junos
/
stress_at_work_analysis
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Prepare data in a separate step. 

Change categorical features.
ml_pipeline
junos 2022-11-16 19:34:18 +01:00
parent c462d55096
commit 389198b17f
1 changed files with 14 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
machine_learning/helper.py
View File

@ -66,8 +66,8 @@ def construct_full_path(folder: Path, filename_prefix: str, data_type: str) -> P
def insert_row(df, row): def insert_row(df, row):
return pd.concat([df, pd.DataFrame([row], columns=df.columns)], ignore_index=True) return pd.concat([df, pd.DataFrame([row], columns=df.columns)], ignore_index=True)
def run_all_models(input_csv): def prepare_model_input(input_csv):
# Prepare data
model_input = pd.read_csv(input_csv) model_input = pd.read_csv(input_csv)
index_columns = ["local_segment", "local_segment_label", "local_segment_start_datetime", "local_segment_end_datetime"] index_columns = ["local_segment", "local_segment_label", "local_segment_start_datetime", "local_segment_end_datetime"]
@ -75,9 +75,11 @@ def run_all_models(input_csv):
data_x, data_y, data_groups = model_input.drop(["target", "pid"], axis=1), model_input["target"], model_input["pid"] data_x, data_y, data_groups = model_input.drop(["target", "pid"], axis=1), model_input["target"], model_input["pid"]
categorical_feature_colnames = ["gender", "startlanguage"] categorical_feature_colnames = ["gender", "startlanguage", "limesurvey_demand_control_ratio_quartile"]
additional_categorical_features = [col for col in data_x.columns if "mostcommonactivity" in col or "homelabel" in col] #TODO: check whether limesurvey_demand_control_ratio_quartile NaNs could be replaced meaningfully
categorical_feature_colnames += additional_categorical_features #additional_categorical_features = [col for col in data_x.columns if "mostcommonactivity" in col or "homelabel" in col]
#TODO: check if mostcommonactivity is indeed a categorical features after aggregating
#categorical_feature_colnames += additional_categorical_features
categorical_features = data_x[categorical_feature_colnames].copy() categorical_features = data_x[categorical_feature_colnames].copy()
mode_categorical_features = categorical_features.mode().iloc[0] mode_categorical_features = categorical_features.mode().iloc[0]
# fillna with mode # fillna with mode
@ -91,6 +93,13 @@ def run_all_models(input_csv):
train_x = pd.concat([numerical_features, categorical_features], axis=1) train_x = pd.concat([numerical_features, categorical_features], axis=1)
return train_x, data_y, data_groups
def run_all_models(input_csv):
# Prepare data
train_x, data_y, data_groups = prepare_model_input(input_csv)
# Prepare cross validation # Prepare cross validation
logo = LeaveOneGroupOut() logo = LeaveOneGroupOut()
logo.get_n_splits( logo.get_n_splits(