From eaf4340afd7db8a43174a4789cfb9ba5a831dac4 Mon Sep 17 00:00:00 2001 From: Primoz Date: Tue, 20 Sep 2022 08:03:48 +0000 Subject: [PATCH 1/2] Small imputation and cleaning corrections. --- Snakefile | 12 ++++++------ config.yaml | 3 +-- rules/models.smk | 8 ++++---- src/features/all_cleaning_individual/straw/main.py | 2 +- src/features/phone_wifi_visible/rapids/main.R | 9 +-------- 5 files changed, 13 insertions(+), 21 deletions(-) diff --git a/Snakefile b/Snakefile index 538a5382..1e9a1fe8 100644 --- a/Snakefile +++ b/Snakefile @@ -410,14 +410,14 @@ for provider in config["ALL_CLEANING_INDIVIDUAL"]["PROVIDERS"].keys(): if config["ALL_CLEANING_INDIVIDUAL"]["PROVIDERS"][provider]["COMPUTE"]: if provider == "STRAW": files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features_cleaned_" + provider.lower() + "_py.csv", pid=config["PIDS"])) - else: + else: files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features_cleaned_" + provider.lower() + "_R.csv", pid=config["PIDS"])) for provider in config["ALL_CLEANING_OVERALL"]["PROVIDERS"].keys(): if config["ALL_CLEANING_OVERALL"]["PROVIDERS"][provider]["COMPUTE"]: if provider == "STRAW": files_to_compute.extend(expand("data/processed/features/all_participants/all_sensor_features_cleaned_" + provider.lower() +"_py.csv")) - else: + else: files_to_compute.extend(expand("data/processed/features/all_participants/all_sensor_features_cleaned_" + provider.lower() +"_R.csv")) # Baseline features @@ -429,10 +429,10 @@ if config["PARAMS_FOR_ANALYSIS"]["BASELINE"]["COMPUTE"]: # Targets (labels) if config["PARAMS_FOR_ANALYSIS"]["TARGET"]["COMPUTE"]: - # files_to_compute.extend(expand("data/processed/models/individual_model/{pid}/input.csv", pid=config["PIDS"])) - # files_to_compute.extend(expand("data/processed/models/population_model/input.csv")) - files_to_compute.extend(expand("data/processed/models/individual_model/{pid}/z_input.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/processed/models/population_model/z_input.csv")) + files_to_compute.extend(expand("data/processed/models/individual_model/{pid}/input.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/processed/models/population_model/input.csv")) + # files_to_compute.extend(expand("data/processed/models/individual_model/{pid}/z_input.csv", pid=config["PIDS"])) + # files_to_compute.extend(expand("data/processed/models/population_model/z_input.csv")) #files_to_compute.extend(expand("data/processed/models/individual_model/{pid}/output_{cv_method}/baselines.csv", pid=config["PIDS"], cv_method=config["PARAMS_FOR_ANALYSIS"]["CV_METHODS"])) diff --git a/config.yaml b/config.yaml index 4adddedb..3e6d43bd 100644 --- a/config.yaml +++ b/config.yaml @@ -3,7 +3,7 @@ ######################################################################################################################## # See https://www.rapids.science/latest/setup/configuration/#participant-files -PIDS: ['p01'] #['p031', 'p032', 'p033', 'p034', 'p035', 'p036', 'p037', 'p038', 'p039', 'p040', 'p042', 'p043', 'p044', 'p045', 'p046', 'p049', 'p050', 'p052', 'p053', 'p054', 'p055', 'p057', 'p058', 'p059', 'p060', 'p061', 'p062', 'p064', 'p067', 'p068', 'p069', 'p070', 'p071', 'p072', 'p073', 'p074', 'p075', 'p076', 'p077', 'p078', 'p079', 'p080', 'p081', 'p082', 'p083', 'p084', 'p085', 'p086', 'p088', 'p089', 'p090', 'p091', 'p092', 'p093', 'p106', 'p107'] +PIDS: ['p01', 'p02'] #['p031', 'p032', 'p033', 'p034', 'p035', 'p036', 'p037', 'p038', 'p039', 'p040', 'p042', 'p043', 'p044', 'p045', 'p046', 'p049', 'p050', 'p052', 'p053', 'p054', 'p055', 'p057', 'p058', 'p059', 'p060', 'p061', 'p062', 'p064', 'p067', 'p068', 'p069', 'p070', 'p071', 'p072', 'p073', 'p074', 'p075', 'p076', 'p077', 'p078', 'p079', 'p080', 'p081', 'p082', 'p083', 'p084', 'p085', 'p086', 'p088', 'p089', 'p090', 'p091', 'p092', 'p093', 'p106', 'p107'] # See https://www.rapids.science/latest/setup/configuration/#automatic-creation-of-participant-files CREATE_PARTICIPANT_FILES: @@ -70,7 +70,6 @@ PHONE_ACCELEROMETER: COMPUTE: False FEATURES: ["maxmagnitude", "minmagnitude", "avgmagnitude", "medianmagnitude", "stdmagnitude"] SRC_SCRIPT: src/features/phone_accelerometer/rapids/main.py - PANDA: COMPUTE: False VALID_SENSED_MINUTES: False diff --git a/rules/models.smk b/rules/models.smk index 3aade69a..cc9b406d 100644 --- a/rules/models.smk +++ b/rules/models.smk @@ -30,22 +30,22 @@ rule baseline_features: rule select_target: input: - cleaned_sensor_features = "data/processed/features/{pid}/z_all_sensor_features_cleaned_straw_py.csv" + cleaned_sensor_features = "data/processed/features/{pid}/all_sensor_features_cleaned_straw_py.csv" params: target_variable = config["PARAMS_FOR_ANALYSIS"]["TARGET"]["LABEL"] output: - "data/processed/models/individual_model/{pid}/z_input.csv" + "data/processed/models/individual_model/{pid}/input.csv" script: "../src/models/select_targets.py" rule merge_features_and_targets_for_population_model: input: - cleaned_sensor_features = "data/processed/features/all_participants/z_all_sensor_features_cleaned_straw_py.csv", + cleaned_sensor_features = "data/processed/features/all_participants/all_sensor_features_cleaned_straw_py.csv", demographic_features = expand("data/processed/features/{pid}/baseline_features.csv", pid=config["PIDS"]), params: target_variable=config["PARAMS_FOR_ANALYSIS"]["TARGET"]["LABEL"] output: - "data/processed/models/population_model/z_input.csv" + "data/processed/models/population_model/input.csv" script: "../src/models/merge_features_and_targets_for_population_model.py" diff --git a/src/features/all_cleaning_individual/straw/main.py b/src/features/all_cleaning_individual/straw/main.py index 55845c89..0b0259f7 100644 --- a/src/features/all_cleaning_individual/straw/main.py +++ b/src/features/all_cleaning_individual/straw/main.py @@ -116,7 +116,7 @@ def straw_cleaning(sensor_data_files, provider): plt.savefig(f'features_nans_af_knn.png', bbox_inches='tight') # VERIFY IF THERE ARE ANY NANS LEFT IN THE DATAFRAME - if features.isna.any().any(): + if features.isna().any().any(): raise ValueError sys.exit() diff --git a/src/features/phone_wifi_visible/rapids/main.R b/src/features/phone_wifi_visible/rapids/main.R index 03221d3a..a8a433fa 100644 --- a/src/features/phone_wifi_visible/rapids/main.R +++ b/src/features/phone_wifi_visible/rapids/main.R @@ -23,13 +23,6 @@ compute_wifi_feature <- function(data, feature, time_segment){ data <- data %>% filter_data_by_segment(time_segment) - print(data %>% - filter(bssid == mostuniquedevice) %>% - group_by(local_segment) %>% - summarise(!!feature := n())) - - raise - return(data %>% filter(bssid == mostuniquedevice) %>% group_by(local_segment) %>% @@ -55,6 +48,6 @@ rapids_features <- function(sensor_data_files, time_segment, provider){ feature <- compute_wifi_feature(wifi_data, feature_name, time_segment) features <- merge(features, feature, by="local_segment", all = TRUE) } - # features <- features %>% mutate_all(~replace(., is.na(.), 0)) + features <- features %>% mutate_all(~replace(., is.na(.), 0)) return(features) } From 7493aaa64368f81adbf478b43411063f56a9a87e Mon Sep 17 00:00:00 2001 From: Primoz Date: Tue, 20 Sep 2022 12:57:55 +0000 Subject: [PATCH 2/2] Small changes in cleaning scrtipt and missing vals testing. --- .../all_cleaning_individual/straw/main.py | 25 +++++++++++-------- tests/scripts/missing_vals.py | 4 +-- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/features/all_cleaning_individual/straw/main.py b/src/features/all_cleaning_individual/straw/main.py index 0b0259f7..5557a2dc 100644 --- a/src/features/all_cleaning_individual/straw/main.py +++ b/src/features/all_cleaning_individual/straw/main.py @@ -17,12 +17,13 @@ def straw_cleaning(sensor_data_files, provider): with open('config.yaml', 'r') as stream: config = yaml.load(stream, Loader=yaml.FullLoader) + excluded_columns = ['local_segment', 'local_segment_label', 'local_segment_start_datetime', 'local_segment_end_datetime'] + # (1) FILTER_OUT THE ROWS THAT DO NOT HAVE THE TARGET COLUMN AVAILABLE if config['PARAMS_FOR_ANALYSIS']['TARGET']['COMPUTE']: target = config['PARAMS_FOR_ANALYSIS']['TARGET']['LABEL'] # get target label from config - features = features[features['phone_esm_straw_' + target].notna()].reset_index() + features = features[features['phone_esm_straw_' + target].notna()].reset_index(drop=True) - # TODO: reorder the cleaning steps so it makes sense for the analysis # TODO: add conditions that differentiates cleaning steps for standardized and nonstandardized features, for this # the snakemake rules will also have to come with additional parameter (in rules/features.smk) @@ -64,10 +65,10 @@ def straw_cleaning(sensor_data_files, provider): if provider["DATA_YIELD_RATIO_THRESHOLD"]: features = features[features[data_yield_column] >= provider["DATA_YIELD_RATIO_THRESHOLD"]] - # (3) REMOVE COLS IF THEIR NAN THRESHOLD IS PASSED (should be <= if even all NaN columns must be preserved) + # (3) REMOVE COLS IF THEIR NAN THRESHOLD IS PASSED (should be <= if even all NaN columns must be preserved - this solution now drops columns with all NaN rows) features = features.loc[:, features.isna().sum() < provider["COLS_NAN_THRESHOLD"] * features.shape[0]] - # (4) REMOVE COLS WHERE VARIANCE IS 0 TODO: preveri za local_segment stolpce + # (4) REMOVE COLS WHERE VARIANCE IS 0 if provider["COLS_VAR_THRESHOLD"]: features.drop(features.std()[features.std() == 0].index.values, axis=1, inplace=True) @@ -91,31 +92,35 @@ def straw_cleaning(sensor_data_files, provider): features.drop(to_drop, axis=1, inplace=True) - # Remove rows if threshold of NaN values is passed + # (6) Remove rows if threshold of NaN values is passed min_count = math.ceil((1 - provider["ROWS_NAN_THRESHOLD"]) * features.shape[1]) # minimal not nan values in row features.dropna(axis=0, thresh=min_count, inplace=True) + sns.set(rc={"figure.figsize":(16, 8)}) sns.heatmap(features.isna(), cbar=False) plt.savefig(f'features_nans_bf_knn.png', bbox_inches='tight') - ## STANDARDIZATION - should it happen before or after kNN imputation? - # TODO: check if there are additional columns that need to be excluded from the standardization - excluded_columns = ['local_segment', 'local_segment_label', 'local_segment_start_datetime', 'local_segment_end_datetime'] + ## (7) STANDARDIZATION if provider["STANDARDIZATION"]: features.loc[:, ~features.columns.isin(excluded_columns)] = StandardScaler().fit_transform(features.loc[:, ~features.columns.isin(excluded_columns)]) - # KNN IMPUTATION + # (8) KNN IMPUTATION impute_cols = [col for col in features.columns if col not in excluded_columns] features[impute_cols] = impute(features[impute_cols], method="knn") + # (9) STANDARDIZATION AGAIN + + if provider["STANDARDIZATION"]: + features.loc[:, ~features.columns.isin(excluded_columns)] = StandardScaler().fit_transform(features.loc[:, ~features.columns.isin(excluded_columns)]) + sns.set(rc={"figure.figsize":(16, 8)}) sns.heatmap(features.isna(), cbar=False) plt.savefig(f'features_nans_af_knn.png', bbox_inches='tight') - # VERIFY IF THERE ARE ANY NANS LEFT IN THE DATAFRAME + # (9) VERIFY IF THERE ARE ANY NANS LEFT IN THE DATAFRAME if features.isna().any().any(): raise ValueError diff --git a/tests/scripts/missing_vals.py b/tests/scripts/missing_vals.py index acbae0bb..41cf4709 100644 --- a/tests/scripts/missing_vals.py +++ b/tests/scripts/missing_vals.py @@ -3,8 +3,8 @@ import seaborn as sns import matplotlib.pyplot as plt -participant = "p031" -all_sensors = ["eda", "bvp", "ibi", "temp", "acc"] +participant = "p01" +all_sensors = ["eda", "ibi", "temp", "acc"] for sensor in all_sensors: