From 19aa8707c0b61790d9d4c1834944c1711d8777aa Mon Sep 17 00:00:00 2001 From: Primoz Date: Thu, 22 Sep 2022 13:45:51 +0000 Subject: [PATCH] Redefined cleaning steps after revision --- .../all_cleaning_individual/straw/main.py | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/src/features/all_cleaning_individual/straw/main.py b/src/features/all_cleaning_individual/straw/main.py index 5557a2dc..0c41676f 100644 --- a/src/features/all_cleaning_individual/straw/main.py +++ b/src/features/all_cleaning_individual/straw/main.py @@ -19,6 +19,7 @@ def straw_cleaning(sensor_data_files, provider): excluded_columns = ['local_segment', 'local_segment_label', 'local_segment_start_datetime', 'local_segment_end_datetime'] + # (1) FILTER_OUT THE ROWS THAT DO NOT HAVE THE TARGET COLUMN AVAILABLE if config['PARAMS_FOR_ANALYSIS']['TARGET']['COMPUTE']: target = config['PARAMS_FOR_ANALYSIS']['TARGET']['LABEL'] # get target label from config @@ -36,9 +37,16 @@ def straw_cleaning(sensor_data_files, provider): # because of the lack of the availability. Secondly, there's a high importance that features data frame is checked if and NaN # values still exist. - # (2) PARTIAL IMPUTATION: IMPUTE DATA DEPENDEND ON THE FEATURES GROUP (e.g., phone or E4 features) + # (2) REMOVE COLS IF THEIR NAN THRESHOLD IS PASSED (should be <= if even all NaN columns must be preserved - this solution now drops columns with all NaN rows) + # TODO: determine the threshold at which the column should be removed because of too many Nans. + features = features.loc[:, features.isna().sum() < provider["COLS_NAN_THRESHOLD"] * features.shape[0]] + + # (3.1) QUALITY CHECK (DATA YIELD COLUMN) which determines if the row stays or not (if either E4 or phone is low quality the row is useless - TODO: determine threshold) + # Here, the imputation is still not executed - only quality check + impute_phone_features = provider["IMPUTE_PHONE_SELECTED_EVENT_FEATURES"] - if impute_phone_features["COMPUTE"]: + + if True: #impute_phone_features["COMPUTE"]: if not 'phone_data_yield_rapids_ratiovalidyieldedminutes' in features.columns: raise KeyError("RAPIDS provider needs to impute the selected event features based on phone_data_yield_rapids_ratiovalidyieldedminutes column, please set config[PHONE_DATA_YIELD][PROVIDERS][RAPIDS][COMPUTE] to True and include 'ratiovalidyieldedminutes' in [FEATURES].") @@ -55,7 +63,9 @@ def straw_cleaning(sensor_data_files, provider): mask = features['phone_data_yield_rapids_ratiovalidyieldedminutes'] > impute_phone_features['MIN_DATA_YIELDED_MINUTES_TO_IMPUTE'] features.loc[mask, phone_cols] = impute(features[mask][phone_cols], method=impute_phone_features["TYPE"].lower()) - # ??? Drop rows with the value of data_yield_column less than data_yield_ratio_threshold ??? + print(features[features['phone_data_yield_rapids_ratiovalidyieldedminutes'] > impute_phone_features['MIN_DATA_YIELDED_MINUTES_TO_IMPUTE']][phone_cols]) + + # ??? Drop rows with the value of data_yield_column less than data_yield_ratio_threshold ??? data_yield_unit = provider["DATA_YIELD_FEATURE"].split("_")[3].lower() data_yield_column = "phone_data_yield_rapids_ratiovalidyielded" + data_yield_unit @@ -65,10 +75,13 @@ def straw_cleaning(sensor_data_files, provider): if provider["DATA_YIELD_RATIO_THRESHOLD"]: features = features[features[data_yield_column] >= provider["DATA_YIELD_RATIO_THRESHOLD"]] - # (3) REMOVE COLS IF THEIR NAN THRESHOLD IS PASSED (should be <= if even all NaN columns must be preserved - this solution now drops columns with all NaN rows) - features = features.loc[:, features.isna().sum() < provider["COLS_NAN_THRESHOLD"] * features.shape[0]] + # (3.2) (optional) DOES ROW CONSIST OF ENOUGH NON-NAN VALUES? Possible some of these examples could still pass previous condition but not this one? - # (4) REMOVE COLS WHERE VARIANCE IS 0 + # (4) IMPUTATION: IMPUTE DATA WITH KNN METHOD + # - no other input restriction for this method except that rows are full enough and have reasonably high quality as assessed by data yield + + + # (5) REMOVE COLS WHERE VARIANCE IS 0 if provider["COLS_VAR_THRESHOLD"]: features.drop(features.std()[features.std() == 0].index.values, axis=1, inplace=True) @@ -77,7 +90,7 @@ def straw_cleaning(sensor_data_files, provider): if esm not in features: features[esm] = esm_cols[esm] - # (5) DROP HIGHLY CORRELATED FEATURES + # (6) DROP HIGHLY CORRELATED FEATURES drop_corr_features = provider["DROP_HIGHLY_CORRELATED_FEATURES"] if drop_corr_features["COMPUTE"]: @@ -92,7 +105,7 @@ def straw_cleaning(sensor_data_files, provider): features.drop(to_drop, axis=1, inplace=True) - # (6) Remove rows if threshold of NaN values is passed + # (7) Remove rows if threshold of NaN values is passed min_count = math.ceil((1 - provider["ROWS_NAN_THRESHOLD"]) * features.shape[1]) # minimal not nan values in row features.dropna(axis=0, thresh=min_count, inplace=True) @@ -101,20 +114,11 @@ def straw_cleaning(sensor_data_files, provider): sns.heatmap(features.isna(), cbar=False) plt.savefig(f'features_nans_bf_knn.png', bbox_inches='tight') - ## (7) STANDARDIZATION + ## (8) STANDARDIZATION if provider["STANDARDIZATION"]: features.loc[:, ~features.columns.isin(excluded_columns)] = StandardScaler().fit_transform(features.loc[:, ~features.columns.isin(excluded_columns)]) - # (8) KNN IMPUTATION - impute_cols = [col for col in features.columns if col not in excluded_columns] - features[impute_cols] = impute(features[impute_cols], method="knn") - - # (9) STANDARDIZATION AGAIN - - if provider["STANDARDIZATION"]: - features.loc[:, ~features.columns.isin(excluded_columns)] = StandardScaler().fit_transform(features.loc[:, ~features.columns.isin(excluded_columns)]) - sns.set(rc={"figure.figsize":(16, 8)}) sns.heatmap(features.isna(), cbar=False)