Redefined cleaning steps after revision

src/features/all_cleaning_individual/straw/main.py

@@ -19,6 +19,7 @@ def straw_cleaning(sensor_data_files, provider):
 
     excluded_columns = ['local_segment', 'local_segment_label', 'local_segment_start_datetime', 'local_segment_end_datetime']
 
+        
     # (1) FILTER_OUT THE ROWS THAT DO NOT HAVE THE TARGET COLUMN AVAILABLE
     if config['PARAMS_FOR_ANALYSIS']['TARGET']['COMPUTE']:
         target = config['PARAMS_FOR_ANALYSIS']['TARGET']['LABEL'] # get target label from config
@@ -36,9 +37,16 @@ def straw_cleaning(sensor_data_files, provider):
     # because of the lack of the availability. Secondly, there's a high importance that features data frame is checked if and NaN 
     # values still exist.  
 
-    # (2) PARTIAL IMPUTATION: IMPUTE DATA DEPENDEND ON THE FEATURES GROUP (e.g., phone or E4 features)
+    # (2) REMOVE COLS IF THEIR NAN THRESHOLD IS PASSED (should be <= if even all NaN columns must be preserved - this solution now drops columns with all NaN rows)
+    # TODO: determine the threshold at which the column should be removed because of too many Nans.  
+    features = features.loc[:, features.isna().sum() < provider["COLS_NAN_THRESHOLD"] * features.shape[0]]
+
+    # (3.1) QUALITY CHECK (DATA YIELD COLUMN) which determines if the row stays or not (if either E4 or phone is low quality the row is useless - TODO: determine threshold)
+    # Here, the imputation is still not executed - only quality check
+    
     impute_phone_features = provider["IMPUTE_PHONE_SELECTED_EVENT_FEATURES"]
-    if impute_phone_features["COMPUTE"]:
+    
+    if True: #impute_phone_features["COMPUTE"]:
         if not 'phone_data_yield_rapids_ratiovalidyieldedminutes' in features.columns:
             raise KeyError("RAPIDS provider needs to impute the selected event features based on phone_data_yield_rapids_ratiovalidyieldedminutes column, please set config[PHONE_DATA_YIELD][PROVIDERS][RAPIDS][COMPUTE] to True and include 'ratiovalidyieldedminutes' in [FEATURES].")
         
@@ -55,6 +63,8 @@ def straw_cleaning(sensor_data_files, provider):
         mask = features['phone_data_yield_rapids_ratiovalidyieldedminutes'] > impute_phone_features['MIN_DATA_YIELDED_MINUTES_TO_IMPUTE']
         features.loc[mask, phone_cols] = impute(features[mask][phone_cols], method=impute_phone_features["TYPE"].lower())
 
+        print(features[features['phone_data_yield_rapids_ratiovalidyieldedminutes'] > impute_phone_features['MIN_DATA_YIELDED_MINUTES_TO_IMPUTE']][phone_cols]) 
+    
     # ??? Drop rows with the value of data_yield_column less than data_yield_ratio_threshold ??? 
     data_yield_unit = provider["DATA_YIELD_FEATURE"].split("_")[3].lower()
     data_yield_column = "phone_data_yield_rapids_ratiovalidyielded" + data_yield_unit
@@ -65,10 +75,13 @@ def straw_cleaning(sensor_data_files, provider):
     if provider["DATA_YIELD_RATIO_THRESHOLD"]:
         features = features[features[data_yield_column] >= provider["DATA_YIELD_RATIO_THRESHOLD"]]
 
-    # (3) REMOVE COLS IF THEIR NAN THRESHOLD IS PASSED (should be <= if even all NaN columns must be preserved - this solution now drops columns with all NaN rows)
+    # (3.2) (optional) DOES ROW CONSIST OF ENOUGH NON-NAN VALUES? Possible some of these examples could still pass previous condition but not this one?
-    features = features.loc[:, features.isna().sum() < provider["COLS_NAN_THRESHOLD"] * features.shape[0]]
 
-    # (4) REMOVE COLS WHERE VARIANCE IS 0
+    # (4) IMPUTATION: IMPUTE DATA WITH KNN METHOD 
+    # - no other input restriction for this method except that rows are full enough and have reasonably high quality as assessed by data yield
+
+
+    # (5) REMOVE COLS WHERE VARIANCE IS 0
     if provider["COLS_VAR_THRESHOLD"]:
         features.drop(features.std()[features.std() == 0].index.values, axis=1, inplace=True)
     
@@ -77,7 +90,7 @@ def straw_cleaning(sensor_data_files, provider):
         if esm not in features:
             features[esm] = esm_cols[esm]
     
-    # (5) DROP HIGHLY CORRELATED FEATURES
+    # (6) DROP HIGHLY CORRELATED FEATURES
     drop_corr_features = provider["DROP_HIGHLY_CORRELATED_FEATURES"]
     if drop_corr_features["COMPUTE"]:
         
@@ -92,7 +105,7 @@ def straw_cleaning(sensor_data_files, provider):
 
         features.drop(to_drop, axis=1, inplace=True)
 
-    # (6) Remove rows if threshold of NaN values is passed
+    # (7) Remove rows if threshold of NaN values is passed
     min_count =  math.ceil((1 - provider["ROWS_NAN_THRESHOLD"]) * features.shape[1]) # minimal not nan values in row
     features.dropna(axis=0, thresh=min_count, inplace=True)
 
@@ -101,16 +114,7 @@ def straw_cleaning(sensor_data_files, provider):
     sns.heatmap(features.isna(), cbar=False)
     plt.savefig(f'features_nans_bf_knn.png', bbox_inches='tight')
         
-    ## (7) STANDARDIZATION 
+    ## (8) STANDARDIZATION 
-    
-    if provider["STANDARDIZATION"]:
-        features.loc[:, ~features.columns.isin(excluded_columns)] = StandardScaler().fit_transform(features.loc[:, ~features.columns.isin(excluded_columns)])
-
-    # (8) KNN IMPUTATION
-    impute_cols = [col for col in features.columns if col not in excluded_columns]
-    features[impute_cols] = impute(features[impute_cols], method="knn")
-
-    # (9) STANDARDIZATION AGAIN
     
     if provider["STANDARDIZATION"]:
         features.loc[:, ~features.columns.isin(excluded_columns)] = StandardScaler().fit_transform(features.loc[:, ~features.columns.isin(excluded_columns)])