Merge branch 'imputation_and_cleaning' of https://repo.ijs.si/junoslukan/rapids into imputation_and_cleaning

Snakefile

@@ -429,10 +429,10 @@ if config["PARAMS_FOR_ANALYSIS"]["BASELINE"]["COMPUTE"]:
 
 # Targets (labels)
 if config["PARAMS_FOR_ANALYSIS"]["TARGET"]["COMPUTE"]:
-    # files_to_compute.extend(expand("data/processed/models/individual_model/{pid}/input.csv", pid=config["PIDS"]))
+    files_to_compute.extend(expand("data/processed/models/individual_model/{pid}/input.csv", pid=config["PIDS"]))
-    # files_to_compute.extend(expand("data/processed/models/population_model/input.csv"))
+    files_to_compute.extend(expand("data/processed/models/population_model/input.csv"))
-    files_to_compute.extend(expand("data/processed/models/individual_model/{pid}/z_input.csv", pid=config["PIDS"]))
+    # files_to_compute.extend(expand("data/processed/models/individual_model/{pid}/z_input.csv", pid=config["PIDS"]))
-    files_to_compute.extend(expand("data/processed/models/population_model/z_input.csv"))
+    # files_to_compute.extend(expand("data/processed/models/population_model/z_input.csv"))
 
 #files_to_compute.extend(expand("data/processed/models/individual_model/{pid}/output_{cv_method}/baselines.csv", pid=config["PIDS"], cv_method=config["PARAMS_FOR_ANALYSIS"]["CV_METHODS"]))
 

config.yaml

@@ -3,7 +3,7 @@
 ########################################################################################################################
 
 # See https://www.rapids.science/latest/setup/configuration/#participant-files
-PIDS: ['p01'] #['p031', 'p032', 'p033', 'p034', 'p035', 'p036', 'p037', 'p038', 'p039', 'p040', 'p042', 'p043', 'p044', 'p045', 'p046', 'p049', 'p050', 'p052', 'p053', 'p054', 'p055', 'p057', 'p058', 'p059', 'p060', 'p061', 'p062', 'p064', 'p067', 'p068', 'p069', 'p070', 'p071', 'p072', 'p073', 'p074', 'p075', 'p076', 'p077', 'p078', 'p079', 'p080', 'p081', 'p082', 'p083', 'p084', 'p085', 'p086', 'p088', 'p089', 'p090', 'p091', 'p092', 'p093', 'p106', 'p107']
+PIDS: ['p01', 'p02'] #['p031', 'p032', 'p033', 'p034', 'p035', 'p036', 'p037', 'p038', 'p039', 'p040', 'p042', 'p043', 'p044', 'p045', 'p046', 'p049', 'p050', 'p052', 'p053', 'p054', 'p055', 'p057', 'p058', 'p059', 'p060', 'p061', 'p062', 'p064', 'p067', 'p068', 'p069', 'p070', 'p071', 'p072', 'p073', 'p074', 'p075', 'p076', 'p077', 'p078', 'p079', 'p080', 'p081', 'p082', 'p083', 'p084', 'p085', 'p086', 'p088', 'p089', 'p090', 'p091', 'p092', 'p093', 'p106', 'p107']
 
 # See https://www.rapids.science/latest/setup/configuration/#automatic-creation-of-participant-files
 CREATE_PARTICIPANT_FILES:
@@ -70,7 +70,6 @@ PHONE_ACCELEROMETER:
       COMPUTE: False
       FEATURES: ["maxmagnitude", "minmagnitude", "avgmagnitude", "medianmagnitude", "stdmagnitude"]
       SRC_SCRIPT: src/features/phone_accelerometer/rapids/main.py
-    
     PANDA:
       COMPUTE: False
       VALID_SENSED_MINUTES: False

rules/models.smk

@@ -30,22 +30,22 @@ rule baseline_features:
 
 rule select_target:
     input:
-        cleaned_sensor_features = "data/processed/features/{pid}/z_all_sensor_features_cleaned_straw_py.csv"
+        cleaned_sensor_features = "data/processed/features/{pid}/all_sensor_features_cleaned_straw_py.csv"
     params:
         target_variable = config["PARAMS_FOR_ANALYSIS"]["TARGET"]["LABEL"]
     output:
-        "data/processed/models/individual_model/{pid}/z_input.csv"
+        "data/processed/models/individual_model/{pid}/input.csv"
     script:
         "../src/models/select_targets.py"
 
 rule merge_features_and_targets_for_population_model:
     input:
-        cleaned_sensor_features = "data/processed/features/all_participants/z_all_sensor_features_cleaned_straw_py.csv",
+        cleaned_sensor_features = "data/processed/features/all_participants/all_sensor_features_cleaned_straw_py.csv",
         demographic_features = expand("data/processed/features/{pid}/baseline_features.csv", pid=config["PIDS"]),
     params:
         target_variable=config["PARAMS_FOR_ANALYSIS"]["TARGET"]["LABEL"]
     output:
-        "data/processed/models/population_model/z_input.csv"
+        "data/processed/models/population_model/input.csv"
     script:
         "../src/models/merge_features_and_targets_for_population_model.py"
 

src/features/all_cleaning_individual/straw/main.py

@@ -17,12 +17,13 @@ def straw_cleaning(sensor_data_files, provider):
     with open('config.yaml', 'r') as stream:
         config = yaml.load(stream, Loader=yaml.FullLoader)
 
+    excluded_columns = ['local_segment', 'local_segment_label', 'local_segment_start_datetime', 'local_segment_end_datetime']
+
     # (1) FILTER_OUT THE ROWS THAT DO NOT HAVE THE TARGET COLUMN AVAILABLE
     if config['PARAMS_FOR_ANALYSIS']['TARGET']['COMPUTE']:
         target = config['PARAMS_FOR_ANALYSIS']['TARGET']['LABEL'] # get target label from config
-        features = features[features['phone_esm_straw_' + target].notna()].reset_index()
+        features = features[features['phone_esm_straw_' + target].notna()].reset_index(drop=True)
 
-    # TODO: reorder the cleaning steps so it makes sense for the analysis
     # TODO: add conditions that differentiates cleaning steps for standardized and nonstandardized features, for this
     # the snakemake rules will also have to come with additional parameter (in rules/features.smk)
 
@@ -64,10 +65,10 @@ def straw_cleaning(sensor_data_files, provider):
     if provider["DATA_YIELD_RATIO_THRESHOLD"]:
         features = features[features[data_yield_column] >= provider["DATA_YIELD_RATIO_THRESHOLD"]]
 
-    # (3) REMOVE COLS IF THEIR NAN THRESHOLD IS PASSED (should be <= if even all NaN columns must be preserved)
+    # (3) REMOVE COLS IF THEIR NAN THRESHOLD IS PASSED (should be <= if even all NaN columns must be preserved - this solution now drops columns with all NaN rows)
     features = features.loc[:, features.isna().sum() < provider["COLS_NAN_THRESHOLD"] * features.shape[0]]
 
-    # (4) REMOVE COLS WHERE VARIANCE IS 0 TODO: preveri za local_segment stolpce
+    # (4) REMOVE COLS WHERE VARIANCE IS 0
     if provider["COLS_VAR_THRESHOLD"]:
         features.drop(features.std()[features.std() == 0].index.values, axis=1, inplace=True)
     
@@ -91,32 +92,36 @@ def straw_cleaning(sensor_data_files, provider):
 
         features.drop(to_drop, axis=1, inplace=True)
 
-    # Remove rows if threshold of NaN values is passed
+    # (6) Remove rows if threshold of NaN values is passed
     min_count =  math.ceil((1 - provider["ROWS_NAN_THRESHOLD"]) * features.shape[1]) # minimal not nan values in row
     features.dropna(axis=0, thresh=min_count, inplace=True)
 
+
     sns.set(rc={"figure.figsize":(16, 8)})
     sns.heatmap(features.isna(), cbar=False)
     plt.savefig(f'features_nans_bf_knn.png', bbox_inches='tight')
         
-    ## STANDARDIZATION - should it happen before or after kNN imputation?
+    ## (7) STANDARDIZATION 
-    # TODO: check if there are additional columns that need to be excluded from the standardization
-    excluded_columns = ['local_segment', 'local_segment_label', 'local_segment_start_datetime', 'local_segment_end_datetime']
     
     if provider["STANDARDIZATION"]:
         features.loc[:, ~features.columns.isin(excluded_columns)] = StandardScaler().fit_transform(features.loc[:, ~features.columns.isin(excluded_columns)])
 
-    # KNN IMPUTATION
+    # (8) KNN IMPUTATION
     impute_cols = [col for col in features.columns if col not in excluded_columns]
     features[impute_cols] = impute(features[impute_cols], method="knn")
 
+    # (9) STANDARDIZATION AGAIN
+    
+    if provider["STANDARDIZATION"]:
+        features.loc[:, ~features.columns.isin(excluded_columns)] = StandardScaler().fit_transform(features.loc[:, ~features.columns.isin(excluded_columns)]) 
+
 
     sns.set(rc={"figure.figsize":(16, 8)})
     sns.heatmap(features.isna(), cbar=False)
     plt.savefig(f'features_nans_af_knn.png', bbox_inches='tight')
 
-    # VERIFY IF THERE ARE ANY NANS LEFT IN THE DATAFRAME
+    # (9) VERIFY IF THERE ARE ANY NANS LEFT IN THE DATAFRAME
-    if features.isna.any().any():
+    if features.isna().any().any():
         raise ValueError
 
     sys.exit()

src/features/phone_wifi_visible/rapids/main.R

@@ -23,13 +23,6 @@ compute_wifi_feature <- function(data, feature, time_segment){
 
     data <- data %>% filter_data_by_segment(time_segment)
 
-    print(data %>% 
-             filter(bssid == mostuniquedevice) %>%
-             group_by(local_segment) %>% 
-             summarise(!!feature := n()))
-
-    raise
-
     return(data %>% 
              filter(bssid == mostuniquedevice) %>%
              group_by(local_segment) %>% 
@@ -55,6 +48,6 @@ rapids_features <- function(sensor_data_files, time_segment, provider){
     feature <- compute_wifi_feature(wifi_data, feature_name, time_segment)
     features <- merge(features, feature, by="local_segment", all = TRUE)
   }
-  # features <- features %>% mutate_all(~replace(., is.na(.), 0))
+  features <- features %>% mutate_all(~replace(., is.na(.), 0))
   return(features)
 }

tests/scripts/missing_vals.py

@@ -3,8 +3,8 @@ import seaborn as sns
 import matplotlib.pyplot as plt
 
 
-participant = "p031"
+participant = "p01"
-all_sensors = ["eda", "bvp", "ibi", "temp", "acc"]
+all_sensors = ["eda", "ibi", "temp", "acc"]
 
 for sensor in all_sensors: