Add a note to think about.

Add todo to rename a column.
Switch config for 30 minutes event related segments.
2023-03-27 11:56:09 +02:00 · 2023-03-27 11:46:04 +02:00 · 2022-10-26 14:17:27 +00:00 · 2022-10-26 14:16:25 +00:00 · 2022-10-26 14:14:32 +00:00 · 2022-10-26 09:59:46 +00:00
44 changed files with 1357 additions and 577 deletions
--- a/NaN.png
+++ b/NaN.png
--- a/130
+++ b/130
@ -33,12 +33,6 @@ for provider in config["PHONE_DATA_YIELD"]["PROVIDERS"].keys():
        files_to_compute.extend(expand("data/processed/features/{pid}/phone_data_yield.csv", pid=config["PIDS"]))
        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
-        if provider in config["STANDARDIZATION"]["PROVIDERS"]["OTHER"]["LIST"] and config["STANDARDIZATION"]["PROVIDERS"]["OTHER"]["COMPUTE"] \
-            and config["PHONE_DATA_YIELD"]["PROVIDERS"][provider]["STANDARDIZE_FEATURES"]:
-            files_to_compute.extend(expand("data/processed/features/{pid}/z_phone_data_yield.csv", pid=config["PIDS"]))
-            if config["STANDARDIZATION"]["MERGE_ALL"]:
-                files_to_compute.extend(expand("data/processed/features/{pid}/z_all_sensor_features.csv", pid=config["PIDS"]))
-                files_to_compute.append("data/processed/features/all_participants/z_all_sensor_features.csv")

 for provider in config["PHONE_MESSAGES"]["PROVIDERS"].keys():
    if config["PHONE_MESSAGES"]["PROVIDERS"][provider]["COMPUTE"]:
@ -48,12 +42,6 @@ for provider in config["PHONE_MESSAGES"]["PROVIDERS"].keys():
        files_to_compute.extend(expand("data/processed/features/{pid}/phone_messages.csv", pid=config["PIDS"]))
        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
-        if provider in config["STANDARDIZATION"]["PROVIDERS"]["OTHER"]["LIST"] and config["STANDARDIZATION"]["PROVIDERS"]["OTHER"]["COMPUTE"] \
-            and config["PHONE_MESSAGES"]["PROVIDERS"][provider]["STANDARDIZE_FEATURES"]:
-            files_to_compute.extend(expand("data/processed/features/{pid}/z_phone_messages.csv", pid=config["PIDS"]))
-            if config["STANDARDIZATION"]["MERGE_ALL"]:
-                files_to_compute.extend(expand("data/processed/features/{pid}/z_all_sensor_features.csv", pid=config["PIDS"]))
-                files_to_compute.append("data/processed/features/all_participants/z_all_sensor_features.csv")

 for provider in config["PHONE_CALLS"]["PROVIDERS"].keys():
    if config["PHONE_CALLS"]["PROVIDERS"][provider]["COMPUTE"]:
@ -68,12 +56,6 @@ for provider in config["PHONE_CALLS"]["PROVIDERS"].keys():
        files_to_compute.extend(expand("data/processed/features/{pid}/phone_calls.csv", pid=config["PIDS"]))
        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
-        if provider in config["STANDARDIZATION"]["PROVIDERS"]["OTHER"]["LIST"] and config["STANDARDIZATION"]["PROVIDERS"]["OTHER"]["COMPUTE"] \
-            and config["PHONE_CALLS"]["PROVIDERS"][provider]["STANDARDIZE_FEATURES"]:
-            files_to_compute.extend(expand("data/processed/features/{pid}/z_phone_calls.csv", pid=config["PIDS"]))
-            if config["STANDARDIZATION"]["MERGE_ALL"]:
-                files_to_compute.extend(expand("data/processed/features/{pid}/z_all_sensor_features.csv", pid=config["PIDS"]))
-                files_to_compute.append("data/processed/features/all_participants/z_all_sensor_features.csv")

 for provider in config["PHONE_BLUETOOTH"]["PROVIDERS"].keys():
    if config["PHONE_BLUETOOTH"]["PROVIDERS"][provider]["COMPUTE"]:
@ -83,12 +65,6 @@ for provider in config["PHONE_BLUETOOTH"]["PROVIDERS"].keys():
        files_to_compute.extend(expand("data/processed/features/{pid}/phone_bluetooth.csv", pid=config["PIDS"]))
        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
-        if provider in config["STANDARDIZATION"]["PROVIDERS"]["OTHER"]["LIST"] and config["STANDARDIZATION"]["PROVIDERS"]["OTHER"]["COMPUTE"] \
-            and config["PHONE_BLUETOOTH"]["PROVIDERS"][provider]["STANDARDIZE_FEATURES"]:
-            files_to_compute.extend(expand("data/processed/features/{pid}/z_phone_bluetooth.csv", pid=config["PIDS"]))
-            if config["STANDARDIZATION"]["MERGE_ALL"]:
-                files_to_compute.extend(expand("data/processed/features/{pid}/z_all_sensor_features.csv", pid=config["PIDS"]))
-                files_to_compute.append("data/processed/features/all_participants/z_all_sensor_features.csv")

 for provider in config["PHONE_ACTIVITY_RECOGNITION"]["PROVIDERS"].keys():
    if config["PHONE_ACTIVITY_RECOGNITION"]["PROVIDERS"][provider]["COMPUTE"]:
@ -101,12 +77,6 @@ for provider in config["PHONE_ACTIVITY_RECOGNITION"]["PROVIDERS"].keys():
        files_to_compute.extend(expand("data/processed/features/{pid}/phone_activity_recognition.csv", pid=config["PIDS"]))
        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
-        if provider in config["STANDARDIZATION"]["PROVIDERS"]["OTHER"]["LIST"] and config["STANDARDIZATION"]["PROVIDERS"]["OTHER"]["COMPUTE"] \
-            and config["PHONE_ACTIVITY_RECOGNITION"]["PROVIDERS"][provider]["STANDARDIZE_FEATURES"]:
-            files_to_compute.extend(expand("data/processed/features/{pid}/z_phone_activity_recognition.csv", pid=config["PIDS"]))
-            if config["STANDARDIZATION"]["MERGE_ALL"]:
-                files_to_compute.extend(expand("data/processed/features/{pid}/z_all_sensor_features.csv", pid=config["PIDS"]))
-                files_to_compute.append("data/processed/features/all_participants/z_all_sensor_features.csv")

 for provider in config["PHONE_BATTERY"]["PROVIDERS"].keys():
    if config["PHONE_BATTERY"]["PROVIDERS"][provider]["COMPUTE"]:
@ -118,12 +88,6 @@ for provider in config["PHONE_BATTERY"]["PROVIDERS"].keys():
        files_to_compute.extend(expand("data/processed/features/{pid}/phone_battery.csv", pid=config["PIDS"]))
        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
-        if provider in config["STANDARDIZATION"]["PROVIDERS"]["OTHER"]["LIST"] and config["STANDARDIZATION"]["PROVIDERS"]["OTHER"]["COMPUTE"] \
-            and config["PHONE_BATTERY"]["PROVIDERS"][provider]["STANDARDIZE_FEATURES"]:
-            files_to_compute.extend(expand("data/processed/features/{pid}/z_phone_battery.csv", pid=config["PIDS"]))
-            if config["STANDARDIZATION"]["MERGE_ALL"]:
-                files_to_compute.extend(expand("data/processed/features/{pid}/z_all_sensor_features.csv", pid=config["PIDS"]))
-                files_to_compute.append("data/processed/features/all_participants/z_all_sensor_features.csv")

 for provider in config["PHONE_SCREEN"]["PROVIDERS"].keys():
    if config["PHONE_SCREEN"]["PROVIDERS"][provider]["COMPUTE"]:
@ -140,12 +104,6 @@ for provider in config["PHONE_SCREEN"]["PROVIDERS"].keys():
        files_to_compute.extend(expand("data/processed/features/{pid}/phone_screen.csv", pid=config["PIDS"]))
        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
-        if provider in config["STANDARDIZATION"]["PROVIDERS"]["OTHER"]["LIST"] and config["STANDARDIZATION"]["PROVIDERS"]["OTHER"]["COMPUTE"] \
-            and config["PHONE_SCREEN"]["PROVIDERS"][provider]["STANDARDIZE_FEATURES"]:
-            files_to_compute.extend(expand("data/processed/features/{pid}/z_phone_screen.csv", pid=config["PIDS"]))
-            if config["STANDARDIZATION"]["MERGE_ALL"]:
-                files_to_compute.extend(expand("data/processed/features/{pid}/z_all_sensor_features.csv", pid=config["PIDS"]))
-                files_to_compute.append("data/processed/features/all_participants/z_all_sensor_features.csv")

 for provider in config["PHONE_LIGHT"]["PROVIDERS"].keys():
    if config["PHONE_LIGHT"]["PROVIDERS"][provider]["COMPUTE"]:
@ -155,12 +113,6 @@ for provider in config["PHONE_LIGHT"]["PROVIDERS"].keys():
        files_to_compute.extend(expand("data/processed/features/{pid}/phone_light.csv", pid=config["PIDS"],))
        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
-        if provider in config["STANDARDIZATION"]["PROVIDERS"]["OTHER"]["LIST"] and config["STANDARDIZATION"]["PROVIDERS"]["OTHER"]["COMPUTE"] \
-            and config["PHONE_LIGHT"]["PROVIDERS"][provider]["STANDARDIZE_FEATURES"]:
-            files_to_compute.extend(expand("data/processed/features/{pid}/z_phone_light.csv", pid=config["PIDS"]))
-            if config["STANDARDIZATION"]["MERGE_ALL"]:
-                files_to_compute.extend(expand("data/processed/features/{pid}/z_all_sensor_features.csv", pid=config["PIDS"]))
-                files_to_compute.append("data/processed/features/all_participants/z_all_sensor_features.csv")

 for provider in config["PHONE_ACCELEROMETER"]["PROVIDERS"].keys():
    if config["PHONE_ACCELEROMETER"]["PROVIDERS"][provider]["COMPUTE"]:
@ -184,12 +136,6 @@ for provider in config["PHONE_APPLICATIONS_FOREGROUND"]["PROVIDERS"].keys():
        files_to_compute.extend(expand("data/processed/features/{pid}/phone_applications_foreground.csv", pid=config["PIDS"]))
        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
-        if provider in config["STANDARDIZATION"]["PROVIDERS"]["OTHER"]["LIST"] and config["STANDARDIZATION"]["PROVIDERS"]["OTHER"]["COMPUTE"] \
-            and config["PHONE_APPLICATIONS_FOREGROUND"]["PROVIDERS"][provider]["STANDARDIZE_FEATURES"]:
-            files_to_compute.extend(expand("data/processed/features/{pid}/z_phone_applications_foreground.csv", pid=config["PIDS"]))
-            if config["STANDARDIZATION"]["MERGE_ALL"]:
-                files_to_compute.extend(expand("data/processed/features/{pid}/z_all_sensor_features.csv", pid=config["PIDS"]))
-                files_to_compute.append("data/processed/features/all_participants/z_all_sensor_features.csv")

 for provider in config["PHONE_WIFI_VISIBLE"]["PROVIDERS"].keys():
    if config["PHONE_WIFI_VISIBLE"]["PROVIDERS"][provider]["COMPUTE"]:
@ -199,12 +145,6 @@ for provider in config["PHONE_WIFI_VISIBLE"]["PROVIDERS"].keys():
        files_to_compute.extend(expand("data/processed/features/{pid}/phone_wifi_visible.csv", pid=config["PIDS"]))
        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
-        if provider in config["STANDARDIZATION"]["PROVIDERS"]["OTHER"]["LIST"] and config["STANDARDIZATION"]["PROVIDERS"]["OTHER"]["COMPUTE"] \
-            and config["PHONE_WIFI_VISIBLE"]["PROVIDERS"][provider]["STANDARDIZE_FEATURES"]:
-            files_to_compute.extend(expand("data/processed/features/{pid}/z_phone_wifi_visible.csv", pid=config["PIDS"]))
-            if config["STANDARDIZATION"]["MERGE_ALL"]:
-                files_to_compute.extend(expand("data/processed/features/{pid}/z_all_sensor_features.csv", pid=config["PIDS"]))
-                files_to_compute.append("data/processed/features/all_participants/z_all_sensor_features.csv")

 for provider in config["PHONE_WIFI_CONNECTED"]["PROVIDERS"].keys():
    if config["PHONE_WIFI_CONNECTED"]["PROVIDERS"][provider]["COMPUTE"]:
@ -233,12 +173,6 @@ for provider in config["PHONE_ESM"]["PROVIDERS"].keys():
        files_to_compute.extend(expand("data/processed/features/{pid}/phone_esm.csv", pid=config["PIDS"]))
        # files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv",pid=config["PIDS"]))
        # files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
-        if provider in config["STANDARDIZATION"]["PROVIDERS"]["OTHER"]["LIST"] and config["STANDARDIZATION"]["PROVIDERS"]["OTHER"]["COMPUTE"] \
-            and config["PHONE_ESM"]["PROVIDERS"][provider]["STANDARDIZE_FEATURES"]:
-            files_to_compute.extend(expand("data/processed/features/{pid}/z_phone_esm.csv", pid=config["PIDS"]))
-            if config["STANDARDIZATION"]["MERGE_ALL"]:
-                files_to_compute.extend(expand("data/processed/features/{pid}/z_all_sensor_features.csv", pid=config["PIDS"]))
-                files_to_compute.append("data/processed/features/all_participants/z_all_sensor_features.csv")

 # We can delete these if's as soon as we add feature PROVIDERS to any of these sensors
 if isinstance(config["PHONE_APPLICATIONS_CRASHES"]["PROVIDERS"], dict):
@ -304,12 +238,6 @@ for provider in config["PHONE_LOCATIONS"]["PROVIDERS"].keys():
        files_to_compute.extend(expand("data/processed/features/{pid}/phone_locations.csv", pid=config["PIDS"]))
        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
-        if provider in config["STANDARDIZATION"]["PROVIDERS"]["OTHER"]["LIST"] and config["STANDARDIZATION"]["PROVIDERS"]["OTHER"]["COMPUTE"] \
-            and config["PHONE_LOCATIONS"]["PROVIDERS"][provider]["STANDARDIZE_FEATURES"]:
-            files_to_compute.extend(expand("data/processed/features/{pid}/z_phone_locations.csv", pid=config["PIDS"]))
-            if config["STANDARDIZATION"]["MERGE_ALL"]:
-                files_to_compute.extend(expand("data/processed/features/{pid}/z_all_sensor_features.csv", pid=config["PIDS"]))
-                files_to_compute.append("data/processed/features/all_participants/z_all_sensor_features.csv")

 for provider in config["FITBIT_CALORIES_INTRADAY"]["PROVIDERS"].keys():
    if config["FITBIT_CALORIES_INTRADAY"]["PROVIDERS"][provider]["COMPUTE"]:
@ -400,13 +328,6 @@ for provider in config["EMPATICA_ACCELEROMETER"]["PROVIDERS"].keys():
        files_to_compute.extend(expand("data/processed/features/{pid}/empatica_accelerometer.csv", pid=config["PIDS"]))
        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
-        if provider in config["STANDARDIZATION"]["PROVIDERS"] and config["STANDARDIZATION"]["PROVIDERS"][provider]["COMPUTE"] \
-            and config["EMPATICA_ACCELEROMETER"]["PROVIDERS"][provider]["WINDOWS"]["STANDARDIZE_FEATURES"]:
-            files_to_compute.extend(expand("data/interim/{pid}/empatica_accelerometer_features/z_empatica_accelerometer_{language}_{provider_key}_windows.csv", pid=config["PIDS"], language=get_script_language(config["STANDARDIZATION"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower()))
-            files_to_compute.extend(expand("data/processed/features/{pid}/z_empatica_accelerometer.csv", pid=config["PIDS"]))
-            if config["STANDARDIZATION"]["MERGE_ALL"]:
-                files_to_compute.extend(expand("data/processed/features/{pid}/z_all_sensor_features.csv", pid=config["PIDS"]))
-                files_to_compute.append("data/processed/features/all_participants/z_all_sensor_features.csv")
     
 for provider in config["EMPATICA_HEARTRATE"]["PROVIDERS"].keys():
    if config["EMPATICA_HEARTRATE"]["PROVIDERS"][provider]["COMPUTE"]:
@ -426,13 +347,6 @@ for provider in config["EMPATICA_TEMPERATURE"]["PROVIDERS"].keys():
        files_to_compute.extend(expand("data/processed/features/{pid}/empatica_temperature.csv", pid=config["PIDS"]))
        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
-        if provider in config["STANDARDIZATION"]["PROVIDERS"] and config["STANDARDIZATION"]["PROVIDERS"][provider]["COMPUTE"] \
-            and config["EMPATICA_TEMPERATURE"]["PROVIDERS"][provider]["WINDOWS"]["STANDARDIZE_FEATURES"]:
-            files_to_compute.extend(expand("data/interim/{pid}/empatica_temperature_features/z_empatica_temperature_{language}_{provider_key}_windows.csv", pid=config["PIDS"], language=get_script_language(config["STANDARDIZATION"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower()))
-            files_to_compute.extend(expand("data/processed/features/{pid}/z_empatica_temperature.csv", pid=config["PIDS"]))
-            if config["STANDARDIZATION"]["MERGE_ALL"]:
-                files_to_compute.extend(expand("data/processed/features/{pid}/z_all_sensor_features.csv", pid=config["PIDS"]))
-                files_to_compute.append("data/processed/features/all_participants/z_all_sensor_features.csv")

 for provider in config["EMPATICA_ELECTRODERMAL_ACTIVITY"]["PROVIDERS"].keys():
    if config["EMPATICA_ELECTRODERMAL_ACTIVITY"]["PROVIDERS"][provider]["COMPUTE"]:
@ -442,13 +356,6 @@ for provider in config["EMPATICA_ELECTRODERMAL_ACTIVITY"]["PROVIDERS"].keys():
        files_to_compute.extend(expand("data/processed/features/{pid}/empatica_electrodermal_activity.csv", pid=config["PIDS"]))
        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
-        if provider in config["STANDARDIZATION"]["PROVIDERS"] and config["STANDARDIZATION"]["PROVIDERS"][provider]["COMPUTE"] \
-            and config["EMPATICA_ELECTRODERMAL_ACTIVITY"]["PROVIDERS"][provider]["WINDOWS"]["STANDARDIZE_FEATURES"]:
-            files_to_compute.extend(expand("data/interim/{pid}/empatica_electrodermal_activity_features/z_empatica_electrodermal_activity_{language}_{provider_key}_windows.csv", pid=config["PIDS"], language=get_script_language(config["STANDARDIZATION"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower()))
-            files_to_compute.extend(expand("data/processed/features/{pid}/z_empatica_electrodermal_activity.csv", pid=config["PIDS"]))
-            if config["STANDARDIZATION"]["MERGE_ALL"]:
-                files_to_compute.extend(expand("data/processed/features/{pid}/z_all_sensor_features.csv", pid=config["PIDS"]))
-                files_to_compute.append("data/processed/features/all_participants/z_all_sensor_features.csv")

 for provider in config["EMPATICA_BLOOD_VOLUME_PULSE"]["PROVIDERS"].keys():
    if config["EMPATICA_BLOOD_VOLUME_PULSE"]["PROVIDERS"][provider]["COMPUTE"]:
@ -458,13 +365,6 @@ for provider in config["EMPATICA_BLOOD_VOLUME_PULSE"]["PROVIDERS"].keys():
        files_to_compute.extend(expand("data/processed/features/{pid}/empatica_blood_volume_pulse.csv", pid=config["PIDS"]))
        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
-        if provider in config["STANDARDIZATION"]["PROVIDERS"] and config["STANDARDIZATION"]["PROVIDERS"][provider]["COMPUTE"] \
-            and config["EMPATICA_BLOOD_VOLUME_PULSE"]["PROVIDERS"][provider]["WINDOWS"]["STANDARDIZE_FEATURES"]:
-            files_to_compute.extend(expand("data/interim/{pid}/empatica_blood_volume_pulse_features/z_empatica_blood_volume_pulse_{language}_{provider_key}_windows.csv", pid=config["PIDS"], language=get_script_language(config["STANDARDIZATION"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower()))
-            files_to_compute.extend(expand("data/processed/features/{pid}/z_empatica_blood_volume_pulse.csv", pid=config["PIDS"]))
-            if config["STANDARDIZATION"]["MERGE_ALL"]:
-                files_to_compute.extend(expand("data/processed/features/{pid}/z_all_sensor_features.csv", pid=config["PIDS"]))
-                files_to_compute.append("data/processed/features/all_participants/z_all_sensor_features.csv")

 for provider in config["EMPATICA_INTER_BEAT_INTERVAL"]["PROVIDERS"].keys():
    if config["EMPATICA_INTER_BEAT_INTERVAL"]["PROVIDERS"][provider]["COMPUTE"]:
@ -474,13 +374,6 @@ for provider in config["EMPATICA_INTER_BEAT_INTERVAL"]["PROVIDERS"].keys():
        files_to_compute.extend(expand("data/processed/features/{pid}/empatica_inter_beat_interval.csv", pid=config["PIDS"]))
        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
-        if provider in config["STANDARDIZATION"]["PROVIDERS"] and config["STANDARDIZATION"]["PROVIDERS"][provider]["COMPUTE"] \
-            and config["EMPATICA_INTER_BEAT_INTERVAL"]["PROVIDERS"][provider]["WINDOWS"]["STANDARDIZE_FEATURES"]:
-            files_to_compute.extend(expand("data/interim/{pid}/empatica_inter_beat_interval_features/z_empatica_inter_beat_interval_{language}_{provider_key}_windows.csv", pid=config["PIDS"], language=get_script_language(config["STANDARDIZATION"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower()))
-            files_to_compute.extend(expand("data/processed/features/{pid}/z_empatica_inter_beat_interval.csv", pid=config["PIDS"]))
-            if config["STANDARDIZATION"]["MERGE_ALL"]:
-                files_to_compute.extend(expand("data/processed/features/{pid}/z_all_sensor_features.csv", pid=config["PIDS"]))
-                files_to_compute.append("data/processed/features/all_participants/z_all_sensor_features.csv")
     
 if isinstance(config["EMPATICA_TAGS"]["PROVIDERS"], dict):
    for provider in config["EMPATICA_TAGS"]["PROVIDERS"].keys():
@ -517,24 +410,16 @@ for provider in config["ALL_CLEANING_INDIVIDUAL"]["PROVIDERS"].keys():
    if config["ALL_CLEANING_INDIVIDUAL"]["PROVIDERS"][provider]["COMPUTE"]:
        if provider == "STRAW":
            files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features_cleaned_" + provider.lower() + "_py.csv", pid=config["PIDS"]))
-            if config["ALL_CLEANING_INDIVIDUAL"]["CLEAN_STANDARDIZED"]:
-                files_to_compute.extend(expand("data/processed/features/{pid}/z_all_sensor_features_cleaned_" + provider.lower() + "_py.csv", pid=config["PIDS"]))
        else:
            files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features_cleaned_" + provider.lower() + "_R.csv", pid=config["PIDS"]))
-            if config["ALL_CLEANING_INDIVIDUAL"]["CLEAN_STANDARDIZED"]:
-                files_to_compute.extend(expand("data/processed/features/{pid}/z_all_sensor_features_cleaned_" + provider.lower() + "_R.csv", pid=config["PIDS"]))

 for provider in config["ALL_CLEANING_OVERALL"]["PROVIDERS"].keys():
    if config["ALL_CLEANING_OVERALL"]["PROVIDERS"][provider]["COMPUTE"]:
        if provider == "STRAW":
-            files_to_compute.extend(expand("data/processed/features/all_participants/all_sensor_features_cleaned_" + provider.lower() +"_py.csv"))
-            if config["ALL_CLEANING_OVERALL"]["CLEAN_STANDARDIZED"]:
-                files_to_compute.extend(expand("data/processed/features/all_participants/z_all_sensor_features_cleaned_" + provider.lower() +"_py.csv"))
+            for target in config["PARAMS_FOR_ANALYSIS"]["TARGET"]["ALL_LABELS"]:
+                files_to_compute.extend(expand("data/processed/features/all_participants/all_sensor_features_cleaned_" + provider.lower() +"_py_(" + target + ").csv"))
        else:
-            files_to_compute.extend(expand("data/processed/features/all_participants/all_sensor_features_cleaned_" + provider.lower() +"_R.csv"))
-            if config["ALL_CLEANING_OVERALL"]["CLEAN_STANDARDIZED"]:
-                files_to_compute.extend(expand("data/processed/features/all_participants/z_all_sensor_features_cleaned_" + provider.lower() +"_R.csv"))
-        
+            files_to_compute.extend(expand("data/processed/features/all_participants/all_sensor_features_cleaned_" + provider.lower() +"_R.csv"))     

 # Baseline features
 if config["PARAMS_FOR_ANALYSIS"]["BASELINE"]["COMPUTE"]:
@ -545,12 +430,9 @@ if config["PARAMS_FOR_ANALYSIS"]["BASELINE"]["COMPUTE"]:

 # Targets (labels)
 if config["PARAMS_FOR_ANALYSIS"]["TARGET"]["COMPUTE"]:
-    # files_to_compute.extend(expand("data/processed/models/individual_model/{pid}/input.csv", pid=config["PIDS"]))
-    # files_to_compute.extend(expand("data/processed/models/population_model/input.csv"))
-    files_to_compute.extend(expand("data/processed/models/individual_model/{pid}/z_input.csv", pid=config["PIDS"]))
-    files_to_compute.extend(expand("data/processed/models/population_model/z_input.csv"))
-
-#files_to_compute.extend(expand("data/processed/models/individual_model/{pid}/output_{cv_method}/baselines.csv", pid=config["PIDS"], cv_method=config["PARAMS_FOR_ANALYSIS"]["CV_METHODS"]))
+    files_to_compute.extend(expand("data/processed/models/individual_model/{pid}/input.csv", pid=config["PIDS"]))
+    for target in config["PARAMS_FOR_ANALYSIS"]["TARGET"]["ALL_LABELS"]:
+        files_to_compute.extend(expand("data/processed/models/population_model/input_" + target + ".csv"))

 rule all:
    input:
--- a/automl_test.py
+++ b/automl_test.py
@ -0,0 +1,57 @@
+from pprint import pprint
+import sklearn.metrics
+import autosklearn.regression
+
+import datetime
+import importlib
+import os
+import sys
+
+import numpy as np
+import matplotlib.pyplot as plt
+import pandas as pd
+import seaborn as sns
+import yaml
+
+from sklearn import linear_model, svm, kernel_ridge, gaussian_process
+from sklearn.model_selection import LeaveOneGroupOut, cross_val_score, train_test_split
+from sklearn.metrics import mean_squared_error, r2_score
+from sklearn.impute import SimpleImputer
+
+model_input = pd.read_csv("data/processed/models/population_model/input_PANAS_negative_affect_mean.csv") # Standardizirani podatki
+
+model_input.dropna(axis=1, how="all", inplace=True)
+model_input.dropna(axis=0, how="any", subset=["target"], inplace=True)
+
+categorical_feature_colnames = ["gender", "startlanguage"]
+categorical_feature_colnames += [col for col in model_input.columns if "mostcommonactivity" in col or "homelabel" in col]
+categorical_features = model_input[categorical_feature_colnames].copy()
+mode_categorical_features = categorical_features.mode().iloc[0]
+categorical_features = categorical_features.fillna(mode_categorical_features)
+categorical_features = categorical_features.apply(lambda col: col.astype("category"))
+if not categorical_features.empty:
+    categorical_features = pd.get_dummies(categorical_features)
+numerical_features = model_input.drop(categorical_feature_colnames, axis=1)
+model_in = pd.concat([numerical_features, categorical_features], axis=1)
+
+index_columns = ["local_segment", "local_segment_label", "local_segment_start_datetime", "local_segment_end_datetime"]
+model_in.set_index(index_columns, inplace=True)
+
+X_train, X_test, y_train, y_test = train_test_split(model_in.drop(["target", "pid"], axis=1), model_in["target"], test_size=0.30)
+
+automl = autosklearn.regression.AutoSklearnRegressor(
+    time_left_for_this_task=7200,
+    per_run_time_limit=120
+)
+automl.fit(X_train, y_train, dataset_name='straw')
+
+print(automl.leaderboard())
+pprint(automl.show_models(), indent=4)
+
+train_predictions = automl.predict(X_train)
+print("Train R2 score:", sklearn.metrics.r2_score(y_train, train_predictions))
+test_predictions = automl.predict(X_test)
+print("Test R2 score:", sklearn.metrics.r2_score(y_test, test_predictions))
+
+import sys
+sys.exit()
--- a/config.yaml
+++ b/config.yaml
@ -3,7 +3,7 @@
 ########################################################################################################################

 # See https://www.rapids.science/latest/setup/configuration/#participant-files
-PIDS: ['p031', 'p032', 'p033', 'p034', 'p035', 'p036', 'p037', 'p038', 'p039', 'p040', 'p042', 'p043', 'p044', 'p045', 'p046', 'p049', 'p050', 'p052', 'p053', 'p054', 'p055', 'p057', 'p058', 'p059', 'p060', 'p061', 'p062', 'p064', 'p067', 'p068', 'p069', 'p070', 'p071', 'p072', 'p073', 'p074', 'p075', 'p076', 'p077', 'p078', 'p079', 'p080', 'p081', 'p082', 'p083', 'p084', 'p085', 'p086', 'p088', 'p089', 'p090', 'p091', 'p092', 'p093', 'p106', 'p107']
+PIDS: ['p03'] #['p031', 'p032', 'p033', 'p034', 'p035', 'p036', 'p037', 'p038', 'p039', 'p040', 'p042', 'p043', 'p044', 'p045', 'p046', 'p049', 'p050', 'p052', 'p053', 'p054', 'p055', 'p057', 'p058', 'p059', 'p060', 'p061', 'p062', 'p064', 'p067', 'p068', 'p069', 'p070', 'p071', 'p072', 'p073', 'p074', 'p075', 'p076', 'p077', 'p078', 'p079', 'p080', 'p081', 'p082', 'p083', 'p084', 'p085', 'p086', 'p088', 'p089', 'p090', 'p091', 'p092', 'p093', 'p106', 'p107']

 # See https://www.rapids.science/latest/setup/configuration/#automatic-creation-of-participant-files
 CREATE_PARTICIPANT_FILES:
@ -21,9 +21,12 @@ CREATE_PARTICIPANT_FILES:

 # See https://www.rapids.science/latest/setup/configuration/#time-segments
 TIME_SEGMENTS: &time_segments
-  TYPE: PERIODIC # FREQUENCY, PERIODIC, EVENT
-  FILE: "data/external/timesegments_daily.csv"
+  TYPE: EVENT # FREQUENCY, PERIODIC, EVENT
+  FILE: "data/external/straw_events.csv"
  INCLUDE_PAST_PERIODIC_SEGMENTS: TRUE # Only relevant if TYPE=PERIODIC, see docs
+  TAILORED_EVENTS: # Only relevant if TYPE=EVENT
+    COMPUTE: True
+    PARAMETER_ONE: "something"

 # See https://www.rapids.science/latest/setup/configuration/#timezone-of-your-study
 TIMEZONE: 
@ -70,7 +73,6 @@ PHONE_ACCELEROMETER:
      COMPUTE: False
      FEATURES: ["maxmagnitude", "minmagnitude", "avgmagnitude", "medianmagnitude", "stdmagnitude"]
      SRC_SCRIPT: src/features/phone_accelerometer/rapids/main.py
-    
    PANDA:
      COMPUTE: False
      VALID_SENSED_MINUTES: False
@ -93,7 +95,6 @@ PHONE_ACTIVITY_RECOGNITION:
        STATIONARY: ["still", "tilting"]
        MOBILE: ["on_foot", "walking", "running", "on_bicycle"]
        VEHICLE: ["in_vehicle"]
-      STANDARDIZE_FEATURES: True
      SRC_SCRIPT: src/features/phone_activity_recognition/rapids/main.py

 # See https://www.rapids.science/latest/features/phone-applications-crashes/
@ -134,7 +135,6 @@ PHONE_APPLICATIONS_FOREGROUND:
        APP_EPISODES: ["countepisode", "minduration", "maxduration", "meanduration", "sumduration"]
      IGNORE_EPISODES_SHORTER_THAN: 0 # in minutes, set to 0 to disable
      IGNORE_EPISODES_LONGER_THAN: 300 # in minutes, set to 0 to disable
-      STANDARDIZE_FEATURES: True
      SRC_SCRIPT: src/features/phone_applications_foreground/rapids/main.py

 # See https://www.rapids.science/latest/features/phone-applications-notifications/
@ -155,7 +155,6 @@ PHONE_BATTERY:
    RAPIDS:
      COMPUTE: True
      FEATURES: ["countdischarge", "sumdurationdischarge", "countcharge", "sumdurationcharge", "avgconsumptionrate", "maxconsumptionrate"]
-      STANDARDIZE_FEATURES: True
      SRC_SCRIPT: src/features/phone_battery/rapids/main.py

 # See https://www.rapids.science/latest/features/phone-bluetooth/
@ -163,9 +162,8 @@ PHONE_BLUETOOTH:
  CONTAINER: bluetooth
  PROVIDERS:
    RAPIDS:
-      COMPUTE: True
+      COMPUTE: False
      FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"]
-      STANDARDIZE_FEATURES: True
      SRC_SCRIPT: src/features/phone_bluetooth/rapids/main.R

    DORYAB:
@ -183,7 +181,6 @@ PHONE_BLUETOOTH:
            DEVICES: ["countscans", "uniquedevices", "meanscans", "stdscans"]
            SCANS_MOST_FREQUENT_DEVICE: ["withinsegments", "acrosssegments", "acrossdataset"]
            SCANS_LEAST_FREQUENT_DEVICE: ["withinsegments", "acrosssegments", "acrossdataset"]
-      STANDARDIZE_FEATURES: True
      SRC_SCRIPT: src/features/phone_bluetooth/doryab/main.py

 # See https://www.rapids.science/latest/features/phone-calls/
@ -198,7 +195,6 @@ PHONE_CALLS:
        missed:  [count, distinctcontacts, timefirstcall, timelastcall, countmostfrequentcontact]
        incoming: [count, distinctcontacts, meanduration, sumduration, minduration, maxduration, stdduration, modeduration, entropyduration, timefirstcall, timelastcall, countmostfrequentcontact]
        outgoing: [count, distinctcontacts, meanduration, sumduration, minduration, maxduration, stdduration, modeduration, entropyduration, timefirstcall, timelastcall, countmostfrequentcontact]
-      STANDARDIZE_FEATURES: True
      SRC_SCRIPT: src/features/phone_calls/rapids/main.R

 # See https://www.rapids.science/latest/features/phone-conversation/
@ -238,7 +234,6 @@ PHONE_DATA_YIELD:
      COMPUTE: True
      FEATURES: [ratiovalidyieldedminutes, ratiovalidyieldedhours]
      MINUTE_RATIO_THRESHOLD_FOR_VALID_YIELDED_HOURS: 0.5 # 0 to 1, minimum percentage of valid minutes in an hour to be considered valid.
-      STANDARDIZE_FEATURES: True
      SRC_SCRIPT: src/features/phone_data_yield/rapids/main.R

 PHONE_ESM:
@ -248,7 +243,6 @@ PHONE_ESM:
      COMPUTE: True
      SCALES: ["PANAS_positive_affect", "PANAS_negative_affect", "JCQ_job_demand", "JCQ_job_control", "JCQ_supervisor_support", "JCQ_coworker_support"]
      FEATURES: [mean]
-      STANDARDIZE_FEATURES: True
      SRC_SCRIPT: src/features/phone_esm/straw/main.py

 # See https://www.rapids.science/latest/features/phone-keyboard/
@ -267,7 +261,6 @@ PHONE_LIGHT:
    RAPIDS:
      COMPUTE: True
      FEATURES: ["count", "maxlux", "minlux", "avglux", "medianlux", "stdlux"]
-      STANDARDIZE_FEATURES: True
      SRC_SCRIPT: src/features/phone_light/rapids/main.py

 # See https://www.rapids.science/latest/features/phone-locations/
@ -292,7 +285,6 @@ PHONE_LOCATIONS:
      MINIMUM_DAYS_TO_DETECT_HOME_CHANGES: 3
      CLUSTERING_ALGORITHM: DBSCAN # DBSCAN, OPTICS
      RADIUS_FOR_HOME: 100
-      STANDARDIZE_FEATURES: True
      SRC_SCRIPT: src/features/phone_locations/doryab/main.py

    BARNETT:
@ -300,7 +292,6 @@ PHONE_LOCATIONS:
      FEATURES: ["hometime","disttravelled","rog","maxdiam","maxhomedist","siglocsvisited","avgflightlen","stdflightlen","avgflightdur","stdflightdur","probpause","siglocentropy","circdnrtn","wkenddayrtn"]
      IF_MULTIPLE_TIMEZONES: USE_MOST_COMMON
      MINUTES_DATA_USED: False # Use this for quality control purposes, how many minutes of data (location coordinates gruped by minute) were used to compute features
-      STANDARDIZE_FEATURES: True
      SRC_SCRIPT: src/features/phone_locations/barnett/main.R

 # See https://www.rapids.science/latest/features/phone-log/
@ -320,7 +311,6 @@ PHONE_MESSAGES:
      FEATURES: 
        received: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact]
        sent: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact]
-      STANDARDIZE_FEATURES: True
      SRC_SCRIPT: src/features/phone_messages/rapids/main.R

 # See https://www.rapids.science/latest/features/phone-screen/
@ -334,7 +324,6 @@ PHONE_SCREEN:
      IGNORE_EPISODES_LONGER_THAN: 360 # in minutes, set to 0 to disable
      FEATURES: ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "stdduration", "firstuseafter"] # "episodepersensedminutes" needs to be added later
      EPISODE_TYPES: ["unlock"]
-      STANDARDIZE_FEATURES: True
      SRC_SCRIPT: src/features/phone_screen/rapids/main.py

 # See https://www.rapids.science/latest/features/phone-wifi-connected/
@ -353,7 +342,6 @@ PHONE_WIFI_VISIBLE:
    RAPIDS:
      COMPUTE: True
      FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"]
-      STANDARDIZE_FEATURES: True
      SRC_SCRIPT: src/features/phone_wifi_visible/rapids/main.R


@ -455,7 +443,6 @@ FITBIT_SLEEP_INTRADAY:
        UNIFIED: [awake, asleep]
      SLEEP_TYPES: [main, nap, all]
      SRC_SCRIPT: src/features/fitbit_sleep_intraday/rapids/main.py
-  
    PRICE:
      COMPUTE: False
      FEATURES: [avgduration, avgratioduration, avgstarttimeofepisodemain, avgendtimeofepisodemain, avgmidpointofepisodemain, stdstarttimeofepisodemain, stdendtimeofepisodemain, stdmidpointofepisodemain, socialjetlag, rmssdmeanstarttimeofepisodemain, rmssdmeanendtimeofepisodemain, rmssdmeanmidpointofepisodemain, rmssdmedianstarttimeofepisodemain, rmssdmedianendtimeofepisodemain, rmssdmedianmidpointofepisodemain]
@ -528,7 +515,6 @@ EMPATICA_ACCELEROMETER:
        COMPUTE: True
        WINDOW_LENGTH: 15 # specify window length in seconds
        SECOND_ORDER_FEATURES: ['mean', 'median', 'sd', 'nlargest', 'nsmallest', 'count_windows']
-        STANDARDIZE_FEATURES: True
      SRC_SCRIPT: src/features/empatica_accelerometer/cr/main.py


@ -557,7 +543,6 @@ EMPATICA_TEMPERATURE:
        COMPUTE: True
        WINDOW_LENGTH: 300 # specify window length in seconds
        SECOND_ORDER_FEATURES: ['mean', 'median', 'sd', 'nlargest', 'nsmallest', 'count_windows']
-        STANDARDIZE_FEATURES: True
      SRC_SCRIPT: src/features/empatica_temperature/cr/main.py

 # See https://www.rapids.science/latest/features/empatica-electrodermal-activity/
@ -579,7 +564,6 @@ EMPATICA_ELECTRODERMAL_ACTIVITY:
        COMPUTE: True
        WINDOW_LENGTH: 60 # specify window length in seconds
        SECOND_ORDER_FEATURES: ['mean', 'median', 'sd', 'nlargest', 'nsmallest', count_windows, eda_num_peaks_non_zero]
-        STANDARDIZE_FEATURES: True
        IMPUTE_NANS: True
      SRC_SCRIPT: src/features/empatica_electrodermal_activity/cr/main.py

@ -599,7 +583,6 @@ EMPATICA_BLOOD_VOLUME_PULSE:
        COMPUTE: True
        WINDOW_LENGTH: 300 # specify window length in seconds
        SECOND_ORDER_FEATURES: ['mean', 'median', 'sd', 'nlargest', 'nsmallest', 'count_windows', 'hrv_num_windows_non_nan']
-        STANDARDIZE_FEATURES: True
      SRC_SCRIPT: src/features/empatica_blood_volume_pulse/cr/main.py

 # See https://www.rapids.science/latest/features/empatica-inter-beat-interval/
@ -619,7 +602,6 @@ EMPATICA_INTER_BEAT_INTERVAL:
        COMPUTE: True
        WINDOW_LENGTH: 300 # specify window length in seconds
        SECOND_ORDER_FEATURES: ['mean', 'median', 'sd', 'nlargest', 'nsmallest', 'count_windows', 'hrv_num_windows_non_nan']
-        STANDARDIZE_FEATURES: True
      SRC_SCRIPT: src/features/empatica_inter_beat_interval/cr/main.py

 # See https://www.rapids.science/latest/features/empatica-tags/
@ -667,10 +649,9 @@ HEATMAP_FEATURE_CORRELATION_MATRIX:
 ########################################################################################################################

 ALL_CLEANING_INDIVIDUAL:
-  CLEAN_STANDARDIZED: True
  PROVIDERS:
    RAPIDS:
-      COMPUTE: True
+      COMPUTE: False
      IMPUTE_SELECTED_EVENT_FEATURES:
        COMPUTE: False
        MIN_DATA_YIELDED_MINUTES_TO_IMPUTE: 0.33
@ -684,28 +665,25 @@ ALL_CLEANING_INDIVIDUAL:
        MIN_OVERLAP_FOR_CORR_THRESHOLD: 0.5
        CORR_THRESHOLD: 0.95
      SRC_SCRIPT: src/features/all_cleaning_individual/rapids/main.R
-    STRAW: # currently the same as RAPIDS provider with a change in selecting the imputation type
+    STRAW:
      COMPUTE: True
-      IMPUTE_PHONE_SELECTED_EVENT_FEATURES:
-        COMPUTE: False
-        TYPE: median # options: zero, mean, median or k-nearest
-        MIN_DATA_YIELDED_MINUTES_TO_IMPUTE: 0.33
-      COLS_NAN_THRESHOLD: 1 # set to 1 to disable
+      PHONE_DATA_YIELD_FEATURE: RATIO_VALID_YIELDED_MINUTES # RATIO_VALID_YIELDED_HOURS or RATIO_VALID_YIELDED_MINUTES
+      PHONE_DATA_YIELD_RATIO_THRESHOLD: 0.5 # set to 0 to disable
+      EMPATICA_DATA_YIELD_RATIO_THRESHOLD: 0.5 # set to 0 to disable
+      ROWS_NAN_THRESHOLD: 0.33 # set to 1 to disable
+      COLS_NAN_THRESHOLD: 0.9 # set to 1 to remove only columns that contains all (100% of) NaN
      COLS_VAR_THRESHOLD: True
-      ROWS_NAN_THRESHOLD: 1 # set to 1 to disable
-      DATA_YIELD_FEATURE: RATIO_VALID_YIELDED_HOURS # RATIO_VALID_YIELDED_HOURS or RATIO_VALID_YIELDED_MINUTES
-      DATA_YIELD_RATIO_THRESHOLD: 0 # set to 0 to disable
      DROP_HIGHLY_CORRELATED_FEATURES:
        COMPUTE: True
        MIN_OVERLAP_FOR_CORR_THRESHOLD: 0.5
        CORR_THRESHOLD: 0.95
+      STANDARDIZATION: True
      SRC_SCRIPT: src/features/all_cleaning_individual/straw/main.py

 ALL_CLEANING_OVERALL:
-  CLEAN_STANDARDIZED: True
  PROVIDERS:
    RAPIDS:
-      COMPUTE: True
+      COMPUTE: False
      IMPUTE_SELECTED_EVENT_FEATURES:
        COMPUTE: False
        MIN_DATA_YIELDED_MINUTES_TO_IMPUTE: 0.33
@ -719,40 +697,22 @@ ALL_CLEANING_OVERALL:
        MIN_OVERLAP_FOR_CORR_THRESHOLD: 0.5
        CORR_THRESHOLD: 0.95
      SRC_SCRIPT: src/features/all_cleaning_overall/rapids/main.R
-    STRAW: # currently the same as RAPIDS provider with a change in selecting the imputation type
+    STRAW:
      COMPUTE: True
-      IMPUTE_PHONE_SELECTED_EVENT_FEATURES:
-        COMPUTE: False
-        TYPE: median # options: zero, mean, median or k-nearest
-        MIN_DATA_YIELDED_MINUTES_TO_IMPUTE: 0.33
-      COLS_NAN_THRESHOLD: 1 # set to 1 to disable
+      PHONE_DATA_YIELD_FEATURE: RATIO_VALID_YIELDED_MINUTES # RATIO_VALID_YIELDED_HOURS or RATIO_VALID_YIELDED_MINUTES
+      PHONE_DATA_YIELD_RATIO_THRESHOLD: 0.5 # set to 0 to disable
+      EMPATICA_DATA_YIELD_RATIO_THRESHOLD: 0.5 # set to 0 to disable
+      ROWS_NAN_THRESHOLD: 0.33 # set to 1 to disable
+      COLS_NAN_THRESHOLD: 0.8 # set to 1 to remove only columns that contains all (100% of) NaN
      COLS_VAR_THRESHOLD: True
-      ROWS_NAN_THRESHOLD: 1 # set to 1 to disable
-      DATA_YIELD_FEATURE: RATIO_VALID_YIELDED_HOURS # RATIO_VALID_YIELDED_HOURS or RATIO_VALID_YIELDED_MINUTES
-      DATA_YIELD_RATIO_THRESHOLD: 0 # set to 0 to disable
      DROP_HIGHLY_CORRELATED_FEATURES:
        COMPUTE: True
        MIN_OVERLAP_FOR_CORR_THRESHOLD: 0.5
        CORR_THRESHOLD: 0.95
+      STANDARDIZATION: True
      SRC_SCRIPT: src/features/all_cleaning_overall/straw/main.py


-########################################################################################################################
-#                                                 Z-score standardization                                              #
-########################################################################################################################
-
-STANDARDIZATION: # Standardization for both providers is executed if only one of two providers is marked COMPUTE: TRUE
-  MERGE_ALL: True # Creates the joint standardized file for each participant and all participants. Similar to merge_sensor_features_for_all_participants rule  
-  PROVIDERS:
-    CR:
-      COMPUTE: True
-      SRC_SCRIPT: src/features/standardization/main.py
-    OTHER:
-      COMPUTE: True
-      LIST: [RAPIDS, DORYAB, BARNETT, STRAW]
-      SRC_SCRIPT: src/features/standardization/main.py
-
-
 ########################################################################################################################
 #                                                      Baseline                                                        #
 ########################################################################################################################
@ -772,3 +732,4 @@ PARAMS_FOR_ANALYSIS:
  TARGET:
    COMPUTE: True
    LABEL: PANAS_negative_affect_mean
+    ALL_LABELS: [PANAS_positive_affect_mean, PANAS_negative_affect_mean, "JCQ_job_demand_mean", "JCQ_job_control_mean", "JCQ_supervisor_support_mean", "JCQ_coworker_support_mean"]
--- a/data/external/participant_files/p01.yaml
+++ b/data/external/participant_files/p01.yaml
@ -1,11 +1,11 @@
 PHONE:
-  DEVICE_IDS: [a748ee1a-1d0b-4ae9-9074-279a2b6ba524] # the participant's AWARE device id
-  PLATFORMS: [android] # or ios
-  LABEL: MyTestP01 # any string
-  START_DATE: 2020-01-01 # this can also be empty
-  END_DATE: 2021-01-01 # this can also be empty
+  DEVICE_IDS: [4b62a655-cbf0-4ac0-a448-06726f45b56a]
+  PLATFORMS: [android]
+  LABEL: uploader_53573
+  START_DATE: 2021-05-21 09:21:24
+  END_DATE: 2021-07-12 17:32:07
 EMPATICA:
-  DEVICE_IDS: [empatica1]
-  LABEL: test01
-  START_DATE: 
-  END_DATE: 
+  DEVICE_IDS: [uploader_53573]
+  LABEL: uploader_53573
+  START_DATE: 2021-05-21 09:21:24
+  END_DATE: 2021-07-12 17:32:07
--- a/data/external/timesegments_daily.csv
+++ b/data/external/timesegments_daily.csv
@ -1,2 +1,3 @@
 label,start_time,length,repeats_on,repeats_value
 daily,04:00:00,23H 59M 59S,every_day,0
+working_day,04:00:00,18H 00M 00S,every_day,0
--- a/environment.yml
+++ b/environment.yml
@ -86,8 +86,6 @@ dependencies:
  - readline=8.0
  - requests=2.25.0
  - retrying=1.3.3
-  - scikit-learn=0.23.2
-  - scipy=1.5.2
  - setuptools=51.0.0
  - six=1.15.0
  - smmap=3.0.4
@ -107,34 +105,61 @@ dependencies:
  - zlib=1.2.11
  - pip:
    - amply==0.1.4
+    - auto-sklearn==0.14.7
    - bidict==0.22.0
    - biosppy==0.8.0
+    - build==0.8.0
    - cached-property==1.5.2
+    - cloudpickle==2.2.0
    - configargparse==0.15.1
+    - configspace==0.4.21
    - cr-features==0.2.1
    - cycler==0.11.0
+    - cython==0.29.32
+    - dask==2022.2.0
    - decorator==4.4.2
+    - distributed==2022.2.0
+    - distro==1.7.0
+    - emcee==3.1.2
    - fonttools==4.33.2
+    - fsspec==2022.8.2
    - h5py==3.6.0
+    - heapdict==1.0.1
    - hmmlearn==0.2.7
    - ipython-genutils==0.2.0
    - jupyter-core==4.6.3
    - kiwisolver==1.4.2
+    - liac-arff==2.5.0
+    - locket==1.0.0
    - matplotlib==3.5.1
+    - msgpack==1.0.4
    - nbformat==5.0.7
    - opencv-python==4.5.5.64
    - packaging==21.3
+    - partd==1.3.0
    - peakutils==1.3.3
+    - pep517==0.13.0
    - pillow==9.1.0
    - pulp==2.4
+    - pynisher==0.6.4
    - pyparsing==2.4.7
+    - pyrfr==0.8.3
    - pyrsistent==0.15.5
    - pywavelets==1.3.0
    - ratelimiter==1.2.0.post0
+    - scikit-learn==0.24.2
+    - scipy==1.7.3
    - seaborn==0.11.2
    - shortuuid==1.0.8
+    - smac==1.2
    - snakemake==5.30.2
+    - sortedcontainers==2.4.0
+    - tblib==1.7.0
+    - tomli==2.0.1
+    - toolz==0.12.0
    - toposort==1.5
+    - tornado==6.2
    - traitlets==4.3.3
    - typing-extensions==4.2.0
+    - zict==2.2.0
 prefix: /opt/conda/envs/rapids
--- a/rules/common.smk
+++ b/rules/common.smk
@ -40,15 +40,6 @@ def find_features_files(wildcards):
            feature_files.extend(expand("data/interim/{{pid}}/{sensor_key}_features/{sensor_key}_{language}_{provider_key}.csv", sensor_key=wildcards.sensor_key.lower(), language=get_script_language(provider["SRC_SCRIPT"]), provider_key=provider_key.lower()))
    return(feature_files)

-def find_empaticas_standardized_features_files(wildcards):
-    feature_files = []
-    if "empatica" in wildcards.sensor_key:
-        for provider_key, provider in config[(wildcards.sensor_key).upper()]["PROVIDERS"].items():
-            if provider["COMPUTE"] and provider.get("WINDOWS", False) and provider["WINDOWS"]["COMPUTE"]:
-                if "empatica" in wildcards.sensor_key:
-                    feature_files.extend(expand("data/interim/{{pid}}/{sensor_key}_features/z_{sensor_key}_{language}_{provider_key}.csv", sensor_key=wildcards.sensor_key.lower(), language=get_script_language(provider["SRC_SCRIPT"]), provider_key=provider_key.lower()))
-    return(feature_files)
-
 def find_joint_non_empatica_sensor_files(wildcards):
    joined_files = []
    for config_key in config.keys():
@ -82,18 +73,6 @@ def input_merge_sensor_features_for_individual_participants(wildcards):
                    break
    return feature_files

-def input_merge_standardized_sensor_features_for_individual_participants(wildcards):
-    feature_files = []
-    for config_key in config.keys():
-        if config_key.startswith(("PHONE", "FITBIT", "EMPATICA")) and "PROVIDERS" in config[config_key] and isinstance(config[config_key]["PROVIDERS"], dict):
-            for provider_key, provider in config[config_key]["PROVIDERS"].items():
-                if "COMPUTE" in provider.keys() and provider["COMPUTE"] and ("STANDARDIZE_FEATURES" in provider.keys() and provider["STANDARDIZE_FEATURES"] or 
-                "WINDOWS" in provider.keys() and "STANDARDIZE_FEATURES" in provider["WINDOWS"].keys() and provider["WINDOWS"]["STANDARDIZE_FEATURES"]):
-                    feature_files.append("data/processed/features/{pid}/z_" + config_key.lower() + ".csv")
-                    break
-    
-    return feature_files
-
 def get_phone_sensor_names():
    phone_sensor_names = []
    for config_key in config.keys():
--- a/rules/features.smk
+++ b/rules/features.smk
@ -796,20 +796,6 @@ rule empatica_accelerometer_python_features:
    script:
        "../src/features/entry.py"

-rule empatica_accelerometer_python_features_standardization:
-    input:
-        windows_features_data = "data/interim/{pid}/empatica_accelerometer_features/empatica_accelerometer_python_{provider_key}_windows.csv"
-    params:
-        provider = config["STANDARDIZATION"]["PROVIDERS"]["CR"],
-        provider_key = "{provider_key}",
-        sensor_key = "empatica_accelerometer",
-        provider_main = config["EMPATICA_ACCELEROMETER"]["PROVIDERS"]["CR"]
-    output:
-        "data/interim/{pid}/empatica_accelerometer_features/z_empatica_accelerometer_python_{provider_key}.csv",
-        "data/interim/{pid}/empatica_accelerometer_features/z_empatica_accelerometer_python_{provider_key}_windows.csv"
-    script:
-        "../src/features/standardization/main.py"
-
 rule empatica_accelerometer_r_features:
    input:
        sensor_data = "data/raw/{pid}/empatica_accelerometer_with_datetime.csv",
@ -864,20 +850,6 @@ rule empatica_temperature_python_features:
    script:
        "../src/features/entry.py"

-rule empatica_temperature_python_features_standardization:
-    input:
-        windows_features_data = "data/interim/{pid}/empatica_temperature_features/empatica_temperature_python_{provider_key}_windows.csv"
-    params:
-        provider = config["STANDARDIZATION"]["PROVIDERS"]["CR"],
-        provider_key = "{provider_key}",
-        sensor_key = "empatica_temperature",
-        provider_main = config["EMPATICA_TEMPERATURE"]["PROVIDERS"]["CR"]
-    output:
-        "data/interim/{pid}/empatica_temperature_features/z_empatica_temperature_python_{provider_key}.csv",
-        "data/interim/{pid}/empatica_temperature_features/z_empatica_temperature_python_{provider_key}_windows.csv"
-    script:
-        "../src/features/standardization/main.py"
-
 rule empatica_temperature_r_features:
    input:
        sensor_data = "data/raw/{pid}/empatica_temperature_with_datetime.csv",
@ -905,20 +877,6 @@ rule empatica_electrodermal_activity_python_features:
    script:
        "../src/features/entry.py"

-rule empatica_electrodermal_activity_python_features_standardization:
-    input:
-        windows_features_data = "data/interim/{pid}/empatica_electrodermal_activity_features/empatica_electrodermal_activity_python_{provider_key}_windows.csv"
-    params:
-        provider = config["STANDARDIZATION"]["PROVIDERS"]["CR"],
-        provider_key = "{provider_key}",
-        sensor_key = "empatica_electrodermal_activity",
-        provider_main = config["EMPATICA_ELECTRODERMAL_ACTIVITY"]["PROVIDERS"]["CR"]
-    output:
-        "data/interim/{pid}/empatica_electrodermal_activity_features/z_empatica_electrodermal_activity_python_{provider_key}.csv",
-        "data/interim/{pid}/empatica_electrodermal_activity_features/z_empatica_electrodermal_activity_python_{provider_key}_windows.csv"
-    script:
-        "../src/features/standardization/main.py"
-
 rule empatica_electrodermal_activity_r_features:
    input:
        sensor_data = "data/raw/{pid}/empatica_electrodermal_activity_with_datetime.csv",
@ -946,20 +904,6 @@ rule empatica_blood_volume_pulse_python_features:
    script:
        "../src/features/entry.py"

-rule empatica_blood_volume_pulse_python_cr_features_standardization:
-    input:
-        windows_features_data = "data/interim/{pid}/empatica_blood_volume_pulse_features/empatica_blood_volume_pulse_python_{provider_key}_windows.csv"
-    params:
-        provider = config["STANDARDIZATION"]["PROVIDERS"]["CR"],
-        provider_key = "{provider_key}",
-        sensor_key = "empatica_blood_volume_pulse",
-        provider_main = config["EMPATICA_BLOOD_VOLUME_PULSE"]["PROVIDERS"]["CR"]
-    output:
-        "data/interim/{pid}/empatica_blood_volume_pulse_features/z_empatica_blood_volume_pulse_python_{provider_key}.csv",
-        "data/interim/{pid}/empatica_blood_volume_pulse_features/z_empatica_blood_volume_pulse_python_{provider_key}_windows.csv"
-    script:
-        "../src/features/standardization/main.py"
-
 rule empatica_blood_volume_pulse_r_features:
    input:
        sensor_data = "data/raw/{pid}/empatica_blood_volume_pulse_with_datetime.csv",
@ -987,20 +931,6 @@ rule empatica_inter_beat_interval_python_features:
    script:
        "../src/features/entry.py"

-rule empatica_inter_beat_interval_python_features_standardization:
-    input:
-        windows_features_data = "data/interim/{pid}/empatica_inter_beat_interval_features/empatica_inter_beat_interval_python_{provider_key}_windows.csv"
-    params:
-        provider = config["STANDARDIZATION"]["PROVIDERS"]["CR"],
-        provider_key = "{provider_key}",
-        sensor_key = "empatica_inter_beat_interval",
-        provider_main = config["EMPATICA_INTER_BEAT_INTERVAL"]["PROVIDERS"]["CR"]
-    output:
-        "data/interim/{pid}/empatica_inter_beat_interval_features/z_empatica_inter_beat_interval_python_{provider_key}.csv",
-        "data/interim/{pid}/empatica_inter_beat_interval_features/z_empatica_inter_beat_interval_python_{provider_key}_windows.csv"
-    script:
-        "../src/features/standardization/main.py"
-
 rule empatica_inter_beat_interval_r_features:
    input:
        sensor_data = "data/raw/{pid}/empatica_inter_beat_interval_with_datetime.csv",
@ -1048,38 +978,6 @@ rule merge_sensor_features_for_individual_participants:
    script:
        "../src/features/utils/merge_sensor_features_for_individual_participants.R"

-rule join_standardized_features_from_empatica:
-    input:
-        sensor_features = find_empaticas_standardized_features_files
-    wildcard_constraints:
-        sensor_key = '(empatica).*'
-    output:
-        "data/processed/features/{pid}/z_{sensor_key}.csv"
-    script:
-        "../src/features/utils/join_features_from_providers.R"
-
-rule standardize_features_from_providers_no_empatica:
-    input:
-        sensor_features = find_joint_non_empatica_sensor_files
-    wildcard_constraints:
-        sensor_key = '(phone|fitbit).*'
-    params:
-        provider = config["STANDARDIZATION"]["PROVIDERS"]["OTHER"],
-        provider_key = "OTHER",
-        sensor_key = "{sensor_key}"
-    output:
-        "data/processed/features/{pid}/z_{sensor_key}.csv"
-    script:
-        "../src/features/standardization/main.py"
-
-rule merge_standardized_sensor_features_for_individual_participants:
-    input:
-        feature_files = input_merge_standardized_sensor_features_for_individual_participants
-    output:
-        "data/processed/features/{pid}/z_all_sensor_features.csv"
-    script:
-        "../src/features/utils/merge_sensor_features_for_individual_participants.R"
-
 rule merge_sensor_features_for_all_participants:
    input:
        feature_files = expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])
@ -1088,14 +986,6 @@ rule merge_sensor_features_for_all_participants:
    script:
        "../src/features/utils/merge_sensor_features_for_all_participants.R"

-rule merge_standardized_sensor_features_for_all_participants:
-    input:
-        feature_files = expand("data/processed/features/{pid}/z_all_sensor_features.csv", pid=config["PIDS"])
-    output:
-        "data/processed/features/all_participants/z_all_sensor_features.csv"
-    script:
-        "../src/features/utils/merge_standardized_sensor_features_for_all_participants.R"
-
 rule clean_sensor_features_for_individual_participants:
    input:
        sensor_data = rules.merge_sensor_features_for_individual_participants.output
@ -1107,7 +997,7 @@ rule clean_sensor_features_for_individual_participants:
        script_extension = "{script_extension}",
        sensor_key = "all_cleaning_individual" 
    output:
-        "data/processed/features/{pid}/all_sensor_features_cleaned_{provider_key}_{script_extension}.csv" # bo predstavljalo probleme za naprej (kako iskati datoteke + standardizacija itd.)
+        "data/processed/features/{pid}/all_sensor_features_cleaned_{provider_key}_{script_extension}.csv" 
    script:
        "../src/features/entry.{params.script_extension}"

@ -1118,37 +1008,9 @@ rule clean_sensor_features_for_all_participants:
        provider = lambda wildcards: config["ALL_CLEANING_OVERALL"]["PROVIDERS"][wildcards.provider_key.upper()],
        provider_key = "{provider_key}",
        script_extension = "{script_extension}",
-        sensor_key = "all_cleaning_overall"
+        sensor_key = "all_cleaning_overall",
+        target = "{target}"
    output:
-        "data/processed/features/all_participants/all_sensor_features_cleaned_{provider_key}_{script_extension}.csv"
+        "data/processed/features/all_participants/all_sensor_features_cleaned_{provider_key}_{script_extension}_({target}).csv"
    script:
        "../src/features/entry.{params.script_extension}"
-
-rule clean_standardized_sensor_features_for_individual_participants:
-    input:
-        sensor_data = rules.merge_standardized_sensor_features_for_individual_participants.output
-    wildcard_constraints:
-        pid = "("+"|".join(config["PIDS"])+")"
-    params:
-        provider = lambda wildcards: config["ALL_CLEANING_INDIVIDUAL"]["PROVIDERS"][wildcards.provider_key.upper()],
-        provider_key = "{provider_key}",
-        script_extension = "{script_extension}",
-        sensor_key = "all_cleaning_individual" 
-    output:
-        "data/processed/features/{pid}/z_all_sensor_features_cleaned_{provider_key}_{script_extension}.csv"
-    script:
-        "../src/features/entry.{params.script_extension}"
-
-rule clean_standardized_sensor_features_for_all_participants:
-    input:
-        sensor_data = rules.merge_standardized_sensor_features_for_all_participants.output
-    params:
-        provider = lambda wildcards: config["ALL_CLEANING_OVERALL"]["PROVIDERS"][wildcards.provider_key.upper()],
-        provider_key = "{provider_key}",
-        script_extension = "{script_extension}",
-        sensor_key = "all_cleaning_overall"
-    output:
-        "data/processed/features/all_participants/z_all_sensor_features_cleaned_{provider_key}_{script_extension}.csv"
-    script:
-        "../src/features/entry.{params.script_extension}"
-
--- a/rules/models.smk
+++ b/rules/models.smk
@ -30,43 +30,23 @@ rule baseline_features:

 rule select_target:
    input:
-        cleaned_sensor_features = "data/processed/features/{pid}/z_all_sensor_features_cleaned_straw_py.csv"
+        cleaned_sensor_features = "data/processed/features/{pid}/all_sensor_features_cleaned_straw_py.csv"
    params:
        target_variable = config["PARAMS_FOR_ANALYSIS"]["TARGET"]["LABEL"]
    output:
-        "data/processed/models/individual_model/{pid}/z_input.csv"
+        "data/processed/models/individual_model/{pid}/input.csv"
    script:
        "../src/models/select_targets.py"

 rule merge_features_and_targets_for_population_model:
    input:
-        cleaned_sensor_features = "data/processed/features/all_participants/z_all_sensor_features_cleaned_straw_py.csv",
+        cleaned_sensor_features = "data/processed/features/all_participants/all_sensor_features_cleaned_straw_py_({target}).csv",
        demographic_features = expand("data/processed/features/{pid}/baseline_features.csv", pid=config["PIDS"]),
    params:
-        target_variable=config["PARAMS_FOR_ANALYSIS"]["TARGET"]["LABEL"]
+        target_variable="{target}"
    output:
-        "data/processed/models/population_model/z_input.csv"
+        "data/processed/models/population_model/input_{target}.csv"
    script:
        "../src/models/merge_features_and_targets_for_population_model.py"

-# rule select_target:
-#     input:
-#         cleaned_sensor_features = "data/processed/features/{pid}/all_sensor_features_cleaned_straw_py.csv"
-#     params:
-#         target_variable = config["PARAMS_FOR_ANALYSIS"]["TARGET"]["LABEL"]
-#     output:
-#         "data/processed/models/individual_model/{pid}/input.csv"
-#     script:
-#         "../src/models/select_targets.py"
-
-# rule merge_features_and_targets_for_population_model:
-#     input:
-#         cleaned_sensor_features = "data/processed/features/all_participants/all_sensor_features_cleaned_straw_py.csv",
-#         demographic_features = expand("data/processed/features/{pid}/baseline_features.csv", pid=config["PIDS"]),
-#     params:
-#         target_variable=config["PARAMS_FOR_ANALYSIS"]["TARGET"]["LABEL"]
-#     output:
-#         "data/processed/models/population_model/input.csv"
-#     script:
-#         "../src/models/merge_features_and_targets_for_population_model.py"

--- a/rules/preprocessing.smk
+++ b/rules/preprocessing.smk
@ -249,3 +249,26 @@ rule empatica_readable_datetime:
        "data/raw/{pid}/empatica_{sensor}_with_datetime.csv"
    script:
        "../src/data/datetime/readable_datetime.R"
+
+
+rule extract_event_information_from_esm:
+    input:
+        esm_raw_input = "data/raw/{pid}/phone_esm_raw.csv",
+        pid_file = "data/external/participant_files/{pid}.yaml"
+    params:
+        stage = "extract",
+        pid = "{pid}"
+    output:
+        "data/raw/ers/{pid}_ers.csv"
+    script:
+        "../src/features/phone_esm/straw/process_user_event_related_segments.py"
+
+rule create_event_related_segments_file:
+    input:
+        ers_files = expand("data/raw/ers/{pid}_ers.csv", pid=config["PIDS"])
+    params:
+        stage = "merge"
+    output:
+        "data/external/straw_events.csv"
+    script:
+        "../src/features/phone_esm/straw/process_user_event_related_segments.py"
--- a/src/data/datetime/assign_to_time_segment.R
+++ b/src/data/datetime/assign_to_time_segment.R
@ -5,13 +5,16 @@ options(scipen=999)

 assign_rows_to_segments <- function(data, segments){
  # This function is used by all segment types, we use data.tables because they are fast
+
  data <- data.table::as.data.table(data)
  data[, assigned_segments := ""]
  for(i in seq_len(nrow(segments))) {
    segment <- segments[i,]
+
    data[segment$segment_start_ts<= timestamp & segment$segment_end_ts >= timestamp,
         assigned_segments := stringi::stri_c(assigned_segments, segment$segment_id, sep = "|")]
  }
+  
  data[,assigned_segments:=substring(assigned_segments, 2)]
  data
 }
--- a/src/features/all_cleaning_individual/straw/init.py
+++ b/src/features/all_cleaning_individual/straw/init.py
--- a/src/features/all_cleaning_individual/straw/main.py
+++ b/src/features/all_cleaning_individual/straw/main.py
@ -1,88 +1,174 @@
 import pandas as pd
 import numpy as np
-import math, sys
+import math, sys, random
+import yaml
+
+from sklearn.impute import KNNImputer
+from sklearn.preprocessing import StandardScaler
+import matplotlib.pyplot as plt
+import seaborn as sns
+
+sys.path.append('/rapids/')
+from src.features import empatica_data_yield as edy
+
+pd.set_option('display.max_columns', 20)

 def straw_cleaning(sensor_data_files, provider):
+    # TODO (maybe): reorganize the script based on the overall 
    
    features = pd.read_csv(sensor_data_files["sensor_data"][0])
-
-    # TODO: reorder the cleaning steps so it makes sense for the analysis
-    # TODO: add conditions that differentiates cleaning steps for standardized and nonstandardized features, for this
-    # the snakemake rules will also have to come with additional parameter (in rules/features.smk)
-
-    # Impute selected features event
-    impute_phone_features = provider["IMPUTE_PHONE_SELECTED_EVENT_FEATURES"]
-    if impute_phone_features["COMPUTE"]:
-        if not 'phone_data_yield_rapids_ratiovalidyieldedminutes' in features.columns:
-            raise KeyError("RAPIDS provider needs to impute the selected event features based on phone_data_yield_rapids_ratiovalidyieldedminutes column, please set config[PHONE_DATA_YIELD][PROVIDERS][RAPIDS][COMPUTE] to True and include 'ratiovalidyieldedminutes' in [FEATURES].")
-        
-        # TODO: if the type of the imputation will vary for different groups of features make conditional imputations here 
-        phone_cols = [col for col in features if \
-            col.startswith('phone_applications_foreground_rapids_') or
-            col.startswith('phone_battery_rapids_') or
-            col.startswith('phone_calls_rapids_') or
-            col.startswith('phone_keyboard_rapids_') or
-            col.startswith('phone_messages_rapids_') or
-            col.startswith('phone_screen_rapids_') or
-            col.startswith('phone_wifi_')]
-
-        mask = features['phone_data_yield_rapids_ratiovalidyieldedminutes'] > impute_phone_features['MIN_DATA_YIELDED_MINUTES_TO_IMPUTE']
-        features.loc[mask, phone_cols] = impute(features[mask][phone_cols], method=impute_phone_features["TYPE"].lower())
-
-    # Drop rows with the value of data_yield_column less than data_yield_ratio_threshold
-    data_yield_unit = provider["DATA_YIELD_FEATURE"].split("_")[3].lower()
-    data_yield_column = "phone_data_yield_rapids_ratiovalidyielded" + data_yield_unit
-
-    if not data_yield_column in features.columns:
-        raise KeyError(f"RAPIDS provider needs to impute the selected event features based on {data_yield_column} column, please set config[PHONE_DATA_YIELD][PROVIDERS][RAPIDS][COMPUTE] to True and include 'ratiovalidyielded{data_yield_unit}' in [FEATURES].")
-        
-    if provider["DATA_YIELD_RATIO_THRESHOLD"]:
-        features = features[features[data_yield_column] >= provider["DATA_YIELD_RATIO_THRESHOLD"]]
-
-    esm_cols = features.loc[:, features.columns.str.startswith('phone_esm')] # For later preservation of esm_cols
-
-    # Remove cols if threshold of NaN values is passed
-    features = features.loc[:, features.isna().sum() < provider["COLS_NAN_THRESHOLD"] * features.shape[0]]    
    
-    # Remove cols where variance is 0
-    if provider["COLS_VAR_THRESHOLD"]:
-        features.drop(features.std()[features.std() == 0].index.values, axis=1, inplace=True)
+    esm_cols = features.loc[:, features.columns.str.startswith('phone_esm_straw')] # Get target (esm) columns
+
+    with open('config.yaml', 'r') as stream:
+        config = yaml.load(stream, Loader=yaml.FullLoader)
+
+    excluded_columns = ['local_segment', 'local_segment_label', 'local_segment_start_datetime', 'local_segment_end_datetime']
+
+    # (1) FILTER_OUT THE ROWS THAT DO NOT HAVE THE TARGET COLUMN AVAILABLE
+    if config['PARAMS_FOR_ANALYSIS']['TARGET']['COMPUTE']:
+        target = config['PARAMS_FOR_ANALYSIS']['TARGET']['LABEL'] # get target label from config
+        features = features[features['phone_esm_straw_' + target].notna()].reset_index(drop=True)
+
+    # (2.1) QUALITY CHECK (DATA YIELD COLUMN) deletes the rows where E4 or phone data is low quality
+    phone_data_yield_unit = provider["PHONE_DATA_YIELD_FEATURE"].split("_")[3].lower()
+    phone_data_yield_column = "phone_data_yield_rapids_ratiovalidyielded" + phone_data_yield_unit
+
+    features = edy.calculate_empatica_data_yield(features)
+
+    if not phone_data_yield_column in features.columns and not "empatica_data_yield" in features.columns:
+        raise KeyError(f"RAPIDS provider needs to clean the selected event features based on {phone_data_yield_column} and empatica_data_yield columns. For phone data yield, please set config[PHONE_DATA_YIELD][PROVIDERS][RAPIDS][COMPUTE] to True and include 'ratiovalidyielded{data_yield_unit}' in [FEATURES].")
+        
+    # Drop rows where phone data yield is less then given threshold
+    if provider["PHONE_DATA_YIELD_RATIO_THRESHOLD"]:
+        features = features[features[phone_data_yield_column] >= provider["PHONE_DATA_YIELD_RATIO_THRESHOLD"]].reset_index(drop=True)
    
+    # Drop rows where empatica data yield is less then given threshold
+    if provider["EMPATICA_DATA_YIELD_RATIO_THRESHOLD"]:
+        features = features[features["empatica_data_yield"] >= provider["EMPATICA_DATA_YIELD_RATIO_THRESHOLD"]].reset_index(drop=True)
+
+    if features.empty:
+        return features
+    
+    # (2.2) DO THE ROWS CONSIST OF ENOUGH NON-NAN VALUES?
+    min_count =  math.ceil((1 - provider["ROWS_NAN_THRESHOLD"]) * features.shape[1]) # minimal not nan values in row
+    features.dropna(axis=0, thresh=min_count, inplace=True) # Thresh => at least this many not-nans
+
+    # (3) REMOVE COLS IF THEIR NAN THRESHOLD IS PASSED (should be <= if even all NaN columns must be preserved - this solution now drops columns with all NaN rows)
+    esm_cols = features.loc[:, features.columns.str.startswith('phone_esm_straw')] # Get target (esm) columns
+
+    features = features.loc[:, features.isna().sum() < provider["COLS_NAN_THRESHOLD"] * features.shape[0]]
+
    # Preserve esm cols if deleted (has to come after drop cols operations)
    for esm in esm_cols:
        if esm not in features:
            features[esm] = esm_cols[esm]
-    
-    # Drop highly correlated features - To-Do še en thershold var, ki je v config + kako se tretirajo NaNs?
+
+    # (4) CONTEXTUAL IMPUTATION
+
+    # Impute selected phone features with a high number
+    impute_w_hn = [col for col in features.columns if \
+        "timeoffirstuse" in col or
+        "timeoflastuse" in col or
+        "timefirstcall" in col or
+        "timelastcall" in col or
+        "firstuseafter" in col or
+        "timefirstmessages" in col or
+        "timelastmessages" in col]
+    features[impute_w_hn] = features[impute_w_hn].fillna(1500)
+
+
+    # Impute special case (mostcommonactivity) and (homelabel)
+    impute_w_sn = [col for col in features.columns if "mostcommonactivity" in col]
+    features[impute_w_sn] = features[impute_w_sn].fillna(4) # Special case of imputation - nominal/ordinal value
+
+    impute_w_sn2 = [col for col in features.columns if "homelabel" in col]
+    features[impute_w_sn2] = features[impute_w_sn2].fillna(1) # Special case of imputation - nominal/ordinal value
+
+    impute_w_sn3 = [col for col in features.columns if "loglocationvariance" in col]
+    features[impute_w_sn2] = features[impute_w_sn2].fillna(-1000000) # Special case of imputation - nominal/ordinal value
+
+
+    # Impute selected phone features with 0
+    impute_zero = [col for col in features if \
+        col.startswith('phone_applications_foreground_rapids_') or
+        col.startswith('phone_battery_rapids_') or
+        col.startswith('phone_bluetooth_rapids_') or
+        col.startswith('phone_light_rapids_') or
+        col.startswith('phone_calls_rapids_') or
+        col.startswith('phone_messages_rapids_') or
+        col.startswith('phone_screen_rapids_') or
+        col.startswith('phone_wifi_visible')]
+
+    features[impute_zero+list(esm_cols.columns)] = features[impute_zero+list(esm_cols.columns)].fillna(0)
+
+    ## (5) STANDARDIZATION 
+    if provider["STANDARDIZATION"]:
+        features.loc[:, ~features.columns.isin(excluded_columns)] = StandardScaler().fit_transform(features.loc[:, ~features.columns.isin(excluded_columns)])
+
+    # (6) IMPUTATION: IMPUTE DATA WITH KNN METHOD
+    impute_cols = [col for col in features.columns if col not in excluded_columns]
+    features.reset_index(drop=True, inplace=True)
+    features[impute_cols] = impute(features[impute_cols], method="knn")
+
+    # (7) REMOVE COLS WHERE VARIANCE IS 0
+    esm_cols = features.loc[:, features.columns.str.startswith('phone_esm_straw')]
+
+    if provider["COLS_VAR_THRESHOLD"]:
+        features.drop(features.std()[features.std() == 0].index.values, axis=1, inplace=True)
+
+    fe5 = features.copy()
+
+    # (8) DROP HIGHLY CORRELATED FEATURES
    drop_corr_features = provider["DROP_HIGHLY_CORRELATED_FEATURES"]
-    if drop_corr_features["COMPUTE"]:
+    if drop_corr_features["COMPUTE"] and features.shape[0]: # If small amount of segments (rows) is present, do not execute correlation check
        
        numerical_cols = features.select_dtypes(include=np.number).columns.tolist()

        # Remove columns where NaN count threshold is passed
        valid_features = features[numerical_cols].loc[:, features[numerical_cols].isna().sum() < drop_corr_features['MIN_OVERLAP_FOR_CORR_THRESHOLD'] * features[numerical_cols].shape[0]]

-        cor_matrix = valid_features.corr(method='spearman').abs()
-        upper_tri = cor_matrix.where(np.triu(np.ones(cor_matrix.shape), k=1).astype(np.bool))
-        to_drop = [column for column in upper_tri.columns if any(upper_tri[column] > drop_corr_features["CORR_THRESHOLD"])]
+        corr_matrix = valid_features.corr().abs()
+        upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(np.bool))
+        to_drop = [column for column in upper.columns if any(upper[column] > drop_corr_features["CORR_THRESHOLD"])]

        features.drop(to_drop, axis=1, inplace=True)

-    # Remove rows if threshold of NaN values is passed
-    min_count =  math.ceil((1 - provider["ROWS_NAN_THRESHOLD"]) * features.shape[1]) # minimal not nan values in row
-    features.dropna(axis=0, thresh=min_count, inplace=True)
+    # Preserve esm cols if deleted (has to come after drop cols operations)
+    for esm in esm_cols:
+        if esm not in features:
+            features[esm] = esm_cols[esm]
+
+    fe6 = features.copy()
+
+    # (9) VERIFY IF THERE ARE ANY NANS LEFT IN THE DATAFRAME
+    if features.isna().any().any():
+        raise ValueError("There are still some NaNs present in the dataframe. Please check for implementation errors.")

    return features

 def impute(df, method='zero'):
    
-    def k_nearest(df): # TODO: if needed, implement k-nearest imputation / interpolation
-        pass
+    def k_nearest(df):
+        pd.set_option('display.max_columns', None)
+        imputer = KNNImputer(n_neighbors=3)
+        return pd.DataFrame(imputer.fit_transform(df), columns=df.columns)

-    return { # rest of the columns should be imputed with the selected method
+    return {
        'zero': df.fillna(0),
+        'high_number': df.fillna(1500),
        'mean': df.fillna(df.mean()),
        'median': df.fillna(df.median()),
-        'k-nearest': k_nearest(df) 
+        'knn': k_nearest(df) 
    }[method]
-    
+
+def graph_bf_af(features, phase_name, plt_flag=False):
+    if plt_flag:
+        sns.set(rc={"figure.figsize":(16, 8)})
+        sns.heatmap(features.isna(), cbar=False) #features.select_dtypes(include=np.number)
+        plt.savefig(f'features_overall_nans_{phase_name}.png', bbox_inches='tight')
+
+    print(f"\n-------------{phase_name}-------------")
+    print("Rows number:", features.shape[0])
+    print("Columns number:", len(features.columns))
+    print("---------------------------------------------\n")
--- a/src/features/all_cleaning_overall/straw/init.py
+++ b/src/features/all_cleaning_overall/straw/init.py
--- a/src/features/all_cleaning_overall/straw/main.py
+++ b/src/features/all_cleaning_overall/straw/main.py
@ -1,88 +1,226 @@
 import pandas as pd
 import numpy as np
-import math, sys
+import math, sys, random, warnings, yaml
+
+from sklearn.impute import KNNImputer
+from sklearn.preprocessing import StandardScaler, minmax_scale 
+import matplotlib.pyplot as plt
+import seaborn as sns
+
+sys.path.append('/rapids/')
+from src.features import empatica_data_yield as edy
+
+def straw_cleaning(sensor_data_files, provider, target):

-def straw_cleaning(sensor_data_files, provider):
-    
    features = pd.read_csv(sensor_data_files["sensor_data"][0])

-    # TODO: reorder the cleaning steps so it makes sense for the analysis
-    # TODO: add conditions that differentiates cleaning steps for standardized and nonstandardized features, for this
-    # the snakemake rules will also have to come with additional parameter (in rules/features.smk)
-
-    # Impute selected features event
-    impute_phone_features = provider["IMPUTE_PHONE_SELECTED_EVENT_FEATURES"]
-    if impute_phone_features["COMPUTE"]:
-        if not 'phone_data_yield_rapids_ratiovalidyieldedminutes' in features.columns:
-            raise KeyError("RAPIDS provider needs to impute the selected event features based on phone_data_yield_rapids_ratiovalidyieldedminutes column, please set config[PHONE_DATA_YIELD][PROVIDERS][RAPIDS][COMPUTE] to True and include 'ratiovalidyieldedminutes' in [FEATURES].")
-        
-        # TODO: if the type of the imputation will vary for different groups of features make conditional imputations here 
-        phone_cols = [col for col in features if \
-            col.startswith('phone_applications_foreground_rapids_') or
-            col.startswith('phone_battery_rapids_') or
-            col.startswith('phone_calls_rapids_') or
-            col.startswith('phone_keyboard_rapids_') or
-            col.startswith('phone_messages_rapids_') or
-            col.startswith('phone_screen_rapids_') or
-            col.startswith('phone_wifi_')]
-
-        mask = features['phone_data_yield_rapids_ratiovalidyieldedminutes'] > impute_phone_features['MIN_DATA_YIELDED_MINUTES_TO_IMPUTE']
-        features.loc[mask, phone_cols] = impute(features[mask][phone_cols], method=impute_phone_features["TYPE"].lower())
-
-    # Drop rows with the value of data_yield_column less than data_yield_ratio_threshold
-    data_yield_unit = provider["DATA_YIELD_FEATURE"].split("_")[3].lower()
-    data_yield_column = "phone_data_yield_rapids_ratiovalidyielded" + data_yield_unit
-
-    if not data_yield_column in features.columns:
-        raise KeyError(f"RAPIDS provider needs to impute the selected event features based on {data_yield_column} column, please set config[PHONE_DATA_YIELD][PROVIDERS][RAPIDS][COMPUTE] to True and include 'ratiovalidyielded{data_yield_unit}' in [FEATURES].")
-        
-    if provider["DATA_YIELD_RATIO_THRESHOLD"]:
-        features = features[features[data_yield_column] >= provider["DATA_YIELD_RATIO_THRESHOLD"]]
-
-    esm_cols = features.loc[:, features.columns.str.startswith('phone_esm')] # For later preservation of esm_cols
-
-    # Remove cols if threshold of NaN values is passed
-    features = features.loc[:, features.isna().sum() < provider["COLS_NAN_THRESHOLD"] * features.shape[0]]    
+    # features = features[features['local_segment_label'] == 'working_day'] # Filtriranje ustreznih časovnih segmentov
    
-    # Remove cols where variance is 0
+    # print(features)
+    # sys.exit()
+    
+    esm_cols = features.loc[:, features.columns.str.startswith('phone_esm_straw')] # Get target (esm) columns
+
+    with open('config.yaml', 'r') as stream:
+        config = yaml.load(stream, Loader=yaml.FullLoader)
+
+    excluded_columns = ['local_segment', 'local_segment_label', 'local_segment_start_datetime', 'local_segment_end_datetime']
+
+    graph_bf_af(features, "1target_rows_before")
+
+    # (1) FILTER_OUT THE ROWS THAT DO NOT HAVE THE TARGET COLUMN AVAILABLE
+    if config['PARAMS_FOR_ANALYSIS']['TARGET']['COMPUTE']:
+        features = features[features['phone_esm_straw_' + target].notna()].reset_index(drop=True)
+    
+    if features.empty:
+        return pd.DataFrame(columns=excluded_columns)
+
+    graph_bf_af(features, "2target_rows_after")
+    print("HERE1", target, features["pid"])
+
+    # (2) QUALITY CHECK (DATA YIELD COLUMN) drops the rows where E4 or phone data is low quality
+    phone_data_yield_unit = provider["PHONE_DATA_YIELD_FEATURE"].split("_")[3].lower()
+    phone_data_yield_column = "phone_data_yield_rapids_ratiovalidyielded" + phone_data_yield_unit
+
+    features = edy.calculate_empatica_data_yield(features)
+
+    if not phone_data_yield_column in features.columns and not "empatica_data_yield" in features.columns:
+        raise KeyError(f"RAPIDS provider needs to clean the selected event features based on {phone_data_yield_column} and empatica_data_yield columns. For phone data yield, please set config[PHONE_DATA_YIELD][PROVIDERS][RAPIDS][COMPUTE] to True and include 'ratiovalidyielded{data_yield_unit}' in [FEATURES].")
+
+    hist = features[["empatica_data_yield", phone_data_yield_column]].hist()
+    plt.savefig(f'phone_E4_histogram.png', bbox_inches='tight')
+
+    # Drop rows where phone data yield is less then given threshold
+    if provider["PHONE_DATA_YIELD_RATIO_THRESHOLD"]:
+        # print("\nThreshold:", provider["PHONE_DATA_YIELD_RATIO_THRESHOLD"])
+        # print("Phone features data yield stats:", features[phone_data_yield_column].describe(), "\n")
+        # print(features[phone_data_yield_column].sort_values())
+        hist = features[phone_data_yield_column].hist(bins=5)
+        plt.close()
+        features = features[features[phone_data_yield_column] >= provider["PHONE_DATA_YIELD_RATIO_THRESHOLD"]].reset_index(drop=True)
+
+    # Drop rows where empatica data yield is less then given threshold
+    if provider["EMPATICA_DATA_YIELD_RATIO_THRESHOLD"]:
+        # print("\nThreshold:", provider["EMPATICA_DATA_YIELD_RATIO_THRESHOLD"])
+        # print("E4 features data yield stats:", features["empatica_data_yield"].describe(), "\n")
+        # print(features["empatica_data_yield"].sort_values())
+        features = features[features["empatica_data_yield"] >= provider["EMPATICA_DATA_YIELD_RATIO_THRESHOLD"]].reset_index(drop=True)
+
+
+    graph_bf_af(features, "3data_yield_drop_rows")
+
+    if features.empty:
+        return pd.DataFrame(columns=excluded_columns)
+
+    # (3) CONTEXTUAL IMPUTATION
+
+    # Impute selected phone features with a high number
+    impute_w_hn = [col for col in features.columns if \
+        "timeoffirstuse" in col or
+        "timeoflastuse" in col or
+        "timefirstcall" in col or
+        "timelastcall" in col or
+        "firstuseafter" in col or
+        "timefirstmessages" in col or
+        "timelastmessages" in col]
+    features[impute_w_hn] = features[impute_w_hn].fillna(1500)
+
+    # Impute special case (mostcommonactivity) and (homelabel)
+    impute_w_sn = [col for col in features.columns if "mostcommonactivity" in col]
+    features[impute_w_sn] = features[impute_w_sn].fillna(4) # Special case of imputation - nominal/ordinal value
+
+    impute_w_sn2 = [col for col in features.columns if "homelabel" in col]
+    features[impute_w_sn2] = features[impute_w_sn2].fillna(1) # Special case of imputation - nominal/ordinal value
+
+    impute_w_sn3 = [col for col in features.columns if "loglocationvariance" in col]
+    features[impute_w_sn2] = features[impute_w_sn2].fillna(-1000000) # Special case of imputation - loglocation
+
+    # Impute selected phone features with 0 + impute ESM features with 0
+    impute_zero = [col for col in features if \
+        col.startswith('phone_applications_foreground_rapids_') or
+        col.startswith('phone_battery_rapids_') or
+        col.startswith('phone_bluetooth_rapids_') or
+        col.startswith('phone_light_rapids_') or
+        col.startswith('phone_calls_rapids_') or
+        col.startswith('phone_messages_rapids_') or
+        col.startswith('phone_screen_rapids_') or
+        col.startswith('phone_wifi_visible')]
+        
+    features[impute_zero+list(esm_cols.columns)] = features[impute_zero+list(esm_cols.columns)].fillna(0)
+
+    graph_bf_af(features, "4context_imp")
+ 
+    # (4) REMOVE COLS IF THEIR NAN THRESHOLD IS PASSED (should be <= if even all NaN columns must be preserved - this solution now drops columns with all NaN rows)
+    esm_cols = features.loc[:, features.columns.str.startswith('phone_esm_straw')] # Get target (esm) columns
+
+    features = features.loc[:, features.isna().sum() < provider["COLS_NAN_THRESHOLD"] * features.shape[0]]
+
+    graph_bf_af(features, "5too_much_nans_cols")
+    # (5) REMOVE COLS WHERE VARIANCE IS 0
+
    if provider["COLS_VAR_THRESHOLD"]:
        features.drop(features.std()[features.std() == 0].index.values, axis=1, inplace=True)
-    
+
+    graph_bf_af(features, "6variance_drop")
+
    # Preserve esm cols if deleted (has to come after drop cols operations)
    for esm in esm_cols:
        if esm not in features:
            features[esm] = esm_cols[esm]
    
-    # Drop highly correlated features - To-Do še en thershold var, ki je v config + kako se tretirajo NaNs?
+    # (6) DO THE ROWS CONSIST OF ENOUGH NON-NAN VALUES?
+    min_count =  math.ceil((1 - provider["ROWS_NAN_THRESHOLD"]) * features.shape[1]) # minimal not nan values in row
+    features.dropna(axis=0, thresh=min_count, inplace=True) # Thresh => at least this many not-nans
+
+    graph_bf_af(features, "7too_much_nans_rows")
+
+    if features.empty:
+        return pd.DataFrame(columns=excluded_columns)
+
+
+    # (7) STANDARDIZATION TODO: exclude nominal features from standardization 
+
+    if provider["STANDARDIZATION"]:
+        # Expected warning within this code block
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", category=RuntimeWarning)
+            features.loc[:, ~features.columns.isin(excluded_columns + ["pid"])] = \
+                features.loc[:, ~features.columns.isin(excluded_columns)].groupby('pid').transform(lambda x: StandardScaler().fit_transform(x.values[:,np.newaxis]).ravel())
+
+    graph_bf_af(features, "8standardization")
+
+    # (8) IMPUTATION: IMPUTE DATA WITH KNN METHOD
+    features.reset_index(drop=True, inplace=True)
+    impute_cols = [col for col in features.columns if col not in excluded_columns and col != "pid"]
+
+    features[impute_cols] = impute(features[impute_cols], method="knn")
+
+    graph_bf_af(features, "9knn_after")
+
+
+    # (9) DROP HIGHLY CORRELATED FEATURES
+    esm_cols = features.loc[:, features.columns.str.startswith('phone_esm_straw')]
+
    drop_corr_features = provider["DROP_HIGHLY_CORRELATED_FEATURES"]
-    if drop_corr_features["COMPUTE"]:
+    if drop_corr_features["COMPUTE"] and features.shape[0] > 5: # If small amount of segments (rows) is present, do not execute correlation check
        
        numerical_cols = features.select_dtypes(include=np.number).columns.tolist()

        # Remove columns where NaN count threshold is passed
        valid_features = features[numerical_cols].loc[:, features[numerical_cols].isna().sum() < drop_corr_features['MIN_OVERLAP_FOR_CORR_THRESHOLD'] * features[numerical_cols].shape[0]]

-        cor_matrix = valid_features.corr(method='spearman').abs()
-        upper_tri = cor_matrix.where(np.triu(np.ones(cor_matrix.shape), k=1).astype(np.bool))
-        to_drop = [column for column in upper_tri.columns if any(upper_tri[column] > drop_corr_features["CORR_THRESHOLD"])]
+        corr_matrix = valid_features.corr().abs()
+        upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(np.bool))
+        to_drop = [column for column in upper.columns if any(upper[column] > drop_corr_features["CORR_THRESHOLD"])]
+
+        # sns.heatmap(corr_matrix, cmap="YlGnBu")
+        # plt.savefig(f'correlation_matrix.png', bbox_inches='tight')
+        # plt.close()
+
+        # s = corr_matrix.unstack()
+        # so = s.sort_values(ascending=False)
+
+        # pd.set_option('display.max_rows', None)
+        # sorted_upper = upper.unstack().sort_values(ascending=False)
+        # print(sorted_upper[sorted_upper > drop_corr_features["CORR_THRESHOLD"]])

        features.drop(to_drop, axis=1, inplace=True)

-    # Remove rows if threshold of NaN values is passed
-    min_count =  math.ceil((1 - provider["ROWS_NAN_THRESHOLD"]) * features.shape[1]) # minimal not nan values in row
-    features.dropna(axis=0, thresh=min_count, inplace=True)
+    # Preserve esm cols if deleted (has to come after drop cols operations)
+    for esm in esm_cols:
+        if esm not in features:
+            features[esm] = esm_cols[esm]
+
+    graph_bf_af(features, "10correlation_drop")
+
+    # (10) VERIFY IF THERE ARE ANY NANS LEFT IN THE DATAFRAME
+    if features.isna().any().any():
+        raise ValueError("There are still some NaNs present in the dataframe. Please check for implementation errors.")

    return features

 def impute(df, method='zero'):
    
-    def k_nearest(df): # TODO: if needed, implement k-nearest imputation / interpolation
-        pass
+    def k_nearest(df):
+        imputer = KNNImputer(n_neighbors=3)
+        return pd.DataFrame(imputer.fit_transform(df), columns=df.columns)

-    return { # rest of the columns should be imputed with the selected method
+    return {
        'zero': df.fillna(0),
+        'high_number': df.fillna(1500),
        'mean': df.fillna(df.mean()),
        'median': df.fillna(df.median()),
-        'k-nearest': k_nearest(df) 
+        'knn': k_nearest(df) 
    }[method]
-    
+
+def graph_bf_af(features, phase_name, plt_flag=True):
+    if plt_flag:
+        sns.set(rc={"figure.figsize":(16, 8)})
+        sns.heatmap(features.isna(), cbar=False) #features.select_dtypes(include=np.number)
+        plt.savefig(f'features_overall_nans_{phase_name}.png', bbox_inches='tight')
+
+    print(f"\n-------------{phase_name}-------------")
+    print("Rows number:", features.shape[0])
+    print("Columns number:", len(features.columns))
+    print("NaN values:", features.isna().sum().sum())
+    print("---------------------------------------------\n")
--- a/src/features/cr_features_helper_methods.py
+++ b/src/features/cr_features_helper_methods.py
@ -21,7 +21,7 @@ def extract_second_order_features(intraday_features, so_features_names, prefix="
            so_features = pd.concat([so_features, intraday_features.drop(prefix+"level_1", axis=1).groupby(groupby_cols).median().add_suffix("_SO_median")], axis=1)
        
        if "sd" in so_features_names:
-            so_features = pd.concat([so_features, intraday_features.drop(prefix+"level_1", axis=1).groupby(groupby_cols).std().add_suffix("_SO_sd")], axis=1)
+            so_features = pd.concat([so_features, intraday_features.drop(prefix+"level_1", axis=1).groupby(groupby_cols).std().fillna(0).add_suffix("_SO_sd")], axis=1)
        
        if "nlargest" in so_features_names: # largest 5 -- maybe there is a faster groupby solution?
            for column in intraday_features.loc[:, ~intraday_features.columns.isin(groupby_cols+[prefix+"level_1"])]:
--- a/src/features/empatica_accelerometer/cr/main.py
+++ b/src/features/empatica_accelerometer/cr/main.py
@ -43,7 +43,11 @@ def extract_acc_features_from_intraday_data(acc_intraday_data, features, window_


 def cr_features(sensor_data_files, time_segment, provider, filter_data_by_segment, *args, **kwargs):
-    acc_intraday_data = pd.read_csv(sensor_data_files["sensor_data"])
+    
+    data_types = {'local_timezone': 'str', 'device_id': 'str', 'timestamp': 'int64', 'double_values_0': 'float64',
+                    'double_values_1': 'float64', 'double_values_2': 'float64', 'local_date_time': 'str', 'local_date': "str",
+                    'local_time': "str", 'local_hour': "str", 'local_minute': "str", 'assigned_segments': "str"}
+    acc_intraday_data = pd.read_csv(sensor_data_files["sensor_data"], dtype=data_types)    

    requested_intraday_features = provider["FEATURES"]
    
--- a/src/features/empatica_data_yield.py
+++ b/src/features/empatica_data_yield.py
@ -0,0 +1,26 @@
+import pandas as pd
+import numpy as np
+from datetime import datetime
+
+import sys
+
+def calculate_empatica_data_yield(features):
+    # Get time segment duration in seconds from dataframe
+    datetime_start = datetime.strptime(features.loc[0, 'local_segment_start_datetime'], '%Y-%m-%d %H:%M:%S')
+    datetime_end = datetime.strptime(features.loc[0, 'local_segment_end_datetime'], '%Y-%m-%d %H:%M:%S')
+    tseg_duration = (datetime_end - datetime_start).total_seconds()
+
+    features["acc_data_yield"] = (features['empatica_accelerometer_cr_SO_windowsCount'] * 15) / tseg_duration \
+        if 'empatica_accelerometer_cr_SO_windowsCount' in features else 0
+    features["temp_data_yield"] = (features['empatica_temperature_cr_SO_windowsCount'] * 300) / tseg_duration \
+        if 'empatica_temperature_cr_SO_windowsCount' in features else 0
+    features["eda_data_yield"] = (features['empatica_electrodermal_activity_cr_SO_windowsCount'] * 60) / tseg_duration \
+        if 'empatica_electrodermal_activity_cr_SO_windowsCount' in features else 0
+    features["ibi_data_yield"] = (features['empatica_inter_beat_interval_cr_SO_windowsCount'] * 300) / tseg_duration \
+        if 'empatica_inter_beat_interval_cr_SO_windowsCount' in features else 0
+
+    empatica_data_yield_cols = ['acc_data_yield', 'temp_data_yield', 'eda_data_yield', 'ibi_data_yield']
+    features["empatica_data_yield"] = features[empatica_data_yield_cols].mean(axis=1).fillna(0)
+    features.drop(empatica_data_yield_cols, axis=1, inplace=True) # In case of if the advanced operations will later not be needed (e.g., weighted average)
+
+    return features
--- a/src/features/empatica_electrodermal_activity/cr/main.py
+++ b/src/features/empatica_electrodermal_activity/cr/main.py
@ -44,7 +44,11 @@ def extract_eda_features_from_intraday_data(eda_intraday_data, features, window_


 def cr_features(sensor_data_files, time_segment, provider, filter_data_by_segment, *args, **kwargs):
-    eda_intraday_data = pd.read_csv(sensor_data_files["sensor_data"])
+
+    data_types = {'local_timezone': 'str', 'device_id': 'str', 'timestamp': 'int64', 'electrodermal_activity': 'float64', 'local_date_time': 'str', 
+                  'local_date': "str", 'local_time': "str", 'local_hour': "str", 'local_minute': "str", 'assigned_segments': "str"}
+
+    eda_intraday_data = pd.read_csv(sensor_data_files["sensor_data"], dtype=data_types)

    requested_intraday_features = provider["FEATURES"]
    
--- a/src/features/empatica_inter_beat_interval/cr/main.py
+++ b/src/features/empatica_inter_beat_interval/cr/main.py
@ -50,6 +50,11 @@ def extract_ibi_features_from_intraday_data(ibi_intraday_data, features, window_


 def cr_features(sensor_data_files, time_segment, provider, filter_data_by_segment, *args, **kwargs):
+
+    data_types = {'local_timezone': 'str', 'device_id': 'str', 'timestamp': 'int64', 'inter_beat_interval': 'float64', 'timings': 'float64', 'local_date_time': 'str', 
+                  'local_date': "str", 'local_time': "str", 'local_hour': "str", 'local_minute': "str", 'assigned_segments': "str"}
+
+    temperature_intraday_data = pd.read_csv(sensor_data_files["sensor_data"], dtype=data_types)
    ibi_intraday_data = pd.read_csv(sensor_data_files["sensor_data"])

    requested_intraday_features = provider["FEATURES"]
--- a/src/features/empatica_temperature/cr/main.py
+++ b/src/features/empatica_temperature/cr/main.py
@ -37,7 +37,10 @@ def extract_temp_features_from_intraday_data(temperature_intraday_data, features


 def cr_features(sensor_data_files, time_segment, provider, filter_data_by_segment, *args, **kwargs):
-    temperature_intraday_data = pd.read_csv(sensor_data_files["sensor_data"])
+    data_types = {'local_timezone': 'str', 'device_id': 'str', 'timestamp': 'int64', 'temperature': 'float64', 'local_date_time': 'str', 
+                'local_date': "str", 'local_time': "str", 'local_hour': "str", 'local_minute': "str", 'assigned_segments': "str"}
+
+    temperature_intraday_data = pd.read_csv(sensor_data_files["sensor_data"], dtype=data_types)

    requested_intraday_features = provider["FEATURES"]

--- a/src/features/entry.py
+++ b/src/features/entry.py
@ -13,7 +13,10 @@ calc_windows = True if (provider.get("WINDOWS", False) and provider["WINDOWS"].g

 if sensor_key == "all_cleaning_individual" or sensor_key == "all_cleaning_overall":
    # Data cleaning
-    sensor_features = run_provider_cleaning_script(provider, provider_key, sensor_key, sensor_data_files)
+    if "overall" in sensor_key:
+        sensor_features = run_provider_cleaning_script(provider, provider_key, sensor_key, sensor_data_files, snakemake.params["target"])
+    else:
+        sensor_features = run_provider_cleaning_script(provider, provider_key, sensor_key, sensor_data_files)
 else:
    # Extract sensor features
    del sensor_data_files["time_segments_labels"]
--- a/src/features/phone_activity_recognition/rapids/main.py
+++ b/src/features/phone_activity_recognition/rapids/main.py
@ -37,6 +37,6 @@ def rapids_features(sensor_data_files, time_segment, provider, filter_data_by_se
            ar_features.index.names = ["local_segment"]
            ar_features = ar_features.reset_index()
    
-    ar_features.fillna(value={"count": 0, "countuniqueactivities": 0, "durationstationary": 0, "durationmobile": 0, "durationvehicle": 0}, inplace=True)
+    ar_features.fillna(value={"count": 0, "countuniqueactivities": 0, "durationstationary": 0, "durationmobile": 0, "durationvehicle": 0, "mostcommonactivity": 4}, inplace=True)

    return ar_features
--- a/src/features/phone_applications_foreground/rapids/main.py
+++ b/src/features/phone_applications_foreground/rapids/main.py
@ -9,19 +9,19 @@ def compute_features(filtered_data, apps_type, requested_features, apps_features
    if "timeoffirstuse" in requested_features:
        time_first_event = filtered_data.sort_values(by="timestamp", ascending=True).drop_duplicates(subset="local_segment", keep="first").set_index("local_segment")
        if time_first_event.empty:
-            apps_features["timeoffirstuse" + apps_type] = np.nan
+            apps_features["timeoffirstuse" + apps_type] = 1500 # np.nan
        else:
            apps_features["timeoffirstuse" + apps_type] = time_first_event["local_hour"] * 60 + time_first_event["local_minute"]
    if "timeoflastuse" in requested_features:
        time_last_event = filtered_data.sort_values(by="timestamp", ascending=False).drop_duplicates(subset="local_segment", keep="first").set_index("local_segment")
        if time_last_event.empty:
-            apps_features["timeoflastuse" + apps_type] = np.nan
+            apps_features["timeoflastuse" + apps_type] = 1500 # np.nan
        else:
            apps_features["timeoflastuse" + apps_type] = time_last_event["local_hour"] * 60 + time_last_event["local_minute"]
    if "frequencyentropy" in requested_features:
        apps_with_count = filtered_data.groupby(["local_segment","application_name"]).count().sort_values(by="timestamp", ascending=False).reset_index()
        if (len(apps_with_count.index) < 2 ):
-            apps_features["frequencyentropy" + apps_type] = np.nan
+            apps_features["frequencyentropy" + apps_type] = 0 # np.nan
        else:    
            apps_features["frequencyentropy" + apps_type] = apps_with_count.groupby("local_segment")["timestamp"].agg(entropy)
    if "countevent" in requested_features:
@ -43,6 +43,7 @@ def compute_features(filtered_data, apps_type, requested_features, apps_features
        apps_features["sumduration" + apps_type] = filtered_data.groupby(by = ["local_segment"])["duration"].sum()
    
    apps_features.index.names = ["local_segment"]
+    
    return apps_features

 def process_app_features(data, requested_features, time_segment, provider, filter_data_by_segment):
--- a/src/features/phone_bluetooth/doryab/main.py
+++ b/src/features/phone_bluetooth/doryab/main.py
@ -14,8 +14,8 @@ def deviceFeatures(devices, ownership, common_devices, features_to_compute, feat
        features = features.join(device_value_counts.groupby("local_segment")["bt_address"].nunique().to_frame("uniquedevices" + ownership), how="outer")
    if "meanscans" in features_to_compute:
        features = features.join(device_value_counts.groupby("local_segment")["scans"].mean().to_frame("meanscans" + ownership), how="outer")
-    if "stdscans" in features_to_compute:
-        features = features.join(device_value_counts.groupby("local_segment")["scans"].std().to_frame("stdscans" + ownership), how="outer")
+    if "stdscans" in features_to_compute: 
+        features = features.join(device_value_counts.groupby("local_segment")["scans"].std().to_frame("stdscans" + ownership).fillna(0), how="outer")
    # Most frequent device within segments, across segments, and across dataset
    if "countscansmostfrequentdevicewithinsegments" in features_to_compute:
        features = features.join(device_value_counts.groupby("local_segment")["scans"].max().to_frame("countscansmostfrequentdevicewithinsegments" + ownership), how="outer")
--- a/src/features/phone_calls/rapids/main.R
+++ b/src/features/phone_calls/rapids/main.R
@ -88,6 +88,16 @@ rapids_features <- function(sensor_data_files, time_segment, provider){
        features <- call_features_of_type(calls_of_type, features_type, call_type, time_segment, requested_features)
        call_features <- merge(call_features, features, all=TRUE)
    }
-    call_features <- call_features %>% mutate_at(vars(contains("countmostfrequentcontact") | contains("distinctcontacts") | contains("count") | contains("sumduration") | contains("minduration") | contains("maxduration") | contains("meanduration") | contains("modeduration")), list( ~ replace_na(., 0)))
+
+    # Fill seleted columns with a high number
+    time_cols <- select(call_features, contains("timefirstcall") |  contains("timelastcall")) %>% 
+        colnames(.)
+
+    call_features <- call_features %>% 
+        mutate_at(., time_cols, ~replace(., is.na(.), 1500))
+
+    # Fill NA values with 0
+    call_features <- call_features %>% mutate_all(~replace(., is.na(.), 0))
+
    return(call_features)
 }
--- a/src/features/phone_esm/straw/esm.py
+++ b/src/features/phone_esm/straw/esm.py
@ -0,0 +1,274 @@
+from collections.abc import Collection
+
+import numpy as np
+import pandas as pd
+from pytz import timezone
+import datetime, json
+
+# from config.models import ESM, Participant
+# from features import helper
+
+ESM_STATUS_ANSWERED = 2
+
+GROUP_SESSIONS_BY = ["device_id", "esm_session"] # 'participant_id
+
+SESSION_STATUS_UNANSWERED = "ema_unanswered"
+SESSION_STATUS_DAY_FINISHED = "day_finished"
+SESSION_STATUS_COMPLETE = "ema_completed"
+
+ANSWER_DAY_FINISHED = "DayFinished3421"
+ANSWER_DAY_OFF = "DayOff3421"
+ANSWER_SET_EVENING = "DayFinishedSetEvening"
+
+MAX_MORNING_LENGTH = 3
+# When the participants was not yet at work at the time of the first (morning) EMA,
+# only three items were answered.
+# Two sleep related items and one indicating NOT starting work yet.
+# Daytime EMAs are all longer, in fact they always consist of at least 6 items.
+
+
+TZ_LJ = timezone("Europe/Ljubljana")
+COLUMN_TIMESTAMP = "timestamp"
+COLUMN_TIMESTAMP_ESM = "double_esm_user_answer_timestamp"
+
+
+def get_date_from_timestamp(df_aware) -> pd.DataFrame:
+    """
+    Transform a UNIX timestamp into a datetime (with Ljubljana timezone).
+    Additionally, extract only the date part, where anything until 4 AM is considered the same day.
+
+    Parameters
+    ----------
+    df_aware: pd.DataFrame
+        Any AWARE-type data as defined in models.py.
+
+    Returns
+    -------
+    df_aware: pd.DataFrame
+        The same dataframe with datetime_lj and date_lj columns added.
+
+    """
+    if COLUMN_TIMESTAMP_ESM in df_aware:
+        column_timestamp = COLUMN_TIMESTAMP_ESM
+    else:
+        column_timestamp = COLUMN_TIMESTAMP
+
+    df_aware["datetime_lj"] = df_aware[column_timestamp].apply(
+        lambda x: datetime.datetime.fromtimestamp(x / 1000.0, tz=TZ_LJ)
+    )
+    df_aware = df_aware.assign(
+        date_lj=lambda x: (x.datetime_lj - datetime.timedelta(hours=4)).dt.date
+    )
+    # Since daytime EMAs could *theoretically* last beyond midnight, but never after 4 AM,
+    # the datetime is first translated to 4 h earlier.
+
+    return df_aware
+
+
+def preprocess_esm(df_esm: pd.DataFrame) -> pd.DataFrame:
+    """
+    Convert timestamps into human-readable datetimes and dates
+    and expand the JSON column into several Pandas DF columns.
+
+    Parameters
+    ----------
+    df_esm: pd.DataFrame
+        A dataframe of esm data.
+
+    Returns
+    -------
+    df_esm_preprocessed: pd.DataFrame
+        A dataframe with added columns: datetime in Ljubljana timezone and all fields from ESM_JSON column.
+    """
+    df_esm = get_date_from_timestamp(df_esm)
+
+    df_esm_json = df_esm["esm_json"].apply(json.loads)
+    df_esm_json = pd.json_normalize(df_esm_json).drop(
+        columns=["esm_trigger"]
+    )  # The esm_trigger column is already present in the main df.
+    return df_esm.join(df_esm_json)
+
+
+def classify_sessions_by_completion(df_esm_preprocessed: pd.DataFrame) -> pd.DataFrame:
+    """
+    For each distinct EMA session, determine how the participant responded to it.
+    Possible outcomes are: SESSION_STATUS_UNANSWERED, SESSION_STATUS_DAY_FINISHED, and SESSION_STATUS_COMPLETE
+
+    This is done in three steps.
+
+    First, the esm_status is considered.
+    If any of the ESMs in a session has a status *other than* "answered", then this session is taken as unfinished.
+
+    Second, the sessions which do not represent full questionnaires are identified.
+    These are sessions where participants only marked they are finished with the day or have not yet started working.
+
+    Third, the sessions with only one item are marked with their trigger.
+    We never offered questionnaires with single items, so we can be sure these are unfinished.
+
+    Finally, all sessions that remain are marked as completed.
+    By going through different possibilities in expl_esm_adherence.ipynb, this turned out to be a reasonable option.
+
+    Parameters
+    ----------
+    df_esm_preprocessed: pd.DataFrame
+        A preprocessed dataframe of esm data, which must include the session ID (esm_session).
+
+    Returns
+    -------
+    df_session_counts: pd.Dataframe
+        A dataframe of all sessions (grouped by GROUP_SESSIONS_BY) with their statuses and the number of items.
+    """
+    sessions_grouped = df_esm_preprocessed.groupby(GROUP_SESSIONS_BY)
+
+    # 0. First, assign all session statuses as NaN.
+    df_session_counts = pd.DataFrame(sessions_grouped.count()["timestamp"]).rename(
+        columns={"timestamp": "esm_session_count"}
+    )
+    df_session_counts["session_response"] = np.nan
+
+    # 1. Identify all ESMs with status other than answered.
+    esm_not_answered = sessions_grouped.apply(
+        lambda x: (x.esm_status != ESM_STATUS_ANSWERED).any()
+    )
+    df_session_counts.loc[
+        esm_not_answered, "session_response"
+    ] = SESSION_STATUS_UNANSWERED
+
+    # 2. Identify non-sessions, i.e. answers about the end of the day.
+    non_session = sessions_grouped.apply(
+        lambda x: (
+            (x.esm_user_answer == ANSWER_DAY_FINISHED)  # I finished working for today.
+            | (x.esm_user_answer == ANSWER_DAY_OFF)  # I am not going to work today.
+            | (
+                x.esm_user_answer == ANSWER_SET_EVENING
+            )  # When would you like to answer the evening EMA?
+        ).any()
+    )
+    df_session_counts.loc[non_session, "session_response"] = SESSION_STATUS_DAY_FINISHED
+
+    # 3. Identify sessions appearing only once, as those were not true EMAs for sure.
+    singleton_sessions = (df_session_counts.esm_session_count == 1) & (
+        df_session_counts.session_response.isna()
+    )
+    df_session_1 = df_session_counts[singleton_sessions]
+    df_esm_unique_session = df_session_1.join(
+        df_esm_preprocessed.set_index(GROUP_SESSIONS_BY), how="left"
+    )
+    df_esm_unique_session = df_esm_unique_session.assign(
+        session_response=lambda x: x.esm_trigger
+    )["session_response"]
+    df_session_counts.loc[
+        df_esm_unique_session.index, "session_response"
+    ] = df_esm_unique_session
+
+    # 4. Mark the remaining sessions as completed.
+    df_session_counts.loc[
+        df_session_counts.session_response.isna(), "session_response"
+    ] = SESSION_STATUS_COMPLETE
+
+    return df_session_counts
+
+
+def classify_sessions_by_time(df_esm_preprocessed: pd.DataFrame) -> pd.DataFrame:
+    """
+    For each EMA session, determine the time of the first user answer and its time type (morning, workday, or evening.)
+
+    Parameters
+    ----------
+    df_esm_preprocessed: pd.DataFrame
+        A preprocessed dataframe of esm data, which must include the session ID (esm_session).
+
+    Returns
+    -------
+    df_session_time: pd.DataFrame
+        A dataframe of all sessions (grouped by GROUP_SESSIONS_BY) with their time type and timestamp of first answer.
+    """
+    df_session_time = (
+        df_esm_preprocessed.sort_values(["datetime_lj"]) # "participant_id"
+        .groupby(GROUP_SESSIONS_BY)
+        .first()[["time", "datetime_lj"]]
+    )
+    return df_session_time
+
+
+def classify_sessions_by_completion_time(
+    df_esm_preprocessed: pd.DataFrame,
+) -> pd.DataFrame:
+    """
+    The point of this function is to not only classify sessions by using the previously defined functions.
+    It also serves to "correct" the time type of some EMA sessions.
+
+    A morning questionnaire could seamlessly transition into a daytime questionnaire,
+        if the participant was already at work.
+    In this case, the "time" label changed mid-session.
+    Because of the way classify_sessions_by_time works, this questionnaire was classified as "morning".
+    But for all intents and purposes, it can be treated as a "daytime" EMA.
+
+    The way this scenario is differentiated from a true "morning" questionnaire,
+        where the participants NOT yet at work, is by considering their length.
+
+    Parameters
+    ----------
+    df_esm_preprocessed: pd.DataFrame
+        A preprocessed dataframe of esm data, which must include the session ID (esm_session).
+
+    Returns
+    -------
+    df_session_counts_time: pd.DataFrame
+        A dataframe of all sessions (grouped by GROUP_SESSIONS_BY) with statuses, the number of items,
+            their time type (with some morning EMAs reclassified) and timestamp of first answer.
+
+    """
+    df_session_counts = classify_sessions_by_completion(df_esm_preprocessed)
+    df_session_time = classify_sessions_by_time(df_esm_preprocessed)
+
+    df_session_counts_time = df_session_time.join(df_session_counts)
+
+    morning_transition_to_daytime = (df_session_counts_time.time == "morning") & (
+        df_session_counts_time.esm_session_count > MAX_MORNING_LENGTH
+    )
+
+    df_session_counts_time.loc[morning_transition_to_daytime, "time"] = "daytime"
+
+    return df_session_counts_time
+
+
+# def clean_up_esm(df_esm_preprocessed: pd.DataFrame) -> pd.DataFrame:
+#     """
+#     This function eliminates invalid ESM responses.
+#     It removes unanswered ESMs and those that indicate end of work and similar.
+#     It also extracts a numeric answer from strings such as "4 - I strongly agree".
+
+#     Parameters
+#     ----------
+#     df_esm_preprocessed: pd.DataFrame
+#         A preprocessed dataframe of esm data.
+
+#     Returns
+#     -------
+#     df_esm_clean: pd.DataFrame
+#         A subset of the original dataframe.
+
+#     """
+#     df_esm_clean = df_esm_preprocessed[
+#         df_esm_preprocessed["esm_status"] == ESM_STATUS_ANSWERED
+#     ]
+#     df_esm_clean = df_esm_clean[
+#         ~df_esm_clean["esm_user_answer"].isin(
+#             [ANSWER_DAY_FINISHED, ANSWER_DAY_OFF, ANSWER_SET_EVENING]
+#         )
+#     ]
+#     df_esm_clean["esm_user_answer_numeric"] = np.nan
+#     esm_type_numeric = [
+#         ESM.ESM_TYPE.get("radio"),
+#         ESM.ESM_TYPE.get("scale"),
+#         ESM.ESM_TYPE.get("number"),
+#     ]
+#     df_esm_clean.loc[
+#         df_esm_clean["esm_type"].isin(esm_type_numeric)
+#     ] = df_esm_clean.loc[df_esm_clean["esm_type"].isin(esm_type_numeric)].assign(
+#         esm_user_answer_numeric=lambda x: x.esm_user_answer.str.slice(stop=1).astype(
+#             int
+#         )
+#     )
+#     return df_esm_clean
--- a/src/features/phone_esm/straw/main.py
+++ b/src/features/phone_esm/straw/main.py
@ -52,7 +52,6 @@ def straw_features(sensor_data_files, time_segment, provider, filter_data_by_seg

        if not esm_data.empty:
            esm_features = pd.DataFrame()
-
            for scale in requested_scales:
                questionnaire_id = QUESTIONNAIRE_IDS[scale]
                mask = esm_data["questionnaire_id"] == questionnaire_id
@ -60,4 +59,7 @@ def straw_features(sensor_data_files, time_segment, provider, filter_data_by_seg
                #TODO Create the column esm_user_score in esm_clean. Currently, this is only done when reversing.

            esm_features = esm_features.reset_index()
+            if 'index' in esm_features: # In calse of empty esm_features df 
+                esm_features.rename(columns={'index': 'local_segment'}, inplace=True)
+
    return esm_features
--- a/src/features/phone_esm/straw/process_user_event_related_segments.py
+++ b/src/features/phone_esm/straw/process_user_event_related_segments.py
@ -0,0 +1,86 @@
+import pandas as pd
+import numpy as np
+import datetime
+
+import math, sys, yaml
+
+from esm_preprocess import clean_up_esm
+from esm import classify_sessions_by_completion_time, preprocess_esm
+
+input_data_files = dict(snakemake.input)
+
+def format_timestamp(x):
+    tstring=""
+    space = False
+    if x//3600 > 0:
+        tstring += f"{x//3600}H"
+        space = True
+    if x % 3600 // 60 > 0:
+        tstring += f" {x % 3600 // 60}M" if "H" in tstring else f"{x % 3600 // 60}M"
+    if x % 60 > 0:  
+        tstring += f" {x % 60}S" if "M" in tstring or "H" in tstring else f"{x % 60}S"
+    
+    return tstring
+
+
+def extract_ers_from_file(esm_df, device_id): # TODO: session_id groupby -> spremeni naziv segmenta
+
+    pd.set_option("display.max_rows", None)
+    pd.set_option("display.max_columns", None)
+
+    # extracted_ers = pd.DataFrame(columns=["label", "event_timestamp", "length", "shift", "shift_direction", "device_id"])
+
+    # esm_df = clean_up_esm(preprocess_esm(esm_df))
+    esm_preprocessed = clean_up_esm(preprocess_esm(esm_df))
+
+    # Take only during work sessions
+    # during_work = esm_df[esm_df["esm_trigger"].str.contains("during_work", na=False)]
+    # esm_trigger_group = esm_df.groupby("esm_session").agg(pd.Series.mode)['esm_trigger'] # Get most frequent esm_trigger within particular session  
+    # esm_filtered_sessions = list(esm_trigger_group[esm_trigger_group == 'during_work'].index) # Take only sessions that contains during work  
+
+    # Take only ema_completed sessions responses
+    classified = classify_sessions_by_completion_time(esm_preprocessed)
+    esm_filtered_sessions = classified[classified["session_response"] == 'ema_completed'].reset_index()['esm_session']
+    
+    esm_df = esm_preprocessed[esm_preprocessed["esm_session"].isin(esm_filtered_sessions)]
+
+    # Extract time-relevant information
+    extracted_ers = esm_df.groupby(["device_id", "esm_session"])['timestamp'].apply(lambda x: math.ceil((x.max() - x.min()) / 1000)).reset_index() # in rounded up seconds
+    extracted_ers = extracted_ers[extracted_ers["timestamp"] <= 15 * 60].reset_index(drop=True) # ensure that the longest duration of the questionnaire anwsering is 15 min
+    # TODO: Rename "timestamp" column meaningfully.
+
+    time_before_questionnaire = 30 * 60 # in seconds (30 minutes)
+
+    extracted_ers["label"] = "straw_event_" + snakemake.params["pid"] + "_" + extracted_ers.index.astype(str).str.zfill(3)
+    extracted_ers["event_timestamp"] = esm_df.groupby("esm_session")['timestamp'].min().reset_index()['timestamp']
+    extracted_ers["length"] = (extracted_ers["timestamp"] + time_before_questionnaire).apply(lambda x: format_timestamp(x))
+    # TODO: Think about adding questionnaire duration.
+    extracted_ers["shift"] = time_before_questionnaire
+    extracted_ers["shift"] = extracted_ers["shift"].apply(lambda x: format_timestamp(x))
+    extracted_ers["shift_direction"] = -1 
+    extracted_ers["device_id"] = device_id 
+
+    return extracted_ers[["label", "event_timestamp", "length", "shift", "shift_direction", "device_id"]]
+
+# TODO: potrebno preveriti kako se izvaja iskanje prek device_id -> na tem temelji tudi proces ekstrahiranja ERS
+
+if snakemake.params["stage"] == "extract": # TODO: najprej preveri ustreznost umeščenosti v RAPIDS pipelineu
+    esm_df = pd.read_csv(input_data_files['esm_raw_input'])
+
+    with open(input_data_files['pid_file'], 'r') as stream:
+        pid_file = yaml.load(stream, Loader=yaml.FullLoader)
+
+    extracted_ers = extract_ers_from_file(esm_df, pid_file["PHONE"]["DEVICE_IDS"][0])
+
+    extracted_ers.to_csv(snakemake.output[0], index=False)
+elif snakemake.params["stage"] == "merge":
+
+    input_data_files = dict(snakemake.input)
+    straw_events = pd.DataFrame(columns=["label", "event_timestamp", "length", "shift", "shift_direction", "device_id"])
+    
+    for input_file in input_data_files["ers_files"]:
+        ers_df = pd.read_csv(input_file)
+        straw_events = pd.concat([straw_events, ers_df], axis=0, ignore_index=True)
+
+    straw_events.to_csv(snakemake.output[0], index=False)
+
--- a/src/features/phone_light/rapids/main.py
+++ b/src/features/phone_light/rapids/main.py
@ -29,7 +29,7 @@ def rapids_features(sensor_data_files, time_segment, provider, filter_data_by_se
            if "medianlux" in features_to_compute:
                light_features["medianlux"] = light_data.groupby(["local_segment"])["double_light_lux"].median()
            if "stdlux" in features_to_compute:
-                light_features["stdlux"] = light_data.groupby(["local_segment"])["double_light_lux"].std()
+                light_features["stdlux"] = light_data.groupby(["local_segment"])["double_light_lux"].std().fillna(0)
            
            light_features = light_features.reset_index()

--- a/src/features/phone_locations/doryab/main.py
+++ b/src/features/phone_locations/doryab/main.py
@ -37,7 +37,8 @@ def variance_and_logvariance_features(location_data, location_features):
    location_data["longitude_for_wvar"] = (location_data["double_longitude"] - location_data["longitude_wavg"]) ** 2 * location_data["duration"] * 60

    location_features["locationvariance"] = ((location_data_grouped["latitude_for_wvar"].sum() + location_data_grouped["longitude_for_wvar"].sum()) / (location_data_grouped["duration"].sum() * 60 - 1)).fillna(0)
-    location_features["loglocationvariance"] = np.log10(location_features["locationvariance"]).replace(-np.inf, np.nan)
+    
+    location_features["loglocationvariance"] = np.log10(location_features["locationvariance"]).replace(-np.inf, -1000000)

    return location_features

--- a/src/features/phone_messages/rapids/main.R
+++ b/src/features/phone_messages/rapids/main.R
@ -65,6 +65,15 @@ rapids_features <- function(sensor_data_files, time_segment, provider){
        features <- message_features_of_type(messages_of_type, message_type, time_segment, requested_features)
        messages_features <- merge(messages_features, features, all=TRUE)
    }
-    messages_features <- messages_features %>% mutate_at(vars(contains("countmostfrequentcontact") | contains("distinctcontacts") | contains("count")), list( ~ replace_na(., 0)))
+    # Fill seleted columns with a high number
+    time_cols <- select(messages_features, contains("timefirstmessages") |  contains("timelastmessages")) %>% 
+    colnames(.)
+
+    messages_features <- messages_features %>% 
+        mutate_at(., time_cols, ~replace(., is.na(.), 1500))
+    
+    # Fill NA values with 0
+    messages_features <- messages_features %>% mutate_all(~replace(., is.na(.), 0))
+    
    return(messages_features)
 }
--- a/src/features/phone_screen/rapids/main.py
+++ b/src/features/phone_screen/rapids/main.py
@ -15,7 +15,7 @@ def getEpisodeDurationFeatures(screen_data, time_segment, episode, features, ref
    if "avgduration" in features:
        duration_helper = pd.concat([duration_helper, screen_data_episode.groupby(["local_segment"])[["duration"]].mean().rename(columns = {"duration":"avgduration" + episode})], axis = 1)
    if "stdduration" in features:
-        duration_helper = pd.concat([duration_helper, screen_data_episode.groupby(["local_segment"])[["duration"]].std().rename(columns = {"duration":"stdduration" + episode})], axis = 1)
+        duration_helper = pd.concat([duration_helper, screen_data_episode.groupby(["local_segment"])[["duration"]].std().fillna(0).rename(columns = {"duration":"stdduration" + episode})], axis = 1)
    if "firstuseafter" + "{0:0=2d}".format(reference_hour_first_use) in features:
        screen_data_episode_after_hour = screen_data_episode.copy()
        screen_data_episode_after_hour["hour"] = pd.to_datetime(screen_data_episode["local_start_date_time"]).dt.hour
--- a/src/features/phone_wifi_visible/rapids/main.R
+++ b/src/features/phone_wifi_visible/rapids/main.R
@ -9,21 +9,26 @@ compute_wifi_feature <- function(data, feature, time_segment){
              "countscans" = data %>% summarise(!!feature := n()),
              "uniquedevices" = data %>% summarise(!!feature := n_distinct(bssid)))
    return(data)
+
   } else if(feature == "countscansmostuniquedevice"){
     # Get the most scanned device
-    mostuniquedevice <- data %>% 
+    mostuniquedevice <- data %>%
+      filter(bssid != "") %>% 
      group_by(bssid) %>% 
      mutate(N=n()) %>% 
      ungroup() %>%
      filter(N == max(N)) %>% 
      head(1) %>% # if there are multiple device with the same amount of scans pick the first one only
      pull(bssid)
+
    data <- data %>% filter_data_by_segment(time_segment)
+
    return(data %>% 
             filter(bssid == mostuniquedevice) %>%
             group_by(local_segment) %>% 
-             summarise(!!feature := n()) %>%
-             replace(is.na(.), 0))
+             summarise(!!feature := n())
+    )
+
  }
 }

@ -43,6 +48,6 @@ rapids_features <- function(sensor_data_files, time_segment, provider){
    feature <- compute_wifi_feature(wifi_data, feature_name, time_segment)
    features <- merge(features, feature, by="local_segment", all = TRUE)
  }
-
+  features <- features %>% mutate_all(~replace(., is.na(.), 0))
  return(features)
 }
--- a/src/features/standardization/main.py
+++ b/src/features/standardization/main.py
@ -1,50 +0,0 @@
-import pandas as pd
-import numpy as np
-from sklearn.preprocessing import StandardScaler
-
-import sys
-
-sensor_data_files = dict(snakemake.input)
-
-provider = snakemake.params["provider"]
-provider_key = snakemake.params["provider_key"]
-sensor_key = snakemake.params["sensor_key"]
-
-pd.set_option('display.max_columns', None)
-
-if provider_key == "cr":
-    sys.path.append('/rapids/src/features/')
-    from cr_features_helper_methods import extract_second_order_features
-
-    provider_main = snakemake.params["provider_main"]
-    prefix = sensor_key + "_" + provider_key + "_"
-
-    windows_features_data = pd.read_csv(sensor_data_files["windows_features_data"])
-    excluded_columns = ['local_segment', 'local_segment_label', 'local_segment_start_datetime', 'local_segment_end_datetime', prefix + "level_1"]
-    
-    if windows_features_data.empty:
-        windows_features_data.to_csv(snakemake.output[1], index=False)
-        windows_features_data.to_csv(snakemake.output[0], index=False)
-    else:
-        windows_features_data.loc[:, ~windows_features_data.columns.isin(excluded_columns)] = StandardScaler().fit_transform(windows_features_data.loc[:, ~windows_features_data.columns.isin(excluded_columns)])
-
-        windows_features_data.to_csv(snakemake.output[1], index=False)
-
-        if provider_main["WINDOWS"]["COMPUTE"] and "SECOND_ORDER_FEATURES" in provider_main["WINDOWS"]:
-            so_features_names = provider_main["WINDOWS"]["SECOND_ORDER_FEATURES"]
-            windows_so_features_data = extract_second_order_features(windows_features_data, so_features_names, prefix)
-            windows_so_features_data.to_csv(snakemake.output[0], index=False)
-        else:
-            pd.DataFrame().to_csv(snakemake.output[0], index=False)
-
-else: 
-    for sensor_features in sensor_data_files["sensor_features"]:
-        if "/" + sensor_key + ".csv" in sensor_features:
-            sensor_data = pd.read_csv(sensor_features)
-            excluded_columns = ['local_segment', 'local_segment_label', 'local_segment_start_datetime', 'local_segment_end_datetime']
-
-            if not sensor_data.empty:
-                sensor_data.loc[:, ~sensor_data.columns.isin(excluded_columns)] = StandardScaler().fit_transform(sensor_data.loc[:, ~sensor_data.columns.isin(excluded_columns)])
-            
-            sensor_data.to_csv(snakemake.output[0], index=False)
-            break
--- a/src/features/utils/utils.py
+++ b/src/features/utils/utils.py
@ -160,12 +160,16 @@ def fetch_provider_features(provider, provider_key, sensor_key, sensor_data_file

        return sensor_features

-def run_provider_cleaning_script(provider, provider_key, sensor_key, sensor_data_files):
+def run_provider_cleaning_script(provider, provider_key, sensor_key, sensor_data_files, target=False):
    from importlib import import_module, util
    print("{} Processing {} {}".format(rapids_log_tag, sensor_key, provider_key))

    cleaning_module = import_path(provider["SRC_SCRIPT"])
    cleaning_function = getattr(cleaning_module,  provider_key.lower() + "_cleaning")
-    sensor_features = cleaning_function(sensor_data_files, provider)
+
+    if target:
+        sensor_features = cleaning_function(sensor_data_files, provider, target)
+    else:
+        sensor_features = cleaning_function(sensor_data_files, provider)

    return sensor_features
--- a/src/models/merge_features_and_targets_for_population_model.py
+++ b/src/models/merge_features_and_targets_for_population_model.py
@ -12,9 +12,13 @@ for baseline_features_path in snakemake.input["demographic_features"]:
    all_baseline_features = pd.concat([all_baseline_features, baseline_features], axis=0)

 # merge sensor features and baseline features
-features = sensor_features.merge(all_baseline_features, on="pid", how="left")
+if not sensor_features.empty:
+    features = sensor_features.merge(all_baseline_features, on="pid", how="left")

-target_variable_name = snakemake.params["target_variable"]
-model_input = retain_target_column(features, target_variable_name)
+    target_variable_name = snakemake.params["target_variable"]
+    model_input = retain_target_column(features, target_variable_name)

-model_input.to_csv(snakemake.output[0], index=False)
+    model_input.to_csv(snakemake.output[0], index=False)
+    
+else:
+    sensor_features.to_csv(snakemake.output[0], index=False)
--- a/src/models/select_targets.py
+++ b/src/models/select_targets.py
@ -6,6 +6,5 @@ cleaned_sensor_features = pd.read_csv(snakemake.input["cleaned_sensor_features"]
 target_variable_name = snakemake.params["target_variable"]

 model_input = retain_target_column(cleaned_sensor_features, target_variable_name)
-model_input.dropna(axis ="index", how="any", subset=["target"], inplace=True)

 model_input.to_csv(snakemake.output[0], index=False)
--- a/tests/scripts/NaN.png
+++ b/tests/scripts/NaN.png
--- a/tests/scripts/missing_vals.py
+++ b/tests/scripts/missing_vals.py
@ -3,8 +3,8 @@ import seaborn as sns
 import matplotlib.pyplot as plt


-participant = "p031"
-all_sensors = ["eda", "bvp", "ibi", "temp", "acc"]
+participant = "p01"
+all_sensors = ["eda", "ibi", "temp", "acc"]

 for sensor in all_sensors:

--- a/tests/scripts/phone_feats.py
+++ b/tests/scripts/phone_feats.py
@ -0,0 +1,285 @@
+import pandas as pd
+import seaborn as sns
+import matplotlib.pyplot as plt
+
+
+path = "/rapids/data/processed/features/all_participants/all_sensor_features.csv"
+df = pd.read_csv(path)
+
+# activity_recognition
+
+cols = [col for col in df.columns if "activity_recognition" in col]
+df_x = df[cols]
+
+print(len(cols))
+print(df_x)
+
+df_x = df_x.dropna(axis=0, how="all")
+sns.heatmap(df_x.isna(), xticklabels=1)
+plt.savefig(f'activity_recognition_values', bbox_inches='tight')
+
+df_q = pd.DataFrame()
+for col in df_x:
+    df_q[col] = pd.to_numeric(pd.cut(df_x[col], bins=[-1,0,0.000000000001,1000], labels=[-1,0,1], right=False))
+
+sns.heatmap(df_q, cbar=False, xticklabels=1)
+plt.savefig(f'cut_activity_recognition_values', bbox_inches='tight')
+plt.close()
+
+# applications_foreground
+
+cols = [col for col in df.columns if "applications_foreground" in col]
+df_x = df[cols]
+
+print(len(cols))
+print(df_x)
+
+df_x = df_x.dropna(axis=0, how="all")
+sns.heatmap(df_x.isna(), xticklabels=1)
+plt.savefig(f'applications_foreground_values', bbox_inches='tight')
+
+df_q = pd.DataFrame()
+for col in df_x:
+    df_q[col] = pd.to_numeric(pd.cut(df_x[col], bins=[-1,0,0.000000000001,1000], labels=[-1,0,1], right=False))
+
+sns.heatmap(df_q, cbar=False, xticklabels=1)
+plt.savefig(f'cut_applications_foreground_values', bbox_inches='tight')
+plt.close()
+
+# battery
+
+cols = [col for col in df.columns if "phone_battery" in col]
+df_x = df[cols]
+
+print(len(cols))
+print(df_x)
+
+df_x = df_x.dropna(axis=0, how="all")
+sns.heatmap(df_x.isna(), xticklabels=1)
+plt.savefig(f'phone_battery_values', bbox_inches='tight')
+
+df_q = pd.DataFrame()
+for col in df_x:
+    df_q[col] = pd.to_numeric(pd.cut(df_x[col], bins=[-1,0,0.000000000001,1000], labels=[-1,0,1], right=False))
+
+sns.heatmap(df_q, cbar=False, xticklabels=1)
+plt.savefig(f'cut_phone_battery_values', bbox_inches='tight')
+plt.close()
+
+# bluetooth_doryab
+
+cols = [col for col in df.columns if "bluetooth_doryab" in col]
+df_x = df[cols]
+
+print(len(cols))
+print(df_x)
+
+df_x = df_x.dropna(axis=0, how="all")
+sns.heatmap(df_x.isna(), xticklabels=1)
+plt.savefig(f'bluetooth_doryab_values', bbox_inches='tight')
+
+df_q = pd.DataFrame()
+for col in df_x:
+    df_q[col] = pd.to_numeric(pd.cut(df_x[col], bins=[-1,0,0.000000000001,1000], labels=[-1,0,1], right=False))
+
+sns.heatmap(df_q, cbar=False, xticklabels=1)
+plt.savefig(f'cut_bluetooth_doryab_values', bbox_inches='tight')
+plt.close()
+
+# bluetooth_rapids
+
+cols = [col for col in df.columns if "bluetooth_rapids" in col]
+df_x = df[cols]
+
+print(len(cols))
+print(df_x)
+
+df_x = df_x.dropna(axis=0, how="all")
+sns.heatmap(df_x.isna(), xticklabels=1)
+plt.savefig(f'bluetooth_rapids_values', bbox_inches='tight')
+
+df_q = pd.DataFrame()
+for col in df_x:
+    df_q[col] = pd.to_numeric(pd.cut(df_x[col], bins=[-1,0,0.000000000001,1000], labels=[-1,0,1], right=False))
+
+sns.heatmap(df_q, cbar=False, xticklabels=1)
+plt.savefig(f'cut_bluetooth_rapids_values', bbox_inches='tight')
+plt.close()
+
+# calls
+
+cols = [col for col in df.columns if "phone_calls" in col]
+df_x = df[cols]
+
+print(len(cols))
+print(df_x)
+
+df_x = df_x.dropna(axis=0, how="all")
+sns.heatmap(df_x.isna(), xticklabels=1)
+plt.savefig(f'phone_calls_values', bbox_inches='tight')
+
+df_q = pd.DataFrame()
+for col in df_x:
+    df_q[col] = pd.to_numeric(pd.cut(df_x[col], bins=[-1,0,0.000000000001,1000], labels=[-1,0,1], right=False))
+
+sns.heatmap(df_q, cbar=False, xticklabels=1)
+plt.savefig(f'cut_phone_calls_values', bbox_inches='tight')
+plt.close()
+
+# data_yield
+
+cols = [col for col in df.columns if "data_yield" in col]
+df_x = df[cols]
+
+print(len(cols))
+print(df_x)
+
+df_x = df_x.dropna(axis=0, how="all")
+sns.heatmap(df_x.isna(), xticklabels=1)
+plt.savefig(f'data_yield_values', bbox_inches='tight')
+
+df_q = pd.DataFrame()
+for col in df_x:
+    df_q[col] = pd.to_numeric(pd.cut(df_x[col], bins=[-1,0,0.000000000001,1000], labels=[-1,0,1], right=False))
+
+sns.heatmap(df_q, cbar=False, xticklabels=1)
+plt.savefig(f'cut_data_yield_values', bbox_inches='tight')
+plt.close()
+
+# esm
+
+cols = [col for col in df.columns if "phone_esm" in col]
+df_x = df[cols]
+
+print(len(cols))
+print(df_x)
+
+df_x = df_x.dropna(axis=0, how="all")
+sns.heatmap(df_x.isna(), xticklabels=1)
+plt.savefig(f'phone_esm_values', bbox_inches='tight')
+
+df_q = pd.DataFrame()
+for col in df_x:
+    df_q[col] = pd.to_numeric(pd.cut(df_x[col], bins=[-1,0,0.000000000001,1000], labels=[-1,0,1], right=False))
+
+sns.heatmap(df_q, cbar=False, xticklabels=1)
+plt.savefig(f'cut_phone_esm_values', bbox_inches='tight')
+plt.close()
+
+# light
+
+cols = [col for col in df.columns if "phone_light" in col]
+df_x = df[cols]
+
+print(len(cols))
+print(df_x)
+
+df_x = df_x.dropna(axis=0, how="all")
+sns.heatmap(df_x.isna(), xticklabels=1)
+plt.savefig(f'phone_light_values', bbox_inches='tight')
+
+df_q = pd.DataFrame()
+for col in df_x:
+    df_q[col] = pd.to_numeric(pd.cut(df_x[col], bins=[-1,0,0.000000000001,1000], labels=[-1,0,1], right=False))
+
+sns.heatmap(df_q, cbar=False, xticklabels=1)
+plt.savefig(f'cut_phone_light_values', bbox_inches='tight')
+plt.close()
+
+# locations_doryab
+
+cols = [col for col in df.columns if "locations_doryab" in col]
+df_x = df[cols]
+
+print(len(cols))
+print(df_x)
+
+df_x = df_x.dropna(axis=0, how="all")
+sns.heatmap(df_x.isna(), xticklabels=1)
+plt.savefig(f'locations_doryab_values', bbox_inches='tight')
+
+df_q = pd.DataFrame()
+for col in df_x:
+    df_q[col] = pd.to_numeric(pd.cut(df_x[col], bins=[-1,0,0.000000000001,1000], labels=[-1,0,1], right=False))
+
+sns.heatmap(df_q, cbar=False, xticklabels=1)
+plt.savefig(f'cut_locations_doryab_values', bbox_inches='tight')
+plt.close()
+
+# locations_barnett
+
+# Not working
+
+# messages
+
+cols = [col for col in df.columns if "phone_messages" in col]
+df_x = df[cols]
+
+print(len(cols))
+print(df_x)
+
+df_x = df_x.dropna(axis=0, how="all")
+sns.heatmap(df_x.isna(), xticklabels=1)
+plt.savefig(f'phone_messages_values', bbox_inches='tight')
+
+df_q = pd.DataFrame()
+for col in df_x:
+    df_q[col] = pd.to_numeric(pd.cut(df_x[col], bins=[-1,0,0.000000000001,1000], labels=[-1,0,1], right=False))
+
+sns.heatmap(df_q, cbar=False, xticklabels=1)
+plt.savefig(f'cut_phone_messages_values', bbox_inches='tight')
+plt.close()
+
+# screen
+
+cols = [col for col in df.columns if "phone_screen" in col]
+df_x = df[cols]
+
+print(len(cols))
+print(df_x)
+
+df_x = df_x.dropna(axis=0, how="all")
+sns.heatmap(df_x.isna(), xticklabels=1)
+plt.savefig(f'phone_screen_values', bbox_inches='tight')
+
+df_q = pd.DataFrame()
+for col in df_x:
+    df_q[col] = pd.to_numeric(pd.cut(df_x[col], bins=[-1,0,0.000000000001,1000], labels=[-1,0,1], right=False))
+
+sns.heatmap(df_q, cbar=False, xticklabels=1)
+plt.savefig(f'cut_phone_screen_values', bbox_inches='tight')
+plt.close()
+
+# wifi_visible
+
+cols = [col for col in df.columns if "wifi_visible" in col]
+df_x = df[cols]
+
+print(len(cols))
+print(df_x)
+
+df_x = df_x.dropna(axis=0, how="all")
+sns.heatmap(df_x.isna(), xticklabels=1)
+plt.savefig(f'wifi_visible_values', bbox_inches='tight')
+
+df_q = pd.DataFrame()
+for col in df_x:
+    df_q[col] = pd.to_numeric(pd.cut(df_x[col], bins=[-1,0,0.000000000001,1000], labels=[-1,0,1], right=False))
+
+sns.heatmap(df_q, cbar=False, xticklabels=1)
+plt.savefig(f'cut_wifi_visible_values', bbox_inches='tight')
+plt.close()
+
+# All features
+
+print(len(df))
+print(df)
+
+# df = df.dropna(axis=0, how="all")
+# df = df.dropna(axis=1, how="all")
+sns.heatmap(df.isna())
+plt.savefig(f'all_features', bbox_inches='tight')
+
+print(df.columns[df.isna().all()].tolist())
+print("All NaNs:", df.isna().sum().sum())
+print("Df shape NaNs:", df.shape)
--- a/tests/scripts/standardization_methods_test.py
+++ b/tests/scripts/standardization_methods_test.py
@ -0,0 +1,70 @@
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt 
+from sklearn.preprocessing import StandardScaler
+import sys
+
+sys.path.append('/rapids/')
+from src.features import cr_features_helper_methods as crhm
+
+pd.set_option("display.max_columns", None)
+features_win = pd.read_csv("data/interim/p031/empatica_temperature_features/empatica_temperature_python_cr_windows.csv", usecols=[0, 1, 2, 3, 4, 5])
+
+# First standardization method
+excluded_columns = ['local_segment', 'local_segment_label', 'local_segment_start_datetime', 'local_segment_end_datetime', "empatica_temperature_cr_level_1"]
+z1_windows = features_win.copy()
+z1_windows.loc[:, ~z1_windows.columns.isin(excluded_columns)] = StandardScaler().fit_transform(z1_windows.loc[:, ~z1_windows.columns.isin(excluded_columns)])
+z1 = crhm.extract_second_order_features(z1_windows, ['mean', 'median', 'sd', 'nlargest', 'nsmallest', 'count_windows'], prefix="empatica_temperature_cr_")
+z1 = z1.iloc[:,4:]
+# print(z1)
+
+# Second standardization method
+so_features_reg = crhm.extract_second_order_features(features_win, ['mean', 'median', 'sd', 'nlargest', 'nsmallest', 'count_windows'], prefix="empatica_temperature_cr_")
+so_features_reg = so_features_reg.iloc[:,4:]
+z2 = pd.DataFrame(StandardScaler().fit_transform(so_features_reg), columns=so_features_reg.columns)
+# print(z2)
+
+# Standardization of the first standardization method values
+z1_z = pd.DataFrame(StandardScaler().fit_transform(z1), columns=z1.columns)
+# print(z1_z)
+
+# For SD
+fig, axs = plt.subplots(3, figsize=(8, 10))
+axs[0].plot(z1['empatica_temperature_cr_squareSumOfComponent_X_SO_sd'])
+axs[0].set_title("Z1 - standardizirana okna, nato ekstrahiranje značilk SO")
+
+axs[1].plot(z2['empatica_temperature_cr_squareSumOfComponent_X_SO_sd'])
+axs[1].set_title("Z2 - ekstrahirane značilke SO 'normalnih' vrednosti, nato standardizacija")
+
+axs[2].plot(z1_z['empatica_temperature_cr_squareSumOfComponent_X_SO_sd'])
+axs[2].set_title("Standardiziran Z1")
+
+fig.suptitle('Z-Score methods for temperature_squareSumOfComponent_SO_sd')
+plt.savefig('z_score_comparison_temperature_squareSumOfComponent_X_SO_sd', bbox_inches='tight')
+
+showcase = pd.DataFrame()
+showcase['Z1__SD'] = z1['empatica_temperature_cr_squareSumOfComponent_X_SO_sd']
+showcase['Z2__SD'] = z2['empatica_temperature_cr_squareSumOfComponent_X_SO_sd']
+showcase['Z1__SD_STANDARDIZED'] = z1_z['empatica_temperature_cr_squareSumOfComponent_X_SO_sd']
+print(showcase)
+
+# For 
+fig, axs = plt.subplots(3, figsize=(8, 10))
+axs[0].plot(z1['empatica_temperature_cr_squareSumOfComponent_X_SO_nlargest'])
+axs[0].set_title("Z1 - standardizirana okna, nato ekstrahiranje značilk SO")
+
+axs[1].plot(z2['empatica_temperature_cr_squareSumOfComponent_X_SO_nlargest'])
+axs[1].set_title("Z2")
+
+axs[2].plot(z1_z['empatica_temperature_cr_squareSumOfComponent_X_SO_nlargest'])
+axs[2].set_title("Standardized Z1")
+
+fig.suptitle('Z-Score methods for temperature_squareSumOfComponent_SO_nlargest')
+plt.savefig('z_score_comparison_temperature_squareSumOfComponent_X_SO_nlargest', bbox_inches='tight')
+
+showcase2 = pd.DataFrame()
+showcase2['Z1__nlargest'] = z1['empatica_temperature_cr_squareSumOfComponent_X_SO_nlargest']
+showcase2['Z2__nlargest'] = z2['empatica_temperature_cr_squareSumOfComponent_X_SO_nlargest']
+showcase2['Z1__nlargest_STANDARDIZED'] = z1_z['empatica_temperature_cr_squareSumOfComponent_X_SO_nlargest']
+print(showcase2)
+
--- a/tests/scripts/test_acc.py
+++ b/tests/scripts/test_acc.py
@ -0,0 +1,38 @@
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+
+import sys
+
+df = pd.read_csv(f"/rapids/data/raw/p03/empatica_accelerometer_raw.csv")
+
+
+df['date'] = pd.to_datetime(df['timestamp'],unit='ms')
+df.set_index('date', inplace=True)
+print(df)
+df = df['double_values_0'].resample("31ms").mean()
+print(df)
+
+st='2021-05-21 12:28:27'
+en='2021-05-21 12:59:12'
+
+df = df.loc[(df.index > st) & (df.index < en)]
+plt.plot(df)
+
+plt.savefig(f'NaN.png')
+sys.exit()
+
+
+plt.plot(df)
+
+esm = pd.read_csv(f"/rapids/data/raw/p03/phone_esm_raw.csv")
+
+esm['date'] = pd.to_datetime(esm['timestamp'],unit='ms')
+esm = esm[esm['date']]
+esm.set_index('date', inplace=True)
+print(esm)
+
+esm = esm['esm_session'].resample("2900ms").mean()
+
+plt.plot(esm)
+plt.savefig(f'NaN.png')
Author	SHA1	Message	Date
junos	bb1e1900a4	Add a note to think about.	2023-03-27 11:56:09 +02:00
junos	c35c75682f	Add todo to rename a column.	2023-03-27 11:46:04 +02:00
Primoz	936324d234	Switch config for 30 minutes event related segments.	2022-10-26 14:17:27 +00:00
Primoz	da0a4596f8	Add additional ESM processing logic for ERS csv extraction.	2022-10-26 14:16:25 +00:00
Primoz	d4d74818e6	Fix a bug - missing time_segment column when df is empty	2022-10-26 14:14:32 +00:00
Primoz	14ff59914b	Fix to correct dtypes.	2022-10-26 09:59:46 +00:00
Primoz	6ab0ac5329	Optimize memory consumption with dtype definition while reading csv file.	2022-10-26 09:57:26 +00:00
Primoz	b92a3aa37a	Remove unwanted output or other error producing code.	2022-10-25 15:25:22 +00:00
Primoz	bfd637eb9c	Improve strings formatting in straw_events file.	2022-10-25 08:53:44 +00:00
Primoz	0d81ad5756	Debug assignment of segments to rows	2022-10-19 13:35:04 +00:00
Primoz	cea451d344	Merge branch 'imputation_and_cleaning' of https://repo.ijs.si/junoslukan/rapids into imputation_and_cleaning	2022-10-18 09:15:06 +00:00
Primoz	e88bbd548f	Add new daily segment and filter by segment in the cleaning script.	2022-10-18 09:15:00 +00:00
Primoz	cf38d9f175	Implement ERS generating logic.	2022-10-17 15:07:33 +00:00
Primoz	f3ca56cdbf	Start with ERS logic integration within Snakemake.	2022-10-14 14:46:28 +00:00
Primoz	797aa98f4f	Config for ERS testing.	2022-10-12 15:51:50 +00:00
Primoz	9baff159cd	Changes needed for testing and starting of the Event-Related Segments.	2022-10-12 15:51:23 +00:00
Primoz	0f21273508	Bugs fix	2022-10-12 12:32:51 +00:00
Primoz	55517eb737	Necessary commit before proceeding.	2022-10-12 12:23:11 +00:00
Primoz	de15a52dba	Bug fix	2022-10-11 08:36:23 +00:00
Primoz	1ad25bb572	Few modifications of some imputation values in cleaning script and feature extraction.	2022-10-11 08:26:17 +00:00
Primoz	9884b383cf	Testing new data with AutoML.	2022-10-10 16:45:38 +00:00
Primoz	2dc89c083c	Small changes in cleaning overall	2022-10-07 08:52:12 +00:00
Primoz	001d400729	Clean features and create input files based on all possible targets.	2022-10-06 14:28:12 +00:00
Primoz	1e38d9bf1e	Standardization and correlation visualization in overall cleaning script.	2022-10-06 13:27:38 +00:00
Primoz	a34412a18d	E4 data yield corrections. Changes in overal cs - standardization.	2022-10-05 14:16:55 +00:00
Primoz	437459648f	Errors fix: individual script - treat participants missing data.	2022-10-05 13:35:05 +00:00
Primoz	53f6cc60d5	Config and cleaning script necessary changes ...	2022-10-03 13:06:39 +00:00
Primoz	bbeabeee6f	Last changes before processing on the server.	2022-10-03 12:53:31 +00:00
Primoz	44531c6d94	Code cleaning, reworking cleaning individual based on changes in overall script. Changes in thresholds.	2022-09-30 10:04:07 +00:00
Primoz	7ac7cd5a37	Preparation of the overall cleaning script.	2022-09-29 14:33:21 +00:00
Primoz	68fd69dada	Cleaning script for individuals: corrections and comments.	2022-09-29 11:55:25 +00:00
Primoz	a4f0d056a0	Fillna for app foreground and activity recognition	2022-09-29 11:44:27 +00:00
Primoz	6286e7a44c	firstuseafter column removed from contextual imputation	2022-09-28 12:47:08 +00:00
Primoz	9b3447febd	Contextual imputation correction	2022-09-28 12:40:05 +00:00
Primoz	d6adda30cf	Contextual imputation on time(first/last) features.	2022-09-28 12:37:51 +00:00
Primoz	8af4ef11dc	Contextual imputation by feature type.	2022-09-28 10:02:47 +00:00
Primoz	536b9494cd	Cleaning script corrections	2022-09-27 14:12:08 +00:00
Primoz	f0b87c9dd0	Debugging of the empatica data yield integration.	2022-09-27 09:54:15 +00:00
Primoz	7fcdb873fe	Merge branch 'imputation_and_cleaning' of https://repo.ijs.si/junoslukan/rapids into imputation_and_cleaning	2022-09-27 07:50:29 +00:00
Primoz	5c7bb0f4c1	Config changes	2022-09-27 07:48:32 +00:00
Primoz	bd53dc1684	Empatica data yield usage in the cleaning script.	2022-09-26 15:54:00 +00:00
Primoz	d9a574c550	Changes in the cleaning script and preparation of empatica data yield method.	2022-09-23 13:24:50 +00:00
Primoz	19aa8707c0	Redefined cleaning steps after revision	2022-09-22 13:45:51 +00:00
Primoz	247d758cb7	Merge branch 'imputation_and_cleaning' of https://repo.ijs.si/junoslukan/rapids into imputation_and_cleaning	2022-09-21 07:18:01 +00:00
Primoz	90ee99e4b9	Remove TODO comments	2022-09-21 07:16:00 +00:00
Primoz	7493aaa643	Small changes in cleaning scrtipt and missing vals testing.	2022-09-20 12:57:55 +00:00
Primoz	eaf4340afd	Small imputation and cleaning corrections.	2022-09-20 08:03:48 +00:00
Primoz	a96ea508c6	Fill NaN of Empatica's SD second order feature (must be tested).	2022-09-19 07:34:02 +00:00
Primoz	52e11cdcab	Configurations for new standardization path.	2022-09-19 07:25:54 +00:00
Primoz	92aff93e65	Remove standardization script.	2022-09-19 07:25:16 +00:00
Primoz	18b63127de	Removed all standardizaton rules and configurations.	2022-09-19 06:16:26 +00:00
Primoz	62982866cd	Phone wifi visible inspection (WIP)	2022-09-16 13:24:21 +00:00
Primoz	0ce6da5444	kNN imputation relocation and execution only on specific columns.	2022-09-16 11:30:08 +00:00
Primoz	e3b78c8a85	Impute selected phone features with 0. Wifi visible, screen, and light.	2022-09-16 10:58:57 +00:00
Primoz	7d85f75d21	Changes in phone features NaN values script.	2022-09-16 09:03:30 +00:00
Primoz	385e21409d	Changes in NaN values testing script.	2022-09-15 14:16:58 +00:00
Primoz	18002f59e1	Doryab bluetooth and locations features fill in NaN values.	2022-09-15 10:48:59 +00:00
Primoz	3cf7ca41aa	Merge branch 'imputation_and_cleaning' of https://repo.ijs.si/junoslukan/rapids into imputation_and_cleaning	2022-09-14 15:38:32 +00:00
Primoz	d5ab5a0394	Writing testing scripts to determine the point of manual imputation.	2022-09-14 14:13:03 +00:00
Primoz	dfbb758902	Changes in AutoML params and environment.yml	2022-09-13 13:54:06 +00:00
Primoz	4ec371ed96	Testing auto-sklearn	2022-09-13 09:51:03 +00:00
Primoz	d27a4a71c8	Reorganisation and reordering of the cleaning script.	2022-09-12 13:44:17 +00:00
Primoz	15d792089d	Changes in cleaning script: - target extracted from config to remove rows where target is nan - prepared sns.heatmap for further missing values analysis - necessary changes in config and participant p01 - picture of heatmap which shows the values state after cleaning	2022-09-01 10:33:36 +00:00
Primoz	cb351e0ff6	Unnecessary line (rows with no target value will be removed in cleaning script).	2022-09-01 10:06:57 +00:00
Primoz	86299d346b	Impute phone and sms NAs with 0	2022-09-01 09:57:21 +00:00
Primoz	3f7ec80c18	Preparation a) phone_calls 0 imputation b) remove rows with NaN target	2022-08-31 10:18:50 +00:00