Migrated testing from Travis to Github Actions and updated config and Snakemake files in the testing directory - part 1

.github/workflows/integrate.yaml

@@ -0,0 +1,33 @@
+name: RAPIDS Continuous Integration
+
+on:
+  push:
+    #branches: [ develop ]
+
+jobs:
+  test-on-latest-ubuntu:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      with:
+         fetch-depth: 0
+      - uses: s-weigand/setup-conda@v1
+      - run : conda --version
+      - run : source activate base
+      - run : |
+          sudo apt update
+          sudo apt install libcurl4-openssl-dev
+          sudo apt install libssl-dev
+          sudo apt install libxml2-dev
+          sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9
+          sudo add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu focal-cran40/' 
+          sudo apt install r-base
+          sudo apt install pandoc
+          sudo Rscript --vanilla -e 'install.packages("rmarkdown", repos="http://cran.us.r-project.org")'
+          cd rapids
+          conda env create -f environment.yml -n test-environment
+          source activate test-environment
+          snakemake -j1 renv_install
+          R -e 'renv::settings$use.cache(FALSE)'
+          snakemake -j1 renv_restore
+          bash tests/scripts/run_tests.sh all test 

tests/Snakefile

@@ -12,20 +12,15 @@ files_to_compute = []
 if len(config["PIDS"]) == 0:
     raise ValueError("Add participants IDs to PIDS in config.yaml. Remember to create their participant files in data/external")
 
-if config["PHONE_VALID_SENSED_BINS"]["COMPUTE"] or config["PHONE_VALID_SENSED_DAYS"]["COMPUTE"]: # valid sensed bins is necessary for sensed days, so we add these files anyways if sensed days are requested
+for provider in config["PHONE_DATA_YIELD"]["PROVIDERS"].keys():
-    if len(config["PHONE_VALID_SENSED_BINS"]["PHONE_SENSORS"]) == 0:
+    if config["PHONE_DATA_YIELD"]["PROVIDERS"][provider]["COMPUTE"]:
-            raise ValueError("If you want to compute PHONE_VALID_SENSED_BINS or PHONE_VALID_SENSED_DAYS, you need to add at least one PHONE_SENSOR to [PHONE_VALID_SENSED_BINS][PHONE_SENSORS] in config.yaml")
+        files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=map(str.lower, config["PHONE_DATA_YIELD"]["SENSORS"])))
-
+        files_to_compute.extend(expand("data/interim/{pid}/phone_yielded_timestamps.csv", pid=config["PIDS"]))
-    files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=map(str.lower, config["PHONE_VALID_SENSED_BINS"]["PHONE_SENSORS"])))
+        files_to_compute.extend(expand("data/interim/{pid}/phone_yielded_timestamps_with_datetime.csv", pid=config["PIDS"]))
-    files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime.csv", pid=config["PIDS"], sensor=map(str.lower, config["PHONE_VALID_SENSED_BINS"]["PHONE_SENSORS"])))
+        files_to_compute.extend(expand("data/interim/{pid}/phone_data_yield_features/phone_data_yield_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_DATA_YIELD"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower()))
-    files_to_compute.extend(expand("data/interim/{pid}/phone_sensed_bins.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/processed/features/{pid}/phone_data_yield.csv", pid=config["PIDS"]))
-    files_to_compute.extend(expand("data/interim/{pid}/phone_sensed_timestamps.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
-
+        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
-if config["PHONE_VALID_SENSED_DAYS"]["COMPUTE"]:
-    files_to_compute.extend(expand("data/interim/{pid}/phone_valid_sensed_days_{min_valid_hours_per_day}hours_{min_valid_bins_per_hour}bins.csv",
-                                pid=config["PIDS"],
-                                min_valid_hours_per_day=config["PHONE_VALID_SENSED_DAYS"]["MIN_VALID_HOURS_PER_DAY"],
-                                min_valid_bins_per_hour=config["PHONE_VALID_SENSED_DAYS"]["MIN_VALID_BINS_PER_HOUR"]))
 
 for provider in config["PHONE_MESSAGES"]["PROVIDERS"].keys():
     if config["PHONE_MESSAGES"]["PROVIDERS"][provider]["COMPUTE"]:
@@ -33,6 +28,8 @@ for provider in config["PHONE_MESSAGES"]["PROVIDERS"].keys():
         files_to_compute.extend(expand("data/raw/{pid}/phone_messages_with_datetime.csv", pid=config["PIDS"]))
         files_to_compute.extend(expand("data/interim/{pid}/phone_messages_features/phone_messages_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_MESSAGES"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower()))
         files_to_compute.extend(expand("data/processed/features/{pid}/phone_messages.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
+        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
 
 for provider in config["PHONE_CALLS"]["PROVIDERS"].keys():
     if config["PHONE_CALLS"]["PROVIDERS"][provider]["COMPUTE"]:
@@ -41,6 +38,8 @@ for provider in config["PHONE_CALLS"]["PROVIDERS"].keys():
         files_to_compute.extend(expand("data/raw/{pid}/phone_calls_with_datetime_unified.csv", pid=config["PIDS"]))
         files_to_compute.extend(expand("data/interim/{pid}/phone_calls_features/phone_calls_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_CALLS"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower()))
         files_to_compute.extend(expand("data/processed/features/{pid}/phone_calls.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
+        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
 
 for provider in config["PHONE_BLUETOOTH"]["PROVIDERS"].keys():
     if config["PHONE_BLUETOOTH"]["PROVIDERS"][provider]["COMPUTE"]:
@@ -48,6 +47,8 @@ for provider in config["PHONE_BLUETOOTH"]["PROVIDERS"].keys():
         files_to_compute.extend(expand("data/raw/{pid}/phone_bluetooth_with_datetime.csv", pid=config["PIDS"]))
         files_to_compute.extend(expand("data/interim/{pid}/phone_bluetooth_features/phone_bluetooth_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_BLUETOOTH"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower()))
         files_to_compute.extend(expand("data/processed/features/{pid}/phone_bluetooth.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
+        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
 
 for provider in config["PHONE_ACTIVITY_RECOGNITION"]["PROVIDERS"].keys():
     if config["PHONE_ACTIVITY_RECOGNITION"]["PROVIDERS"][provider]["COMPUTE"]:
@@ -59,7 +60,8 @@ for provider in config["PHONE_ACTIVITY_RECOGNITION"]["PROVIDERS"].keys():
         files_to_compute.extend(expand("data/interim/{pid}/phone_activity_recognition_episodes_resampled_with_datetime.csv", pid=config["PIDS"]))
         files_to_compute.extend(expand("data/interim/{pid}/phone_activity_recognition_features/phone_activity_recognition_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_ACTIVITY_RECOGNITION"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower()))
         files_to_compute.extend(expand("data/processed/features/{pid}/phone_activity_recognition.csv", pid=config["PIDS"]))
-
+        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
+        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
 
 for provider in config["PHONE_BATTERY"]["PROVIDERS"].keys():
     if config["PHONE_BATTERY"]["PROVIDERS"][provider]["COMPUTE"]:
@@ -69,14 +71,15 @@ for provider in config["PHONE_BATTERY"]["PROVIDERS"].keys():
         files_to_compute.extend(expand("data/interim/{pid}/phone_battery_episodes_resampled_with_datetime.csv", pid=config["PIDS"]))
         files_to_compute.extend(expand("data/interim/{pid}/phone_battery_features/phone_battery_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_BATTERY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower()))
         files_to_compute.extend(expand("data/processed/features/{pid}/phone_battery.csv", pid=config["PIDS"]))
-
+        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
+        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
 
 for provider in config["PHONE_SCREEN"]["PROVIDERS"].keys():
     if config["PHONE_SCREEN"]["PROVIDERS"][provider]["COMPUTE"]:
-        if "PHONE_SCREEN" in config["PHONE_VALID_SENSED_BINS"]["PHONE_SENSORS"]:
+        # if "PHONE_SCREEN" in config["PHONE_DATA_YIELD"]["SENSORS"]:# not used for now because we took episodepersensedminutes out of the list of supported features
-            files_to_compute.extend(expand("data/interim/{pid}/phone_sensed_bins.csv", pid=config["PIDS"]))
+        #     files_to_compute.extend(expand("data/interim/{pid}/phone_yielded_timestamps.csv", pid=config["PIDS"]))
-        else:
+        # else:
-            raise ValueError("Error: Add PHONE_SCREEN (and as many phone sensor as you have in your database) to [PHONE_VALID_SENSED_BINS][PHONE_SENSORS] in config.yaml. This is necessary to compute phone_sensed_bins (bins of time when the smartphone was sensing data)")
+        #     raise ValueError("Error: Add PHONE_SCREEN (and as many PHONE_SENSORS as you have in your database) to [PHONE_DATA_YIELD][SENSORS] in config.yaml. This is necessary to compute phone_yielded_timestamps (time when the smartphone was sensing data)")
         files_to_compute.extend(expand("data/raw/{pid}/phone_screen_raw.csv", pid=config["PIDS"]))
         files_to_compute.extend(expand("data/raw/{pid}/phone_screen_with_datetime.csv", pid=config["PIDS"]))
         files_to_compute.extend(expand("data/raw/{pid}/phone_screen_with_datetime_unified.csv", pid=config["PIDS"]))
@@ -85,6 +88,8 @@ for provider in config["PHONE_SCREEN"]["PROVIDERS"].keys():
         files_to_compute.extend(expand("data/interim/{pid}/phone_screen_episodes_resampled_with_datetime.csv", pid=config["PIDS"]))
         files_to_compute.extend(expand("data/interim/{pid}/phone_screen_features/phone_screen_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_SCREEN"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower()))
         files_to_compute.extend(expand("data/processed/features/{pid}/phone_screen.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
+        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
 
 for provider in config["PHONE_LIGHT"]["PROVIDERS"].keys():
     if config["PHONE_LIGHT"]["PROVIDERS"][provider]["COMPUTE"]:
@@ -92,6 +97,8 @@ for provider in config["PHONE_LIGHT"]["PROVIDERS"].keys():
         files_to_compute.extend(expand("data/raw/{pid}/phone_light_with_datetime.csv", pid=config["PIDS"]))
         files_to_compute.extend(expand("data/interim/{pid}/phone_light_features/phone_light_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_LIGHT"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower()))
         files_to_compute.extend(expand("data/processed/features/{pid}/phone_light.csv", pid=config["PIDS"],))
+        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
+        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
 
 for provider in config["PHONE_ACCELEROMETER"]["PROVIDERS"].keys():
     if config["PHONE_ACCELEROMETER"]["PROVIDERS"][provider]["COMPUTE"]:
@@ -99,6 +106,8 @@ for provider in config["PHONE_ACCELEROMETER"]["PROVIDERS"].keys():
         files_to_compute.extend(expand("data/raw/{pid}/phone_accelerometer_with_datetime.csv", pid=config["PIDS"]))
         files_to_compute.extend(expand("data/interim/{pid}/phone_accelerometer_features/phone_accelerometer_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_ACCELEROMETER"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower()))
         files_to_compute.extend(expand("data/processed/features/{pid}/phone_accelerometer.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
+        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
 
 for provider in config["PHONE_APPLICATIONS_FOREGROUND"]["PROVIDERS"].keys():
     if config["PHONE_APPLICATIONS_FOREGROUND"]["PROVIDERS"][provider]["COMPUTE"]:
@@ -107,6 +116,8 @@ for provider in config["PHONE_APPLICATIONS_FOREGROUND"]["PROVIDERS"].keys():
         files_to_compute.extend(expand("data/raw/{pid}/phone_applications_foreground_with_datetime_with_categories.csv", pid=config["PIDS"]))
         files_to_compute.extend(expand("data/interim/{pid}/phone_applications_foreground_features/phone_applications_foreground_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_APPLICATIONS_FOREGROUND"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower()))
         files_to_compute.extend(expand("data/processed/features/{pid}/phone_applications_foreground.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
+        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
 
 for provider in config["PHONE_WIFI_VISIBLE"]["PROVIDERS"].keys():
     if config["PHONE_WIFI_VISIBLE"]["PROVIDERS"][provider]["COMPUTE"]:
@@ -114,6 +125,8 @@ for provider in config["PHONE_WIFI_VISIBLE"]["PROVIDERS"].keys():
         files_to_compute.extend(expand("data/raw/{pid}/phone_wifi_visible_with_datetime.csv", pid=config["PIDS"]))
         files_to_compute.extend(expand("data/interim/{pid}/phone_wifi_visible_features/phone_wifi_visible_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_WIFI_VISIBLE"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower()))
         files_to_compute.extend(expand("data/processed/features/{pid}/phone_wifi_visible.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
+        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
 
 for provider in config["PHONE_WIFI_CONNECTED"]["PROVIDERS"].keys():
     if config["PHONE_WIFI_CONNECTED"]["PROVIDERS"][provider]["COMPUTE"]:
@@ -121,6 +134,8 @@ for provider in config["PHONE_WIFI_CONNECTED"]["PROVIDERS"].keys():
         files_to_compute.extend(expand("data/raw/{pid}/phone_wifi_connected_with_datetime.csv", pid=config["PIDS"]))
         files_to_compute.extend(expand("data/interim/{pid}/phone_wifi_connected_features/phone_wifi_connected_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_WIFI_CONNECTED"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower()))
         files_to_compute.extend(expand("data/processed/features/{pid}/phone_wifi_connected.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
+        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
 
 for provider in config["PHONE_CONVERSATION"]["PROVIDERS"].keys():    
     if config["PHONE_CONVERSATION"]["PROVIDERS"][provider]["COMPUTE"]:
@@ -129,75 +144,106 @@ for provider in config["PHONE_CONVERSATION"]["PROVIDERS"].keys():
         files_to_compute.extend(expand("data/raw/{pid}/phone_conversation_with_datetime_unified.csv", pid=config["PIDS"]))
         files_to_compute.extend(expand("data/interim/{pid}/phone_conversation_features/phone_conversation_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_CONVERSATION"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower()))
         files_to_compute.extend(expand("data/processed/features/{pid}/phone_conversation.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
+        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
 
 for provider in config["PHONE_LOCATIONS"]["PROVIDERS"].keys():
     if config["PHONE_LOCATIONS"]["PROVIDERS"][provider]["COMPUTE"]:
-        if config["PHONE_LOCATIONS"]["LOCATIONS_TO_USE"] == "RESAMPLE_FUSED":
+        if config["PHONE_LOCATIONS"]["LOCATIONS_TO_USE"] == "FUSED_RESAMPLED":
-            if "PHONE_LOCATIONS" in config["PHONE_VALID_SENSED_BINS"]["PHONE_SENSORS"]:
+            if "PHONE_LOCATIONS" in config["PHONE_DATA_YIELD"]["SENSORS"]:
-                files_to_compute.extend(expand("data/interim/{pid}/phone_sensed_bins.csv", pid=config["PIDS"]))
+                files_to_compute.extend(expand("data/interim/{pid}/phone_yielded_timestamps.csv", pid=config["PIDS"]))
             else:
-                raise ValueError("Error: Add PHONE_LOCATIONS (and as many PHONE_SENSORS as you have) to [PHONE_VALID_SENSED_BINS][PHONE_SENSORS] in config.yaml. This is necessary to compute phone_sensed_bins (bins of time when the smartphone was sensing data) which is used to resample fused location data (RESAMPLED_FUSED)")
+                raise ValueError("Error: Add PHONE_LOCATIONS (and as many PHONE_SENSORS as you have) to [PHONE_DATA_YIELD][SENSORS] in config.yaml. This is necessary to compute phone_yielded_timestamps (time when the smartphone was sensing data) which is used to resample fused location data (RESAMPLED_FUSED)")
 
         files_to_compute.extend(expand("data/raw/{pid}/phone_locations_raw.csv", pid=config["PIDS"]))
         files_to_compute.extend(expand("data/interim/{pid}/phone_locations_processed.csv", pid=config["PIDS"]))
         files_to_compute.extend(expand("data/interim/{pid}/phone_locations_processed_with_datetime.csv", pid=config["PIDS"]))
         files_to_compute.extend(expand("data/interim/{pid}/phone_locations_features/phone_locations_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_LOCATIONS"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower()))
         files_to_compute.extend(expand("data/processed/features/{pid}/phone_locations.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
+        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
 
-if config["FITBIT_HEARTRATE"]["TABLE_FORMAT"] not in ["JSON", "CSV"]:
+for provider in config["FITBIT_HEARTRATE_SUMMARY"]["PROVIDERS"].keys():
-    raise ValueError("config['FITBIT_HEARTRATE']['TABLE_FORMAT'] should be JSON or CSV but you typed" + config["FITBIT_HEARTRATE"]["TABLE_FORMAT"])
+    if config["FITBIT_HEARTRATE_SUMMARY"]["PROVIDERS"][provider]["COMPUTE"]:
+        files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_summary_raw.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_summary_parsed.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_summary_parsed_with_datetime.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/interim/{pid}/fitbit_heartrate_summary_features/fitbit_heartrate_summary_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["FITBIT_HEARTRATE_SUMMARY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower()))
+        files_to_compute.extend(expand("data/processed/features/{pid}/fitbit_heartrate_summary.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
+        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
 
-if config["FITBIT_STEPS"]["TABLE_FORMAT"] not in ["JSON", "CSV"]:
+for provider in config["FITBIT_HEARTRATE_INTRADAY"]["PROVIDERS"].keys():
-    raise ValueError("config['FITBIT_STEPS']['TABLE_FORMAT'] should be JSON or CSV but you typed" + config["FITBIT_STEPS"]["TABLE_FORMAT"])
+    if config["FITBIT_HEARTRATE_INTRADAY"]["PROVIDERS"][provider]["COMPUTE"]:
+        files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_intraday_raw.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_intraday_parsed.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_intraday_parsed_with_datetime.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/interim/{pid}/fitbit_heartrate_intraday_features/fitbit_heartrate_intraday_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["FITBIT_HEARTRATE_INTRADAY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower()))
+        files_to_compute.extend(expand("data/processed/features/{pid}/fitbit_heartrate_intraday.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
+        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
 
-if config["FITBIT_CALORIES"]["TABLE_FORMAT"] not in ["JSON", "CSV"]:
+for provider in config["FITBIT_SLEEP_SUMMARY"]["PROVIDERS"].keys():
-    raise ValueError("config['FITBIT_CALORIES']['TABLE_FORMAT'] should be JSON or CSV but you typed" + config["FITBIT_CALORIES"]["TABLE_FORMAT"])
+    if config["FITBIT_SLEEP_SUMMARY"]["PROVIDERS"][provider]["COMPUTE"]:
+        files_to_compute.extend(expand("data/raw/{pid}/fitbit_sleep_summary_raw.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/raw/{pid}/fitbit_sleep_summary_parsed.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/raw/{pid}/fitbit_sleep_summary_parsed_with_datetime.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/interim/{pid}/fitbit_sleep_summary_features/fitbit_sleep_summary_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["FITBIT_SLEEP_SUMMARY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower()))
+        files_to_compute.extend(expand("data/processed/features/{pid}/fitbit_sleep_summary.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
+        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
 
-if config["FITBIT_SLEEP"]["TABLE_FORMAT"] not in ["JSON", "CSV"]:
+# for provider in config["FITBIT_SLEEP_INTRADAY"]["PROVIDERS"].keys():
-    raise ValueError("config['FITBIT_SLEEP']['TABLE_FORMAT'] should be JSON or CSV but you typed" + config["FITBIT_SLEEP"]["TABLE_FORMAT"])
+#     if config["FITBIT_SLEEP_INTRADAY"]["PROVIDERS"][provider]["COMPUTE"]:
+#         files_to_compute.extend(expand("data/raw/{pid}/fitbit_sleep_intraday_raw.csv", pid=config["PIDS"]))
+#         files_to_compute.extend(expand("data/raw/{pid}/fitbit_sleep_intraday_parsed.csv", pid=config["PIDS"]))
+#         files_to_compute.extend(expand("data/raw/{pid}/fitbit_sleep_intraday_parsed_with_datetime.csv", pid=config["PIDS"]))
 
-for provider in config["FITBIT_HEARTRATE"]["PROVIDERS"].keys():
+for provider in config["FITBIT_STEPS_SUMMARY"]["PROVIDERS"].keys():
-    if config["FITBIT_HEARTRATE"]["PROVIDERS"][provider]["COMPUTE"]:
+    if config["FITBIT_STEPS_SUMMARY"]["PROVIDERS"][provider]["COMPUTE"]:
-        files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_{fitbit_data_type}_raw.csv", pid=config["PIDS"], fitbit_data_type=(["json"] if config["FITBIT_HEARTRATE"]["TABLE_FORMAT"] == "JSON" else ["summary", "intraday"])))
+        files_to_compute.extend(expand("data/raw/{pid}/fitbit_steps_summary_raw.csv", pid=config["PIDS"]))
-        files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_{fitbit_data_type}_parsed.csv", pid=config["PIDS"], fitbit_data_type=["summary", "intraday"]))
+        files_to_compute.extend(expand("data/raw/{pid}/fitbit_steps_summary_parsed.csv", pid=config["PIDS"]))
-        files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_{fitbit_data_type}_parsed_with_datetime.csv", pid=config["PIDS"], fitbit_data_type=["summary", "intraday"]))
+        files_to_compute.extend(expand("data/raw/{pid}/fitbit_steps_summary_parsed_with_datetime.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/interim/{pid}/fitbit_steps_summary_features/fitbit_steps_summary_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["FITBIT_STEPS_SUMMARY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower()))
+        files_to_compute.extend(expand("data/processed/features/{pid}/fitbit_steps_summary.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
+        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
 
-for provider in config["FITBIT_STEPS"]["PROVIDERS"].keys():
+for provider in config["FITBIT_STEPS_INTRADAY"]["PROVIDERS"].keys():
-    if config["FITBIT_STEPS"]["PROVIDERS"][provider]["COMPUTE"]:
+    if config["FITBIT_STEPS_INTRADAY"]["PROVIDERS"][provider]["COMPUTE"]:
-        files_to_compute.extend(expand("data/raw/{pid}/fitbit_steps_{fitbit_data_type}_raw.csv", pid=config["PIDS"], fitbit_data_type=(["json"] if config["FITBIT_STEPS"]["TABLE_FORMAT"] == "JSON" else ["summary", "intraday"])))
+        files_to_compute.extend(expand("data/raw/{pid}/fitbit_steps_intraday_raw.csv", pid=config["PIDS"]))
-        files_to_compute.extend(expand("data/raw/{pid}/fitbit_steps_{fitbit_data_type}_parsed.csv", pid=config["PIDS"], fitbit_data_type=["summary", "intraday"]))
+        files_to_compute.extend(expand("data/raw/{pid}/fitbit_steps_intraday_parsed.csv", pid=config["PIDS"]))
-        files_to_compute.extend(expand("data/raw/{pid}/fitbit_steps_{fitbit_data_type}_parsed_with_datetime.csv", pid=config["PIDS"], fitbit_data_type=["summary", "intraday"]))
+        files_to_compute.extend(expand("data/raw/{pid}/fitbit_steps_intraday_parsed_with_datetime.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/interim/{pid}/fitbit_steps_intraday_features/fitbit_steps_intraday_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["FITBIT_STEPS_INTRADAY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower()))
+        files_to_compute.extend(expand("data/processed/features/{pid}/fitbit_steps_intraday.csv", pid=config["PIDS"]))
+        files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
+        files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
 
-for provider in config["FITBIT_CALORIES"]["PROVIDERS"].keys():
+# for provider in config["FITBIT_CALORIES"]["PROVIDERS"].keys():
-    if config["FITBIT_CALORIES"]["PROVIDERS"][provider]["COMPUTE"]:
+#     if config["FITBIT_CALORIES"]["PROVIDERS"][provider]["COMPUTE"]:
-        files_to_compute.extend(expand("data/raw/{pid}/fitbit_calories_{fitbit_data_type}_raw.csv", pid=config["PIDS"], fitbit_data_type=(["json"] if config["FITBIT_CALORIES"]["TABLE_FORMAT"] == "JSON" else ["summary", "intraday"])))
+#         files_to_compute.extend(expand("data/raw/{pid}/fitbit_calories_{fitbit_data_type}_raw.csv", pid=config["PIDS"], fitbit_data_type=(["json"] if config["FITBIT_CALORIES"]["TABLE_FORMAT"] == "JSON" else ["summary", "intraday"])))
-        files_to_compute.extend(expand("data/raw/{pid}/fitbit_calories_{fitbit_data_type}_parsed.csv", pid=config["PIDS"], fitbit_data_type=["summary", "intraday"]))
+#         files_to_compute.extend(expand("data/raw/{pid}/fitbit_calories_{fitbit_data_type}_parsed.csv", pid=config["PIDS"], fitbit_data_type=["summary", "intraday"]))
-        files_to_compute.extend(expand("data/raw/{pid}/fitbit_calories_{fitbit_data_type}_parsed_with_datetime.csv", pid=config["PIDS"], fitbit_data_type=["summary", "intraday"]))
+#         files_to_compute.extend(expand("data/raw/{pid}/fitbit_calories_{fitbit_data_type}_parsed_with_datetime.csv", pid=config["PIDS"], fitbit_data_type=["summary", "intraday"]))
+#         files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
+#         files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv")
 
-for provider in config["FITBIT_SLEEP"]["PROVIDERS"].keys():
+# Visualization for Data Exploration
-    if config["FITBIT_SLEEP"]["PROVIDERS"][provider]["COMPUTE"]:
+if config["HISTOGRAM_PHONE_DATA_YIELD"]["PLOT"]:
-        files_to_compute.extend(expand("data/raw/{pid}/fitbit_sleep_{fitbit_data_type}_raw.csv", pid=config["PIDS"], fitbit_data_type=(["json"] if config["FITBIT_SLEEP"]["TABLE_FORMAT"] == "JSON" else ["summary", "intraday"])))
+    files_to_compute.append("reports/data_exploration/histogram_phone_data_yield.html")
-        files_to_compute.extend(expand("data/raw/{pid}/fitbit_sleep_{fitbit_data_type}_parsed_episodes.csv", pid=config["PIDS"], fitbit_data_type=["summary"]))
-        files_to_compute.extend(expand("data/raw/{pid}/fitbit_sleep_{fitbit_data_type}_parsed.csv", pid=config["PIDS"], fitbit_data_type=["intraday"]))
-        files_to_compute.extend(expand("data/raw/{pid}/fitbit_sleep_{fitbit_data_type}_parsed_with_datetime.csv", pid=config["PIDS"], fitbit_data_type=["intraday"]))
 
-# visualization for data exploration
+if config["HEATMAP_SENSORS_PER_MINUTE_PER_TIME_SEGMENT"]["PLOT"]:
-if config["HEATMAP_FEATURES_CORRELATIONS"]["PLOT"]:
+    files_to_compute.extend(expand("reports/interim/{pid}/heatmap_sensors_per_minute_per_time_segment.html", pid=config["PIDS"]))
-    files_to_compute.extend(expand("reports/data_exploration/{min_valid_hours_per_day}hours_{min_valid_bins_per_hour}bins/heatmap_features_correlations.html", min_valid_hours_per_day=config["HEATMAP_FEATURES_CORRELATIONS"]["MIN_VALID_HOURS_PER_DAY"], min_valid_bins_per_hour=config["PHONE_VALID_SENSED_DAYS"]["MIN_VALID_BINS_PER_HOUR"]))
+    files_to_compute.append("reports/data_exploration/heatmap_sensors_per_minute_per_time_segment.html")
-    
-if config["HISTOGRAM_VALID_SENSED_HOURS"]["PLOT"]:
-    files_to_compute.extend(expand("reports/data_exploration/{min_valid_hours_per_day}hours_{min_valid_bins_per_hour}bins/histogram_valid_sensed_hours.html", min_valid_hours_per_day=config["HISTOGRAM_VALID_SENSED_HOURS"]["MIN_VALID_HOURS_PER_DAY"], min_valid_bins_per_hour=config["PHONE_VALID_SENSED_DAYS"]["MIN_VALID_BINS_PER_HOUR"]))
 
-if config["HEATMAP_DAYS_BY_SENSORS"]["PLOT"]:
+if config["HEATMAP_SENSOR_ROW_COUNT_PER_TIME_SEGMENT"]["PLOT"]:
-    files_to_compute.extend(expand("reports/interim/{min_valid_hours_per_day}hours_{min_valid_bins_per_hour}bins/{pid}/heatmap_days_by_sensors.html", pid=config["PIDS"], min_valid_hours_per_day=config["HEATMAP_DAYS_BY_SENSORS"]["MIN_VALID_HOURS_PER_DAY"], min_valid_bins_per_hour=config["PHONE_VALID_SENSED_DAYS"]["MIN_VALID_BINS_PER_HOUR"]))
+    files_to_compute.extend(expand("reports/interim/{pid}/heatmap_sensor_row_count_per_time_segment.html", pid=config["PIDS"]))
-    files_to_compute.extend(expand("reports/data_exploration/{min_valid_hours_per_day}hours_{min_valid_bins_per_hour}bins/heatmap_days_by_sensors_all_participants.html", min_valid_hours_per_day=config["HEATMAP_DAYS_BY_SENSORS"]["MIN_VALID_HOURS_PER_DAY"], min_valid_bins_per_hour=config["PHONE_VALID_SENSED_DAYS"]["MIN_VALID_BINS_PER_HOUR"]))
+    files_to_compute.append("reports/data_exploration/heatmap_sensor_row_count_per_time_segment.html")
 
-if config["HEATMAP_SENSED_BINS"]["PLOT"]:
+if config["HEATMAP_PHONE_DATA_YIELD_PER_PARTICIPANT_PER_TIME_SEGMENT"]["PLOT"]:
-    files_to_compute.extend(expand("reports/interim/heatmap_sensed_bins/{pid}/heatmap_sensed_bins.html", pid=config["PIDS"]))
+    files_to_compute.append("reports/data_exploration/heatmap_phone_data_yield_per_participant_per_time_segment.html")
-    files_to_compute.extend(["reports/data_exploration/heatmap_sensed_bins_all_participants.html"])
 
-if config["OVERALL_COMPLIANCE_HEATMAP"]["PLOT"]:
+if config["HEATMAP_FEATURE_CORRELATION_MATRIX"]["PLOT"]:
-    files_to_compute.extend(expand("reports/data_exploration/{min_valid_hours_per_day}hours_{min_valid_bins_per_hour}bins/overall_compliance_heatmap.html", min_valid_hours_per_day=config["OVERALL_COMPLIANCE_HEATMAP"]["MIN_VALID_HOURS_PER_DAY"], min_valid_bins_per_hour=config["PHONE_VALID_SENSED_DAYS"]["MIN_VALID_BINS_PER_HOUR"]))
+    files_to_compute.append("reports/data_exploration/heatmap_feature_correlation_matrix.html")
 
 
 rule all:

tests/data/processed/features/frequency/test03/phone_bluetooth.csv

@@ -1 +1 @@
-"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","bluetooth_rapids_countscans","bluetooth_rapids_uniquedevices","bluetooth_rapids_countscansmostuniquedevice"
+"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","phone_bluetooth_rapids_countscans","phone_bluetooth_rapids_uniquedevices","phone_bluetooth_rapids_countscansmostuniquedevice"

tests/data/processed/features/frequency/test04/phone_bluetooth.csv

@@ -1 +1 @@
-"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","bluetooth_rapids_countscans","bluetooth_rapids_uniquedevices","bluetooth_rapids_countscansmostuniquedevice"
+"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","phone_bluetooth_rapids_countscans","phone_bluetooth_rapids_uniquedevices","phone_bluetooth_rapids_countscansmostuniquedevice"

tests/data/processed/features/frequency/test05/phone_bluetooth.csv

@@ -1,4 +1,4 @@
-"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","bluetooth_rapids_countscans","bluetooth_rapids_uniquedevices","bluetooth_rapids_countscansmostuniquedevice"
+"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","phone_bluetooth_rapids_countscans","phone_bluetooth_rapids_uniquedevices","phone_bluetooth_rapids_countscansmostuniquedevice"
 "thirtyminutes0000#2020-07-02 00:00:00,2020-07-02 00:29:59","thirtyminutes0000","2020-07-02 00:00:00","2020-07-02 00:29:59",1,1,1
 "thirtyminutes0001#2020-07-02 00:30:00,2020-07-02 00:59:59","thirtyminutes0001","2020-07-02 00:30:00","2020-07-02 00:59:59",1,1,1
 "thirtyminutes0007#2020-07-02 03:30:00,2020-07-02 03:59:59","thirtyminutes0007","2020-07-02 03:30:00","2020-07-02 03:59:59",1,1,1

tests/data/processed/features/frequency/test06/phone_bluetooth.csv

@@ -1,4 +1,4 @@
-"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","bluetooth_rapids_countscans","bluetooth_rapids_uniquedevices","bluetooth_rapids_countscansmostuniquedevice"
+"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","phone_bluetooth_rapids_countscans","phone_bluetooth_rapids_uniquedevices","phone_bluetooth_rapids_countscansmostuniquedevice"
 "thirtyminutes0000#2020-07-02 00:00:00,2020-07-02 00:29:59","thirtyminutes0000","2020-07-02 00:00:00","2020-07-02 00:29:59",1,1,1
 "thirtyminutes0001#2020-07-02 00:30:00,2020-07-02 00:59:59","thirtyminutes0001","2020-07-02 00:30:00","2020-07-02 00:59:59",1,1,1
 "thirtyminutes0007#2020-07-02 03:30:00,2020-07-02 03:59:59","thirtyminutes0007","2020-07-02 03:30:00","2020-07-02 03:59:59",1,1,1

tests/data/processed/features/periodic/test01/phone_bluetooth.csv

@@ -1,4 +1,4 @@
-"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","bluetooth_rapids_countscans","bluetooth_rapids_uniquedevices","bluetooth_rapids_countscansmostuniquedevice"
+"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","phone_bluetooth_rapids_countscans","phone_bluetooth_rapids_uniquedevices","phone_bluetooth_rapids_countscansmostuniquedevice"
 "afternoon#2020-07-02 12:00:00,2020-07-02 17:59:59","afternoon","2020-07-02 12:00:00","2020-07-02 17:59:59",2,2,1
 "daily#2020-07-02 00:00:00,2020-07-02 23:59:59","daily","2020-07-02 00:00:00","2020-07-02 23:59:59",14,5,4
 "evening#2020-07-02 18:00:00,2020-07-02 23:59:59","evening","2020-07-02 18:00:00","2020-07-02 23:59:59",5,4,2

tests/data/processed/features/periodic/test02/phone_bluetooth.csv

@@ -1,4 +1,4 @@
-"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","bluetooth_rapids_countscans","bluetooth_rapids_uniquedevices","bluetooth_rapids_countscansmostuniquedevice"
+"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","phone_bluetooth_rapids_countscans","phone_bluetooth_rapids_uniquedevices","phone_bluetooth_rapids_countscansmostuniquedevice"
 "afternoon#2020-07-02 12:00:00,2020-07-02 17:59:59","afternoon","2020-07-02 12:00:00","2020-07-02 17:59:59",2,2,1
 "daily#2020-07-02 00:00:00,2020-07-02 23:59:59","daily","2020-07-02 00:00:00","2020-07-02 23:59:59",14,5,5
 "evening#2020-07-02 18:00:00,2020-07-02 23:59:59","evening","2020-07-02 18:00:00","2020-07-02 23:59:59",5,3,3

tests/data/processed/features/periodic/test03/phone_bluetooth.csv

@@ -1 +1 @@
-"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","bluetooth_rapids_countscans","bluetooth_rapids_uniquedevices","bluetooth_rapids_countscansmostuniquedevice"
+"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","phone_bluetooth_rapids_countscans","phone_bluetooth_rapids_uniquedevices","phone_bluetooth_rapids_countscansmostuniquedevice"

tests/data/processed/features/periodic/test04/phone_bluetooth.csv

@@ -1 +1 @@
-"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","bluetooth_rapids_countscans","bluetooth_rapids_uniquedevices","bluetooth_rapids_countscansmostuniquedevice"
+"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","phone_bluetooth_rapids_countscans","phone_bluetooth_rapids_uniquedevices","phone_bluetooth_rapids_countscansmostuniquedevice"

tests/scripts/run_tests.sh

@@ -64,7 +64,7 @@ cp -r tests/data/raw/* data/raw
 cp -r tests/data/external/* data/external
 
 echo Disabling downloading of dataset...
-sed -e '27,53 s/^/#/' -e  's/rules.download_dataset.output/"data\/raw\/\{pid\}\/\{sensor\}_raw\.csv"/' rules/preprocessing.smk > tmp
+sed -e '26,51 s/^/#/' -e  's/rules.download_dataset.output/"data\/raw\/\{pid\}\/\{sensor\}_raw\.csv"/' rules/preprocessing.smk > tmp
 mv tmp rules/preprocessing.smk
 
 if [ $# -eq 1 ]

tests/settings/frequency/testing_config.yaml

@@ -1,85 +1,139 @@
-# Participants to include in the analysis
+# See https://www.rapids.science/latest/setup/configuration/#database-credentials
-# You must create a file for each participant named pXXX containing their device_id. This can be done manually or automatically
-PIDS: [test03, test04,test05, test06]
-
-# Global var with common time segments
-TIME_SEGMENTS: &time_segments
-  TYPE: FREQUENCY # FREQUENCY, PERIODIC, EVENT
-  FILE: "data/external/timesegments_frequency.csv"
-  INCLUDE_PAST_PERIODIC_SEGMENTS: FALSE # Only relevant if TYPE=PERIODIC, if set to TRUE we consider time segments back enough in the past as to include the first day of data
-
-# Use tz codes from https://en.wikipedia.org/wiki/List_of_tz_database_time_zones. Double check your code, for example EST is not US Eastern Time.
-TIMEZONE: &timezone
-  America/New_York
-
 DATABASE_GROUP: &database_group
   MY_GROUP
 
-# config section for the script that creates participant files automatically
+# See https://www.rapids.science/latest/setup/configuration/#timezone-of-your-study
-PARTICIPANT_FILES: # run snakemake -j1 -R parse_participant_files
+TIMEZONE: &timezone
+  America/New_York
+
+# See https://www.rapids.science/latest/setup/configuration/#participant-files
+PIDS: [test03, test04,test05, test06]
+
+# See https://www.rapids.science/latest/setup/configuration/#automatic-creation-of-participant-files
+CREATE_PARTICIPANT_FILES:
+  SOURCE:
+    TYPE: AWARE_DEVICE_TABLE #AWARE_DEVICE_TABLE or CSV_FILE
+    DATABASE_GROUP: *database_group
+    CSV_FILE_PATH: "data/external/example_participants.csv" # see docs for required format
+    TIMEZONE: *timezone
   PHONE_SECTION:
-    ADD: FALSE
+    ADD: TRUE
-    PARSED_FROM: AWARE_DEVICE_TABLE #AWARE_DEVICE_TABLE or CSV_FILE
+    DEVICE_ID_COLUMN: device_id # column name
-    PARSED_SOURCE: *database_group # DB credentials group or CSV file path. If CSV file, it should have: device_id, pid (optional), label (optional), start_date (optional), end_date (optional)
     IGNORED_DEVICE_IDS: []
   FITBIT_SECTION:
-    ADD: FALSE
+    ADD: TRUE
-    SAME_AS_PHONE: FALSE # If TRUE, all config below is ignored
+    DEVICE_ID_COLUMN: device_id # column name
-    PARSED_FROM: CSV_FILE
+    IGNORED_DEVICE_IDS: []
-    PARSED_SOURCE: "external/my_fitbit_participants.csv" # CSV file should have: device_id, pid (optional), label (optional), start_date (optional), end_date (optional)
 
-DEVICE_DATA:
+# See https://www.rapids.science/latest/setup/configuration/#time-segments
-  PHONE:
+TIME_SEGMENTS: &time_segments
-    SOURCE: 
+  TYPE: FREQUENCY # FREQUENCY, PERIODIC, EVENT
-      TYPE: DATABASE # Phone only supports DATABASE for now
+  FILE: "data/external/timesegments_frequency.csv"
-      DATABASE_GROUP: *database_group
+  INCLUDE_PAST_PERIODIC_SEGMENTS: FALSE # Only relevant if TYPE=PERIODIC, see docs
-      DEVICE_ID_COLUMN: device_id # column name
-    TIMEZONE: 
-      TYPE: SINGLE # SINGLE or MULTIPLE
-      VALUE: *timezone # IF TYPE=SINGLE, timezone code (e.g. America/New_York, see attribute TIMEZONE above). If TYPE=MULTIPLE, a table in your database with two columns (timestamp, timezone) where timestamp is a unix timestamp and timezone is one of https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
-  FITBIT:
-    SOURCE: 
-      TYPE: DATABASE # DATABASE or FILES (set each FITBIT_SENSOR TABLE attribute accordingly with a table name or a file path)
-      DATABASE_GROUP: *database_group
-      DEVICE_ID_COLUMN: device_id # column name
-    TIMEZONE: 
-      TYPE: SINGLE # Fitbit only supports SINGLE timezones
-      VALUE: *timezone # timezone code (e.g. America/New_York, see attribute TIMEZONE above and https://en.wikipedia.org/wiki/List_of_tz_database_time_zones)
 
-PHONE_VALID_SENSED_BINS:
-  COMPUTE: False # This flag is automatically ignored (set to True) if you are extracting PHONE_VALID_SENSED_DAYS or screen or Barnett's location features
-  BIN_SIZE: &bin_size 5 # (in minutes)
-  # Add as many PHONE sensors as you have, they all improve the computation of PHONE_VALID_SENSED_BINS and PHONE_VALID_SENSED_DAYS. 
-  # If you are extracting screen or Barnett/Doryab location features, PHONE_SCREEN and PHONE_LOCATIONS tables are mandatory.
-  # You can choose any of the keys shown below, just make sure its TABLE exists in your database!
-  # PHONE_MESSAGES, PHONE_CALLS, PHONE_LOCATIONS, PHONE_BLUETOOTH, PHONE_ACTIVITY_RECOGNITION, PHONE_BATTERY, PHONE_SCREEN, PHONE_LIGHT,
-  # PHONE_ACCELEROMETER, PHONE_APPLICATIONS_FOREGROUND, PHONE_WIFI_VISIBLE, PHONE_WIFI_CONNECTED, PHONE_CONVERSATION
-  PHONE_SENSORS: []
 
-PHONE_VALID_SENSED_DAYS:
-  COMPUTE: False
-  MIN_VALID_HOURS_PER_DAY: &min_valid_hours_per_day [16] # (out of 24) MIN_HOURS_PER_DAY
-  MIN_VALID_BINS_PER_HOUR: &min_valid_bins_per_hour [6] # (out of 60min/BIN_SIZE bins)
 
-# Communication SMS features config, TYPES and FEATURES keys need to match
+########################################################################################################################
-PHONE_MESSAGES:
+#                                                 PHONE                                                                #
-  TABLE: messages
+########################################################################################################################
+
+# See https://www.rapids.science/latest/setup/configuration/#device-data-source-configuration
+PHONE_DATA_CONFIGURATION:
+  SOURCE: 
+    TYPE: DATABASE
+    DATABASE_GROUP: *database_group
+    DEVICE_ID_COLUMN: device_id # column name
+  TIMEZONE: 
+    TYPE: SINGLE
+    VALUE: *timezone
+
+# Sensors ------
+
+# https://www.rapids.science/latest/features/phone-accelerometer/
+PHONE_ACCELEROMETER:
+  TABLE: accelerometer
   PROVIDERS:
     RAPIDS:
-      COMPUTE: TRUE
+      COMPUTE: False
-      MESSAGES_TYPES : [received, sent]
+      FEATURES: ["maxmagnitude", "minmagnitude", "avgmagnitude", "medianmagnitude", "stdmagnitude"]
-      FEATURES: 
+      SRC_FOLDER: "rapids" # inside src/features/phone_accelerometer
-        received: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact]
+      SRC_LANGUAGE: "python"
-        sent: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact]
+    
-      SRC_LANGUAGE: "r"
+    PANDA:
-      SRC_FOLDER: "rapids" # inside src/features/phone_messages
+      COMPUTE: False
+      VALID_SENSED_MINUTES: False
+      FEATURES:
+        exertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"]
+        nonexertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"]
+      SRC_FOLDER: "panda" # inside src/features/phone_accelerometer
+      SRC_LANGUAGE: "python"
 
-# Communication call features config, TYPES and FEATURES keys need to match
+# See https://www.rapids.science/latest/features/phone-activity-recognition/
+PHONE_ACTIVITY_RECOGNITION:
+  TABLE: 
+    ANDROID: plugin_google_activity_recognition
+    IOS: plugin_ios_activity_recognition
+  EPISODE_THRESHOLD_BETWEEN_ROWS: 5 # minutes. Max time difference for two consecutive rows to be considered within the same battery episode.
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: False
+      FEATURES: ["count", "mostcommonactivity", "countuniqueactivities", "durationstationary", "durationmobile", "durationvehicle"]
+      ACTIVITY_CLASSES:
+        STATIONARY: ["still", "tilting"]
+        MOBILE: ["on_foot", "walking", "running", "on_bicycle"]
+        VEHICLE: ["in_vehicle"]
+      SRC_FOLDER: "rapids" # inside src/features/phone_activity_recognition
+      SRC_LANGUAGE: "python"
+
+# See https://www.rapids.science/latest/features/phone-applications-foreground/
+PHONE_APPLICATIONS_FOREGROUND:
+  TABLE: applications_foreground
+  APPLICATION_CATEGORIES:
+    CATALOGUE_SOURCE: FILE # FILE (genres are read from CATALOGUE_FILE) or GOOGLE (genres are scrapped from the Play Store)
+    CATALOGUE_FILE: "data/external/stachl_application_genre_catalogue.csv"
+    UPDATE_CATALOGUE_FILE: False # if CATALOGUE_SOURCE is equal to FILE, whether or not to update CATALOGUE_FILE, if CATALOGUE_SOURCE is equal to GOOGLE all scraped genres will be saved to CATALOGUE_FILE
+    SCRAPE_MISSING_CATEGORIES: False # whether or not to scrape missing genres, only effective if CATALOGUE_SOURCE is equal to FILE. If CATALOGUE_SOURCE is equal to GOOGLE, all genres are scraped anyway
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: False
+      SINGLE_CATEGORIES: ["all", "email"]
+      MULTIPLE_CATEGORIES:
+        social: ["socialnetworks", "socialmediatools"]
+        entertainment: ["entertainment", "gamingknowledge", "gamingcasual", "gamingadventure", "gamingstrategy", "gamingtoolscommunity", "gamingroleplaying", "gamingaction", "gaminglogic", "gamingsports", "gamingsimulation"]
+      SINGLE_APPS: ["top1global", "com.facebook.moments", "com.google.android.youtube", "com.twitter.android"] # There's no entropy for single apps
+      EXCLUDED_CATEGORIES: []
+      EXCLUDED_APPS: ["com.fitbit.FitbitMobile", "com.aware.plugin.upmc.cancer"]
+      FEATURES: ["count", "timeoffirstuse", "timeoflastuse", "frequencyentropy"]
+      SRC_FOLDER: "rapids" # inside src/features/phone_applications_foreground
+      SRC_LANGUAGE: "python"
+
+# See https://www.rapids.science/latest/features/phone-battery/
+PHONE_BATTERY:
+  TABLE: battery
+  EPISODE_THRESHOLD_BETWEEN_ROWS: 30 # minutes. Max time difference for two consecutive rows to be considered within the same battery episode.
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: False
+      FEATURES: ["countdischarge", "sumdurationdischarge", "countcharge", "sumdurationcharge", "avgconsumptionrate", "maxconsumptionrate"]
+      SRC_FOLDER: "rapids" # inside src/features/phone_battery
+      SRC_LANGUAGE: "python"
+
+# See https://www.rapids.science/latest/features/phone-bluetooth/
+PHONE_BLUETOOTH:
+  TABLE: bluetooth
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: True
+      FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"]
+      SRC_FOLDER: "rapids" # inside src/features/phone_bluetooth
+      SRC_LANGUAGE: "r"
+
+# See https://www.rapids.science/latest/features/phone-calls/
 PHONE_CALLS:
   TABLE: calls
   PROVIDERS:
     RAPIDS:
-      COMPUTE: TRUE
+      COMPUTE: False
       CALL_TYPES: [missed, incoming, outgoing]
       FEATURES:
         missed:  [count, distinctcontacts, timefirstcall, timelastcall, countmostfrequentcontact]
@@ -88,6 +142,47 @@ PHONE_CALLS:
       SRC_LANGUAGE: "r"
       SRC_FOLDER: "rapids" # inside src/features/phone_calls
 
+# See https://www.rapids.science/latest/features/phone-conversation/
+PHONE_CONVERSATION:
+  TABLE: 
+    ANDROID: plugin_studentlife_audio_android
+    IOS: plugin_studentlife_audio
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: False
+      FEATURES: ["minutessilence", "minutesnoise", "minutesvoice", "minutesunknown","sumconversationduration","avgconversationduration",
+    "sdconversationduration","minconversationduration","maxconversationduration","timefirstconversation","timelastconversation","noisesumenergy",
+    "noiseavgenergy","noisesdenergy","noiseminenergy","noisemaxenergy","voicesumenergy",
+    "voiceavgenergy","voicesdenergy","voiceminenergy","voicemaxenergy","silencesensedfraction","noisesensedfraction",
+    "voicesensedfraction","unknownsensedfraction","silenceexpectedfraction","noiseexpectedfraction","voiceexpectedfraction",
+    "unknownexpectedfraction","countconversation"]
+      RECORDING_MINUTES: 1
+      PAUSED_MINUTES : 3
+      SRC_FOLDER: "rapids" # inside src/features/phone_conversation
+      SRC_LANGUAGE: "python"
+
+# See https://www.rapids.science/latest/features/phone-data-yield/
+PHONE_DATA_YIELD:
+  SENSORS: []
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: False
+      FEATURES: [ratiovalidyieldedminutes, ratiovalidyieldedhours]
+      MINUTE_RATIO_THRESHOLD_FOR_VALID_YIELDED_HOURS: 0.5 # 0 to 1 representing the number of minutes with at least
+      SRC_LANGUAGE: "r"
+      SRC_FOLDER: "rapids" # inside src/features/phone_data_yield
+
+# See https://www.rapids.science/latest/features/phone-light/
+PHONE_LIGHT:
+  TABLE: light
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: False
+      FEATURES: ["count", "maxlux", "minlux", "avglux", "medianlux", "stdlux"]
+      SRC_FOLDER: "rapids" # inside src/features/phone_light
+      SRC_LANGUAGE: "python"
+
+# See https://www.rapids.science/latest/features/phone-locations/
 PHONE_LOCATIONS:
   TABLE: locations
   LOCATIONS_TO_USE: FUSED_RESAMPLED # ALL, GPS OR FUSED_RESAMPLED  
@@ -115,42 +210,20 @@ PHONE_LOCATIONS:
       SRC_FOLDER: "barnett" # inside src/features/phone_locations
       SRC_LANGUAGE: "r"
 
-PHONE_BLUETOOTH:
+# See https://www.rapids.science/latest/features/phone-messages/
-  TABLE: bluetooth
+PHONE_MESSAGES:
+  TABLE: messages
   PROVIDERS:
     RAPIDS:
-      COMPUTE: TRUE
+      COMPUTE: False
-      FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"]
+      MESSAGES_TYPES : [received, sent]
-      SRC_FOLDER: "rapids" # inside src/features/phone_bluetooth
+      FEATURES: 
+        received: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact]
+        sent: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact]
       SRC_LANGUAGE: "r"
+      SRC_FOLDER: "rapids" # inside src/features/phone_messages
 
-
+# See https://www.rapids.science/latest/features/phone-screen/
-PHONE_ACTIVITY_RECOGNITION:
-  TABLE: 
-    ANDROID: plugin_google_activity_recognition
-    IOS: plugin_ios_activity_recognition
-  EPISODE_THRESHOLD_BETWEEN_ROWS: 5 # minutes. Max time difference for two consecutive rows to be considered within the same battery episode.
-  PROVIDERS:
-    RAPIDS:
-      COMPUTE: False
-      FEATURES: ["count", "mostcommonactivity", "countuniqueactivities", "durationstationary", "durationmobile", "durationvehicle"]
-      ACTIVITY_CLASSES:
-        STATIONARY: ["still", "tilting"]
-        MOBILE: ["on_foot", "walking", "running", "on_bicycle"]
-        VEHICLE: ["in_vehicle"]
-      SRC_FOLDER: "rapids" # inside src/features/phone_activity_recognition
-      SRC_LANGUAGE: "python"
-
-PHONE_BATTERY:
-  TABLE: battery
-  EPISODE_THRESHOLD_BETWEEN_ROWS: 30 # minutes. Max time difference for two consecutive rows to be considered within the same battery episode.
-  PROVIDERS:
-    RAPIDS:
-      COMPUTE: False
-      FEATURES: ["countdischarge", "sumdurationdischarge", "countcharge", "sumdurationcharge", "avgconsumptionrate", "maxconsumptionrate"]
-      SRC_FOLDER: "rapids" # inside src/features/phone_battery
-      SRC_LANGUAGE: "python"
-
 PHONE_SCREEN:
   TABLE: screen
   PROVIDERS:
@@ -164,190 +237,138 @@ PHONE_SCREEN:
       SRC_FOLDER: "rapids" # inside src/features/phone_screen
       SRC_LANGUAGE: "python"
 
-PHONE_LIGHT:
+# See https://www.rapids.science/latest/features/phone-wifi-connected/
-  TABLE: light
-  PROVIDERS:
-    RAPIDS:
-      COMPUTE: TRUE
-      FEATURES: ["count", "maxlux", "minlux", "avglux", "medianlux", "stdlux"]
-      SRC_FOLDER: "rapids" # inside src/features/phone_light
-      SRC_LANGUAGE: "python"
-
-PHONE_ACCELEROMETER:
-  TABLE: accelerometer
-  PROVIDERS:
-    RAPIDS:
-      COMPUTE: False
-      FEATURES: ["maxmagnitude", "minmagnitude", "avgmagnitude", "medianmagnitude", "stdmagnitude"]
-      SRC_FOLDER: "rapids" # inside src/features/phone_accelerometer
-      SRC_LANGUAGE: "python"
-    
-    PANDA:
-      COMPUTE: False
-      VALID_SENSED_MINUTES: False
-      FEATURES:
-        exertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"]
-        nonexertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"]
-      SRC_FOLDER: "panda" # inside src/features/phone_accelerometer
-      SRC_LANGUAGE: "python"
-
-PHONE_APPLICATIONS_FOREGROUND:
-  TABLE: applications_foreground
-  APPLICATION_CATEGORIES:
-    CATALOGUE_SOURCE: FILE # FILE (genres are read from CATALOGUE_FILE) or GOOGLE (genres are scrapped from the Play Store)
-    CATALOGUE_FILE: "data/external/stachl_application_genre_catalogue.csv"
-    UPDATE_CATALOGUE_FILE: False # if CATALOGUE_SOURCE is equal to FILE, whether or not to update CATALOGUE_FILE, if CATALOGUE_SOURCE is equal to GOOGLE all scraped genres will be saved to CATALOGUE_FILE
-    SCRAPE_MISSING_CATEGORIES: False # whether or not to scrape missing genres, only effective if CATALOGUE_SOURCE is equal to FILE. If CATALOGUE_SOURCE is equal to GOOGLE, all genres are scraped anyway
-  PROVIDERS:
-    RAPIDS:
-      COMPUTE: TRUE
-      SINGLE_CATEGORIES: ["all", "email"]
-      MULTIPLE_CATEGORIES:
-        social: ["socialnetworks", "socialmediatools"]
-        entertainment: ["entertainment", "gamingstrategy"]
-      SINGLE_APPS: ["top1global", "com.facebook.moments"] # There's no entropy for single apps
-      EXCLUDED_CATEGORIES: ["systemapp", "tvvideoapps"]
-      EXCLUDED_APPS: ["com.fitbit.FitbitMobile", "com.aware.plugin.upmc.cancer"]
-      FEATURES: ["count", "timeoffirstuse", "timeoflastuse", "frequencyentropy"]
-      SRC_FOLDER: "rapids" # inside src/features/phone_applications_foreground
-      SRC_LANGUAGE: "python"
-
-PHONE_WIFI_VISIBLE:
-  TABLE: "wifi"
-  PROVIDERS:
-    RAPIDS:
-      COMPUTE: TRUE
-      FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"]
-      SRC_FOLDER: "rapids" # inside src/features/phone_wifi_visible
-      SRC_LANGUAGE: "r"
-
 PHONE_WIFI_CONNECTED:
   TABLE: "sensor_wifi"
   PROVIDERS:
     RAPIDS:
-      COMPUTE: TRUE
+      COMPUTE: False
       FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"]
       SRC_FOLDER: "rapids" # inside src/features/phone_wifi_connected
       SRC_LANGUAGE: "r"
 
-PHONE_CONVERSATION:
+# See https://www.rapids.science/latest/features/phone-wifi-visible/
-  TABLE: 
+PHONE_WIFI_VISIBLE:
-    ANDROID: plugin_studentlife_audio_android
+  TABLE: "wifi"
-    IOS: plugin_studentlife_audio
-  PROVIDERS:
-    RAPIDS:
-      COMPUTE: TRUE
-      FEATURES: ["minutessilence", "minutesnoise", "minutesvoice", "minutesunknown","sumconversationduration","avgconversationduration",
-        "sdconversationduration","minconversationduration","maxconversationduration","timefirstconversation","timelastconversation","noisesumenergy",
-        "noiseavgenergy","noisesdenergy","noiseminenergy","noisemaxenergy","voicesumenergy",
-        "voiceavgenergy","voicesdenergy","voiceminenergy","voicemaxenergy","silencesensedfraction","noisesensedfraction",
-        "voicesensedfraction","unknownsensedfraction","silenceexpectedfraction","noiseexpectedfraction","voiceexpectedfraction",
-        "unknownexpectedfraction","countconversation"]
-      RECORDING_MINUTES: 1
-      PAUSED_MINUTES : 3
-      SRC_FOLDER: "rapids" # inside src/features/phone_conversation
-      SRC_LANGUAGE: "python"
-
-############## FITBIT ##########################################################
-################################################################################
-
-FITBIT_HEARTRATE:
-  TABLE_FORMAT: JSON # JSON or CSV
-  TABLE:
-    JSON: fitbit_heartrate
-    CSV:
-      SUMMARY: heartrate_summary.csv
-      INTRADAY: heartrate_intraday.csv
   PROVIDERS:
     RAPIDS:
       COMPUTE: False
-      SUMMARY_FEATURES: ["restinghr"] # calories features' accuracy depend on the accuracy of the participants fitbit profile (e.g. height, weight) use these with care: ["caloriesoutofrange", "caloriesfatburn", "caloriescardio", "caloriespeak"] 
+      FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"]
-      INTRADAY_FEATURES: ["maxhr", "minhr", "avghr", "medianhr", "modehr", "stdhr", "diffmaxmodehr", "diffminmodehr", "entropyhr", "minutesonoutofrangezone", "minutesonfatburnzone", "minutesoncardiozone", "minutesonpeakzone"]
+      SRC_FOLDER: "rapids" # inside src/features/phone_wifi_visible
+      SRC_LANGUAGE: "r"
 
 
-FITBIT_STEPS:
-  TABLE_FORMAT: JSON # JSON or CSV
-  TABLE:
-    JSON: fitbit_steps
-    CSV:
-      SUMMARY: steps_summary.csv
-      INTRADAY: steps_intraday.csv
-  EXCLUDE_SLEEP: # you can exclude sleep periods from the step features computation
-    EXCLUDE: False
-    TYPE: FIXED # FIXED OR FITBIT_BASED (configure FITBIT_SLEEP section)
-    FIXED:
-      START: "23:00"
-      END: "07:00"
 
+########################################################################################################################
+#                                                 FITBIT                                                               #
+########################################################################################################################
+
+# See https://www.rapids.science/latest/setup/configuration/#device-data-source-configuration
+FITBIT_DATA_CONFIGURATION:
+  SOURCE: 
+    TYPE: DATABASE # DATABASE or FILES (set each [FITBIT_SENSOR][TABLE] attribute with a table name or a file path accordingly)
+    COLUMN_FORMAT: JSON # JSON or PLAIN_TEXT
+    DATABASE_GROUP: *database_group
+    DEVICE_ID_COLUMN: device_id # column name
+  TIMEZONE: 
+    TYPE: SINGLE # Fitbit devices don't support time zones so we read this data in the timezone indicated by VALUE 
+    VALUE: *timezone
+
+# Sensors ------
+
+# See https://www.rapids.science/latest/features/fitbit-heartrate-summary/
+FITBIT_HEARTRATE_SUMMARY:
+  TABLE: heartrate_summary
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: False
+      FEATURES: ["maxrestinghr", "minrestinghr", "avgrestinghr", "medianrestinghr", "moderestinghr", "stdrestinghr", "diffmaxmoderestinghr", "diffminmoderestinghr", "entropyrestinghr"] # calories features' accuracy depend on the accuracy of the participants fitbit profile (e.g. height, weight) use these with care: ["sumcaloriesoutofrange", "maxcaloriesoutofrange", "mincaloriesoutofrange", "avgcaloriesoutofrange", "mediancaloriesoutofrange", "stdcaloriesoutofrange", "entropycaloriesoutofrange", "sumcaloriesfatburn", "maxcaloriesfatburn", "mincaloriesfatburn", "avgcaloriesfatburn", "mediancaloriesfatburn", "stdcaloriesfatburn", "entropycaloriesfatburn", "sumcaloriescardio", "maxcaloriescardio", "mincaloriescardio", "avgcaloriescardio", "mediancaloriescardio", "stdcaloriescardio", "entropycaloriescardio", "sumcaloriespeak", "maxcaloriespeak", "mincaloriespeak", "avgcaloriespeak", "mediancaloriespeak", "stdcaloriespeak", "entropycaloriespeak"]
+      SRC_FOLDER: "rapids" # inside src/features/fitbit_heartrate_summary
+      SRC_LANGUAGE: "python"
+
+# See https://www.rapids.science/latest/features/fitbit-heartrate-intraday/
+FITBIT_HEARTRATE_INTRADAY:
+  TABLE: heartrate_intraday
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: False
+      FEATURES: ["maxhr", "minhr", "avghr", "medianhr", "modehr", "stdhr", "diffmaxmodehr", "diffminmodehr", "entropyhr", "minutesonoutofrangezone", "minutesonfatburnzone", "minutesoncardiozone", "minutesonpeakzone"]
+      SRC_FOLDER: "rapids" # inside src/features/fitbit_heartrate_intraday
+      SRC_LANGUAGE: "python"
+
+# See https://www.rapids.science/latest/features/fitbit-sleep-summary/
+FITBIT_SLEEP_SUMMARY:
+  TABLE: sleep_summary
+  SLEEP_EPISODE_TIMESTAMP: end # summary sleep episodes are considered as events based on either the start timestamp or end timestamp.
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: False
+      FEATURES: ["countepisode", "avgefficiency", "sumdurationafterwakeup", "sumdurationasleep", "sumdurationawake", "sumdurationtofallasleep", "sumdurationinbed", "avgdurationafterwakeup", "avgdurationasleep", "avgdurationawake", "avgdurationtofallasleep", "avgdurationinbed"]
+      SLEEP_TYPES: ["main", "nap", "all"]
+      SRC_FOLDER: "rapids" # inside src/features/fitbit_sleep_summary
+      SRC_LANGUAGE: "python"
+
+# See https://www.rapids.science/latest/features/fitbit-steps-summary/
+FITBIT_STEPS_SUMMARY:
+  TABLE: steps_summary
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: False
+      FEATURES: ["maxsumsteps", "minsumsteps", "avgsumsteps", "mediansumsteps", "stdsumsteps"]
+      SRC_FOLDER: "rapids" # inside src/features/fitbit_steps_summary
+      SRC_LANGUAGE: "python"
+
+# See https://www.rapids.science/latest/features/fitbit-steps-intraday/
+FITBIT_STEPS_INTRADAY:
+  TABLE: steps_intraday
   PROVIDERS:
     RAPIDS:
       COMPUTE: False
       FEATURES:
-        ALL_STEPS: ["sumallsteps", "maxallsteps", "minallsteps", "avgallsteps", "stdallsteps"]
+        STEPS: ["sum", "max", "min", "avg", "std"]
         SEDENTARY_BOUT: ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "stdduration"]
         ACTIVE_BOUT: ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "stdduration"]
       THRESHOLD_ACTIVE_BOUT: 10 # steps
       INCLUDE_ZERO_STEP_ROWS: False
+      SRC_FOLDER: "rapids" # inside src/features/fitbit_steps_intraday
+      SRC_LANGUAGE: "python"
 
-FITBIT_SLEEP:
+# FITBIT_CALORIES:
-  TABLE_FORMAT: JSON # JSON or CSV
+#   TABLE_FORMAT: JSON # JSON or CSV. If your JSON or CSV data are files change [DEVICE_DATA][FITBIT][SOURCE][TYPE] to FILES
-  TABLE:
+#   TABLE:
-    JSON: fitbit_sleep
+#     JSON: fitbit_calories
-    CSV:
+#     CSV:
-      SUMMARY: sleep_summary.csv
+#       SUMMARY: calories_summary
-      INTRADAY: sleep_intraday.csv
+#       INTRADAY: calories_intraday
-  PROVIDERS:
+#   PROVIDERS:
-    RAPIDS:
+#     RAPIDS:
-      COMPUTE: False
+#       COMPUTE: False
-      SLEEP_TYPES: ["main", "nap", "all"]
+#       FEATURES: []
-      SUMMARY_FEATURES: ["sumdurationafterwakeup", "sumdurationasleep", "sumdurationawake", "sumdurationtofallasleep", "sumdurationinbed", "avgefficiency", "countepisode"]
 
-FITBIT_CALORIES:
-  TABLE_FORMAT: JSON # JSON or CSV
-  TABLE:
-    JSON: fitbit_calories
-    CSV:
-      SUMMARY: calories_summary.csv
-      INTRADAY: calories_intraday.csv
-  PROVIDERS:
-    RAPIDS:
-      COMPUTE: False
-      FEATURES: []
 
-### Visualizations #############################################################
-################################################################################
 
-HEATMAP_FEATURES_CORRELATIONS:
+########################################################################################################################
+#                                                 PLOTS                                                                #
+########################################################################################################################
+
+# Data quality
+HISTOGRAM_PHONE_DATA_YIELD:
+  PLOT: False
+
+HEATMAP_PHONE_DATA_YIELD_PER_PARTICIPANT_PER_TIME_SEGMENT:
+  PLOT: False
+
+HEATMAP_SENSORS_PER_MINUTE_PER_TIME_SEGMENT:
+  PLOT: False
+
+HEATMAP_SENSOR_ROW_COUNT_PER_TIME_SEGMENT:
+  PLOT: False
+  SENSORS: [PHONE_ACCELEROMETER, PHONE_ACTIVITY_RECOGNITION, PHONE_APPLICATIONS_FOREGROUND, PHONE_BATTERY, PHONE_BLUETOOTH, PHONE_CALLS, PHONE_CONVERSATION, PHONE_LIGHT, PHONE_LOCATIONS, PHONE_MESSAGES, PHONE_SCREEN, PHONE_WIFI_CONNECTED, PHONE_WIFI_VISIBLE]
+
+# Features
+HEATMAP_FEATURE_CORRELATION_MATRIX:
   PLOT: False
   MIN_ROWS_RATIO: 0.5
-  MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day
-  MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour
-  PHONE_FEATURES: [accelerometer, activity_recognition, applications_foreground, battery, calls_incoming, calls_missed, calls_outgoing, conversation, light, location_doryab, messages_received, messages_sent, screen]
-  FITBIT_FEATURES: [fitbit_heartrate, fitbit_step, fitbit_sleep]
   CORR_THRESHOLD: 0.1
   CORR_METHOD: "pearson" # choose from {"pearson", "kendall", "spearman"}
 
-HISTOGRAM_VALID_SENSED_HOURS:
-  PLOT: False
-  MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day
-  MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour
-
-HEATMAP_DAYS_BY_SENSORS:
-  PLOT: False
-  MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day
-  MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour
-  EXPECTED_NUM_OF_DAYS: -1
-  DB_TABLES: [accelerometer, applications_foreground, battery, bluetooth, calls, light, locations, messages, screen, wifi, sensor_wifi, plugin_google_activity_recognition, plugin_ios_activity_recognition, plugin_studentlife_audio_android, plugin_studentlife_audio]
-
-HEATMAP_SENSED_BINS:
-  PLOT: False
-  BIN_SIZE: *bin_size
-
-OVERALL_COMPLIANCE_HEATMAP:
-  PLOT: False
-  ONLY_SHOW_VALID_DAYS: False
-  EXPECTED_NUM_OF_DAYS: -1
-  BIN_SIZE: *bin_size
-  MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day
-  MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour
-

tests/settings/periodic/testing_config.yaml

@@ -1,85 +1,139 @@
-# Participants to include in the analysis
+# See https://www.rapids.science/latest/setup/configuration/#database-credentials
-# You must create a file for each participant named pXXX containing their device_id. This can be done manually or automatically
-PIDS: [test01, test02, test03, test04]
-
-# Global var with common time segments
-TIME_SEGMENTS: &time_segments
-  TYPE: PERIODIC # FREQUENCY, PERIODIC, EVENT
-  FILE: "data/external/timesegments_periodic.csv"
-  INCLUDE_PAST_PERIODIC_SEGMENTS: FALSE # Only relevant if TYPE=PERIODIC, if set to TRUE we consider time segments back enough in the past as to include the first day of data
-
-# Use tz codes from https://en.wikipedia.org/wiki/List_of_tz_database_time_zones. Double check your code, for example EST is not US Eastern Time.
-TIMEZONE: &timezone
-  America/New_York
-
 DATABASE_GROUP: &database_group
   MY_GROUP
 
-# config section for the script that creates participant files automatically
+# See https://www.rapids.science/latest/setup/configuration/#timezone-of-your-study
-PARTICIPANT_FILES: # run snakemake -j1 -R parse_participant_files
+TIMEZONE: &timezone
+  America/New_York
+
+# See https://www.rapids.science/latest/setup/configuration/#participant-files
+PIDS: [test01, test02, test03, test04]
+
+# See https://www.rapids.science/latest/setup/configuration/#automatic-creation-of-participant-files
+CREATE_PARTICIPANT_FILES:
+  SOURCE:
+    TYPE: AWARE_DEVICE_TABLE #AWARE_DEVICE_TABLE or CSV_FILE
+    DATABASE_GROUP: *database_group
+    CSV_FILE_PATH: "data/external/example_participants.csv" # see docs for required format
+    TIMEZONE: *timezone
   PHONE_SECTION:
-    ADD: FALSE
+    ADD: TRUE
-    PARSED_FROM: AWARE_DEVICE_TABLE #AWARE_DEVICE_TABLE or CSV_FILE
+    DEVICE_ID_COLUMN: device_id # column name
-    PARSED_SOURCE: *database_group # DB credentials group or CSV file path. If CSV file, it should have: device_id, pid (optional), label (optional), start_date (optional), end_date (optional)
     IGNORED_DEVICE_IDS: []
   FITBIT_SECTION:
-    ADD: FALSE
+    ADD: TRUE
-    SAME_AS_PHONE: FALSE # If TRUE, all config below is ignored
+    DEVICE_ID_COLUMN: device_id # column name
-    PARSED_FROM: CSV_FILE
+    IGNORED_DEVICE_IDS: []
-    PARSED_SOURCE: "external/my_fitbit_participants.csv" # CSV file should have: device_id, pid (optional), label (optional), start_date (optional), end_date (optional)
 
-DEVICE_DATA:
+# See https://www.rapids.science/latest/setup/configuration/#time-segments
-  PHONE:
+TIME_SEGMENTS: &time_segments
-    SOURCE: 
+  TYPE: PERIODIC # FREQUENCY, PERIODIC, EVENT
-      TYPE: DATABASE # Phone only supports DATABASE for now
+  FILE: "data/external/timesegments_periodic.csv"
-      DATABASE_GROUP: *database_group
+  INCLUDE_PAST_PERIODIC_SEGMENTS: FALSE # Only relevant if TYPE=PERIODIC, see docs
-      DEVICE_ID_COLUMN: device_id # column name
-    TIMEZONE: 
-      TYPE: SINGLE # SINGLE or MULTIPLE
-      VALUE: *timezone # IF TYPE=SINGLE, timezone code (e.g. America/New_York, see attribute TIMEZONE above). If TYPE=MULTIPLE, a table in your database with two columns (timestamp, timezone) where timestamp is a unix timestamp and timezone is one of https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
-  FITBIT:
-    SOURCE: 
-      TYPE: DATABASE # DATABASE or FILES (set each FITBIT_SENSOR TABLE attribute accordingly with a table name or a file path)
-      DATABASE_GROUP: *database_group
-      DEVICE_ID_COLUMN: device_id # column name
-    TIMEZONE: 
-      TYPE: SINGLE # Fitbit only supports SINGLE timezones
-      VALUE: *timezone # timezone code (e.g. America/New_York, see attribute TIMEZONE above and https://en.wikipedia.org/wiki/List_of_tz_database_time_zones)
 
-PHONE_VALID_SENSED_BINS:
-  COMPUTE: False # This flag is automatically ignored (set to True) if you are extracting PHONE_VALID_SENSED_DAYS or screen or Barnett's location features
-  BIN_SIZE: &bin_size 5 # (in minutes)
-  # Add as many PHONE sensors as you have, they all improve the computation of PHONE_VALID_SENSED_BINS and PHONE_VALID_SENSED_DAYS. 
-  # If you are extracting screen or Barnett/Doryab location features, PHONE_SCREEN and PHONE_LOCATIONS tables are mandatory.
-  # You can choose any of the keys shown below, just make sure its TABLE exists in your database!
-  # PHONE_MESSAGES, PHONE_CALLS, PHONE_LOCATIONS, PHONE_BLUETOOTH, PHONE_ACTIVITY_RECOGNITION, PHONE_BATTERY, PHONE_SCREEN, PHONE_LIGHT,
-  # PHONE_ACCELEROMETER, PHONE_APPLICATIONS_FOREGROUND, PHONE_WIFI_VISIBLE, PHONE_WIFI_CONNECTED, PHONE_CONVERSATION
-  PHONE_SENSORS: []
 
-PHONE_VALID_SENSED_DAYS:
-  COMPUTE: False
-  MIN_VALID_HOURS_PER_DAY: &min_valid_hours_per_day [16] # (out of 24) MIN_HOURS_PER_DAY
-  MIN_VALID_BINS_PER_HOUR: &min_valid_bins_per_hour [6] # (out of 60min/BIN_SIZE bins)
 
-# Communication SMS features config, TYPES and FEATURES keys need to match
+########################################################################################################################
-PHONE_MESSAGES:
+#                                                 PHONE                                                                #
-  TABLE: messages
+########################################################################################################################
+
+# See https://www.rapids.science/latest/setup/configuration/#device-data-source-configuration
+PHONE_DATA_CONFIGURATION:
+  SOURCE: 
+    TYPE: DATABASE
+    DATABASE_GROUP: *database_group
+    DEVICE_ID_COLUMN: device_id # column name
+  TIMEZONE: 
+    TYPE: SINGLE
+    VALUE: *timezone
+
+# Sensors ------
+
+# https://www.rapids.science/latest/features/phone-accelerometer/
+PHONE_ACCELEROMETER:
+  TABLE: accelerometer
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: False
+      FEATURES: ["maxmagnitude", "minmagnitude", "avgmagnitude", "medianmagnitude", "stdmagnitude"]
+      SRC_FOLDER: "rapids" # inside src/features/phone_accelerometer
+      SRC_LANGUAGE: "python"
+    
+    PANDA:
+      COMPUTE: False
+      VALID_SENSED_MINUTES: False
+      FEATURES:
+        exertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"]
+        nonexertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"]
+      SRC_FOLDER: "panda" # inside src/features/phone_accelerometer
+      SRC_LANGUAGE: "python"
+
+# See https://www.rapids.science/latest/features/phone-activity-recognition/
+PHONE_ACTIVITY_RECOGNITION:
+  TABLE: 
+    ANDROID: plugin_google_activity_recognition
+    IOS: plugin_ios_activity_recognition
+  EPISODE_THRESHOLD_BETWEEN_ROWS: 5 # minutes. Max time difference for two consecutive rows to be considered within the same battery episode.
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: False
+      FEATURES: ["count", "mostcommonactivity", "countuniqueactivities", "durationstationary", "durationmobile", "durationvehicle"]
+      ACTIVITY_CLASSES:
+        STATIONARY: ["still", "tilting"]
+        MOBILE: ["on_foot", "walking", "running", "on_bicycle"]
+        VEHICLE: ["in_vehicle"]
+      SRC_FOLDER: "rapids" # inside src/features/phone_activity_recognition
+      SRC_LANGUAGE: "python"
+
+# See https://www.rapids.science/latest/features/phone-applications-foreground/
+PHONE_APPLICATIONS_FOREGROUND:
+  TABLE: applications_foreground
+  APPLICATION_CATEGORIES:
+    CATALOGUE_SOURCE: FILE # FILE (genres are read from CATALOGUE_FILE) or GOOGLE (genres are scrapped from the Play Store)
+    CATALOGUE_FILE: "data/external/stachl_application_genre_catalogue.csv"
+    UPDATE_CATALOGUE_FILE: False # if CATALOGUE_SOURCE is equal to FILE, whether or not to update CATALOGUE_FILE, if CATALOGUE_SOURCE is equal to GOOGLE all scraped genres will be saved to CATALOGUE_FILE
+    SCRAPE_MISSING_CATEGORIES: False # whether or not to scrape missing genres, only effective if CATALOGUE_SOURCE is equal to FILE. If CATALOGUE_SOURCE is equal to GOOGLE, all genres are scraped anyway
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: False
+      SINGLE_CATEGORIES: ["all", "email"]
+      MULTIPLE_CATEGORIES:
+        social: ["socialnetworks", "socialmediatools"]
+        entertainment: ["entertainment", "gamingknowledge", "gamingcasual", "gamingadventure", "gamingstrategy", "gamingtoolscommunity", "gamingroleplaying", "gamingaction", "gaminglogic", "gamingsports", "gamingsimulation"]
+      SINGLE_APPS: ["top1global", "com.facebook.moments", "com.google.android.youtube", "com.twitter.android"] # There's no entropy for single apps
+      EXCLUDED_CATEGORIES: []
+      EXCLUDED_APPS: ["com.fitbit.FitbitMobile", "com.aware.plugin.upmc.cancer"]
+      FEATURES: ["count", "timeoffirstuse", "timeoflastuse", "frequencyentropy"]
+      SRC_FOLDER: "rapids" # inside src/features/phone_applications_foreground
+      SRC_LANGUAGE: "python"
+
+# See https://www.rapids.science/latest/features/phone-battery/
+PHONE_BATTERY:
+  TABLE: battery
+  EPISODE_THRESHOLD_BETWEEN_ROWS: 30 # minutes. Max time difference for two consecutive rows to be considered within the same battery episode.
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: False
+      FEATURES: ["countdischarge", "sumdurationdischarge", "countcharge", "sumdurationcharge", "avgconsumptionrate", "maxconsumptionrate"]
+      SRC_FOLDER: "rapids" # inside src/features/phone_battery
+      SRC_LANGUAGE: "python"
+
+# See https://www.rapids.science/latest/features/phone-bluetooth/
+PHONE_BLUETOOTH:
+  TABLE: bluetooth
   PROVIDERS:
     RAPIDS:
       COMPUTE: TRUE
-      MESSAGES_TYPES : [received, sent]
+      FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"]
-      FEATURES: 
+      SRC_FOLDER: "rapids" # inside src/features/phone_bluetooth
-        received: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact]
-        sent: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact]
       SRC_LANGUAGE: "r"
-      SRC_FOLDER: "rapids" # inside src/features/phone_messages
 
-# Communication call features config, TYPES and FEATURES keys need to match
+# See https://www.rapids.science/latest/features/phone-calls/
 PHONE_CALLS:
   TABLE: calls
   PROVIDERS:
     RAPIDS:
-      COMPUTE: TRUE
+      COMPUTE: False
       CALL_TYPES: [missed, incoming, outgoing]
       FEATURES:
         missed:  [count, distinctcontacts, timefirstcall, timelastcall, countmostfrequentcontact]
@@ -88,6 +142,47 @@ PHONE_CALLS:
       SRC_LANGUAGE: "r"
       SRC_FOLDER: "rapids" # inside src/features/phone_calls
 
+# See https://www.rapids.science/latest/features/phone-conversation/
+PHONE_CONVERSATION:
+  TABLE: 
+    ANDROID: plugin_studentlife_audio_android
+    IOS: plugin_studentlife_audio
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: False
+      FEATURES: ["minutessilence", "minutesnoise", "minutesvoice", "minutesunknown","sumconversationduration","avgconversationduration",
+    "sdconversationduration","minconversationduration","maxconversationduration","timefirstconversation","timelastconversation","noisesumenergy",
+    "noiseavgenergy","noisesdenergy","noiseminenergy","noisemaxenergy","voicesumenergy",
+    "voiceavgenergy","voicesdenergy","voiceminenergy","voicemaxenergy","silencesensedfraction","noisesensedfraction",
+    "voicesensedfraction","unknownsensedfraction","silenceexpectedfraction","noiseexpectedfraction","voiceexpectedfraction",
+    "unknownexpectedfraction","countconversation"]
+      RECORDING_MINUTES: 1
+      PAUSED_MINUTES : 3
+      SRC_FOLDER: "rapids" # inside src/features/phone_conversation
+      SRC_LANGUAGE: "python"
+
+# See https://www.rapids.science/latest/features/phone-data-yield/
+PHONE_DATA_YIELD:
+  SENSORS: []
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: False
+      FEATURES: [ratiovalidyieldedminutes, ratiovalidyieldedhours]
+      MINUTE_RATIO_THRESHOLD_FOR_VALID_YIELDED_HOURS: 0.5 # 0 to 1 representing the number of minutes with at least
+      SRC_LANGUAGE: "r"
+      SRC_FOLDER: "rapids" # inside src/features/phone_data_yield
+
+# See https://www.rapids.science/latest/features/phone-light/
+PHONE_LIGHT:
+  TABLE: light
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: False
+      FEATURES: ["count", "maxlux", "minlux", "avglux", "medianlux", "stdlux"]
+      SRC_FOLDER: "rapids" # inside src/features/phone_light
+      SRC_LANGUAGE: "python"
+
+# See https://www.rapids.science/latest/features/phone-locations/
 PHONE_LOCATIONS:
   TABLE: locations
   LOCATIONS_TO_USE: FUSED_RESAMPLED # ALL, GPS OR FUSED_RESAMPLED  
@@ -115,42 +210,20 @@ PHONE_LOCATIONS:
       SRC_FOLDER: "barnett" # inside src/features/phone_locations
       SRC_LANGUAGE: "r"
 
-PHONE_BLUETOOTH:
+# See https://www.rapids.science/latest/features/phone-messages/
-  TABLE: bluetooth
+PHONE_MESSAGES:
+  TABLE: messages
   PROVIDERS:
     RAPIDS:
-      COMPUTE: TRUE
+      COMPUTE: False
-      FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"]
+      MESSAGES_TYPES : [received, sent]
-      SRC_FOLDER: "rapids" # inside src/features/phone_bluetooth
+      FEATURES: 
+        received: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact]
+        sent: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact]
       SRC_LANGUAGE: "r"
+      SRC_FOLDER: "rapids" # inside src/features/phone_messages
 
-
+# See https://www.rapids.science/latest/features/phone-screen/
-PHONE_ACTIVITY_RECOGNITION:
-  TABLE: 
-    ANDROID: plugin_google_activity_recognition
-    IOS: plugin_ios_activity_recognition
-  EPISODE_THRESHOLD_BETWEEN_ROWS: 5 # minutes. Max time difference for two consecutive rows to be considered within the same battery episode.
-  PROVIDERS:
-    RAPIDS:
-      COMPUTE: False
-      FEATURES: ["count", "mostcommonactivity", "countuniqueactivities", "durationstationary", "durationmobile", "durationvehicle"]
-      ACTIVITY_CLASSES:
-        STATIONARY: ["still", "tilting"]
-        MOBILE: ["on_foot", "walking", "running", "on_bicycle"]
-        VEHICLE: ["in_vehicle"]
-      SRC_FOLDER: "rapids" # inside src/features/phone_activity_recognition
-      SRC_LANGUAGE: "python"
-
-PHONE_BATTERY:
-  TABLE: battery
-  EPISODE_THRESHOLD_BETWEEN_ROWS: 30 # minutes. Max time difference for two consecutive rows to be considered within the same battery episode.
-  PROVIDERS:
-    RAPIDS:
-      COMPUTE: False
-      FEATURES: ["countdischarge", "sumdurationdischarge", "countcharge", "sumdurationcharge", "avgconsumptionrate", "maxconsumptionrate"]
-      SRC_FOLDER: "rapids" # inside src/features/phone_battery
-      SRC_LANGUAGE: "python"
-
 PHONE_SCREEN:
   TABLE: screen
   PROVIDERS:
@@ -164,190 +237,138 @@ PHONE_SCREEN:
       SRC_FOLDER: "rapids" # inside src/features/phone_screen
       SRC_LANGUAGE: "python"
 
-PHONE_LIGHT:
+# See https://www.rapids.science/latest/features/phone-wifi-connected/
-  TABLE: light
-  PROVIDERS:
-    RAPIDS:
-      COMPUTE: TRUE
-      FEATURES: ["count", "maxlux", "minlux", "avglux", "medianlux", "stdlux"]
-      SRC_FOLDER: "rapids" # inside src/features/phone_light
-      SRC_LANGUAGE: "python"
-
-PHONE_ACCELEROMETER:
-  TABLE: accelerometer
-  PROVIDERS:
-    RAPIDS:
-      COMPUTE: False
-      FEATURES: ["maxmagnitude", "minmagnitude", "avgmagnitude", "medianmagnitude", "stdmagnitude"]
-      SRC_FOLDER: "rapids" # inside src/features/phone_accelerometer
-      SRC_LANGUAGE: "python"
-    
-    PANDA:
-      COMPUTE: False
-      VALID_SENSED_MINUTES: False
-      FEATURES:
-        exertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"]
-        nonexertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"]
-      SRC_FOLDER: "panda" # inside src/features/phone_accelerometer
-      SRC_LANGUAGE: "python"
-
-PHONE_APPLICATIONS_FOREGROUND:
-  TABLE: applications_foreground
-  APPLICATION_CATEGORIES:
-    CATALOGUE_SOURCE: FILE # FILE (genres are read from CATALOGUE_FILE) or GOOGLE (genres are scrapped from the Play Store)
-    CATALOGUE_FILE: "data/external/stachl_application_genre_catalogue.csv"
-    UPDATE_CATALOGUE_FILE: False # if CATALOGUE_SOURCE is equal to FILE, whether or not to update CATALOGUE_FILE, if CATALOGUE_SOURCE is equal to GOOGLE all scraped genres will be saved to CATALOGUE_FILE
-    SCRAPE_MISSING_CATEGORIES: False # whether or not to scrape missing genres, only effective if CATALOGUE_SOURCE is equal to FILE. If CATALOGUE_SOURCE is equal to GOOGLE, all genres are scraped anyway
-  PROVIDERS:
-    RAPIDS:
-      COMPUTE: TRUE
-      SINGLE_CATEGORIES: ["all", "email"]
-      MULTIPLE_CATEGORIES:
-        social: ["socialnetworks", "socialmediatools"]
-        entertainment: ["entertainment", "gamingstrategy"]
-      SINGLE_APPS: ["top1global", "com.facebook.moments"] # There's no entropy for single apps
-      EXCLUDED_CATEGORIES: ["systemapp", "tvvideoapps"]
-      EXCLUDED_APPS: ["com.fitbit.FitbitMobile", "com.aware.plugin.upmc.cancer"]
-      FEATURES: ["count", "timeoffirstuse", "timeoflastuse", "frequencyentropy"]
-      SRC_FOLDER: "rapids" # inside src/features/phone_applications_foreground
-      SRC_LANGUAGE: "python"
-
-PHONE_WIFI_VISIBLE:
-  TABLE: "wifi"
-  PROVIDERS:
-    RAPIDS:
-      COMPUTE: TRUE
-      FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"]
-      SRC_FOLDER: "rapids" # inside src/features/phone_wifi_visible
-      SRC_LANGUAGE: "r"
-
 PHONE_WIFI_CONNECTED:
   TABLE: "sensor_wifi"
   PROVIDERS:
     RAPIDS:
-      COMPUTE: TRUE
+      COMPUTE: False
       FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"]
       SRC_FOLDER: "rapids" # inside src/features/phone_wifi_connected
       SRC_LANGUAGE: "r"
 
-PHONE_CONVERSATION:
+# See https://www.rapids.science/latest/features/phone-wifi-visible/
-  TABLE: 
+PHONE_WIFI_VISIBLE:
-    ANDROID: plugin_studentlife_audio_android
+  TABLE: "wifi"
-    IOS: plugin_studentlife_audio
-  PROVIDERS:
-    RAPIDS:
-      COMPUTE: TRUE
-      FEATURES: ["minutessilence", "minutesnoise", "minutesvoice", "minutesunknown","sumconversationduration","avgconversationduration",
-        "sdconversationduration","minconversationduration","maxconversationduration","timefirstconversation","timelastconversation","noisesumenergy",
-        "noiseavgenergy","noisesdenergy","noiseminenergy","noisemaxenergy","voicesumenergy",
-        "voiceavgenergy","voicesdenergy","voiceminenergy","voicemaxenergy","silencesensedfraction","noisesensedfraction",
-        "voicesensedfraction","unknownsensedfraction","silenceexpectedfraction","noiseexpectedfraction","voiceexpectedfraction",
-        "unknownexpectedfraction","countconversation"]
-      RECORDING_MINUTES: 1
-      PAUSED_MINUTES : 3
-      SRC_FOLDER: "rapids" # inside src/features/phone_conversation
-      SRC_LANGUAGE: "python"
-
-############## FITBIT ##########################################################
-################################################################################
-
-FITBIT_HEARTRATE:
-  TABLE_FORMAT: JSON # JSON or CSV
-  TABLE:
-    JSON: fitbit_heartrate
-    CSV:
-      SUMMARY: heartrate_summary.csv
-      INTRADAY: heartrate_intraday.csv
   PROVIDERS:
     RAPIDS:
       COMPUTE: False
-      SUMMARY_FEATURES: ["restinghr"] # calories features' accuracy depend on the accuracy of the participants fitbit profile (e.g. height, weight) use these with care: ["caloriesoutofrange", "caloriesfatburn", "caloriescardio", "caloriespeak"] 
+      FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"]
-      INTRADAY_FEATURES: ["maxhr", "minhr", "avghr", "medianhr", "modehr", "stdhr", "diffmaxmodehr", "diffminmodehr", "entropyhr", "minutesonoutofrangezone", "minutesonfatburnzone", "minutesoncardiozone", "minutesonpeakzone"]
+      SRC_FOLDER: "rapids" # inside src/features/phone_wifi_visible
+      SRC_LANGUAGE: "r"
 
 
-FITBIT_STEPS:
-  TABLE_FORMAT: JSON # JSON or CSV
-  TABLE:
-    JSON: fitbit_steps
-    CSV:
-      SUMMARY: steps_summary.csv
-      INTRADAY: steps_intraday.csv
-  EXCLUDE_SLEEP: # you can exclude sleep periods from the step features computation
-    EXCLUDE: False
-    TYPE: FIXED # FIXED OR FITBIT_BASED (configure FITBIT_SLEEP section)
-    FIXED:
-      START: "23:00"
-      END: "07:00"
 
+########################################################################################################################
+#                                                 FITBIT                                                               #
+########################################################################################################################
+
+# See https://www.rapids.science/latest/setup/configuration/#device-data-source-configuration
+FITBIT_DATA_CONFIGURATION:
+  SOURCE: 
+    TYPE: DATABASE # DATABASE or FILES (set each [FITBIT_SENSOR][TABLE] attribute with a table name or a file path accordingly)
+    COLUMN_FORMAT: JSON # JSON or PLAIN_TEXT
+    DATABASE_GROUP: *database_group
+    DEVICE_ID_COLUMN: device_id # column name
+  TIMEZONE: 
+    TYPE: SINGLE # Fitbit devices don't support time zones so we read this data in the timezone indicated by VALUE 
+    VALUE: *timezone
+
+# Sensors ------
+
+# See https://www.rapids.science/latest/features/fitbit-heartrate-summary/
+FITBIT_HEARTRATE_SUMMARY:
+  TABLE: heartrate_summary
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: False
+      FEATURES: ["maxrestinghr", "minrestinghr", "avgrestinghr", "medianrestinghr", "moderestinghr", "stdrestinghr", "diffmaxmoderestinghr", "diffminmoderestinghr", "entropyrestinghr"] # calories features' accuracy depend on the accuracy of the participants fitbit profile (e.g. height, weight) use these with care: ["sumcaloriesoutofrange", "maxcaloriesoutofrange", "mincaloriesoutofrange", "avgcaloriesoutofrange", "mediancaloriesoutofrange", "stdcaloriesoutofrange", "entropycaloriesoutofrange", "sumcaloriesfatburn", "maxcaloriesfatburn", "mincaloriesfatburn", "avgcaloriesfatburn", "mediancaloriesfatburn", "stdcaloriesfatburn", "entropycaloriesfatburn", "sumcaloriescardio", "maxcaloriescardio", "mincaloriescardio", "avgcaloriescardio", "mediancaloriescardio", "stdcaloriescardio", "entropycaloriescardio", "sumcaloriespeak", "maxcaloriespeak", "mincaloriespeak", "avgcaloriespeak", "mediancaloriespeak", "stdcaloriespeak", "entropycaloriespeak"]
+      SRC_FOLDER: "rapids" # inside src/features/fitbit_heartrate_summary
+      SRC_LANGUAGE: "python"
+
+# See https://www.rapids.science/latest/features/fitbit-heartrate-intraday/
+FITBIT_HEARTRATE_INTRADAY:
+  TABLE: heartrate_intraday
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: False
+      FEATURES: ["maxhr", "minhr", "avghr", "medianhr", "modehr", "stdhr", "diffmaxmodehr", "diffminmodehr", "entropyhr", "minutesonoutofrangezone", "minutesonfatburnzone", "minutesoncardiozone", "minutesonpeakzone"]
+      SRC_FOLDER: "rapids" # inside src/features/fitbit_heartrate_intraday
+      SRC_LANGUAGE: "python"
+
+# See https://www.rapids.science/latest/features/fitbit-sleep-summary/
+FITBIT_SLEEP_SUMMARY:
+  TABLE: sleep_summary
+  SLEEP_EPISODE_TIMESTAMP: end # summary sleep episodes are considered as events based on either the start timestamp or end timestamp.
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: False
+      FEATURES: ["countepisode", "avgefficiency", "sumdurationafterwakeup", "sumdurationasleep", "sumdurationawake", "sumdurationtofallasleep", "sumdurationinbed", "avgdurationafterwakeup", "avgdurationasleep", "avgdurationawake", "avgdurationtofallasleep", "avgdurationinbed"]
+      SLEEP_TYPES: ["main", "nap", "all"]
+      SRC_FOLDER: "rapids" # inside src/features/fitbit_sleep_summary
+      SRC_LANGUAGE: "python"
+
+# See https://www.rapids.science/latest/features/fitbit-steps-summary/
+FITBIT_STEPS_SUMMARY:
+  TABLE: steps_summary
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: False
+      FEATURES: ["maxsumsteps", "minsumsteps", "avgsumsteps", "mediansumsteps", "stdsumsteps"]
+      SRC_FOLDER: "rapids" # inside src/features/fitbit_steps_summary
+      SRC_LANGUAGE: "python"
+
+# See https://www.rapids.science/latest/features/fitbit-steps-intraday/
+FITBIT_STEPS_INTRADAY:
+  TABLE: steps_intraday
   PROVIDERS:
     RAPIDS:
       COMPUTE: False
       FEATURES:
-        ALL_STEPS: ["sumallsteps", "maxallsteps", "minallsteps", "avgallsteps", "stdallsteps"]
+        STEPS: ["sum", "max", "min", "avg", "std"]
         SEDENTARY_BOUT: ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "stdduration"]
         ACTIVE_BOUT: ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "stdduration"]
       THRESHOLD_ACTIVE_BOUT: 10 # steps
       INCLUDE_ZERO_STEP_ROWS: False
+      SRC_FOLDER: "rapids" # inside src/features/fitbit_steps_intraday
+      SRC_LANGUAGE: "python"
 
-FITBIT_SLEEP:
+# FITBIT_CALORIES:
-  TABLE_FORMAT: JSON # JSON or CSV
+#   TABLE_FORMAT: JSON # JSON or CSV. If your JSON or CSV data are files change [DEVICE_DATA][FITBIT][SOURCE][TYPE] to FILES
-  TABLE:
+#   TABLE:
-    JSON: fitbit_sleep
+#     JSON: fitbit_calories
-    CSV:
+#     CSV:
-      SUMMARY: sleep_summary.csv
+#       SUMMARY: calories_summary
-      INTRADAY: sleep_intraday.csv
+#       INTRADAY: calories_intraday
-  PROVIDERS:
+#   PROVIDERS:
-    RAPIDS:
+#     RAPIDS:
-      COMPUTE: False
+#       COMPUTE: False
-      SLEEP_TYPES: ["main", "nap", "all"]
+#       FEATURES: []
-      SUMMARY_FEATURES: ["sumdurationafterwakeup", "sumdurationasleep", "sumdurationawake", "sumdurationtofallasleep", "sumdurationinbed", "avgefficiency", "countepisode"]
 
-FITBIT_CALORIES:
-  TABLE_FORMAT: JSON # JSON or CSV
-  TABLE:
-    JSON: fitbit_calories
-    CSV:
-      SUMMARY: calories_summary.csv
-      INTRADAY: calories_intraday.csv
-  PROVIDERS:
-    RAPIDS:
-      COMPUTE: False
-      FEATURES: []
 
-### Visualizations #############################################################
-################################################################################
 
-HEATMAP_FEATURES_CORRELATIONS:
+########################################################################################################################
+#                                                 PLOTS                                                                #
+########################################################################################################################
+
+# Data quality
+HISTOGRAM_PHONE_DATA_YIELD:
+  PLOT: False
+
+HEATMAP_PHONE_DATA_YIELD_PER_PARTICIPANT_PER_TIME_SEGMENT:
+  PLOT: False
+
+HEATMAP_SENSORS_PER_MINUTE_PER_TIME_SEGMENT:
+  PLOT: False
+
+HEATMAP_SENSOR_ROW_COUNT_PER_TIME_SEGMENT:
+  PLOT: False
+  SENSORS: [PHONE_ACCELEROMETER, PHONE_ACTIVITY_RECOGNITION, PHONE_APPLICATIONS_FOREGROUND, PHONE_BATTERY, PHONE_BLUETOOTH, PHONE_CALLS, PHONE_CONVERSATION, PHONE_LIGHT, PHONE_LOCATIONS, PHONE_MESSAGES, PHONE_SCREEN, PHONE_WIFI_CONNECTED, PHONE_WIFI_VISIBLE]
+
+# Features
+HEATMAP_FEATURE_CORRELATION_MATRIX:
   PLOT: False
   MIN_ROWS_RATIO: 0.5
-  MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day
-  MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour
-  PHONE_FEATURES: [accelerometer, activity_recognition, applications_foreground, battery, calls_incoming, calls_missed, calls_outgoing, conversation, light, location_doryab, messages_received, messages_sent, screen]
-  FITBIT_FEATURES: [fitbit_heartrate, fitbit_step, fitbit_sleep]
   CORR_THRESHOLD: 0.1
   CORR_METHOD: "pearson" # choose from {"pearson", "kendall", "spearman"}
 
-HISTOGRAM_VALID_SENSED_HOURS:
-  PLOT: False
-  MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day
-  MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour
-
-HEATMAP_DAYS_BY_SENSORS:
-  PLOT: False
-  MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day
-  MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour
-  EXPECTED_NUM_OF_DAYS: -1
-  DB_TABLES: [accelerometer, applications_foreground, battery, bluetooth, calls, light, locations, messages, screen, wifi, sensor_wifi, plugin_google_activity_recognition, plugin_ios_activity_recognition, plugin_studentlife_audio_android, plugin_studentlife_audio]
-
-HEATMAP_SENSED_BINS:
-  PLOT: False
-  BIN_SIZE: *bin_size
-
-OVERALL_COMPLIANCE_HEATMAP:
-  PLOT: False
-  ONLY_SHOW_VALID_DAYS: False
-  EXPECTED_NUM_OF_DAYS: -1
-  BIN_SIZE: *bin_size
-  MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day
-  MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour
-