From 87b23d7848dce016670cb7b02c788fe595876b75 Mon Sep 17 00:00:00 2001 From: abhineethreddyk Date: Mon, 19 Oct 2020 17:13:10 -0400 Subject: [PATCH] Update Snakemake, testing_config.yaml, and messages' filenames in the tests directory --- tests/Snakefile | 217 ++++++++++----- .../{messages.csv => phone_messages.csv} | 0 .../{messages.csv => phone_messages.csv} | 0 .../{messages.csv => phone_messages.csv} | 0 .../{messages.csv => phone_messages.csv} | 0 .../{messages.csv => phone_messages.csv} | 0 .../{messages.csv => phone_messages.csv} | 0 .../{messages.csv => phone_messages.csv} | 0 .../{messages.csv => phone_messages.csv} | 0 ...essages_raw.csv => phone_messages_raw.csv} | 0 ...essages_raw.csv => phone_messages_raw.csv} | 0 ...essages_raw.csv => phone_messages_raw.csv} | 0 ...essages_raw.csv => phone_messages_raw.csv} | 0 ...essages_raw.csv => phone_messages_raw.csv} | 0 ...essages_raw.csv => phone_messages_raw.csv} | 0 tests/settings/frequency/testing_config.yaml | 258 ++++++++++++++--- tests/settings/periodic/testing_config.yaml | 259 +++++++++++++++--- 17 files changed, 605 insertions(+), 129 deletions(-) rename tests/data/processed/features/frequency/test03/{messages.csv => phone_messages.csv} (100%) rename tests/data/processed/features/frequency/test04/{messages.csv => phone_messages.csv} (100%) rename tests/data/processed/features/frequency/test05/{messages.csv => phone_messages.csv} (100%) rename tests/data/processed/features/frequency/test06/{messages.csv => phone_messages.csv} (100%) rename tests/data/processed/features/periodic/test01/{messages.csv => phone_messages.csv} (100%) rename tests/data/processed/features/periodic/test02/{messages.csv => phone_messages.csv} (100%) rename tests/data/processed/features/periodic/test03/{messages.csv => phone_messages.csv} (100%) rename tests/data/processed/features/periodic/test04/{messages.csv => phone_messages.csv} (100%) rename tests/data/raw/test01/{messages_raw.csv => phone_messages_raw.csv} (100%) rename tests/data/raw/test02/{messages_raw.csv => phone_messages_raw.csv} (100%) rename tests/data/raw/test03/{messages_raw.csv => phone_messages_raw.csv} (100%) rename tests/data/raw/test04/{messages_raw.csv => phone_messages_raw.csv} (100%) rename tests/data/raw/test05/{messages_raw.csv => phone_messages_raw.csv} (100%) rename tests/data/raw/test06/{messages_raw.csv => phone_messages_raw.csv} (100%) diff --git a/tests/Snakefile b/tests/Snakefile index 8cd9c333..9e2da9d7 100644 --- a/tests/Snakefile +++ b/tests/Snakefile @@ -13,18 +13,13 @@ if len(config["PIDS"]) == 0: raise ValueError("Add participants IDs to PIDS in config.yaml. Remember to create their participant files in data/external") if config["PHONE_VALID_SENSED_BINS"]["COMPUTE"] or config["PHONE_VALID_SENSED_DAYS"]["COMPUTE"]: # valid sensed bins is necessary for sensed days, so we add these files anyways if sensed days are requested - if len(config["PHONE_VALID_SENSED_BINS"]["DB_TABLES"]) == 0: - raise ValueError("If you want to compute PHONE_VALID_SENSED_BINS or PHONE_VALID_SENSED_DAYS, you need to add at least one table to [PHONE_VALID_SENSED_BINS][DB_TABLES] in config.yaml") + if len(config["PHONE_VALID_SENSED_BINS"]["PHONE_SENSORS"]) == 0: + raise ValueError("If you want to compute PHONE_VALID_SENSED_BINS or PHONE_VALID_SENSED_DAYS, you need to add at least one PHONE_SENSOR to [PHONE_VALID_SENSED_BINS][PHONE_SENSORS] in config.yaml") - pids_android = list(filter(lambda pid: infer_participant_platform("data/external/" + pid) == "android", config["PIDS"])) - pids_ios = list(filter(lambda pid: infer_participant_platform("data/external/" + pid) == "ios", config["PIDS"])) - tables_android = [table for table in config["PHONE_VALID_SENSED_BINS"]["DB_TABLES"] if table not in [config["CONVERSATION"]["DB_TABLE"]["IOS"], config["ACTIVITY_RECOGNITION"]["DB_TABLE"]["IOS"]]] # for android, discard any ios tables that may exist - tables_ios = [table for table in config["PHONE_VALID_SENSED_BINS"]["DB_TABLES"] if table not in [config["CONVERSATION"]["DB_TABLE"]["ANDROID"], config["ACTIVITY_RECOGNITION"]["DB_TABLE"]["ANDROID"]]] # for ios, discard any android tables that may exist - - for pids,table in zip([pids_android, pids_ios], [tables_android, tables_ios]): - files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=pids, sensor=table)) - files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime.csv", pid=pids, sensor=table)) + files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=map(str.lower, config["PHONE_VALID_SENSED_BINS"]["PHONE_SENSORS"]))) + files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime.csv", pid=config["PIDS"], sensor=map(str.lower, config["PHONE_VALID_SENSED_BINS"]["PHONE_SENSORS"]))) files_to_compute.extend(expand("data/interim/{pid}/phone_sensed_bins.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/phone_sensed_timestamps.csv", pid=config["PIDS"])) if config["PHONE_VALID_SENSED_DAYS"]["COMPUTE"]: files_to_compute.extend(expand("data/interim/{pid}/phone_valid_sensed_days_{min_valid_hours_per_day}hours_{min_valid_bins_per_hour}bins.csv", @@ -32,68 +27,158 @@ if config["PHONE_VALID_SENSED_DAYS"]["COMPUTE"]: min_valid_hours_per_day=config["PHONE_VALID_SENSED_DAYS"]["MIN_VALID_HOURS_PER_DAY"], min_valid_bins_per_hour=config["PHONE_VALID_SENSED_DAYS"]["MIN_VALID_BINS_PER_HOUR"])) -for provider in config["MESSAGES"]["PROVIDERS"].keys(): - if config["MESSAGES"]["PROVIDERS"][provider]["COMPUTE"]: - files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=config["MESSAGES"]["DB_TABLE"])) - files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime.csv", pid=config["PIDS"], sensor=config["MESSAGES"]["DB_TABLE"])) - files_to_compute.extend(expand("data/interim/{pid}/{sensor_key}_features/{sensor_key}_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["MESSAGES"]["PROVIDERS"][provider]["SRC_LANGUAGE"], provider_key=provider, sensor_key="MESSAGES".lower())) - files_to_compute.extend(expand("data/processed/features/{pid}/{sensor_key}.csv", pid=config["PIDS"], sensor_key="MESSAGES".lower())) +for provider in config["PHONE_MESSAGES"]["PROVIDERS"].keys(): + if config["PHONE_MESSAGES"]["PROVIDERS"][provider]["COMPUTE"]: + files_to_compute.extend(expand("data/raw/{pid}/phone_messages_raw.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/raw/{pid}/phone_messages_with_datetime.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/phone_messages_features/phone_messages_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_MESSAGES"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/processed/features/{pid}/phone_messages.csv", pid=config["PIDS"])) -for provider in config["CALLS"]["PROVIDERS"].keys(): - if config["CALLS"]["PROVIDERS"][provider]["COMPUTE"]: - files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=config["CALLS"]["DB_TABLE"])) - files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime.csv", pid=config["PIDS"], sensor=config["CALLS"]["DB_TABLE"])) - files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime_unified.csv", pid=config["PIDS"], sensor=config["CALLS"]["DB_TABLE"])) - files_to_compute.extend(expand("data/interim/{pid}/{sensor_key}_features/{sensor_key}_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["CALLS"]["PROVIDERS"][provider]["SRC_LANGUAGE"], provider_key=provider, sensor_key="CALLS".lower())) - files_to_compute.extend(expand("data/processed/features/{pid}/{sensor_key}.csv", pid=config["PIDS"], sensor_key="CALLS".lower())) +for provider in config["PHONE_CALLS"]["PROVIDERS"].keys(): + if config["PHONE_CALLS"]["PROVIDERS"][provider]["COMPUTE"]: + files_to_compute.extend(expand("data/raw/{pid}/phone_calls_raw.csv", pid=config["PIDS"], sensor=config["PHONE_CALLS"]["DB_TABLE"])) + files_to_compute.extend(expand("data/raw/{pid}/phone_calls_with_datetime.csv", pid=config["PIDS"], sensor=config["PHONE_CALLS"]["DB_TABLE"])) + files_to_compute.extend(expand("data/raw/{pid}/phone_calls_with_datetime_unified.csv", pid=config["PIDS"], sensor=config["PHONE_CALLS"]["DB_TABLE"])) + files_to_compute.extend(expand("data/interim/{pid}/phone_calls_features/phone_calls_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_CALLS"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/processed/features/{pid}/phone_calls.csv", pid=config["PIDS"])) -for provider in config["BLUETOOTH"]["PROVIDERS"].keys(): - if config["BLUETOOTH"]["PROVIDERS"][provider]["COMPUTE"]: - files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=config["BLUETOOTH"]["DB_TABLE"])) - files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime.csv", pid=config["PIDS"], sensor=config["BLUETOOTH"]["DB_TABLE"])) - files_to_compute.extend(expand("data/interim/{pid}/{sensor_key}_features/{sensor_key}_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["BLUETOOTH"]["PROVIDERS"][provider]["SRC_LANGUAGE"], provider_key=provider, sensor_key="BLUETOOTH".lower())) - files_to_compute.extend(expand("data/processed/features/{pid}/{sensor_key}.csv", pid=config["PIDS"], sensor_key="BLUETOOTH".lower())) +for provider in config["PHONE_BLUETOOTH"]["PROVIDERS"].keys(): + if config["PHONE_BLUETOOTH"]["PROVIDERS"][provider]["COMPUTE"]: + files_to_compute.extend(expand("data/raw/{pid}/phone_bluetooth_raw.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/raw/{pid}/phone_bluetooth_with_datetime.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/phone_bluetooth_features/phone_bluetooth_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_BLUETOOTH"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/processed/features/{pid}/phone_bluetooth.csv", pid=config["PIDS"])) -for provider in config["LIGHT"]["PROVIDERS"].keys(): - if config["LIGHT"]["PROVIDERS"][provider]["COMPUTE"]: - files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=config["LIGHT"]["DB_TABLE"])) - files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime.csv", pid=config["PIDS"], sensor=config["LIGHT"]["DB_TABLE"])) - files_to_compute.extend(expand("data/interim/{pid}/{sensor_key}_features/{sensor_key}_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["LIGHT"]["PROVIDERS"][provider]["SRC_LANGUAGE"], provider_key=provider, sensor_key="LIGHT".lower())) - files_to_compute.extend(expand("data/processed/features/{pid}/{sensor_key}.csv", pid=config["PIDS"], sensor_key="LIGHT".lower())) +for provider in config["PHONE_ACTIVITY_RECOGNITION"]["PROVIDERS"].keys(): + if config["PHONE_ACTIVITY_RECOGNITION"]["PROVIDERS"][provider]["COMPUTE"]: + files_to_compute.extend(expand("data/raw/{pid}/phone_activity_recognition_raw.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/raw/{pid}/phone_activity_recognition_with_datetime.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/raw/{pid}/phone_activity_recognition_with_datetime_unified.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/phone_activity_recognition_episodes.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/phone_activity_recognition_episodes_resampled.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/phone_activity_recognition_episodes_resampled_with_datetime.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/phone_activity_recognition_features/phone_activity_recognition_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_ACTIVITY_RECOGNITION"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/processed/features/{pid}/phone_activity_recognition.csv", pid=config["PIDS"])) -for provider in config["APPLICATIONS_FOREGROUND"]["PROVIDERS"].keys(): - if config["APPLICATIONS_FOREGROUND"]["PROVIDERS"][provider]["COMPUTE"]: - files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=config["APPLICATIONS_FOREGROUND"]["DB_TABLE"])) - files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime.csv", pid=config["PIDS"], sensor=config["APPLICATIONS_FOREGROUND"]["DB_TABLE"])) - files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime_with_genre.csv", pid=config["PIDS"], sensor=config["APPLICATIONS_FOREGROUND"]["DB_TABLE"])) - files_to_compute.extend(expand("data/interim/{pid}/{sensor_key}_features/{sensor_key}_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["APPLICATIONS_FOREGROUND"]["PROVIDERS"][provider]["SRC_LANGUAGE"], provider_key=provider, sensor_key="APPLICATIONS_FOREGROUND".lower())) - files_to_compute.extend(expand("data/processed/features/{pid}/{sensor_key}.csv", pid=config["PIDS"], sensor_key="APPLICATIONS_FOREGROUND".lower())) -for provider in config["WIFI"]["PROVIDERS"].keys(): - if config["WIFI"]["PROVIDERS"][provider]["COMPUTE"]: - if len(config["WIFI"]["DB_TABLE"]["VISIBLE_ACCESS_POINTS"]) > 0: - files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=config["WIFI"]["DB_TABLE"]["VISIBLE_ACCESS_POINTS"])) - files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime.csv", pid=config["PIDS"], sensor=config["WIFI"]["DB_TABLE"]["VISIBLE_ACCESS_POINTS"])) - files_to_compute.extend(expand("data/raw/{pid}/{sensor_key}_with_datetime_visibleandconnected.csv", pid=config["PIDS"], sensor_key="WIFI".lower())) - files_to_compute.extend(expand("data/interim/{pid}/{sensor_key}_features/{sensor_key}_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["WIFI"]["PROVIDERS"][provider]["SRC_LANGUAGE"], provider_key=provider, sensor_key="WIFI".lower())) - files_to_compute.extend(expand("data/processed/features/{pid}/{sensor_key}.csv", pid=config["PIDS"], sensor_key="WIFI".lower())) - if len(config["WIFI"]["DB_TABLE"]["CONNECTED_ACCESS_POINTS"]) > 0: - files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=config["WIFI"]["DB_TABLE"]["CONNECTED_ACCESS_POINTS"])) - files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime.csv", pid=config["PIDS"], sensor=config["WIFI"]["DB_TABLE"]["CONNECTED_ACCESS_POINTS"])) - files_to_compute.extend(expand("data/raw/{pid}/{sensor_key}_with_datetime_visibleandconnected.csv", pid=config["PIDS"], sensor_key="WIFI".lower())) - files_to_compute.extend(expand("data/interim/{pid}/{sensor_key}_features/{sensor_key}_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["WIFI"]["PROVIDERS"][provider]["SRC_LANGUAGE"], provider_key=provider, sensor_key="WIFI".lower())) - files_to_compute.extend(expand("data/processed/features/{pid}/{sensor_key}.csv", pid=config["PIDS"], sensor_key="WIFI".lower())) +for provider in config["PHONE_BATTERY"]["PROVIDERS"].keys(): + if config["PHONE_BATTERY"]["PROVIDERS"][provider]["COMPUTE"]: + files_to_compute.extend(expand("data/raw/{pid}/phone_battery_raw.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/phone_battery_episodes.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/phone_battery_episodes_resampled.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/phone_battery_episodes_resampled_with_datetime.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/phone_battery_features/phone_battery_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_BATTERY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/processed/features/{pid}/phone_battery.csv", pid=config["PIDS"])) + + +for provider in config["PHONE_SCREEN"]["PROVIDERS"].keys(): + if config["PHONE_SCREEN"]["PROVIDERS"][provider]["COMPUTE"]: + if "PHONE_SCREEN" in config["PHONE_VALID_SENSED_BINS"]["PHONE_SENSORS"]: + files_to_compute.extend(expand("data/interim/{pid}/phone_sensed_bins.csv", pid=config["PIDS"])) + else: + raise ValueError("Error: Add PHONE_SCREEN (and as many phone sensor as you have in your database) to [PHONE_VALID_SENSED_BINS][PHONE_SENSORS] in config.yaml. This is necessary to compute phone_sensed_bins (bins of time when the smartphone was sensing data)") + files_to_compute.extend(expand("data/raw/{pid}/phone_screen_raw.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/raw/{pid}/phone_screen_with_datetime.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/raw/{pid}/phone_screen_with_datetime_unified.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/phone_screen_episodes.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/phone_screen_episodes_resampled.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/phone_screen_episodes_resampled_with_datetime.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/phone_screen_features/phone_screen_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_SCREEN"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/processed/features/{pid}/phone_screen.csv", pid=config["PIDS"])) + +for provider in config["PHONE_LIGHT"]["PROVIDERS"].keys(): + if config["PHONE_LIGHT"]["PROVIDERS"][provider]["COMPUTE"]: + files_to_compute.extend(expand("data/raw/{pid}/phone_light_raw.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/raw/{pid}/phone_light_with_datetime.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/phone_light_features/phone_light_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_LIGHT"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/processed/features/{pid}/phone_light.csv", pid=config["PIDS"],)) + +for provider in config["PHONE_ACCELEROMETER"]["PROVIDERS"].keys(): + if config["PHONE_ACCELEROMETER"]["PROVIDERS"][provider]["COMPUTE"]: + files_to_compute.extend(expand("data/raw/{pid}/phone_accelerometer_raw.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/raw/{pid}/phone_accelerometer_with_datetime.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/phone_accelerometer_features/phone_accelerometer_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_ACCELEROMETER"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/processed/features/{pid}/phone_accelerometer.csv", pid=config["PIDS"])) + +for provider in config["PHONE_APPLICATIONS_FOREGROUND"]["PROVIDERS"].keys(): + if config["PHONE_APPLICATIONS_FOREGROUND"]["PROVIDERS"][provider]["COMPUTE"]: + files_to_compute.extend(expand("data/raw/{pid}/phone_applications_foreground_raw.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/raw/{pid}/phone_applications_foreground_with_datetime.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/raw/{pid}/phone_applications_foreground_with_datetime_with_categories.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/phone_applications_foreground_features/phone_applications_foreground_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_APPLICATIONS_FOREGROUND"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/processed/features/{pid}/phone_applications_foreground.csv", pid=config["PIDS"])) + +for provider in config["PHONE_WIFI_VISIBLE"]["PROVIDERS"].keys(): + if config["PHONE_WIFI_VISIBLE"]["PROVIDERS"][provider]["COMPUTE"]: + files_to_compute.extend(expand("data/raw/{pid}/phone_wifi_visible_raw.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/raw/{pid}/phone_wifi_visible_with_datetime.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/phone_wifi_visible_features/phone_wifi_visible_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_WIFI_VISIBLE"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/processed/features/{pid}/phone_wifi_visible.csv", pid=config["PIDS"])) + +for provider in config["PHONE_WIFI_CONNECTED"]["PROVIDERS"].keys(): + if config["PHONE_WIFI_CONNECTED"]["PROVIDERS"][provider]["COMPUTE"]: + files_to_compute.extend(expand("data/raw/{pid}/phone_wifi_connected_raw.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/raw/{pid}/phone_wifi_connected_with_datetime.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/phone_wifi_connected_features/phone_wifi_connected_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_WIFI_CONNECTED"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/processed/features/{pid}/phone_wifi_connected.csv", pid=config["PIDS"])) + +if config["HEARTRATE"]["COMPUTE"]: + files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=config["HEARTRATE"]["DB_TABLE"])) + files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_{fitbit_data_type}_with_datetime.csv", pid=config["PIDS"], fitbit_data_type=["summary", "intraday"])) + files_to_compute.extend(expand("data/processed/{pid}/fitbit_heartrate_{day_segment}.csv", pid = config["PIDS"], day_segment = config["HEARTRATE"]["DAY_SEGMENTS"])) + +if config["STEP"]["COMPUTE"]: + if config["STEP"]["EXCLUDE_SLEEP"]["EXCLUDE"] == True and config["STEP"]["EXCLUDE_SLEEP"]["TYPE"] == "FITBIT_BASED": + files_to_compute.extend(expand("data/raw/{pid}/fitbit_sleep_{fitbit_data_type}_with_datetime.csv", pid=config["PIDS"], fitbit_data_type=["summary"])) + files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=config["STEP"]["DB_TABLE"])) + files_to_compute.extend(expand("data/raw/{pid}/fitbit_step_{fitbit_data_type}_with_datetime.csv", pid=config["PIDS"], fitbit_data_type=["intraday"])) + files_to_compute.extend(expand("data/processed/{pid}/fitbit_step_{day_segment}.csv", pid = config["PIDS"], day_segment = config["STEP"]["DAY_SEGMENTS"])) + +if config["SLEEP"]["COMPUTE"]: + files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=config["SLEEP"]["DB_TABLE"])) + files_to_compute.extend(expand("data/raw/{pid}/fitbit_sleep_{fitbit_data_type}_with_datetime.csv", pid=config["PIDS"], fitbit_data_type=["intraday", "summary"])) + files_to_compute.extend(expand("data/processed/{pid}/fitbit_sleep_{day_segment}.csv", pid = config["PIDS"], day_segment = config["SLEEP"]["DAY_SEGMENTS"])) + +for provider in config["PHONE_CONVERSATION"]["PROVIDERS"].keys(): + if config["PHONE_CONVERSATION"]["PROVIDERS"][provider]["COMPUTE"]: + files_to_compute.extend(expand("data/raw/{pid}/phone_conversation_raw.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/raw/{pid}/phone_conversation_with_datetime.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/raw/{pid}/phone_conversation_with_datetime_unified.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/phone_conversation_features/phone_conversation_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_CONVERSATION"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/processed/features/{pid}/phone_conversation.csv", pid=config["PIDS"])) + +for provider in config["PHONE_LOCATIONS"]["PROVIDERS"].keys(): + if config["PHONE_LOCATIONS"]["PROVIDERS"][provider]["COMPUTE"]: + if config["PHONE_LOCATIONS"]["LOCATIONS_TO_USE"] == "RESAMPLE_FUSED": + if config["PHONE_LOCATIONS"]["DB_TABLE"] in config["PHONE_VALID_SENSED_BINS"]["DB_TABLES"]: + files_to_compute.extend(expand("data/interim/{pid}/phone_sensed_bins.csv", pid=config["PIDS"])) + else: + raise ValueError("Error: Add your locations table (and as many sensor tables as you have) to [PHONE_VALID_SENSED_BINS][DB_TABLES] in config.yaml. This is necessary to compute phone_sensed_bins (bins of time when the smartphone was sensing data) which is used to resample fused location data (RESAMPLED_FUSED)") + + files_to_compute.extend(expand("data/raw/{pid}/phone_locations_raw.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/phone_locations_processed.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/phone_locations_processed_with_datetime.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/phone_locations_features/phone_locations_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_LOCATIONS"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/processed/features/{pid}/phone_locations.csv", pid=config["PIDS"])) + +# visualization for data exploration +if config["HEATMAP_FEATURES_CORRELATIONS"]["PLOT"]: + files_to_compute.extend(expand("reports/data_exploration/{min_valid_hours_per_day}hours_{min_valid_bins_per_hour}bins/heatmap_features_correlations.html", min_valid_hours_per_day=config["HEATMAP_FEATURES_CORRELATIONS"]["MIN_VALID_HOURS_PER_DAY"], min_valid_bins_per_hour=config["PHONE_VALID_SENSED_DAYS"]["MIN_VALID_BINS_PER_HOUR"])) + +if config["HISTOGRAM_VALID_SENSED_HOURS"]["PLOT"]: + files_to_compute.extend(expand("reports/data_exploration/{min_valid_hours_per_day}hours_{min_valid_bins_per_hour}bins/histogram_valid_sensed_hours.html", min_valid_hours_per_day=config["HISTOGRAM_VALID_SENSED_HOURS"]["MIN_VALID_HOURS_PER_DAY"], min_valid_bins_per_hour=config["PHONE_VALID_SENSED_DAYS"]["MIN_VALID_BINS_PER_HOUR"])) + +if config["HEATMAP_DAYS_BY_SENSORS"]["PLOT"]: + files_to_compute.extend(expand("reports/interim/{min_valid_hours_per_day}hours_{min_valid_bins_per_hour}bins/{pid}/heatmap_days_by_sensors.html", pid=config["PIDS"], min_valid_hours_per_day=config["HEATMAP_DAYS_BY_SENSORS"]["MIN_VALID_HOURS_PER_DAY"], min_valid_bins_per_hour=config["PHONE_VALID_SENSED_DAYS"]["MIN_VALID_BINS_PER_HOUR"])) + files_to_compute.extend(expand("reports/data_exploration/{min_valid_hours_per_day}hours_{min_valid_bins_per_hour}bins/heatmap_days_by_sensors_all_participants.html", min_valid_hours_per_day=config["HEATMAP_DAYS_BY_SENSORS"]["MIN_VALID_HOURS_PER_DAY"], min_valid_bins_per_hour=config["PHONE_VALID_SENSED_DAYS"]["MIN_VALID_BINS_PER_HOUR"])) + +if config["HEATMAP_SENSED_BINS"]["PLOT"]: + files_to_compute.extend(expand("reports/interim/heatmap_sensed_bins/{pid}/heatmap_sensed_bins.html", pid=config["PIDS"])) + files_to_compute.extend(["reports/data_exploration/heatmap_sensed_bins_all_participants.html"]) + +if config["OVERALL_COMPLIANCE_HEATMAP"]["PLOT"]: + files_to_compute.extend(expand("reports/data_exploration/{min_valid_hours_per_day}hours_{min_valid_bins_per_hour}bins/overall_compliance_heatmap.html", min_valid_hours_per_day=config["OVERALL_COMPLIANCE_HEATMAP"]["MIN_VALID_HOURS_PER_DAY"], min_valid_bins_per_hour=config["PHONE_VALID_SENSED_DAYS"]["MIN_VALID_BINS_PER_HOUR"])) -for provider in config["CONVERSATION"]["PROVIDERS"].keys(): - if config["CONVERSATION"]["PROVIDERS"][provider]["COMPUTE"]: - pids_android = list(filter(lambda pid: infer_participant_platform("data/external/" + pid) == "android", config["PIDS"])) - pids_ios = list(filter(lambda pid: infer_participant_platform("data/external/" + pid) == "ios", config["PIDS"])) - for pids,table in zip([pids_android, pids_ios], [config["CONVERSATION"]["DB_TABLE"]["ANDROID"], config["CONVERSATION"]["DB_TABLE"]["IOS"]]): - files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=pids, sensor=table)) - files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime.csv", pid=pids, sensor=table)) - files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime_unified.csv", pid=pids, sensor=table)) - files_to_compute.extend(expand("data/interim/{pid}/{sensor_key}_features/{sensor_key}_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["CONVERSATION"]["PROVIDERS"][provider]["SRC_LANGUAGE"], provider_key=provider, sensor_key="CONVERSATION".lower())) - files_to_compute.extend(expand("data/processed/features/{pid}/{sensor_key}.csv", pid=config["PIDS"], sensor_key="CONVERSATION".lower())) rule all: input: diff --git a/tests/data/processed/features/frequency/test03/messages.csv b/tests/data/processed/features/frequency/test03/phone_messages.csv similarity index 100% rename from tests/data/processed/features/frequency/test03/messages.csv rename to tests/data/processed/features/frequency/test03/phone_messages.csv diff --git a/tests/data/processed/features/frequency/test04/messages.csv b/tests/data/processed/features/frequency/test04/phone_messages.csv similarity index 100% rename from tests/data/processed/features/frequency/test04/messages.csv rename to tests/data/processed/features/frequency/test04/phone_messages.csv diff --git a/tests/data/processed/features/frequency/test05/messages.csv b/tests/data/processed/features/frequency/test05/phone_messages.csv similarity index 100% rename from tests/data/processed/features/frequency/test05/messages.csv rename to tests/data/processed/features/frequency/test05/phone_messages.csv diff --git a/tests/data/processed/features/frequency/test06/messages.csv b/tests/data/processed/features/frequency/test06/phone_messages.csv similarity index 100% rename from tests/data/processed/features/frequency/test06/messages.csv rename to tests/data/processed/features/frequency/test06/phone_messages.csv diff --git a/tests/data/processed/features/periodic/test01/messages.csv b/tests/data/processed/features/periodic/test01/phone_messages.csv similarity index 100% rename from tests/data/processed/features/periodic/test01/messages.csv rename to tests/data/processed/features/periodic/test01/phone_messages.csv diff --git a/tests/data/processed/features/periodic/test02/messages.csv b/tests/data/processed/features/periodic/test02/phone_messages.csv similarity index 100% rename from tests/data/processed/features/periodic/test02/messages.csv rename to tests/data/processed/features/periodic/test02/phone_messages.csv diff --git a/tests/data/processed/features/periodic/test03/messages.csv b/tests/data/processed/features/periodic/test03/phone_messages.csv similarity index 100% rename from tests/data/processed/features/periodic/test03/messages.csv rename to tests/data/processed/features/periodic/test03/phone_messages.csv diff --git a/tests/data/processed/features/periodic/test04/messages.csv b/tests/data/processed/features/periodic/test04/phone_messages.csv similarity index 100% rename from tests/data/processed/features/periodic/test04/messages.csv rename to tests/data/processed/features/periodic/test04/phone_messages.csv diff --git a/tests/data/raw/test01/messages_raw.csv b/tests/data/raw/test01/phone_messages_raw.csv similarity index 100% rename from tests/data/raw/test01/messages_raw.csv rename to tests/data/raw/test01/phone_messages_raw.csv diff --git a/tests/data/raw/test02/messages_raw.csv b/tests/data/raw/test02/phone_messages_raw.csv similarity index 100% rename from tests/data/raw/test02/messages_raw.csv rename to tests/data/raw/test02/phone_messages_raw.csv diff --git a/tests/data/raw/test03/messages_raw.csv b/tests/data/raw/test03/phone_messages_raw.csv similarity index 100% rename from tests/data/raw/test03/messages_raw.csv rename to tests/data/raw/test03/phone_messages_raw.csv diff --git a/tests/data/raw/test04/messages_raw.csv b/tests/data/raw/test04/phone_messages_raw.csv similarity index 100% rename from tests/data/raw/test04/messages_raw.csv rename to tests/data/raw/test04/phone_messages_raw.csv diff --git a/tests/data/raw/test05/messages_raw.csv b/tests/data/raw/test05/phone_messages_raw.csv similarity index 100% rename from tests/data/raw/test05/messages_raw.csv rename to tests/data/raw/test05/phone_messages_raw.csv diff --git a/tests/data/raw/test06/messages_raw.csv b/tests/data/raw/test06/phone_messages_raw.csv similarity index 100% rename from tests/data/raw/test06/messages_raw.csv rename to tests/data/raw/test06/phone_messages_raw.csv diff --git a/tests/settings/frequency/testing_config.yaml b/tests/settings/frequency/testing_config.yaml index a1106b64..ec3168f7 100644 --- a/tests/settings/frequency/testing_config.yaml +++ b/tests/settings/frequency/testing_config.yaml @@ -6,12 +6,46 @@ PIDS: [test03, test04,test05, test06] DAY_SEGMENTS: &day_segments TYPE: FREQUENCY # FREQUENCY, PERIODIC, EVENT FILE: "data/external/daysegments_frequency.csv" + INCLUDE_PAST_PERIODIC_SEGMENTS: FALSE # Only relevant if TYPE=PERIODIC, if set to TRUE we consider day segments back enough in the past as to include the first day of data + +# Global timezone +# Use codes from https://en.wikipedia.org/wiki/List_of_tz_database_time_zones +# Double check your code, for example EST is not US Eastern Time. +TIMEZONE: &timezone + America/New_York + +DATABASE_GROUP: &database_group + MY_GROUP + +DOWNLOAD_PARTICIPANTS: + IGNORED_DEVICE_IDS: [] # for example "5a1dd68c-6cd1-48fe-ae1e-14344ac5215f" + GROUP: *database_group + +# Download data config +DOWNLOAD_DATASET: + GROUP: *database_group + +# Readable datetime config +READABLE_DATETIME: + FIXED_TIMEZONE: *timezone PHONE_VALID_SENSED_BINS: - DB_TABLES: [bluetooth] + COMPUTE: False # This flag is automatically ignored (set to True) if you are extracting PHONE_VALID_SENSED_DAYS or screen or Barnett's location features + BIN_SIZE: &bin_size 5 # (in minutes) + # Add as many PHONE sensors as you have, they all improve the computation of PHONE_VALID_SENSED_BINS and PHONE_VALID_SENSED_DAYS. + # If you are extracting screen or Barnett/Doryab location features, PHONE_SCREEN and PHONE_LOCATIONS tables are mandatory. + # You can choose any of the keys shown below, just make sure its DB_TABLE exists in your database! + # PHONE_MESSAGES, PHONE_CALLS, PHONE_LOCATIONS, PHONE_BLUETOOTH, PHONE_ACTIVITY_RECOGNITION, PHONE_BATTERY, PHONE_SCREEN, PHONE_LIGHT, + # PHONE_ACCELEROMETER, PHONE_APPLICATIONS_FOREGROUND, PHONE_WIFI_VISIBLE, PHONE_WIFI_CONNECTED, PHONE_CONVERSATION + PHONE_SENSORS: [] + +PHONE_VALID_SENSED_DAYS: + COMPUTE: False + MIN_VALID_HOURS_PER_DAY: &min_valid_hours_per_day [16] # (out of 24) MIN_HOURS_PER_DAY + MIN_VALID_BINS_PER_HOUR: &min_valid_bins_per_hour [6] # (out of 60min/BIN_SIZE bins) # Communication SMS features config, TYPES and FEATURES keys need to match -MESSAGES: +PHONE_MESSAGES: DB_TABLE: messages PROVIDERS: RAPIDS: @@ -21,68 +55,163 @@ MESSAGES: received: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact] sent: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact] SRC_LANGUAGE: "r" - SRC_FOLDER: "rapids" # inside src/features/messages + SRC_FOLDER: "rapids" # inside src/features/phone_messages # Communication call features config, TYPES and FEATURES keys need to match -CALLS: +PHONE_CALLS: DB_TABLE: calls PROVIDERS: RAPIDS: - COMPUTE: True + COMPUTE: False CALL_TYPES: [missed, incoming, outgoing] FEATURES: missed: [count, distinctcontacts, timefirstcall, timelastcall, countmostfrequentcontact] incoming: [count, distinctcontacts, meanduration, sumduration, minduration, maxduration, stdduration, modeduration, entropyduration, timefirstcall, timelastcall, countmostfrequentcontact] outgoing: [count, distinctcontacts, meanduration, sumduration, minduration, maxduration, stdduration, modeduration, entropyduration, timefirstcall, timelastcall, countmostfrequentcontact] SRC_LANGUAGE: "r" - SRC_FOLDER: "rapids" # inside src/features/calls + SRC_FOLDER: "rapids" # inside src/features/phone_calls -BLUETOOTH: +PHONE_LOCATIONS: + DB_TABLE: locations + LOCATIONS_TO_USE: FUSED_RESAMPLED # ALL, GPS OR FUSED_RESAMPLED + FUSED_RESAMPLED_CONSECUTIVE_THRESHOLD: 30 # minutes, only replicate location samples to the next sensed bin if the phone did not stop collecting data for more than this threshold + FUSED_RESAMPLED_TIME_SINCE_VALID_LOCATION: 720 # minutes, only replicate location samples to consecutive sensed bins if they were logged within this threshold after a valid location row + PROVIDERS: + DORYAB: + COMPUTE: False + FEATURES: ["locationvariance","loglocationvariance","totaldistance","averagespeed","varspeed","circadianmovement","numberofsignificantplaces","numberlocationtransitions","radiusgyration","timeattop1location","timeattop2location","timeattop3location","movingtostaticratio","outlierstimepercent","maxlengthstayatclusters","minlengthstayatclusters","meanlengthstayatclusters","stdlengthstayatclusters","locationentropy","normalizedlocationentropy"] + DBSCAN_EPS: 10 # meters + DBSCAN_MINSAMPLES: 5 + THRESHOLD_STATIC : 1 # km/h + MAXIMUM_GAP_ALLOWED: 300 + MINUTES_DATA_USED: False + SAMPLING_FREQUENCY: 0 + SRC_FOLDER: "doryab" # inside src/features/phone_locations + SRC_LANGUAGE: "python" + + BARNETT: + COMPUTE: False + FEATURES: ["hometime","disttravelled","rog","maxdiam","maxhomedist","siglocsvisited","avgflightlen","stdflightlen","avgflightdur","stdflightdur","probpause","siglocentropy","circdnrtn","wkenddayrtn"] + ACCURACY_LIMIT: 51 # meters, drops location coordinates with an accuracy higher than this. This number means there's a 68% probability the true location is within this radius + TIMEZONE: *timezone + MINUTES_DATA_USED: False # Use this for quality control purposes, how many minutes of data (location coordinates gruped by minute) were used to compute features + SRC_FOLDER: "barnett" # inside src/features/phone_locations + SRC_LANGUAGE: "r" + +PHONE_BLUETOOTH: DB_TABLE: bluetooth PROVIDERS: RAPIDS: - COMPUTE: True + COMPUTE: False FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] - SRC_FOLDER: "rapids" # inside src/features/bluetooth + SRC_FOLDER: "rapids" # inside src/features/phone_bluetooth SRC_LANGUAGE: "r" -LIGHT: + +PHONE_ACTIVITY_RECOGNITION: + DB_TABLE: + ANDROID: plugin_google_activity_recognition + IOS: plugin_ios_activity_recognition + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["count", "mostcommonactivity", "countuniqueactivities", "durationstationary", "durationmobile", "durationvehicle"] + ACTIVITY_CLASSES: + STATIONARY: ["still", "tilting"] + MOBILE: ["on_foot", "walking", "running", "on_bicycle"] + VEHICLE: ["in_vehicle"] + SRC_FOLDER: "rapids" # inside src/features/phone_activity_recognition + SRC_LANGUAGE: "python" + +PHONE_BATTERY: + DB_TABLE: battery + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["countdischarge", "sumdurationdischarge", "countcharge", "sumdurationcharge", "avgconsumptionrate", "maxconsumptionrate"] + SRC_FOLDER: "rapids" # inside src/features/phone_battery + SRC_LANGUAGE: "python" + +PHONE_SCREEN: + DB_TABLE: screen + PROVIDERS: + RAPIDS: + COMPUTE: False + REFERENCE_HOUR_FIRST_USE: 0 + IGNORE_EPISODES_SHORTER_THAN: 0 # in minutes, set to 0 to disable + IGNORE_EPISODES_LONGER_THAN: 0 # in minutes, set to 0 to disable + FEATURES: ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "stdduration", "firstuseafter"] # "episodepersensedminutes" needs to be added later + EPISODE_TYPES: ["unlock"] + SRC_FOLDER: "rapids" # inside src/features/phone_screen + SRC_LANGUAGE: "python" + +PHONE_LIGHT: DB_TABLE: light PROVIDERS: RAPIDS: - COMPUTE: True + COMPUTE: False FEATURES: ["count", "maxlux", "minlux", "avglux", "medianlux", "stdlux"] - SRC_FOLDER: "rapids" # inside src/features/light + SRC_FOLDER: "rapids" # inside src/features/phone_light SRC_LANGUAGE: "python" -WIFI: - DB_TABLE: - VISIBLE_ACCESS_POINTS: "wifi" # if you only have a CONNECTED_ACCESS_POINTS table, set this value to "" - CONNECTED_ACCESS_POINTS: "sensor_wifi" # if you only have a VISIBLE_ACCESS_POINTS table, set this value to "" +PHONE_ACCELEROMETER: + DB_TABLE: accelerometer PROVIDERS: RAPIDS: - COMPUTE: True - FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] - SRC_FOLDER: "rapids" # inside src/features/bluetooth - SRC_LANGUAGE: "r" + COMPUTE: False + FEATURES: ["maxmagnitude", "minmagnitude", "avgmagnitude", "medianmagnitude", "stdmagnitude"] + SRC_FOLDER: "rapids" # inside src/features/phone_accelerometer + SRC_LANGUAGE: "python" + + PANDA: + COMPUTE: False + VALID_SENSED_MINUTES: False + FEATURES: + exertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"] + nonexertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"] + SRC_FOLDER: "panda" # inside src/features/phone_accelerometer + SRC_LANGUAGE: "python" -APPLICATIONS_FOREGROUND: +PHONE_APPLICATIONS_FOREGROUND: DB_TABLE: applications_foreground + APPLICATION_CATEGORIES: + CATALOGUE_SOURCE: FILE # FILE (genres are read from CATALOGUE_FILE) or GOOGLE (genres are scrapped from the Play Store) + CATALOGUE_FILE: "data/external/stachl_application_genre_catalogue.csv" + UPDATE_CATALOGUE_FILE: False # if CATALOGUE_SOURCE is equal to FILE, whether or not to update CATALOGUE_FILE, if CATALOGUE_SOURCE is equal to GOOGLE all scraped genres will be saved to CATALOGUE_FILE + SCRAPE_MISSING_CATEGORIES: False # whether or not to scrape missing genres, only effective if CATALOGUE_SOURCE is equal to FILE. If CATALOGUE_SOURCE is equal to GOOGLE, all genres are scraped anyway PROVIDERS: RAPIDS: - COMPUTE: True + COMPUTE: False SINGLE_CATEGORIES: ["all", "email"] MULTIPLE_CATEGORIES: social: ["socialnetworks", "socialmediatools"] - entertainment: ["entertainment", "gamingstrategy"] - SINGLE_APPS: ["top1global", "com.facebook.moments"] # There's no entropy for single apps - EXCLUDED_CATEGORIES: ["systemapp", "tvvideoapps"] + entertainment: ["entertainment", "gamingknowledge", "gamingcasual", "gamingadventure", "gamingstrategy", "gamingtoolscommunity", "gamingroleplaying", "gamingaction", "gaminglogic", "gamingsports", "gamingsimulation"] + SINGLE_APPS: ["top1global", "com.facebook.moments", "com.google.android.youtube", "com.twitter.android"] # There's no entropy for single apps + EXCLUDED_CATEGORIES: [] EXCLUDED_APPS: ["com.fitbit.FitbitMobile", "com.aware.plugin.upmc.cancer"] FEATURES: ["count", "timeoffirstuse", "timeoflastuse", "frequencyentropy"] - SRC_FOLDER: "rapids" # inside src/features/applications_foreground + SRC_FOLDER: "rapids" # inside src/features/phone_applications_foreground SRC_LANGUAGE: "python" -CONVERSATION: +PHONE_WIFI_VISIBLE: + DB_TABLE: "wifi" + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] + SRC_FOLDER: "rapids" # inside src/features/phone_wifi_visible + SRC_LANGUAGE: "r" + +PHONE_WIFI_CONNECTED: + DB_TABLE: "sensor_wifi" + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] + SRC_FOLDER: "rapids" # inside src/features/phone_wifi_connected + SRC_LANGUAGE: "r" + +PHONE_CONVERSATION: DB_TABLE: ANDROID: plugin_studentlife_audio_android IOS: plugin_studentlife_audio @@ -90,12 +219,79 @@ CONVERSATION: RAPIDS: COMPUTE: False FEATURES: ["minutessilence", "minutesnoise", "minutesvoice", "minutesunknown","sumconversationduration","avgconversationduration", - "sdconversationduration","minconversationduration","maxconversationduration","timefirstconversation","timelastconversation","noisesumenergy", - "noiseavgenergy","noisesdenergy","noiseminenergy","noisemaxenergy","voicesumenergy", - "voiceavgenergy","voicesdenergy","voiceminenergy","voicemaxenergy","silencesensedfraction","noisesensedfraction", + "sdconversationduration","minconversationduration","maxconversationduration","timefirstconversation","timelastconversation","sumenergy", + "avgenergy","sdenergy","minenergy","maxenergy","silencesensedfraction","noisesensedfraction", "voicesensedfraction","unknownsensedfraction","silenceexpectedfraction","noiseexpectedfraction","voiceexpectedfraction", "unknownexpectedfraction","countconversation"] RECORDING_MINUTES: 1 PAUSED_MINUTES : 3 - SRC_FOLDER: "rapids" # inside src/features/conversation + SRC_FOLDER: "rapids" # inside src/features/phone_conversation SRC_LANGUAGE: "python" + + +HEARTRATE: + COMPUTE: False + DB_TABLE: fitbit_data + DAY_SEGMENTS: *day_segments + SUMMARY_FEATURES: ["restinghr"] # calories features' accuracy depend on the accuracy of the participants fitbit profile (e.g. heigh, weight) use with care: ["caloriesoutofrange", "caloriesfatburn", "caloriescardio", "caloriespeak"] + INTRADAY_FEATURES: ["maxhr", "minhr", "avghr", "medianhr", "modehr", "stdhr", "diffmaxmodehr", "diffminmodehr", "entropyhr", "minutesonoutofrangezone", "minutesonfatburnzone", "minutesoncardiozone", "minutesonpeakzone"] + +STEP: + COMPUTE: False + DB_TABLE: fitbit_data + DAY_SEGMENTS: *day_segments + EXCLUDE_SLEEP: + EXCLUDE: False + TYPE: FIXED # FIXED OR FITBIT_BASED (CONFIGURE FITBIT's SLEEP DB_TABLE) + FIXED: + START: "23:00" + END: "07:00" + FEATURES: + ALL_STEPS: ["sumallsteps", "maxallsteps", "minallsteps", "avgallsteps", "stdallsteps"] + SEDENTARY_BOUT: ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "stdduration"] + ACTIVE_BOUT: ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "stdduration"] + THRESHOLD_ACTIVE_BOUT: 10 # steps + INCLUDE_ZERO_STEP_ROWS: False + +SLEEP: + COMPUTE: False + DB_TABLE: fitbit_data + DAY_SEGMENTS: *day_segments + SLEEP_TYPES: ["main", "nap", "all"] + SUMMARY_FEATURES: ["sumdurationafterwakeup", "sumdurationasleep", "sumdurationawake", "sumdurationtofallasleep", "sumdurationinbed", "avgefficiency", "countepisode"] + +### Visualizations ################################################################ +HEATMAP_FEATURES_CORRELATIONS: + PLOT: False + MIN_ROWS_RATIO: 0.5 + MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day + MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour + PHONE_FEATURES: [accelerometer, activity_recognition, applications_foreground, battery, calls_incoming, calls_missed, calls_outgoing, conversation, light, location_doryab, messages_received, messages_sent, screen] + FITBIT_FEATURES: [fitbit_heartrate, fitbit_step, fitbit_sleep] + CORR_THRESHOLD: 0.1 + CORR_METHOD: "pearson" # choose from {"pearson", "kendall", "spearman"} + +HISTOGRAM_VALID_SENSED_HOURS: + PLOT: False + MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day + MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour + +HEATMAP_DAYS_BY_SENSORS: + PLOT: False + MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day + MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour + EXPECTED_NUM_OF_DAYS: -1 + DB_TABLES: [accelerometer, applications_foreground, battery, bluetooth, calls, light, locations, messages, screen, wifi, sensor_wifi, plugin_google_activity_recognition, plugin_ios_activity_recognition, plugin_studentlife_audio_android, plugin_studentlife_audio] + +HEATMAP_SENSED_BINS: + PLOT: False + BIN_SIZE: *bin_size + +OVERALL_COMPLIANCE_HEATMAP: + PLOT: False + ONLY_SHOW_VALID_DAYS: False + EXPECTED_NUM_OF_DAYS: -1 + BIN_SIZE: *bin_size + MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day + MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour + diff --git a/tests/settings/periodic/testing_config.yaml b/tests/settings/periodic/testing_config.yaml index 6351ba15..e244bb00 100644 --- a/tests/settings/periodic/testing_config.yaml +++ b/tests/settings/periodic/testing_config.yaml @@ -2,17 +2,50 @@ # You must create a file for each participant named pXXX containing their device_id. This can be done manually or automatically PIDS: [test01, test02, test03, test04] - # Global var with common day segments DAY_SEGMENTS: &day_segments TYPE: PERIODIC # FREQUENCY, PERIODIC, EVENT FILE: "data/external/daysegments_periodic.csv" + INCLUDE_PAST_PERIODIC_SEGMENTS: FALSE # Only relevant if TYPE=PERIODIC, if set to TRUE we consider day segments back enough in the past as to include the first day of data + +# Global timezone +# Use codes from https://en.wikipedia.org/wiki/List_of_tz_database_time_zones +# Double check your code, for example EST is not US Eastern Time. +TIMEZONE: &timezone + America/New_York + +DATABASE_GROUP: &database_group + MY_GROUP + +DOWNLOAD_PARTICIPANTS: + IGNORED_DEVICE_IDS: [] # for example "5a1dd68c-6cd1-48fe-ae1e-14344ac5215f" + GROUP: *database_group + +# Download data config +DOWNLOAD_DATASET: + GROUP: *database_group + +# Readable datetime config +READABLE_DATETIME: + FIXED_TIMEZONE: *timezone PHONE_VALID_SENSED_BINS: - DB_TABLES: [messages, calls, bluetooth, light, wifi, applications_foreground] + COMPUTE: False # This flag is automatically ignored (set to True) if you are extracting PHONE_VALID_SENSED_DAYS or screen or Barnett's location features + BIN_SIZE: &bin_size 5 # (in minutes) + # Add as many PHONE sensors as you have, they all improve the computation of PHONE_VALID_SENSED_BINS and PHONE_VALID_SENSED_DAYS. + # If you are extracting screen or Barnett/Doryab location features, PHONE_SCREEN and PHONE_LOCATIONS tables are mandatory. + # You can choose any of the keys shown below, just make sure its DB_TABLE exists in your database! + # PHONE_MESSAGES, PHONE_CALLS, PHONE_LOCATIONS, PHONE_BLUETOOTH, PHONE_ACTIVITY_RECOGNITION, PHONE_BATTERY, PHONE_SCREEN, PHONE_LIGHT, + # PHONE_ACCELEROMETER, PHONE_APPLICATIONS_FOREGROUND, PHONE_WIFI_VISIBLE, PHONE_WIFI_CONNECTED, PHONE_CONVERSATION + PHONE_SENSORS: [] + +PHONE_VALID_SENSED_DAYS: + COMPUTE: False + MIN_VALID_HOURS_PER_DAY: &min_valid_hours_per_day [16] # (out of 24) MIN_HOURS_PER_DAY + MIN_VALID_BINS_PER_HOUR: &min_valid_bins_per_hour [6] # (out of 60min/BIN_SIZE bins) # Communication SMS features config, TYPES and FEATURES keys need to match -MESSAGES: +PHONE_MESSAGES: DB_TABLE: messages PROVIDERS: RAPIDS: @@ -22,68 +55,163 @@ MESSAGES: received: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact] sent: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact] SRC_LANGUAGE: "r" - SRC_FOLDER: "rapids" # inside src/features/messages + SRC_FOLDER: "rapids" # inside src/features/phone_messages # Communication call features config, TYPES and FEATURES keys need to match -CALLS: +PHONE_CALLS: DB_TABLE: calls PROVIDERS: RAPIDS: - COMPUTE: True + COMPUTE: False CALL_TYPES: [missed, incoming, outgoing] FEATURES: missed: [count, distinctcontacts, timefirstcall, timelastcall, countmostfrequentcontact] incoming: [count, distinctcontacts, meanduration, sumduration, minduration, maxduration, stdduration, modeduration, entropyduration, timefirstcall, timelastcall, countmostfrequentcontact] outgoing: [count, distinctcontacts, meanduration, sumduration, minduration, maxduration, stdduration, modeduration, entropyduration, timefirstcall, timelastcall, countmostfrequentcontact] SRC_LANGUAGE: "r" - SRC_FOLDER: "rapids" # inside src/features/calls + SRC_FOLDER: "rapids" # inside src/features/phone_calls -BLUETOOTH: +PHONE_LOCATIONS: + DB_TABLE: locations + LOCATIONS_TO_USE: FUSED_RESAMPLED # ALL, GPS OR FUSED_RESAMPLED + FUSED_RESAMPLED_CONSECUTIVE_THRESHOLD: 30 # minutes, only replicate location samples to the next sensed bin if the phone did not stop collecting data for more than this threshold + FUSED_RESAMPLED_TIME_SINCE_VALID_LOCATION: 720 # minutes, only replicate location samples to consecutive sensed bins if they were logged within this threshold after a valid location row + PROVIDERS: + DORYAB: + COMPUTE: False + FEATURES: ["locationvariance","loglocationvariance","totaldistance","averagespeed","varspeed","circadianmovement","numberofsignificantplaces","numberlocationtransitions","radiusgyration","timeattop1location","timeattop2location","timeattop3location","movingtostaticratio","outlierstimepercent","maxlengthstayatclusters","minlengthstayatclusters","meanlengthstayatclusters","stdlengthstayatclusters","locationentropy","normalizedlocationentropy"] + DBSCAN_EPS: 10 # meters + DBSCAN_MINSAMPLES: 5 + THRESHOLD_STATIC : 1 # km/h + MAXIMUM_GAP_ALLOWED: 300 + MINUTES_DATA_USED: False + SAMPLING_FREQUENCY: 0 + SRC_FOLDER: "doryab" # inside src/features/phone_locations + SRC_LANGUAGE: "python" + + BARNETT: + COMPUTE: False + FEATURES: ["hometime","disttravelled","rog","maxdiam","maxhomedist","siglocsvisited","avgflightlen","stdflightlen","avgflightdur","stdflightdur","probpause","siglocentropy","circdnrtn","wkenddayrtn"] + ACCURACY_LIMIT: 51 # meters, drops location coordinates with an accuracy higher than this. This number means there's a 68% probability the true location is within this radius + TIMEZONE: *timezone + MINUTES_DATA_USED: False # Use this for quality control purposes, how many minutes of data (location coordinates gruped by minute) were used to compute features + SRC_FOLDER: "barnett" # inside src/features/phone_locations + SRC_LANGUAGE: "r" + +PHONE_BLUETOOTH: DB_TABLE: bluetooth PROVIDERS: RAPIDS: - COMPUTE: True + COMPUTE: False FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] - SRC_FOLDER: "rapids" # inside src/features/bluetooth + SRC_FOLDER: "rapids" # inside src/features/phone_bluetooth SRC_LANGUAGE: "r" -LIGHT: + +PHONE_ACTIVITY_RECOGNITION: + DB_TABLE: + ANDROID: plugin_google_activity_recognition + IOS: plugin_ios_activity_recognition + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["count", "mostcommonactivity", "countuniqueactivities", "durationstationary", "durationmobile", "durationvehicle"] + ACTIVITY_CLASSES: + STATIONARY: ["still", "tilting"] + MOBILE: ["on_foot", "walking", "running", "on_bicycle"] + VEHICLE: ["in_vehicle"] + SRC_FOLDER: "rapids" # inside src/features/phone_activity_recognition + SRC_LANGUAGE: "python" + +PHONE_BATTERY: + DB_TABLE: battery + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["countdischarge", "sumdurationdischarge", "countcharge", "sumdurationcharge", "avgconsumptionrate", "maxconsumptionrate"] + SRC_FOLDER: "rapids" # inside src/features/phone_battery + SRC_LANGUAGE: "python" + +PHONE_SCREEN: + DB_TABLE: screen + PROVIDERS: + RAPIDS: + COMPUTE: False + REFERENCE_HOUR_FIRST_USE: 0 + IGNORE_EPISODES_SHORTER_THAN: 0 # in minutes, set to 0 to disable + IGNORE_EPISODES_LONGER_THAN: 0 # in minutes, set to 0 to disable + FEATURES: ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "stdduration", "firstuseafter"] # "episodepersensedminutes" needs to be added later + EPISODE_TYPES: ["unlock"] + SRC_FOLDER: "rapids" # inside src/features/phone_screen + SRC_LANGUAGE: "python" + +PHONE_LIGHT: DB_TABLE: light PROVIDERS: RAPIDS: - COMPUTE: True + COMPUTE: False FEATURES: ["count", "maxlux", "minlux", "avglux", "medianlux", "stdlux"] - SRC_FOLDER: "rapids" # inside src/features/light + SRC_FOLDER: "rapids" # inside src/features/phone_light SRC_LANGUAGE: "python" -WIFI: - DB_TABLE: - VISIBLE_ACCESS_POINTS: "wifi" # if you only have a CONNECTED_ACCESS_POINTS table, set this value to "" - CONNECTED_ACCESS_POINTS: "sensor_wifi" # if you only have a VISIBLE_ACCESS_POINTS table, set this value to "" +PHONE_ACCELEROMETER: + DB_TABLE: accelerometer PROVIDERS: RAPIDS: - COMPUTE: True - FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] - SRC_FOLDER: "rapids" # inside src/features/bluetooth - SRC_LANGUAGE: "r" + COMPUTE: False + FEATURES: ["maxmagnitude", "minmagnitude", "avgmagnitude", "medianmagnitude", "stdmagnitude"] + SRC_FOLDER: "rapids" # inside src/features/phone_accelerometer + SRC_LANGUAGE: "python" + + PANDA: + COMPUTE: False + VALID_SENSED_MINUTES: False + FEATURES: + exertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"] + nonexertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"] + SRC_FOLDER: "panda" # inside src/features/phone_accelerometer + SRC_LANGUAGE: "python" -APPLICATIONS_FOREGROUND: +PHONE_APPLICATIONS_FOREGROUND: DB_TABLE: applications_foreground + APPLICATION_CATEGORIES: + CATALOGUE_SOURCE: FILE # FILE (genres are read from CATALOGUE_FILE) or GOOGLE (genres are scrapped from the Play Store) + CATALOGUE_FILE: "data/external/stachl_application_genre_catalogue.csv" + UPDATE_CATALOGUE_FILE: False # if CATALOGUE_SOURCE is equal to FILE, whether or not to update CATALOGUE_FILE, if CATALOGUE_SOURCE is equal to GOOGLE all scraped genres will be saved to CATALOGUE_FILE + SCRAPE_MISSING_CATEGORIES: False # whether or not to scrape missing genres, only effective if CATALOGUE_SOURCE is equal to FILE. If CATALOGUE_SOURCE is equal to GOOGLE, all genres are scraped anyway PROVIDERS: RAPIDS: - COMPUTE: True + COMPUTE: False SINGLE_CATEGORIES: ["all", "email"] MULTIPLE_CATEGORIES: social: ["socialnetworks", "socialmediatools"] - entertainment: ["entertainment", "gamingstrategy"] - SINGLE_APPS: ["top1global", "com.facebook.moments"] # There's no entropy for single apps - EXCLUDED_CATEGORIES: ["systemapp", "tvvideoapps"] + entertainment: ["entertainment", "gamingknowledge", "gamingcasual", "gamingadventure", "gamingstrategy", "gamingtoolscommunity", "gamingroleplaying", "gamingaction", "gaminglogic", "gamingsports", "gamingsimulation"] + SINGLE_APPS: ["top1global", "com.facebook.moments", "com.google.android.youtube", "com.twitter.android"] # There's no entropy for single apps + EXCLUDED_CATEGORIES: [] EXCLUDED_APPS: ["com.fitbit.FitbitMobile", "com.aware.plugin.upmc.cancer"] FEATURES: ["count", "timeoffirstuse", "timeoflastuse", "frequencyentropy"] - SRC_FOLDER: "rapids" # inside src/features/applications_foreground + SRC_FOLDER: "rapids" # inside src/features/phone_applications_foreground SRC_LANGUAGE: "python" -CONVERSATION: +PHONE_WIFI_VISIBLE: + DB_TABLE: "wifi" + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] + SRC_FOLDER: "rapids" # inside src/features/phone_wifi_visible + SRC_LANGUAGE: "r" + +PHONE_WIFI_CONNECTED: + DB_TABLE: "sensor_wifi" + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] + SRC_FOLDER: "rapids" # inside src/features/phone_wifi_connected + SRC_LANGUAGE: "r" + +PHONE_CONVERSATION: DB_TABLE: ANDROID: plugin_studentlife_audio_android IOS: plugin_studentlife_audio @@ -91,12 +219,79 @@ CONVERSATION: RAPIDS: COMPUTE: False FEATURES: ["minutessilence", "minutesnoise", "minutesvoice", "minutesunknown","sumconversationduration","avgconversationduration", - "sdconversationduration","minconversationduration","maxconversationduration","timefirstconversation","timelastconversation","noisesumenergy", - "noiseavgenergy","noisesdenergy","noiseminenergy","noisemaxenergy","voicesumenergy", - "voiceavgenergy","voicesdenergy","voiceminenergy","voicemaxenergy","silencesensedfraction","noisesensedfraction", + "sdconversationduration","minconversationduration","maxconversationduration","timefirstconversation","timelastconversation","sumenergy", + "avgenergy","sdenergy","minenergy","maxenergy","silencesensedfraction","noisesensedfraction", "voicesensedfraction","unknownsensedfraction","silenceexpectedfraction","noiseexpectedfraction","voiceexpectedfraction", "unknownexpectedfraction","countconversation"] RECORDING_MINUTES: 1 PAUSED_MINUTES : 3 - SRC_FOLDER: "rapids" # inside src/features/conversation + SRC_FOLDER: "rapids" # inside src/features/phone_conversation SRC_LANGUAGE: "python" + + +HEARTRATE: + COMPUTE: False + DB_TABLE: fitbit_data + DAY_SEGMENTS: *day_segments + SUMMARY_FEATURES: ["restinghr"] # calories features' accuracy depend on the accuracy of the participants fitbit profile (e.g. heigh, weight) use with care: ["caloriesoutofrange", "caloriesfatburn", "caloriescardio", "caloriespeak"] + INTRADAY_FEATURES: ["maxhr", "minhr", "avghr", "medianhr", "modehr", "stdhr", "diffmaxmodehr", "diffminmodehr", "entropyhr", "minutesonoutofrangezone", "minutesonfatburnzone", "minutesoncardiozone", "minutesonpeakzone"] + +STEP: + COMPUTE: False + DB_TABLE: fitbit_data + DAY_SEGMENTS: *day_segments + EXCLUDE_SLEEP: + EXCLUDE: False + TYPE: FIXED # FIXED OR FITBIT_BASED (CONFIGURE FITBIT's SLEEP DB_TABLE) + FIXED: + START: "23:00" + END: "07:00" + FEATURES: + ALL_STEPS: ["sumallsteps", "maxallsteps", "minallsteps", "avgallsteps", "stdallsteps"] + SEDENTARY_BOUT: ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "stdduration"] + ACTIVE_BOUT: ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "stdduration"] + THRESHOLD_ACTIVE_BOUT: 10 # steps + INCLUDE_ZERO_STEP_ROWS: False + +SLEEP: + COMPUTE: False + DB_TABLE: fitbit_data + DAY_SEGMENTS: *day_segments + SLEEP_TYPES: ["main", "nap", "all"] + SUMMARY_FEATURES: ["sumdurationafterwakeup", "sumdurationasleep", "sumdurationawake", "sumdurationtofallasleep", "sumdurationinbed", "avgefficiency", "countepisode"] + +### Visualizations ################################################################ +HEATMAP_FEATURES_CORRELATIONS: + PLOT: False + MIN_ROWS_RATIO: 0.5 + MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day + MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour + PHONE_FEATURES: [accelerometer, activity_recognition, applications_foreground, battery, calls_incoming, calls_missed, calls_outgoing, conversation, light, location_doryab, messages_received, messages_sent, screen] + FITBIT_FEATURES: [fitbit_heartrate, fitbit_step, fitbit_sleep] + CORR_THRESHOLD: 0.1 + CORR_METHOD: "pearson" # choose from {"pearson", "kendall", "spearman"} + +HISTOGRAM_VALID_SENSED_HOURS: + PLOT: False + MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day + MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour + +HEATMAP_DAYS_BY_SENSORS: + PLOT: False + MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day + MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour + EXPECTED_NUM_OF_DAYS: -1 + DB_TABLES: [accelerometer, applications_foreground, battery, bluetooth, calls, light, locations, messages, screen, wifi, sensor_wifi, plugin_google_activity_recognition, plugin_ios_activity_recognition, plugin_studentlife_audio_android, plugin_studentlife_audio] + +HEATMAP_SENSED_BINS: + PLOT: False + BIN_SIZE: *bin_size + +OVERALL_COMPLIANCE_HEATMAP: + PLOT: False + ONLY_SHOW_VALID_DAYS: False + EXPECTED_NUM_OF_DAYS: -1 + BIN_SIZE: *bin_size + MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day + MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour +