diff --git a/.github/workflows/integrate.yaml b/.github/workflows/integrate.yaml new file mode 100644 index 00000000..e7407d00 --- /dev/null +++ b/.github/workflows/integrate.yaml @@ -0,0 +1,33 @@ +name: RAPIDS Continuous Integration + +on: + push: + #branches: [ develop ] + +jobs: + test-on-latest-ubuntu: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: s-weigand/setup-conda@v1 + - run : conda --version + - run : source activate base + - run : | + sudo apt update + sudo apt install libcurl4-openssl-dev + sudo apt install libssl-dev + sudo apt install libxml2-dev + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 + sudo add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu focal-cran40/' + sudo apt install r-base + sudo apt install pandoc + sudo Rscript --vanilla -e 'install.packages("rmarkdown", repos="http://cran.us.r-project.org")' + cd rapids + conda env create -f environment.yml -n test-environment + source activate test-environment + snakemake -j1 renv_install + R -e 'renv::settings$use.cache(FALSE)' + snakemake -j1 renv_restore + bash tests/scripts/run_tests.sh all test \ No newline at end of file diff --git a/tests/Snakefile b/tests/Snakefile index e4e8574c..4d78b5f2 100644 --- a/tests/Snakefile +++ b/tests/Snakefile @@ -12,20 +12,15 @@ files_to_compute = [] if len(config["PIDS"]) == 0: raise ValueError("Add participants IDs to PIDS in config.yaml. Remember to create their participant files in data/external") -if config["PHONE_VALID_SENSED_BINS"]["COMPUTE"] or config["PHONE_VALID_SENSED_DAYS"]["COMPUTE"]: # valid sensed bins is necessary for sensed days, so we add these files anyways if sensed days are requested - if len(config["PHONE_VALID_SENSED_BINS"]["PHONE_SENSORS"]) == 0: - raise ValueError("If you want to compute PHONE_VALID_SENSED_BINS or PHONE_VALID_SENSED_DAYS, you need to add at least one PHONE_SENSOR to [PHONE_VALID_SENSED_BINS][PHONE_SENSORS] in config.yaml") - - files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=map(str.lower, config["PHONE_VALID_SENSED_BINS"]["PHONE_SENSORS"]))) - files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime.csv", pid=config["PIDS"], sensor=map(str.lower, config["PHONE_VALID_SENSED_BINS"]["PHONE_SENSORS"]))) - files_to_compute.extend(expand("data/interim/{pid}/phone_sensed_bins.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_sensed_timestamps.csv", pid=config["PIDS"])) - -if config["PHONE_VALID_SENSED_DAYS"]["COMPUTE"]: - files_to_compute.extend(expand("data/interim/{pid}/phone_valid_sensed_days_{min_valid_hours_per_day}hours_{min_valid_bins_per_hour}bins.csv", - pid=config["PIDS"], - min_valid_hours_per_day=config["PHONE_VALID_SENSED_DAYS"]["MIN_VALID_HOURS_PER_DAY"], - min_valid_bins_per_hour=config["PHONE_VALID_SENSED_DAYS"]["MIN_VALID_BINS_PER_HOUR"])) +for provider in config["PHONE_DATA_YIELD"]["PROVIDERS"].keys(): + if config["PHONE_DATA_YIELD"]["PROVIDERS"][provider]["COMPUTE"]: + files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=map(str.lower, config["PHONE_DATA_YIELD"]["SENSORS"]))) + files_to_compute.extend(expand("data/interim/{pid}/phone_yielded_timestamps.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/phone_yielded_timestamps_with_datetime.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/phone_data_yield_features/phone_data_yield_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_DATA_YIELD"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/processed/features/{pid}/phone_data_yield.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) + files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") for provider in config["PHONE_MESSAGES"]["PROVIDERS"].keys(): if config["PHONE_MESSAGES"]["PROVIDERS"][provider]["COMPUTE"]: @@ -33,6 +28,8 @@ for provider in config["PHONE_MESSAGES"]["PROVIDERS"].keys(): files_to_compute.extend(expand("data/raw/{pid}/phone_messages_with_datetime.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_messages_features/phone_messages_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_MESSAGES"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_messages.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) + files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") for provider in config["PHONE_CALLS"]["PROVIDERS"].keys(): if config["PHONE_CALLS"]["PROVIDERS"][provider]["COMPUTE"]: @@ -41,6 +38,8 @@ for provider in config["PHONE_CALLS"]["PROVIDERS"].keys(): files_to_compute.extend(expand("data/raw/{pid}/phone_calls_with_datetime_unified.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_calls_features/phone_calls_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_CALLS"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_calls.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) + files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") for provider in config["PHONE_BLUETOOTH"]["PROVIDERS"].keys(): if config["PHONE_BLUETOOTH"]["PROVIDERS"][provider]["COMPUTE"]: @@ -48,6 +47,8 @@ for provider in config["PHONE_BLUETOOTH"]["PROVIDERS"].keys(): files_to_compute.extend(expand("data/raw/{pid}/phone_bluetooth_with_datetime.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_bluetooth_features/phone_bluetooth_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_BLUETOOTH"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_bluetooth.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) + files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") for provider in config["PHONE_ACTIVITY_RECOGNITION"]["PROVIDERS"].keys(): if config["PHONE_ACTIVITY_RECOGNITION"]["PROVIDERS"][provider]["COMPUTE"]: @@ -59,7 +60,8 @@ for provider in config["PHONE_ACTIVITY_RECOGNITION"]["PROVIDERS"].keys(): files_to_compute.extend(expand("data/interim/{pid}/phone_activity_recognition_episodes_resampled_with_datetime.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_activity_recognition_features/phone_activity_recognition_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_ACTIVITY_RECOGNITION"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_activity_recognition.csv", pid=config["PIDS"])) - + files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) + files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") for provider in config["PHONE_BATTERY"]["PROVIDERS"].keys(): if config["PHONE_BATTERY"]["PROVIDERS"][provider]["COMPUTE"]: @@ -69,14 +71,15 @@ for provider in config["PHONE_BATTERY"]["PROVIDERS"].keys(): files_to_compute.extend(expand("data/interim/{pid}/phone_battery_episodes_resampled_with_datetime.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_battery_features/phone_battery_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_BATTERY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_battery.csv", pid=config["PIDS"])) - + files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) + files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") for provider in config["PHONE_SCREEN"]["PROVIDERS"].keys(): if config["PHONE_SCREEN"]["PROVIDERS"][provider]["COMPUTE"]: - if "PHONE_SCREEN" in config["PHONE_VALID_SENSED_BINS"]["PHONE_SENSORS"]: - files_to_compute.extend(expand("data/interim/{pid}/phone_sensed_bins.csv", pid=config["PIDS"])) - else: - raise ValueError("Error: Add PHONE_SCREEN (and as many phone sensor as you have in your database) to [PHONE_VALID_SENSED_BINS][PHONE_SENSORS] in config.yaml. This is necessary to compute phone_sensed_bins (bins of time when the smartphone was sensing data)") + # if "PHONE_SCREEN" in config["PHONE_DATA_YIELD"]["SENSORS"]:# not used for now because we took episodepersensedminutes out of the list of supported features + # files_to_compute.extend(expand("data/interim/{pid}/phone_yielded_timestamps.csv", pid=config["PIDS"])) + # else: + # raise ValueError("Error: Add PHONE_SCREEN (and as many PHONE_SENSORS as you have in your database) to [PHONE_DATA_YIELD][SENSORS] in config.yaml. This is necessary to compute phone_yielded_timestamps (time when the smartphone was sensing data)") files_to_compute.extend(expand("data/raw/{pid}/phone_screen_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_screen_with_datetime.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_screen_with_datetime_unified.csv", pid=config["PIDS"])) @@ -85,6 +88,8 @@ for provider in config["PHONE_SCREEN"]["PROVIDERS"].keys(): files_to_compute.extend(expand("data/interim/{pid}/phone_screen_episodes_resampled_with_datetime.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_screen_features/phone_screen_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_SCREEN"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_screen.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) + files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") for provider in config["PHONE_LIGHT"]["PROVIDERS"].keys(): if config["PHONE_LIGHT"]["PROVIDERS"][provider]["COMPUTE"]: @@ -92,6 +97,8 @@ for provider in config["PHONE_LIGHT"]["PROVIDERS"].keys(): files_to_compute.extend(expand("data/raw/{pid}/phone_light_with_datetime.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_light_features/phone_light_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_LIGHT"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_light.csv", pid=config["PIDS"],)) + files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) + files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") for provider in config["PHONE_ACCELEROMETER"]["PROVIDERS"].keys(): if config["PHONE_ACCELEROMETER"]["PROVIDERS"][provider]["COMPUTE"]: @@ -99,6 +106,8 @@ for provider in config["PHONE_ACCELEROMETER"]["PROVIDERS"].keys(): files_to_compute.extend(expand("data/raw/{pid}/phone_accelerometer_with_datetime.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_accelerometer_features/phone_accelerometer_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_ACCELEROMETER"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_accelerometer.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) + files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") for provider in config["PHONE_APPLICATIONS_FOREGROUND"]["PROVIDERS"].keys(): if config["PHONE_APPLICATIONS_FOREGROUND"]["PROVIDERS"][provider]["COMPUTE"]: @@ -107,6 +116,8 @@ for provider in config["PHONE_APPLICATIONS_FOREGROUND"]["PROVIDERS"].keys(): files_to_compute.extend(expand("data/raw/{pid}/phone_applications_foreground_with_datetime_with_categories.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_applications_foreground_features/phone_applications_foreground_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_APPLICATIONS_FOREGROUND"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_applications_foreground.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) + files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") for provider in config["PHONE_WIFI_VISIBLE"]["PROVIDERS"].keys(): if config["PHONE_WIFI_VISIBLE"]["PROVIDERS"][provider]["COMPUTE"]: @@ -114,6 +125,8 @@ for provider in config["PHONE_WIFI_VISIBLE"]["PROVIDERS"].keys(): files_to_compute.extend(expand("data/raw/{pid}/phone_wifi_visible_with_datetime.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_wifi_visible_features/phone_wifi_visible_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_WIFI_VISIBLE"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_wifi_visible.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) + files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") for provider in config["PHONE_WIFI_CONNECTED"]["PROVIDERS"].keys(): if config["PHONE_WIFI_CONNECTED"]["PROVIDERS"][provider]["COMPUTE"]: @@ -121,6 +134,8 @@ for provider in config["PHONE_WIFI_CONNECTED"]["PROVIDERS"].keys(): files_to_compute.extend(expand("data/raw/{pid}/phone_wifi_connected_with_datetime.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_wifi_connected_features/phone_wifi_connected_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_WIFI_CONNECTED"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_wifi_connected.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) + files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") for provider in config["PHONE_CONVERSATION"]["PROVIDERS"].keys(): if config["PHONE_CONVERSATION"]["PROVIDERS"][provider]["COMPUTE"]: @@ -129,75 +144,106 @@ for provider in config["PHONE_CONVERSATION"]["PROVIDERS"].keys(): files_to_compute.extend(expand("data/raw/{pid}/phone_conversation_with_datetime_unified.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_conversation_features/phone_conversation_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_CONVERSATION"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_conversation.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) + files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") for provider in config["PHONE_LOCATIONS"]["PROVIDERS"].keys(): if config["PHONE_LOCATIONS"]["PROVIDERS"][provider]["COMPUTE"]: - if config["PHONE_LOCATIONS"]["LOCATIONS_TO_USE"] == "RESAMPLE_FUSED": - if "PHONE_LOCATIONS" in config["PHONE_VALID_SENSED_BINS"]["PHONE_SENSORS"]: - files_to_compute.extend(expand("data/interim/{pid}/phone_sensed_bins.csv", pid=config["PIDS"])) + if config["PHONE_LOCATIONS"]["LOCATIONS_TO_USE"] == "FUSED_RESAMPLED": + if "PHONE_LOCATIONS" in config["PHONE_DATA_YIELD"]["SENSORS"]: + files_to_compute.extend(expand("data/interim/{pid}/phone_yielded_timestamps.csv", pid=config["PIDS"])) else: - raise ValueError("Error: Add PHONE_LOCATIONS (and as many PHONE_SENSORS as you have) to [PHONE_VALID_SENSED_BINS][PHONE_SENSORS] in config.yaml. This is necessary to compute phone_sensed_bins (bins of time when the smartphone was sensing data) which is used to resample fused location data (RESAMPLED_FUSED)") + raise ValueError("Error: Add PHONE_LOCATIONS (and as many PHONE_SENSORS as you have) to [PHONE_DATA_YIELD][SENSORS] in config.yaml. This is necessary to compute phone_yielded_timestamps (time when the smartphone was sensing data) which is used to resample fused location data (RESAMPLED_FUSED)") files_to_compute.extend(expand("data/raw/{pid}/phone_locations_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_locations_processed.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_locations_processed_with_datetime.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_locations_features/phone_locations_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_LOCATIONS"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_locations.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) + files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") -if config["FITBIT_HEARTRATE"]["TABLE_FORMAT"] not in ["JSON", "CSV"]: - raise ValueError("config['FITBIT_HEARTRATE']['TABLE_FORMAT'] should be JSON or CSV but you typed" + config["FITBIT_HEARTRATE"]["TABLE_FORMAT"]) +for provider in config["FITBIT_HEARTRATE_SUMMARY"]["PROVIDERS"].keys(): + if config["FITBIT_HEARTRATE_SUMMARY"]["PROVIDERS"][provider]["COMPUTE"]: + files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_summary_raw.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_summary_parsed.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_summary_parsed_with_datetime.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/fitbit_heartrate_summary_features/fitbit_heartrate_summary_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["FITBIT_HEARTRATE_SUMMARY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/processed/features/{pid}/fitbit_heartrate_summary.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) + files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") -if config["FITBIT_STEPS"]["TABLE_FORMAT"] not in ["JSON", "CSV"]: - raise ValueError("config['FITBIT_STEPS']['TABLE_FORMAT'] should be JSON or CSV but you typed" + config["FITBIT_STEPS"]["TABLE_FORMAT"]) +for provider in config["FITBIT_HEARTRATE_INTRADAY"]["PROVIDERS"].keys(): + if config["FITBIT_HEARTRATE_INTRADAY"]["PROVIDERS"][provider]["COMPUTE"]: + files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_intraday_raw.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_intraday_parsed.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_intraday_parsed_with_datetime.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/fitbit_heartrate_intraday_features/fitbit_heartrate_intraday_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["FITBIT_HEARTRATE_INTRADAY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/processed/features/{pid}/fitbit_heartrate_intraday.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) + files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") -if config["FITBIT_CALORIES"]["TABLE_FORMAT"] not in ["JSON", "CSV"]: - raise ValueError("config['FITBIT_CALORIES']['TABLE_FORMAT'] should be JSON or CSV but you typed" + config["FITBIT_CALORIES"]["TABLE_FORMAT"]) +for provider in config["FITBIT_SLEEP_SUMMARY"]["PROVIDERS"].keys(): + if config["FITBIT_SLEEP_SUMMARY"]["PROVIDERS"][provider]["COMPUTE"]: + files_to_compute.extend(expand("data/raw/{pid}/fitbit_sleep_summary_raw.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/raw/{pid}/fitbit_sleep_summary_parsed.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/raw/{pid}/fitbit_sleep_summary_parsed_with_datetime.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/fitbit_sleep_summary_features/fitbit_sleep_summary_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["FITBIT_SLEEP_SUMMARY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/processed/features/{pid}/fitbit_sleep_summary.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) + files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") -if config["FITBIT_SLEEP"]["TABLE_FORMAT"] not in ["JSON", "CSV"]: - raise ValueError("config['FITBIT_SLEEP']['TABLE_FORMAT'] should be JSON or CSV but you typed" + config["FITBIT_SLEEP"]["TABLE_FORMAT"]) +# for provider in config["FITBIT_SLEEP_INTRADAY"]["PROVIDERS"].keys(): +# if config["FITBIT_SLEEP_INTRADAY"]["PROVIDERS"][provider]["COMPUTE"]: +# files_to_compute.extend(expand("data/raw/{pid}/fitbit_sleep_intraday_raw.csv", pid=config["PIDS"])) +# files_to_compute.extend(expand("data/raw/{pid}/fitbit_sleep_intraday_parsed.csv", pid=config["PIDS"])) +# files_to_compute.extend(expand("data/raw/{pid}/fitbit_sleep_intraday_parsed_with_datetime.csv", pid=config["PIDS"])) -for provider in config["FITBIT_HEARTRATE"]["PROVIDERS"].keys(): - if config["FITBIT_HEARTRATE"]["PROVIDERS"][provider]["COMPUTE"]: - files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_{fitbit_data_type}_raw.csv", pid=config["PIDS"], fitbit_data_type=(["json"] if config["FITBIT_HEARTRATE"]["TABLE_FORMAT"] == "JSON" else ["summary", "intraday"]))) - files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_{fitbit_data_type}_parsed.csv", pid=config["PIDS"], fitbit_data_type=["summary", "intraday"])) - files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_{fitbit_data_type}_parsed_with_datetime.csv", pid=config["PIDS"], fitbit_data_type=["summary", "intraday"])) +for provider in config["FITBIT_STEPS_SUMMARY"]["PROVIDERS"].keys(): + if config["FITBIT_STEPS_SUMMARY"]["PROVIDERS"][provider]["COMPUTE"]: + files_to_compute.extend(expand("data/raw/{pid}/fitbit_steps_summary_raw.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/raw/{pid}/fitbit_steps_summary_parsed.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/raw/{pid}/fitbit_steps_summary_parsed_with_datetime.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/fitbit_steps_summary_features/fitbit_steps_summary_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["FITBIT_STEPS_SUMMARY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/processed/features/{pid}/fitbit_steps_summary.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) + files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") -for provider in config["FITBIT_STEPS"]["PROVIDERS"].keys(): - if config["FITBIT_STEPS"]["PROVIDERS"][provider]["COMPUTE"]: - files_to_compute.extend(expand("data/raw/{pid}/fitbit_steps_{fitbit_data_type}_raw.csv", pid=config["PIDS"], fitbit_data_type=(["json"] if config["FITBIT_STEPS"]["TABLE_FORMAT"] == "JSON" else ["summary", "intraday"]))) - files_to_compute.extend(expand("data/raw/{pid}/fitbit_steps_{fitbit_data_type}_parsed.csv", pid=config["PIDS"], fitbit_data_type=["summary", "intraday"])) - files_to_compute.extend(expand("data/raw/{pid}/fitbit_steps_{fitbit_data_type}_parsed_with_datetime.csv", pid=config["PIDS"], fitbit_data_type=["summary", "intraday"])) +for provider in config["FITBIT_STEPS_INTRADAY"]["PROVIDERS"].keys(): + if config["FITBIT_STEPS_INTRADAY"]["PROVIDERS"][provider]["COMPUTE"]: + files_to_compute.extend(expand("data/raw/{pid}/fitbit_steps_intraday_raw.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/raw/{pid}/fitbit_steps_intraday_parsed.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/raw/{pid}/fitbit_steps_intraday_parsed_with_datetime.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/interim/{pid}/fitbit_steps_intraday_features/fitbit_steps_intraday_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["FITBIT_STEPS_INTRADAY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/processed/features/{pid}/fitbit_steps_intraday.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) + files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") -for provider in config["FITBIT_CALORIES"]["PROVIDERS"].keys(): - if config["FITBIT_CALORIES"]["PROVIDERS"][provider]["COMPUTE"]: - files_to_compute.extend(expand("data/raw/{pid}/fitbit_calories_{fitbit_data_type}_raw.csv", pid=config["PIDS"], fitbit_data_type=(["json"] if config["FITBIT_CALORIES"]["TABLE_FORMAT"] == "JSON" else ["summary", "intraday"]))) - files_to_compute.extend(expand("data/raw/{pid}/fitbit_calories_{fitbit_data_type}_parsed.csv", pid=config["PIDS"], fitbit_data_type=["summary", "intraday"])) - files_to_compute.extend(expand("data/raw/{pid}/fitbit_calories_{fitbit_data_type}_parsed_with_datetime.csv", pid=config["PIDS"], fitbit_data_type=["summary", "intraday"])) +# for provider in config["FITBIT_CALORIES"]["PROVIDERS"].keys(): +# if config["FITBIT_CALORIES"]["PROVIDERS"][provider]["COMPUTE"]: +# files_to_compute.extend(expand("data/raw/{pid}/fitbit_calories_{fitbit_data_type}_raw.csv", pid=config["PIDS"], fitbit_data_type=(["json"] if config["FITBIT_CALORIES"]["TABLE_FORMAT"] == "JSON" else ["summary", "intraday"]))) +# files_to_compute.extend(expand("data/raw/{pid}/fitbit_calories_{fitbit_data_type}_parsed.csv", pid=config["PIDS"], fitbit_data_type=["summary", "intraday"])) +# files_to_compute.extend(expand("data/raw/{pid}/fitbit_calories_{fitbit_data_type}_parsed_with_datetime.csv", pid=config["PIDS"], fitbit_data_type=["summary", "intraday"])) +# files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) +# files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") -for provider in config["FITBIT_SLEEP"]["PROVIDERS"].keys(): - if config["FITBIT_SLEEP"]["PROVIDERS"][provider]["COMPUTE"]: - files_to_compute.extend(expand("data/raw/{pid}/fitbit_sleep_{fitbit_data_type}_raw.csv", pid=config["PIDS"], fitbit_data_type=(["json"] if config["FITBIT_SLEEP"]["TABLE_FORMAT"] == "JSON" else ["summary", "intraday"]))) - files_to_compute.extend(expand("data/raw/{pid}/fitbit_sleep_{fitbit_data_type}_parsed_episodes.csv", pid=config["PIDS"], fitbit_data_type=["summary"])) - files_to_compute.extend(expand("data/raw/{pid}/fitbit_sleep_{fitbit_data_type}_parsed.csv", pid=config["PIDS"], fitbit_data_type=["intraday"])) - files_to_compute.extend(expand("data/raw/{pid}/fitbit_sleep_{fitbit_data_type}_parsed_with_datetime.csv", pid=config["PIDS"], fitbit_data_type=["intraday"])) +# Visualization for Data Exploration +if config["HISTOGRAM_PHONE_DATA_YIELD"]["PLOT"]: + files_to_compute.append("reports/data_exploration/histogram_phone_data_yield.html") -# visualization for data exploration -if config["HEATMAP_FEATURES_CORRELATIONS"]["PLOT"]: - files_to_compute.extend(expand("reports/data_exploration/{min_valid_hours_per_day}hours_{min_valid_bins_per_hour}bins/heatmap_features_correlations.html", min_valid_hours_per_day=config["HEATMAP_FEATURES_CORRELATIONS"]["MIN_VALID_HOURS_PER_DAY"], min_valid_bins_per_hour=config["PHONE_VALID_SENSED_DAYS"]["MIN_VALID_BINS_PER_HOUR"])) - -if config["HISTOGRAM_VALID_SENSED_HOURS"]["PLOT"]: - files_to_compute.extend(expand("reports/data_exploration/{min_valid_hours_per_day}hours_{min_valid_bins_per_hour}bins/histogram_valid_sensed_hours.html", min_valid_hours_per_day=config["HISTOGRAM_VALID_SENSED_HOURS"]["MIN_VALID_HOURS_PER_DAY"], min_valid_bins_per_hour=config["PHONE_VALID_SENSED_DAYS"]["MIN_VALID_BINS_PER_HOUR"])) +if config["HEATMAP_SENSORS_PER_MINUTE_PER_TIME_SEGMENT"]["PLOT"]: + files_to_compute.extend(expand("reports/interim/{pid}/heatmap_sensors_per_minute_per_time_segment.html", pid=config["PIDS"])) + files_to_compute.append("reports/data_exploration/heatmap_sensors_per_minute_per_time_segment.html") -if config["HEATMAP_DAYS_BY_SENSORS"]["PLOT"]: - files_to_compute.extend(expand("reports/interim/{min_valid_hours_per_day}hours_{min_valid_bins_per_hour}bins/{pid}/heatmap_days_by_sensors.html", pid=config["PIDS"], min_valid_hours_per_day=config["HEATMAP_DAYS_BY_SENSORS"]["MIN_VALID_HOURS_PER_DAY"], min_valid_bins_per_hour=config["PHONE_VALID_SENSED_DAYS"]["MIN_VALID_BINS_PER_HOUR"])) - files_to_compute.extend(expand("reports/data_exploration/{min_valid_hours_per_day}hours_{min_valid_bins_per_hour}bins/heatmap_days_by_sensors_all_participants.html", min_valid_hours_per_day=config["HEATMAP_DAYS_BY_SENSORS"]["MIN_VALID_HOURS_PER_DAY"], min_valid_bins_per_hour=config["PHONE_VALID_SENSED_DAYS"]["MIN_VALID_BINS_PER_HOUR"])) +if config["HEATMAP_SENSOR_ROW_COUNT_PER_TIME_SEGMENT"]["PLOT"]: + files_to_compute.extend(expand("reports/interim/{pid}/heatmap_sensor_row_count_per_time_segment.html", pid=config["PIDS"])) + files_to_compute.append("reports/data_exploration/heatmap_sensor_row_count_per_time_segment.html") -if config["HEATMAP_SENSED_BINS"]["PLOT"]: - files_to_compute.extend(expand("reports/interim/heatmap_sensed_bins/{pid}/heatmap_sensed_bins.html", pid=config["PIDS"])) - files_to_compute.extend(["reports/data_exploration/heatmap_sensed_bins_all_participants.html"]) +if config["HEATMAP_PHONE_DATA_YIELD_PER_PARTICIPANT_PER_TIME_SEGMENT"]["PLOT"]: + files_to_compute.append("reports/data_exploration/heatmap_phone_data_yield_per_participant_per_time_segment.html") -if config["OVERALL_COMPLIANCE_HEATMAP"]["PLOT"]: - files_to_compute.extend(expand("reports/data_exploration/{min_valid_hours_per_day}hours_{min_valid_bins_per_hour}bins/overall_compliance_heatmap.html", min_valid_hours_per_day=config["OVERALL_COMPLIANCE_HEATMAP"]["MIN_VALID_HOURS_PER_DAY"], min_valid_bins_per_hour=config["PHONE_VALID_SENSED_DAYS"]["MIN_VALID_BINS_PER_HOUR"])) +if config["HEATMAP_FEATURE_CORRELATION_MATRIX"]["PLOT"]: + files_to_compute.append("reports/data_exploration/heatmap_feature_correlation_matrix.html") rule all: diff --git a/tests/data/processed/features/frequency/test03/phone_bluetooth.csv b/tests/data/processed/features/frequency/test03/phone_bluetooth.csv index 2e48244e..3f1d551d 100644 --- a/tests/data/processed/features/frequency/test03/phone_bluetooth.csv +++ b/tests/data/processed/features/frequency/test03/phone_bluetooth.csv @@ -1 +1 @@ -"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","bluetooth_rapids_countscans","bluetooth_rapids_uniquedevices","bluetooth_rapids_countscansmostuniquedevice" +"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","phone_bluetooth_rapids_countscans","phone_bluetooth_rapids_uniquedevices","phone_bluetooth_rapids_countscansmostuniquedevice" diff --git a/tests/data/processed/features/frequency/test04/phone_bluetooth.csv b/tests/data/processed/features/frequency/test04/phone_bluetooth.csv index 2e48244e..3f1d551d 100644 --- a/tests/data/processed/features/frequency/test04/phone_bluetooth.csv +++ b/tests/data/processed/features/frequency/test04/phone_bluetooth.csv @@ -1 +1 @@ -"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","bluetooth_rapids_countscans","bluetooth_rapids_uniquedevices","bluetooth_rapids_countscansmostuniquedevice" +"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","phone_bluetooth_rapids_countscans","phone_bluetooth_rapids_uniquedevices","phone_bluetooth_rapids_countscansmostuniquedevice" diff --git a/tests/data/processed/features/frequency/test05/phone_bluetooth.csv b/tests/data/processed/features/frequency/test05/phone_bluetooth.csv index cfd4b23b..87893d18 100644 --- a/tests/data/processed/features/frequency/test05/phone_bluetooth.csv +++ b/tests/data/processed/features/frequency/test05/phone_bluetooth.csv @@ -1,4 +1,4 @@ -"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","bluetooth_rapids_countscans","bluetooth_rapids_uniquedevices","bluetooth_rapids_countscansmostuniquedevice" +"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","phone_bluetooth_rapids_countscans","phone_bluetooth_rapids_uniquedevices","phone_bluetooth_rapids_countscansmostuniquedevice" "thirtyminutes0000#2020-07-02 00:00:00,2020-07-02 00:29:59","thirtyminutes0000","2020-07-02 00:00:00","2020-07-02 00:29:59",1,1,1 "thirtyminutes0001#2020-07-02 00:30:00,2020-07-02 00:59:59","thirtyminutes0001","2020-07-02 00:30:00","2020-07-02 00:59:59",1,1,1 "thirtyminutes0007#2020-07-02 03:30:00,2020-07-02 03:59:59","thirtyminutes0007","2020-07-02 03:30:00","2020-07-02 03:59:59",1,1,1 diff --git a/tests/data/processed/features/frequency/test06/phone_bluetooth.csv b/tests/data/processed/features/frequency/test06/phone_bluetooth.csv index 0f8bec90..4f8dc3cb 100644 --- a/tests/data/processed/features/frequency/test06/phone_bluetooth.csv +++ b/tests/data/processed/features/frequency/test06/phone_bluetooth.csv @@ -1,4 +1,4 @@ -"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","bluetooth_rapids_countscans","bluetooth_rapids_uniquedevices","bluetooth_rapids_countscansmostuniquedevice" +"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","phone_bluetooth_rapids_countscans","phone_bluetooth_rapids_uniquedevices","phone_bluetooth_rapids_countscansmostuniquedevice" "thirtyminutes0000#2020-07-02 00:00:00,2020-07-02 00:29:59","thirtyminutes0000","2020-07-02 00:00:00","2020-07-02 00:29:59",1,1,1 "thirtyminutes0001#2020-07-02 00:30:00,2020-07-02 00:59:59","thirtyminutes0001","2020-07-02 00:30:00","2020-07-02 00:59:59",1,1,1 "thirtyminutes0007#2020-07-02 03:30:00,2020-07-02 03:59:59","thirtyminutes0007","2020-07-02 03:30:00","2020-07-02 03:59:59",1,1,1 diff --git a/tests/data/processed/features/periodic/test01/phone_bluetooth.csv b/tests/data/processed/features/periodic/test01/phone_bluetooth.csv index ab777e3c..c627ca42 100644 --- a/tests/data/processed/features/periodic/test01/phone_bluetooth.csv +++ b/tests/data/processed/features/periodic/test01/phone_bluetooth.csv @@ -1,4 +1,4 @@ -"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","bluetooth_rapids_countscans","bluetooth_rapids_uniquedevices","bluetooth_rapids_countscansmostuniquedevice" +"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","phone_bluetooth_rapids_countscans","phone_bluetooth_rapids_uniquedevices","phone_bluetooth_rapids_countscansmostuniquedevice" "afternoon#2020-07-02 12:00:00,2020-07-02 17:59:59","afternoon","2020-07-02 12:00:00","2020-07-02 17:59:59",2,2,1 "daily#2020-07-02 00:00:00,2020-07-02 23:59:59","daily","2020-07-02 00:00:00","2020-07-02 23:59:59",14,5,4 "evening#2020-07-02 18:00:00,2020-07-02 23:59:59","evening","2020-07-02 18:00:00","2020-07-02 23:59:59",5,4,2 diff --git a/tests/data/processed/features/periodic/test02/phone_bluetooth.csv b/tests/data/processed/features/periodic/test02/phone_bluetooth.csv index 15b8e071..994a3b80 100644 --- a/tests/data/processed/features/periodic/test02/phone_bluetooth.csv +++ b/tests/data/processed/features/periodic/test02/phone_bluetooth.csv @@ -1,4 +1,4 @@ -"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","bluetooth_rapids_countscans","bluetooth_rapids_uniquedevices","bluetooth_rapids_countscansmostuniquedevice" +"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","phone_bluetooth_rapids_countscans","phone_bluetooth_rapids_uniquedevices","phone_bluetooth_rapids_countscansmostuniquedevice" "afternoon#2020-07-02 12:00:00,2020-07-02 17:59:59","afternoon","2020-07-02 12:00:00","2020-07-02 17:59:59",2,2,1 "daily#2020-07-02 00:00:00,2020-07-02 23:59:59","daily","2020-07-02 00:00:00","2020-07-02 23:59:59",14,5,5 "evening#2020-07-02 18:00:00,2020-07-02 23:59:59","evening","2020-07-02 18:00:00","2020-07-02 23:59:59",5,3,3 diff --git a/tests/data/processed/features/periodic/test03/phone_bluetooth.csv b/tests/data/processed/features/periodic/test03/phone_bluetooth.csv index 2e48244e..3f1d551d 100644 --- a/tests/data/processed/features/periodic/test03/phone_bluetooth.csv +++ b/tests/data/processed/features/periodic/test03/phone_bluetooth.csv @@ -1 +1 @@ -"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","bluetooth_rapids_countscans","bluetooth_rapids_uniquedevices","bluetooth_rapids_countscansmostuniquedevice" +"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","phone_bluetooth_rapids_countscans","phone_bluetooth_rapids_uniquedevices","phone_bluetooth_rapids_countscansmostuniquedevice" diff --git a/tests/data/processed/features/periodic/test04/phone_bluetooth.csv b/tests/data/processed/features/periodic/test04/phone_bluetooth.csv index 2e48244e..3f1d551d 100644 --- a/tests/data/processed/features/periodic/test04/phone_bluetooth.csv +++ b/tests/data/processed/features/periodic/test04/phone_bluetooth.csv @@ -1 +1 @@ -"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","bluetooth_rapids_countscans","bluetooth_rapids_uniquedevices","bluetooth_rapids_countscansmostuniquedevice" +"local_segment","local_segment_label","local_segment_start_datetime","local_segment_end_datetime","phone_bluetooth_rapids_countscans","phone_bluetooth_rapids_uniquedevices","phone_bluetooth_rapids_countscansmostuniquedevice" diff --git a/tests/scripts/run_tests.sh b/tests/scripts/run_tests.sh index 9a8958b3..59a60f1a 100755 --- a/tests/scripts/run_tests.sh +++ b/tests/scripts/run_tests.sh @@ -64,7 +64,7 @@ cp -r tests/data/raw/* data/raw cp -r tests/data/external/* data/external echo Disabling downloading of dataset... -sed -e '27,53 s/^/#/' -e 's/rules.download_dataset.output/"data\/raw\/\{pid\}\/\{sensor\}_raw\.csv"/' rules/preprocessing.smk > tmp +sed -e '26,51 s/^/#/' -e 's/rules.download_dataset.output/"data\/raw\/\{pid\}\/\{sensor\}_raw\.csv"/' rules/preprocessing.smk > tmp mv tmp rules/preprocessing.smk if [ $# -eq 1 ] diff --git a/tests/settings/frequency/testing_config.yaml b/tests/settings/frequency/testing_config.yaml index 54d2e379..e2128568 100644 --- a/tests/settings/frequency/testing_config.yaml +++ b/tests/settings/frequency/testing_config.yaml @@ -1,85 +1,139 @@ -# Participants to include in the analysis -# You must create a file for each participant named pXXX containing their device_id. This can be done manually or automatically -PIDS: [test03, test04,test05, test06] - -# Global var with common time segments -TIME_SEGMENTS: &time_segments - TYPE: FREQUENCY # FREQUENCY, PERIODIC, EVENT - FILE: "data/external/timesegments_frequency.csv" - INCLUDE_PAST_PERIODIC_SEGMENTS: FALSE # Only relevant if TYPE=PERIODIC, if set to TRUE we consider time segments back enough in the past as to include the first day of data - -# Use tz codes from https://en.wikipedia.org/wiki/List_of_tz_database_time_zones. Double check your code, for example EST is not US Eastern Time. -TIMEZONE: &timezone - America/New_York - +# See https://www.rapids.science/latest/setup/configuration/#database-credentials DATABASE_GROUP: &database_group MY_GROUP -# config section for the script that creates participant files automatically -PARTICIPANT_FILES: # run snakemake -j1 -R parse_participant_files +# See https://www.rapids.science/latest/setup/configuration/#timezone-of-your-study +TIMEZONE: &timezone + America/New_York + +# See https://www.rapids.science/latest/setup/configuration/#participant-files +PIDS: [test03, test04,test05, test06] + +# See https://www.rapids.science/latest/setup/configuration/#automatic-creation-of-participant-files +CREATE_PARTICIPANT_FILES: + SOURCE: + TYPE: AWARE_DEVICE_TABLE #AWARE_DEVICE_TABLE or CSV_FILE + DATABASE_GROUP: *database_group + CSV_FILE_PATH: "data/external/example_participants.csv" # see docs for required format + TIMEZONE: *timezone PHONE_SECTION: - ADD: FALSE - PARSED_FROM: AWARE_DEVICE_TABLE #AWARE_DEVICE_TABLE or CSV_FILE - PARSED_SOURCE: *database_group # DB credentials group or CSV file path. If CSV file, it should have: device_id, pid (optional), label (optional), start_date (optional), end_date (optional) + ADD: TRUE + DEVICE_ID_COLUMN: device_id # column name IGNORED_DEVICE_IDS: [] FITBIT_SECTION: - ADD: FALSE - SAME_AS_PHONE: FALSE # If TRUE, all config below is ignored - PARSED_FROM: CSV_FILE - PARSED_SOURCE: "external/my_fitbit_participants.csv" # CSV file should have: device_id, pid (optional), label (optional), start_date (optional), end_date (optional) + ADD: TRUE + DEVICE_ID_COLUMN: device_id # column name + IGNORED_DEVICE_IDS: [] -DEVICE_DATA: - PHONE: - SOURCE: - TYPE: DATABASE # Phone only supports DATABASE for now - DATABASE_GROUP: *database_group - DEVICE_ID_COLUMN: device_id # column name - TIMEZONE: - TYPE: SINGLE # SINGLE or MULTIPLE - VALUE: *timezone # IF TYPE=SINGLE, timezone code (e.g. America/New_York, see attribute TIMEZONE above). If TYPE=MULTIPLE, a table in your database with two columns (timestamp, timezone) where timestamp is a unix timestamp and timezone is one of https://en.wikipedia.org/wiki/List_of_tz_database_time_zones - FITBIT: - SOURCE: - TYPE: DATABASE # DATABASE or FILES (set each FITBIT_SENSOR TABLE attribute accordingly with a table name or a file path) - DATABASE_GROUP: *database_group - DEVICE_ID_COLUMN: device_id # column name - TIMEZONE: - TYPE: SINGLE # Fitbit only supports SINGLE timezones - VALUE: *timezone # timezone code (e.g. America/New_York, see attribute TIMEZONE above and https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) +# See https://www.rapids.science/latest/setup/configuration/#time-segments +TIME_SEGMENTS: &time_segments + TYPE: FREQUENCY # FREQUENCY, PERIODIC, EVENT + FILE: "data/external/timesegments_frequency.csv" + INCLUDE_PAST_PERIODIC_SEGMENTS: FALSE # Only relevant if TYPE=PERIODIC, see docs -PHONE_VALID_SENSED_BINS: - COMPUTE: False # This flag is automatically ignored (set to True) if you are extracting PHONE_VALID_SENSED_DAYS or screen or Barnett's location features - BIN_SIZE: &bin_size 5 # (in minutes) - # Add as many PHONE sensors as you have, they all improve the computation of PHONE_VALID_SENSED_BINS and PHONE_VALID_SENSED_DAYS. - # If you are extracting screen or Barnett/Doryab location features, PHONE_SCREEN and PHONE_LOCATIONS tables are mandatory. - # You can choose any of the keys shown below, just make sure its TABLE exists in your database! - # PHONE_MESSAGES, PHONE_CALLS, PHONE_LOCATIONS, PHONE_BLUETOOTH, PHONE_ACTIVITY_RECOGNITION, PHONE_BATTERY, PHONE_SCREEN, PHONE_LIGHT, - # PHONE_ACCELEROMETER, PHONE_APPLICATIONS_FOREGROUND, PHONE_WIFI_VISIBLE, PHONE_WIFI_CONNECTED, PHONE_CONVERSATION - PHONE_SENSORS: [] -PHONE_VALID_SENSED_DAYS: - COMPUTE: False - MIN_VALID_HOURS_PER_DAY: &min_valid_hours_per_day [16] # (out of 24) MIN_HOURS_PER_DAY - MIN_VALID_BINS_PER_HOUR: &min_valid_bins_per_hour [6] # (out of 60min/BIN_SIZE bins) -# Communication SMS features config, TYPES and FEATURES keys need to match -PHONE_MESSAGES: - TABLE: messages +######################################################################################################################## +# PHONE # +######################################################################################################################## + +# See https://www.rapids.science/latest/setup/configuration/#device-data-source-configuration +PHONE_DATA_CONFIGURATION: + SOURCE: + TYPE: DATABASE + DATABASE_GROUP: *database_group + DEVICE_ID_COLUMN: device_id # column name + TIMEZONE: + TYPE: SINGLE + VALUE: *timezone + +# Sensors ------ + +# https://www.rapids.science/latest/features/phone-accelerometer/ +PHONE_ACCELEROMETER: + TABLE: accelerometer PROVIDERS: RAPIDS: - COMPUTE: TRUE - MESSAGES_TYPES : [received, sent] - FEATURES: - received: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact] - sent: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact] - SRC_LANGUAGE: "r" - SRC_FOLDER: "rapids" # inside src/features/phone_messages + COMPUTE: False + FEATURES: ["maxmagnitude", "minmagnitude", "avgmagnitude", "medianmagnitude", "stdmagnitude"] + SRC_FOLDER: "rapids" # inside src/features/phone_accelerometer + SRC_LANGUAGE: "python" + + PANDA: + COMPUTE: False + VALID_SENSED_MINUTES: False + FEATURES: + exertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"] + nonexertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"] + SRC_FOLDER: "panda" # inside src/features/phone_accelerometer + SRC_LANGUAGE: "python" -# Communication call features config, TYPES and FEATURES keys need to match +# See https://www.rapids.science/latest/features/phone-activity-recognition/ +PHONE_ACTIVITY_RECOGNITION: + TABLE: + ANDROID: plugin_google_activity_recognition + IOS: plugin_ios_activity_recognition + EPISODE_THRESHOLD_BETWEEN_ROWS: 5 # minutes. Max time difference for two consecutive rows to be considered within the same battery episode. + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["count", "mostcommonactivity", "countuniqueactivities", "durationstationary", "durationmobile", "durationvehicle"] + ACTIVITY_CLASSES: + STATIONARY: ["still", "tilting"] + MOBILE: ["on_foot", "walking", "running", "on_bicycle"] + VEHICLE: ["in_vehicle"] + SRC_FOLDER: "rapids" # inside src/features/phone_activity_recognition + SRC_LANGUAGE: "python" + +# See https://www.rapids.science/latest/features/phone-applications-foreground/ +PHONE_APPLICATIONS_FOREGROUND: + TABLE: applications_foreground + APPLICATION_CATEGORIES: + CATALOGUE_SOURCE: FILE # FILE (genres are read from CATALOGUE_FILE) or GOOGLE (genres are scrapped from the Play Store) + CATALOGUE_FILE: "data/external/stachl_application_genre_catalogue.csv" + UPDATE_CATALOGUE_FILE: False # if CATALOGUE_SOURCE is equal to FILE, whether or not to update CATALOGUE_FILE, if CATALOGUE_SOURCE is equal to GOOGLE all scraped genres will be saved to CATALOGUE_FILE + SCRAPE_MISSING_CATEGORIES: False # whether or not to scrape missing genres, only effective if CATALOGUE_SOURCE is equal to FILE. If CATALOGUE_SOURCE is equal to GOOGLE, all genres are scraped anyway + PROVIDERS: + RAPIDS: + COMPUTE: False + SINGLE_CATEGORIES: ["all", "email"] + MULTIPLE_CATEGORIES: + social: ["socialnetworks", "socialmediatools"] + entertainment: ["entertainment", "gamingknowledge", "gamingcasual", "gamingadventure", "gamingstrategy", "gamingtoolscommunity", "gamingroleplaying", "gamingaction", "gaminglogic", "gamingsports", "gamingsimulation"] + SINGLE_APPS: ["top1global", "com.facebook.moments", "com.google.android.youtube", "com.twitter.android"] # There's no entropy for single apps + EXCLUDED_CATEGORIES: [] + EXCLUDED_APPS: ["com.fitbit.FitbitMobile", "com.aware.plugin.upmc.cancer"] + FEATURES: ["count", "timeoffirstuse", "timeoflastuse", "frequencyentropy"] + SRC_FOLDER: "rapids" # inside src/features/phone_applications_foreground + SRC_LANGUAGE: "python" + +# See https://www.rapids.science/latest/features/phone-battery/ +PHONE_BATTERY: + TABLE: battery + EPISODE_THRESHOLD_BETWEEN_ROWS: 30 # minutes. Max time difference for two consecutive rows to be considered within the same battery episode. + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["countdischarge", "sumdurationdischarge", "countcharge", "sumdurationcharge", "avgconsumptionrate", "maxconsumptionrate"] + SRC_FOLDER: "rapids" # inside src/features/phone_battery + SRC_LANGUAGE: "python" + +# See https://www.rapids.science/latest/features/phone-bluetooth/ +PHONE_BLUETOOTH: + TABLE: bluetooth + PROVIDERS: + RAPIDS: + COMPUTE: True + FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] + SRC_FOLDER: "rapids" # inside src/features/phone_bluetooth + SRC_LANGUAGE: "r" + +# See https://www.rapids.science/latest/features/phone-calls/ PHONE_CALLS: TABLE: calls PROVIDERS: RAPIDS: - COMPUTE: TRUE + COMPUTE: False CALL_TYPES: [missed, incoming, outgoing] FEATURES: missed: [count, distinctcontacts, timefirstcall, timelastcall, countmostfrequentcontact] @@ -88,6 +142,47 @@ PHONE_CALLS: SRC_LANGUAGE: "r" SRC_FOLDER: "rapids" # inside src/features/phone_calls +# See https://www.rapids.science/latest/features/phone-conversation/ +PHONE_CONVERSATION: + TABLE: + ANDROID: plugin_studentlife_audio_android + IOS: plugin_studentlife_audio + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["minutessilence", "minutesnoise", "minutesvoice", "minutesunknown","sumconversationduration","avgconversationduration", + "sdconversationduration","minconversationduration","maxconversationduration","timefirstconversation","timelastconversation","noisesumenergy", + "noiseavgenergy","noisesdenergy","noiseminenergy","noisemaxenergy","voicesumenergy", + "voiceavgenergy","voicesdenergy","voiceminenergy","voicemaxenergy","silencesensedfraction","noisesensedfraction", + "voicesensedfraction","unknownsensedfraction","silenceexpectedfraction","noiseexpectedfraction","voiceexpectedfraction", + "unknownexpectedfraction","countconversation"] + RECORDING_MINUTES: 1 + PAUSED_MINUTES : 3 + SRC_FOLDER: "rapids" # inside src/features/phone_conversation + SRC_LANGUAGE: "python" + +# See https://www.rapids.science/latest/features/phone-data-yield/ +PHONE_DATA_YIELD: + SENSORS: [] + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: [ratiovalidyieldedminutes, ratiovalidyieldedhours] + MINUTE_RATIO_THRESHOLD_FOR_VALID_YIELDED_HOURS: 0.5 # 0 to 1 representing the number of minutes with at least + SRC_LANGUAGE: "r" + SRC_FOLDER: "rapids" # inside src/features/phone_data_yield + +# See https://www.rapids.science/latest/features/phone-light/ +PHONE_LIGHT: + TABLE: light + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["count", "maxlux", "minlux", "avglux", "medianlux", "stdlux"] + SRC_FOLDER: "rapids" # inside src/features/phone_light + SRC_LANGUAGE: "python" + +# See https://www.rapids.science/latest/features/phone-locations/ PHONE_LOCATIONS: TABLE: locations LOCATIONS_TO_USE: FUSED_RESAMPLED # ALL, GPS OR FUSED_RESAMPLED @@ -115,42 +210,20 @@ PHONE_LOCATIONS: SRC_FOLDER: "barnett" # inside src/features/phone_locations SRC_LANGUAGE: "r" -PHONE_BLUETOOTH: - TABLE: bluetooth +# See https://www.rapids.science/latest/features/phone-messages/ +PHONE_MESSAGES: + TABLE: messages PROVIDERS: RAPIDS: - COMPUTE: TRUE - FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] - SRC_FOLDER: "rapids" # inside src/features/phone_bluetooth + COMPUTE: False + MESSAGES_TYPES : [received, sent] + FEATURES: + received: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact] + sent: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact] SRC_LANGUAGE: "r" + SRC_FOLDER: "rapids" # inside src/features/phone_messages - -PHONE_ACTIVITY_RECOGNITION: - TABLE: - ANDROID: plugin_google_activity_recognition - IOS: plugin_ios_activity_recognition - EPISODE_THRESHOLD_BETWEEN_ROWS: 5 # minutes. Max time difference for two consecutive rows to be considered within the same battery episode. - PROVIDERS: - RAPIDS: - COMPUTE: False - FEATURES: ["count", "mostcommonactivity", "countuniqueactivities", "durationstationary", "durationmobile", "durationvehicle"] - ACTIVITY_CLASSES: - STATIONARY: ["still", "tilting"] - MOBILE: ["on_foot", "walking", "running", "on_bicycle"] - VEHICLE: ["in_vehicle"] - SRC_FOLDER: "rapids" # inside src/features/phone_activity_recognition - SRC_LANGUAGE: "python" - -PHONE_BATTERY: - TABLE: battery - EPISODE_THRESHOLD_BETWEEN_ROWS: 30 # minutes. Max time difference for two consecutive rows to be considered within the same battery episode. - PROVIDERS: - RAPIDS: - COMPUTE: False - FEATURES: ["countdischarge", "sumdurationdischarge", "countcharge", "sumdurationcharge", "avgconsumptionrate", "maxconsumptionrate"] - SRC_FOLDER: "rapids" # inside src/features/phone_battery - SRC_LANGUAGE: "python" - +# See https://www.rapids.science/latest/features/phone-screen/ PHONE_SCREEN: TABLE: screen PROVIDERS: @@ -164,190 +237,138 @@ PHONE_SCREEN: SRC_FOLDER: "rapids" # inside src/features/phone_screen SRC_LANGUAGE: "python" -PHONE_LIGHT: - TABLE: light - PROVIDERS: - RAPIDS: - COMPUTE: TRUE - FEATURES: ["count", "maxlux", "minlux", "avglux", "medianlux", "stdlux"] - SRC_FOLDER: "rapids" # inside src/features/phone_light - SRC_LANGUAGE: "python" - -PHONE_ACCELEROMETER: - TABLE: accelerometer - PROVIDERS: - RAPIDS: - COMPUTE: False - FEATURES: ["maxmagnitude", "minmagnitude", "avgmagnitude", "medianmagnitude", "stdmagnitude"] - SRC_FOLDER: "rapids" # inside src/features/phone_accelerometer - SRC_LANGUAGE: "python" - - PANDA: - COMPUTE: False - VALID_SENSED_MINUTES: False - FEATURES: - exertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"] - nonexertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"] - SRC_FOLDER: "panda" # inside src/features/phone_accelerometer - SRC_LANGUAGE: "python" - -PHONE_APPLICATIONS_FOREGROUND: - TABLE: applications_foreground - APPLICATION_CATEGORIES: - CATALOGUE_SOURCE: FILE # FILE (genres are read from CATALOGUE_FILE) or GOOGLE (genres are scrapped from the Play Store) - CATALOGUE_FILE: "data/external/stachl_application_genre_catalogue.csv" - UPDATE_CATALOGUE_FILE: False # if CATALOGUE_SOURCE is equal to FILE, whether or not to update CATALOGUE_FILE, if CATALOGUE_SOURCE is equal to GOOGLE all scraped genres will be saved to CATALOGUE_FILE - SCRAPE_MISSING_CATEGORIES: False # whether or not to scrape missing genres, only effective if CATALOGUE_SOURCE is equal to FILE. If CATALOGUE_SOURCE is equal to GOOGLE, all genres are scraped anyway - PROVIDERS: - RAPIDS: - COMPUTE: TRUE - SINGLE_CATEGORIES: ["all", "email"] - MULTIPLE_CATEGORIES: - social: ["socialnetworks", "socialmediatools"] - entertainment: ["entertainment", "gamingstrategy"] - SINGLE_APPS: ["top1global", "com.facebook.moments"] # There's no entropy for single apps - EXCLUDED_CATEGORIES: ["systemapp", "tvvideoapps"] - EXCLUDED_APPS: ["com.fitbit.FitbitMobile", "com.aware.plugin.upmc.cancer"] - FEATURES: ["count", "timeoffirstuse", "timeoflastuse", "frequencyentropy"] - SRC_FOLDER: "rapids" # inside src/features/phone_applications_foreground - SRC_LANGUAGE: "python" - -PHONE_WIFI_VISIBLE: - TABLE: "wifi" - PROVIDERS: - RAPIDS: - COMPUTE: TRUE - FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] - SRC_FOLDER: "rapids" # inside src/features/phone_wifi_visible - SRC_LANGUAGE: "r" - +# See https://www.rapids.science/latest/features/phone-wifi-connected/ PHONE_WIFI_CONNECTED: TABLE: "sensor_wifi" PROVIDERS: RAPIDS: - COMPUTE: TRUE + COMPUTE: False FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] SRC_FOLDER: "rapids" # inside src/features/phone_wifi_connected SRC_LANGUAGE: "r" -PHONE_CONVERSATION: - TABLE: - ANDROID: plugin_studentlife_audio_android - IOS: plugin_studentlife_audio - PROVIDERS: - RAPIDS: - COMPUTE: TRUE - FEATURES: ["minutessilence", "minutesnoise", "minutesvoice", "minutesunknown","sumconversationduration","avgconversationduration", - "sdconversationduration","minconversationduration","maxconversationduration","timefirstconversation","timelastconversation","noisesumenergy", - "noiseavgenergy","noisesdenergy","noiseminenergy","noisemaxenergy","voicesumenergy", - "voiceavgenergy","voicesdenergy","voiceminenergy","voicemaxenergy","silencesensedfraction","noisesensedfraction", - "voicesensedfraction","unknownsensedfraction","silenceexpectedfraction","noiseexpectedfraction","voiceexpectedfraction", - "unknownexpectedfraction","countconversation"] - RECORDING_MINUTES: 1 - PAUSED_MINUTES : 3 - SRC_FOLDER: "rapids" # inside src/features/phone_conversation - SRC_LANGUAGE: "python" - -############## FITBIT ########################################################## -################################################################################ - -FITBIT_HEARTRATE: - TABLE_FORMAT: JSON # JSON or CSV - TABLE: - JSON: fitbit_heartrate - CSV: - SUMMARY: heartrate_summary.csv - INTRADAY: heartrate_intraday.csv +# See https://www.rapids.science/latest/features/phone-wifi-visible/ +PHONE_WIFI_VISIBLE: + TABLE: "wifi" PROVIDERS: RAPIDS: COMPUTE: False - SUMMARY_FEATURES: ["restinghr"] # calories features' accuracy depend on the accuracy of the participants fitbit profile (e.g. height, weight) use these with care: ["caloriesoutofrange", "caloriesfatburn", "caloriescardio", "caloriespeak"] - INTRADAY_FEATURES: ["maxhr", "minhr", "avghr", "medianhr", "modehr", "stdhr", "diffmaxmodehr", "diffminmodehr", "entropyhr", "minutesonoutofrangezone", "minutesonfatburnzone", "minutesoncardiozone", "minutesonpeakzone"] + FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] + SRC_FOLDER: "rapids" # inside src/features/phone_wifi_visible + SRC_LANGUAGE: "r" -FITBIT_STEPS: - TABLE_FORMAT: JSON # JSON or CSV - TABLE: - JSON: fitbit_steps - CSV: - SUMMARY: steps_summary.csv - INTRADAY: steps_intraday.csv - EXCLUDE_SLEEP: # you can exclude sleep periods from the step features computation - EXCLUDE: False - TYPE: FIXED # FIXED OR FITBIT_BASED (configure FITBIT_SLEEP section) - FIXED: - START: "23:00" - END: "07:00" +######################################################################################################################## +# FITBIT # +######################################################################################################################## + +# See https://www.rapids.science/latest/setup/configuration/#device-data-source-configuration +FITBIT_DATA_CONFIGURATION: + SOURCE: + TYPE: DATABASE # DATABASE or FILES (set each [FITBIT_SENSOR][TABLE] attribute with a table name or a file path accordingly) + COLUMN_FORMAT: JSON # JSON or PLAIN_TEXT + DATABASE_GROUP: *database_group + DEVICE_ID_COLUMN: device_id # column name + TIMEZONE: + TYPE: SINGLE # Fitbit devices don't support time zones so we read this data in the timezone indicated by VALUE + VALUE: *timezone + +# Sensors ------ + +# See https://www.rapids.science/latest/features/fitbit-heartrate-summary/ +FITBIT_HEARTRATE_SUMMARY: + TABLE: heartrate_summary + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["maxrestinghr", "minrestinghr", "avgrestinghr", "medianrestinghr", "moderestinghr", "stdrestinghr", "diffmaxmoderestinghr", "diffminmoderestinghr", "entropyrestinghr"] # calories features' accuracy depend on the accuracy of the participants fitbit profile (e.g. height, weight) use these with care: ["sumcaloriesoutofrange", "maxcaloriesoutofrange", "mincaloriesoutofrange", "avgcaloriesoutofrange", "mediancaloriesoutofrange", "stdcaloriesoutofrange", "entropycaloriesoutofrange", "sumcaloriesfatburn", "maxcaloriesfatburn", "mincaloriesfatburn", "avgcaloriesfatburn", "mediancaloriesfatburn", "stdcaloriesfatburn", "entropycaloriesfatburn", "sumcaloriescardio", "maxcaloriescardio", "mincaloriescardio", "avgcaloriescardio", "mediancaloriescardio", "stdcaloriescardio", "entropycaloriescardio", "sumcaloriespeak", "maxcaloriespeak", "mincaloriespeak", "avgcaloriespeak", "mediancaloriespeak", "stdcaloriespeak", "entropycaloriespeak"] + SRC_FOLDER: "rapids" # inside src/features/fitbit_heartrate_summary + SRC_LANGUAGE: "python" + +# See https://www.rapids.science/latest/features/fitbit-heartrate-intraday/ +FITBIT_HEARTRATE_INTRADAY: + TABLE: heartrate_intraday + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["maxhr", "minhr", "avghr", "medianhr", "modehr", "stdhr", "diffmaxmodehr", "diffminmodehr", "entropyhr", "minutesonoutofrangezone", "minutesonfatburnzone", "minutesoncardiozone", "minutesonpeakzone"] + SRC_FOLDER: "rapids" # inside src/features/fitbit_heartrate_intraday + SRC_LANGUAGE: "python" + +# See https://www.rapids.science/latest/features/fitbit-sleep-summary/ +FITBIT_SLEEP_SUMMARY: + TABLE: sleep_summary + SLEEP_EPISODE_TIMESTAMP: end # summary sleep episodes are considered as events based on either the start timestamp or end timestamp. + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["countepisode", "avgefficiency", "sumdurationafterwakeup", "sumdurationasleep", "sumdurationawake", "sumdurationtofallasleep", "sumdurationinbed", "avgdurationafterwakeup", "avgdurationasleep", "avgdurationawake", "avgdurationtofallasleep", "avgdurationinbed"] + SLEEP_TYPES: ["main", "nap", "all"] + SRC_FOLDER: "rapids" # inside src/features/fitbit_sleep_summary + SRC_LANGUAGE: "python" + +# See https://www.rapids.science/latest/features/fitbit-steps-summary/ +FITBIT_STEPS_SUMMARY: + TABLE: steps_summary + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["maxsumsteps", "minsumsteps", "avgsumsteps", "mediansumsteps", "stdsumsteps"] + SRC_FOLDER: "rapids" # inside src/features/fitbit_steps_summary + SRC_LANGUAGE: "python" + +# See https://www.rapids.science/latest/features/fitbit-steps-intraday/ +FITBIT_STEPS_INTRADAY: + TABLE: steps_intraday PROVIDERS: RAPIDS: COMPUTE: False FEATURES: - ALL_STEPS: ["sumallsteps", "maxallsteps", "minallsteps", "avgallsteps", "stdallsteps"] + STEPS: ["sum", "max", "min", "avg", "std"] SEDENTARY_BOUT: ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "stdduration"] ACTIVE_BOUT: ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "stdduration"] THRESHOLD_ACTIVE_BOUT: 10 # steps INCLUDE_ZERO_STEP_ROWS: False + SRC_FOLDER: "rapids" # inside src/features/fitbit_steps_intraday + SRC_LANGUAGE: "python" -FITBIT_SLEEP: - TABLE_FORMAT: JSON # JSON or CSV - TABLE: - JSON: fitbit_sleep - CSV: - SUMMARY: sleep_summary.csv - INTRADAY: sleep_intraday.csv - PROVIDERS: - RAPIDS: - COMPUTE: False - SLEEP_TYPES: ["main", "nap", "all"] - SUMMARY_FEATURES: ["sumdurationafterwakeup", "sumdurationasleep", "sumdurationawake", "sumdurationtofallasleep", "sumdurationinbed", "avgefficiency", "countepisode"] +# FITBIT_CALORIES: +# TABLE_FORMAT: JSON # JSON or CSV. If your JSON or CSV data are files change [DEVICE_DATA][FITBIT][SOURCE][TYPE] to FILES +# TABLE: +# JSON: fitbit_calories +# CSV: +# SUMMARY: calories_summary +# INTRADAY: calories_intraday +# PROVIDERS: +# RAPIDS: +# COMPUTE: False +# FEATURES: [] -FITBIT_CALORIES: - TABLE_FORMAT: JSON # JSON or CSV - TABLE: - JSON: fitbit_calories - CSV: - SUMMARY: calories_summary.csv - INTRADAY: calories_intraday.csv - PROVIDERS: - RAPIDS: - COMPUTE: False - FEATURES: [] -### Visualizations ############################################################# -################################################################################ -HEATMAP_FEATURES_CORRELATIONS: +######################################################################################################################## +# PLOTS # +######################################################################################################################## + +# Data quality +HISTOGRAM_PHONE_DATA_YIELD: + PLOT: False + +HEATMAP_PHONE_DATA_YIELD_PER_PARTICIPANT_PER_TIME_SEGMENT: + PLOT: False + +HEATMAP_SENSORS_PER_MINUTE_PER_TIME_SEGMENT: + PLOT: False + +HEATMAP_SENSOR_ROW_COUNT_PER_TIME_SEGMENT: + PLOT: False + SENSORS: [PHONE_ACCELEROMETER, PHONE_ACTIVITY_RECOGNITION, PHONE_APPLICATIONS_FOREGROUND, PHONE_BATTERY, PHONE_BLUETOOTH, PHONE_CALLS, PHONE_CONVERSATION, PHONE_LIGHT, PHONE_LOCATIONS, PHONE_MESSAGES, PHONE_SCREEN, PHONE_WIFI_CONNECTED, PHONE_WIFI_VISIBLE] + +# Features +HEATMAP_FEATURE_CORRELATION_MATRIX: PLOT: False MIN_ROWS_RATIO: 0.5 - MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day - MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour - PHONE_FEATURES: [accelerometer, activity_recognition, applications_foreground, battery, calls_incoming, calls_missed, calls_outgoing, conversation, light, location_doryab, messages_received, messages_sent, screen] - FITBIT_FEATURES: [fitbit_heartrate, fitbit_step, fitbit_sleep] CORR_THRESHOLD: 0.1 CORR_METHOD: "pearson" # choose from {"pearson", "kendall", "spearman"} -HISTOGRAM_VALID_SENSED_HOURS: - PLOT: False - MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day - MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour - -HEATMAP_DAYS_BY_SENSORS: - PLOT: False - MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day - MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour - EXPECTED_NUM_OF_DAYS: -1 - DB_TABLES: [accelerometer, applications_foreground, battery, bluetooth, calls, light, locations, messages, screen, wifi, sensor_wifi, plugin_google_activity_recognition, plugin_ios_activity_recognition, plugin_studentlife_audio_android, plugin_studentlife_audio] - -HEATMAP_SENSED_BINS: - PLOT: False - BIN_SIZE: *bin_size - -OVERALL_COMPLIANCE_HEATMAP: - PLOT: False - ONLY_SHOW_VALID_DAYS: False - EXPECTED_NUM_OF_DAYS: -1 - BIN_SIZE: *bin_size - MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day - MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour - diff --git a/tests/settings/periodic/testing_config.yaml b/tests/settings/periodic/testing_config.yaml index 5fe9c839..27b0df80 100644 --- a/tests/settings/periodic/testing_config.yaml +++ b/tests/settings/periodic/testing_config.yaml @@ -1,85 +1,139 @@ -# Participants to include in the analysis -# You must create a file for each participant named pXXX containing their device_id. This can be done manually or automatically -PIDS: [test01, test02, test03, test04] - -# Global var with common time segments -TIME_SEGMENTS: &time_segments - TYPE: PERIODIC # FREQUENCY, PERIODIC, EVENT - FILE: "data/external/timesegments_periodic.csv" - INCLUDE_PAST_PERIODIC_SEGMENTS: FALSE # Only relevant if TYPE=PERIODIC, if set to TRUE we consider time segments back enough in the past as to include the first day of data - -# Use tz codes from https://en.wikipedia.org/wiki/List_of_tz_database_time_zones. Double check your code, for example EST is not US Eastern Time. -TIMEZONE: &timezone - America/New_York - +# See https://www.rapids.science/latest/setup/configuration/#database-credentials DATABASE_GROUP: &database_group MY_GROUP -# config section for the script that creates participant files automatically -PARTICIPANT_FILES: # run snakemake -j1 -R parse_participant_files +# See https://www.rapids.science/latest/setup/configuration/#timezone-of-your-study +TIMEZONE: &timezone + America/New_York + +# See https://www.rapids.science/latest/setup/configuration/#participant-files +PIDS: [test01, test02, test03, test04] + +# See https://www.rapids.science/latest/setup/configuration/#automatic-creation-of-participant-files +CREATE_PARTICIPANT_FILES: + SOURCE: + TYPE: AWARE_DEVICE_TABLE #AWARE_DEVICE_TABLE or CSV_FILE + DATABASE_GROUP: *database_group + CSV_FILE_PATH: "data/external/example_participants.csv" # see docs for required format + TIMEZONE: *timezone PHONE_SECTION: - ADD: FALSE - PARSED_FROM: AWARE_DEVICE_TABLE #AWARE_DEVICE_TABLE or CSV_FILE - PARSED_SOURCE: *database_group # DB credentials group or CSV file path. If CSV file, it should have: device_id, pid (optional), label (optional), start_date (optional), end_date (optional) + ADD: TRUE + DEVICE_ID_COLUMN: device_id # column name IGNORED_DEVICE_IDS: [] FITBIT_SECTION: - ADD: FALSE - SAME_AS_PHONE: FALSE # If TRUE, all config below is ignored - PARSED_FROM: CSV_FILE - PARSED_SOURCE: "external/my_fitbit_participants.csv" # CSV file should have: device_id, pid (optional), label (optional), start_date (optional), end_date (optional) + ADD: TRUE + DEVICE_ID_COLUMN: device_id # column name + IGNORED_DEVICE_IDS: [] -DEVICE_DATA: - PHONE: - SOURCE: - TYPE: DATABASE # Phone only supports DATABASE for now - DATABASE_GROUP: *database_group - DEVICE_ID_COLUMN: device_id # column name - TIMEZONE: - TYPE: SINGLE # SINGLE or MULTIPLE - VALUE: *timezone # IF TYPE=SINGLE, timezone code (e.g. America/New_York, see attribute TIMEZONE above). If TYPE=MULTIPLE, a table in your database with two columns (timestamp, timezone) where timestamp is a unix timestamp and timezone is one of https://en.wikipedia.org/wiki/List_of_tz_database_time_zones - FITBIT: - SOURCE: - TYPE: DATABASE # DATABASE or FILES (set each FITBIT_SENSOR TABLE attribute accordingly with a table name or a file path) - DATABASE_GROUP: *database_group - DEVICE_ID_COLUMN: device_id # column name - TIMEZONE: - TYPE: SINGLE # Fitbit only supports SINGLE timezones - VALUE: *timezone # timezone code (e.g. America/New_York, see attribute TIMEZONE above and https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) +# See https://www.rapids.science/latest/setup/configuration/#time-segments +TIME_SEGMENTS: &time_segments + TYPE: PERIODIC # FREQUENCY, PERIODIC, EVENT + FILE: "data/external/timesegments_periodic.csv" + INCLUDE_PAST_PERIODIC_SEGMENTS: FALSE # Only relevant if TYPE=PERIODIC, see docs -PHONE_VALID_SENSED_BINS: - COMPUTE: False # This flag is automatically ignored (set to True) if you are extracting PHONE_VALID_SENSED_DAYS or screen or Barnett's location features - BIN_SIZE: &bin_size 5 # (in minutes) - # Add as many PHONE sensors as you have, they all improve the computation of PHONE_VALID_SENSED_BINS and PHONE_VALID_SENSED_DAYS. - # If you are extracting screen or Barnett/Doryab location features, PHONE_SCREEN and PHONE_LOCATIONS tables are mandatory. - # You can choose any of the keys shown below, just make sure its TABLE exists in your database! - # PHONE_MESSAGES, PHONE_CALLS, PHONE_LOCATIONS, PHONE_BLUETOOTH, PHONE_ACTIVITY_RECOGNITION, PHONE_BATTERY, PHONE_SCREEN, PHONE_LIGHT, - # PHONE_ACCELEROMETER, PHONE_APPLICATIONS_FOREGROUND, PHONE_WIFI_VISIBLE, PHONE_WIFI_CONNECTED, PHONE_CONVERSATION - PHONE_SENSORS: [] -PHONE_VALID_SENSED_DAYS: - COMPUTE: False - MIN_VALID_HOURS_PER_DAY: &min_valid_hours_per_day [16] # (out of 24) MIN_HOURS_PER_DAY - MIN_VALID_BINS_PER_HOUR: &min_valid_bins_per_hour [6] # (out of 60min/BIN_SIZE bins) -# Communication SMS features config, TYPES and FEATURES keys need to match -PHONE_MESSAGES: - TABLE: messages +######################################################################################################################## +# PHONE # +######################################################################################################################## + +# See https://www.rapids.science/latest/setup/configuration/#device-data-source-configuration +PHONE_DATA_CONFIGURATION: + SOURCE: + TYPE: DATABASE + DATABASE_GROUP: *database_group + DEVICE_ID_COLUMN: device_id # column name + TIMEZONE: + TYPE: SINGLE + VALUE: *timezone + +# Sensors ------ + +# https://www.rapids.science/latest/features/phone-accelerometer/ +PHONE_ACCELEROMETER: + TABLE: accelerometer + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["maxmagnitude", "minmagnitude", "avgmagnitude", "medianmagnitude", "stdmagnitude"] + SRC_FOLDER: "rapids" # inside src/features/phone_accelerometer + SRC_LANGUAGE: "python" + + PANDA: + COMPUTE: False + VALID_SENSED_MINUTES: False + FEATURES: + exertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"] + nonexertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"] + SRC_FOLDER: "panda" # inside src/features/phone_accelerometer + SRC_LANGUAGE: "python" + +# See https://www.rapids.science/latest/features/phone-activity-recognition/ +PHONE_ACTIVITY_RECOGNITION: + TABLE: + ANDROID: plugin_google_activity_recognition + IOS: plugin_ios_activity_recognition + EPISODE_THRESHOLD_BETWEEN_ROWS: 5 # minutes. Max time difference for two consecutive rows to be considered within the same battery episode. + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["count", "mostcommonactivity", "countuniqueactivities", "durationstationary", "durationmobile", "durationvehicle"] + ACTIVITY_CLASSES: + STATIONARY: ["still", "tilting"] + MOBILE: ["on_foot", "walking", "running", "on_bicycle"] + VEHICLE: ["in_vehicle"] + SRC_FOLDER: "rapids" # inside src/features/phone_activity_recognition + SRC_LANGUAGE: "python" + +# See https://www.rapids.science/latest/features/phone-applications-foreground/ +PHONE_APPLICATIONS_FOREGROUND: + TABLE: applications_foreground + APPLICATION_CATEGORIES: + CATALOGUE_SOURCE: FILE # FILE (genres are read from CATALOGUE_FILE) or GOOGLE (genres are scrapped from the Play Store) + CATALOGUE_FILE: "data/external/stachl_application_genre_catalogue.csv" + UPDATE_CATALOGUE_FILE: False # if CATALOGUE_SOURCE is equal to FILE, whether or not to update CATALOGUE_FILE, if CATALOGUE_SOURCE is equal to GOOGLE all scraped genres will be saved to CATALOGUE_FILE + SCRAPE_MISSING_CATEGORIES: False # whether or not to scrape missing genres, only effective if CATALOGUE_SOURCE is equal to FILE. If CATALOGUE_SOURCE is equal to GOOGLE, all genres are scraped anyway + PROVIDERS: + RAPIDS: + COMPUTE: False + SINGLE_CATEGORIES: ["all", "email"] + MULTIPLE_CATEGORIES: + social: ["socialnetworks", "socialmediatools"] + entertainment: ["entertainment", "gamingknowledge", "gamingcasual", "gamingadventure", "gamingstrategy", "gamingtoolscommunity", "gamingroleplaying", "gamingaction", "gaminglogic", "gamingsports", "gamingsimulation"] + SINGLE_APPS: ["top1global", "com.facebook.moments", "com.google.android.youtube", "com.twitter.android"] # There's no entropy for single apps + EXCLUDED_CATEGORIES: [] + EXCLUDED_APPS: ["com.fitbit.FitbitMobile", "com.aware.plugin.upmc.cancer"] + FEATURES: ["count", "timeoffirstuse", "timeoflastuse", "frequencyentropy"] + SRC_FOLDER: "rapids" # inside src/features/phone_applications_foreground + SRC_LANGUAGE: "python" + +# See https://www.rapids.science/latest/features/phone-battery/ +PHONE_BATTERY: + TABLE: battery + EPISODE_THRESHOLD_BETWEEN_ROWS: 30 # minutes. Max time difference for two consecutive rows to be considered within the same battery episode. + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["countdischarge", "sumdurationdischarge", "countcharge", "sumdurationcharge", "avgconsumptionrate", "maxconsumptionrate"] + SRC_FOLDER: "rapids" # inside src/features/phone_battery + SRC_LANGUAGE: "python" + +# See https://www.rapids.science/latest/features/phone-bluetooth/ +PHONE_BLUETOOTH: + TABLE: bluetooth PROVIDERS: RAPIDS: COMPUTE: TRUE - MESSAGES_TYPES : [received, sent] - FEATURES: - received: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact] - sent: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact] + FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] + SRC_FOLDER: "rapids" # inside src/features/phone_bluetooth SRC_LANGUAGE: "r" - SRC_FOLDER: "rapids" # inside src/features/phone_messages -# Communication call features config, TYPES and FEATURES keys need to match +# See https://www.rapids.science/latest/features/phone-calls/ PHONE_CALLS: TABLE: calls PROVIDERS: RAPIDS: - COMPUTE: TRUE + COMPUTE: False CALL_TYPES: [missed, incoming, outgoing] FEATURES: missed: [count, distinctcontacts, timefirstcall, timelastcall, countmostfrequentcontact] @@ -88,6 +142,47 @@ PHONE_CALLS: SRC_LANGUAGE: "r" SRC_FOLDER: "rapids" # inside src/features/phone_calls +# See https://www.rapids.science/latest/features/phone-conversation/ +PHONE_CONVERSATION: + TABLE: + ANDROID: plugin_studentlife_audio_android + IOS: plugin_studentlife_audio + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["minutessilence", "minutesnoise", "minutesvoice", "minutesunknown","sumconversationduration","avgconversationduration", + "sdconversationduration","minconversationduration","maxconversationduration","timefirstconversation","timelastconversation","noisesumenergy", + "noiseavgenergy","noisesdenergy","noiseminenergy","noisemaxenergy","voicesumenergy", + "voiceavgenergy","voicesdenergy","voiceminenergy","voicemaxenergy","silencesensedfraction","noisesensedfraction", + "voicesensedfraction","unknownsensedfraction","silenceexpectedfraction","noiseexpectedfraction","voiceexpectedfraction", + "unknownexpectedfraction","countconversation"] + RECORDING_MINUTES: 1 + PAUSED_MINUTES : 3 + SRC_FOLDER: "rapids" # inside src/features/phone_conversation + SRC_LANGUAGE: "python" + +# See https://www.rapids.science/latest/features/phone-data-yield/ +PHONE_DATA_YIELD: + SENSORS: [] + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: [ratiovalidyieldedminutes, ratiovalidyieldedhours] + MINUTE_RATIO_THRESHOLD_FOR_VALID_YIELDED_HOURS: 0.5 # 0 to 1 representing the number of minutes with at least + SRC_LANGUAGE: "r" + SRC_FOLDER: "rapids" # inside src/features/phone_data_yield + +# See https://www.rapids.science/latest/features/phone-light/ +PHONE_LIGHT: + TABLE: light + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["count", "maxlux", "minlux", "avglux", "medianlux", "stdlux"] + SRC_FOLDER: "rapids" # inside src/features/phone_light + SRC_LANGUAGE: "python" + +# See https://www.rapids.science/latest/features/phone-locations/ PHONE_LOCATIONS: TABLE: locations LOCATIONS_TO_USE: FUSED_RESAMPLED # ALL, GPS OR FUSED_RESAMPLED @@ -115,42 +210,20 @@ PHONE_LOCATIONS: SRC_FOLDER: "barnett" # inside src/features/phone_locations SRC_LANGUAGE: "r" -PHONE_BLUETOOTH: - TABLE: bluetooth +# See https://www.rapids.science/latest/features/phone-messages/ +PHONE_MESSAGES: + TABLE: messages PROVIDERS: RAPIDS: - COMPUTE: TRUE - FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] - SRC_FOLDER: "rapids" # inside src/features/phone_bluetooth + COMPUTE: False + MESSAGES_TYPES : [received, sent] + FEATURES: + received: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact] + sent: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact] SRC_LANGUAGE: "r" + SRC_FOLDER: "rapids" # inside src/features/phone_messages - -PHONE_ACTIVITY_RECOGNITION: - TABLE: - ANDROID: plugin_google_activity_recognition - IOS: plugin_ios_activity_recognition - EPISODE_THRESHOLD_BETWEEN_ROWS: 5 # minutes. Max time difference for two consecutive rows to be considered within the same battery episode. - PROVIDERS: - RAPIDS: - COMPUTE: False - FEATURES: ["count", "mostcommonactivity", "countuniqueactivities", "durationstationary", "durationmobile", "durationvehicle"] - ACTIVITY_CLASSES: - STATIONARY: ["still", "tilting"] - MOBILE: ["on_foot", "walking", "running", "on_bicycle"] - VEHICLE: ["in_vehicle"] - SRC_FOLDER: "rapids" # inside src/features/phone_activity_recognition - SRC_LANGUAGE: "python" - -PHONE_BATTERY: - TABLE: battery - EPISODE_THRESHOLD_BETWEEN_ROWS: 30 # minutes. Max time difference for two consecutive rows to be considered within the same battery episode. - PROVIDERS: - RAPIDS: - COMPUTE: False - FEATURES: ["countdischarge", "sumdurationdischarge", "countcharge", "sumdurationcharge", "avgconsumptionrate", "maxconsumptionrate"] - SRC_FOLDER: "rapids" # inside src/features/phone_battery - SRC_LANGUAGE: "python" - +# See https://www.rapids.science/latest/features/phone-screen/ PHONE_SCREEN: TABLE: screen PROVIDERS: @@ -164,190 +237,138 @@ PHONE_SCREEN: SRC_FOLDER: "rapids" # inside src/features/phone_screen SRC_LANGUAGE: "python" -PHONE_LIGHT: - TABLE: light - PROVIDERS: - RAPIDS: - COMPUTE: TRUE - FEATURES: ["count", "maxlux", "minlux", "avglux", "medianlux", "stdlux"] - SRC_FOLDER: "rapids" # inside src/features/phone_light - SRC_LANGUAGE: "python" - -PHONE_ACCELEROMETER: - TABLE: accelerometer - PROVIDERS: - RAPIDS: - COMPUTE: False - FEATURES: ["maxmagnitude", "minmagnitude", "avgmagnitude", "medianmagnitude", "stdmagnitude"] - SRC_FOLDER: "rapids" # inside src/features/phone_accelerometer - SRC_LANGUAGE: "python" - - PANDA: - COMPUTE: False - VALID_SENSED_MINUTES: False - FEATURES: - exertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"] - nonexertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"] - SRC_FOLDER: "panda" # inside src/features/phone_accelerometer - SRC_LANGUAGE: "python" - -PHONE_APPLICATIONS_FOREGROUND: - TABLE: applications_foreground - APPLICATION_CATEGORIES: - CATALOGUE_SOURCE: FILE # FILE (genres are read from CATALOGUE_FILE) or GOOGLE (genres are scrapped from the Play Store) - CATALOGUE_FILE: "data/external/stachl_application_genre_catalogue.csv" - UPDATE_CATALOGUE_FILE: False # if CATALOGUE_SOURCE is equal to FILE, whether or not to update CATALOGUE_FILE, if CATALOGUE_SOURCE is equal to GOOGLE all scraped genres will be saved to CATALOGUE_FILE - SCRAPE_MISSING_CATEGORIES: False # whether or not to scrape missing genres, only effective if CATALOGUE_SOURCE is equal to FILE. If CATALOGUE_SOURCE is equal to GOOGLE, all genres are scraped anyway - PROVIDERS: - RAPIDS: - COMPUTE: TRUE - SINGLE_CATEGORIES: ["all", "email"] - MULTIPLE_CATEGORIES: - social: ["socialnetworks", "socialmediatools"] - entertainment: ["entertainment", "gamingstrategy"] - SINGLE_APPS: ["top1global", "com.facebook.moments"] # There's no entropy for single apps - EXCLUDED_CATEGORIES: ["systemapp", "tvvideoapps"] - EXCLUDED_APPS: ["com.fitbit.FitbitMobile", "com.aware.plugin.upmc.cancer"] - FEATURES: ["count", "timeoffirstuse", "timeoflastuse", "frequencyentropy"] - SRC_FOLDER: "rapids" # inside src/features/phone_applications_foreground - SRC_LANGUAGE: "python" - -PHONE_WIFI_VISIBLE: - TABLE: "wifi" - PROVIDERS: - RAPIDS: - COMPUTE: TRUE - FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] - SRC_FOLDER: "rapids" # inside src/features/phone_wifi_visible - SRC_LANGUAGE: "r" - +# See https://www.rapids.science/latest/features/phone-wifi-connected/ PHONE_WIFI_CONNECTED: TABLE: "sensor_wifi" PROVIDERS: RAPIDS: - COMPUTE: TRUE + COMPUTE: False FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] SRC_FOLDER: "rapids" # inside src/features/phone_wifi_connected SRC_LANGUAGE: "r" -PHONE_CONVERSATION: - TABLE: - ANDROID: plugin_studentlife_audio_android - IOS: plugin_studentlife_audio - PROVIDERS: - RAPIDS: - COMPUTE: TRUE - FEATURES: ["minutessilence", "minutesnoise", "minutesvoice", "minutesunknown","sumconversationduration","avgconversationduration", - "sdconversationduration","minconversationduration","maxconversationduration","timefirstconversation","timelastconversation","noisesumenergy", - "noiseavgenergy","noisesdenergy","noiseminenergy","noisemaxenergy","voicesumenergy", - "voiceavgenergy","voicesdenergy","voiceminenergy","voicemaxenergy","silencesensedfraction","noisesensedfraction", - "voicesensedfraction","unknownsensedfraction","silenceexpectedfraction","noiseexpectedfraction","voiceexpectedfraction", - "unknownexpectedfraction","countconversation"] - RECORDING_MINUTES: 1 - PAUSED_MINUTES : 3 - SRC_FOLDER: "rapids" # inside src/features/phone_conversation - SRC_LANGUAGE: "python" - -############## FITBIT ########################################################## -################################################################################ - -FITBIT_HEARTRATE: - TABLE_FORMAT: JSON # JSON or CSV - TABLE: - JSON: fitbit_heartrate - CSV: - SUMMARY: heartrate_summary.csv - INTRADAY: heartrate_intraday.csv +# See https://www.rapids.science/latest/features/phone-wifi-visible/ +PHONE_WIFI_VISIBLE: + TABLE: "wifi" PROVIDERS: RAPIDS: COMPUTE: False - SUMMARY_FEATURES: ["restinghr"] # calories features' accuracy depend on the accuracy of the participants fitbit profile (e.g. height, weight) use these with care: ["caloriesoutofrange", "caloriesfatburn", "caloriescardio", "caloriespeak"] - INTRADAY_FEATURES: ["maxhr", "minhr", "avghr", "medianhr", "modehr", "stdhr", "diffmaxmodehr", "diffminmodehr", "entropyhr", "minutesonoutofrangezone", "minutesonfatburnzone", "minutesoncardiozone", "minutesonpeakzone"] + FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] + SRC_FOLDER: "rapids" # inside src/features/phone_wifi_visible + SRC_LANGUAGE: "r" -FITBIT_STEPS: - TABLE_FORMAT: JSON # JSON or CSV - TABLE: - JSON: fitbit_steps - CSV: - SUMMARY: steps_summary.csv - INTRADAY: steps_intraday.csv - EXCLUDE_SLEEP: # you can exclude sleep periods from the step features computation - EXCLUDE: False - TYPE: FIXED # FIXED OR FITBIT_BASED (configure FITBIT_SLEEP section) - FIXED: - START: "23:00" - END: "07:00" +######################################################################################################################## +# FITBIT # +######################################################################################################################## + +# See https://www.rapids.science/latest/setup/configuration/#device-data-source-configuration +FITBIT_DATA_CONFIGURATION: + SOURCE: + TYPE: DATABASE # DATABASE or FILES (set each [FITBIT_SENSOR][TABLE] attribute with a table name or a file path accordingly) + COLUMN_FORMAT: JSON # JSON or PLAIN_TEXT + DATABASE_GROUP: *database_group + DEVICE_ID_COLUMN: device_id # column name + TIMEZONE: + TYPE: SINGLE # Fitbit devices don't support time zones so we read this data in the timezone indicated by VALUE + VALUE: *timezone + +# Sensors ------ + +# See https://www.rapids.science/latest/features/fitbit-heartrate-summary/ +FITBIT_HEARTRATE_SUMMARY: + TABLE: heartrate_summary + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["maxrestinghr", "minrestinghr", "avgrestinghr", "medianrestinghr", "moderestinghr", "stdrestinghr", "diffmaxmoderestinghr", "diffminmoderestinghr", "entropyrestinghr"] # calories features' accuracy depend on the accuracy of the participants fitbit profile (e.g. height, weight) use these with care: ["sumcaloriesoutofrange", "maxcaloriesoutofrange", "mincaloriesoutofrange", "avgcaloriesoutofrange", "mediancaloriesoutofrange", "stdcaloriesoutofrange", "entropycaloriesoutofrange", "sumcaloriesfatburn", "maxcaloriesfatburn", "mincaloriesfatburn", "avgcaloriesfatburn", "mediancaloriesfatburn", "stdcaloriesfatburn", "entropycaloriesfatburn", "sumcaloriescardio", "maxcaloriescardio", "mincaloriescardio", "avgcaloriescardio", "mediancaloriescardio", "stdcaloriescardio", "entropycaloriescardio", "sumcaloriespeak", "maxcaloriespeak", "mincaloriespeak", "avgcaloriespeak", "mediancaloriespeak", "stdcaloriespeak", "entropycaloriespeak"] + SRC_FOLDER: "rapids" # inside src/features/fitbit_heartrate_summary + SRC_LANGUAGE: "python" + +# See https://www.rapids.science/latest/features/fitbit-heartrate-intraday/ +FITBIT_HEARTRATE_INTRADAY: + TABLE: heartrate_intraday + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["maxhr", "minhr", "avghr", "medianhr", "modehr", "stdhr", "diffmaxmodehr", "diffminmodehr", "entropyhr", "minutesonoutofrangezone", "minutesonfatburnzone", "minutesoncardiozone", "minutesonpeakzone"] + SRC_FOLDER: "rapids" # inside src/features/fitbit_heartrate_intraday + SRC_LANGUAGE: "python" + +# See https://www.rapids.science/latest/features/fitbit-sleep-summary/ +FITBIT_SLEEP_SUMMARY: + TABLE: sleep_summary + SLEEP_EPISODE_TIMESTAMP: end # summary sleep episodes are considered as events based on either the start timestamp or end timestamp. + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["countepisode", "avgefficiency", "sumdurationafterwakeup", "sumdurationasleep", "sumdurationawake", "sumdurationtofallasleep", "sumdurationinbed", "avgdurationafterwakeup", "avgdurationasleep", "avgdurationawake", "avgdurationtofallasleep", "avgdurationinbed"] + SLEEP_TYPES: ["main", "nap", "all"] + SRC_FOLDER: "rapids" # inside src/features/fitbit_sleep_summary + SRC_LANGUAGE: "python" + +# See https://www.rapids.science/latest/features/fitbit-steps-summary/ +FITBIT_STEPS_SUMMARY: + TABLE: steps_summary + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["maxsumsteps", "minsumsteps", "avgsumsteps", "mediansumsteps", "stdsumsteps"] + SRC_FOLDER: "rapids" # inside src/features/fitbit_steps_summary + SRC_LANGUAGE: "python" + +# See https://www.rapids.science/latest/features/fitbit-steps-intraday/ +FITBIT_STEPS_INTRADAY: + TABLE: steps_intraday PROVIDERS: RAPIDS: COMPUTE: False FEATURES: - ALL_STEPS: ["sumallsteps", "maxallsteps", "minallsteps", "avgallsteps", "stdallsteps"] + STEPS: ["sum", "max", "min", "avg", "std"] SEDENTARY_BOUT: ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "stdduration"] ACTIVE_BOUT: ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "stdduration"] THRESHOLD_ACTIVE_BOUT: 10 # steps INCLUDE_ZERO_STEP_ROWS: False + SRC_FOLDER: "rapids" # inside src/features/fitbit_steps_intraday + SRC_LANGUAGE: "python" -FITBIT_SLEEP: - TABLE_FORMAT: JSON # JSON or CSV - TABLE: - JSON: fitbit_sleep - CSV: - SUMMARY: sleep_summary.csv - INTRADAY: sleep_intraday.csv - PROVIDERS: - RAPIDS: - COMPUTE: False - SLEEP_TYPES: ["main", "nap", "all"] - SUMMARY_FEATURES: ["sumdurationafterwakeup", "sumdurationasleep", "sumdurationawake", "sumdurationtofallasleep", "sumdurationinbed", "avgefficiency", "countepisode"] +# FITBIT_CALORIES: +# TABLE_FORMAT: JSON # JSON or CSV. If your JSON or CSV data are files change [DEVICE_DATA][FITBIT][SOURCE][TYPE] to FILES +# TABLE: +# JSON: fitbit_calories +# CSV: +# SUMMARY: calories_summary +# INTRADAY: calories_intraday +# PROVIDERS: +# RAPIDS: +# COMPUTE: False +# FEATURES: [] -FITBIT_CALORIES: - TABLE_FORMAT: JSON # JSON or CSV - TABLE: - JSON: fitbit_calories - CSV: - SUMMARY: calories_summary.csv - INTRADAY: calories_intraday.csv - PROVIDERS: - RAPIDS: - COMPUTE: False - FEATURES: [] -### Visualizations ############################################################# -################################################################################ -HEATMAP_FEATURES_CORRELATIONS: +######################################################################################################################## +# PLOTS # +######################################################################################################################## + +# Data quality +HISTOGRAM_PHONE_DATA_YIELD: + PLOT: False + +HEATMAP_PHONE_DATA_YIELD_PER_PARTICIPANT_PER_TIME_SEGMENT: + PLOT: False + +HEATMAP_SENSORS_PER_MINUTE_PER_TIME_SEGMENT: + PLOT: False + +HEATMAP_SENSOR_ROW_COUNT_PER_TIME_SEGMENT: + PLOT: False + SENSORS: [PHONE_ACCELEROMETER, PHONE_ACTIVITY_RECOGNITION, PHONE_APPLICATIONS_FOREGROUND, PHONE_BATTERY, PHONE_BLUETOOTH, PHONE_CALLS, PHONE_CONVERSATION, PHONE_LIGHT, PHONE_LOCATIONS, PHONE_MESSAGES, PHONE_SCREEN, PHONE_WIFI_CONNECTED, PHONE_WIFI_VISIBLE] + +# Features +HEATMAP_FEATURE_CORRELATION_MATRIX: PLOT: False MIN_ROWS_RATIO: 0.5 - MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day - MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour - PHONE_FEATURES: [accelerometer, activity_recognition, applications_foreground, battery, calls_incoming, calls_missed, calls_outgoing, conversation, light, location_doryab, messages_received, messages_sent, screen] - FITBIT_FEATURES: [fitbit_heartrate, fitbit_step, fitbit_sleep] CORR_THRESHOLD: 0.1 CORR_METHOD: "pearson" # choose from {"pearson", "kendall", "spearman"} -HISTOGRAM_VALID_SENSED_HOURS: - PLOT: False - MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day - MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour - -HEATMAP_DAYS_BY_SENSORS: - PLOT: False - MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day - MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour - EXPECTED_NUM_OF_DAYS: -1 - DB_TABLES: [accelerometer, applications_foreground, battery, bluetooth, calls, light, locations, messages, screen, wifi, sensor_wifi, plugin_google_activity_recognition, plugin_ios_activity_recognition, plugin_studentlife_audio_android, plugin_studentlife_audio] - -HEATMAP_SENSED_BINS: - PLOT: False - BIN_SIZE: *bin_size - -OVERALL_COMPLIANCE_HEATMAP: - PLOT: False - ONLY_SHOW_VALID_DAYS: False - EXPECTED_NUM_OF_DAYS: -1 - BIN_SIZE: *bin_size - MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day - MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour -