From dc2b4621451a3d36cf08d7446f3feff3795868ec Mon Sep 17 00:00:00 2001 From: Primoz Date: Wed, 16 Mar 2022 13:30:19 +0000 Subject: [PATCH] Reseting files to defaults - for Minimal Working Example --- Snakefile | 7 -- config.yaml | 126 +++++++++--------------- data/external/timesegments_periodic.csv | 6 -- rules/preprocessing.smk | 32 +----- 4 files changed, 48 insertions(+), 123 deletions(-) diff --git a/Snakefile b/Snakefile index 4ea33827..7dc5ad3a 100644 --- a/Snakefile +++ b/Snakefile @@ -5,7 +5,6 @@ include: "rules/common.smk" include: "rules/renv.smk" include: "rules/preprocessing.smk" include: "rules/features.smk" -include: "rules/models.smk" include: "rules/reports.smk" import itertools @@ -403,12 +402,6 @@ for provider in config["ALL_CLEANING_OVERALL"]["PROVIDERS"].keys(): if config["ALL_CLEANING_OVERALL"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/processed/features/all_participants/all_sensor_features_cleaned_" + provider.lower() +".csv")) -# Baseline features -files_to_compute.extend(expand("data/raw/baseline_merged.csv")) -files_to_compute.extend(expand("data/raw/{pid}/participant_baseline_raw.csv", pid=config["PIDS"])) -files_to_compute.extend(expand("data/interim/{pid}/baseline_questionnaires.csv", pid=config["PIDS"])) -files_to_compute.extend(expand("data/processed/features/{pid}/baseline_features.csv", pid=config["PIDS"])) - rule all: input: files_to_compute diff --git a/config.yaml b/config.yaml index 4d7a699c..4b277ac5 100644 --- a/config.yaml +++ b/config.yaml @@ -3,38 +3,36 @@ ######################################################################################################################## # See https://www.rapids.science/latest/setup/configuration/#participant-files -PIDS: ['p031', 'p032', 'p033', 'p034', 'p035', 'p036', 'p037', 'p038', 'p039', 'p040', 'p042', 'p043', 'p044', 'p045', 'p046', 'p049', 'p050', 'p052', 'p053', 'p054', 'p055', 'p057', 'p058', 'p059', 'p060', 'p061', 'p062', 'p064', 'p067', 'p068', 'p069', 'p070', 'p071', 'p072', 'p073', 'p074', 'p075', 'p076', 'p077', 'p078', 'p079', 'p080', 'p081', 'p082', 'p083', 'p084', 'p085', 'p086', 'p088', 'p089', 'p090', 'p091', 'p092', 'p093', 'p106', 'p107'] +PIDS: [p01] # See https://www.rapids.science/latest/setup/configuration/#automatic-creation-of-participant-files CREATE_PARTICIPANT_FILES: - USERNAMES_CSV: "data/external/main_study_usernames.csv" - CSV_FILE_PATH: "data/external/main_study_participants.csv" # see docs for required format + CSV_FILE_PATH: "data/external/example_participants.csv" # see docs for required format PHONE_SECTION: ADD: True IGNORED_DEVICE_IDS: [] FITBIT_SECTION: - ADD: False + ADD: True IGNORED_DEVICE_IDS: [] EMPATICA_SECTION: - ADD: False + ADD: True IGNORED_DEVICE_IDS: [] # See https://www.rapids.science/latest/setup/configuration/#time-segments TIME_SEGMENTS: &time_segments TYPE: PERIODIC # FREQUENCY, PERIODIC, EVENT - FILE: "data/external/timesegments_daily.csv" - INCLUDE_PAST_PERIODIC_SEGMENTS: TRUE # Only relevant if TYPE=PERIODIC, see docs + FILE: "data/external/timesegments_periodic.csv" + INCLUDE_PAST_PERIODIC_SEGMENTS: FALSE # Only relevant if TYPE=PERIODIC, see docs # See https://www.rapids.science/latest/setup/configuration/#timezone-of-your-study TIMEZONE: - TYPE: MULTIPLE + TYPE: SINGLE SINGLE: - TZCODE: Europe/Ljubljana + TZCODE: America/New_York MULTIPLE: - TZ_FILE: data/external/timezone.csv - TZCODES_FILE: data/external/multiple_timezones.csv - IF_MISSING_TZCODE: USE_DEFAULT - DEFAULT_TZCODE: Europe/Ljubljana + TZCODES_FILE: data/external/multiple_timezones_example.csv + IF_MISSING_TZCODE: STOP + DEFAULT_TZCODE: America/New_York FITBIT: ALLOW_MULTIPLE_TZ_PER_DEVICE: False INFER_FROM_SMARTPHONE_TZ: False @@ -45,15 +43,12 @@ TIMEZONE: # See https://www.rapids.science/latest/setup/configuration/#data-stream-configuration PHONE_DATA_STREAMS: - USE: aware_postgresql + USE: aware_csv # AVAILABLE: aware_mysql: DATABASE_GROUP: MY_GROUP - aware_postgresql: - DATABASE_GROUP: PSQL_STRAW - aware_csv: FOLDER: data/external/aware_csv @@ -82,12 +77,12 @@ PHONE_ACCELEROMETER: # See https://www.rapids.science/latest/features/phone-activity-recognition/ PHONE_ACTIVITY_RECOGNITION: CONTAINER: - ANDROID: google_ar + ANDROID: plugin_google_activity_recognition IOS: plugin_ios_activity_recognition EPISODE_THRESHOLD_BETWEEN_ROWS: 5 # minutes. Max time difference for two consecutive rows to be considered within the same AR episode. PROVIDERS: RAPIDS: - COMPUTE: True + COMPUTE: False FEATURES: ["count", "mostcommonactivity", "countuniqueactivities", "durationstationary", "durationmobile", "durationvehicle"] ACTIVITY_CLASSES: STATIONARY: ["still", "tilting"] @@ -107,17 +102,16 @@ PHONE_APPLICATIONS_CRASHES: # See https://www.rapids.science/latest/features/phone-applications-foreground/ PHONE_APPLICATIONS_FOREGROUND: - CONTAINER: applications + CONTAINER: applications_foreground APPLICATION_CATEGORIES: CATALOGUE_SOURCE: FILE # FILE (genres are read from CATALOGUE_FILE) or GOOGLE (genres are scrapped from the Play Store) CATALOGUE_FILE: "data/external/stachl_application_genre_catalogue.csv" - PACKAGE_NAMES_HASHED: True UPDATE_CATALOGUE_FILE: False # if CATALOGUE_SOURCE is equal to FILE, whether or not to update CATALOGUE_FILE, if CATALOGUE_SOURCE is equal to GOOGLE all scraped genres will be saved to CATALOGUE_FILE SCRAPE_MISSING_CATEGORIES: False # whether or not to scrape missing genres, only effective if CATALOGUE_SOURCE is equal to FILE. If CATALOGUE_SOURCE is equal to GOOGLE, all genres are scraped anyway PROVIDERS: RAPIDS: - COMPUTE: True - INCLUDE_EPISODE_FEATURES: True + COMPUTE: False + INCLUDE_EPISODE_FEATURES: False SINGLE_CATEGORIES: ["all", "email"] MULTIPLE_CATEGORIES: social: ["socialnetworks", "socialmediatools"] @@ -127,7 +121,7 @@ PHONE_APPLICATIONS_FOREGROUND: dating: ["com.tinder", "com.relance.happycouple", "com.kiwi.joyride"] SINGLE_APPS: ["top1global", "com.facebook.moments", "com.google.android.youtube", "com.twitter.android"] # There's no entropy for single apps EXCLUDED_CATEGORIES: [] - EXCLUDED_APPS: ["com.fitbit.FitbitMobile", "com.aware.plugin.upmc.cancer"] # TODO list system apps? + EXCLUDED_APPS: ["com.fitbit.FitbitMobile", "com.aware.plugin.upmc.cancer"] FEATURES: APP_EVENTS: ["countevent", "timeoffirstuse", "timeoflastuse", "frequencyentropy"] APP_EPISODES: ["countepisode", "minduration", "maxduration", "meanduration", "sumduration"] @@ -137,7 +131,7 @@ PHONE_APPLICATIONS_FOREGROUND: # See https://www.rapids.science/latest/features/phone-applications-notifications/ PHONE_APPLICATIONS_NOTIFICATIONS: - CONTAINER: notifications + CONTAINER: applications_notifications APPLICATION_CATEGORIES: CATALOGUE_SOURCE: FILE # FILE (genres are read from CATALOGUE_FILE) or GOOGLE (genres are scrapped from the Play Store) CATALOGUE_FILE: "data/external/stachl_application_genre_catalogue.csv" @@ -151,7 +145,7 @@ PHONE_BATTERY: EPISODE_THRESHOLD_BETWEEN_ROWS: 30 # minutes. Max time difference for two consecutive rows to be considered within the same battery episode. PROVIDERS: RAPIDS: - COMPUTE: True + COMPUTE: False FEATURES: ["countdischarge", "sumdurationdischarge", "countcharge", "sumdurationcharge", "avgconsumptionrate", "maxconsumptionrate"] SRC_SCRIPT: src/features/phone_battery/rapids/main.py @@ -160,12 +154,12 @@ PHONE_BLUETOOTH: CONTAINER: bluetooth PROVIDERS: RAPIDS: - COMPUTE: True + COMPUTE: False FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] SRC_SCRIPT: src/features/phone_bluetooth/rapids/main.R DORYAB: - COMPUTE: True + COMPUTE: False FEATURES: ALL: DEVICES: ["countscans", "uniquedevices", "meanscans", "stdscans"] @@ -183,7 +177,7 @@ PHONE_BLUETOOTH: # See https://www.rapids.science/latest/features/phone-calls/ PHONE_CALLS: - CONTAINER: call + CONTAINER: calls.csv PROVIDERS: RAPIDS: COMPUTE: True @@ -196,7 +190,7 @@ PHONE_CALLS: SRC_SCRIPT: src/features/phone_calls/rapids/main.R # See https://www.rapids.science/latest/features/phone-conversation/ -PHONE_CONVERSATION: # TODO Adapt for speech +PHONE_CONVERSATION: CONTAINER: ANDROID: plugin_studentlife_audio_android IOS: plugin_studentlife_audio @@ -215,21 +209,10 @@ PHONE_CONVERSATION: # TODO Adapt for speech # See https://www.rapids.science/latest/features/phone-data-yield/ PHONE_DATA_YIELD: - SENSORS: [#PHONE_ACCELEROMETER, - PHONE_ACTIVITY_RECOGNITION, - PHONE_APPLICATIONS_FOREGROUND, - PHONE_APPLICATIONS_NOTIFICATIONS, - PHONE_BATTERY, - PHONE_BLUETOOTH, - PHONE_CALLS, - PHONE_LIGHT, - PHONE_LOCATIONS, - PHONE_MESSAGES, - PHONE_SCREEN, - PHONE_WIFI_VISIBLE] + SENSORS: [] PROVIDERS: RAPIDS: - COMPUTE: True + COMPUTE: False FEATURES: [ratiovalidyieldedminutes, ratiovalidyieldedhours] MINUTE_RATIO_THRESHOLD_FOR_VALID_YIELDED_HOURS: 0.5 # 0 to 1, minimum percentage of valid minutes in an hour to be considered valid. SRC_SCRIPT: src/features/phone_data_yield/rapids/main.R @@ -245,10 +228,10 @@ PHONE_KEYBOARD: # See https://www.rapids.science/latest/features/phone-light/ PHONE_LIGHT: - CONTAINER: light_sensor + CONTAINER: light PROVIDERS: RAPIDS: - COMPUTE: True + COMPUTE: False FEATURES: ["count", "maxlux", "minlux", "avglux", "medianlux", "stdlux"] SRC_SCRIPT: src/features/phone_light/rapids/main.py @@ -262,7 +245,7 @@ PHONE_LOCATIONS: PROVIDERS: DORYAB: - COMPUTE: True + COMPUTE: False FEATURES: ["locationvariance","loglocationvariance","totaldistance","avgspeed","varspeed", "numberofsignificantplaces","numberlocationtransitions","radiusgyration","timeattop1location","timeattop2location","timeattop3location","movingtostaticratio","outlierstimepercent","maxlengthstayatclusters","minlengthstayatclusters","avglengthstayatclusters","stdlengthstayatclusters","locationentropy","normalizedlocationentropy","timeathome", "homelabel"] DBSCAN_EPS: 100 # meters DBSCAN_MINSAMPLES: 5 @@ -277,7 +260,7 @@ PHONE_LOCATIONS: SRC_SCRIPT: src/features/phone_locations/doryab/main.py BARNETT: - COMPUTE: True + COMPUTE: False FEATURES: ["hometime","disttravelled","rog","maxdiam","maxhomedist","siglocsvisited","avgflightlen","stdflightlen","avgflightdur","stdflightdur","probpause","siglocentropy","circdnrtn","wkenddayrtn"] IF_MULTIPLE_TIMEZONES: USE_MOST_COMMON MINUTES_DATA_USED: False # Use this for quality control purposes, how many minutes of data (location coordinates gruped by minute) were used to compute features @@ -292,10 +275,10 @@ PHONE_LOG: # See https://www.rapids.science/latest/features/phone-messages/ PHONE_MESSAGES: - CONTAINER: sms + CONTAINER: messages PROVIDERS: RAPIDS: - COMPUTE: True + COMPUTE: False MESSAGES_TYPES : [received, sent] FEATURES: received: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact] @@ -307,7 +290,7 @@ PHONE_SCREEN: CONTAINER: screen PROVIDERS: RAPIDS: - COMPUTE: True + COMPUTE: False REFERENCE_HOUR_FIRST_USE: 0 IGNORE_EPISODES_SHORTER_THAN: 0 # in minutes, set to 0 to disable IGNORE_EPISODES_LONGER_THAN: 360 # in minutes, set to 0 to disable @@ -329,7 +312,7 @@ PHONE_WIFI_VISIBLE: CONTAINER: wifi PROVIDERS: RAPIDS: - COMPUTE: True + COMPUTE: False FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] SRC_SCRIPT: src/features/phone_wifi_visible/rapids/main.R @@ -557,16 +540,16 @@ EMPATICA_TAGS: # See https://www.rapids.science/latest/visualizations/data-quality-visualizations/#1-histograms-of-phone-data-yield HISTOGRAM_PHONE_DATA_YIELD: - PLOT: True + PLOT: False # See https://www.rapids.science/latest/visualizations/data-quality-visualizations/#2-heatmaps-of-overall-data-yield HEATMAP_PHONE_DATA_YIELD_PER_PARTICIPANT_PER_TIME_SEGMENT: - PLOT: True + PLOT: False TIME: RELATIVE_TIME # ABSOLUTE_TIME or RELATIVE_TIME # See https://www.rapids.science/latest/visualizations/data-quality-visualizations/#3-heatmap-of-recorded-phone-sensors HEATMAP_SENSORS_PER_MINUTE_PER_TIME_SEGMENT: - PLOT: True + PLOT: False # See https://www.rapids.science/latest/visualizations/data-quality-visualizations/#4-heatmap-of-sensor-row-count HEATMAP_SENSOR_ROW_COUNT_PER_TIME_SEGMENT: @@ -577,7 +560,7 @@ HEATMAP_SENSOR_ROW_COUNT_PER_TIME_SEGMENT: # See https://www.rapids.science/latest/visualizations/feature-visualizations/#1-heatmap-correlation-matrix HEATMAP_FEATURE_CORRELATION_MATRIX: - PLOT: True + PLOT: False MIN_ROWS_RATIO: 0.5 CORR_THRESHOLD: 0.1 CORR_METHOD: "pearson" # choose from {"pearson", "kendall", "spearman"} @@ -590,17 +573,17 @@ HEATMAP_FEATURE_CORRELATION_MATRIX: ALL_CLEANING_INDIVIDUAL: PROVIDERS: RAPIDS: - COMPUTE: True + COMPUTE: False IMPUTE_SELECTED_EVENT_FEATURES: COMPUTE: True MIN_DATA_YIELDED_MINUTES_TO_IMPUTE: 0.33 COLS_NAN_THRESHOLD: 0.3 # set to 1 to disable COLS_VAR_THRESHOLD: True - ROWS_NAN_THRESHOLD: 1 # set to 1 to disable + ROWS_NAN_THRESHOLD: 0.3 # set to 1 to disable DATA_YIELD_FEATURE: RATIO_VALID_YIELDED_HOURS # RATIO_VALID_YIELDED_HOURS or RATIO_VALID_YIELDED_MINUTES - DATA_YIELD_RATIO_THRESHOLD: 0.3 # set to 0 to disable + DATA_YIELD_RATIO_THRESHOLD: 0.5 # set to 0 to disable DROP_HIGHLY_CORRELATED_FEATURES: - COMPUTE: False + COMPUTE: True MIN_OVERLAP_FOR_CORR_THRESHOLD: 0.5 CORR_THRESHOLD: 0.95 SRC_SCRIPT: src/features/all_cleaning_individual/rapids/main.R @@ -608,32 +591,17 @@ ALL_CLEANING_INDIVIDUAL: ALL_CLEANING_OVERALL: PROVIDERS: RAPIDS: - COMPUTE: True + COMPUTE: False IMPUTE_SELECTED_EVENT_FEATURES: COMPUTE: True MIN_DATA_YIELDED_MINUTES_TO_IMPUTE: 0.33 COLS_NAN_THRESHOLD: 0.3 # set to 1 to disable COLS_VAR_THRESHOLD: True - ROWS_NAN_THRESHOLD: 1 # set to 1 to disable + ROWS_NAN_THRESHOLD: 0.3 # set to 1 to disable DATA_YIELD_FEATURE: RATIO_VALID_YIELDED_HOURS # RATIO_VALID_YIELDED_HOURS or RATIO_VALID_YIELDED_MINUTES - DATA_YIELD_RATIO_THRESHOLD: 0.3 # set to 0 to disable + DATA_YIELD_RATIO_THRESHOLD: 0.5 # set to 0 to disable DROP_HIGHLY_CORRELATED_FEATURES: - COMPUTE: False + COMPUTE: True MIN_OVERLAP_FOR_CORR_THRESHOLD: 0.5 CORR_THRESHOLD: 0.95 - SRC_SCRIPT: src/features/all_cleaning_overall/rapids/main.R - -######################################################################################################################## -# Analysis Workflow Example # -######################################################################################################################## - -PARAMS_FOR_ANALYSIS: - BASELINE: - FOLDER: data/external/baseline - CONTAINER: [results-survey637813_final.csv, # Slovenia - results-survey358134_final.csv, # Belgium 1 - results-survey413767_final.csv # Belgium 2 - ] - QUESTION_LIST: survey637813+question_text.csv - FEATURES: [age, gender, startlanguage, demand, control, demand_control_ratio] - CATEGORICAL_FEATURES: [gender] + SRC_SCRIPT: src/features/all_cleaning_overall/rapids/main.R \ No newline at end of file diff --git a/data/external/timesegments_periodic.csv b/data/external/timesegments_periodic.csv index d52d92fa..0a26eb6b 100644 --- a/data/external/timesegments_periodic.csv +++ b/data/external/timesegments_periodic.csv @@ -1,9 +1,3 @@ label,start_time,length,repeats_on,repeats_value -threeday,00:00:00,2D 23H 59M 59S,every_day,0 daily,00:00:00,23H 59M 59S,every_day,0 -morning,06:00:00,5H 59M 59S,every_day,0 -afternoon,12:00:00,5H 59M 59S,every_day,0 -evening,18:00:00,5H 59M 59S,every_day,0 night,00:00:00,5H 59M 59S,every_day,0 -two_weeks_overlapping,00:00:00,13D 23H 59M 59S,every_day,0 -weekends,00:00:00,2D 23H 59M 59S,wday,5 diff --git a/rules/preprocessing.smk b/rules/preprocessing.smk index cd83b95b..f40495fc 100644 --- a/rules/preprocessing.smk +++ b/rules/preprocessing.smk @@ -4,36 +4,6 @@ rule create_example_participant_files: shell: "echo 'PHONE:\n DEVICE_IDS: [a748ee1a-1d0b-4ae9-9074-279a2b6ba524]\n PLATFORMS: [android]\n LABEL: test-01\n START_DATE: 2020-04-23 00:00:00\n END_DATE: 2020-05-04 23:59:59\nFITBIT:\n DEVICE_IDS: [a748ee1a-1d0b-4ae9-9074-279a2b6ba524]\n LABEL: test-01\n START_DATE: 2020-04-23 00:00:00\n END_DATE: 2020-05-04 23:59:59\n' >> ./data/external/participant_files/example01.yaml && echo 'PHONE:\n DEVICE_IDS: [13dbc8a3-dae3-4834-823a-4bc96a7d459d]\n PLATFORMS: [ios]\n LABEL: test-02\n START_DATE: 2020-04-23 00:00:00\n END_DATE: 2020-05-04 23:59:59\nFITBIT:\n DEVICE_IDS: [13dbc8a3-dae3-4834-823a-4bc96a7d459d]\n LABEL: test-02\n START_DATE: 2020-04-23 00:00:00\n END_DATE: 2020-05-04 23:59:59\n' >> ./data/external/participant_files/example02.yaml" -rule query_usernames_device_empatica_ids: - params: - baseline_folder = "/mnt/e/STRAWbaseline/" - output: - usernames_file = config["CREATE_PARTICIPANT_FILES"]["USERNAMES_CSV"], - timezone_file = config["TIMEZONE"]["MULTIPLE"]["TZ_FILE"] - script: - "../../participants/prepare_usernames_file.py" - -rule prepare_tzcodes_file: - input: - timezone_file = config["TIMEZONE"]["MULTIPLE"]["TZ_FILE"] - output: - tzcodes_file = config["TIMEZONE"]["MULTIPLE"]["TZCODES_FILE"] - script: - "../tools/create_multi_timezones_file.py" - -rule prepare_participants_csv: - input: - username_list = config["CREATE_PARTICIPANT_FILES"]["USERNAMES_CSV"] - params: - data_configuration = config["PHONE_DATA_STREAMS"][config["PHONE_DATA_STREAMS"]["USE"]], - participants_table = "participants", - device_id_table = "esm", - start_end_date_table = "esm" - output: - participants_file = config["CREATE_PARTICIPANT_FILES"]["CSV_FILE_PATH"] - script: - "../src/data/translate_usernames_into_participants_data.R" - rule create_participants_files: input: participants_file = config["CREATE_PARTICIPANT_FILES"]["CSV_FILE_PATH"] @@ -249,4 +219,4 @@ rule empatica_readable_datetime: output: "data/raw/{pid}/empatica_{sensor}_with_datetime.csv" script: - "../src/data/datetime/readable_datetime.R" + "../src/data/datetime/readable_datetime.R" \ No newline at end of file