From 712ff748981b735277acd8aef3184cbdcf894fb6 Mon Sep 17 00:00:00 2001 From: junos Date: Wed, 8 Dec 2021 18:37:34 +0100 Subject: [PATCH] Set table names and calculate all relevant features. --- config.yaml | 212 +++++++--------------------------------------------- 1 file changed, 26 insertions(+), 186 deletions(-) diff --git a/config.yaml b/config.yaml index 69f6b46d..15a9aabe 100644 --- a/config.yaml +++ b/config.yaml @@ -63,12 +63,12 @@ PHONE_ACCELEROMETER: CONTAINER: accelerometer PROVIDERS: RAPIDS: - COMPUTE: False + COMPUTE: True FEATURES: ["maxmagnitude", "minmagnitude", "avgmagnitude", "medianmagnitude", "stdmagnitude"] SRC_SCRIPT: src/features/phone_accelerometer/rapids/main.py PANDA: - COMPUTE: False + COMPUTE: True VALID_SENSED_MINUTES: False FEATURES: exertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"] @@ -78,12 +78,12 @@ PHONE_ACCELEROMETER: # See https://www.rapids.science/latest/features/phone-activity-recognition/ PHONE_ACTIVITY_RECOGNITION: CONTAINER: - ANDROID: plugin_google_activity_recognition + ANDROID: google_ar IOS: plugin_ios_activity_recognition EPISODE_THRESHOLD_BETWEEN_ROWS: 5 # minutes. Max time difference for two consecutive rows to be considered within the same AR episode. PROVIDERS: RAPIDS: - COMPUTE: False + COMPUTE: True FEATURES: ["count", "mostcommonactivity", "countuniqueactivities", "durationstationary", "durationmobile", "durationvehicle"] ACTIVITY_CLASSES: STATIONARY: ["still", "tilting"] @@ -91,19 +91,9 @@ PHONE_ACTIVITY_RECOGNITION: VEHICLE: ["in_vehicle"] SRC_SCRIPT: src/features/phone_activity_recognition/rapids/main.py -# See https://www.rapids.science/latest/features/phone-applications-crashes/ -PHONE_APPLICATIONS_CRASHES: - CONTAINER: applications_crashes - APPLICATION_CATEGORIES: - CATALOGUE_SOURCE: FILE # FILE (genres are read from CATALOGUE_FILE) or GOOGLE (genres are scrapped from the Play Store) - CATALOGUE_FILE: "data/external/stachl_application_genre_catalogue.csv" - UPDATE_CATALOGUE_FILE: False # if CATALOGUE_SOURCE is equal to FILE, whether or not to update CATALOGUE_FILE, if CATALOGUE_SOURCE is equal to GOOGLE all scraped genres will be saved to CATALOGUE_FILE - SCRAPE_MISSING_CATEGORIES: False # whether or not to scrape missing genres, only effective if CATALOGUE_SOURCE is equal to FILE. If CATALOGUE_SOURCE is equal to GOOGLE, all genres are scraped anyway - PROVIDERS: # None implemented yet but this sensor can be used in PHONE_DATA_YIELD - # See https://www.rapids.science/latest/features/phone-applications-foreground/ PHONE_APPLICATIONS_FOREGROUND: - CONTAINER: applications_foreground + CONTAINER: applications APPLICATION_CATEGORIES: CATALOGUE_SOURCE: FILE # FILE (genres are read from CATALOGUE_FILE) or GOOGLE (genres are scrapped from the Play Store) CATALOGUE_FILE: "data/external/stachl_application_genre_catalogue.csv" @@ -111,8 +101,8 @@ PHONE_APPLICATIONS_FOREGROUND: SCRAPE_MISSING_CATEGORIES: False # whether or not to scrape missing genres, only effective if CATALOGUE_SOURCE is equal to FILE. If CATALOGUE_SOURCE is equal to GOOGLE, all genres are scraped anyway PROVIDERS: RAPIDS: - COMPUTE: False - INCLUDE_EPISODE_FEATURES: False + COMPUTE: True + INCLUDE_EPISODE_FEATURES: True SINGLE_CATEGORIES: ["all", "email"] MULTIPLE_CATEGORIES: social: ["socialnetworks", "socialmediatools"] @@ -122,7 +112,7 @@ PHONE_APPLICATIONS_FOREGROUND: dating: ["com.tinder", "com.relance.happycouple", "com.kiwi.joyride"] SINGLE_APPS: ["top1global", "com.facebook.moments", "com.google.android.youtube", "com.twitter.android"] # There's no entropy for single apps EXCLUDED_CATEGORIES: [] - EXCLUDED_APPS: ["com.fitbit.FitbitMobile", "com.aware.plugin.upmc.cancer"] + EXCLUDED_APPS: ["com.fitbit.FitbitMobile", "com.aware.plugin.upmc.cancer"] # TODO list system apps? FEATURES: APP_EVENTS: ["countevent", "timeoffirstuse", "timeoflastuse", "frequencyentropy"] APP_EPISODES: ["countepisode", "minduration", "maxduration", "meanduration", "sumduration"] @@ -132,7 +122,7 @@ PHONE_APPLICATIONS_FOREGROUND: # See https://www.rapids.science/latest/features/phone-applications-notifications/ PHONE_APPLICATIONS_NOTIFICATIONS: - CONTAINER: applications_notifications + CONTAINER: notifications APPLICATION_CATEGORIES: CATALOGUE_SOURCE: FILE # FILE (genres are read from CATALOGUE_FILE) or GOOGLE (genres are scrapped from the Play Store) CATALOGUE_FILE: "data/external/stachl_application_genre_catalogue.csv" @@ -146,7 +136,7 @@ PHONE_BATTERY: EPISODE_THRESHOLD_BETWEEN_ROWS: 30 # minutes. Max time difference for two consecutive rows to be considered within the same battery episode. PROVIDERS: RAPIDS: - COMPUTE: False + COMPUTE: True FEATURES: ["countdischarge", "sumdurationdischarge", "countcharge", "sumdurationcharge", "avgconsumptionrate", "maxconsumptionrate"] SRC_SCRIPT: src/features/phone_battery/rapids/main.py @@ -155,12 +145,12 @@ PHONE_BLUETOOTH: CONTAINER: bluetooth PROVIDERS: RAPIDS: - COMPUTE: False + COMPUTE: True FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] SRC_SCRIPT: src/features/phone_bluetooth/rapids/main.R DORYAB: - COMPUTE: False + COMPUTE: True FEATURES: ALL: DEVICES: ["countscans", "uniquedevices", "meanscans", "stdscans"] @@ -191,7 +181,7 @@ PHONE_CALLS: SRC_SCRIPT: src/features/phone_calls/rapids/main.R # See https://www.rapids.science/latest/features/phone-conversation/ -PHONE_CONVERSATION: +PHONE_CONVERSATION: # TODO Adapt for speech CONTAINER: ANDROID: plugin_studentlife_audio_android IOS: plugin_studentlife_audio @@ -210,29 +200,20 @@ PHONE_CONVERSATION: # See https://www.rapids.science/latest/features/phone-data-yield/ PHONE_DATA_YIELD: - SENSORS: [] + SENSORS: [PHONE_LIGHT] PROVIDERS: RAPIDS: - COMPUTE: False + COMPUTE: True FEATURES: [ratiovalidyieldedminutes, ratiovalidyieldedhours] MINUTE_RATIO_THRESHOLD_FOR_VALID_YIELDED_HOURS: 0.5 # 0 to 1, minimum percentage of valid minutes in an hour to be considered valid. SRC_SCRIPT: src/features/phone_data_yield/rapids/main.R -# See https://www.rapids.science/latest/features/phone-keyboard/ -PHONE_KEYBOARD: - CONTAINER: keyboard - PROVIDERS: - RAPIDS: - COMPUTE: False - FEATURES: ["sessioncount","averageinterkeydelay","averagesessionlength","changeintextlengthlessthanminusone","changeintextlengthequaltominusone","changeintextlengthequaltoone","changeintextlengthmorethanone","maxtextlength","lastmessagelength","totalkeyboardtouches"] - SRC_SCRIPT: src/features/phone_keyboard/rapids/main.py - # See https://www.rapids.science/latest/features/phone-light/ PHONE_LIGHT: CONTAINER: light PROVIDERS: RAPIDS: - COMPUTE: False + COMPUTE: True FEATURES: ["count", "maxlux", "minlux", "avglux", "medianlux", "stdlux"] SRC_SCRIPT: src/features/phone_light/rapids/main.py @@ -246,7 +227,7 @@ PHONE_LOCATIONS: PROVIDERS: DORYAB: - COMPUTE: False + COMPUTE: True FEATURES: ["locationvariance","loglocationvariance","totaldistance","avgspeed","varspeed", "numberofsignificantplaces","numberlocationtransitions","radiusgyration","timeattop1location","timeattop2location","timeattop3location","movingtostaticratio","outlierstimepercent","maxlengthstayatclusters","minlengthstayatclusters","avglengthstayatclusters","stdlengthstayatclusters","locationentropy","normalizedlocationentropy","timeathome", "homelabel"] DBSCAN_EPS: 100 # meters DBSCAN_MINSAMPLES: 5 @@ -261,7 +242,7 @@ PHONE_LOCATIONS: SRC_SCRIPT: src/features/phone_locations/doryab/main.py BARNETT: - COMPUTE: False + COMPUTE: True FEATURES: ["hometime","disttravelled","rog","maxdiam","maxhomedist","siglocsvisited","avgflightlen","stdflightlen","avgflightdur","stdflightdur","probpause","siglocentropy","circdnrtn","wkenddayrtn"] IF_MULTIPLE_TIMEZONES: USE_MOST_COMMON MINUTES_DATA_USED: False # Use this for quality control purposes, how many minutes of data (location coordinates gruped by minute) were used to compute features @@ -276,10 +257,10 @@ PHONE_LOG: # See https://www.rapids.science/latest/features/phone-messages/ PHONE_MESSAGES: - CONTAINER: messages + CONTAINER: sms PROVIDERS: RAPIDS: - COMPUTE: False + COMPUTE: True MESSAGES_TYPES : [received, sent] FEATURES: received: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact] @@ -291,7 +272,7 @@ PHONE_SCREEN: CONTAINER: screen PROVIDERS: RAPIDS: - COMPUTE: False + COMPUTE: True REFERENCE_HOUR_FIRST_USE: 0 IGNORE_EPISODES_SHORTER_THAN: 0 # in minutes, set to 0 to disable IGNORE_EPISODES_LONGER_THAN: 360 # in minutes, set to 0 to disable @@ -313,152 +294,11 @@ PHONE_WIFI_VISIBLE: CONTAINER: wifi PROVIDERS: RAPIDS: - COMPUTE: False + COMPUTE: True FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] SRC_SCRIPT: src/features/phone_wifi_visible/rapids/main.R - -######################################################################################################################## -# FITBIT # -######################################################################################################################## - -# See https://www.rapids.science/latest/setup/configuration/#data-stream-configuration -FITBIT_DATA_STREAMS: - USE: fitbitjson_mysql - - # AVAILABLE: - fitbitjson_mysql: - DATABASE_GROUP: MY_GROUP - SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00). - - fitbitparsed_mysql: - DATABASE_GROUP: MY_GROUP - SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00). - - fitbitjson_csv: - FOLDER: data/external/fitbit_csv - SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00). - - fitbitparsed_csv: - FOLDER: data/external/fitbit_csv - SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00). - -# Sensors ------ - -# See https://www.rapids.science/latest/features/fitbit-calories-intraday/ -FITBIT_CALORIES_INTRADAY: - CONTAINER: fitbit_data - PROVIDERS: - RAPIDS: - COMPUTE: False - EPISODE_TYPE: [sedentary, lightlyactive, fairlyactive, veryactive, mvpa, lowmet, highmet] - EPISODE_TIME_THRESHOLD: 5 # minutes - EPISODE_MET_THRESHOLD: 3 - EPISODE_MVPA_CATEGORIES: [fairlyactive, veryactive] - EPISODE_REFERENCE_TIME: MIDNIGHT # or START_OF_THE_SEGMENT - FEATURES: [count, sumduration, avgduration, minduration, maxduration, stdduration, starttimefirst, endtimefirst, starttimelast, endtimelast, starttimelongest, endtimelongest, summet, avgmet, maxmet, minmet, stdmet, sumcalories, avgcalories, maxcalories, mincalories, stdcalories] - SRC_SCRIPT: src/features/fitbit_calories_intraday/rapids/main.R - -# See https://www.rapids.science/latest/features/fitbit-data-yield/ -FITBIT_DATA_YIELD: - SENSOR: FITBIT_HEARTRATE_INTRADAY - PROVIDERS: - RAPIDS: - COMPUTE: False - FEATURES: [ratiovalidyieldedminutes, ratiovalidyieldedhours] - MINUTE_RATIO_THRESHOLD_FOR_VALID_YIELDED_HOURS: 0.5 # 0 to 1, minimum percentage of valid minutes in an hour to be considered valid. - SRC_SCRIPT: src/features/fitbit_data_yield/rapids/main.R - - -# See https://www.rapids.science/latest/features/fitbit-heartrate-summary/ -FITBIT_HEARTRATE_SUMMARY: - CONTAINER: heartrate_summary - PROVIDERS: - RAPIDS: - COMPUTE: False - FEATURES: ["maxrestinghr", "minrestinghr", "avgrestinghr", "medianrestinghr", "moderestinghr", "stdrestinghr", "diffmaxmoderestinghr", "diffminmoderestinghr", "entropyrestinghr"] # calories features' accuracy depend on the accuracy of the participants fitbit profile (e.g. height, weight) use these with care: ["sumcaloriesoutofrange", "maxcaloriesoutofrange", "mincaloriesoutofrange", "avgcaloriesoutofrange", "mediancaloriesoutofrange", "stdcaloriesoutofrange", "entropycaloriesoutofrange", "sumcaloriesfatburn", "maxcaloriesfatburn", "mincaloriesfatburn", "avgcaloriesfatburn", "mediancaloriesfatburn", "stdcaloriesfatburn", "entropycaloriesfatburn", "sumcaloriescardio", "maxcaloriescardio", "mincaloriescardio", "avgcaloriescardio", "mediancaloriescardio", "stdcaloriescardio", "entropycaloriescardio", "sumcaloriespeak", "maxcaloriespeak", "mincaloriespeak", "avgcaloriespeak", "mediancaloriespeak", "stdcaloriespeak", "entropycaloriespeak"] - SRC_SCRIPT: src/features/fitbit_heartrate_summary/rapids/main.py - -# See https://www.rapids.science/latest/features/fitbit-heartrate-intraday/ -FITBIT_HEARTRATE_INTRADAY: - CONTAINER: heartrate_intraday - PROVIDERS: - RAPIDS: - COMPUTE: False - FEATURES: ["maxhr", "minhr", "avghr", "medianhr", "modehr", "stdhr", "diffmaxmodehr", "diffminmodehr", "entropyhr", "minutesonoutofrangezone", "minutesonfatburnzone", "minutesoncardiozone", "minutesonpeakzone"] - SRC_SCRIPT: src/features/fitbit_heartrate_intraday/rapids/main.py - -# See https://www.rapids.science/latest/features/fitbit-sleep-summary/ -FITBIT_SLEEP_SUMMARY: - CONTAINER: sleep_summary - PROVIDERS: - RAPIDS: - COMPUTE: False - FEATURES: ["firstwaketime", "lastwaketime", "firstbedtime", "lastbedtime", "countepisode", "avgefficiency", "sumdurationafterwakeup", "sumdurationasleep", "sumdurationawake", "sumdurationtofallasleep", "sumdurationinbed", "avgdurationafterwakeup", "avgdurationasleep", "avgdurationawake", "avgdurationtofallasleep", "avgdurationinbed"] - SLEEP_TYPES: ["main", "nap", "all"] - SRC_SCRIPT: src/features/fitbit_sleep_summary/rapids/main.py - -# See https://www.rapids.science/latest/features/fitbit-sleep-intraday/ -FITBIT_SLEEP_INTRADAY: - CONTAINER: sleep_intraday - PROVIDERS: - RAPIDS: - COMPUTE: False - FEATURES: - LEVELS_AND_TYPES: [countepisode, sumduration, maxduration, minduration, avgduration, medianduration, stdduration] - RATIOS_TYPE: [count, duration] - RATIOS_SCOPE: [ACROSS_LEVELS, ACROSS_TYPES, WITHIN_LEVELS, WITHIN_TYPES] - SLEEP_LEVELS: - INCLUDE_ALL_GROUPS: True - CLASSIC: [awake, restless, asleep] - STAGES: [wake, deep, light, rem] - UNIFIED: [awake, asleep] - SLEEP_TYPES: [main, nap, all] - SRC_SCRIPT: src/features/fitbit_sleep_intraday/rapids/main.py - - PRICE: - COMPUTE: False - FEATURES: [avgduration, avgratioduration, avgstarttimeofepisodemain, avgendtimeofepisodemain, avgmidpointofepisodemain, stdstarttimeofepisodemain, stdendtimeofepisodemain, stdmidpointofepisodemain, socialjetlag, rmssdmeanstarttimeofepisodemain, rmssdmeanendtimeofepisodemain, rmssdmeanmidpointofepisodemain, rmssdmedianstarttimeofepisodemain, rmssdmedianendtimeofepisodemain, rmssdmedianmidpointofepisodemain] - SLEEP_LEVELS: - INCLUDE_ALL_GROUPS: True - CLASSIC: [awake, restless, asleep] - STAGES: [wake, deep, light, rem] - UNIFIED: [awake, asleep] - DAY_TYPES: [WEEKEND, WEEK, ALL] - LAST_NIGHT_END: 660 # number of minutes after midnight (11:00) 11*60 - SRC_SCRIPT: src/features/fitbit_sleep_intraday/price/main.py - -# See https://www.rapids.science/latest/features/fitbit-steps-summary/ -FITBIT_STEPS_SUMMARY: - CONTAINER: steps_summary - PROVIDERS: - RAPIDS: - COMPUTE: False - FEATURES: ["maxsumsteps", "minsumsteps", "avgsumsteps", "mediansumsteps", "stdsumsteps"] - SRC_SCRIPT: src/features/fitbit_steps_summary/rapids/main.py - -# See https://www.rapids.science/latest/features/fitbit-steps-intraday/ -FITBIT_STEPS_INTRADAY: - CONTAINER: steps_intraday - EXCLUDE_SLEEP: # you can exclude step data that was logged during sleep periods - TIME_BASED: - EXCLUDE: False - START_TIME: "23:00" - END_TIME: "07:00" - FITBIT_BASED: - EXCLUDE: False - PROVIDERS: - RAPIDS: - COMPUTE: False - FEATURES: - STEPS: ["sum", "max", "min", "avg", "std"] - SEDENTARY_BOUT: ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "stdduration"] - ACTIVE_BOUT: ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "stdduration"] - THRESHOLD_ACTIVE_BOUT: 10 # steps - INCLUDE_ZERO_STEP_ROWS: False - SRC_SCRIPT: src/features/fitbit_steps_intraday/rapids/main.py - ######################################################################################################################## # EMPATICA # ######################################################################################################################## @@ -540,16 +380,16 @@ EMPATICA_TAGS: # See https://www.rapids.science/latest/visualizations/data-quality-visualizations/#1-histograms-of-phone-data-yield HISTOGRAM_PHONE_DATA_YIELD: - PLOT: False + PLOT: True # See https://www.rapids.science/latest/visualizations/data-quality-visualizations/#2-heatmaps-of-overall-data-yield HEATMAP_PHONE_DATA_YIELD_PER_PARTICIPANT_PER_TIME_SEGMENT: - PLOT: False + PLOT: True TIME: RELATIVE_TIME # ABSOLUTE_TIME or RELATIVE_TIME # See https://www.rapids.science/latest/visualizations/data-quality-visualizations/#3-heatmap-of-recorded-phone-sensors HEATMAP_SENSORS_PER_MINUTE_PER_TIME_SEGMENT: - PLOT: False + PLOT: True # See https://www.rapids.science/latest/visualizations/data-quality-visualizations/#4-heatmap-of-sensor-row-count HEATMAP_SENSOR_ROW_COUNT_PER_TIME_SEGMENT: @@ -560,7 +400,7 @@ HEATMAP_SENSOR_ROW_COUNT_PER_TIME_SEGMENT: # See https://www.rapids.science/latest/visualizations/feature-visualizations/#1-heatmap-correlation-matrix HEATMAP_FEATURE_CORRELATION_MATRIX: - PLOT: False + PLOT: True MIN_ROWS_RATIO: 0.5 CORR_THRESHOLD: 0.1 CORR_METHOD: "pearson" # choose from {"pearson", "kendall", "spearman"}