Set table names and calculate all relevant features.

config.yaml

@@ -63,12 +63,12 @@ PHONE_ACCELEROMETER:
   CONTAINER: accelerometer
   PROVIDERS:
     RAPIDS:
-      COMPUTE: False
+      COMPUTE: True
       FEATURES: ["maxmagnitude", "minmagnitude", "avgmagnitude", "medianmagnitude", "stdmagnitude"]
       SRC_SCRIPT: src/features/phone_accelerometer/rapids/main.py
     
     PANDA:
-      COMPUTE: False
+      COMPUTE: True
       VALID_SENSED_MINUTES: False
       FEATURES:
         exertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"]
@@ -78,12 +78,12 @@ PHONE_ACCELEROMETER:
 # See https://www.rapids.science/latest/features/phone-activity-recognition/
 PHONE_ACTIVITY_RECOGNITION:
   CONTAINER: 
-    ANDROID: plugin_google_activity_recognition
+    ANDROID: google_ar
     IOS: plugin_ios_activity_recognition
   EPISODE_THRESHOLD_BETWEEN_ROWS: 5 # minutes. Max time difference for two consecutive rows to be considered within the same AR episode.
   PROVIDERS:
     RAPIDS:
-      COMPUTE: False
+      COMPUTE: True
       FEATURES: ["count", "mostcommonactivity", "countuniqueactivities", "durationstationary", "durationmobile", "durationvehicle"]
       ACTIVITY_CLASSES:
         STATIONARY: ["still", "tilting"]
@@ -91,19 +91,9 @@ PHONE_ACTIVITY_RECOGNITION:
         VEHICLE: ["in_vehicle"]
       SRC_SCRIPT: src/features/phone_activity_recognition/rapids/main.py
 
-# See https://www.rapids.science/latest/features/phone-applications-crashes/
-PHONE_APPLICATIONS_CRASHES:
-  CONTAINER: applications_crashes
-  APPLICATION_CATEGORIES:
-    CATALOGUE_SOURCE: FILE # FILE (genres are read from CATALOGUE_FILE) or GOOGLE (genres are scrapped from the Play Store)
-    CATALOGUE_FILE: "data/external/stachl_application_genre_catalogue.csv"
-    UPDATE_CATALOGUE_FILE: False # if CATALOGUE_SOURCE is equal to FILE, whether or not to update CATALOGUE_FILE, if CATALOGUE_SOURCE is equal to GOOGLE all scraped genres will be saved to CATALOGUE_FILE
-    SCRAPE_MISSING_CATEGORIES: False # whether or not to scrape missing genres, only effective if CATALOGUE_SOURCE is equal to FILE. If CATALOGUE_SOURCE is equal to GOOGLE, all genres are scraped anyway
-  PROVIDERS: # None implemented yet but this sensor can be used in PHONE_DATA_YIELD
-
 # See https://www.rapids.science/latest/features/phone-applications-foreground/
 PHONE_APPLICATIONS_FOREGROUND:
-  CONTAINER: applications_foreground
+  CONTAINER: applications
   APPLICATION_CATEGORIES:
     CATALOGUE_SOURCE: FILE # FILE (genres are read from CATALOGUE_FILE) or GOOGLE (genres are scrapped from the Play Store)
     CATALOGUE_FILE: "data/external/stachl_application_genre_catalogue.csv"
@@ -111,8 +101,8 @@ PHONE_APPLICATIONS_FOREGROUND:
     SCRAPE_MISSING_CATEGORIES: False # whether or not to scrape missing genres, only effective if CATALOGUE_SOURCE is equal to FILE. If CATALOGUE_SOURCE is equal to GOOGLE, all genres are scraped anyway
   PROVIDERS:
     RAPIDS:
-      COMPUTE: False
+      COMPUTE: True
-      INCLUDE_EPISODE_FEATURES: False
+      INCLUDE_EPISODE_FEATURES: True
       SINGLE_CATEGORIES: ["all", "email"]
       MULTIPLE_CATEGORIES:
         social: ["socialnetworks", "socialmediatools"]
@@ -122,7 +112,7 @@ PHONE_APPLICATIONS_FOREGROUND:
         dating: ["com.tinder", "com.relance.happycouple", "com.kiwi.joyride"]
       SINGLE_APPS: ["top1global", "com.facebook.moments", "com.google.android.youtube", "com.twitter.android"] # There's no entropy for single apps
       EXCLUDED_CATEGORIES: []
-      EXCLUDED_APPS: ["com.fitbit.FitbitMobile", "com.aware.plugin.upmc.cancer"]
+      EXCLUDED_APPS: ["com.fitbit.FitbitMobile", "com.aware.plugin.upmc.cancer"] # TODO list system apps?
       FEATURES: 
         APP_EVENTS: ["countevent", "timeoffirstuse", "timeoflastuse", "frequencyentropy"]
         APP_EPISODES: ["countepisode", "minduration", "maxduration", "meanduration", "sumduration"]
@@ -132,7 +122,7 @@ PHONE_APPLICATIONS_FOREGROUND:
 
 # See https://www.rapids.science/latest/features/phone-applications-notifications/
 PHONE_APPLICATIONS_NOTIFICATIONS:
-  CONTAINER: applications_notifications
+  CONTAINER: notifications
   APPLICATION_CATEGORIES:
     CATALOGUE_SOURCE: FILE # FILE (genres are read from CATALOGUE_FILE) or GOOGLE (genres are scrapped from the Play Store)
     CATALOGUE_FILE: "data/external/stachl_application_genre_catalogue.csv"
@@ -146,7 +136,7 @@ PHONE_BATTERY:
   EPISODE_THRESHOLD_BETWEEN_ROWS: 30 # minutes. Max time difference for two consecutive rows to be considered within the same battery episode.
   PROVIDERS:
     RAPIDS:
-      COMPUTE: False
+      COMPUTE: True
       FEATURES: ["countdischarge", "sumdurationdischarge", "countcharge", "sumdurationcharge", "avgconsumptionrate", "maxconsumptionrate"]
       SRC_SCRIPT: src/features/phone_battery/rapids/main.py
 
@@ -155,12 +145,12 @@ PHONE_BLUETOOTH:
   CONTAINER: bluetooth
   PROVIDERS:
     RAPIDS:
-      COMPUTE: False
+      COMPUTE: True
       FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"]
       SRC_SCRIPT: src/features/phone_bluetooth/rapids/main.R
 
     DORYAB:
-      COMPUTE: False
+      COMPUTE: True
       FEATURES: 
         ALL: 
             DEVICES: ["countscans", "uniquedevices", "meanscans", "stdscans"]
@@ -191,7 +181,7 @@ PHONE_CALLS:
       SRC_SCRIPT: src/features/phone_calls/rapids/main.R
 
 # See https://www.rapids.science/latest/features/phone-conversation/
-PHONE_CONVERSATION:
+PHONE_CONVERSATION: # TODO Adapt for speech
   CONTAINER: 
     ANDROID: plugin_studentlife_audio_android
     IOS: plugin_studentlife_audio
@@ -210,29 +200,20 @@ PHONE_CONVERSATION:
 
 # See https://www.rapids.science/latest/features/phone-data-yield/
 PHONE_DATA_YIELD:
-  SENSORS: []
+  SENSORS: [PHONE_LIGHT]
   PROVIDERS:
     RAPIDS:
-      COMPUTE: False
+      COMPUTE: True
       FEATURES: [ratiovalidyieldedminutes, ratiovalidyieldedhours]
       MINUTE_RATIO_THRESHOLD_FOR_VALID_YIELDED_HOURS: 0.5 # 0 to 1, minimum percentage of valid minutes in an hour to be considered valid.
       SRC_SCRIPT: src/features/phone_data_yield/rapids/main.R
 
-# See https://www.rapids.science/latest/features/phone-keyboard/
-PHONE_KEYBOARD:
-  CONTAINER: keyboard
-  PROVIDERS:
-    RAPIDS:
-      COMPUTE: False
-      FEATURES: ["sessioncount","averageinterkeydelay","averagesessionlength","changeintextlengthlessthanminusone","changeintextlengthequaltominusone","changeintextlengthequaltoone","changeintextlengthmorethanone","maxtextlength","lastmessagelength","totalkeyboardtouches"]
-      SRC_SCRIPT: src/features/phone_keyboard/rapids/main.py
-
 # See https://www.rapids.science/latest/features/phone-light/
 PHONE_LIGHT:
   CONTAINER: light
   PROVIDERS:
     RAPIDS:
-      COMPUTE: False
+      COMPUTE: True
       FEATURES: ["count", "maxlux", "minlux", "avglux", "medianlux", "stdlux"]
       SRC_SCRIPT: src/features/phone_light/rapids/main.py
 
@@ -246,7 +227,7 @@ PHONE_LOCATIONS:
 
   PROVIDERS:
     DORYAB:
-      COMPUTE: False
+      COMPUTE: True
       FEATURES: ["locationvariance","loglocationvariance","totaldistance","avgspeed","varspeed", "numberofsignificantplaces","numberlocationtransitions","radiusgyration","timeattop1location","timeattop2location","timeattop3location","movingtostaticratio","outlierstimepercent","maxlengthstayatclusters","minlengthstayatclusters","avglengthstayatclusters","stdlengthstayatclusters","locationentropy","normalizedlocationentropy","timeathome", "homelabel"]
       DBSCAN_EPS: 100 # meters
       DBSCAN_MINSAMPLES: 5
@@ -261,7 +242,7 @@ PHONE_LOCATIONS:
       SRC_SCRIPT: src/features/phone_locations/doryab/main.py
 
     BARNETT:
-      COMPUTE: False
+      COMPUTE: True
       FEATURES: ["hometime","disttravelled","rog","maxdiam","maxhomedist","siglocsvisited","avgflightlen","stdflightlen","avgflightdur","stdflightdur","probpause","siglocentropy","circdnrtn","wkenddayrtn"]
       IF_MULTIPLE_TIMEZONES: USE_MOST_COMMON
       MINUTES_DATA_USED: False # Use this for quality control purposes, how many minutes of data (location coordinates gruped by minute) were used to compute features
@@ -276,10 +257,10 @@ PHONE_LOG:
 
 # See https://www.rapids.science/latest/features/phone-messages/
 PHONE_MESSAGES:
-  CONTAINER: messages
+  CONTAINER: sms
   PROVIDERS:
     RAPIDS:
-      COMPUTE: False
+      COMPUTE: True
       MESSAGES_TYPES : [received, sent]
       FEATURES: 
         received: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact]
@@ -291,7 +272,7 @@ PHONE_SCREEN:
   CONTAINER: screen
   PROVIDERS:
     RAPIDS:
-      COMPUTE: False
+      COMPUTE: True
       REFERENCE_HOUR_FIRST_USE: 0
       IGNORE_EPISODES_SHORTER_THAN: 0 # in minutes, set to 0 to disable
       IGNORE_EPISODES_LONGER_THAN: 360 # in minutes, set to 0 to disable
@@ -313,152 +294,11 @@ PHONE_WIFI_VISIBLE:
   CONTAINER: wifi
   PROVIDERS:
     RAPIDS:
-      COMPUTE: False
+      COMPUTE: True
       FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"]
       SRC_SCRIPT: src/features/phone_wifi_visible/rapids/main.R
 
 
-
-########################################################################################################################
-#                                                 FITBIT                                                               #
-########################################################################################################################
-
-# See https://www.rapids.science/latest/setup/configuration/#data-stream-configuration
-FITBIT_DATA_STREAMS:
-  USE: fitbitjson_mysql
-  
-  # AVAILABLE:
-  fitbitjson_mysql: 
-    DATABASE_GROUP: MY_GROUP
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
-  
-  fitbitparsed_mysql: 
-    DATABASE_GROUP: MY_GROUP
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
-  
-  fitbitjson_csv: 
-    FOLDER: data/external/fitbit_csv
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
-
-  fitbitparsed_csv: 
-    FOLDER: data/external/fitbit_csv
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
-
-# Sensors ------
-
-# See https://www.rapids.science/latest/features/fitbit-calories-intraday/
-FITBIT_CALORIES_INTRADAY:
-  CONTAINER: fitbit_data
-  PROVIDERS:
-    RAPIDS:
-      COMPUTE: False
-      EPISODE_TYPE: [sedentary, lightlyactive, fairlyactive, veryactive, mvpa, lowmet, highmet]
-      EPISODE_TIME_THRESHOLD: 5 # minutes
-      EPISODE_MET_THRESHOLD: 3
-      EPISODE_MVPA_CATEGORIES: [fairlyactive, veryactive]
-      EPISODE_REFERENCE_TIME: MIDNIGHT # or START_OF_THE_SEGMENT
-      FEATURES: [count, sumduration, avgduration, minduration, maxduration, stdduration, starttimefirst, endtimefirst, starttimelast, endtimelast, starttimelongest, endtimelongest, summet, avgmet, maxmet, minmet, stdmet, sumcalories, avgcalories, maxcalories, mincalories, stdcalories]
-      SRC_SCRIPT: src/features/fitbit_calories_intraday/rapids/main.R
-
-# See https://www.rapids.science/latest/features/fitbit-data-yield/
-FITBIT_DATA_YIELD:
-  SENSOR: FITBIT_HEARTRATE_INTRADAY
-  PROVIDERS:
-    RAPIDS:
-      COMPUTE: False
-      FEATURES: [ratiovalidyieldedminutes, ratiovalidyieldedhours]
-      MINUTE_RATIO_THRESHOLD_FOR_VALID_YIELDED_HOURS: 0.5 #  0 to 1, minimum percentage of valid minutes in an hour to be considered valid.
-      SRC_SCRIPT: src/features/fitbit_data_yield/rapids/main.R
-
-
-# See https://www.rapids.science/latest/features/fitbit-heartrate-summary/
-FITBIT_HEARTRATE_SUMMARY:
-  CONTAINER: heartrate_summary
-  PROVIDERS:
-    RAPIDS:
-      COMPUTE: False
-      FEATURES: ["maxrestinghr", "minrestinghr", "avgrestinghr", "medianrestinghr", "moderestinghr", "stdrestinghr", "diffmaxmoderestinghr", "diffminmoderestinghr", "entropyrestinghr"] # calories features' accuracy depend on the accuracy of the participants fitbit profile (e.g. height, weight) use these with care: ["sumcaloriesoutofrange", "maxcaloriesoutofrange", "mincaloriesoutofrange", "avgcaloriesoutofrange", "mediancaloriesoutofrange", "stdcaloriesoutofrange", "entropycaloriesoutofrange", "sumcaloriesfatburn", "maxcaloriesfatburn", "mincaloriesfatburn", "avgcaloriesfatburn", "mediancaloriesfatburn", "stdcaloriesfatburn", "entropycaloriesfatburn", "sumcaloriescardio", "maxcaloriescardio", "mincaloriescardio", "avgcaloriescardio", "mediancaloriescardio", "stdcaloriescardio", "entropycaloriescardio", "sumcaloriespeak", "maxcaloriespeak", "mincaloriespeak", "avgcaloriespeak", "mediancaloriespeak", "stdcaloriespeak", "entropycaloriespeak"]
-      SRC_SCRIPT: src/features/fitbit_heartrate_summary/rapids/main.py
-
-# See https://www.rapids.science/latest/features/fitbit-heartrate-intraday/
-FITBIT_HEARTRATE_INTRADAY:
-  CONTAINER: heartrate_intraday
-  PROVIDERS:
-    RAPIDS:
-      COMPUTE: False
-      FEATURES: ["maxhr", "minhr", "avghr", "medianhr", "modehr", "stdhr", "diffmaxmodehr", "diffminmodehr", "entropyhr", "minutesonoutofrangezone", "minutesonfatburnzone", "minutesoncardiozone", "minutesonpeakzone"]
-      SRC_SCRIPT: src/features/fitbit_heartrate_intraday/rapids/main.py
-
-# See https://www.rapids.science/latest/features/fitbit-sleep-summary/
-FITBIT_SLEEP_SUMMARY:
-  CONTAINER: sleep_summary
-  PROVIDERS:
-    RAPIDS:
-      COMPUTE: False
-      FEATURES: ["firstwaketime", "lastwaketime", "firstbedtime", "lastbedtime", "countepisode", "avgefficiency", "sumdurationafterwakeup", "sumdurationasleep", "sumdurationawake", "sumdurationtofallasleep", "sumdurationinbed", "avgdurationafterwakeup", "avgdurationasleep", "avgdurationawake", "avgdurationtofallasleep", "avgdurationinbed"]
-      SLEEP_TYPES: ["main", "nap", "all"]
-      SRC_SCRIPT: src/features/fitbit_sleep_summary/rapids/main.py
-
-# See https://www.rapids.science/latest/features/fitbit-sleep-intraday/
-FITBIT_SLEEP_INTRADAY:
-  CONTAINER: sleep_intraday
-  PROVIDERS:
-    RAPIDS:
-      COMPUTE: False
-      FEATURES:
-        LEVELS_AND_TYPES: [countepisode, sumduration, maxduration, minduration, avgduration, medianduration, stdduration]
-        RATIOS_TYPE: [count, duration]
-        RATIOS_SCOPE: [ACROSS_LEVELS, ACROSS_TYPES, WITHIN_LEVELS, WITHIN_TYPES]
-      SLEEP_LEVELS:
-        INCLUDE_ALL_GROUPS: True
-        CLASSIC: [awake, restless, asleep]
-        STAGES: [wake, deep, light, rem]
-        UNIFIED: [awake, asleep]
-      SLEEP_TYPES: [main, nap, all]
-      SRC_SCRIPT: src/features/fitbit_sleep_intraday/rapids/main.py
-  
-    PRICE:
-      COMPUTE: False
-      FEATURES: [avgduration, avgratioduration, avgstarttimeofepisodemain, avgendtimeofepisodemain, avgmidpointofepisodemain, stdstarttimeofepisodemain, stdendtimeofepisodemain, stdmidpointofepisodemain, socialjetlag, rmssdmeanstarttimeofepisodemain, rmssdmeanendtimeofepisodemain, rmssdmeanmidpointofepisodemain, rmssdmedianstarttimeofepisodemain, rmssdmedianendtimeofepisodemain, rmssdmedianmidpointofepisodemain]
-      SLEEP_LEVELS:
-        INCLUDE_ALL_GROUPS: True
-        CLASSIC: [awake, restless, asleep]
-        STAGES: [wake, deep, light, rem]
-        UNIFIED: [awake, asleep]
-      DAY_TYPES: [WEEKEND, WEEK, ALL]
-      LAST_NIGHT_END: 660 # number of minutes after midnight (11:00) 11*60
-      SRC_SCRIPT: src/features/fitbit_sleep_intraday/price/main.py
-
-# See https://www.rapids.science/latest/features/fitbit-steps-summary/
-FITBIT_STEPS_SUMMARY:
-  CONTAINER: steps_summary
-  PROVIDERS:
-    RAPIDS:
-      COMPUTE: False
-      FEATURES: ["maxsumsteps", "minsumsteps", "avgsumsteps", "mediansumsteps", "stdsumsteps"]
-      SRC_SCRIPT: src/features/fitbit_steps_summary/rapids/main.py
-
-# See https://www.rapids.science/latest/features/fitbit-steps-intraday/
-FITBIT_STEPS_INTRADAY:
-  CONTAINER: steps_intraday
-  EXCLUDE_SLEEP: # you can exclude step data that was logged during sleep periods
-    TIME_BASED:
-      EXCLUDE: False
-      START_TIME: "23:00"
-      END_TIME: "07:00"
-    FITBIT_BASED:
-      EXCLUDE: False
-  PROVIDERS:
-    RAPIDS:
-      COMPUTE: False
-      FEATURES:
-        STEPS: ["sum", "max", "min", "avg", "std"]
-        SEDENTARY_BOUT: ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "stdduration"]
-        ACTIVE_BOUT: ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "stdduration"]
-      THRESHOLD_ACTIVE_BOUT: 10 # steps
-      INCLUDE_ZERO_STEP_ROWS: False
-      SRC_SCRIPT: src/features/fitbit_steps_intraday/rapids/main.py
-
 ########################################################################################################################
 #                                                 EMPATICA                                                             #
 ########################################################################################################################
@@ -540,16 +380,16 @@ EMPATICA_TAGS:
 
 # See https://www.rapids.science/latest/visualizations/data-quality-visualizations/#1-histograms-of-phone-data-yield
 HISTOGRAM_PHONE_DATA_YIELD:
-  PLOT: False
+  PLOT: True
 
 # See https://www.rapids.science/latest/visualizations/data-quality-visualizations/#2-heatmaps-of-overall-data-yield
 HEATMAP_PHONE_DATA_YIELD_PER_PARTICIPANT_PER_TIME_SEGMENT:
-  PLOT: False
+  PLOT: True
   TIME: RELATIVE_TIME # ABSOLUTE_TIME or RELATIVE_TIME
 
 # See https://www.rapids.science/latest/visualizations/data-quality-visualizations/#3-heatmap-of-recorded-phone-sensors
 HEATMAP_SENSORS_PER_MINUTE_PER_TIME_SEGMENT:
-  PLOT: False
+  PLOT: True
 
 # See https://www.rapids.science/latest/visualizations/data-quality-visualizations/#4-heatmap-of-sensor-row-count
 HEATMAP_SENSOR_ROW_COUNT_PER_TIME_SEGMENT:
@@ -560,7 +400,7 @@ HEATMAP_SENSOR_ROW_COUNT_PER_TIME_SEGMENT:
 
 # See https://www.rapids.science/latest/visualizations/feature-visualizations/#1-heatmap-correlation-matrix
 HEATMAP_FEATURE_CORRELATION_MATRIX:
-  PLOT: False
+  PLOT: True
   MIN_ROWS_RATIO: 0.5
   CORR_THRESHOLD: 0.1
   CORR_METHOD: "pearson" # choose from {"pearson", "kendall", "spearman"}