Initial testing restructure for 30mins segments

data/external/daysegments_frequency.csv

@@ -1,2 +1,2 @@
 label,length
-tenminutes,10
+thirtyminutes,30

data/external/daysegments_periodic.csv

@@ -1,7 +1,6 @@
 label,start_time,length,repeats_on,repeats_value
 daily,00:00:00,23H 59M 59S,every_day,0
-weekly,00:00:00,6D 23H 59M 59S,wday,1
 morning,06:00:00,5H 59M 59S,every_day,0
 afternoon,12:00:00,5H 59M 59S,every_day,0
-evening,18:00:00,5H 59M 59S,wday,2
-night,00:00:00,5H 59M 59S,qday,5
+evening,18:00:00,5H 59M 59S,every_day,0
+night,00:00:00,5H 59M 59S,every_day,0

src/features/bluetooth/rapids/main.R

@@ -18,6 +18,7 @@ compute_bluetooth_feature <- function(data, feature, day_segment){
       filter(N == max(N)) %>% 
       head(1) %>% # if there are multiple device with the same amount of scans pick the first one only
       pull(bt_address)
+    mostuniquedevice
     return(data %>% 
              filter(bt_address == mostuniquedevice) %>%
              group_by(local_segment) %>% 

tests/data/processed/features/frequency/test01/bluetooth.csv

@@ -0,0 +1,14 @@
+"local_segment","local_segment_label","local_start_date","local_start_time","local_end_date","local_end_time","bluetooth_rapids_countscans","bluetooth_rapids_uniquedevices","bluetooth_rapids_countscansmostuniquedevice"
+"thirtyminutes0000#2020-07-02#00:00:00#2020-07-02#00:29:59","thirtyminutes0000","2020-07-02","00:00:00","2020-07-02","00:29:59",1,1,1
+"thirtyminutes0001#2020-07-02#00:30:00#2020-07-02#00:59:59","thirtyminutes0001","2020-07-02","00:30:00","2020-07-02","00:59:59",1,1,1
+"thirtyminutes0007#2020-07-02#03:30:00#2020-07-02#03:59:59","thirtyminutes0007","2020-07-02","03:30:00","2020-07-02","03:59:59",1,1,1
+"thirtyminutes0011#2020-07-02#05:30:00#2020-07-02#05:59:59","thirtyminutes0011","2020-07-02","05:30:00","2020-07-02","05:59:59",1,1,1
+"thirtyminutes0012#2020-07-02#06:00:00#2020-07-02#06:29:59","thirtyminutes0012","2020-07-02","06:00:00","2020-07-02","06:29:59",1,1,1
+"thirtyminutes0014#2020-07-02#07:00:00#2020-07-02#07:29:59","thirtyminutes0014","2020-07-02","07:00:00","2020-07-02","07:29:59",1,1,1
+"thirtyminutes0023#2020-07-02#11:30:00#2020-07-02#11:59:59","thirtyminutes0023","2020-07-02","11:30:00","2020-07-02","11:59:59",1,1,1
+"thirtyminutes0024#2020-07-02#12:00:00#2020-07-02#12:29:59","thirtyminutes0024","2020-07-02","12:00:00","2020-07-02","12:29:59",1,1,1
+"thirtyminutes0035#2020-07-02#17:30:00#2020-07-02#17:59:59","thirtyminutes0035","2020-07-02","17:30:00","2020-07-02","17:59:59",1,1,1
+"thirtyminutes0036#2020-07-02#18:00:00#2020-07-02#18:29:59","thirtyminutes0036","2020-07-02","18:00:00","2020-07-02","18:29:59",1,1,1
+"thirtyminutes0039#2020-07-02#19:30:00#2020-07-02#19:59:59","thirtyminutes0039","2020-07-02","19:30:00","2020-07-02","19:59:59",1,1,1
+"thirtyminutes0042#2020-07-02#21:00:00#2020-07-02#21:29:59","thirtyminutes0042","2020-07-02","21:00:00","2020-07-02","21:29:59",1,1,1
+"thirtyminutes0047#2020-07-02#23:30:00#2020-07-02#23:59:59","thirtyminutes0047","2020-07-02","23:30:00","2020-07-02","23:59:59",2,2,1

tests/data/processed/features/frequency/test02/bluetooth.csv

@@ -0,0 +1,14 @@
+"local_segment","local_segment_label","local_start_date","local_start_time","local_end_date","local_end_time","bluetooth_rapids_countscans","bluetooth_rapids_uniquedevices","bluetooth_rapids_countscansmostuniquedevice"
+"thirtyminutes0000#2020-07-02#00:00:00#2020-07-02#00:29:59","thirtyminutes0000","2020-07-02","00:00:00","2020-07-02","00:29:59",1,1,1
+"thirtyminutes0001#2020-07-02#00:30:00#2020-07-02#00:59:59","thirtyminutes0001","2020-07-02","00:30:00","2020-07-02","00:59:59",1,1,1
+"thirtyminutes0007#2020-07-02#03:30:00#2020-07-02#03:59:59","thirtyminutes0007","2020-07-02","03:30:00","2020-07-02","03:59:59",1,1,1
+"thirtyminutes0011#2020-07-02#05:30:00#2020-07-02#05:59:59","thirtyminutes0011","2020-07-02","05:30:00","2020-07-02","05:59:59",1,1,1
+"thirtyminutes0012#2020-07-02#06:00:00#2020-07-02#06:29:59","thirtyminutes0012","2020-07-02","06:00:00","2020-07-02","06:29:59",1,1,1
+"thirtyminutes0014#2020-07-02#07:00:00#2020-07-02#07:29:59","thirtyminutes0014","2020-07-02","07:00:00","2020-07-02","07:29:59",1,1,1
+"thirtyminutes0023#2020-07-02#11:30:00#2020-07-02#11:59:59","thirtyminutes0023","2020-07-02","11:30:00","2020-07-02","11:59:59",1,1,1
+"thirtyminutes0024#2020-07-02#12:00:00#2020-07-02#12:29:59","thirtyminutes0024","2020-07-02","12:00:00","2020-07-02","12:29:59",1,1,1
+"thirtyminutes0035#2020-07-02#17:30:00#2020-07-02#17:59:59","thirtyminutes0035","2020-07-02","17:30:00","2020-07-02","17:59:59",1,1,1
+"thirtyminutes0036#2020-07-02#18:00:00#2020-07-02#18:29:59","thirtyminutes0036","2020-07-02","18:00:00","2020-07-02","18:29:59",1,1,1
+"thirtyminutes0039#2020-07-02#19:30:00#2020-07-02#19:59:59","thirtyminutes0039","2020-07-02","19:30:00","2020-07-02","19:59:59",1,1,1
+"thirtyminutes0042#2020-07-02#21:00:00#2020-07-02#21:29:59","thirtyminutes0042","2020-07-02","21:00:00","2020-07-02","21:29:59",1,1,1
+"thirtyminutes0047#2020-07-02#23:30:00#2020-07-02#23:59:59","thirtyminutes0047","2020-07-02","23:30:00","2020-07-02","23:59:59",2,1,2

tests/data/processed/features/periodic/test03/bluetooth.csv

@@ -0,0 +1 @@
+"local_segment","local_segment_label","local_start_date","local_start_time","local_end_date","local_end_time","bluetooth_rapids_countscans","bluetooth_rapids_uniquedevices","bluetooth_rapids_countscansmostuniquedevice"

tests/data/processed/features/periodic/test04/bluetooth.csv

@@ -0,0 +1 @@
+"local_segment","local_segment_label","local_start_date","local_start_time","local_end_date","local_end_time","bluetooth_rapids_countscans","bluetooth_rapids_uniquedevices","bluetooth_rapids_countscansmostuniquedevice"

tests/scripts/run_tests.sh

@@ -22,13 +22,41 @@ echo Disabling downloading of dataset...
 sed -e '27,39 s/^/#/' -e  's/rules.download_dataset.output/"data\/raw\/\{pid\}\/\{sensor\}_raw\.csv"/' rules/preprocessing.smk > tmp
 cp tmp rules/preprocessing.smk
 
-echo Running RAPIDS Pipeline on testdata...
-snakemake --profile tests/settings 
+echo Running RAPIDS Pipeline periodic segment on testdata...
+snakemake --profile tests/settings/periodic/ 
 
-echo Running tests on data produced...
+echo Moving produced data from previous pipeline run ...
+# rm -rf data/raw/*
+mkdir data/processed/features/periodic
+mv data/processed/features/test* data/processed/features/periodic/
+rm -rf data/interim/*
+# rm -rf data/external/test*
+
+echo Running RAPIDS Pipeline frequnecy segment on testdata...
+snakemake --profile tests/settings/frequency/ 
+
+echo Moving produced data from previous pipeline run...
+mkdir data/processed/features/frequency
+mv data/processed/features/test* data/processed/features/frequency/
+
+echo Running tests on periodic data produced...
 python -m unittest discover tests/scripts/ -v 
 
+echo Backing up Testing script...
+cp tests/scripts/test_sensor_features.py test_bak
+
+echo Re-writing the config file being loaded for testing
+sed -e  's/tests\/settings\/periodic\/testing_config\.yaml/tests\/settings\/frequency\/testing_config\.yaml/' tests/scripts/test_sensor_features.py > test_tmp
+cp test_tmp tests/scripts/test_sensor_features.py
+
+echo Running tests on frequency data produced...
+./env/bin/python -m unittest discover tests/scripts/ -v
+
 # Uncomment to return snakemake back to the original version when testing locally
 # echo Cleaning up...
 # mv bak rules/preprocessing.smk
-# rm tmp
+# mv test_bak tests/scripts/test_sensor_features.py
+# rm test_bak 
+# rm test_tmp
+# rm bak
+# rm tmp

tests/scripts/test_sensor_features.py

@@ -4,6 +4,7 @@ import hashlib
 import pandas as pd
 import utils
 import yaml
+import sys
 import os
 
 class TestSensorFeatures(unittest.TestCase):
@@ -15,13 +16,14 @@ class TestSensorFeatures(unittest.TestCase):
     def setUpClass(cls):
         # Runs once to Setup env
         global configs 
-        with open(r'tests/settings/testing_config.yaml') as file:
+        with open(r'tests/settings/periodic/testing_config.yaml') as file:
             configs = yaml.full_load(file)
 
 
     def test_sensors_files_exist(self):
         # Loop through the file_list dictionary and check if the files exist. 
     
+        #print("Testing existance of files")
         file_lists = utils.generate_sensor_file_lists(configs)
         for each in file_lists:
             #for out_file, _ in file_lists[each]:
@@ -30,7 +32,7 @@ class TestSensorFeatures(unittest.TestCase):
 
     def test_sensors_features_calculations(self):
 
-                       
+        # print("Testing calculations..")               
         sensor_file_list = utils.generate_sensor_file_lists(configs)        
         for each in sensor_file_list:
             for act_result, exp_result in sensor_file_list:
@@ -48,4 +50,5 @@ class TestSensorFeatures(unittest.TestCase):
 
 if __name__ == '__main__':
 
-    unittest.main()
+    unittest.main()
+   

tests/scripts/utils.py

@@ -49,8 +49,8 @@ def generate_file_list(configs, sensor):
     # i.e. The sensor passed into the function. 
 
     # Initialize string of file path for both expected and actual metric values
-    act_str = "data/processed/{pid}/{sensor}_{sensor_type}{day_segment}.csv"
-    exp_str = "tests/data/processed/{pid}/{sensor}_{sensor_type}{day_segment}.csv"
+    act_str = "data/processed/features/{pid}/{sensor}_{sensor_type}{day_segment}.csv"
+    exp_str = "tests/data/processed/features/period/{pid}/{sensor}_{sensor_type}{day_segment}.csv"
     
     sensor_cap = sensor.upper()
     if 'DAY_SEGMENTS' and 'FEATURES' in configs[sensor_cap]:
@@ -79,8 +79,10 @@ def generate_sensor_file_lists(configs):
     # actual files for each sensor listed in the config file. Added for Travis.
 
     # Initialize string of file path for both expected and actual metric values
-    act_str = "data/processed/features/{pid}/{sensor_key}.csv"
-    exp_str = "tests/data/processed/features/{pid}/{sensor_key}.csv"
+    segment = configs['DAY_SEGMENTS']['TYPE'].lower()
+    print(segment)
+    act_str = "data/processed/features/"+segment+"/{pid}/{sensor_key}.csv"
+    exp_str = "tests/data/processed/features/"+segment+"/{pid}/{sensor_key}.csv"
 
     # List of available sensors that can be tested by the testing suite
     TESTABLE_SENSORS = ['MESSAGES', 'CALLS', 'SCREEN', 'BATTERY', 'BLUETOOTH', 'WIFI', 'LIGHT', 'APPLICATIONS_FOREGROUND', 'ACTIVITY_RECOGNITION', 'CONVERSATION']

tests/settings/config.yaml

@@ -1,5 +0,0 @@
-directory: ./
-configfile: ./tests/settings/testing_config.yaml
-snakefile: ./tests/Snakefile
-cores: 1
-forcerun: [join_features_from_providers]

tests/settings/frequency/config.yaml

@@ -0,0 +1,5 @@
+directory: ./
+configfile: ./tests/settings/frequency/testing_config.yaml
+snakefile: ./tests/Snakefile
+cores: 1
+forcerun: [join_features_from_providers]

tests/settings/frequency/testing_config.yaml

@@ -0,0 +1,101 @@
+# Participants to include in the analysis
+# You must create a file for each participant named pXXX containing their device_id. This can be done manually or automatically
+PIDS: [test01, test02, test03, test04]
+
+# Global var with common day segments
+DAY_SEGMENTS: &day_segments
+  TYPE: FREQUENCY # FREQUENCY, PERIODIC, EVENT
+  FILE: "data/external/daysegments_frequency.csv"
+
+PHONE_VALID_SENSED_BINS:
+  DB_TABLES: [bluetooth]
+
+# Communication SMS features config, TYPES and FEATURES keys need to match
+MESSAGES:
+  DB_TABLE: messages
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: False
+      MESSAGES_TYPES : [received, sent]
+      FEATURES: 
+        received: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact]
+        sent: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact]
+      SRC_LANGUAGE: "r"
+      SRC_FOLDER: "rapids" # inside src/features/messages
+
+# Communication call features config, TYPES and FEATURES keys need to match
+CALLS:
+  DB_TABLE: calls
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: False
+      CALL_TYPES: [missed, incoming, outgoing]
+      FEATURES:
+        missed:  [count, distinctcontacts, timefirstcall, timelastcall, countmostfrequentcontact]
+        incoming: [count, distinctcontacts, meanduration, sumduration, minduration, maxduration, stdduration, modeduration, entropyduration, timefirstcall, timelastcall, countmostfrequentcontact]
+        outgoing: [count, distinctcontacts, meanduration, sumduration, minduration, maxduration, stdduration, modeduration, entropyduration, timefirstcall, timelastcall, countmostfrequentcontact]
+      SRC_LANGUAGE: "r"
+      SRC_FOLDER: "rapids" # inside src/features/calls
+
+BLUETOOTH:
+  DB_TABLE: bluetooth
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: True
+      FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"]
+      SRC_FOLDER: "rapids" # inside src/features/bluetooth
+      SRC_LANGUAGE: "r"
+
+LIGHT:
+  DB_TABLE: light
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: False
+      FEATURES: ["count", "maxlux", "minlux", "avglux", "medianlux", "stdlux"]
+      SRC_FOLDER: "rapids" # inside src/features/light
+      SRC_LANGUAGE: "python"
+
+WIFI:
+  DB_TABLE: 
+    VISIBLE_ACCESS_POINTS: "wifi" # if you only have a CONNECTED_ACCESS_POINTS table, set this value to ""
+    CONNECTED_ACCESS_POINTS: "sensor_wifi" # if you only have a VISIBLE_ACCESS_POINTS table, set this value to ""
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: False
+      FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"]
+      SRC_FOLDER: "rapids" # inside src/features/bluetooth
+      SRC_LANGUAGE: "r"
+
+APPLICATIONS_FOREGROUND:
+  DB_TABLE: applications_foreground
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: False
+      SINGLE_CATEGORIES: ["all", "email"]
+      MULTIPLE_CATEGORIES:
+        social: ["socialnetworks", "socialmediatools"]
+        entertainment: ["entertainment", "gamingstrategy"]
+      SINGLE_APPS: ["top1global", "com.facebook.moments"] # There's no entropy for single apps
+      EXCLUDED_CATEGORIES: ["systemapp", "tvvideoapps"]
+      EXCLUDED_APPS: ["com.fitbit.FitbitMobile", "com.aware.plugin.upmc.cancer"]
+      FEATURES: ["count", "timeoffirstuse", "timeoflastuse", "frequencyentropy"]
+      SRC_FOLDER: "rapids" # inside src/features/applications_foreground
+      SRC_LANGUAGE: "python"
+
+CONVERSATION:
+  DB_TABLE: 
+    ANDROID: plugin_studentlife_audio_android
+    IOS: plugin_studentlife_audio
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: False
+      FEATURES: ["minutessilence", "minutesnoise", "minutesvoice", "minutesunknown","sumconversationduration","avgconversationduration",
+        "sdconversationduration","minconversationduration","maxconversationduration","timefirstconversation","timelastconversation","noisesumenergy",
+        "noiseavgenergy","noisesdenergy","noiseminenergy","noisemaxenergy","voicesumenergy",
+        "voiceavgenergy","voicesdenergy","voiceminenergy","voicemaxenergy","silencesensedfraction","noisesensedfraction",
+        "voicesensedfraction","unknownsensedfraction","silenceexpectedfraction","noiseexpectedfraction","voiceexpectedfraction",
+        "unknownexpectedfraction","countconversation"]
+      RECORDING_MINUTES: 1
+      PAUSED_MINUTES : 3
+      SRC_FOLDER: "rapids" # inside src/features/conversation
+      SRC_LANGUAGE: "python"

tests/settings/periodic/config.yaml

@@ -0,0 +1,5 @@
+directory: ./
+configfile: ./tests/settings/periodic/testing_config.yaml
+snakefile: ./tests/Snakefile
+cores: 1
+forcerun: [join_features_from_providers]

tests/settings/periodic/testing_config.yaml

@@ -2,6 +2,7 @@
 # You must create a file for each participant named pXXX containing their device_id. This can be done manually or automatically
 PIDS: [test01, test02, test03, test04]
 
+
 # Global var with common day segments
 DAY_SEGMENTS: &day_segments
   TYPE: PERIODIC # FREQUENCY, PERIODIC, EVENT
@@ -15,7 +16,7 @@ MESSAGES:
   DB_TABLE: messages
   PROVIDERS:
     RAPIDS:
-      COMPUTE: True
+      COMPUTE: False
       MESSAGES_TYPES : [received, sent]
       FEATURES: 
         received: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact]
@@ -28,7 +29,7 @@ CALLS:
   DB_TABLE: calls
   PROVIDERS:
     RAPIDS:
-      COMPUTE: True
+      COMPUTE: False
       CALL_TYPES: [missed, incoming, outgoing]
       FEATURES:
         missed:  [count, distinctcontacts, timefirstcall, timelastcall, countmostfrequentcontact]
@@ -50,7 +51,7 @@ LIGHT:
   DB_TABLE: light
   PROVIDERS:
     RAPIDS:
-      COMPUTE: True
+      COMPUTE: False
       FEATURES: ["count", "maxlux", "minlux", "avglux", "medianlux", "stdlux"]
       SRC_FOLDER: "rapids" # inside src/features/light
       SRC_LANGUAGE: "python"
@@ -61,7 +62,7 @@ WIFI:
     CONNECTED_ACCESS_POINTS: "sensor_wifi" # if you only have a VISIBLE_ACCESS_POINTS table, set this value to ""
   PROVIDERS:
     RAPIDS:
-      COMPUTE: True
+      COMPUTE: False
       FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"]
       SRC_FOLDER: "rapids" # inside src/features/bluetooth
       SRC_LANGUAGE: "r"
@@ -70,7 +71,7 @@ APPLICATIONS_FOREGROUND:
   DB_TABLE: applications_foreground
   PROVIDERS:
     RAPIDS:
-      COMPUTE: True
+      COMPUTE: False
       SINGLE_CATEGORIES: ["all", "email"]
       MULTIPLE_CATEGORIES:
         social: ["socialnetworks", "socialmediatools"]
@@ -88,7 +89,7 @@ CONVERSATION:
     IOS: plugin_studentlife_audio
   PROVIDERS:
     RAPIDS:
-      COMPUTE: True
+      COMPUTE: False
       FEATURES: ["minutessilence", "minutesnoise", "minutesvoice", "minutesunknown","sumconversationduration","avgconversationduration",
         "sdconversationduration","minconversationduration","maxconversationduration","timefirstconversation","timelastconversation","noisesumenergy",
         "noiseavgenergy","noisesdenergy","noiseminenergy","noisemaxenergy","voicesumenergy",