From ca14eba8e13192b1e958eb8857ca226ba573497d Mon Sep 17 00:00:00 2001 From: kaguillera Date: Fri, 18 Sep 2020 19:14:07 -0400 Subject: [PATCH] Initial testing restructure for 30mins segments --- data/external/daysegments_frequency.csv | 2 +- data/external/daysegments_periodic.csv | 5 +- src/features/bluetooth/rapids/main.R | 1 + .../features/frequency/test01/bluetooth.csv | 14 +++ .../features/frequency/test02/bluetooth.csv | 14 +++ .../{ => frequency}/test03/bluetooth.csv | 0 .../{ => frequency}/test04/bluetooth.csv | 0 .../test01/applications_foreground.csv | 0 .../{ => periodic}/test01/bluetooth.csv | 0 .../features/{ => periodic}/test01/calls.csv | 0 .../{ => periodic}/test01/conversation.csv | 0 .../features/{ => periodic}/test01/light.csv | 0 .../{ => periodic}/test01/messages.csv | 0 .../features/{ => periodic}/test01/wifi.csv | 0 .../test02/applications_foreground.csv | 0 .../{ => periodic}/test02/bluetooth.csv | 0 .../features/{ => periodic}/test02/calls.csv | 0 .../{ => periodic}/test02/conversation.csv | 0 .../features/{ => periodic}/test02/light.csv | 0 .../{ => periodic}/test02/messages.csv | 0 .../features/{ => periodic}/test02/wifi.csv | 0 .../test03/applications_foreground.csv | 0 .../features/periodic/test03/bluetooth.csv | 1 + .../features/{ => periodic}/test03/calls.csv | 0 .../{ => periodic}/test03/conversation.csv | 0 .../features/{ => periodic}/test03/light.csv | 0 .../{ => periodic}/test03/messages.csv | 0 .../features/{ => periodic}/test03/wifi.csv | 0 .../test04/applications_foreground.csv | 0 .../features/periodic/test04/bluetooth.csv | 1 + .../features/{ => periodic}/test04/calls.csv | 0 .../{ => periodic}/test04/conversation.csv | 0 .../features/{ => periodic}/test04/light.csv | 0 .../{ => periodic}/test04/messages.csv | 0 .../features/{ => periodic}/test04/wifi.csv | 0 tests/scripts/run_tests.sh | 36 ++++++- tests/scripts/test_sensor_features.py | 9 +- tests/scripts/utils.py | 10 +- tests/settings/config.yaml | 5 - tests/settings/frequency/config.yaml | 5 + tests/settings/frequency/testing_config.yaml | 101 ++++++++++++++++++ tests/settings/periodic/config.yaml | 5 + .../{ => periodic}/testing_config.yaml | 13 +-- 43 files changed, 196 insertions(+), 26 deletions(-) create mode 100644 tests/data/processed/features/frequency/test01/bluetooth.csv create mode 100644 tests/data/processed/features/frequency/test02/bluetooth.csv rename tests/data/processed/features/{ => frequency}/test03/bluetooth.csv (100%) rename tests/data/processed/features/{ => frequency}/test04/bluetooth.csv (100%) rename tests/data/processed/features/{ => periodic}/test01/applications_foreground.csv (100%) rename tests/data/processed/features/{ => periodic}/test01/bluetooth.csv (100%) rename tests/data/processed/features/{ => periodic}/test01/calls.csv (100%) rename tests/data/processed/features/{ => periodic}/test01/conversation.csv (100%) rename tests/data/processed/features/{ => periodic}/test01/light.csv (100%) rename tests/data/processed/features/{ => periodic}/test01/messages.csv (100%) rename tests/data/processed/features/{ => periodic}/test01/wifi.csv (100%) rename tests/data/processed/features/{ => periodic}/test02/applications_foreground.csv (100%) rename tests/data/processed/features/{ => periodic}/test02/bluetooth.csv (100%) rename tests/data/processed/features/{ => periodic}/test02/calls.csv (100%) rename tests/data/processed/features/{ => periodic}/test02/conversation.csv (100%) rename tests/data/processed/features/{ => periodic}/test02/light.csv (100%) rename tests/data/processed/features/{ => periodic}/test02/messages.csv (100%) rename tests/data/processed/features/{ => periodic}/test02/wifi.csv (100%) rename tests/data/processed/features/{ => periodic}/test03/applications_foreground.csv (100%) create mode 100644 tests/data/processed/features/periodic/test03/bluetooth.csv rename tests/data/processed/features/{ => periodic}/test03/calls.csv (100%) rename tests/data/processed/features/{ => periodic}/test03/conversation.csv (100%) rename tests/data/processed/features/{ => periodic}/test03/light.csv (100%) rename tests/data/processed/features/{ => periodic}/test03/messages.csv (100%) rename tests/data/processed/features/{ => periodic}/test03/wifi.csv (100%) rename tests/data/processed/features/{ => periodic}/test04/applications_foreground.csv (100%) create mode 100644 tests/data/processed/features/periodic/test04/bluetooth.csv rename tests/data/processed/features/{ => periodic}/test04/calls.csv (100%) rename tests/data/processed/features/{ => periodic}/test04/conversation.csv (100%) rename tests/data/processed/features/{ => periodic}/test04/light.csv (100%) rename tests/data/processed/features/{ => periodic}/test04/messages.csv (100%) rename tests/data/processed/features/{ => periodic}/test04/wifi.csv (100%) delete mode 100644 tests/settings/config.yaml create mode 100644 tests/settings/frequency/config.yaml create mode 100644 tests/settings/frequency/testing_config.yaml create mode 100644 tests/settings/periodic/config.yaml rename tests/settings/{ => periodic}/testing_config.yaml (97%) diff --git a/data/external/daysegments_frequency.csv b/data/external/daysegments_frequency.csv index 55bcfba3..e56782d1 100644 --- a/data/external/daysegments_frequency.csv +++ b/data/external/daysegments_frequency.csv @@ -1,2 +1,2 @@ label,length -tenminutes,10 \ No newline at end of file +thirtyminutes,30 \ No newline at end of file diff --git a/data/external/daysegments_periodic.csv b/data/external/daysegments_periodic.csv index 7da05379..d47a2d2e 100644 --- a/data/external/daysegments_periodic.csv +++ b/data/external/daysegments_periodic.csv @@ -1,7 +1,6 @@ label,start_time,length,repeats_on,repeats_value daily,00:00:00,23H 59M 59S,every_day,0 -weekly,00:00:00,6D 23H 59M 59S,wday,1 morning,06:00:00,5H 59M 59S,every_day,0 afternoon,12:00:00,5H 59M 59S,every_day,0 -evening,18:00:00,5H 59M 59S,wday,2 -night,00:00:00,5H 59M 59S,qday,5 \ No newline at end of file +evening,18:00:00,5H 59M 59S,every_day,0 +night,00:00:00,5H 59M 59S,every_day,0 \ No newline at end of file diff --git a/src/features/bluetooth/rapids/main.R b/src/features/bluetooth/rapids/main.R index 19052856..cba86eed 100644 --- a/src/features/bluetooth/rapids/main.R +++ b/src/features/bluetooth/rapids/main.R @@ -18,6 +18,7 @@ compute_bluetooth_feature <- function(data, feature, day_segment){ filter(N == max(N)) %>% head(1) %>% # if there are multiple device with the same amount of scans pick the first one only pull(bt_address) + mostuniquedevice return(data %>% filter(bt_address == mostuniquedevice) %>% group_by(local_segment) %>% diff --git a/tests/data/processed/features/frequency/test01/bluetooth.csv b/tests/data/processed/features/frequency/test01/bluetooth.csv new file mode 100644 index 00000000..04ff0840 --- /dev/null +++ b/tests/data/processed/features/frequency/test01/bluetooth.csv @@ -0,0 +1,14 @@ +"local_segment","local_segment_label","local_start_date","local_start_time","local_end_date","local_end_time","bluetooth_rapids_countscans","bluetooth_rapids_uniquedevices","bluetooth_rapids_countscansmostuniquedevice" +"thirtyminutes0000#2020-07-02#00:00:00#2020-07-02#00:29:59","thirtyminutes0000","2020-07-02","00:00:00","2020-07-02","00:29:59",1,1,1 +"thirtyminutes0001#2020-07-02#00:30:00#2020-07-02#00:59:59","thirtyminutes0001","2020-07-02","00:30:00","2020-07-02","00:59:59",1,1,1 +"thirtyminutes0007#2020-07-02#03:30:00#2020-07-02#03:59:59","thirtyminutes0007","2020-07-02","03:30:00","2020-07-02","03:59:59",1,1,1 +"thirtyminutes0011#2020-07-02#05:30:00#2020-07-02#05:59:59","thirtyminutes0011","2020-07-02","05:30:00","2020-07-02","05:59:59",1,1,1 +"thirtyminutes0012#2020-07-02#06:00:00#2020-07-02#06:29:59","thirtyminutes0012","2020-07-02","06:00:00","2020-07-02","06:29:59",1,1,1 +"thirtyminutes0014#2020-07-02#07:00:00#2020-07-02#07:29:59","thirtyminutes0014","2020-07-02","07:00:00","2020-07-02","07:29:59",1,1,1 +"thirtyminutes0023#2020-07-02#11:30:00#2020-07-02#11:59:59","thirtyminutes0023","2020-07-02","11:30:00","2020-07-02","11:59:59",1,1,1 +"thirtyminutes0024#2020-07-02#12:00:00#2020-07-02#12:29:59","thirtyminutes0024","2020-07-02","12:00:00","2020-07-02","12:29:59",1,1,1 +"thirtyminutes0035#2020-07-02#17:30:00#2020-07-02#17:59:59","thirtyminutes0035","2020-07-02","17:30:00","2020-07-02","17:59:59",1,1,1 +"thirtyminutes0036#2020-07-02#18:00:00#2020-07-02#18:29:59","thirtyminutes0036","2020-07-02","18:00:00","2020-07-02","18:29:59",1,1,1 +"thirtyminutes0039#2020-07-02#19:30:00#2020-07-02#19:59:59","thirtyminutes0039","2020-07-02","19:30:00","2020-07-02","19:59:59",1,1,1 +"thirtyminutes0042#2020-07-02#21:00:00#2020-07-02#21:29:59","thirtyminutes0042","2020-07-02","21:00:00","2020-07-02","21:29:59",1,1,1 +"thirtyminutes0047#2020-07-02#23:30:00#2020-07-02#23:59:59","thirtyminutes0047","2020-07-02","23:30:00","2020-07-02","23:59:59",2,2,1 diff --git a/tests/data/processed/features/frequency/test02/bluetooth.csv b/tests/data/processed/features/frequency/test02/bluetooth.csv new file mode 100644 index 00000000..448a29fc --- /dev/null +++ b/tests/data/processed/features/frequency/test02/bluetooth.csv @@ -0,0 +1,14 @@ +"local_segment","local_segment_label","local_start_date","local_start_time","local_end_date","local_end_time","bluetooth_rapids_countscans","bluetooth_rapids_uniquedevices","bluetooth_rapids_countscansmostuniquedevice" +"thirtyminutes0000#2020-07-02#00:00:00#2020-07-02#00:29:59","thirtyminutes0000","2020-07-02","00:00:00","2020-07-02","00:29:59",1,1,1 +"thirtyminutes0001#2020-07-02#00:30:00#2020-07-02#00:59:59","thirtyminutes0001","2020-07-02","00:30:00","2020-07-02","00:59:59",1,1,1 +"thirtyminutes0007#2020-07-02#03:30:00#2020-07-02#03:59:59","thirtyminutes0007","2020-07-02","03:30:00","2020-07-02","03:59:59",1,1,1 +"thirtyminutes0011#2020-07-02#05:30:00#2020-07-02#05:59:59","thirtyminutes0011","2020-07-02","05:30:00","2020-07-02","05:59:59",1,1,1 +"thirtyminutes0012#2020-07-02#06:00:00#2020-07-02#06:29:59","thirtyminutes0012","2020-07-02","06:00:00","2020-07-02","06:29:59",1,1,1 +"thirtyminutes0014#2020-07-02#07:00:00#2020-07-02#07:29:59","thirtyminutes0014","2020-07-02","07:00:00","2020-07-02","07:29:59",1,1,1 +"thirtyminutes0023#2020-07-02#11:30:00#2020-07-02#11:59:59","thirtyminutes0023","2020-07-02","11:30:00","2020-07-02","11:59:59",1,1,1 +"thirtyminutes0024#2020-07-02#12:00:00#2020-07-02#12:29:59","thirtyminutes0024","2020-07-02","12:00:00","2020-07-02","12:29:59",1,1,1 +"thirtyminutes0035#2020-07-02#17:30:00#2020-07-02#17:59:59","thirtyminutes0035","2020-07-02","17:30:00","2020-07-02","17:59:59",1,1,1 +"thirtyminutes0036#2020-07-02#18:00:00#2020-07-02#18:29:59","thirtyminutes0036","2020-07-02","18:00:00","2020-07-02","18:29:59",1,1,1 +"thirtyminutes0039#2020-07-02#19:30:00#2020-07-02#19:59:59","thirtyminutes0039","2020-07-02","19:30:00","2020-07-02","19:59:59",1,1,1 +"thirtyminutes0042#2020-07-02#21:00:00#2020-07-02#21:29:59","thirtyminutes0042","2020-07-02","21:00:00","2020-07-02","21:29:59",1,1,1 +"thirtyminutes0047#2020-07-02#23:30:00#2020-07-02#23:59:59","thirtyminutes0047","2020-07-02","23:30:00","2020-07-02","23:59:59",2,1,2 diff --git a/tests/data/processed/features/test03/bluetooth.csv b/tests/data/processed/features/frequency/test03/bluetooth.csv similarity index 100% rename from tests/data/processed/features/test03/bluetooth.csv rename to tests/data/processed/features/frequency/test03/bluetooth.csv diff --git a/tests/data/processed/features/test04/bluetooth.csv b/tests/data/processed/features/frequency/test04/bluetooth.csv similarity index 100% rename from tests/data/processed/features/test04/bluetooth.csv rename to tests/data/processed/features/frequency/test04/bluetooth.csv diff --git a/tests/data/processed/features/test01/applications_foreground.csv b/tests/data/processed/features/periodic/test01/applications_foreground.csv similarity index 100% rename from tests/data/processed/features/test01/applications_foreground.csv rename to tests/data/processed/features/periodic/test01/applications_foreground.csv diff --git a/tests/data/processed/features/test01/bluetooth.csv b/tests/data/processed/features/periodic/test01/bluetooth.csv similarity index 100% rename from tests/data/processed/features/test01/bluetooth.csv rename to tests/data/processed/features/periodic/test01/bluetooth.csv diff --git a/tests/data/processed/features/test01/calls.csv b/tests/data/processed/features/periodic/test01/calls.csv similarity index 100% rename from tests/data/processed/features/test01/calls.csv rename to tests/data/processed/features/periodic/test01/calls.csv diff --git a/tests/data/processed/features/test01/conversation.csv b/tests/data/processed/features/periodic/test01/conversation.csv similarity index 100% rename from tests/data/processed/features/test01/conversation.csv rename to tests/data/processed/features/periodic/test01/conversation.csv diff --git a/tests/data/processed/features/test01/light.csv b/tests/data/processed/features/periodic/test01/light.csv similarity index 100% rename from tests/data/processed/features/test01/light.csv rename to tests/data/processed/features/periodic/test01/light.csv diff --git a/tests/data/processed/features/test01/messages.csv b/tests/data/processed/features/periodic/test01/messages.csv similarity index 100% rename from tests/data/processed/features/test01/messages.csv rename to tests/data/processed/features/periodic/test01/messages.csv diff --git a/tests/data/processed/features/test01/wifi.csv b/tests/data/processed/features/periodic/test01/wifi.csv similarity index 100% rename from tests/data/processed/features/test01/wifi.csv rename to tests/data/processed/features/periodic/test01/wifi.csv diff --git a/tests/data/processed/features/test02/applications_foreground.csv b/tests/data/processed/features/periodic/test02/applications_foreground.csv similarity index 100% rename from tests/data/processed/features/test02/applications_foreground.csv rename to tests/data/processed/features/periodic/test02/applications_foreground.csv diff --git a/tests/data/processed/features/test02/bluetooth.csv b/tests/data/processed/features/periodic/test02/bluetooth.csv similarity index 100% rename from tests/data/processed/features/test02/bluetooth.csv rename to tests/data/processed/features/periodic/test02/bluetooth.csv diff --git a/tests/data/processed/features/test02/calls.csv b/tests/data/processed/features/periodic/test02/calls.csv similarity index 100% rename from tests/data/processed/features/test02/calls.csv rename to tests/data/processed/features/periodic/test02/calls.csv diff --git a/tests/data/processed/features/test02/conversation.csv b/tests/data/processed/features/periodic/test02/conversation.csv similarity index 100% rename from tests/data/processed/features/test02/conversation.csv rename to tests/data/processed/features/periodic/test02/conversation.csv diff --git a/tests/data/processed/features/test02/light.csv b/tests/data/processed/features/periodic/test02/light.csv similarity index 100% rename from tests/data/processed/features/test02/light.csv rename to tests/data/processed/features/periodic/test02/light.csv diff --git a/tests/data/processed/features/test02/messages.csv b/tests/data/processed/features/periodic/test02/messages.csv similarity index 100% rename from tests/data/processed/features/test02/messages.csv rename to tests/data/processed/features/periodic/test02/messages.csv diff --git a/tests/data/processed/features/test02/wifi.csv b/tests/data/processed/features/periodic/test02/wifi.csv similarity index 100% rename from tests/data/processed/features/test02/wifi.csv rename to tests/data/processed/features/periodic/test02/wifi.csv diff --git a/tests/data/processed/features/test03/applications_foreground.csv b/tests/data/processed/features/periodic/test03/applications_foreground.csv similarity index 100% rename from tests/data/processed/features/test03/applications_foreground.csv rename to tests/data/processed/features/periodic/test03/applications_foreground.csv diff --git a/tests/data/processed/features/periodic/test03/bluetooth.csv b/tests/data/processed/features/periodic/test03/bluetooth.csv new file mode 100644 index 00000000..0251cdb2 --- /dev/null +++ b/tests/data/processed/features/periodic/test03/bluetooth.csv @@ -0,0 +1 @@ +"local_segment","local_segment_label","local_start_date","local_start_time","local_end_date","local_end_time","bluetooth_rapids_countscans","bluetooth_rapids_uniquedevices","bluetooth_rapids_countscansmostuniquedevice" diff --git a/tests/data/processed/features/test03/calls.csv b/tests/data/processed/features/periodic/test03/calls.csv similarity index 100% rename from tests/data/processed/features/test03/calls.csv rename to tests/data/processed/features/periodic/test03/calls.csv diff --git a/tests/data/processed/features/test03/conversation.csv b/tests/data/processed/features/periodic/test03/conversation.csv similarity index 100% rename from tests/data/processed/features/test03/conversation.csv rename to tests/data/processed/features/periodic/test03/conversation.csv diff --git a/tests/data/processed/features/test03/light.csv b/tests/data/processed/features/periodic/test03/light.csv similarity index 100% rename from tests/data/processed/features/test03/light.csv rename to tests/data/processed/features/periodic/test03/light.csv diff --git a/tests/data/processed/features/test03/messages.csv b/tests/data/processed/features/periodic/test03/messages.csv similarity index 100% rename from tests/data/processed/features/test03/messages.csv rename to tests/data/processed/features/periodic/test03/messages.csv diff --git a/tests/data/processed/features/test03/wifi.csv b/tests/data/processed/features/periodic/test03/wifi.csv similarity index 100% rename from tests/data/processed/features/test03/wifi.csv rename to tests/data/processed/features/periodic/test03/wifi.csv diff --git a/tests/data/processed/features/test04/applications_foreground.csv b/tests/data/processed/features/periodic/test04/applications_foreground.csv similarity index 100% rename from tests/data/processed/features/test04/applications_foreground.csv rename to tests/data/processed/features/periodic/test04/applications_foreground.csv diff --git a/tests/data/processed/features/periodic/test04/bluetooth.csv b/tests/data/processed/features/periodic/test04/bluetooth.csv new file mode 100644 index 00000000..0251cdb2 --- /dev/null +++ b/tests/data/processed/features/periodic/test04/bluetooth.csv @@ -0,0 +1 @@ +"local_segment","local_segment_label","local_start_date","local_start_time","local_end_date","local_end_time","bluetooth_rapids_countscans","bluetooth_rapids_uniquedevices","bluetooth_rapids_countscansmostuniquedevice" diff --git a/tests/data/processed/features/test04/calls.csv b/tests/data/processed/features/periodic/test04/calls.csv similarity index 100% rename from tests/data/processed/features/test04/calls.csv rename to tests/data/processed/features/periodic/test04/calls.csv diff --git a/tests/data/processed/features/test04/conversation.csv b/tests/data/processed/features/periodic/test04/conversation.csv similarity index 100% rename from tests/data/processed/features/test04/conversation.csv rename to tests/data/processed/features/periodic/test04/conversation.csv diff --git a/tests/data/processed/features/test04/light.csv b/tests/data/processed/features/periodic/test04/light.csv similarity index 100% rename from tests/data/processed/features/test04/light.csv rename to tests/data/processed/features/periodic/test04/light.csv diff --git a/tests/data/processed/features/test04/messages.csv b/tests/data/processed/features/periodic/test04/messages.csv similarity index 100% rename from tests/data/processed/features/test04/messages.csv rename to tests/data/processed/features/periodic/test04/messages.csv diff --git a/tests/data/processed/features/test04/wifi.csv b/tests/data/processed/features/periodic/test04/wifi.csv similarity index 100% rename from tests/data/processed/features/test04/wifi.csv rename to tests/data/processed/features/periodic/test04/wifi.csv diff --git a/tests/scripts/run_tests.sh b/tests/scripts/run_tests.sh index f6297a9b..c4b6f6db 100755 --- a/tests/scripts/run_tests.sh +++ b/tests/scripts/run_tests.sh @@ -22,13 +22,41 @@ echo Disabling downloading of dataset... sed -e '27,39 s/^/#/' -e 's/rules.download_dataset.output/"data\/raw\/\{pid\}\/\{sensor\}_raw\.csv"/' rules/preprocessing.smk > tmp cp tmp rules/preprocessing.smk -echo Running RAPIDS Pipeline on testdata... -snakemake --profile tests/settings +echo Running RAPIDS Pipeline periodic segment on testdata... +snakemake --profile tests/settings/periodic/ -echo Running tests on data produced... +echo Moving produced data from previous pipeline run ... +# rm -rf data/raw/* +mkdir data/processed/features/periodic +mv data/processed/features/test* data/processed/features/periodic/ +rm -rf data/interim/* +# rm -rf data/external/test* + +echo Running RAPIDS Pipeline frequnecy segment on testdata... +snakemake --profile tests/settings/frequency/ + +echo Moving produced data from previous pipeline run... +mkdir data/processed/features/frequency +mv data/processed/features/test* data/processed/features/frequency/ + +echo Running tests on periodic data produced... python -m unittest discover tests/scripts/ -v +echo Backing up Testing script... +cp tests/scripts/test_sensor_features.py test_bak + +echo Re-writing the config file being loaded for testing +sed -e 's/tests\/settings\/periodic\/testing_config\.yaml/tests\/settings\/frequency\/testing_config\.yaml/' tests/scripts/test_sensor_features.py > test_tmp +cp test_tmp tests/scripts/test_sensor_features.py + +echo Running tests on frequency data produced... +./env/bin/python -m unittest discover tests/scripts/ -v + # Uncomment to return snakemake back to the original version when testing locally # echo Cleaning up... # mv bak rules/preprocessing.smk -# rm tmp \ No newline at end of file +# mv test_bak tests/scripts/test_sensor_features.py +# rm test_bak +# rm test_tmp +# rm bak +# rm tmp diff --git a/tests/scripts/test_sensor_features.py b/tests/scripts/test_sensor_features.py index 097353d2..43d79f34 100644 --- a/tests/scripts/test_sensor_features.py +++ b/tests/scripts/test_sensor_features.py @@ -4,6 +4,7 @@ import hashlib import pandas as pd import utils import yaml +import sys import os class TestSensorFeatures(unittest.TestCase): @@ -15,13 +16,14 @@ class TestSensorFeatures(unittest.TestCase): def setUpClass(cls): # Runs once to Setup env global configs - with open(r'tests/settings/testing_config.yaml') as file: + with open(r'tests/settings/periodic/testing_config.yaml') as file: configs = yaml.full_load(file) def test_sensors_files_exist(self): # Loop through the file_list dictionary and check if the files exist. + #print("Testing existance of files") file_lists = utils.generate_sensor_file_lists(configs) for each in file_lists: #for out_file, _ in file_lists[each]: @@ -30,7 +32,7 @@ class TestSensorFeatures(unittest.TestCase): def test_sensors_features_calculations(self): - + # print("Testing calculations..") sensor_file_list = utils.generate_sensor_file_lists(configs) for each in sensor_file_list: for act_result, exp_result in sensor_file_list: @@ -48,4 +50,5 @@ class TestSensorFeatures(unittest.TestCase): if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() + \ No newline at end of file diff --git a/tests/scripts/utils.py b/tests/scripts/utils.py index ec653e86..6c7a518f 100644 --- a/tests/scripts/utils.py +++ b/tests/scripts/utils.py @@ -49,8 +49,8 @@ def generate_file_list(configs, sensor): # i.e. The sensor passed into the function. # Initialize string of file path for both expected and actual metric values - act_str = "data/processed/{pid}/{sensor}_{sensor_type}{day_segment}.csv" - exp_str = "tests/data/processed/{pid}/{sensor}_{sensor_type}{day_segment}.csv" + act_str = "data/processed/features/{pid}/{sensor}_{sensor_type}{day_segment}.csv" + exp_str = "tests/data/processed/features/period/{pid}/{sensor}_{sensor_type}{day_segment}.csv" sensor_cap = sensor.upper() if 'DAY_SEGMENTS' and 'FEATURES' in configs[sensor_cap]: @@ -79,8 +79,10 @@ def generate_sensor_file_lists(configs): # actual files for each sensor listed in the config file. Added for Travis. # Initialize string of file path for both expected and actual metric values - act_str = "data/processed/features/{pid}/{sensor_key}.csv" - exp_str = "tests/data/processed/features/{pid}/{sensor_key}.csv" + segment = configs['DAY_SEGMENTS']['TYPE'].lower() + print(segment) + act_str = "data/processed/features/"+segment+"/{pid}/{sensor_key}.csv" + exp_str = "tests/data/processed/features/"+segment+"/{pid}/{sensor_key}.csv" # List of available sensors that can be tested by the testing suite TESTABLE_SENSORS = ['MESSAGES', 'CALLS', 'SCREEN', 'BATTERY', 'BLUETOOTH', 'WIFI', 'LIGHT', 'APPLICATIONS_FOREGROUND', 'ACTIVITY_RECOGNITION', 'CONVERSATION'] diff --git a/tests/settings/config.yaml b/tests/settings/config.yaml deleted file mode 100644 index ba36a53c..00000000 --- a/tests/settings/config.yaml +++ /dev/null @@ -1,5 +0,0 @@ -directory: ./ -configfile: ./tests/settings/testing_config.yaml -snakefile: ./tests/Snakefile -cores: 1 -forcerun: [join_features_from_providers] \ No newline at end of file diff --git a/tests/settings/frequency/config.yaml b/tests/settings/frequency/config.yaml new file mode 100644 index 00000000..0ebef6c0 --- /dev/null +++ b/tests/settings/frequency/config.yaml @@ -0,0 +1,5 @@ +directory: ./ +configfile: ./tests/settings/frequency/testing_config.yaml +snakefile: ./tests/Snakefile +cores: 1 +forcerun: [join_features_from_providers] \ No newline at end of file diff --git a/tests/settings/frequency/testing_config.yaml b/tests/settings/frequency/testing_config.yaml new file mode 100644 index 00000000..617bfe76 --- /dev/null +++ b/tests/settings/frequency/testing_config.yaml @@ -0,0 +1,101 @@ +# Participants to include in the analysis +# You must create a file for each participant named pXXX containing their device_id. This can be done manually or automatically +PIDS: [test01, test02, test03, test04] + +# Global var with common day segments +DAY_SEGMENTS: &day_segments + TYPE: FREQUENCY # FREQUENCY, PERIODIC, EVENT + FILE: "data/external/daysegments_frequency.csv" + +PHONE_VALID_SENSED_BINS: + DB_TABLES: [bluetooth] + +# Communication SMS features config, TYPES and FEATURES keys need to match +MESSAGES: + DB_TABLE: messages + PROVIDERS: + RAPIDS: + COMPUTE: False + MESSAGES_TYPES : [received, sent] + FEATURES: + received: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact] + sent: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact] + SRC_LANGUAGE: "r" + SRC_FOLDER: "rapids" # inside src/features/messages + +# Communication call features config, TYPES and FEATURES keys need to match +CALLS: + DB_TABLE: calls + PROVIDERS: + RAPIDS: + COMPUTE: False + CALL_TYPES: [missed, incoming, outgoing] + FEATURES: + missed: [count, distinctcontacts, timefirstcall, timelastcall, countmostfrequentcontact] + incoming: [count, distinctcontacts, meanduration, sumduration, minduration, maxduration, stdduration, modeduration, entropyduration, timefirstcall, timelastcall, countmostfrequentcontact] + outgoing: [count, distinctcontacts, meanduration, sumduration, minduration, maxduration, stdduration, modeduration, entropyduration, timefirstcall, timelastcall, countmostfrequentcontact] + SRC_LANGUAGE: "r" + SRC_FOLDER: "rapids" # inside src/features/calls + +BLUETOOTH: + DB_TABLE: bluetooth + PROVIDERS: + RAPIDS: + COMPUTE: True + FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] + SRC_FOLDER: "rapids" # inside src/features/bluetooth + SRC_LANGUAGE: "r" + +LIGHT: + DB_TABLE: light + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["count", "maxlux", "minlux", "avglux", "medianlux", "stdlux"] + SRC_FOLDER: "rapids" # inside src/features/light + SRC_LANGUAGE: "python" + +WIFI: + DB_TABLE: + VISIBLE_ACCESS_POINTS: "wifi" # if you only have a CONNECTED_ACCESS_POINTS table, set this value to "" + CONNECTED_ACCESS_POINTS: "sensor_wifi" # if you only have a VISIBLE_ACCESS_POINTS table, set this value to "" + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] + SRC_FOLDER: "rapids" # inside src/features/bluetooth + SRC_LANGUAGE: "r" + +APPLICATIONS_FOREGROUND: + DB_TABLE: applications_foreground + PROVIDERS: + RAPIDS: + COMPUTE: False + SINGLE_CATEGORIES: ["all", "email"] + MULTIPLE_CATEGORIES: + social: ["socialnetworks", "socialmediatools"] + entertainment: ["entertainment", "gamingstrategy"] + SINGLE_APPS: ["top1global", "com.facebook.moments"] # There's no entropy for single apps + EXCLUDED_CATEGORIES: ["systemapp", "tvvideoapps"] + EXCLUDED_APPS: ["com.fitbit.FitbitMobile", "com.aware.plugin.upmc.cancer"] + FEATURES: ["count", "timeoffirstuse", "timeoflastuse", "frequencyentropy"] + SRC_FOLDER: "rapids" # inside src/features/applications_foreground + SRC_LANGUAGE: "python" + +CONVERSATION: + DB_TABLE: + ANDROID: plugin_studentlife_audio_android + IOS: plugin_studentlife_audio + PROVIDERS: + RAPIDS: + COMPUTE: False + FEATURES: ["minutessilence", "minutesnoise", "minutesvoice", "minutesunknown","sumconversationduration","avgconversationduration", + "sdconversationduration","minconversationduration","maxconversationduration","timefirstconversation","timelastconversation","noisesumenergy", + "noiseavgenergy","noisesdenergy","noiseminenergy","noisemaxenergy","voicesumenergy", + "voiceavgenergy","voicesdenergy","voiceminenergy","voicemaxenergy","silencesensedfraction","noisesensedfraction", + "voicesensedfraction","unknownsensedfraction","silenceexpectedfraction","noiseexpectedfraction","voiceexpectedfraction", + "unknownexpectedfraction","countconversation"] + RECORDING_MINUTES: 1 + PAUSED_MINUTES : 3 + SRC_FOLDER: "rapids" # inside src/features/conversation + SRC_LANGUAGE: "python" diff --git a/tests/settings/periodic/config.yaml b/tests/settings/periodic/config.yaml new file mode 100644 index 00000000..5b99f5c9 --- /dev/null +++ b/tests/settings/periodic/config.yaml @@ -0,0 +1,5 @@ +directory: ./ +configfile: ./tests/settings/periodic/testing_config.yaml +snakefile: ./tests/Snakefile +cores: 1 +forcerun: [join_features_from_providers] \ No newline at end of file diff --git a/tests/settings/testing_config.yaml b/tests/settings/periodic/testing_config.yaml similarity index 97% rename from tests/settings/testing_config.yaml rename to tests/settings/periodic/testing_config.yaml index 29f7e98f..9400f8f6 100644 --- a/tests/settings/testing_config.yaml +++ b/tests/settings/periodic/testing_config.yaml @@ -2,6 +2,7 @@ # You must create a file for each participant named pXXX containing their device_id. This can be done manually or automatically PIDS: [test01, test02, test03, test04] + # Global var with common day segments DAY_SEGMENTS: &day_segments TYPE: PERIODIC # FREQUENCY, PERIODIC, EVENT @@ -15,7 +16,7 @@ MESSAGES: DB_TABLE: messages PROVIDERS: RAPIDS: - COMPUTE: True + COMPUTE: False MESSAGES_TYPES : [received, sent] FEATURES: received: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact] @@ -28,7 +29,7 @@ CALLS: DB_TABLE: calls PROVIDERS: RAPIDS: - COMPUTE: True + COMPUTE: False CALL_TYPES: [missed, incoming, outgoing] FEATURES: missed: [count, distinctcontacts, timefirstcall, timelastcall, countmostfrequentcontact] @@ -50,7 +51,7 @@ LIGHT: DB_TABLE: light PROVIDERS: RAPIDS: - COMPUTE: True + COMPUTE: False FEATURES: ["count", "maxlux", "minlux", "avglux", "medianlux", "stdlux"] SRC_FOLDER: "rapids" # inside src/features/light SRC_LANGUAGE: "python" @@ -61,7 +62,7 @@ WIFI: CONNECTED_ACCESS_POINTS: "sensor_wifi" # if you only have a VISIBLE_ACCESS_POINTS table, set this value to "" PROVIDERS: RAPIDS: - COMPUTE: True + COMPUTE: False FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] SRC_FOLDER: "rapids" # inside src/features/bluetooth SRC_LANGUAGE: "r" @@ -70,7 +71,7 @@ APPLICATIONS_FOREGROUND: DB_TABLE: applications_foreground PROVIDERS: RAPIDS: - COMPUTE: True + COMPUTE: False SINGLE_CATEGORIES: ["all", "email"] MULTIPLE_CATEGORIES: social: ["socialnetworks", "socialmediatools"] @@ -88,7 +89,7 @@ CONVERSATION: IOS: plugin_studentlife_audio PROVIDERS: RAPIDS: - COMPUTE: True + COMPUTE: False FEATURES: ["minutessilence", "minutesnoise", "minutesvoice", "minutesunknown","sumconversationduration","avgconversationduration", "sdconversationduration","minconversationduration","maxconversationduration","timefirstconversation","timelastconversation","noisesumenergy", "noiseavgenergy","noisesdenergy","noiseminenergy","noisemaxenergy","voicesumenergy",