From cf2a8f13a3c5861fff4d6d3eed3b3855f1def52c Mon Sep 17 00:00:00 2001 From: kaguillera Date: Fri, 4 Sep 2020 12:33:13 -0400 Subject: [PATCH] New testing and minor bug fix for light --- src/features/light/rapids/main.py | 2 +- tests/Snakefile | 7 +++++++ tests/data/processed/features/test01/light.csv | 6 ++++++ tests/data/processed/features/test02/light.csv | 1 + tests/data/processed/features/test03/light.csv | 1 + tests/data/processed/features/test04/light.csv | 1 + tests/settings/testing_config.yaml | 11 ++++++++++- 7 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 tests/data/processed/features/test01/light.csv create mode 100644 tests/data/processed/features/test02/light.csv create mode 100644 tests/data/processed/features/test03/light.csv create mode 100644 tests/data/processed/features/test04/light.csv diff --git a/src/features/light/rapids/main.py b/src/features/light/rapids/main.py index 105c1f49..ffe31ea0 100644 --- a/src/features/light/rapids/main.py +++ b/src/features/light/rapids/main.py @@ -27,7 +27,7 @@ def rapids_features(light_data, day_segment, provider, filter_data_by_segment, * if "medianlux" in features_to_compute: light_features["light_rapids_" + "_medianlux"] = light_data.groupby(["local_segment"])["double_light_lux"].median() if "stdlux" in features_to_compute: - light_features["light_rapids_" + "_stdlux"] = light_data.groupby(["local_segment"])["double_light_lux"].std().fillna('NA') + light_features["light_rapids_" + "_stdlux"] = light_data.groupby(["local_segment"])["double_light_lux"].std() light_features = light_features.reset_index() diff --git a/tests/Snakefile b/tests/Snakefile index 117634a9..fceece6c 100644 --- a/tests/Snakefile +++ b/tests/Snakefile @@ -54,6 +54,13 @@ for provider in config["BLUETOOTH"]["PROVIDERS"].keys(): files_to_compute.extend(expand("data/interim/{pid}/{sensor_key}_features/{sensor_key}_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["BLUETOOTH"]["PROVIDERS"][provider]["SRC_LANGUAGE"], provider_key=provider, sensor_key="BLUETOOTH".lower())) files_to_compute.extend(expand("data/processed/features/{pid}/{sensor_key}.csv", pid=config["PIDS"], sensor_key="BLUETOOTH".lower())) +for provider in config["LIGHT"]["PROVIDERS"].keys(): + if config["LIGHT"]["PROVIDERS"][provider]["COMPUTE"]: + files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=config["LIGHT"]["DB_TABLE"])) + files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime.csv", pid=config["PIDS"], sensor=config["LIGHT"]["DB_TABLE"])) + files_to_compute.extend(expand("data/interim/{pid}/{sensor_key}_features/{sensor_key}_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["LIGHT"]["PROVIDERS"][provider]["SRC_LANGUAGE"], provider_key=provider, sensor_key="LIGHT".lower())) + files_to_compute.extend(expand("data/processed/features/{pid}/{sensor_key}.csv", pid=config["PIDS"], sensor_key="LIGHT".lower())) + rule all: input: files_to_compute diff --git a/tests/data/processed/features/test01/light.csv b/tests/data/processed/features/test01/light.csv new file mode 100644 index 00000000..9381685e --- /dev/null +++ b/tests/data/processed/features/test01/light.csv @@ -0,0 +1,6 @@ +"local_segment","local_segment_label","local_start_date","local_start_time","local_end_date","local_end_time","light_rapids__count","light_rapids__maxlux","light_rapids__minlux","light_rapids__avglux","light_rapids__medianlux","light_rapids__stdlux" +"afternoon#2020-07-04#12:00:00#2020-07-04#17:59:59","afternoon","2020-07-04","12:00:00","2020-07-04","17:59:59",4,97656,10351,55761.5,57519.5,44778.6855136831 +"daily#2020-07-04#00:00:00#2020-07-04#23:59:59","daily","2020-07-04","00:00:00","2020-07-04","23:59:59",12,114615,0.065,40207.8950833333,19836,44686.694225665 +"evening#2020-07-04#18:00:00#2020-07-04#23:59:59","evening","2020-07-04","18:00:00","2020-07-04","23:59:59",3,15258,84.156,5603.71866666667,1469,8389.47610244199 +"morning#2020-07-04#06:00:00#2020-07-04#11:59:59","morning","2020-07-04","06:00:00","2020-07-04","11:59:59",4,114615,472.52,60659.38,63775,51510.5823438666 +"night#2020-07-04#00:00:00#2020-07-04#05:59:59","night","2020-07-04","00:00:00","2020-07-04","05:59:59",1,0.065,0.065,0.065,0.065,NA diff --git a/tests/data/processed/features/test02/light.csv b/tests/data/processed/features/test02/light.csv new file mode 100644 index 00000000..21560584 --- /dev/null +++ b/tests/data/processed/features/test02/light.csv @@ -0,0 +1 @@ +"local_segment","local_segment_label","local_start_date","local_start_time","local_end_date","local_end_time","light_rapids__maxlux","light_rapids__stdlux","light_rapids__count","light_rapids__avglux","light_rapids__medianlux","light_rapids__minlux" diff --git a/tests/data/processed/features/test03/light.csv b/tests/data/processed/features/test03/light.csv new file mode 100644 index 00000000..94109a8b --- /dev/null +++ b/tests/data/processed/features/test03/light.csv @@ -0,0 +1 @@ +"local_segment","local_segment_label","local_start_date","local_start_time","local_end_date","local_end_time","light_rapids__avglux","light_rapids__maxlux","light_rapids__stdlux","light_rapids__medianlux","light_rapids__count","light_rapids__minlux" diff --git a/tests/data/processed/features/test04/light.csv b/tests/data/processed/features/test04/light.csv new file mode 100644 index 00000000..1d50198b --- /dev/null +++ b/tests/data/processed/features/test04/light.csv @@ -0,0 +1 @@ +"local_segment","local_segment_label","local_start_date","local_start_time","local_end_date","local_end_time","light_rapids__medianlux","light_rapids__minlux","light_rapids__stdlux","light_rapids__count","light_rapids__avglux","light_rapids__maxlux" diff --git a/tests/settings/testing_config.yaml b/tests/settings/testing_config.yaml index 690e712f..c8a43077 100644 --- a/tests/settings/testing_config.yaml +++ b/tests/settings/testing_config.yaml @@ -8,7 +8,7 @@ DAY_SEGMENTS: &day_segments FILE: "data/external/daysegments_interval.csv" PHONE_VALID_SENSED_BINS: - DB_TABLES: [messages, calls, bluetooth] + DB_TABLES: [messages, calls, bluetooth, light] # Communication SMS features config, TYPES and FEATURES keys need to match MESSAGES: @@ -46,3 +46,12 @@ BLUETOOTH: SRC_FOLDER: "rapids" # inside src/features/bluetooth SRC_LANGUAGE: "r" +LIGHT: + DB_TABLE: light + PROVIDERS: + RAPIDS: + COMPUTE: True + FEATURES: ["count", "maxlux", "minlux", "avglux", "medianlux", "stdlux"] + SRC_FOLDER: "rapids" # inside src/features/light + SRC_LANGUAGE: "python" +