diff --git a/tests/Snakefile b/tests/Snakefile index 917e91ef..117634a9 100644 --- a/tests/Snakefile +++ b/tests/Snakefile @@ -39,6 +39,14 @@ for provider in config["MESSAGES"]["PROVIDERS"].keys(): files_to_compute.extend(expand("data/interim/{pid}/{sensor_key}_features/{sensor_key}_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["MESSAGES"]["PROVIDERS"][provider]["SRC_LANGUAGE"], provider_key=provider, sensor_key="MESSAGES".lower())) files_to_compute.extend(expand("data/processed/features/{pid}/{sensor_key}.csv", pid=config["PIDS"], sensor_key="MESSAGES".lower())) +for provider in config["CALLS"]["PROVIDERS"].keys(): + if config["CALLS"]["PROVIDERS"][provider]["COMPUTE"]: + files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=config["CALLS"]["DB_TABLE"])) + files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime.csv", pid=config["PIDS"], sensor=config["CALLS"]["DB_TABLE"])) + files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime_unified.csv", pid=config["PIDS"], sensor=config["CALLS"]["DB_TABLE"])) + files_to_compute.extend(expand("data/interim/{pid}/{sensor_key}_features/{sensor_key}_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["CALLS"]["PROVIDERS"][provider]["SRC_LANGUAGE"], provider_key=provider, sensor_key="CALLS".lower())) + files_to_compute.extend(expand("data/processed/features/{pid}/{sensor_key}.csv", pid=config["PIDS"], sensor_key="CALLS".lower())) + for provider in config["BLUETOOTH"]["PROVIDERS"].keys(): if config["BLUETOOTH"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=config["BLUETOOTH"]["DB_TABLE"])) diff --git a/tests/data/processed/features/test01/calls.csv b/tests/data/processed/features/test01/calls.csv new file mode 100644 index 00000000..e78ad697 --- /dev/null +++ b/tests/data/processed/features/test01/calls.csv @@ -0,0 +1,9 @@ +"local_segment","local_segment_label","local_start_date","local_start_time","local_end_date","local_end_time","calls_rapids_missed_count","calls_rapids_missed_distinctcontacts","calls_rapids_missed_timefirstcall","calls_rapids_missed_timelastcall","calls_rapids_missed_countmostfrequentcontact","calls_rapids_incoming_count","calls_rapids_incoming_distinctcontacts","calls_rapids_incoming_meanduration","calls_rapids_incoming_sumduration","calls_rapids_incoming_minduration","calls_rapids_incoming_maxduration","calls_rapids_incoming_stdduration","calls_rapids_incoming_modeduration","calls_rapids_incoming_entropyduration","calls_rapids_incoming_timefirstcall","calls_rapids_incoming_timelastcall","calls_rapids_incoming_countmostfrequentcontact","calls_rapids_outgoing_count","calls_rapids_outgoing_distinctcontacts","calls_rapids_outgoing_meanduration","calls_rapids_outgoing_sumduration","calls_rapids_outgoing_minduration","calls_rapids_outgoing_maxduration","calls_rapids_outgoing_stdduration","calls_rapids_outgoing_modeduration","calls_rapids_outgoing_entropyduration","calls_rapids_outgoing_timefirstcall","calls_rapids_outgoing_timelastcall","calls_rapids_outgoing_countmostfrequentcontact" +"afternoon#2020-06-01#12:00:00#2020-06-01#17:59:59","afternoon","2020-06-01","12:00:00","2020-06-01","17:59:59",1,1,874,874,1,3,2,642.666666666667,1928,213,1053,420.333597673721,1053,0.941278069255821,753,921,2,2,2,1237.5,2475,1186,1289,72.8319984622144,1289,0.692482998176928,869,1051,1 +"daily#2020-06-01#00:00:00#2020-06-01#23:59:59","daily","2020-06-01","00:00:00","2020-06-01","23:59:59",6,3,13,1167,4,10,6,976.4,9764,213,1719,465.141603289913,439,2.18820020272087,163,1331,5,8,6,1168,9344,759,1543,250.842010607702,970,2.05922274128194,172,1277,3 +"daily#2020-06-02#00:00:00#2020-06-02#23:59:59","daily","2020-06-02","00:00:00","2020-06-02","23:59:59",2,2,589,1167,1,5,5,1213.2,6066,667,1719,375.767481296612,667,1.56941860966338,519,1331,1,5,5,1179.8,5899,759,1543,310.478179587552,1116,1.58127936063292,418,1277,1 +"evening#2020-06-01#18:00:00#2020-06-01#23:59:59","evening","2020-06-01","18:00:00","2020-06-01","23:59:59",1,1,1167,1167,1,3,3,1366.66666666667,4100,1157,1719,306.963081384934,1157,1.08260122332248,1144,1331,1,2,2,1482,2964,1421,1543,86.2670273047588,1421,0.692468534923961,1156,1277,1 +"evening#2020-06-02#18:00:00#2020-06-02#23:59:59","evening","2020-06-02","18:00:00","2020-06-02","23:59:59",1,1,1167,1167,1,3,3,1366.66666666667,4100,1157,1719,306.963081384934,1157,1.08260122332248,1144,1331,1,2,2,1482,2964,1421,1543,86.2670273047588,1421,0.692468534923961,1156,1277,1 +"morning#2020-06-01#06:00:00#2020-06-01#11:59:59","morning","2020-06-01","06:00:00","2020-06-01","11:59:59",1,1,589,589,1,2,2,983,1966,667,1299,446.891485709898,667,0.640802774623272,519,600,1,3,3,978.333333333333,2935,759,1116,192.000868053593,1116,1.08558305836162,418,687,1 +"morning#2020-06-02#06:00:00#2020-06-02#11:59:59","morning","2020-06-02","06:00:00","2020-06-02","11:59:59",1,1,589,589,1,2,2,983,1966,667,1299,446.891485709898,667,0.640802774623272,519,600,1,3,3,978.333333333333,2935,759,1116,192.000868053593,1116,1.08558305836162,418,687,1 +"night#2020-06-01#00:00:00#2020-06-01#05:59:59","night","2020-06-01","00:00:00","2020-06-01","05:59:59",3,1,13,257,3,2,1,885,1770,439,1331,630.7392488184,439,0.560434787927257,163,257,2,1,1,970,970,970,970,NA,970,0,172,172,1 diff --git a/tests/data/processed/features/test02/calls.csv b/tests/data/processed/features/test02/calls.csv new file mode 100644 index 00000000..ab3338c3 --- /dev/null +++ b/tests/data/processed/features/test02/calls.csv @@ -0,0 +1,9 @@ +"local_segment","local_segment_label","local_start_date","local_start_time","local_end_date","local_end_time","calls_rapids_missed_count","calls_rapids_missed_distinctcontacts","calls_rapids_missed_timefirstcall","calls_rapids_missed_timelastcall","calls_rapids_missed_countmostfrequentcontact","calls_rapids_incoming_count","calls_rapids_incoming_distinctcontacts","calls_rapids_incoming_meanduration","calls_rapids_incoming_sumduration","calls_rapids_incoming_minduration","calls_rapids_incoming_maxduration","calls_rapids_incoming_stdduration","calls_rapids_incoming_modeduration","calls_rapids_incoming_entropyduration","calls_rapids_incoming_timefirstcall","calls_rapids_incoming_timelastcall","calls_rapids_incoming_countmostfrequentcontact","calls_rapids_outgoing_count","calls_rapids_outgoing_distinctcontacts","calls_rapids_outgoing_meanduration","calls_rapids_outgoing_sumduration","calls_rapids_outgoing_minduration","calls_rapids_outgoing_maxduration","calls_rapids_outgoing_stdduration","calls_rapids_outgoing_modeduration","calls_rapids_outgoing_entropyduration","calls_rapids_outgoing_timefirstcall","calls_rapids_outgoing_timelastcall","calls_rapids_outgoing_countmostfrequentcontact" +"afternoon#2020-06-01#12:00:00#2020-06-01#17:59:59","afternoon","2020-06-01","12:00:00","2020-06-01","17:59:59",NA,NA,NA,NA,NA,3,3,629.333333333333,1888,198,1040,421.38027164704,1040,0.932593293857646,753,921,1,3,3,816.333333333333,2449,0,1279,709.062996731132,1279,0.692360538889388,869,1051,1 +"daily#2020-06-01#00:00:00#2020-06-01#23:59:59","daily","2020-06-01","00:00:00","2020-06-01","23:59:59",3,3,13,1167,1,10,10,961.4,9614,198,1700,464.326800624062,420,2.18431625877259,163,1331,1,11,11,837.909090909091,9217,0,1530,577.720772440364,0,2.0585997872903,57,1277,1 +"daily#2020-06-02#00:00:00#2020-06-02#23:59:59","daily","2020-06-02","00:00:00","2020-06-02","23:59:59",1,1,589,589,0,5,5,1202,6010,660,1710,375.326524508993,660,1.56885323664414,519,1331,0,6,6,973.333333333333,5840,0,1530,551.313582878807,1100,1.58092421564451,418,1277,0 +"evening#2020-06-01#18:00:00#2020-06-01#23:59:59","evening","2020-06-01","18:00:00","2020-06-01","23:59:59",1,1,1167,1167,1,3,3,1350,4050,1140,1700,305.122926047847,1140,1.08239814505043,1144,1331,1,2,2,1465,2930,1400,1530,91.9238815542512,1400,0.692333219466401,1156,1277,1 +"evening#2020-06-02#18:00:00#2020-06-02#23:59:59","evening","2020-06-02","18:00:00","2020-06-02","23:59:59",NA,NA,NA,NA,NA,3,3,1353.33333333333,4060,1140,1710,310.859024854247,1140,1.08186609729599,1144,1331,0,3,3,980,2940,0,1530,850.823130856232,1410,0.692484030888463,1156,1277,0 +"morning#2020-06-01#06:00:00#2020-06-01#11:59:59","morning","2020-06-01","06:00:00","2020-06-01","11:59:59",1,1,589,589,1,2,2,968,1936,657,1279,439.820417898033,657,0.640867990789142,519,600,1,3,3,959.333333333333,2878,742,1096,190.287501779106,1096,1.08529592058152,418,687,1 +"morning#2020-06-02#06:00:00#2020-06-02#11:59:59","morning","2020-06-02","06:00:00","2020-06-02","11:59:59",1,1,589,589,0,2,2,975,1950,660,1290,445.477272147525,660,0.640266157864261,519,600,0,3,3,966.666666666667,2900,750,1100,189.296944860009,1100,1.08563855912214,418,687,0 +"night#2020-06-01#00:00:00#2020-06-01#05:59:59","night","2020-06-01","00:00:00","2020-06-01","05:59:59",1,1,13,13,1,2,2,870,1740,420,1320,636.396103067893,420,0.552951978816239,163,257,1,3,3,320,960,0,960,554.256258422041,0,0,57,257,1 diff --git a/tests/data/processed/features/test03/calls.csv b/tests/data/processed/features/test03/calls.csv new file mode 100644 index 00000000..3bc8c7fe --- /dev/null +++ b/tests/data/processed/features/test03/calls.csv @@ -0,0 +1 @@ +"local_segment","local_segment_label","local_start_date","local_start_time","local_end_date","local_end_time","calls_rapids_missed_count","calls_rapids_missed_distinctcontacts","calls_rapids_missed_timefirstcall","calls_rapids_missed_timelastcall","calls_rapids_missed_countmostfrequentcontact","calls_rapids_incoming_count","calls_rapids_incoming_distinctcontacts","calls_rapids_incoming_meanduration","calls_rapids_incoming_sumduration","calls_rapids_incoming_minduration","calls_rapids_incoming_maxduration","calls_rapids_incoming_stdduration","calls_rapids_incoming_modeduration","calls_rapids_incoming_entropyduration","calls_rapids_incoming_timefirstcall","calls_rapids_incoming_timelastcall","calls_rapids_incoming_countmostfrequentcontact","calls_rapids_outgoing_count","calls_rapids_outgoing_distinctcontacts","calls_rapids_outgoing_meanduration","calls_rapids_outgoing_sumduration","calls_rapids_outgoing_minduration","calls_rapids_outgoing_maxduration","calls_rapids_outgoing_stdduration","calls_rapids_outgoing_modeduration","calls_rapids_outgoing_entropyduration","calls_rapids_outgoing_timefirstcall","calls_rapids_outgoing_timelastcall","calls_rapids_outgoing_countmostfrequentcontact" diff --git a/tests/data/processed/features/test04/calls.csv b/tests/data/processed/features/test04/calls.csv new file mode 100644 index 00000000..3bc8c7fe --- /dev/null +++ b/tests/data/processed/features/test04/calls.csv @@ -0,0 +1 @@ +"local_segment","local_segment_label","local_start_date","local_start_time","local_end_date","local_end_time","calls_rapids_missed_count","calls_rapids_missed_distinctcontacts","calls_rapids_missed_timefirstcall","calls_rapids_missed_timelastcall","calls_rapids_missed_countmostfrequentcontact","calls_rapids_incoming_count","calls_rapids_incoming_distinctcontacts","calls_rapids_incoming_meanduration","calls_rapids_incoming_sumduration","calls_rapids_incoming_minduration","calls_rapids_incoming_maxduration","calls_rapids_incoming_stdduration","calls_rapids_incoming_modeduration","calls_rapids_incoming_entropyduration","calls_rapids_incoming_timefirstcall","calls_rapids_incoming_timelastcall","calls_rapids_incoming_countmostfrequentcontact","calls_rapids_outgoing_count","calls_rapids_outgoing_distinctcontacts","calls_rapids_outgoing_meanduration","calls_rapids_outgoing_sumduration","calls_rapids_outgoing_minduration","calls_rapids_outgoing_maxduration","calls_rapids_outgoing_stdduration","calls_rapids_outgoing_modeduration","calls_rapids_outgoing_entropyduration","calls_rapids_outgoing_timefirstcall","calls_rapids_outgoing_timelastcall","calls_rapids_outgoing_countmostfrequentcontact" diff --git a/tests/settings/testing_config.yaml b/tests/settings/testing_config.yaml index 4033d642..690e712f 100644 --- a/tests/settings/testing_config.yaml +++ b/tests/settings/testing_config.yaml @@ -8,7 +8,7 @@ DAY_SEGMENTS: &day_segments FILE: "data/external/daysegments_interval.csv" PHONE_VALID_SENSED_BINS: - DB_TABLES: [bluetooth] + DB_TABLES: [messages, calls, bluetooth] # Communication SMS features config, TYPES and FEATURES keys need to match MESSAGES: @@ -23,6 +23,19 @@ MESSAGES: SRC_LANGUAGE: "r" SRC_FOLDER: "rapids" # inside src/features/messages +# Communication call features config, TYPES and FEATURES keys need to match +CALLS: + DB_TABLE: calls + PROVIDERS: + RAPIDS: + COMPUTE: True + CALL_TYPES: [missed, incoming, outgoing] + FEATURES: + missed: [count, distinctcontacts, timefirstcall, timelastcall, countmostfrequentcontact] + incoming: [count, distinctcontacts, meanduration, sumduration, minduration, maxduration, stdduration, modeduration, entropyduration, timefirstcall, timelastcall, countmostfrequentcontact] + outgoing: [count, distinctcontacts, meanduration, sumduration, minduration, maxduration, stdduration, modeduration, entropyduration, timefirstcall, timelastcall, countmostfrequentcontact] + SRC_LANGUAGE: "r" + SRC_FOLDER: "rapids" # inside src/features/calls BLUETOOTH: DB_TABLE: bluetooth