WIFI Bug fix and testing

docs/develop/test_cases.rst

@@ -59,7 +59,16 @@ Bluetooth
 """"""""""
 
     - The raw bluetooth data file contains data for 1 day. 
-    - The raw bluetooth data contains at least 2 records for each ``epoch``.
+    - The raw bluetooth data contains at least 2 records for each ``epoch``. Each ``epoch`` has a record with a ``timestamp`` for the beginning boundary for that ``epoch`` and a record with a ``timestamp`` for the ending boundary for that ``epoch``. (e.g. For the ``morning`` epoch there is a record with a ``timestamp`` for ``6:00AM`` and another record with a ``timestamp`` for ``11:59:59AM``. These are to test edge cases) 
     - An option of 5 bluetooth devices are randomly distributed throughout the data records.
     - There is one raw bluetooth data file each, for testing both iOS and Android data formats.
-    - There is also an additional empty data file for both android and iOS for testing empty data files
+    - There is also an additional empty data file for both android and iOS for testing empty data files.
+
+WIFI
+"""""
+
+    - The raw WIFI data file contains data for 1 day. 
+    - The raw WIFI data contains at least 2 records for each ``epoch``. Each ``epoch`` has a record with a ``timestamp`` for the beginning boundary for that ``epoch`` and a record with a ``timestamp`` for the ending boundary for that ``epoch``. (e.g. For the ``morning`` epoch there is a record with a ``timestamp`` for ``6:00AM`` and another record with a ``timestamp`` for ``11:59:59AM``. These are to test edge cases) 
+    - An option of 5 access point devices is randomly distributed throughout the data records.
+    - There is one raw WIFI data file each, for testing both iOS and Android data formats.
+    - There is also an additional empty data file for both android and iOS for testing empty data files.

src/features/wifi/wifi_base.R

@@ -8,21 +8,26 @@ filter_by_day_segment <- function(data, day_segment) {
 }
 
 compute_wifi_feature <- function(data, feature, day_segment){
+  data <- data %>% filter_by_day_segment(day_segment)
   if(feature %in% c("countscans", "uniquedevices")){
-    data <- data %>% filter_by_day_segment(day_segment)
     data <- switch(feature,
               "countscans" = data %>% summarise(!!paste("wifi", day_segment, feature, sep = "_") := n()),
               "uniquedevices" = data %>% summarise(!!paste("wifi", day_segment, feature, sep = "_") := n_distinct(bssid)))
     return(data)
    } else if(feature == "countscansmostuniquedevice"){
      # Get the most scanned device
-    data <- data %>% group_by(bssid) %>% 
+    mostuniquedevice <- data %>% 
+      group_by(bssid) %>% 
       mutate(N=n()) %>% 
       ungroup() %>%
-      filter(N == max(N))
+      filter(N == max(N)) %>% 
+      head(1) %>% # if there are multiple device with the same amount of scans pick the first one only
+      pull(bssid)
     return(data %>% 
-             filter_by_day_segment(day_segment) %>%
+             filter(bssid == mostuniquedevice) %>%
-             summarise(!!paste("wifi", day_segment, feature, sep = "_") := n()))
+             group_by(local_date) %>% 
+             summarise(!!paste("wifi", day_segment, feature, sep = "_") := n()) %>%
+             replace(is.na(.), 0))
   }
 }
 

tests/Snakefile

@@ -42,6 +42,11 @@ if config["BLUETOOTH"]["COMPUTE"]:
     files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime.csv", pid=config["PIDS"], sensor=config["BLUETOOTH"]["DB_TABLE"]))
     files_to_compute.extend(expand("data/processed/{pid}/bluetooth_{segment}.csv", pid=config["PIDS"], segment = config["BLUETOOTH"]["DAY_SEGMENTS"]))
 
+if config["WIFI"]["COMPUTE"]:
+    files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=config["WIFI"]["DB_TABLE"]))
+    files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime.csv", pid=config["PIDS"], sensor=config["WIFI"]["DB_TABLE"]))
+    files_to_compute.extend(expand("data/processed/{pid}/wifi_{day_segment}.csv", pid = config["PIDS"], day_segment = config["WIFI"]["DAY_SEGMENTS"]))
+
 rule all:
     input:
         files_to_compute

tests/data/processed/test01/wifi_afternoon.csv

@@ -0,0 +1,2 @@
+"local_date","wifi_afternoon_countscans","wifi_afternoon_uniquedevices","wifi_afternoon_countscansmostuniquedevice"
+"2020-07-03",2,2,1

tests/data/processed/test01/wifi_daily.csv

@@ -0,0 +1,2 @@
+"local_date","wifi_daily_countscans","wifi_daily_uniquedevices","wifi_daily_countscansmostuniquedevice"
+"2020-07-03",14,5,6

tests/data/processed/test01/wifi_evening.csv

@@ -0,0 +1,2 @@
+"local_date","wifi_evening_countscans","wifi_evening_uniquedevices","wifi_evening_countscansmostuniquedevice"
+"2020-07-03",3,3,1

tests/data/processed/test01/wifi_morning.csv

@@ -0,0 +1,2 @@
+"local_date","wifi_morning_countscans","wifi_morning_uniquedevices","wifi_morning_countscansmostuniquedevice"
+"2020-07-03",4,3,2

tests/data/processed/test01/wifi_night.csv

@@ -0,0 +1,2 @@
+"local_date","wifi_night_countscans","wifi_night_uniquedevices","wifi_night_countscansmostuniquedevice"
+"2020-07-03",5,4,2

tests/data/processed/test02/wifi_afternoon.csv

@@ -0,0 +1,2 @@
+"local_date","wifi_afternoon_countscans","wifi_afternoon_uniquedevices","wifi_afternoon_countscansmostuniquedevice"
+"2020-07-03",2,2,1

tests/data/processed/test02/wifi_daily.csv

@@ -0,0 +1,2 @@
+"local_date","wifi_daily_countscans","wifi_daily_uniquedevices","wifi_daily_countscansmostuniquedevice"
+"2020-07-03",14,5,4

tests/data/processed/test02/wifi_evening.csv

@@ -0,0 +1,2 @@
+"local_date","wifi_evening_countscans","wifi_evening_uniquedevices","wifi_evening_countscansmostuniquedevice"
+"2020-07-03",5,4,2

tests/data/processed/test02/wifi_morning.csv

@@ -0,0 +1,2 @@
+"local_date","wifi_morning_countscans","wifi_morning_uniquedevices","wifi_morning_countscansmostuniquedevice"
+"2020-07-03",3,2,2

tests/data/processed/test02/wifi_night.csv

@@ -0,0 +1,2 @@
+"local_date","wifi_night_countscans","wifi_night_uniquedevices","wifi_night_countscansmostuniquedevice"
+"2020-07-03",4,4,1

tests/data/processed/test03/wifi_afternoon.csv

@@ -0,0 +1 @@
+"local_date","wifi_afternoon_countscans","wifi_afternoon_uniquedevices","wifi_afternoon_countscansmostuniquedevice"

tests/data/processed/test03/wifi_daily.csv

@@ -0,0 +1 @@
+"local_date","wifi_daily_countscans","wifi_daily_uniquedevices","wifi_daily_countscansmostuniquedevice"

tests/data/processed/test03/wifi_evening.csv

@@ -0,0 +1 @@
+"local_date","wifi_evening_countscans","wifi_evening_uniquedevices","wifi_evening_countscansmostuniquedevice"

tests/data/processed/test03/wifi_morning.csv

@@ -0,0 +1 @@
+"local_date","wifi_morning_countscans","wifi_morning_uniquedevices","wifi_morning_countscansmostuniquedevice"

tests/data/processed/test03/wifi_night.csv

@@ -0,0 +1 @@
+"local_date","wifi_night_countscans","wifi_night_uniquedevices","wifi_night_countscansmostuniquedevice"

tests/data/processed/test04/wifi_afternoon.csv

@@ -0,0 +1 @@
+"local_date","wifi_afternoon_countscans","wifi_afternoon_uniquedevices","wifi_afternoon_countscansmostuniquedevice"

tests/data/processed/test04/wifi_daily.csv

@@ -0,0 +1 @@
+"local_date","wifi_daily_countscans","wifi_daily_uniquedevices","wifi_daily_countscansmostuniquedevice"

tests/data/processed/test04/wifi_evening.csv

@@ -0,0 +1 @@
+"local_date","wifi_evening_countscans","wifi_evening_uniquedevices","wifi_evening_countscansmostuniquedevice"

tests/data/processed/test04/wifi_morning.csv

@@ -0,0 +1 @@
+"local_date","wifi_morning_countscans","wifi_morning_uniquedevices","wifi_morning_countscansmostuniquedevice"

tests/data/processed/test04/wifi_night.csv

@@ -0,0 +1 @@
+"local_date","wifi_night_countscans","wifi_night_uniquedevices","wifi_night_countscansmostuniquedevice"

tests/data/raw/test01/wifi_raw.csv

@@ -0,0 +1,15 @@
+timestamp,device_id,bssid,ssid,security,frequency,rssi,label
+1593770400826,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,e6:06:da:af:f9:91,xfinity,[WPA2-EAP-CCMP][ESS],5180,-89,
+1593778094559,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,86:90:7b:8a:3e:43,WIRELESS-PITTNET,[WPA2-EAP-CCMP][ESS],5805,-95,
+1593785574027,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,e6:06:da:af:f9:91,xfinity,[WPA2-EAP-CCMP][ESS],5180,-93,
+1593791999936,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,d0:b8:2f:eb:0e:f8,CMU-SECURE,[WPA2-EAP-CCMP][ESS],5765,-83,
+1593792000442,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,3d:12:ec:de:96:e1,,[WPA2-EAP-CCMP][ESS],5765,-97,
+1593812070708,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,e6:06:da:af:f9:91,xfinity,[WPA2-EAP-CCMP][ESS],5765,-98,
+1593817199253,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,86:90:7b:8a:3e:43,WIRELESS-PITTNET,[WPA2-EAP-CCMP][ESS],5765,-79,
+1593813600578,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,3d:12:ec:de:96:e1,,[WPA2-EAP-CCMP][ESS],5765,-80,
+1593835199363,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,e6:06:da:af:f9:91,xfinity,[WPA2-EAP-CCMP][ESS],5180,-82,
+1593748800505,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,e6:06:da:af:f9:91,xfinity,[WPA2-EAP-CCMP][ESS],5180,-92,
+1593761511861,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,86:90:7b:8a:3e:43,WIRELESS-PITTNET,[WPA2-EAP-CCMP][ESS],5805,-84,
+1593760292500,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,e6:06:da:af:f9:91,xfinity,[WPA2-EAP-CCMP][ESS],5765,-91,
+1593762343842,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,24:2b:a2:55:8a:e0,eduroam,[WPA2-EAP-CCMP][ESS],5805,-100,
+1593770399830,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,d0:b8:2f:eb:0e:f8,CMU-SECURE,[WPA2-EAP-CCMP][ESS],5765,-75,

tests/data/raw/test02/wifi_raw.csv

@@ -0,0 +1,15 @@
+timestamp,device_id,bssid,ssid,security,frequency,rssi,label
+1593770400547,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,E6:06:DA:AF:F9:91,xfinity,,0,0,
+1593791999289,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,E6:06:DA:AF:F9:91,xfinity,,0,0,
+1593774038251,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,D0:B8:2F:EB:0E:F8,CMU-SECURE,,0,0,
+1593792000898,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,3D:12:EC:DE:96:E1,,,0,0,
+1593813599019,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,24:2B:A2:55:8A:E0,eduroam,,0,0,
+1593813600589,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,E6:06:DA:AF:F9:91,xfinity,,0,0,
+1593835199588,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,D0:B8:2F:EB:0E:F8,CMU-SECURE,,0,0,
+1593825019372,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,24:2B:A2:55:8A:E0,eduroam,,0,0,
+1593833925830,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,3D:12:EC:DE:96:E1,,,0,0,
+1593819918273,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,3D:12:EC:DE:96:E1,,,0,0,
+1593748800099,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,86:90:7B:8A:3E:43,WIRELESS-PITTNET,,0,0,
+1593770399772,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,3D:12:EC:DE:96:E1,,,0,0,
+1593761699709,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,24:2B:A2:55:8A:E0,eduroam,,0,0,
+1593751744305,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,D0:B8:2F:EB:0E:F8,CMU-SECURE,,0,0,

tests/data/raw/test03/wifi_raw.csv

@@ -0,0 +1 @@
+timestamp,device_id,bssid,ssid,security,frequency,rssi,label

tests/data/raw/test04/wifi_raw.csv

@@ -0,0 +1 @@
+timestamp,device_id,bssid,ssid,security,frequency,rssi,label

tests/settings/config.yaml

@@ -2,4 +2,4 @@ directory: ./
 configfile: ./tests/settings/testing_config.yaml
 snakefile: ./tests/Snakefile
 cores: 1
-forcerun: [messages_features, call_features, screen_features, battery_features, bluetooth_features]
+forcerun: [messages_features, call_features, screen_features, battery_features, bluetooth_features, wifi_features]

tests/settings/testing_config.yaml

@@ -1,10 +1,9 @@
 # Add as many sensor tables as you have, they all improve the computation of PHONE_SENSED_BINS. 
 # If you are extracting screen or Barnett's location features, screen and locations tables are mandatory.
-TABLES_FOR_SENSED_BINS: [messages, calls, screen, battery, bluetooth]
+TABLES_FOR_SENSED_BINS: [messages, calls, screen, battery, bluetooth, wifi]
 
 # Participants to include in the analysis
 # You must create a file for each participant named pXXX containing their device_id. This can be done manually or automatically
-# PIDS: [test01, test02, test03, test04]
 PIDS: [test01, test02, test03, test04]
 
 
@@ -51,4 +50,10 @@ BLUETOOTH:
   COMPUTE: True
   DB_TABLE: bluetooth
   DAY_SEGMENTS: *day_segments
-  FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"]
+  FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"]
+
+WIFI:
+  COMPUTE: True
+  DB_TABLE: wifi
+  DAY_SEGMENTS: *day_segments
+  FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"]