From dbc51de4e384820027e85f7ef604eae9b1f87123 Mon Sep 17 00:00:00 2001 From: kaguillera Date: Wed, 29 Jul 2020 14:14:47 -0400 Subject: [PATCH] Updated WIFI testing, bugfix and docs --- docs/develop/environments.rst | 2 +- docs/develop/test_cases.rst | 34 ++++++++++--------- docs/features/extracted.rst | 9 ++--- src/features/wifi_features.R | 1 + tests/Snakefile | 17 +++++++--- .../data/processed/test01/wifi_afternoon.csv | 2 +- tests/data/processed/test01/wifi_daily.csv | 2 +- tests/data/processed/test01/wifi_evening.csv | 2 +- tests/data/processed/test01/wifi_morning.csv | 2 +- tests/data/processed/test01/wifi_night.csv | 2 +- tests/data/raw/test01/sensor_wifi_raw.csv | 13 +++++++ tests/data/raw/test02/sensor_wifi_raw.csv | 15 ++++++++ tests/data/raw/test02/wifi_raw.csv | 16 +-------- tests/data/raw/test03/sensor_wifi_raw.csv | 1 + tests/data/raw/test04/sensor_wifi_raw.csv | 1 + tests/scripts/utils.py | 12 +++---- 16 files changed, 80 insertions(+), 51 deletions(-) create mode 100644 tests/data/raw/test01/sensor_wifi_raw.csv create mode 100644 tests/data/raw/test02/sensor_wifi_raw.csv create mode 100644 tests/data/raw/test03/sensor_wifi_raw.csv create mode 100644 tests/data/raw/test04/sensor_wifi_raw.csv diff --git a/docs/develop/environments.rst b/docs/develop/environments.rst index 8d4aafb8..76abcb1a 100644 --- a/docs/develop/environments.rst +++ b/docs/develop/environments.rst @@ -1,5 +1,5 @@ Manage virtual environments -==================== +============================= **Add new packages** diff --git a/docs/develop/test_cases.rst b/docs/develop/test_cases.rst index 5ff267f9..1f0f1129 100644 --- a/docs/develop/test_cases.rst +++ b/docs/develop/test_cases.rst @@ -7,7 +7,7 @@ Along with the continued development and the addition of new sensors and feature List of Sensor with Tests ^^^^^^^^^^^^^^^^^^^^^^^^^^ -The following is a list of the sesors that testing is currently available. +The following is a list of the sensors that testing is currently available. Messages (SMS) @@ -50,43 +50,45 @@ Battery Due to the difference in the format of the raw battery data for iOS and Android as well as versions of iOS (see the **Assumptions/Observations** section of :ref:`Battery`) the following is the expected results the ``battery_deltas.csv``. This would give a better idea of the use cases being tested since the ``battery_deltas.csv`` would make both the iOS and Android data comparable. These files are used to calculate the features for the battery sensor. - The battery delta data file contains data for 1 day. - - The battery delta data contains 1 record each for a ``charging`` and ``discharging`` episode that falls within an ``epoch`` for every ``epoch``. Thus for the ``daily`` epoch there would be multiple ``charging`` and ``discharging`` episodes - - Since either a ``charging`` episode or a ``discharging`` episode and not both can occur across epochs, in order to test epsiodes that occur across epochs alternating episodes of ``charging`` and ``discharging`` episodes that fall across ``night`` to ``morning``, ``morning`` to ``afternoon`` and finally ``afternoon`` to ``night`` are present in the battery delta data. This starts with a ``discharging`` episode that begins in ``night`` and end in ``morning``. + - The battery delta data contains 1 record each for a ``charging`` and ``discharging`` episode that falls within an ``epoch`` for every ``epoch``. Thus, for the ``daily`` epoch there would be multiple ``charging`` and ``discharging`` episodes + - Since either a ``charging`` episode or a ``discharging`` episode and not both can occur across epochs, in order to test episodes that occur across epochs alternating episodes of ``charging`` and ``discharging`` episodes that fall across ``night`` to ``morning``, ``morning`` to ``afternoon`` and finally ``afternoon`` to ``night`` are present in the battery delta data. This starts with a ``discharging`` episode that begins in ``night`` and end in ``morning``. - There is one battery data file each, for testing both iOS and Android data formats. - There is also an additional empty data file for both android and iOS for testing empty data files Bluetooth """""""""" - - The raw bluetooth data file contains data for 1 day. - - The raw bluetooth data contains at least 2 records for each ``epoch``. Each ``epoch`` has a record with a ``timestamp`` for the beginning boundary for that ``epoch`` and a record with a ``timestamp`` for the ending boundary for that ``epoch``. (e.g. For the ``morning`` epoch there is a record with a ``timestamp`` for ``6:00AM`` and another record with a ``timestamp`` for ``11:59:59AM``. These are to test edge cases) - - An option of 5 bluetooth devices are randomly distributed throughout the data records. - - There is one raw bluetooth data file each, for testing both iOS and Android data formats. + - The raw Bluetooth data file contains data for 1 day. + - The raw Bluetooth data contains at least 2 records for each ``epoch``. Each ``epoch`` has a record with a ``timestamp`` for the beginning boundary for that ``epoch`` and a record with a ``timestamp`` for the ending boundary for that ``epoch``. (e.g. For the ``morning`` epoch there is a record with a ``timestamp`` for ``6:00AM`` and another record with a ``timestamp`` for ``11:59:59AM``. These are to test edge cases) + - An option of 5 Bluetooth devices are randomly distributed throughout the data records. + - There is one raw Bluetooth data file each, for testing both iOS and Android data formats. - There is also an additional empty data file for both android and iOS for testing empty data files. WIFI """"" - - The raw WIFI data file contains data for 1 day. - - The raw WIFI data contains at least 2 records for each ``epoch``. Each ``epoch`` has a record with a ``timestamp`` for the beginning boundary for that ``epoch`` and a record with a ``timestamp`` for the ending boundary for that ``epoch``. (e.g. For the ``morning`` epoch there is a record with a ``timestamp`` for ``6:00AM`` and another record with a ``timestamp`` for ``11:59:59AM``. These are to test edge cases) - - An option of 5 access point devices is randomly distributed throughout the data records. - - There is one raw WIFI data file each, for testing both iOS and Android data formats. - - There is also an additional empty data file for both android and iOS for testing empty data files. + - There are 2 data files (``wifi_raw.csv`` and ``sensor_wifi_raw.csv``) for each fake participant for each phone platform. (see the **Assumptions/Observations** section of :ref:`WIFI`) + - The raw WIFI data files contain data for 1 day. + - The ``sensor_wifi_raw.csv`` data contains at least 2 records for each ``epoch``. Each ``epoch`` has a record with a ``timestamp`` for the beginning boundary for that ``epoch`` and a record with a ``timestamp`` for the ending boundary for that ``epoch``. (e.g. For the ``morning`` epoch there is a record with a ``timestamp`` for ``6:00AM`` and another record with a ``timestamp`` for ``11:59:59AM``. These are to test edge cases) + - The ``wifi_raw.csv`` data contains 3 records with random timestamps for each ``epoch`` to represent visible broadcasting WIFI network. This file is empty for the iOS phone testing data. + - An option of 10 access point devices is randomly distributed throughout the data records. 5 each for ``sensor_wifi_raw.csv`` and ``wifi_raw.csv``. + - There data files for testing both iOS and Android data formats. + - There are also additional empty data files for both android and iOS for testing empty data files. Light """"""" - The raw light data file contains data for 1 day. - The raw light data contains 3 or 4 rows of data for each ``epoch`` except ``night``. The single row of data for ``night`` is for testing features for single values inputs. (Example testing the standard deviation of one input value) - - Since light is only available for Android there is only one file that constains data for Android. All other files (i.e. for iPhone) are empty data files. + - Since light is only available for Android there is only one file that contains data for Android. All other files (i.e. for iPhone) are empty data files. Application Foreground """"""""""""""""""""""" - The raw application foreground data file contains data for 1 day. - - The raw application foreground data contains 7 - 9 rows of data for each ``epoch``. The records for each ``epoch`` contains apps that are randomly selected from a list of apps that are from the ``MULTIPLE_CATEGORIES`` and ``SINGLE_CATEGORIES`` (See `test_config.yaml`_). There are also records in each epoch that have apps randomly selected from a list of apps that are from the ``EXCLUDED_CATEGORIES`` and ``EXCLUDED_APPS``. This is to test that these apps are actually being excluded from the calculations of features. There are also records to test ``SINGLE_APPS`` calculations. - - Since application foreground is only available for Android there is only one file that constains data for Android. All other files (i.e. for iPhone) are empty data files. + - The raw application foreground data contains 7 - 9 rows of data for each ``epoch``. The records for each ``epoch`` contains apps that are randomly selected from a list of apps that are from the ``MULTIPLE_CATEGORIES`` and ``SINGLE_CATEGORIES`` (See `testing_config.yaml`_). There are also records in each epoch that have apps randomly selected from a list of apps that are from the ``EXCLUDED_CATEGORIES`` and ``EXCLUDED_APPS``. This is to test that these apps are actually being excluded from the calculations of features. There are also records to test ``SINGLE_APPS`` calculations. + - Since application foreground is only available for Android there is only one file that contains data for Android. All other files (i.e. for iPhone) are empty data files. - .. _`test_config.yaml`: TBD \ No newline at end of file + .. _`testing_config.yaml`: https://github.com/carissalow/rapids/blob/c498b8d2dfd7cc29d1e4d53e978d30cff6cdf3f2/tests/settings/testing_config.yaml#L70 diff --git a/docs/features/extracted.rst b/docs/features/extracted.rst index 43064078..d81d0708 100644 --- a/docs/features/extracted.rst +++ b/docs/features/extracted.rst @@ -287,7 +287,8 @@ uniquedevices devices Number of unique access point during countscansmostuniquedevice scans Number of scans of the most scanned access point during a ``day_segment`` across the whole monitoring period =========================== ========= ============= -**Assumptions/Observations:** N/A +**Assumptions/Observations:** +Both phone platforms record the wifi networks a phone is connected to in ``sensor_wifi`` and those networks that are being broadcasted around a phone in ``wifi``. However, iOS cannot record any broadcasting network due to API restrictions, therefore iOS wifi data only exists in ``sensor_wifi``. .. _accelerometer-sensor-doc: @@ -655,7 +656,7 @@ See `Location (Doryab's) Config Code`_ - Rule ``rules/preprocessing.snakefile/resample_fused_location`` (only relevant if setting ``location_to_use`` to ````RESAMPLE_FUSED``. - Rule ``rules/features.snakefile/location_doryab_features`` -.. _location-parameters: +.. _location-doryab-parameters: **Location Rule Parameters (location_doryab_features):** @@ -672,7 +673,7 @@ maximum_gap_allowed The maximum gap (in seconds) allowed between any two cons minutes_data_used This is NOT a feature. This is just a quality control check, and if set to TRUE, a new column is added to the output file with the number of minutes containing location data that were used to compute all features. The more data minutes exist for a period, the more reliable its features should be. For fused location, a single minute can contain more than one coordinate pair if the participant is moving fast enough. =================== =================== -.. _location-available-features: +.. _location-doryab-available-features: **Available Location Features** @@ -682,7 +683,7 @@ Name Units Description locationvariance :math:`meters^2` The sum of the variances of the latitude and longitude columns. loglocationvariance Log of the sum of the variances of the latitude and longitude columns. totaldistance meters Total distance travelled in a ``day_segment`` using the haversine formula. -averagespeed km/hr Average speed in a ``day_segment` considering only the instances labeled as Moving. +averagespeed km/hr Average speed in a ``day_segment`` considering only the instances labeled as Moving. varspeed km/hr Speed variance in a ``day_segment`` considering only the instances labeled as Moving. circadianmovement "It encodes the extent to which a person’s location patterns follow a 24-hour circadian cycle." (Doryab et. al. 2019) numberofsignificantplaces places Number of significant locations visited. It is calculated using the DBSCAN clustering algorithm which takes in EPS and MIN_SAMPLES as paramters to identify clusters. Each cluster is a significant place. diff --git a/src/features/wifi_features.R b/src/features/wifi_features.R index 14d77327..54fcc99e 100644 --- a/src/features/wifi_features.R +++ b/src/features/wifi_features.R @@ -10,6 +10,7 @@ if(!is.null(snakemake@input[["visible_access_points"]]) && is.null(snakemake@inp wifi_data <- wifi_data %>% mutate(connected = 1) } else if(!is.null(snakemake@input[["visible_access_points"]]) && !is.null(snakemake@input[["connected_access_points"]])){ visible_access_points <- read.csv(snakemake@input[["visible_access_points"]], stringsAsFactors = FALSE) + visible_access_points <- visible_access_points %>% mutate(connected = 0) connected_access_points <- read.csv(snakemake@input[["connected_access_points"]], stringsAsFactors = FALSE) connected_access_points <- connected_access_points %>% mutate(connected = 1) wifi_data <- bind_rows(visible_access_points, connected_access_points) %>% arrange(timestamp) diff --git a/tests/Snakefile b/tests/Snakefile index 50540ac7..ca1b597d 100644 --- a/tests/Snakefile +++ b/tests/Snakefile @@ -23,10 +23,13 @@ if config["CALLS"]["COMPUTE"]: files_to_compute.extend(expand("data/processed/{pid}/calls_{call_type}_{segment}.csv", pid=config["PIDS"], call_type=config["CALLS"]["TYPES"], segment = config["CALLS"]["DAY_SEGMENTS"])) if config["SCREEN"]["COMPUTE"]: - if config["SCREEN"]["DB_TABLE"] not in config["PHONE_VALID_SENSED_BINS"]["TABLES"]: + if config["SCREEN"]["DB_TABLE"] in config["PHONE_VALID_SENSED_BINS"]["TABLES"]: + files_to_compute.extend(expand("data/interim/{pid}/phone_sensed_bins.csv", pid=config["PIDS"])) + else: raise ValueError("Error: Add your screen table (and as many sensor tables as you have) to [PHONE_VALID_SENSED_BINS][TABLES] in config.yaml. This is necessary to compute phone_sensed_bins (bins of time when the smartphone was sensing data)") files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=config["SCREEN"]["DB_TABLE"])) files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime.csv", pid=config["PIDS"], sensor=config["SCREEN"]["DB_TABLE"])) + files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime_unified.csv", pid=config["PIDS"], sensor=config["SCREEN"]["DB_TABLE"])) files_to_compute.extend(expand("data/processed/{pid}/screen_deltas.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/{pid}/screen_{day_segment}.csv", pid = config["PIDS"], day_segment = config["SCREEN"]["DAY_SEGMENTS"])) @@ -43,9 +46,15 @@ if config["BLUETOOTH"]["COMPUTE"]: files_to_compute.extend(expand("data/processed/{pid}/bluetooth_{segment}.csv", pid=config["PIDS"], segment = config["BLUETOOTH"]["DAY_SEGMENTS"])) if config["WIFI"]["COMPUTE"]: - files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=config["WIFI"]["DB_TABLE"])) - files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime.csv", pid=config["PIDS"], sensor=config["WIFI"]["DB_TABLE"])) - files_to_compute.extend(expand("data/processed/{pid}/wifi_{day_segment}.csv", pid = config["PIDS"], day_segment = config["WIFI"]["DAY_SEGMENTS"])) + if len(config["WIFI"]["DB_TABLE"]["VISIBLE_ACCESS_POINTS"]) > 0: + files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=config["WIFI"]["DB_TABLE"]["VISIBLE_ACCESS_POINTS"])) + files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime.csv", pid=config["PIDS"], sensor=config["WIFI"]["DB_TABLE"]["VISIBLE_ACCESS_POINTS"])) + files_to_compute.extend(expand("data/processed/{pid}/wifi_{day_segment}.csv", pid = config["PIDS"], day_segment = config["WIFI"]["DAY_SEGMENTS"])) + + if len(config["WIFI"]["DB_TABLE"]["CONNECTED_ACCESS_POINTS"]) > 0: + files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=config["WIFI"]["DB_TABLE"]["CONNECTED_ACCESS_POINTS"])) + files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime.csv", pid=config["PIDS"], sensor=config["WIFI"]["DB_TABLE"]["CONNECTED_ACCESS_POINTS"])) + files_to_compute.extend(expand("data/processed/{pid}/wifi_{day_segment}.csv", pid = config["PIDS"], day_segment = config["WIFI"]["DAY_SEGMENTS"])) if config["LIGHT"]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=config["LIGHT"]["DB_TABLE"])) diff --git a/tests/data/processed/test01/wifi_afternoon.csv b/tests/data/processed/test01/wifi_afternoon.csv index f89b0613..2c670e59 100644 --- a/tests/data/processed/test01/wifi_afternoon.csv +++ b/tests/data/processed/test01/wifi_afternoon.csv @@ -1,2 +1,2 @@ "local_date","wifi_afternoon_countscans","wifi_afternoon_uniquedevices","wifi_afternoon_countscansmostuniquedevice" -"2020-07-03",2,2,1 +"2020-07-03",4,4,1 diff --git a/tests/data/processed/test01/wifi_daily.csv b/tests/data/processed/test01/wifi_daily.csv index b699c63f..66cb5e2a 100644 --- a/tests/data/processed/test01/wifi_daily.csv +++ b/tests/data/processed/test01/wifi_daily.csv @@ -1,2 +1,2 @@ "local_date","wifi_daily_countscans","wifi_daily_uniquedevices","wifi_daily_countscansmostuniquedevice" -"2020-07-03",14,5,6 +"2020-07-03",26,10,6 diff --git a/tests/data/processed/test01/wifi_evening.csv b/tests/data/processed/test01/wifi_evening.csv index 2b7c52b4..e63ca233 100644 --- a/tests/data/processed/test01/wifi_evening.csv +++ b/tests/data/processed/test01/wifi_evening.csv @@ -1,2 +1,2 @@ "local_date","wifi_evening_countscans","wifi_evening_uniquedevices","wifi_evening_countscansmostuniquedevice" -"2020-07-03",3,3,1 +"2020-07-03",7,6,2 diff --git a/tests/data/processed/test01/wifi_morning.csv b/tests/data/processed/test01/wifi_morning.csv index 3c43cca2..2359f24e 100644 --- a/tests/data/processed/test01/wifi_morning.csv +++ b/tests/data/processed/test01/wifi_morning.csv @@ -1,2 +1,2 @@ "local_date","wifi_morning_countscans","wifi_morning_uniquedevices","wifi_morning_countscansmostuniquedevice" -"2020-07-03",4,3,2 +"2020-07-03",7,5,2 diff --git a/tests/data/processed/test01/wifi_night.csv b/tests/data/processed/test01/wifi_night.csv index 3e28eee8..89fef7fb 100644 --- a/tests/data/processed/test01/wifi_night.csv +++ b/tests/data/processed/test01/wifi_night.csv @@ -1,2 +1,2 @@ "local_date","wifi_night_countscans","wifi_night_uniquedevices","wifi_night_countscansmostuniquedevice" -"2020-07-03",5,4,2 +"2020-07-03",8,6,2 diff --git a/tests/data/raw/test01/sensor_wifi_raw.csv b/tests/data/raw/test01/sensor_wifi_raw.csv new file mode 100644 index 00000000..547fe57a --- /dev/null +++ b/tests/data/raw/test01/sensor_wifi_raw.csv @@ -0,0 +1,13 @@ +timestamp,device_id,mac_address,ssid,bssid +1593771645547,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,af:13:50:8b:5f:ab,HOME-1234,E0:A9:8A:1E:48:1B +1593791393289,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,af:13:50:8b:5f:ab,HOME-1234,E0:A9:8A:1E:48:1B +1593779101251,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,af:13:50:8b:5f:ab,Stewart Family,0D:FB:75:AA:43:0A +1593813989898,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,af:13:50:8b:5f:ab,NETGEAR07,EF:E7:56:4A:F4:1D +1593799538019,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,af:13:50:8b:5f:ab,Fios-H4S9a,48:6B:8C:51:08:F2 +1593797221589,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,af:13:50:8b:5f:ab,HOME-1234,E0:A9:8A:1E:48:1B +1593820668588,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,af:13:50:8b:5f:ab,Stewart Family,0D:FB:75:AA:43:0A +1593831618372,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,af:13:50:8b:5f:ab,Fios-H4S9a,48:6B:8C:51:08:F2 +1593821661830,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,af:13:50:8b:5f:ab,NETGEAR07,EF:E7:56:4A:F4:1D +1593768441273,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,af:13:50:8b:5f:ab,NETGEAR07,EF:E7:56:4A:F4:1D +1593764562099,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,af:13:50:8b:5f:ab,Galactica,E6:4C:9A:3F:D0:1B +1593765323772,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,af:13:50:8b:5f:ab,NETGEAR07,EF:E7:56:4A:F4:1D \ No newline at end of file diff --git a/tests/data/raw/test02/sensor_wifi_raw.csv b/tests/data/raw/test02/sensor_wifi_raw.csv new file mode 100644 index 00000000..bda8b521 --- /dev/null +++ b/tests/data/raw/test02/sensor_wifi_raw.csv @@ -0,0 +1,15 @@ +timestamp,device_id,mac_address,ssid,bssid +1593770400547,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,af:13:50:8b:5f:ab,xfinity,E6:06:DA:AF:F9:91 +1593791999289,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,af:13:50:8b:5f:ab,xfinity,E6:06:DA:AF:F9:91 +1593774038251,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,af:13:50:8b:5f:ab,CMU-SECURE,D0:B8:2F:EB:0E:F8 +1593792000898,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,af:13:50:8b:5f:ab,,3D:12:EC:DE:96:E1 +1593813599019,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,af:13:50:8b:5f:ab,eduroam,24:2B:A2:55:8A:E0 +1593813600589,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,af:13:50:8b:5f:ab,xfinity,E6:06:DA:AF:F9:91 +1593835199588,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,af:13:50:8b:5f:ab,CMU-SECURE,D0:B8:2F:EB:0E:F8 +1593825019372,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,af:13:50:8b:5f:ab,eduroam,24:2B:A2:55:8A:E0 +1593833925830,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,af:13:50:8b:5f:ab,,3D:12:EC:DE:96:E1 +1593819918273,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,af:13:50:8b:5f:ab,,3D:12:EC:DE:96:E1 +1593748800099,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,af:13:50:8b:5f:ab,WIRELESS-PITTNET,86:90:7B:8A:3E:43 +1593770399772,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,af:13:50:8b:5f:ab,,3D:12:EC:DE:96:E1 +1593761699709,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,af:13:50:8b:5f:ab,eduroam,24:2B:A2:55:8A:E0 +1593751744305,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,af:13:50:8b:5f:ab,CMU-SECURE,D0:B8:2F:EB:0E:F8 \ No newline at end of file diff --git a/tests/data/raw/test02/wifi_raw.csv b/tests/data/raw/test02/wifi_raw.csv index 8240b9b5..a6acabd0 100644 --- a/tests/data/raw/test02/wifi_raw.csv +++ b/tests/data/raw/test02/wifi_raw.csv @@ -1,15 +1 @@ -timestamp,device_id,bssid,ssid,security,frequency,rssi,label -1593770400547,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,E6:06:DA:AF:F9:91,xfinity,,0,0, -1593791999289,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,E6:06:DA:AF:F9:91,xfinity,,0,0, -1593774038251,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,D0:B8:2F:EB:0E:F8,CMU-SECURE,,0,0, -1593792000898,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,3D:12:EC:DE:96:E1,,,0,0, -1593813599019,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,24:2B:A2:55:8A:E0,eduroam,,0,0, -1593813600589,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,E6:06:DA:AF:F9:91,xfinity,,0,0, -1593835199588,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,D0:B8:2F:EB:0E:F8,CMU-SECURE,,0,0, -1593825019372,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,24:2B:A2:55:8A:E0,eduroam,,0,0, -1593833925830,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,3D:12:EC:DE:96:E1,,,0,0, -1593819918273,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,3D:12:EC:DE:96:E1,,,0,0, -1593748800099,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,86:90:7B:8A:3E:43,WIRELESS-PITTNET,,0,0, -1593770399772,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,3D:12:EC:DE:96:E1,,,0,0, -1593761699709,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,24:2B:A2:55:8A:E0,eduroam,,0,0, -1593751744305,7yKzcQm4-xKTC-0bhC-PZXC-3jAbRIXOsf5w,D0:B8:2F:EB:0E:F8,CMU-SECURE,,0,0, \ No newline at end of file +timestamp,device_id,bssid,ssid,security,frequency,rssi,label \ No newline at end of file diff --git a/tests/data/raw/test03/sensor_wifi_raw.csv b/tests/data/raw/test03/sensor_wifi_raw.csv new file mode 100644 index 00000000..baa63da7 --- /dev/null +++ b/tests/data/raw/test03/sensor_wifi_raw.csv @@ -0,0 +1 @@ +timestamp,device_id,mac_address,ssid,bssid \ No newline at end of file diff --git a/tests/data/raw/test04/sensor_wifi_raw.csv b/tests/data/raw/test04/sensor_wifi_raw.csv new file mode 100644 index 00000000..baa63da7 --- /dev/null +++ b/tests/data/raw/test04/sensor_wifi_raw.csv @@ -0,0 +1 @@ +timestamp,device_id,mac_address,ssid,bssid \ No newline at end of file diff --git a/tests/scripts/utils.py b/tests/scripts/utils.py index 5412f6d5..38d804f8 100644 --- a/tests/scripts/utils.py +++ b/tests/scripts/utils.py @@ -100,26 +100,26 @@ def generate_sensor_file_lists(config): if 'TYPES' in config[sensor]: for each in config[sensor]['TYPES']: sensor_type.append(each+'_') - + lower_sensor = sensor.lower() if sensor_type: act_file_list = expand(act_str, pid=config["PIDS"], - sensor = config[sensor]["DB_TABLE"], + sensor = lower_sensor, sensor_type = sensor_type, day_segment = config[sensor]["DAY_SEGMENTS"]) exp_file_list = expand(exp_str, pid=config["PIDS"], - sensor = config[sensor]["DB_TABLE"], + sensor = lower_sensor, sensor_type = sensor_type, day_segment = config[sensor]["DAY_SEGMENTS"]) else: act_file_list = expand(act_str, pid=config["PIDS"], - sensor = config[sensor]["DB_TABLE"], + sensor = lower_sensor, sensor_type = '', day_segment = config[sensor]["DAY_SEGMENTS"]) exp_file_list = expand(exp_str, pid=config["PIDS"], - sensor = config[sensor]["DB_TABLE"], + sensor = lower_sensor, sensor_type = '', day_segment = config[sensor]["DAY_SEGMENTS"]) sensor_file_lists[sensor] = list(zip(act_file_list,exp_file_list)) - return sensor_file_lists + return sensor_file_lists \ No newline at end of file