Apps_foreground, bugfixes, testing & testcase docs

docs/develop/test_cases.rst

@@ -79,3 +79,14 @@ Light
     - The raw light data file contains data for 1 day. 
     - The raw light data contains 3 or 4 rows of data for each ``epoch`` except ``night``. The single row of data for ``night`` is for testing features for single values inputs. (Example testing the standard deviation of one input value)
     - Since light is only available for Android there is only one file that constains data for Android. All other files (i.e. for iPhone) are empty data files.
+
+Application Foreground 
+"""""""""""""""""""""""
+
+    - The raw application foreground data file contains data for 1 day. 
+    - The raw application foreground data contains 7 - 9 rows of data for each ``epoch``. The records for each ``epoch`` contains apps that are randomly selected from a list of apps that are from the ``MULTIPLE_CATEGORIES`` and ``SINGLE_CATEGORIES`` (See `test_config.yaml`_). There are also records in each epoch that have apps randomly selected from a list of apps that are from the ``EXCLUDED_CATEGORIES`` and ``EXCLUDED_APPS``. This is to test that these apps are actually being excluded from the calculations of features. There are also records to test ``SINGLE_APPS`` calculations. 
+    - Since application foreground is only available for Android there is only one file that constains data for Android. All other files (i.e. for iPhone) are empty data files.
+
+
+
+ .. _`test_config.yaml`: TBD

src/features/applications_foreground/applications_foreground_base.py

@@ -4,14 +4,24 @@ from scipy.stats import entropy
 
 
 def compute_features(filtered_data, apps_type, requested_features, apps_features, day_segment):        
+    # There is the rare occasion that filtered_data is empty (found in testing)
     if "timeoffirstuse" in requested_features:
         time_first_event = filtered_data.sort_values(by="timestamp", ascending=True).drop_duplicates(subset="local_date", keep="first").set_index("local_date")
+        if time_first_event.empty:
+            apps_features["apps_" + day_segment + "_timeoffirstuse" + apps_type] = 'NA'
+        else:
             apps_features["apps_" + day_segment + "_timeoffirstuse" + apps_type] = time_first_event["local_hour"] * 60 + time_first_event["local_minute"]
     if "timeoflastuse" in requested_features:
         time_last_event = filtered_data.sort_values(by="timestamp", ascending=False).drop_duplicates(subset="local_date", keep="first").set_index("local_date")
+        if time_last_event.empty:
+            apps_features["apps_" + day_segment + "_timeoflastuse" + apps_type] = 'NA'
+        else:
             apps_features["apps_" + day_segment + "_timeoflastuse" + apps_type] = time_last_event["local_hour"] * 60 + time_last_event["local_minute"]
     if "frequencyentropy" in requested_features:
         apps_with_count = filtered_data.groupby(["local_date","application_name"]).count().sort_values(by="timestamp", ascending=False).reset_index()
+        if (len(apps_with_count.index) < 2 ):
+            apps_features["apps_" + day_segment + "_frequencyentropy" + apps_type] = 'NA'
+        else:    
             apps_features["apps_" + day_segment + "_frequencyentropy" + apps_type] = apps_with_count.groupby("local_date")["timestamp"].agg(entropy)
     if "count" in requested_features:
         apps_features["apps_" + day_segment + "_count" + apps_type] = filtered_data.groupby(["local_date"]).count()["timestamp"]
@@ -36,6 +46,7 @@ def base_applications_foreground_features(apps_data, day_segment, requested_feat
         if not apps_data.empty:
             apps_features = pd.DataFrame()
             # single category
+            single_categories.sort()
             for sc in single_categories:
                 if sc == "all":
                     apps_features = compute_features(apps_data, "all", requested_features, apps_features, day_segment)

tests/Snakefile

@@ -52,6 +52,12 @@ if config["LIGHT"]["COMPUTE"]:
     files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime.csv", pid=config["PIDS"], sensor=config["LIGHT"]["DB_TABLE"]))
     files_to_compute.extend(expand("data/processed/{pid}/light_{day_segment}.csv", pid = config["PIDS"], day_segment = config["LIGHT"]["DAY_SEGMENTS"]))
 
+if config["APPLICATIONS_FOREGROUND"]["COMPUTE"]:
+    files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=config["APPLICATIONS_FOREGROUND"]["DB_TABLE"]))
+    files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime.csv", pid=config["PIDS"], sensor=config["APPLICATIONS_FOREGROUND"]["DB_TABLE"]))
+    files_to_compute.extend(expand("data/interim/{pid}/{sensor}_with_datetime_with_genre.csv", pid=config["PIDS"], sensor=config["APPLICATIONS_FOREGROUND"]["DB_TABLE"]))
+    files_to_compute.extend(expand("data/processed/{pid}/applications_foreground_{day_segment}.csv", pid = config["PIDS"], day_segment = config["APPLICATIONS_FOREGROUND"]["DAY_SEGMENTS"]))
+
 rule all:
     input:
         files_to_compute

tests/data/processed/test01/applications_foreground_afternoon.csv

@@ -0,0 +1,2 @@
+local_date,apps_afternoon_timeoffirstuseall,apps_afternoon_timeoflastuseall,apps_afternoon_frequencyentropyall,apps_afternoon_countall,apps_afternoon_timeoffirstuseemail,apps_afternoon_timeoflastuseemail,apps_afternoon_frequencyentropyemail,apps_afternoon_countemail,apps_afternoon_timeoffirstusesocial,apps_afternoon_timeoflastusesocial,apps_afternoon_frequencyentropysocial,apps_afternoon_countsocial,apps_afternoon_timeoffirstuseentertainment,apps_afternoon_timeoflastuseentertainment,apps_afternoon_frequencyentropyentertainment,apps_afternoon_countentertainment,apps_afternoon_timeoffirstusetop1global,apps_afternoon_timeoflastusetop1global,apps_afternoon_frequencyentropytop1global,apps_afternoon_counttop1global,apps_afternoon_timeoffirstusecom.facebook.moments,apps_afternoon_timeoflastusecom.facebook.moments,apps_afternoon_frequencyentropycom.facebook.moments,apps_afternoon_countcom.facebook.moments,apps_afternoon_timeoffirstusecom.google.android.youtube,apps_afternoon_timeoflastusecom.google.android.youtube,apps_afternoon_frequencyentropycom.google.android.youtube,apps_afternoon_countcom.google.android.youtube
+2020-07-05,721,889,1.0397207708399179,4,889,889,NA,1,NA,NA,NA,0.0,721,721,NA,1,NA,NA,NA,0.0,798,877,NA,2,NA,NA,NA,0.0

tests/data/processed/test01/applications_foreground_daily.csv

@@ -0,0 +1,2 @@
+local_date,apps_daily_timeoffirstuseall,apps_daily_timeoflastuseall,apps_daily_frequencyentropyall,apps_daily_countall,apps_daily_timeoffirstuseemail,apps_daily_timeoflastuseemail,apps_daily_frequencyentropyemail,apps_daily_countemail,apps_daily_timeoffirstuseentertainment,apps_daily_timeoflastuseentertainment,apps_daily_frequencyentropyentertainment,apps_daily_countentertainment,apps_daily_timeoffirstusesocial,apps_daily_timeoflastusesocial,apps_daily_frequencyentropysocial,apps_daily_countsocial,apps_daily_timeoffirstusetop1global,apps_daily_timeoflastusetop1global,apps_daily_frequencyentropytop1global,apps_daily_counttop1global,apps_daily_timeoffirstusecom.google.android.youtube,apps_daily_timeoflastusecom.google.android.youtube,apps_daily_frequencyentropycom.google.android.youtube,apps_daily_countcom.google.android.youtube,apps_daily_timeoffirstusecom.facebook.moments,apps_daily_timeoflastusecom.facebook.moments,apps_daily_frequencyentropycom.facebook.moments,apps_daily_countcom.facebook.moments
+2020-07-05,17,1359,1.5443819809168389,17,889,1308,NA,2,195,721,0.5982695885852573,7,302,1359,NA,4,195,719,NA,5,NA,NA,NA,0.0,17,877,NA,4

tests/data/processed/test01/applications_foreground_evening.csv

@@ -0,0 +1,2 @@
+local_date,apps_evening_timeoffirstuseall,apps_evening_timeoflastuseall,apps_evening_frequencyentropyall,apps_evening_countall,apps_evening_timeoffirstuseemail,apps_evening_timeoflastuseemail,apps_evening_frequencyentropyemail,apps_evening_countemail,apps_evening_timeoffirstusesocial,apps_evening_timeoflastusesocial,apps_evening_frequencyentropysocial,apps_evening_countsocial,apps_evening_timeoffirstuseentertainment,apps_evening_timeoflastuseentertainment,apps_evening_frequencyentropyentertainment,apps_evening_countentertainment,apps_evening_timeoffirstusecom.google.android.youtube,apps_evening_timeoflastusecom.google.android.youtube,apps_evening_frequencyentropycom.google.android.youtube,apps_evening_countcom.google.android.youtube,apps_evening_timeoffirstusetop1global,apps_evening_timeoflastusetop1global,apps_evening_frequencyentropytop1global,apps_evening_counttop1global,apps_evening_timeoffirstusecom.facebook.moments,apps_evening_timeoflastusecom.facebook.moments,apps_evening_frequencyentropycom.facebook.moments,apps_evening_countcom.facebook.moments
+2020-07-05,1168,1359,0.6365141682948128,3,1308,1308,NA,1,1168,1359,NA,2,NA,NA,NA,0.0,NA,NA,NA,0.0,NA,NA,NA,0.0,NA,NA,NA,0.0

tests/data/processed/test01/applications_foreground_morning.csv

@@ -0,0 +1,2 @@
+local_date,apps_morning_timeoffirstuseall,apps_morning_timeoflastuseall,apps_morning_frequencyentropyall,apps_morning_countall,apps_morning_timeoffirstuseemail,apps_morning_timeoflastuseemail,apps_morning_frequencyentropyemail,apps_morning_countemail,apps_morning_timeoffirstusesocial,apps_morning_timeoflastusesocial,apps_morning_frequencyentropysocial,apps_morning_countsocial,apps_morning_timeoffirstuseentertainment,apps_morning_timeoflastuseentertainment,apps_morning_frequencyentropyentertainment,apps_morning_countentertainment,apps_morning_timeoffirstusecom.facebook.moments,apps_morning_timeoflastusecom.facebook.moments,apps_morning_frequencyentropycom.facebook.moments,apps_morning_countcom.facebook.moments,apps_morning_timeoffirstusetop1global,apps_morning_timeoflastusetop1global,apps_morning_frequencyentropytop1global,apps_morning_counttop1global,apps_morning_timeoffirstusecom.google.android.youtube,apps_morning_timeoflastusecom.google.android.youtube,apps_morning_frequencyentropycom.google.android.youtube,apps_morning_countcom.google.android.youtube
+2020-07-05,412,719,0.9502705392332347,5,NA,NA,NA,0.0,427,427,NA,1,412,719,0.5623351446188083,4,NA,NA,NA,0.0,412,719,NA,3,NA,NA,NA,0.0

tests/data/processed/test01/applications_foreground_night.csv

@@ -0,0 +1,2 @@
+local_date,apps_night_timeoffirstuseall,apps_night_timeoflastuseall,apps_night_frequencyentropyall,apps_night_countall,apps_night_timeoffirstuseemail,apps_night_timeoflastuseemail,apps_night_frequencyentropyemail,apps_night_countemail,apps_night_timeoffirstusesocial,apps_night_timeoflastusesocial,apps_night_frequencyentropysocial,apps_night_countsocial,apps_night_timeoffirstuseentertainment,apps_night_timeoflastuseentertainment,apps_night_frequencyentropyentertainment,apps_night_countentertainment,apps_night_timeoffirstusecom.facebook.moments,apps_night_timeoflastusecom.facebook.moments,apps_night_frequencyentropycom.facebook.moments,apps_night_countcom.facebook.moments,apps_night_timeoffirstusecom.google.android.youtube,apps_night_timeoflastusecom.google.android.youtube,apps_night_frequencyentropycom.google.android.youtube,apps_night_countcom.google.android.youtube,apps_night_timeoffirstusetop1global,apps_night_timeoflastusetop1global,apps_night_frequencyentropytop1global,apps_night_counttop1global
+2020-07-05,17,359,1.0549201679861442,5,NA,NA,NA,0.0,302,302,NA,1,195,359,NA,2,17,59,NA,2,NA,NA,NA,0.0,195,359,NA,2

tests/data/processed/test02/applications_foreground_afternoon.csv

@@ -0,0 +1 @@
+local_date,apps_afternoon_countemail,apps_afternoon_countall,apps_afternoon_countentertainment,apps_afternoon_countsocial,apps_afternoon_countcom.facebook.moments,apps_afternoon_countcom.google.android.youtube,apps_afternoon_counttop1global,apps_afternoon_timeoffirstuseemail,apps_afternoon_timeoffirstuseall,apps_afternoon_timeoffirstuseentertainment,apps_afternoon_timeoffirstusesocial,apps_afternoon_timeoffirstusecom.facebook.moments,apps_afternoon_timeoffirstusecom.google.android.youtube,apps_afternoon_timeoffirstusetop1global,apps_afternoon_timeoflastuseemail,apps_afternoon_timeoflastuseall,apps_afternoon_timeoflastuseentertainment,apps_afternoon_timeoflastusesocial,apps_afternoon_timeoflastusecom.facebook.moments,apps_afternoon_timeoflastusecom.google.android.youtube,apps_afternoon_timeoflastusetop1global,apps_afternoon_frequencyentropyemail,apps_afternoon_frequencyentropyall,apps_afternoon_frequencyentropyentertainment,apps_afternoon_frequencyentropysocial,apps_afternoon_frequencyentropycom.facebook.moments,apps_afternoon_frequencyentropycom.google.android.youtube,apps_afternoon_frequencyentropytop1global

tests/data/processed/test02/applications_foreground_daily.csv

@@ -0,0 +1 @@
+local_date,apps_daily_countall,apps_daily_countemail,apps_daily_countentertainment,apps_daily_countsocial,apps_daily_countcom.facebook.moments,apps_daily_countcom.google.android.youtube,apps_daily_counttop1global,apps_daily_timeoffirstuseall,apps_daily_timeoffirstuseemail,apps_daily_timeoffirstuseentertainment,apps_daily_timeoffirstusesocial,apps_daily_timeoffirstusecom.facebook.moments,apps_daily_timeoffirstusecom.google.android.youtube,apps_daily_timeoffirstusetop1global,apps_daily_timeoflastuseall,apps_daily_timeoflastuseemail,apps_daily_timeoflastuseentertainment,apps_daily_timeoflastusesocial,apps_daily_timeoflastusecom.facebook.moments,apps_daily_timeoflastusecom.google.android.youtube,apps_daily_timeoflastusetop1global,apps_daily_frequencyentropyall,apps_daily_frequencyentropyemail,apps_daily_frequencyentropyentertainment,apps_daily_frequencyentropysocial,apps_daily_frequencyentropycom.facebook.moments,apps_daily_frequencyentropycom.google.android.youtube,apps_daily_frequencyentropytop1global

tests/data/processed/test02/applications_foreground_evening.csv

@@ -0,0 +1 @@
+local_date,apps_evening_countemail,apps_evening_countall,apps_evening_countsocial,apps_evening_countentertainment,apps_evening_counttop1global,apps_evening_countcom.facebook.moments,apps_evening_countcom.google.android.youtube,apps_evening_timeoffirstuseemail,apps_evening_timeoffirstuseall,apps_evening_timeoffirstusesocial,apps_evening_timeoffirstuseentertainment,apps_evening_timeoffirstusetop1global,apps_evening_timeoffirstusecom.facebook.moments,apps_evening_timeoffirstusecom.google.android.youtube,apps_evening_timeoflastuseemail,apps_evening_timeoflastuseall,apps_evening_timeoflastusesocial,apps_evening_timeoflastuseentertainment,apps_evening_timeoflastusetop1global,apps_evening_timeoflastusecom.facebook.moments,apps_evening_timeoflastusecom.google.android.youtube,apps_evening_frequencyentropyemail,apps_evening_frequencyentropyall,apps_evening_frequencyentropysocial,apps_evening_frequencyentropyentertainment,apps_evening_frequencyentropytop1global,apps_evening_frequencyentropycom.facebook.moments,apps_evening_frequencyentropycom.google.android.youtube

tests/data/processed/test02/applications_foreground_morning.csv

@@ -0,0 +1 @@
+local_date,apps_morning_countemail,apps_morning_countall,apps_morning_countentertainment,apps_morning_countsocial,apps_morning_counttop1global,apps_morning_countcom.google.android.youtube,apps_morning_countcom.facebook.moments,apps_morning_timeoffirstuseemail,apps_morning_timeoffirstuseall,apps_morning_timeoffirstuseentertainment,apps_morning_timeoffirstusesocial,apps_morning_timeoffirstusetop1global,apps_morning_timeoffirstusecom.google.android.youtube,apps_morning_timeoffirstusecom.facebook.moments,apps_morning_timeoflastuseemail,apps_morning_timeoflastuseall,apps_morning_timeoflastuseentertainment,apps_morning_timeoflastusesocial,apps_morning_timeoflastusetop1global,apps_morning_timeoflastusecom.google.android.youtube,apps_morning_timeoflastusecom.facebook.moments,apps_morning_frequencyentropyemail,apps_morning_frequencyentropyall,apps_morning_frequencyentropyentertainment,apps_morning_frequencyentropysocial,apps_morning_frequencyentropytop1global,apps_morning_frequencyentropycom.google.android.youtube,apps_morning_frequencyentropycom.facebook.moments

tests/data/processed/test02/applications_foreground_night.csv

@@ -0,0 +1 @@
+local_date,apps_night_countall,apps_night_countemail,apps_night_countentertainment,apps_night_countsocial,apps_night_countcom.google.android.youtube,apps_night_counttop1global,apps_night_countcom.facebook.moments,apps_night_timeoffirstuseall,apps_night_timeoffirstuseemail,apps_night_timeoffirstuseentertainment,apps_night_timeoffirstusesocial,apps_night_timeoffirstusecom.google.android.youtube,apps_night_timeoffirstusetop1global,apps_night_timeoffirstusecom.facebook.moments,apps_night_timeoflastuseall,apps_night_timeoflastuseemail,apps_night_timeoflastuseentertainment,apps_night_timeoflastusesocial,apps_night_timeoflastusecom.google.android.youtube,apps_night_timeoflastusetop1global,apps_night_timeoflastusecom.facebook.moments,apps_night_frequencyentropyall,apps_night_frequencyentropyemail,apps_night_frequencyentropyentertainment,apps_night_frequencyentropysocial,apps_night_frequencyentropycom.google.android.youtube,apps_night_frequencyentropytop1global,apps_night_frequencyentropycom.facebook.moments

tests/data/processed/test03/applications_foreground_afternoon.csv

@@ -0,0 +1 @@
+local_date,apps_afternoon_countemail,apps_afternoon_countall,apps_afternoon_countsocial,apps_afternoon_countentertainment,apps_afternoon_counttop1global,apps_afternoon_countcom.facebook.moments,apps_afternoon_countcom.google.android.youtube,apps_afternoon_timeoffirstuseemail,apps_afternoon_timeoffirstuseall,apps_afternoon_timeoffirstusesocial,apps_afternoon_timeoffirstuseentertainment,apps_afternoon_timeoffirstusetop1global,apps_afternoon_timeoffirstusecom.facebook.moments,apps_afternoon_timeoffirstusecom.google.android.youtube,apps_afternoon_timeoflastuseemail,apps_afternoon_timeoflastuseall,apps_afternoon_timeoflastusesocial,apps_afternoon_timeoflastuseentertainment,apps_afternoon_timeoflastusetop1global,apps_afternoon_timeoflastusecom.facebook.moments,apps_afternoon_timeoflastusecom.google.android.youtube,apps_afternoon_frequencyentropyemail,apps_afternoon_frequencyentropyall,apps_afternoon_frequencyentropysocial,apps_afternoon_frequencyentropyentertainment,apps_afternoon_frequencyentropytop1global,apps_afternoon_frequencyentropycom.facebook.moments,apps_afternoon_frequencyentropycom.google.android.youtube

tests/data/processed/test03/applications_foreground_daily.csv

@@ -0,0 +1 @@
+local_date,apps_daily_countall,apps_daily_countemail,apps_daily_countsocial,apps_daily_countentertainment,apps_daily_counttop1global,apps_daily_countcom.google.android.youtube,apps_daily_countcom.facebook.moments,apps_daily_timeoffirstuseall,apps_daily_timeoffirstuseemail,apps_daily_timeoffirstusesocial,apps_daily_timeoffirstuseentertainment,apps_daily_timeoffirstusetop1global,apps_daily_timeoffirstusecom.google.android.youtube,apps_daily_timeoffirstusecom.facebook.moments,apps_daily_timeoflastuseall,apps_daily_timeoflastuseemail,apps_daily_timeoflastusesocial,apps_daily_timeoflastuseentertainment,apps_daily_timeoflastusetop1global,apps_daily_timeoflastusecom.google.android.youtube,apps_daily_timeoflastusecom.facebook.moments,apps_daily_frequencyentropyall,apps_daily_frequencyentropyemail,apps_daily_frequencyentropysocial,apps_daily_frequencyentropyentertainment,apps_daily_frequencyentropytop1global,apps_daily_frequencyentropycom.google.android.youtube,apps_daily_frequencyentropycom.facebook.moments

tests/data/processed/test03/applications_foreground_evening.csv

@@ -0,0 +1 @@
+local_date,apps_evening_countall,apps_evening_countemail,apps_evening_countsocial,apps_evening_countentertainment,apps_evening_counttop1global,apps_evening_countcom.facebook.moments,apps_evening_countcom.google.android.youtube,apps_evening_timeoffirstuseall,apps_evening_timeoffirstuseemail,apps_evening_timeoffirstusesocial,apps_evening_timeoffirstuseentertainment,apps_evening_timeoffirstusetop1global,apps_evening_timeoffirstusecom.facebook.moments,apps_evening_timeoffirstusecom.google.android.youtube,apps_evening_timeoflastuseall,apps_evening_timeoflastuseemail,apps_evening_timeoflastusesocial,apps_evening_timeoflastuseentertainment,apps_evening_timeoflastusetop1global,apps_evening_timeoflastusecom.facebook.moments,apps_evening_timeoflastusecom.google.android.youtube,apps_evening_frequencyentropyall,apps_evening_frequencyentropyemail,apps_evening_frequencyentropysocial,apps_evening_frequencyentropyentertainment,apps_evening_frequencyentropytop1global,apps_evening_frequencyentropycom.facebook.moments,apps_evening_frequencyentropycom.google.android.youtube

tests/data/processed/test03/applications_foreground_morning.csv

@@ -0,0 +1 @@
+local_date,apps_morning_countall,apps_morning_countemail,apps_morning_countsocial,apps_morning_countentertainment,apps_morning_countcom.facebook.moments,apps_morning_counttop1global,apps_morning_countcom.google.android.youtube,apps_morning_timeoffirstuseall,apps_morning_timeoffirstuseemail,apps_morning_timeoffirstusesocial,apps_morning_timeoffirstuseentertainment,apps_morning_timeoffirstusecom.facebook.moments,apps_morning_timeoffirstusetop1global,apps_morning_timeoffirstusecom.google.android.youtube,apps_morning_timeoflastuseall,apps_morning_timeoflastuseemail,apps_morning_timeoflastusesocial,apps_morning_timeoflastuseentertainment,apps_morning_timeoflastusecom.facebook.moments,apps_morning_timeoflastusetop1global,apps_morning_timeoflastusecom.google.android.youtube,apps_morning_frequencyentropyall,apps_morning_frequencyentropyemail,apps_morning_frequencyentropysocial,apps_morning_frequencyentropyentertainment,apps_morning_frequencyentropycom.facebook.moments,apps_morning_frequencyentropytop1global,apps_morning_frequencyentropycom.google.android.youtube

tests/data/processed/test03/applications_foreground_night.csv

@@ -0,0 +1 @@
+local_date,apps_night_countemail,apps_night_countall,apps_night_countsocial,apps_night_countentertainment,apps_night_counttop1global,apps_night_countcom.facebook.moments,apps_night_countcom.google.android.youtube,apps_night_timeoffirstuseemail,apps_night_timeoffirstuseall,apps_night_timeoffirstusesocial,apps_night_timeoffirstuseentertainment,apps_night_timeoffirstusetop1global,apps_night_timeoffirstusecom.facebook.moments,apps_night_timeoffirstusecom.google.android.youtube,apps_night_timeoflastuseemail,apps_night_timeoflastuseall,apps_night_timeoflastusesocial,apps_night_timeoflastuseentertainment,apps_night_timeoflastusetop1global,apps_night_timeoflastusecom.facebook.moments,apps_night_timeoflastusecom.google.android.youtube,apps_night_frequencyentropyemail,apps_night_frequencyentropyall,apps_night_frequencyentropysocial,apps_night_frequencyentropyentertainment,apps_night_frequencyentropytop1global,apps_night_frequencyentropycom.facebook.moments,apps_night_frequencyentropycom.google.android.youtube

tests/data/processed/test04/applications_foreground_afternoon.csv

@@ -0,0 +1 @@
+local_date,apps_afternoon_countemail,apps_afternoon_countall,apps_afternoon_countentertainment,apps_afternoon_countsocial,apps_afternoon_countcom.facebook.moments,apps_afternoon_countcom.google.android.youtube,apps_afternoon_counttop1global,apps_afternoon_timeoffirstuseemail,apps_afternoon_timeoffirstuseall,apps_afternoon_timeoffirstuseentertainment,apps_afternoon_timeoffirstusesocial,apps_afternoon_timeoffirstusecom.facebook.moments,apps_afternoon_timeoffirstusecom.google.android.youtube,apps_afternoon_timeoffirstusetop1global,apps_afternoon_timeoflastuseemail,apps_afternoon_timeoflastuseall,apps_afternoon_timeoflastuseentertainment,apps_afternoon_timeoflastusesocial,apps_afternoon_timeoflastusecom.facebook.moments,apps_afternoon_timeoflastusecom.google.android.youtube,apps_afternoon_timeoflastusetop1global,apps_afternoon_frequencyentropyemail,apps_afternoon_frequencyentropyall,apps_afternoon_frequencyentropyentertainment,apps_afternoon_frequencyentropysocial,apps_afternoon_frequencyentropycom.facebook.moments,apps_afternoon_frequencyentropycom.google.android.youtube,apps_afternoon_frequencyentropytop1global

tests/data/processed/test04/applications_foreground_daily.csv

@@ -0,0 +1 @@
+local_date,apps_daily_countall,apps_daily_countemail,apps_daily_countsocial,apps_daily_countentertainment,apps_daily_countcom.facebook.moments,apps_daily_counttop1global,apps_daily_countcom.google.android.youtube,apps_daily_timeoffirstuseall,apps_daily_timeoffirstuseemail,apps_daily_timeoffirstusesocial,apps_daily_timeoffirstuseentertainment,apps_daily_timeoffirstusecom.facebook.moments,apps_daily_timeoffirstusetop1global,apps_daily_timeoffirstusecom.google.android.youtube,apps_daily_timeoflastuseall,apps_daily_timeoflastuseemail,apps_daily_timeoflastusesocial,apps_daily_timeoflastuseentertainment,apps_daily_timeoflastusecom.facebook.moments,apps_daily_timeoflastusetop1global,apps_daily_timeoflastusecom.google.android.youtube,apps_daily_frequencyentropyall,apps_daily_frequencyentropyemail,apps_daily_frequencyentropysocial,apps_daily_frequencyentropyentertainment,apps_daily_frequencyentropycom.facebook.moments,apps_daily_frequencyentropytop1global,apps_daily_frequencyentropycom.google.android.youtube

tests/data/processed/test04/applications_foreground_evening.csv

@@ -0,0 +1 @@
+local_date,apps_evening_countemail,apps_evening_countall,apps_evening_countentertainment,apps_evening_countsocial,apps_evening_counttop1global,apps_evening_countcom.google.android.youtube,apps_evening_countcom.facebook.moments,apps_evening_timeoffirstuseemail,apps_evening_timeoffirstuseall,apps_evening_timeoffirstuseentertainment,apps_evening_timeoffirstusesocial,apps_evening_timeoffirstusetop1global,apps_evening_timeoffirstusecom.google.android.youtube,apps_evening_timeoffirstusecom.facebook.moments,apps_evening_timeoflastuseemail,apps_evening_timeoflastuseall,apps_evening_timeoflastuseentertainment,apps_evening_timeoflastusesocial,apps_evening_timeoflastusetop1global,apps_evening_timeoflastusecom.google.android.youtube,apps_evening_timeoflastusecom.facebook.moments,apps_evening_frequencyentropyemail,apps_evening_frequencyentropyall,apps_evening_frequencyentropyentertainment,apps_evening_frequencyentropysocial,apps_evening_frequencyentropytop1global,apps_evening_frequencyentropycom.google.android.youtube,apps_evening_frequencyentropycom.facebook.moments

tests/data/processed/test04/applications_foreground_morning.csv

@@ -0,0 +1 @@
+local_date,apps_morning_countemail,apps_morning_countall,apps_morning_countsocial,apps_morning_countentertainment,apps_morning_countcom.google.android.youtube,apps_morning_countcom.facebook.moments,apps_morning_counttop1global,apps_morning_timeoffirstuseemail,apps_morning_timeoffirstuseall,apps_morning_timeoffirstusesocial,apps_morning_timeoffirstuseentertainment,apps_morning_timeoffirstusecom.google.android.youtube,apps_morning_timeoffirstusecom.facebook.moments,apps_morning_timeoffirstusetop1global,apps_morning_timeoflastuseemail,apps_morning_timeoflastuseall,apps_morning_timeoflastusesocial,apps_morning_timeoflastuseentertainment,apps_morning_timeoflastusecom.google.android.youtube,apps_morning_timeoflastusecom.facebook.moments,apps_morning_timeoflastusetop1global,apps_morning_frequencyentropyemail,apps_morning_frequencyentropyall,apps_morning_frequencyentropysocial,apps_morning_frequencyentropyentertainment,apps_morning_frequencyentropycom.google.android.youtube,apps_morning_frequencyentropycom.facebook.moments,apps_morning_frequencyentropytop1global

tests/data/processed/test04/applications_foreground_night.csv

@@ -0,0 +1 @@
+local_date,apps_night_countall,apps_night_countemail,apps_night_countentertainment,apps_night_countsocial,apps_night_countcom.google.android.youtube,apps_night_counttop1global,apps_night_countcom.facebook.moments,apps_night_timeoffirstuseall,apps_night_timeoffirstuseemail,apps_night_timeoffirstuseentertainment,apps_night_timeoffirstusesocial,apps_night_timeoffirstusecom.google.android.youtube,apps_night_timeoffirstusetop1global,apps_night_timeoffirstusecom.facebook.moments,apps_night_timeoflastuseall,apps_night_timeoflastuseemail,apps_night_timeoflastuseentertainment,apps_night_timeoflastusesocial,apps_night_timeoflastusecom.google.android.youtube,apps_night_timeoflastusetop1global,apps_night_timeoflastusecom.facebook.moments,apps_night_frequencyentropyall,apps_night_frequencyentropyemail,apps_night_frequencyentropyentertainment,apps_night_frequencyentropysocial,apps_night_frequencyentropycom.google.android.youtube,apps_night_frequencyentropytop1global,apps_night_frequencyentropycom.facebook.moments

tests/data/raw/test01/applications_foreground_raw.csv

@@ -0,0 +1,33 @@
+timestamp,device_id,package_name,application_name,is_system_app
+1593946320761,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,tv.twitch.android.app,Twitch,0
+1593961974942,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,com.google.android.gms,Google,1
+1593958144033,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,tv.twitch.android.app,Twitch,0
+1593947228964,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,com.google.android.apps.youtube.creator,Youtube Video Creater,0
+1593951572326,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,com.netflix.mediaclient,Netflix,0
+1593950554868,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,com.supercell.clashofclans,Clash of Clans,0
+1593964799620,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,tv.twitch.android.app,Twitch,0
+1593974241305,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,com.facebook.moments,Facebook Moments,0
+1593969483540,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,com.facebook.moments,Facebook Moments,0
+1593977289581,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,com.fitbit.FitbitMobile,Fitbit,0
+1593970763367,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,com.netflix.mediaclient,Netflix,0
+1593964867720,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,com.supercell.clashofclans,Clash of Clans,0
+1593974942995,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,com.google.android.gm,Gmail,0
+1593986399351,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,com.netflix.mediaclient,Netflix,0
+1594000139073,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,com.google.android.gm,Gmail,0
+1593994717099,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,com.fitbit.FitbitMobile,Fitbit,0
+1593985854872,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,com.google.android.youtube,Youtube,0
+1594003154390,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,com.google.android.apps.youtube.creator,Youtube Video Creater,0
+1594003853415,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,com.google.android.gms,Google,1
+1593991680045,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,com.google.android.apps.youtube.creator,Youtube Video Creater,0
+1594007999202,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,com.aware.plugin.upmc.cancer,AWARE,0
+1593939733998,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,com.google.android.apps.youtube.creator,Youtube Video Creater,0
+1593933324739,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,tv.twitch.android.app,Twitch,0
+1593925161482,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,com.facebook.moments,Facebook Moments,0
+1593936918763,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,com.google.android.gms,Google,1
+1593924155524,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,com.google.android.youtube,Youtube,0
+1593922625358,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,com.facebook.moments,Facebook Moments,0
+1593943199317,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,tv.twitch.android.app,Twitch,0
+1593951550550,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,com.google.android.gms,Google,1
+1593981544544,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,com.google.android.gms,Google,1
+1593999779779,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,com.google.android.gms,Google,1
+1593933565565,wYESbVwI-4GfR-G5I6-7iKL-tOmCKs02MBun,com.google.android.gms,Google,1

tests/data/raw/test02/applications_foreground_raw.csv

@@ -0,0 +1 @@
+timestamp,device_id,package_name,application_name,is_system_app

tests/data/raw/test03/applications_foreground_raw.csv

@@ -0,0 +1 @@
+timestamp,device_id,package_name,application_name,is_system_app

tests/data/raw/test04/applications_foreground_raw.csv

@@ -0,0 +1 @@
+timestamp,device_id,package_name,application_name,is_system_app

tests/scripts/test_sensor_features.py

@@ -36,6 +36,10 @@ class TestSensorFeatures(unittest.TestCase):
                 if df_act.empty:
                     self.assertTrue(df_exp.empty)
                 else:
+                    # The order in which the columns varies from time to time so
+                    # the columns are sorted before doing the comparision 
+                    df_exp = df_exp.reindex(sorted(df_exp.columns), axis=1)
+                    df_act = df_act.reindex(sorted(df_act.columns), axis=1)
                     pd.testing.assert_frame_equal(df_exp, df_act, obj=df_exp)
 
 

tests/scripts/utils.py

@@ -83,7 +83,7 @@ def generate_sensor_file_lists(config):
     exp_str = "tests/data/processed/{pid}/{sensor}_{sensor_type}{day_segment}.csv"
 
     # List of available sensors that can be tested by the testing suite
-    TESTABLE_SENSORS = ['MESSAGES', 'CALLS', 'SCREEN']
+    TESTABLE_SENSORS = ['MESSAGES', 'CALLS', 'SCREEN', 'BATTERY', 'BLUETOOTH', 'WIFI', 'LIGHT', 'APPLICATIONS_FOREGROUND']
 
     # Build list of sensors to be tested. 
     sensors = []

tests/settings/config.yaml

@@ -2,4 +2,4 @@ directory: ./
 configfile: ./tests/settings/testing_config.yaml
 snakefile: ./tests/Snakefile
 cores: 1
-forcerun: [messages_features, call_features, screen_features, battery_features, bluetooth_features, wifi_features, light_features]
+forcerun: [messages_features, call_features, screen_features, battery_features, bluetooth_features, wifi_features, light_features, applications_foreground_features]

tests/settings/testing_config.yaml

@@ -7,11 +7,11 @@ DAY_SEGMENTS: &day_segments
   [daily, morning, afternoon, evening, night]
 
 PHONE_VALID_SENSED_BINS:
-  TABLES: [messages, calls, screen, battery, bluetooth, wifi, light]
+  TABLES: [messages, calls, screen, battery, bluetooth, wifi, light, applications_foreground]
 
 # Communication SMS features config, TYPES and FEATURES keys need to match
 MESSAGES:
-  COMPUTE: True
+  COMPUTE: False
   DB_TABLE: messages
   TYPES : [received, sent]
   FEATURES: 
@@ -21,7 +21,7 @@ MESSAGES:
 
 # Communication call features config, TYPES and FEATURES keys need to match
 CALLS:
-  COMPUTE: True
+  COMPUTE: False
   DB_TABLE: calls
   TYPES: [missed, incoming, outgoing]
   FEATURES:
@@ -31,7 +31,7 @@ CALLS:
   DAY_SEGMENTS: *day_segments
 
 SCREEN:
-  COMPUTE: True
+  COMPUTE: False
   DB_TABLE: screen
   DAY_SEGMENTS: *day_segments
   REFERENCE_HOUR_FIRST_USE: 0
@@ -39,25 +39,38 @@ SCREEN:
   EPISODE_TYPES: ["unlock"]
 
 BATTERY:
-  COMPUTE: True
+  COMPUTE: False
   DB_TABLE: battery
   DAY_SEGMENTS: *day_segments
   FEATURES: ["countdischarge", "sumdurationdischarge", "countcharge", "sumdurationcharge", "avgconsumptionrate", "maxconsumptionrate"]
 
 BLUETOOTH:
-  COMPUTE: True
+  COMPUTE: False
   DB_TABLE: bluetooth
   DAY_SEGMENTS: *day_segments
   FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"]
 
 WIFI:
-  COMPUTE: True
+  COMPUTE: False
   DB_TABLE: wifi
   DAY_SEGMENTS: *day_segments
   FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"]
 
 LIGHT:
-  COMPUTE: True
+  COMPUTE: False
   DB_TABLE: light
   DAY_SEGMENTS: *day_segments
   FEATURES: ["count", "maxlux", "minlux", "avglux", "medianlux", "stdlux"]
+
+APPLICATIONS_FOREGROUND:
+  COMPUTE: True
+  DB_TABLE: applications_foreground
+  DAY_SEGMENTS: *day_segments
+  SINGLE_CATEGORIES: ["all", "email"]
+  MULTIPLE_CATEGORIES:
+    social: ["socialnetworks", "socialmediatools"]
+    entertainment: ["entertainment", "gamingstrategy"]
+  SINGLE_APPS: ["top1global", "com.facebook.moments", "com.google.android.youtube"] # There's no entropy for single apps
+  EXCLUDED_CATEGORIES: ["systemapp", "tvvideoapps"]
+  EXCLUDED_APPS: ["com.fitbit.FitbitMobile", "com.aware.plugin.upmc.cancer"]
+  FEATURES: ["count", "timeoffirstuse", "timeoflastuse", "frequencyentropy"]