New Application_foreground testing & bugfixes

.travis.yml

@@ -1,7 +1,6 @@
 services:
 - mysql
 - docker
-sudo: required
 language: python
 jobs:
   include:
@@ -80,16 +79,13 @@ jobs:
     - docker login -u "agamk" -p $DOCKERPWD
     - docker tag rapids agamk/rapids:travislatest
     - docker push agamk/rapids:travislatest
-
 #branches:
 #  only:
 #  - master
 #  - day_segment
-
 stages:
   - name: deploy
     if: branch = master AND type = push
-
 notifications:
   email: false
   slack:

src/features/applications_foreground/rapids/main.py

@@ -54,10 +54,11 @@ def rapids_features(apps_data, day_segment, provider, filter_data_by_segment, *a
 
     apps_features = pd.DataFrame(columns=["local_segment"] + ["apps_rapids_" + "_" + x for x in ["".join(feature) for feature in itertools.product(requested_features, single_categories + multiple_categories + single_apps)]])
     if not apps_data.empty:
-        apps_data = filter_data_by_segment(apps_data, day_segment)
         # deep copy the apps_data for the top1global computation
         apps_data_global = apps_data.copy()
-
+        
+        apps_data = filter_data_by_segment(apps_data, day_segment)
+        
         if not apps_data.empty:
             apps_features = pd.DataFrame()
             # single category
@@ -77,12 +78,12 @@ def rapids_features(apps_data, day_segment, provider, filter_data_by_segment, *a
                 col_name = app
                 if app == "top1global":
                     # get the most used app
-                    apps_with_count = apps_data_global.groupby(["local_segment","package_name"]).count().sort_values(by="timestamp", ascending=False).reset_index()
+                    apps_with_count = apps_data_global.groupby(["package_name"]).count().sort_values(by="timestamp", ascending=False).reset_index()
                     app = apps_with_count.iloc[0]["package_name"]
                     col_name = "top1global"
                 filtered_data = apps_data[apps_data["package_name"].isin([app])]
                 apps_features = compute_features(filtered_data, col_name, requested_features, apps_features, day_segment)
-
+ 
             apps_features = apps_features.reset_index()
     
     return apps_features

tests/Snakefile

@@ -61,6 +61,14 @@ for provider in config["LIGHT"]["PROVIDERS"].keys():
         files_to_compute.extend(expand("data/interim/{pid}/{sensor_key}_features/{sensor_key}_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["LIGHT"]["PROVIDERS"][provider]["SRC_LANGUAGE"], provider_key=provider, sensor_key="LIGHT".lower()))
         files_to_compute.extend(expand("data/processed/features/{pid}/{sensor_key}.csv", pid=config["PIDS"], sensor_key="LIGHT".lower()))
 
+for provider in config["APPLICATIONS_FOREGROUND"]["PROVIDERS"].keys():
+    if config["APPLICATIONS_FOREGROUND"]["PROVIDERS"][provider]["COMPUTE"]:
+        files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=config["APPLICATIONS_FOREGROUND"]["DB_TABLE"]))
+        files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime.csv", pid=config["PIDS"], sensor=config["APPLICATIONS_FOREGROUND"]["DB_TABLE"]))
+        files_to_compute.extend(expand("data/raw/{pid}/{sensor}_with_datetime_with_genre.csv", pid=config["PIDS"], sensor=config["APPLICATIONS_FOREGROUND"]["DB_TABLE"]))
+        files_to_compute.extend(expand("data/interim/{pid}/{sensor_key}_features/{sensor_key}_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["APPLICATIONS_FOREGROUND"]["PROVIDERS"][provider]["SRC_LANGUAGE"], provider_key=provider, sensor_key="APPLICATIONS_FOREGROUND".lower()))
+        files_to_compute.extend(expand("data/processed/features/{pid}/{sensor_key}.csv", pid=config["PIDS"], sensor_key="APPLICATIONS_FOREGROUND".lower()))
+
 for provider in config["WIFI"]["PROVIDERS"].keys():
     if config["WIFI"]["PROVIDERS"][provider]["COMPUTE"]:
         if len(config["WIFI"]["DB_TABLE"]["VISIBLE_ACCESS_POINTS"]) > 0:

tests/data/processed/features/test01/applications_foreground.csv

@@ -0,0 +1,6 @@
+"local_segment","local_segment_label","local_start_date","local_start_time","local_end_date","local_end_time","apps_rapids_timeoffirstuseall","apps_rapids_timeoflastuseall","apps_rapids_frequencyentropyall","apps_rapids_countall","apps_rapids_timeoffirstuseemail","apps_rapids_timeoflastuseemail","apps_rapids_frequencyentropyemail","apps_rapids_countemail","apps_rapids_timeoffirstusesocial","apps_rapids_timeoflastusesocial","apps_rapids_frequencyentropysocial","apps_rapids_countsocial","apps_rapids_timeoffirstuseentertainment","apps_rapids_timeoflastuseentertainment","apps_rapids_frequencyentropyentertainment","apps_rapids_countentertainment","apps_rapids_timeoffirstusetop1global","apps_rapids_timeoflastusetop1global","apps_rapids_frequencyentropytop1global","apps_rapids_counttop1global","apps_rapids_timeoffirstusecom.facebook.moments","apps_rapids_timeoflastusecom.facebook.moments","apps_rapids_frequencyentropycom.facebook.moments","apps_rapids_countcom.facebook.moments"
+"afternoon#2020-07-05#12:00:00#2020-07-05#17:59:59","afternoon","2020-07-05","12:00:00","2020-07-05","17:59:59",721,889,1.03972077083992,4,889,889,NA,1,NA,NA,NA,0,721,721,NA,1,NA,NA,NA,0,798,877,NA,2
+"daily#2020-07-05#00:00:00#2020-07-05#23:59:59","daily","2020-07-05","00:00:00","2020-07-05","23:59:59",17,1359,1.54438198091684,17,889,1308,NA,2,302,1359,NA,4,195,721,0.598269588585257,7,195,719,NA,5,17,877,NA,4
+"evening#2020-07-05#18:00:00#2020-07-05#23:59:59","evening","2020-07-05","18:00:00","2020-07-05","23:59:59",1168,1359,0.636514168294813,3,1308,1308,NA,1,1168,1359,NA,2,NA,NA,NA,0,NA,NA,NA,0,NA,NA,NA,0
+"morning#2020-07-05#06:00:00#2020-07-05#11:59:59","morning","2020-07-05","06:00:00","2020-07-05","11:59:59",412,719,0.950270539233235,5,NA,NA,NA,0,427,427,NA,1,412,719,0.562335144618808,4,412,719,NA,3,NA,NA,NA,0
+"night#2020-07-05#00:00:00#2020-07-05#05:59:59","night","2020-07-05","00:00:00","2020-07-05","05:59:59",17,359,1.05492016798614,5,NA,NA,NA,0,302,302,NA,1,195,359,NA,2,195,359,NA,2,17,59,NA,2

tests/data/processed/features/test02/applications_foreground.csv

@@ -0,0 +1 @@
+"local_segment","local_segment_label","local_start_date","local_start_time","local_end_date","local_end_time","apps_rapids__countall","apps_rapids__countemail","apps_rapids__countsocial","apps_rapids__countentertainment","apps_rapids__counttop1global","apps_rapids__countcom.facebook.moments","apps_rapids__timeoffirstuseall","apps_rapids__timeoffirstuseemail","apps_rapids__timeoffirstusesocial","apps_rapids__timeoffirstuseentertainment","apps_rapids__timeoffirstusetop1global","apps_rapids__timeoffirstusecom.facebook.moments","apps_rapids__timeoflastuseall","apps_rapids__timeoflastuseemail","apps_rapids__timeoflastusesocial","apps_rapids__timeoflastuseentertainment","apps_rapids__timeoflastusetop1global","apps_rapids__timeoflastusecom.facebook.moments","apps_rapids__frequencyentropyall","apps_rapids__frequencyentropyemail","apps_rapids__frequencyentropysocial","apps_rapids__frequencyentropyentertainment","apps_rapids__frequencyentropytop1global","apps_rapids__frequencyentropycom.facebook.moments"

tests/data/processed/features/test03/applications_foreground.csv

@@ -0,0 +1 @@
+"local_segment","local_segment_label","local_start_date","local_start_time","local_end_date","local_end_time","apps_rapids__countall","apps_rapids__countemail","apps_rapids__countsocial","apps_rapids__countentertainment","apps_rapids__counttop1global","apps_rapids__countcom.facebook.moments","apps_rapids__timeoffirstuseall","apps_rapids__timeoffirstuseemail","apps_rapids__timeoffirstusesocial","apps_rapids__timeoffirstuseentertainment","apps_rapids__timeoffirstusetop1global","apps_rapids__timeoffirstusecom.facebook.moments","apps_rapids__timeoflastuseall","apps_rapids__timeoflastuseemail","apps_rapids__timeoflastusesocial","apps_rapids__timeoflastuseentertainment","apps_rapids__timeoflastusetop1global","apps_rapids__timeoflastusecom.facebook.moments","apps_rapids__frequencyentropyall","apps_rapids__frequencyentropyemail","apps_rapids__frequencyentropysocial","apps_rapids__frequencyentropyentertainment","apps_rapids__frequencyentropytop1global","apps_rapids__frequencyentropycom.facebook.moments"

tests/data/processed/features/test04/applications_foreground.csv

@@ -0,0 +1 @@
+"local_segment","local_segment_label","local_start_date","local_start_time","local_end_date","local_end_time","apps_rapids__countall","apps_rapids__countemail","apps_rapids__countsocial","apps_rapids__countentertainment","apps_rapids__counttop1global","apps_rapids__countcom.facebook.moments","apps_rapids__timeoffirstuseall","apps_rapids__timeoffirstuseemail","apps_rapids__timeoffirstusesocial","apps_rapids__timeoffirstuseentertainment","apps_rapids__timeoffirstusetop1global","apps_rapids__timeoffirstusecom.facebook.moments","apps_rapids__timeoflastuseall","apps_rapids__timeoflastuseemail","apps_rapids__timeoflastusesocial","apps_rapids__timeoflastuseentertainment","apps_rapids__timeoflastusetop1global","apps_rapids__timeoflastusecom.facebook.moments","apps_rapids__frequencyentropyall","apps_rapids__frequencyentropyemail","apps_rapids__frequencyentropysocial","apps_rapids__frequencyentropyentertainment","apps_rapids__frequencyentropytop1global","apps_rapids__frequencyentropycom.facebook.moments"

tests/settings/testing_config.yaml

@@ -8,7 +8,7 @@ DAY_SEGMENTS: &day_segments
   FILE: "data/external/daysegments_interval.csv"
 
 PHONE_VALID_SENSED_BINS:
-  DB_TABLES: [messages, calls, bluetooth, light, wifi]
+  DB_TABLES: [messages, calls, bluetooth, light, wifi, applications_foreground]
 
 # Communication SMS features config, TYPES and FEATURES keys need to match
 MESSAGES:
@@ -66,6 +66,22 @@ WIFI:
       SRC_FOLDER: "rapids" # inside src/features/bluetooth
       SRC_LANGUAGE: "r"
 
+APPLICATIONS_FOREGROUND:
+  DB_TABLE: applications_foreground
+  PROVIDERS:
+    RAPIDS:
+      COMPUTE: True
+      SINGLE_CATEGORIES: ["all", "email"]
+      MULTIPLE_CATEGORIES:
+        social: ["socialnetworks", "socialmediatools"]
+        entertainment: ["entertainment", "gamingstrategy"]
+      SINGLE_APPS: ["top1global", "com.facebook.moments"] # There's no entropy for single apps
+      EXCLUDED_CATEGORIES: ["systemapp", "tvvideoapps"]
+      EXCLUDED_APPS: ["com.fitbit.FitbitMobile", "com.aware.plugin.upmc.cancer"]
+      FEATURES: ["count", "timeoffirstuse", "timeoflastuse", "frequencyentropy"]
+      SRC_FOLDER: "rapids" # inside src/features/applications_foreground
+      SRC_LANGUAGE: "python"
+
 CONVERSATION:
   DB_TABLE: 
     ANDROID: plugin_studentlife_audio_android