Use methods in helper.py.

Update coding files for app categories.

.idea/vcs.xml

@@ -3,6 +3,5 @@
   <component name="VcsDirectoryMappings">
     <mapping directory="$PROJECT_DIR$" vcs="Git" />
     <mapping directory="$PROJECT_DIR$/rapids" vcs="Git" />
-    <mapping directory="$PROJECT_DIR$/rapids/calculatingfeatures" vcs="Git" />
   </component>
 </project>

config/environment.yml

@@ -1,9 +1,8 @@
 name: straw2analysis
 channels:
-  - defaults
   - conda-forge
 dependencies:
-  - python=3.9
+  - python=3.11
   - black
   - isort
   - flake8
@@ -24,3 +23,4 @@ dependencies:
   - sqlalchemy
   - statsmodels
   - tabulate
+  - xgboost

exploration/ml_pipeline_regression.py

@@ -15,91 +15,34 @@
 
 # %% jupyter={"source_hidden": true}
 # %matplotlib inline
-import datetime
-import importlib
 import os
 import sys
 
 import numpy as np
-import matplotlib.pyplot as plt
 import pandas as pd
-import seaborn as sns
-import yaml
-from pyprojroot import here
-from sklearn import linear_model, svm, kernel_ridge, gaussian_process
-from sklearn.model_selection import LeaveOneGroupOut, cross_val_score, cross_validate
-from sklearn.metrics import mean_squared_error, r2_score
-from sklearn.impute import SimpleImputer
-from sklearn.dummy import DummyRegressor
 import xgboost as xg
-from IPython.core.interactiveshell import InteractiveShell
-InteractiveShell.ast_node_interactivity = "all"
+from machine_learning.helper import prepare_regression_model_input
+from sklearn import gaussian_process, kernel_ridge, linear_model, svm
+from sklearn.dummy import DummyRegressor
+from sklearn.impute import SimpleImputer
+from sklearn.model_selection import LeaveOneGroupOut, cross_validate
+
+# from IPython.core.interactiveshell import InteractiveShell
+# InteractiveShell.ast_node_interactivity = "all"
 
 nb_dir = os.path.split(os.getcwd())[0]
 if nb_dir not in sys.path:
     sys.path.append(nb_dir)
 
-import machine_learning.features_sensor
-import machine_learning.labels
-import machine_learning.model
-
-# %% [markdown]
-# # RAPIDS models
-
-# %% [markdown]
-# ## PANAS negative affect
+# %% jupyter={"source_hidden": true}
+model_input = pd.read_csv(
+    "../data/intradaily_30_min_all_targets/input_JCQ_job_demand_mean.csv"
+)
 
 # %% jupyter={"source_hidden": true}
-model_input = pd.read_csv("../data/intradaily_30_min_all_targets/input_JCQ_job_demand_mean.csv")
-
-# %% jupyter={"source_hidden": true}
-index_columns = ["local_segment", "local_segment_label", "local_segment_start_datetime", "local_segment_end_datetime"]
-#if "pid" in model_input.columns:
-#    index_columns.append("pid")
-model_input.set_index(index_columns, inplace=True)
-
-cv_method = 'half_logo' # logo, half_logo, 5kfold
-if cv_method == 'logo':
-    data_x, data_y, data_groups = model_input.drop(["target", "pid"], axis=1), model_input["target"], model_input["pid"]
-else:
-    model_input['pid_index'] = model_input.groupby('pid').cumcount()
-    model_input['pid_count'] = model_input.groupby('pid')['pid'].transform('count')
-
-    model_input["pid_index"] = (model_input['pid_index'] / model_input['pid_count'] + 1).round()
-    model_input["pid_half"] = model_input["pid"] + "_" +  model_input["pid_index"].astype(int).astype(str)
-
-    data_x, data_y, data_groups = model_input.drop(["target", "pid", "pid_index", "pid_half"], axis=1), model_input["target"], model_input["pid_half"]
-
-# %% jupyter={"source_hidden": true}
-categorical_feature_colnames = ["gender", "startlanguage"]
-additional_categorical_features = [col for col in data_x.columns if "mostcommonactivity" in col or "homelabel" in col]
-categorical_feature_colnames += additional_categorical_features
-
-# %% jupyter={"source_hidden": true}
-categorical_features = data_x[categorical_feature_colnames].copy()
-
-# %% jupyter={"source_hidden": true}
-mode_categorical_features = categorical_features.mode().iloc[0]
-
-# %% jupyter={"source_hidden": true}
-# fillna with mode
-categorical_features = categorical_features.fillna(mode_categorical_features)
-
-# %% jupyter={"source_hidden": true}
-# one-hot encoding
-categorical_features = categorical_features.apply(lambda col: col.astype("category"))
-if not categorical_features.empty:
-    categorical_features = pd.get_dummies(categorical_features)
-
-# %% jupyter={"source_hidden": true}
-numerical_features = data_x.drop(categorical_feature_colnames, axis=1)
-
-# %% jupyter={"source_hidden": true}
-train_x = pd.concat([numerical_features, categorical_features], axis=1)
-
-# %% jupyter={"source_hidden": true}
-train_x.dtypes
+cv_method = "half_logo"  # logo, half_logo, 5kfold
 
+train_x, data_y, data_groups = prepare_regression_model_input(model_input, cv_method)
 # %% jupyter={"source_hidden": true}
 logo = LeaveOneGroupOut()
 logo.get_n_splits(
@@ -109,7 +52,7 @@ logo.get_n_splits(
 )
 
 # Defaults to 5 k folds in cross_validate method
-if cv_method != 'logo' and cv_method != 'half_logo':
+if cv_method != "logo" and cv_method != "half_logo":
     logo = None
 
 # %% jupyter={"source_hidden": true}
@@ -120,7 +63,7 @@ sum(data_y.isna())
 dummy_regr = DummyRegressor(strategy="mean")
 
 # %% jupyter={"source_hidden": true}
-imputer = SimpleImputer(missing_values=np.nan, strategy='mean')
+imputer = SimpleImputer(missing_values=np.nan, strategy="mean")
 
 # %% jupyter={"source_hidden": true}
 dummy_regressor = cross_validate(
@@ -130,12 +73,26 @@ dummy_regressor = cross_validate(
     groups=data_groups,
     cv=logo,
     n_jobs=-1,
-    scoring=('r2', 'neg_mean_squared_error', 'neg_mean_absolute_error', 'neg_root_mean_squared_error')
+    scoring=(
+        "r2",
+        "neg_mean_squared_error",
+        "neg_mean_absolute_error",
+        "neg_root_mean_squared_error",
+    ),
 )
-print("Negative Mean Squared Error", np.median(dummy_regressor['test_neg_mean_squared_error']))
-print("Negative Mean Absolute Error", np.median(dummy_regressor['test_neg_mean_absolute_error']))
-print("Negative Root Mean Squared Error", np.median(dummy_regressor['test_neg_root_mean_squared_error']))
-print("R2", np.median(dummy_regressor['test_r2']))
+print(
+    "Negative Mean Squared Error",
+    np.median(dummy_regressor["test_neg_mean_squared_error"]),
+)
+print(
+    "Negative Mean Absolute Error",
+    np.median(dummy_regressor["test_neg_mean_absolute_error"]),
+)
+print(
+    "Negative Root Mean Squared Error",
+    np.median(dummy_regressor["test_neg_root_mean_squared_error"]),
+)
+print("R2", np.median(dummy_regressor["test_r2"]))
 
 # %% [markdown]
 # ### Linear Regression
@@ -143,7 +100,7 @@ print("R2", np.median(dummy_regressor['test_r2']))
 # %% jupyter={"source_hidden": true}
 lin_reg_rapids = linear_model.LinearRegression()
 # %% jupyter={"source_hidden": true}
-imputer = SimpleImputer(missing_values=np.nan, strategy='mean')
+imputer = SimpleImputer(missing_values=np.nan, strategy="mean")
 
 # %% jupyter={"source_hidden": true}
 lin_reg_scores = cross_validate(
@@ -153,19 +110,33 @@ lin_reg_scores = cross_validate(
     groups=data_groups,
     cv=logo,
     n_jobs=-1,
-    scoring=('r2', 'neg_mean_squared_error', 'neg_mean_absolute_error', 'neg_root_mean_squared_error')
+    scoring=(
+        "r2",
+        "neg_mean_squared_error",
+        "neg_mean_absolute_error",
+        "neg_root_mean_squared_error",
+    ),
 )
-print("Negative Mean Squared Error", np.median(lin_reg_scores['test_neg_mean_squared_error']))
-print("Negative Mean Absolute Error", np.median(lin_reg_scores['test_neg_mean_absolute_error']))
-print("Negative Root Mean Squared Error", np.median(lin_reg_scores['test_neg_root_mean_squared_error']))
-print("R2", np.median(lin_reg_scores['test_r2']))
+print(
+    "Negative Mean Squared Error",
+    np.median(lin_reg_scores["test_neg_mean_squared_error"]),
+)
+print(
+    "Negative Mean Absolute Error",
+    np.median(lin_reg_scores["test_neg_mean_absolute_error"]),
+)
+print(
+    "Negative Root Mean Squared Error",
+    np.median(lin_reg_scores["test_neg_root_mean_squared_error"]),
+)
+print("R2", np.median(lin_reg_scores["test_r2"]))
 
 # %% [markdown]
 # ### XGBRegressor Linear Regression
 # %% jupyter={"source_hidden": true}
-xgb_r = xg.XGBRegressor(objective ='reg:squarederror', n_estimators = 10)
+xgb_r = xg.XGBRegressor(objective="reg:squarederror", n_estimators=10)
 # %% jupyter={"source_hidden": true}
-imputer = SimpleImputer(missing_values=np.nan, strategy='mean')
+imputer = SimpleImputer(missing_values=np.nan, strategy="mean")
 
 # %% jupyter={"source_hidden": true}
 xgb_reg_scores = cross_validate(
@@ -175,19 +146,33 @@ xgb_reg_scores = cross_validate(
     groups=data_groups,
     cv=logo,
     n_jobs=-1,
-    scoring=('r2', 'neg_mean_squared_error', 'neg_mean_absolute_error', 'neg_root_mean_squared_error')
+    scoring=(
+        "r2",
+        "neg_mean_squared_error",
+        "neg_mean_absolute_error",
+        "neg_root_mean_squared_error",
+    ),
 )
-print("Negative Mean Squared Error", np.median(xgb_reg_scores['test_neg_mean_squared_error']))
-print("Negative Mean Absolute Error", np.median(xgb_reg_scores['test_neg_mean_absolute_error']))
-print("Negative Root Mean Squared Error", np.median(xgb_reg_scores['test_neg_root_mean_squared_error']))
-print("R2", np.median(xgb_reg_scores['test_r2']))
+print(
+    "Negative Mean Squared Error",
+    np.median(xgb_reg_scores["test_neg_mean_squared_error"]),
+)
+print(
+    "Negative Mean Absolute Error",
+    np.median(xgb_reg_scores["test_neg_mean_absolute_error"]),
+)
+print(
+    "Negative Root Mean Squared Error",
+    np.median(xgb_reg_scores["test_neg_root_mean_squared_error"]),
+)
+print("R2", np.median(xgb_reg_scores["test_r2"]))
 
 # %% [markdown]
 # ### XGBRegressor Pseudo Huber Error Regression
 # %% jupyter={"source_hidden": true}
-xgb_psuedo_huber_r = xg.XGBRegressor(objective ='reg:pseudohubererror', n_estimators = 10)
+xgb_psuedo_huber_r = xg.XGBRegressor(objective="reg:pseudohubererror", n_estimators=10)
 # %% jupyter={"source_hidden": true}
-imputer = SimpleImputer(missing_values=np.nan, strategy='mean')
+imputer = SimpleImputer(missing_values=np.nan, strategy="mean")
 
 # %% jupyter={"source_hidden": true}
 xgb_psuedo_huber_reg_scores = cross_validate(
@@ -197,18 +182,32 @@ xgb_psuedo_huber_reg_scores = cross_validate(
     groups=data_groups,
     cv=logo,
     n_jobs=-1,
-    scoring=('r2', 'neg_mean_squared_error', 'neg_mean_absolute_error', 'neg_root_mean_squared_error')
+    scoring=(
+        "r2",
+        "neg_mean_squared_error",
+        "neg_mean_absolute_error",
+        "neg_root_mean_squared_error",
+    ),
 )
-print("Negative Mean Squared Error", np.median(xgb_psuedo_huber_reg_scores['test_neg_mean_squared_error']))
-print("Negative Mean Absolute Error", np.median(xgb_psuedo_huber_reg_scores['test_neg_mean_absolute_error']))
-print("Negative Root Mean Squared Error", np.median(xgb_psuedo_huber_reg_scores['test_neg_root_mean_squared_error']))
-print("R2", np.median(xgb_psuedo_huber_reg_scores['test_r2']))
+print(
+    "Negative Mean Squared Error",
+    np.median(xgb_psuedo_huber_reg_scores["test_neg_mean_squared_error"]),
+)
+print(
+    "Negative Mean Absolute Error",
+    np.median(xgb_psuedo_huber_reg_scores["test_neg_mean_absolute_error"]),
+)
+print(
+    "Negative Root Mean Squared Error",
+    np.median(xgb_psuedo_huber_reg_scores["test_neg_root_mean_squared_error"]),
+)
+print("R2", np.median(xgb_psuedo_huber_reg_scores["test_r2"]))
 
 # %% [markdown]
 # ### Ridge regression
 
 # %% jupyter={"source_hidden": true}
-ridge_reg = linear_model.Ridge(alpha=.5)
+ridge_reg = linear_model.Ridge(alpha=0.5)
 
 # %% tags=[] jupyter={"source_hidden": true}
 ridge_reg_scores = cross_validate(
@@ -218,12 +217,26 @@ ridge_reg_scores = cross_validate(
     groups=data_groups,
     cv=logo,
     n_jobs=-1,
-    scoring=('r2', 'neg_mean_squared_error', 'neg_mean_absolute_error', 'neg_root_mean_squared_error')
+    scoring=(
+        "r2",
+        "neg_mean_squared_error",
+        "neg_mean_absolute_error",
+        "neg_root_mean_squared_error",
+    ),
 )
-print("Negative Mean Squared Error", np.median(ridge_reg_scores['test_neg_mean_squared_error']))
-print("Negative Mean Absolute Error", np.median(ridge_reg_scores['test_neg_mean_absolute_error']))
-print("Negative Root Mean Squared Error", np.median(ridge_reg_scores['test_neg_root_mean_squared_error']))
-print("R2", np.median(ridge_reg_scores['test_r2']))
+print(
+    "Negative Mean Squared Error",
+    np.median(ridge_reg_scores["test_neg_mean_squared_error"]),
+)
+print(
+    "Negative Mean Absolute Error",
+    np.median(ridge_reg_scores["test_neg_mean_absolute_error"]),
+)
+print(
+    "Negative Root Mean Squared Error",
+    np.median(ridge_reg_scores["test_neg_root_mean_squared_error"]),
+)
+print("R2", np.median(ridge_reg_scores["test_r2"]))
 
 # %% [markdown]
 # ### Lasso
@@ -239,12 +252,26 @@ lasso_reg_score = cross_validate(
     groups=data_groups,
     cv=logo,
     n_jobs=-1,
-    scoring=('r2', 'neg_mean_squared_error', 'neg_mean_absolute_error', 'neg_root_mean_squared_error')
+    scoring=(
+        "r2",
+        "neg_mean_squared_error",
+        "neg_mean_absolute_error",
+        "neg_root_mean_squared_error",
+    ),
 )
-print("Negative Mean Squared Error", np.median(lasso_reg_score['test_neg_mean_squared_error']))
-print("Negative Mean Absolute Error", np.median(lasso_reg_score['test_neg_mean_absolute_error']))
-print("Negative Root Mean Squared Error", np.median(lasso_reg_score['test_neg_root_mean_squared_error']))
-print("R2", np.median(lasso_reg_score['test_r2']))
+print(
+    "Negative Mean Squared Error",
+    np.median(lasso_reg_score["test_neg_mean_squared_error"]),
+)
+print(
+    "Negative Mean Absolute Error",
+    np.median(lasso_reg_score["test_neg_mean_absolute_error"]),
+)
+print(
+    "Negative Root Mean Squared Error",
+    np.median(lasso_reg_score["test_neg_root_mean_squared_error"]),
+)
+print("R2", np.median(lasso_reg_score["test_r2"]))
 
 # %% [markdown]
 # ### Bayesian Ridge
@@ -260,12 +287,26 @@ bayesian_ridge_reg_score = cross_validate(
     groups=data_groups,
     cv=logo,
     n_jobs=-1,
-    scoring=('r2', 'neg_mean_squared_error', 'neg_mean_absolute_error', 'neg_root_mean_squared_error')
+    scoring=(
+        "r2",
+        "neg_mean_squared_error",
+        "neg_mean_absolute_error",
+        "neg_root_mean_squared_error",
+    ),
 )
-print("Negative Mean Squared Error", np.median(bayesian_ridge_reg_score['test_neg_mean_squared_error']))
-print("Negative Mean Absolute Error", np.median(bayesian_ridge_reg_score['test_neg_mean_absolute_error']))
-print("Negative Root Mean Squared Error", np.median(bayesian_ridge_reg_score['test_neg_root_mean_squared_error']))
-print("R2", np.median(bayesian_ridge_reg_score['test_r2']))
+print(
+    "Negative Mean Squared Error",
+    np.median(bayesian_ridge_reg_score["test_neg_mean_squared_error"]),
+)
+print(
+    "Negative Mean Absolute Error",
+    np.median(bayesian_ridge_reg_score["test_neg_mean_absolute_error"]),
+)
+print(
+    "Negative Root Mean Squared Error",
+    np.median(bayesian_ridge_reg_score["test_neg_root_mean_squared_error"]),
+)
+print("R2", np.median(bayesian_ridge_reg_score["test_r2"]))
 
 # %% [markdown]
 # ### RANSAC (outlier robust regression)
@@ -281,12 +322,26 @@ ransac_reg_scores = cross_validate(
     groups=data_groups,
     cv=logo,
     n_jobs=-1,
-    scoring=('r2', 'neg_mean_squared_error', 'neg_mean_absolute_error', 'neg_root_mean_squared_error')
+    scoring=(
+        "r2",
+        "neg_mean_squared_error",
+        "neg_mean_absolute_error",
+        "neg_root_mean_squared_error",
+    ),
 )
-print("Negative Mean Squared Error", np.median(ransac_reg_scores['test_neg_mean_squared_error']))
-print("Negative Mean Absolute Error", np.median(ransac_reg_scores['test_neg_mean_absolute_error']))
-print("Negative Root Mean Squared Error", np.median(ransac_reg_scores['test_neg_root_mean_squared_error']))
-print("R2", np.median(ransac_reg_scores['test_r2']))
+print(
+    "Negative Mean Squared Error",
+    np.median(ransac_reg_scores["test_neg_mean_squared_error"]),
+)
+print(
+    "Negative Mean Absolute Error",
+    np.median(ransac_reg_scores["test_neg_mean_absolute_error"]),
+)
+print(
+    "Negative Root Mean Squared Error",
+    np.median(ransac_reg_scores["test_neg_root_mean_squared_error"]),
+)
+print("R2", np.median(ransac_reg_scores["test_r2"]))
 
 # %% [markdown]
 # ### Support vector regression
@@ -302,12 +357,25 @@ svr_scores = cross_validate(
     groups=data_groups,
     cv=logo,
     n_jobs=-1,
-    scoring=('r2', 'neg_mean_squared_error', 'neg_mean_absolute_error', 'neg_root_mean_squared_error')
+    scoring=(
+        "r2",
+        "neg_mean_squared_error",
+        "neg_mean_absolute_error",
+        "neg_root_mean_squared_error",
+    ),
 )
-print("Negative Mean Squared Error", np.median(svr_scores['test_neg_mean_squared_error']))
-print("Negative Mean Absolute Error", np.median(svr_scores['test_neg_mean_absolute_error']))
-print("Negative Root Mean Squared Error", np.median(svr_scores['test_neg_root_mean_squared_error']))
-print("R2", np.median(svr_scores['test_r2']))
+print(
+    "Negative Mean Squared Error", np.median(svr_scores["test_neg_mean_squared_error"])
+)
+print(
+    "Negative Mean Absolute Error",
+    np.median(svr_scores["test_neg_mean_absolute_error"]),
+)
+print(
+    "Negative Root Mean Squared Error",
+    np.median(svr_scores["test_neg_root_mean_squared_error"]),
+)
+print("R2", np.median(svr_scores["test_r2"]))
 
 # %% [markdown]
 # ### Kernel Ridge regression
@@ -323,12 +391,26 @@ kridge_scores = cross_validate(
     groups=data_groups,
     cv=logo,
     n_jobs=-1,
-    scoring=('r2', 'neg_mean_squared_error', 'neg_mean_absolute_error', 'neg_root_mean_squared_error')
+    scoring=(
+        "r2",
+        "neg_mean_squared_error",
+        "neg_mean_absolute_error",
+        "neg_root_mean_squared_error",
+    ),
 )
-print("Negative Mean Squared Error", np.median(kridge_scores['test_neg_mean_squared_error']))
-print("Negative Mean Absolute Error", np.median(kridge_scores['test_neg_mean_absolute_error']))
-print("Negative Root Mean Squared Error", np.median(kridge_scores['test_neg_root_mean_squared_error']))
-print("R2", np.median(kridge_scores['test_r2']))
+print(
+    "Negative Mean Squared Error",
+    np.median(kridge_scores["test_neg_mean_squared_error"]),
+)
+print(
+    "Negative Mean Absolute Error",
+    np.median(kridge_scores["test_neg_mean_absolute_error"]),
+)
+print(
+    "Negative Root Mean Squared Error",
+    np.median(kridge_scores["test_neg_root_mean_squared_error"]),
+)
+print("R2", np.median(kridge_scores["test_r2"]))
 
 # %% [markdown]
 # ### Gaussian Process Regression
@@ -345,11 +427,24 @@ gpr_scores = cross_validate(
     groups=data_groups,
     cv=logo,
     n_jobs=-1,
-    scoring=('r2', 'neg_mean_squared_error', 'neg_mean_absolute_error', 'neg_root_mean_squared_error')
+    scoring=(
+        "r2",
+        "neg_mean_squared_error",
+        "neg_mean_absolute_error",
+        "neg_root_mean_squared_error",
+    ),
 )
-print("Negative Mean Squared Error", np.median(gpr_scores['test_neg_mean_squared_error']))
-print("Negative Mean Absolute Error", np.median(gpr_scores['test_neg_mean_absolute_error']))
-print("Negative Root Mean Squared Error", np.median(gpr_scores['test_neg_root_mean_squared_error']))
-print("R2", np.median(gpr_scores['test_r2']))
+print(
+    "Negative Mean Squared Error", np.median(gpr_scores["test_neg_mean_squared_error"])
+)
+print(
+    "Negative Mean Absolute Error",
+    np.median(gpr_scores["test_neg_mean_absolute_error"]),
+)
+print(
+    "Negative Root Mean Squared Error",
+    np.median(gpr_scores["test_neg_root_mean_squared_error"]),
+)
+print("R2", np.median(gpr_scores["test_r2"]))
 
 # %%

machine_learning/helper.py

@@ -1,15 +1,18 @@
 from pathlib import Path
-from sklearn import linear_model, svm, kernel_ridge, gaussian_process, ensemble, naive_bayes, neighbors, tree
-from sklearn.model_selection import LeaveOneGroupOut, cross_validate, cross_validate
-from sklearn.metrics import mean_squared_error, r2_score
-from sklearn.impute import SimpleImputer
-from sklearn.dummy import DummyRegressor, DummyClassifier
 
-from xgboost import XGBRegressor, XGBClassifier
-import xgboost as xg
-
-import pandas as pd
 import numpy as np
+import pandas as pd
+from sklearn import (
+    ensemble,
+    gaussian_process,
+    kernel_ridge,
+    linear_model,
+    naive_bayes,
+    svm,
+)
+from sklearn.dummy import DummyClassifier, DummyRegressor
+from sklearn.model_selection import LeaveOneGroupOut, cross_validate
+from xgboost import XGBClassifier, XGBRegressor
 
 
 def safe_outer_merge_on_index(left: pd.DataFrame, right: pd.DataFrame) -> pd.DataFrame:
@@ -65,28 +68,64 @@ def construct_full_path(folder: Path, filename_prefix: str, data_type: str) -> P
     full_path = folder / export_filename
     return full_path
 
+
 def insert_row(df, row):
     return pd.concat([df, pd.DataFrame([row], columns=df.columns)], ignore_index=True)
 
-def prepare_regression_model_input(input_csv):
 
-    model_input = pd.read_csv(input_csv)
-
-    index_columns = ["local_segment", "local_segment_label", "local_segment_start_datetime", "local_segment_end_datetime"]
+def prepare_regression_model_input(model_input, cv_method="logo"):
+    index_columns = [
+        "local_segment",
+        "local_segment_label",
+        "local_segment_start_datetime",
+        "local_segment_end_datetime",
+    ]
     model_input.set_index(index_columns, inplace=True)
 
-    data_x, data_y, data_groups = model_input.drop(["target", "pid"], axis=1), model_input["target"], model_input["pid"]
+    if cv_method == "logo":
+        data_x, data_y, data_groups = (
+            model_input.drop(["target", "pid"], axis=1),
+            model_input["target"],
+            model_input["pid"],
+        )
+    else:
+        model_input["pid_index"] = model_input.groupby("pid").cumcount()
+        model_input["pid_count"] = model_input.groupby("pid")["pid"].transform("count")
 
-    categorical_feature_colnames = ["gender", "startlanguage", "limesurvey_demand_control_ratio_quartile"]
-    additional_categorical_features = [col for col in data_x.columns if "mostcommonactivity" in col or "homelabel" in col]
+        model_input["pid_index"] = (
+            model_input["pid_index"] / model_input["pid_count"] + 1
+        ).round()
+        model_input["pid_half"] = (
+            model_input["pid"] + "_" + model_input["pid_index"].astype(int).astype(str)
+        )
+
+        data_x, data_y, data_groups = (
+            model_input.drop(["target", "pid", "pid_index", "pid_half"], axis=1),
+            model_input["target"],
+            model_input["pid_half"],
+        )
+
+    categorical_feature_colnames = [
+        "gender",
+        "startlanguage",
+        "limesurvey_demand_control_ratio_quartile",
+    ]
+    additional_categorical_features = [
+        col
+        for col in data_x.columns
+        if "mostcommonactivity" in col or "homelabel" in col
+    ]
     categorical_feature_colnames += additional_categorical_features
-    #TODO: check whether limesurvey_demand_control_ratio_quartile NaNs could be replaced meaningfully
+
     categorical_features = data_x[categorical_feature_colnames].copy()
+
     mode_categorical_features = categorical_features.mode().iloc[0]
     # fillna with mode
     categorical_features = categorical_features.fillna(mode_categorical_features)
     # one-hot encoding
-    categorical_features = categorical_features.apply(lambda col: col.astype("category"))
+    categorical_features = categorical_features.apply(
+        lambda col: col.astype("category")
+    )
     if not categorical_features.empty:
         categorical_features = pd.get_dummies(categorical_features)
 
@@ -108,7 +147,7 @@ def run_all_regression_models(input_csv):
         data_y,
         groups=data_groups,
     )
-    metrics = ['r2', 'neg_mean_absolute_error', 'neg_root_mean_squared_error']
+    metrics = ["r2", "neg_mean_absolute_error", "neg_root_mean_squared_error"]
     test_metrics = ["test_" + metric for metric in metrics]
     scores = pd.DataFrame(columns=["method", "max", "nanmedian"])
 
@@ -121,13 +160,13 @@ def run_all_regression_models(input_csv):
         groups=data_groups,
         cv=logo,
         n_jobs=-1,
-        scoring=metrics
+        scoring=metrics,
     )
     print("Dummy model:")
-    print("R^2: ", np.nanmedian(dummy_regr_scores['test_r2']))
+    print("R^2: ", np.nanmedian(dummy_regr_scores["test_r2"]))
 
     scores_df = pd.DataFrame(dummy_regr_scores)[test_metrics]
-    scores_df = scores_df.agg(['max', np.nanmedian]).transpose()
+    scores_df = scores_df.agg(["max", np.nanmedian]).transpose()
     scores_df["method"] = "dummy"
     scores = pd.concat([scores, scores_df])
 
@@ -139,17 +178,17 @@ def run_all_regression_models(input_csv):
         groups=data_groups,
         cv=logo,
         n_jobs=-1,
-        scoring=metrics
+        scoring=metrics,
     )
     print("Linear regression:")
-    print("R^2: ", np.nanmedian(lin_reg_scores['test_r2']))
+    print("R^2: ", np.nanmedian(lin_reg_scores["test_r2"]))
 
     scores_df = pd.DataFrame(lin_reg_scores)[test_metrics]
-    scores_df = scores_df.agg(['max', np.nanmedian]).transpose()
+    scores_df = scores_df.agg(["max", np.nanmedian]).transpose()
     scores_df["method"] = "linear_reg"
     scores = pd.concat([scores, scores_df])
 
-    ridge_reg = linear_model.Ridge(alpha=.5)
+    ridge_reg = linear_model.Ridge(alpha=0.5)
     ridge_reg_scores = cross_validate(
         ridge_reg,
         X=data_x,
@@ -157,16 +196,15 @@ def run_all_regression_models(input_csv):
         groups=data_groups,
         cv=logo,
         n_jobs=-1,
-        scoring=metrics
+        scoring=metrics,
     )
     print("Ridge regression")
 
     scores_df = pd.DataFrame(ridge_reg_scores)[test_metrics]
-    scores_df = scores_df.agg(['max', np.nanmedian]).transpose()
+    scores_df = scores_df.agg(["max", np.nanmedian]).transpose()
     scores_df["method"] = "ridge_reg"
     scores = pd.concat([scores, scores_df])
 
-
     lasso_reg = linear_model.Lasso(alpha=0.1)
     lasso_reg_score = cross_validate(
         lasso_reg,
@@ -175,12 +213,12 @@ def run_all_regression_models(input_csv):
         groups=data_groups,
         cv=logo,
         n_jobs=-1,
-        scoring=metrics
+        scoring=metrics,
     )
     print("Lasso regression")
 
     scores_df = pd.DataFrame(lasso_reg_score)[test_metrics]
-    scores_df = scores_df.agg(['max', np.nanmedian]).transpose()
+    scores_df = scores_df.agg(["max", np.nanmedian]).transpose()
     scores_df["method"] = "lasso_reg"
     scores = pd.concat([scores, scores_df])
 
@@ -192,12 +230,12 @@ def run_all_regression_models(input_csv):
         groups=data_groups,
         cv=logo,
         n_jobs=-1,
-        scoring=metrics
+        scoring=metrics,
     )
     print("Bayesian Ridge")
 
     scores_df = pd.DataFrame(bayesian_ridge_reg_score)[test_metrics]
-    scores_df = scores_df.agg(['max', np.nanmedian]).transpose()
+    scores_df = scores_df.agg(["max", np.nanmedian]).transpose()
     scores_df["method"] = "bayesian_ridge"
     scores = pd.concat([scores, scores_df])
 
@@ -209,29 +247,23 @@ def run_all_regression_models(input_csv):
         groups=data_groups,
         cv=logo,
         n_jobs=-1,
-        scoring=metrics
+        scoring=metrics,
     )
     print("RANSAC (outlier robust regression)")
 
     scores_df = pd.DataFrame(ransac_reg_score)[test_metrics]
-    scores_df = scores_df.agg(['max', np.nanmedian]).transpose()
+    scores_df = scores_df.agg(["max", np.nanmedian]).transpose()
     scores_df["method"] = "RANSAC"
     scores = pd.concat([scores, scores_df])
 
     svr = svm.SVR()
     svr_score = cross_validate(
-        svr,
-        X=data_x,
-        y=data_y,
-        groups=data_groups,
-        cv=logo,
-        n_jobs=-1,
-        scoring=metrics
+        svr, X=data_x, y=data_y, groups=data_groups, cv=logo, n_jobs=-1, scoring=metrics
     )
     print("Support vector regression")
 
     scores_df = pd.DataFrame(svr_score)[test_metrics]
-    scores_df = scores_df.agg(['max', np.nanmedian]).transpose()
+    scores_df = scores_df.agg(["max", np.nanmedian]).transpose()
     scores_df["method"] = "SVR"
     scores = pd.concat([scores, scores_df])
 
@@ -243,80 +275,56 @@ def run_all_regression_models(input_csv):
         groups=data_groups,
         cv=logo,
         n_jobs=-1,
-        scoring=metrics
+        scoring=metrics,
     )
     print("Kernel Ridge regression")
 
     scores_df = pd.DataFrame(kridge_score)[test_metrics]
-    scores_df = scores_df.agg(['max', np.nanmedian]).transpose()
+    scores_df = scores_df.agg(["max", np.nanmedian]).transpose()
     scores_df["method"] = "kernel_ridge"
     scores = pd.concat([scores, scores_df])
 
     gpr = gaussian_process.GaussianProcessRegressor()
     gpr_score = cross_validate(
-        gpr,
-        X=data_x,
-        y=data_y,
-        groups=data_groups,
-        cv=logo,
-        n_jobs=-1,
-        scoring=metrics
+        gpr, X=data_x, y=data_y, groups=data_groups, cv=logo, n_jobs=-1, scoring=metrics
     )
     print("Gaussian Process Regression")
 
     scores_df = pd.DataFrame(gpr_score)[test_metrics]
-    scores_df = scores_df.agg(['max', np.nanmedian]).transpose()
+    scores_df = scores_df.agg(["max", np.nanmedian]).transpose()
     scores_df["method"] = "gaussian_proc"
     scores = pd.concat([scores, scores_df])
 
     rfr = ensemble.RandomForestRegressor(max_features=0.3, n_jobs=-1)
     rfr_score = cross_validate(
-        rfr,
-        X=data_x,
-        y=data_y,
-        groups=data_groups,
-        cv=logo,
-        n_jobs=-1,
-        scoring=metrics
+        rfr, X=data_x, y=data_y, groups=data_groups, cv=logo, n_jobs=-1, scoring=metrics
     )
     print("Random Forest Regression")
 
     scores_df = pd.DataFrame(rfr_score)[test_metrics]
-    scores_df = scores_df.agg(['max', np.nanmedian]).transpose()
+    scores_df = scores_df.agg(["max", np.nanmedian]).transpose()
     scores_df["method"] = "random_forest"
     scores = pd.concat([scores, scores_df])
 
     xgb = XGBRegressor()
     xgb_score = cross_validate(
-        xgb,
-        X=data_x,
-        y=data_y,
-        groups=data_groups,
-        cv=logo,
-        n_jobs=-1,
-        scoring=metrics
+        xgb, X=data_x, y=data_y, groups=data_groups, cv=logo, n_jobs=-1, scoring=metrics
     )
     print("XGBoost Regressor")
 
     scores_df = pd.DataFrame(xgb_score)[test_metrics]
-    scores_df = scores_df.agg(['max', np.nanmedian]).transpose()
+    scores_df = scores_df.agg(["max", np.nanmedian]).transpose()
     scores_df["method"] = "XGBoost"
     scores = pd.concat([scores, scores_df])
 
     ada = ensemble.AdaBoostRegressor()
     ada_score = cross_validate(
-        ada,
-        X=data_x,
-        y=data_y,
-        groups=data_groups,
-        cv=logo,
-        n_jobs=-1,
-        scoring=metrics
+        ada, X=data_x, y=data_y, groups=data_groups, cv=logo, n_jobs=-1, scoring=metrics
     )
     print("ADA Boost Regressor")
 
     scores_df = pd.DataFrame(ada_score)[test_metrics]
-    scores_df = scores_df.agg(['max', np.nanmedian]).transpose()
+    scores_df = scores_df.agg(["max", np.nanmedian]).transpose()
     scores_df["method"] = "ADA_boost"
     scores = pd.concat([scores, scores_df])
 
@@ -324,7 +332,7 @@ def run_all_regression_models(input_csv):
 
 
 def run_all_classification_models(data_x, data_y, data_groups, cv_method):
-    metrics = ['accuracy', 'average_precision', 'recall', 'f1']
+    metrics = ["accuracy", "average_precision", "recall", "f1"]
     test_metrics = ["test_" + metric for metric in metrics]
 
     scores = pd.DataFrame(columns=["method", "max", "mean"])
@@ -338,13 +346,13 @@ def run_all_classification_models(data_x, data_y, data_groups, cv_method):
         groups=data_groups,
         cv=cv_method,
         n_jobs=-1,
-    error_score='raise',
-    scoring=metrics
+        error_score="raise",
+        scoring=metrics,
     )
     print("Dummy")
 
     scores_df = pd.DataFrame(dummy_score)[test_metrics]
-    scores_df = scores_df.agg(['max', 'mean']).transpose()
+    scores_df = scores_df.agg(["max", "mean"]).transpose()
     scores_df["method"] = "Dummy"
     scores = pd.concat([scores, scores_df])
 
@@ -357,12 +365,12 @@ def run_all_classification_models(data_x, data_y, data_groups, cv_method):
         groups=data_groups,
         cv=cv_method,
         n_jobs=-1,
-    scoring=metrics
+        scoring=metrics,
     )
     print("Logistic regression")
 
     scores_df = pd.DataFrame(log_reg_scores)[test_metrics]
-    scores_df = scores_df.agg(['max', 'mean']).transpose()
+    scores_df = scores_df.agg(["max", "mean"]).transpose()
     scores_df["method"] = "logistic_reg"
     scores = pd.concat([scores, scores_df])
 
@@ -375,12 +383,12 @@ def run_all_classification_models(data_x, data_y, data_groups, cv_method):
         groups=data_groups,
         cv=cv_method,
         n_jobs=-1,
-    scoring=metrics
+        scoring=metrics,
     )
     print("Support Vector Machine")
 
     scores_df = pd.DataFrame(svc_scores)[test_metrics]
-    scores_df = scores_df.agg(['max', 'mean']).transpose()
+    scores_df = scores_df.agg(["max", "mean"]).transpose()
     scores_df["method"] = "svc"
     scores = pd.concat([scores, scores_df])
 
@@ -393,12 +401,12 @@ def run_all_classification_models(data_x, data_y, data_groups, cv_method):
         groups=data_groups,
         cv=cv_method,
         n_jobs=-1,
-    scoring=metrics
+        scoring=metrics,
     )
     print("Gaussian Naive Bayes")
 
     scores_df = pd.DataFrame(gaussian_nb_scores)[test_metrics]
-    scores_df = scores_df.agg(['max', 'mean']).transpose()
+    scores_df = scores_df.agg(["max", "mean"]).transpose()
     scores_df["method"] = "gaussian_naive_bayes"
     scores = pd.concat([scores, scores_df])
 
@@ -411,12 +419,12 @@ def run_all_classification_models(data_x, data_y, data_groups, cv_method):
         groups=data_groups,
         cv=cv_method,
         n_jobs=-1,
-    scoring=metrics
+        scoring=metrics,
     )
     print("Stochastic Gradient Descent")
 
     scores_df = pd.DataFrame(sgdc_scores)[test_metrics]
-    scores_df = scores_df.agg(['max', 'mean']).transpose()
+    scores_df = scores_df.agg(["max", "mean"]).transpose()
     scores_df["method"] = "stochastic_gradient_descent"
     scores = pd.concat([scores, scores_df])
 
@@ -429,12 +437,12 @@ def run_all_classification_models(data_x, data_y, data_groups, cv_method):
         groups=data_groups,
         cv=cv_method,
         n_jobs=-1,
-    scoring=metrics
+        scoring=metrics,
     )
     print("Random Forest")
 
     scores_df = pd.DataFrame(rfc_scores)[test_metrics]
-    scores_df = scores_df.agg(['max', 'mean']).transpose()
+    scores_df = scores_df.agg(["max", "mean"]).transpose()
     scores_df["method"] = "random_forest"
     scores = pd.concat([scores, scores_df])
 
@@ -447,12 +455,12 @@ def run_all_classification_models(data_x, data_y, data_groups, cv_method):
         groups=data_groups,
         cv=cv_method,
         n_jobs=-1,
-    scoring=metrics
+        scoring=metrics,
     )
     print("XGBoost")
 
     scores_df = pd.DataFrame(xgb_scores)[test_metrics]
-    scores_df = scores_df.agg(['max', 'mean']).transpose()
+    scores_df = scores_df.agg(["max", "mean"]).transpose()
     scores_df["method"] = "xgboost"
     scores = pd.concat([scores, scores_df])
 

presentation/ApplicationCategories.R

@@ -34,18 +34,114 @@ df_app_categories <- tbl(con, "app_categories") %>%
 head(df_app_categories)
 table(df_app_categories$play_store_genre)
 
+df_app_categories %>%
+  filter(play_store_genre == "not_found") %>%
+  group_by(play_store_response) %>%
+  count()
+# All "not_found" have an HTTP status of 404.
+
+df_app_categories %>%
+  filter(play_store_genre == "not_found") %>%
+  group_by(package_name) %>%
+  count() %>%
+  arrange(desc(n))
+# All "not_found" apps are unique.
+
+# Exclude phone manufacturers, custom ROM names and similar.
+manufacturers <- c(
+  "samsung",
+  "oneplus",
+  "huawei",
+  "xiaomi",
+  "lge",
+  "motorola",
+  "miui",
+  "lenovo",
+  "oppo",
+  "mediatek"
+)
+custom_rom <- c("coloros", "lineageos", "myos", "cyanogenmod", "foundation.e")
+other <- c("android", "wssyncmldm")
+
+grep_pattern <- paste(c(manufacturers, custom_rom, other), collapse = "|")
+
+rows_os_manufacturer <- grepl(grep_pattern, df_app_categories$package_name)
+
+# Explore what remains after excluding above.
+df_app_categories[!rows_os_manufacturer, ] %>%
+  filter(play_store_genre == "not_found")
+
+# Also check the relationship between is_system_app and System category.
+tbl(con, "applications") %>% 
+  filter(is_system_app, play_store_genre != "System") %>% 
+  count()
+# They are perfectly correlated.
+
+# Manually classify apps
+df_app_categories[df_app_categories$play_store_genre == "not_found",] <- 
+  df_app_categories %>% 
+  filter(play_store_genre == "not_found") %>% 
+  mutate(
+    play_store_genre =
+      case_when(
+        str_detect(str_to_lower(package_name), grep_pattern) ~ "System",
+        str_detect(str_to_lower(package_name), "straw") ~ "STRAW",
+        str_detect(str_to_lower(package_name), "chromium") ~ "Communication", # Same as chrome.
+        str_detect(str_to_lower(package_name), "skype") ~ "Communication", # Skype Lite not classified.
+        str_detect(str_to_lower(package_name), "imsservice") ~ "Communication", # IP Multimedia Subsystem
+        str_detect(str_to_lower(package_name), paste(c("covid", "empatica"), collapse = "|")) ~ "Medical",
+        str_detect(str_to_lower(package_name), paste(c("libri", "tachiyomi"), collapse = "|")) ~ "Books & Reference",
+        str_detect(str_to_lower(package_name), paste(c("bricks", "chess"), collapse = "|")) ~ "Casual",
+        str_detect(str_to_lower(package_name), "weather") ~ "Weather",
+        str_detect(str_to_lower(package_name), "excel") ~ "Productivity",
+        str_detect(str_to_lower(package_name), paste(c("qr", "barcode", "archimedes", "mixplorer", "winrar", "filemanager", "shot", "faceunlock", "signin", "milink"), collapse = "|")) ~ "Tools",
+        str_detect(str_to_lower(package_name), "stupeflix") ~ "Photography",
+        str_detect(str_to_lower(package_name), "anyme") ~ "Entertainment",
+        str_detect(str_to_lower(package_name), "vanced") ~ "Video Players & Editors",
+        str_detect(str_to_lower(package_name), paste(c("music", "radio", "dolby"), collapse = "|")) ~ "Music & Audio",
+        str_detect(str_to_lower(package_name), paste(c("tensorflow", "object_detection"), collapse = "|")) ~ "Education",
+        .default = play_store_genre
+      )
+  )
+
+# Explore what remains after classifying above.
+df_app_categories %>%
+  filter(play_store_genre == "not_found")
+
+# After this, 13 applications remain, which I will classify as "Other".
+
 # Correct some mistakes
-df_app_categories %<>% mutate(
+# And classify 'not_found'
+df_app_categories %<>%
+  mutate(
     play_store_genre = {
       function(x) {
         case_when(
           x == "Education,Education" ~ "Education",
           x == "EducationEducation" ~ "Education",
-        x == "not_found" ~ "System",
+          x == "not_found" ~ "Other",
           .default = x
         )
       }
     }(play_store_genre)
+  ) %>%
+  select(-package_name) %>%
+  rename(
+    genre = play_store_genre,
+    package_name = package_hash
+  )
+
+table(df_app_categories$genre)
+
+df_app_categories %>%
+  group_by(genre) %>%
+  count() %>%
+  arrange(desc(n)) %>%
+  write_csv("play_store_categories_count.csv")
+
+write_csv(
+  x = select(df_app_categories, c(package_name, genre)),
+  file = "play_store_application_genre_catalogue.csv"
 )
 
 dbDisconnect(con)

presentation/play_store_categories_count.csv

@@ -0,0 +1,45 @@
+genre,n
+System,261
+Tools,96
+Productivity,71
+Health & Fitness,60
+Finance,54
+Communication,39
+Music & Audio,39
+Shopping,38
+Lifestyle,33
+Education,28
+News & Magazines,24
+Maps & Navigation,23
+Entertainment,21
+Business,18
+Travel & Local,18
+Books & Reference,16
+Social,16
+Weather,16
+Food & Drink,14
+Sports,14
+Other,13
+Photography,13
+Puzzle,13
+Video Players & Editors,12
+Card,9
+Casual,9
+Personalization,8
+Medical,7
+Board,5
+Strategy,4
+House & Home,3
+Trivia,3
+Word,3
+Adventure,2
+Art & Design,2
+Auto & Vehicles,2
+Dating,2
+Role Playing,2
+STRAW,2
+Simulation,2
+"Board,Brain Games",1
+"Entertainment,Music & Video",1
+Parenting,1
+Racing,1

rapids

@@ -1 +1 @@
-Subproject commit 03687a1ac204f0a4347eb758dada8005f68b0bb1
+Subproject commit 63f5a526fce4d288499168e1701adadb8b885d82