Rename merge metrics for models and add filter valid sensed days

Co-authored-by: Meng Li <AnnieLM1996@gmail.com>

.gitignore

@@ -107,6 +107,3 @@ reports/
 *.Rproj
 .RData
 .Rhistory
-
-# analysis part
-models/*

Snakefile

@@ -57,11 +57,11 @@ rule all:
                             pid = config["PIDS"],
                             day_segment = config["STEP"]["DAY_SEGMENTS"]),
         # Models
-        expand("models/input/merged_single_participant/{pid}/{source}_{day_segment}.csv",
+        expand("data/processed/{pid}/metrics_for_individual_model/{source}_{day_segment}.csv",
                                 pid = config["PIDS"],
                                 source = config["METRICS_FOR_ANALYSIS"]["SOURCES"],
                                 day_segment = config["METRICS_FOR_ANALYSIS"]["DAY_SEGMENTS"]),
-        expand("models/input/merged_all_participants/{source}_{day_segment}.csv",
+        expand("data/processed/metrics_for_population_model/{source}_{day_segment}.csv",
                                 source = config["METRICS_FOR_ANALYSIS"]["SOURCES"],
                                 day_segment = config["METRICS_FOR_ANALYSIS"]["DAY_SEGMENTS"]),
         # Reports

config.yaml

@@ -130,3 +130,4 @@ METRICS_FOR_ANALYSIS:
   PHONE_METRICS: [accelerometer, applications_foreground, battery, call_incoming, call_missed, call_outgoing, google_activity_recognition, light, location_barnett, screen, sms_received, sms_sent]
   FITBIT_METRICS: [fitbit_heartrate, fitbit_step]
   PHONE_FITBIT_METRICS: "" # This array is merged in the input_merge_features_of_single_participant function in models.snakefile
+  DROP_VALID_SENSED_DAYS: True

rules/models.snakefile

@@ -4,18 +4,22 @@ def input_merge_metrics_of_single_participant(wildcards):
     else:
         return expand("data/processed/{pid}/{metrics}_{day_segment}.csv", pid=wildcards.pid, metrics=config["METRICS_FOR_ANALYSIS"][wildcards.source.upper()], day_segment=wildcards.day_segment)
 
-rule merge_metrics_of_single_participant:
+rule merge_metrics_for_individual_model:
     input:
-        metric_files = input_merge_metrics_of_single_participant
+        metric_files = input_merge_metrics_of_single_participant,
+        phone_valid_sensed_days = "data/interim/{pid}/phone_valid_sensed_days.csv"
+    params:
+        drop_valid_sensed_days = config["METRICS_FOR_ANALYSIS"]["DROP_VALID_SENSED_DAYS"],
+        source = "{source}"
     output:
-        "models/input/merged_single_participant/{pid}/{source}_{day_segment}.csv"
+        "data/processed/{pid}/metrics_for_individual_model/{source}_{day_segment}.csv"
     script:
-        "../src/models/merge_metrics_of_single_participant.R"
+        "../src/models/merge_metrics_for_individual_model.R"
 
-rule merge_metrics_of_all_participants:
+rule merge_metrics_for_population_model:
     input:
-        metric_files = expand("models/input/merged_single_participant/{pid}/{{source}}_{{day_segment}}.csv", pid=config["PIDS"])
+        metric_files = expand("data/processed/{pid}/metrics_for_individual_model/{{source}}_{{day_segment}}.csv", pid=config["PIDS"])
     output:
-        "models/input/merged_all_participants/{source}_{day_segment}.csv" 
+        "data/processed/metrics_for_population_model/{source}_{day_segment}.csv" 
     script:
-        "../src/models/merge_metrics_of_all_participants.R"
+        "../src/models/merge_metrics_for_population_model.R"

src/models/merge_metrics_for_individual_model.R

@@ -0,0 +1,20 @@
+source("packrat/init.R")
+
+library(tidyr)
+library(purrr)
+library(dplyr)
+
+metric_files  <- snakemake@input[["metric_files"]]
+phone_valid_sensed_days  <- read.csv(snakemake@input[["phone_valid_sensed_days"]])
+drop_valid_sensed_days <- snakemake@params[["drop_valid_sensed_days"]]
+source <- snakemake@params[["source"]]
+
+metrics_for_individual_model <- metric_files %>%
+  map(read.csv, stringsAsFactors = F, colClasses = c(local_date = "character")) %>%
+  reduce(full_join, by="local_date")
+
+if(drop_valid_sensed_days && source == "phone_metrics"){
+    metrics_for_individual_model <- merge(metrics_for_individual_model, phone_valid_sensed_days, by="local_date") %>% select(-valid_hours)
+  }
+
+write.csv(metrics_for_individual_model, snakemake@output[[1]], row.names = FALSE)

src/models/merge_metrics_of_single_participant.R

@@ -1,13 +0,0 @@
-source("packrat/init.R")
-
-library(tidyr)
-library(purrr)
-library(dplyr)
-
-metric_files  <- snakemake@input[["metric_files"]]
-
-metrics_of_single_participant <- metric_files %>%
-  map(read.csv, stringsAsFactors = F, colClasses = c(local_date = "character")) %>%
-  reduce(full_join, by="local_date")
-
-write.csv(metrics_of_single_participant, snakemake@output[[1]], row.names = FALSE)