From 48259623612d862276a2e622076ea88e42ca4e2d Mon Sep 17 00:00:00 2001 From: Meng Li <34143965+Meng6@users.noreply.github.com> Date: Wed, 10 Mar 2021 00:15:06 -0500 Subject: [PATCH] Migrate phone screen sensor to new data stream --- Snakefile | 1 - docs/datastreams/aware-mysql.md | 50 +++++++++++++++++++ docs/datastreams/mandatory-phone-format.md | 9 ++++ docs/features/phone-screen.md | 1 - rules/features.smk | 2 +- src/data/streams/aware_mysql/format.yaml | 22 ++++++++ .../phone/aware/screen_ios_unification.R | 17 +++++++ src/data/streams/rapids_columns.yaml | 5 ++ 8 files changed, 104 insertions(+), 3 deletions(-) create mode 100644 src/data/streams/mutations/phone/aware/screen_ios_unification.R diff --git a/Snakefile b/Snakefile index 15e32de3..56d89a6b 100644 --- a/Snakefile +++ b/Snakefile @@ -91,7 +91,6 @@ for provider in config["PHONE_SCREEN"]["PROVIDERS"].keys(): # raise ValueError("Error: Add PHONE_SCREEN (and as many PHONE_SENSORS as you have in your database) to [PHONE_DATA_YIELD][SENSORS] in config.yaml. This is necessary to compute phone_yielded_timestamps (time when the smartphone was sensing data)") files_to_compute.extend(expand("data/raw/{pid}/phone_screen_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_screen_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/raw/{pid}/phone_screen_with_datetime_unified.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_screen_episodes.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_screen_episodes_resampled.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_screen_episodes_resampled_with_datetime.csv", pid=config["PIDS"])) diff --git a/docs/datastreams/aware-mysql.md b/docs/datastreams/aware-mysql.md index 11c6059b..d0ef0467 100644 --- a/docs/datastreams/aware-mysql.md +++ b/docs/datastreams/aware-mysql.md @@ -418,3 +418,53 @@ Stream columns named `FLAG_TO_MUTATE` means they are extracted based on the `MUT === "IOS" This sensor is not supported by iOS devices. + + +??? info "PHONE_SCREEN" + + === "ANDROID" + + **RAPIDS_COLUMN_MAPPINGS** + + | RAPIDS column | Stream column | + |----------------------|---------------------| + | TIMESTAMP | timestamp | + | DEVICE_ID | device_id | + | SCREEN_STATUS | screen_status | + + **MUTATION** + + - **COLUMN_MAPPINGS** (None) + - **SCRIPTS** (None) + + === "IOS" + + **RAPIDS_COLUMN_MAPPINGS** + + | RAPIDS column | Stream column | + |----------------------|---------------------| + | TIMESTAMP | timestamp | + | DEVICE_ID | device_id | + | SCREEN_STATUS | FLAG_TO_MUTATE | + + **MUTATION** + + - **COLUMN_MAPPINGS** + + | Script column | Stream column | + |----------------------|---------------------| + | SCREEN_STATUS | screen_status | + + - **SCRIPTS** + + ```bash + src/data/streams/mutations/phone/aware/screen_ios_unification.R + ``` + + !!! note + For `SCREEN_STATUS` RAPIDS column: + + - if stream's `screen_status` field is 2 (lock episode), set `SCREEN_STATUS` = 0 (off episode). + + + diff --git a/docs/datastreams/mandatory-phone-format.md b/docs/datastreams/mandatory-phone-format.md index f6f27c5c..92c4ee29 100644 --- a/docs/datastreams/mandatory-phone-format.md +++ b/docs/datastreams/mandatory-phone-format.md @@ -114,3 +114,12 @@ This is a description of the format RAPIDS needs to process data for the followi | MESSAGE_TYPE | An integer that denotes message type: 1 = received, 2 = sent | | TRACE | SHA-1 one-way source/target of the message | + +??? info "PHONE_SCREEN" + + | RAPIDS column | Description | + |--------------------|-----------------------------------------------------------------------------------| + | TIMESTAMP | An UNIX timestamp (13 digits) when a row of data was logged | + | DEVICE_ID | A string that uniquely identifies a device | + | SCREEN_STATUS | An integer that denotes screen status: 0 = off, 1 = on, 2 = locked, 3 = unlocked | + diff --git a/docs/features/phone-screen.md b/docs/features/phone-screen.md index a08cc632..96cee1bd 100644 --- a/docs/features/phone-screen.md +++ b/docs/features/phone-screen.md @@ -16,7 +16,6 @@ Sensor parameters description for `[PHONE_SCREEN]`: ```bash - data/raw/{pid}/phone_screen_raw.csv - data/raw/{pid}/phone_screen_with_datetime.csv - - data/raw/{pid}/phone_screen_with_datetime_unified.csv - data/interim/{pid}/phone_screen_episodes.csv - data/interim/{pid}/phone_screen_episodes_resampled.csv - data/interim/{pid}/phone_screen_episodes_resampled_with_datetime.csv diff --git a/rules/features.smk b/rules/features.smk index 94548a75..07b3600e 100644 --- a/rules/features.smk +++ b/rules/features.smk @@ -420,7 +420,7 @@ rule phone_messages_r_features: rule screen_episodes: input: - screen = "data/raw/{pid}/phone_screen_with_datetime_unified.csv" + screen = "data/raw/{pid}/phone_screen_with_datetime.csv" output: "data/interim/{pid}/phone_screen_episodes.csv" script: diff --git a/src/data/streams/aware_mysql/format.yaml b/src/data/streams/aware_mysql/format.yaml index 52640cf0..33235959 100644 --- a/src/data/streams/aware_mysql/format.yaml +++ b/src/data/streams/aware_mysql/format.yaml @@ -193,3 +193,25 @@ PHONE_MESSAGES: MUTATION: COLUMN_MAPPINGS: SCRIPTS: # List any python or r scripts that mutate your raw data + +PHONE_SCREEN: + ANDROID: + RAPIDS_COLUMN_MAPPINGS: + TIMESTAMP: timestamp + DEVICE_ID: device_id + SCREEN_STATUS: screen_status + MUTATION: + COLUMN_MAPPINGS: + SCRIPTS: # List any python or r scripts that mutate your raw data + IOS: + RAPIDS_COLUMN_MAPPINGS: + TIMESTAMP: timestamp + DEVICE_ID: device_id + SCREEN_STATUS: FLAG_TO_MUTATE + MUTATION: + COLUMN_MAPPINGS: + SCREEN_STATUS: screen_status + SCRIPTS: # List any python or r scripts that mutate your raw data + - "src/data/streams/mutations/phone/aware/screen_ios_unification.R" + + diff --git a/src/data/streams/mutations/phone/aware/screen_ios_unification.R b/src/data/streams/mutations/phone/aware/screen_ios_unification.R new file mode 100644 index 00000000..5ef5c9be --- /dev/null +++ b/src/data/streams/mutations/phone/aware/screen_ios_unification.R @@ -0,0 +1,17 @@ +source("renv/activate.R") +library("dplyr", warn.conflicts = F) + + +unify_ios_screen <- function(ios_screen){ + # In Android we only process UNLOCK to OFF episodes. In iOS we only process UNLOCK to LOCKED episodes, + # thus, we replace LOCKED with OFF episodes (2 to 0) so we can use Android's code for iOS + ios_screen <- ios_screen %>% + # only keep consecutive pairs of 3,2 events + filter( (screen_status == 3 & lead(screen_status) == 2) | (screen_status == 2 & lag(screen_status) == 3) ) %>% + mutate(screen_status = replace(screen_status, screen_status == 2, 0)) + return(ios_screen) +} + +main <- function(data, stream_parameters){ + return(unify_ios_screen(data)) +} \ No newline at end of file diff --git a/src/data/streams/rapids_columns.yaml b/src/data/streams/rapids_columns.yaml index 9ab899e1..7d93bd88 100644 --- a/src/data/streams/rapids_columns.yaml +++ b/src/data/streams/rapids_columns.yaml @@ -71,6 +71,11 @@ PHONE_MESSAGES: - MESSAGE_TYPE - TRACE +PHONE_SCREEN: + - TIMESTAMP + - DEVICE_ID + - SCREEN_STATUS + FITBIT_STEPS_SUMMARY: - TIMESTAMP