From 47f449555a6c31fe4a3c89b74b73bd24c176b057 Mon Sep 17 00:00:00 2001 From: JulioV Date: Wed, 10 Mar 2021 12:07:24 -0500 Subject: [PATCH] Add hr summary to fitbitjson_mysql --- Snakefile | 3 +- docs/datastreams/fitbitjson-mysql.md | 31 +++++--- docs/datastreams/mandatory-fitbit-format.md | 11 +-- rules/features.smk | 4 +- src/data/streams/fitbitjson_mysql/format.yaml | 16 ++++ .../fitbit/parse_heartrate_summary_json.py | 74 +++++++++++++++++++ src/data/streams/rapids_columns.yaml | 10 +++ 7 files changed, 131 insertions(+), 18 deletions(-) create mode 100644 src/data/streams/mutations/fitbit/parse_heartrate_summary_json.py diff --git a/Snakefile b/Snakefile index d8288f73..3b57d270 100644 --- a/Snakefile +++ b/Snakefile @@ -226,8 +226,7 @@ for provider in config["FITBIT_DATA_YIELD"]["PROVIDERS"].keys(): for provider in config["FITBIT_HEARTRATE_SUMMARY"]["PROVIDERS"].keys(): if config["FITBIT_HEARTRATE_SUMMARY"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_summary_raw.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_summary_parsed.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_summary_parsed_with_datetime.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_summary_with_datetime.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/fitbit_heartrate_summary_features/fitbit_heartrate_summary_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["FITBIT_HEARTRATE_SUMMARY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/fitbit_heartrate_summary.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) diff --git a/docs/datastreams/fitbitjson-mysql.md b/docs/datastreams/fitbitjson-mysql.md index b25ae2d8..4d654497 100644 --- a/docs/datastreams/fitbitjson-mysql.md +++ b/docs/datastreams/fitbitjson-mysql.md @@ -32,20 +32,33 @@ If you want RAPIDS to process Fitbit sensor data using this stream, you will nee | HEARTRATE_DAILY_CALORIESFATBURN | FLAG_TO_MUTATE | | HEARTRATE_DAILY_CALORIESCARDIO | FLAG_TO_MUTATE | | HEARTRATE_DAILY_CALORIESPEAK | FLAG_TO_MUTATE | - | FLAG_AS_EXTRA: | fitbit_data | - **MUTATION_SCRIPTS** + **MUTATION** - TODO list our parsing script + - **COLUMN_MAPPINGS** - ??? "Example of the raw data RAPIDS expects for this data stream" + | Script column | Stream column | + |-----------------|-----------------| + | JSON_FITBIT_COLUMN | fitbit_data | + + - **SCRIPTS** + + ```bash + src/data/streams/mutations/fitbit/parse_heartrate_summary_json.py + ``` - |device_id |fitbit_data | - |---------------------------------------- |--------------------------------------------------------- | - |a748ee1a-1d0b-4ae9-9074-279a2b6ba524 |{"activities-heart":[{"dateTime":"2020-10-07","value":{"customHeartRateZones":[],"heartRateZones":[{"caloriesOut":1200.6102,"max":88,"min":31,"minutes":1058,"name":"Out of Range"},{"caloriesOut":760.3020,"max":120,"min":86,"minutes":366,"name":"Fat Burn"},{"caloriesOut":15.2048,"max":146,"min":120,"minutes":2,"name":"Cardio"},{"caloriesOut":0,"max":221,"min":148,"minutes":0,"name":"Peak"}],"restingHeartRate":72}}],"activities-heart-intraday":{"dataset":[{"time":"00:00:00","value":68},{"time":"00:01:00","value":67},{"time":"00:02:00","value":67},...],"datasetInterval":1,"datasetType":"minute"}} - |a748ee1a-1d0b-4ae9-9074-279a2b6ba524 |{"activities-heart":[{"dateTime":"2020-10-08","value":{"customHeartRateZones":[],"heartRateZones":[{"caloriesOut":1100.1120,"max":89,"min":30,"minutes":921,"name":"Out of Range"},{"caloriesOut":660.0012,"max":118,"min":82,"minutes":361,"name":"Fat Burn"},{"caloriesOut":23.7088,"max":142,"min":108,"minutes":3,"name":"Cardio"},{"caloriesOut":0,"max":221,"min":148,"minutes":0,"name":"Peak"}],"restingHeartRate":70}}],"activities-heart-intraday":{"dataset":[{"time":"00:00:00","value":77},{"time":"00:01:00","value":75},{"time":"00:02:00","value":73},...],"datasetInterval":1,"datasetType":"minute"}} - |a748ee1a-1d0b-4ae9-9074-279a2b6ba524 |{"activities-heart":[{"dateTime":"2020-10-09","value":{"customHeartRateZones":[],"heartRateZones":[{"caloriesOut":750.3615,"max":77,"min":30,"minutes":851,"name":"Out of Range"},{"caloriesOut":734.1516,"max":107,"min":77,"minutes":550,"name":"Fat Burn"},{"caloriesOut":131.8579,"max":130,"min":107,"minutes":29,"name":"Cardio"},{"caloriesOut":0,"max":220,"min":130,"minutes":0,"name":"Peak"}],"restingHeartRate":69}}],"activities-heart-intraday":{"dataset":[{"time":"00:00:00","value":90},{"time":"00:01:00","value":89},{"time":"00:02:00","value":88},...],"datasetInterval":1,"datasetType":"minute"}} + !!! note + All columns except `DEVICE_ID` are parsed from `JSON_FITBIT_COLUMN`. `JSON_FITBIT_COLUMN` is a string column containing the JSON objects returned by Fitbit's API. See an example of the raw data RAPIDS expects for this data stream: + + + ??? "Example of the raw data RAPIDS expects for this data stream" + + |device_id |fitbit_data | + |---------------------------------------- |--------------------------------------------------------- | + |a748ee1a-1d0b-4ae9-9074-279a2b6ba524 |{"activities-heart":[{"dateTime":"2020-10-07","value":{"customHeartRateZones":[],"heartRateZones":[{"caloriesOut":1200.6102,"max":88,"min":31,"minutes":1058,"name":"Out of Range"},{"caloriesOut":760.3020,"max":120,"min":86,"minutes":366,"name":"Fat Burn"},{"caloriesOut":15.2048,"max":146,"min":120,"minutes":2,"name":"Cardio"},{"caloriesOut":0,"max":221,"min":148,"minutes":0,"name":"Peak"}],"restingHeartRate":72}}],"activities-heart-intraday":{"dataset":[{"time":"00:00:00","value":68},{"time":"00:01:00","value":67},{"time":"00:02:00","value":67},...],"datasetInterval":1,"datasetType":"minute"}} + |a748ee1a-1d0b-4ae9-9074-279a2b6ba524 |{"activities-heart":[{"dateTime":"2020-10-08","value":{"customHeartRateZones":[],"heartRateZones":[{"caloriesOut":1100.1120,"max":89,"min":30,"minutes":921,"name":"Out of Range"},{"caloriesOut":660.0012,"max":118,"min":82,"minutes":361,"name":"Fat Burn"},{"caloriesOut":23.7088,"max":142,"min":108,"minutes":3,"name":"Cardio"},{"caloriesOut":0,"max":221,"min":148,"minutes":0,"name":"Peak"}],"restingHeartRate":70}}],"activities-heart-intraday":{"dataset":[{"time":"00:00:00","value":77},{"time":"00:01:00","value":75},{"time":"00:02:00","value":73},...],"datasetInterval":1,"datasetType":"minute"}} + |a748ee1a-1d0b-4ae9-9074-279a2b6ba524 |{"activities-heart":[{"dateTime":"2020-10-09","value":{"customHeartRateZones":[],"heartRateZones":[{"caloriesOut":750.3615,"max":77,"min":30,"minutes":851,"name":"Out of Range"},{"caloriesOut":734.1516,"max":107,"min":77,"minutes":550,"name":"Fat Burn"},{"caloriesOut":131.8579,"max":130,"min":107,"minutes":29,"name":"Cardio"},{"caloriesOut":0,"max":220,"min":130,"minutes":0,"name":"Peak"}],"restingHeartRate":69}}],"activities-heart-intraday":{"dataset":[{"time":"00:00:00","value":90},{"time":"00:01:00","value":89},{"time":"00:02:00","value":88},...],"datasetInterval":1,"datasetType":"minute"}} ??? info "FITBIT_STEPS_SUMMARY" diff --git a/docs/datastreams/mandatory-fitbit-format.md b/docs/datastreams/mandatory-fitbit-format.md index 2c52bae6..ab86601f 100644 --- a/docs/datastreams/mandatory-fitbit-format.md +++ b/docs/datastreams/mandatory-fitbit-format.md @@ -6,13 +6,14 @@ This is a description of the format RAPIDS needs to process data for the followi | RAPIDS column | Description | |-----------------|-----------------| + | TIMESTAMP | An UNIX timestamp (13 digits) when a row of data was logged | | LOCAL_DATE_TIME | Date time string with format `yyyy-mm-dd hh:mm:ss` | | DEVICE_ID | A string that uniquely identifies a device | - | HEARTRATE_DAILY_RESTINGHR | TODO | - | HEARTRATE_DAILY_CALORIESOUTOFRANGE | TODO | - | HEARTRATE_DAILY_CALORIESFATBURN | TODO | - | HEARTRATE_DAILY_CALORIESCARDIO | TODO | - | HEARTRATE_DAILY_CALORIESPEAK | TODO | + | HEARTRATE_DAILY_RESTINGHR | Daily resting heartrate | + | HEARTRATE_DAILY_CALORIESOUTOFRANGE | Calories spent while heartrate was oustide a heartrate [zone](https://help.fitbit.com/articles/en_US/Help_article/1565.htm#) | + | HEARTRATE_DAILY_CALORIESFATBURN | Calories spent while heartrate was inside the fat burn [zone](https://help.fitbit.com/articles/en_US/Help_article/1565.htm#) | + | HEARTRATE_DAILY_CALORIESCARDIO | Calories spent while heartrate was inside the cardio [zone](https://help.fitbit.com/articles/en_US/Help_article/1565.htm#) | + | HEARTRATE_DAILY_CALORIESPEAK | Calories spent while heartrate was inside the peak [zone](https://help.fitbit.com/articles/en_US/Help_article/1565.htm#) | ??? info "FITBIT_STEPS_SUMMARY" diff --git a/rules/features.smk b/rules/features.smk index a2b0045d..397d8ed1 100644 --- a/rules/features.smk +++ b/rules/features.smk @@ -532,7 +532,7 @@ rule fitbit_data_yield_r_features: rule fitbit_heartrate_summary_python_features: input: - sensor_data = "data/raw/{pid}/fitbit_heartrate_summary_parsed_with_datetime.csv", + sensor_data = "data/raw/{pid}/fitbit_heartrate_summary_with_datetime.csv", time_segments_labels = "data/interim/time_segments/{pid}_time_segments_labels.csv" params: provider = lambda wildcards: config["FITBIT_HEARTRATE_SUMMARY"]["PROVIDERS"][wildcards.provider_key.upper()], @@ -545,7 +545,7 @@ rule fitbit_heartrate_summary_python_features: rule fitbit_heartrate_summary_r_features: input: - sensor_data = "data/raw/{pid}/fitbit_heartrate_summary_parsed_with_datetime.csv", + sensor_data = "data/raw/{pid}/fitbit_heartrate_summary_with_datetime.csv", time_segments_labels = "data/interim/time_segments/{pid}_time_segments_labels.csv" params: provider = lambda wildcards: config["FITBIT_HEARTRATE_SUMMARY"]["PROVIDERS"][wildcards.provider_key.upper()], diff --git a/src/data/streams/fitbitjson_mysql/format.yaml b/src/data/streams/fitbitjson_mysql/format.yaml index 5c055293..31ea979d 100644 --- a/src/data/streams/fitbitjson_mysql/format.yaml +++ b/src/data/streams/fitbitjson_mysql/format.yaml @@ -1,3 +1,19 @@ +FITBIT_HEARTRATE_SUMMARY: + RAPIDS_COLUMN_MAPPINGS: + TIMESTAMP: FLAG_TO_MUTATE + DEVICE_ID: device_id + LOCAL_DATE_TIME: FLAG_TO_MUTATE + HEARTRATE_DAILY_RESTINGHR: FLAG_TO_MUTATE + HEARTRATE_DAILY_CALORIESOUTOFRANGE: FLAG_TO_MUTATE + HEARTRATE_DAILY_CALORIESFATBURN: FLAG_TO_MUTATE + HEARTRATE_DAILY_CALORIESCARDIO: FLAG_TO_MUTATE + HEARTRATE_DAILY_CALORIESPEAK: FLAG_TO_MUTATE + MUTATION: + COLUMN_MAPPINGS: + JSON_FITBIT_COLUMN: fitbit_data # text column with JSON objects + SCRIPTS: # List any python or r scripts that mutate your raw data + - src/data/streams/mutations/fitbit/parse_heartrate_summary_json.py + FITBIT_STEPS_SUMMARY: RAPIDS_COLUMN_MAPPINGS: TIMESTAMP: FLAG_TO_MUTATE diff --git a/src/data/streams/mutations/fitbit/parse_heartrate_summary_json.py b/src/data/streams/mutations/fitbit/parse_heartrate_summary_json.py new file mode 100644 index 00000000..daab3b5b --- /dev/null +++ b/src/data/streams/mutations/fitbit/parse_heartrate_summary_json.py @@ -0,0 +1,74 @@ +import yaml, json, sys +import pandas as pd +import numpy as np +from datetime import datetime, timezone +from math import trunc + + +HR_SUMMARY_COLUMNS = ("device_id", + "local_date_time", + "timestamp", + "heartrate_daily_restinghr", + "heartrate_daily_caloriesoutofrange", + "heartrate_daily_caloriesfatburn", + "heartrate_daily_caloriescardio", + "heartrate_daily_caloriespeak") + + +def parseHeartrateSummaryData(record_summary, device_id, curr_date): + # API Version X: not sure the exact version + if "heartRateZones" in record_summary: + heartrate_zones = record_summary["heartRateZones"] + d_resting_heartrate = record_summary["value"] if "value" in record_summary else None + # API VERSION Y: not sure the exact version + elif "value" in record_summary: + heartrate_zones = record_summary["value"]["heartRateZones"] + d_resting_heartrate = record_summary["value"]["restingHeartRate"] if "restingHeartRate" in record_summary["value"] else None + else: + ValueError("Heartrate zone are stored in an unkown format, this could mean Fitbit's heartrate API changed") + + if "caloriesOut" in heartrate_zones[0]: + d_calories_outofrange = heartrate_zones[0]["caloriesOut"] + d_calories_fatburn = heartrate_zones[1]["caloriesOut"] + d_calories_cardio = heartrate_zones[2]["caloriesOut"] + d_calories_peak = heartrate_zones[3]["caloriesOut"] + else: + d_calories_outofrange, d_calories_fatburn, d_calories_cardio, d_calories_peak = None, None, None, None + + row_summary = (device_id, + curr_date, + 0, + d_resting_heartrate, + d_calories_outofrange, + d_calories_fatburn, + d_calories_cardio, + d_calories_peak) + return row_summary + +def parseHeartrateData(heartrate_data): + if heartrate_data.empty: + return pd.DataFrame(columns=HR_SUMMARY_COLUMNS) + + device_id = heartrate_data["device_id"].iloc[0] + records_summary = [] + + + # Parse JSON into individual records + for record in heartrate_data.json_fitbit_column: + record = json.loads(record) # Parse text into JSON + if "activities-heart" in record: + curr_date = datetime.strptime(record["activities-heart"][0]["dateTime"], "%Y-%m-%d") + + record_summary = record["activities-heart"][0] + row_summary = parseHeartrateSummaryData(record_summary, device_id, curr_date) + records_summary.append(row_summary) + parsed_data = pd.DataFrame(data=records_summary, columns=HR_SUMMARY_COLUMNS) + + return parsed_data + + +def main(json_raw, stream_parameters): + parsed_data = parseHeartrateData(json_raw) + parsed_data["timestamp"] = 0 # this column is added at readable_datetime.R because we neeed to take into account multiple timezones + parsed_data['local_date_time'] = parsed_data['local_date_time'].dt.strftime('%Y-%m-%d %H:%M:%S') + return(parsed_data) diff --git a/src/data/streams/rapids_columns.yaml b/src/data/streams/rapids_columns.yaml index f97e9c79..6aea61b3 100644 --- a/src/data/streams/rapids_columns.yaml +++ b/src/data/streams/rapids_columns.yaml @@ -92,6 +92,16 @@ PHONE_WIFI_VISIBLE: - FREQUENCY - RSSI +FITBIT_HEARTRATE_SUMMARY: + - TIMESTAMP + - DEVICE_ID + - LOCAL_DATE_TIME + - HEARTRATE_DAILY_RESTINGHR + - HEARTRATE_DAILY_CALORIESOUTOFRANGE + - HEARTRATE_DAILY_CALORIESFATBURN + - HEARTRATE_DAILY_CALORIESCARDIO + - HEARTRATE_DAILY_CALORIESPEAK + FITBIT_STEPS_SUMMARY: - TIMESTAMP - DEVICE_ID