Add hr summary to fitbitjson_mysql
JulioV
parent
72f6b2d621
commit
47f449555a
|
|
@ -226,8 +226,7 @@ for provider in config["FITBIT_DATA_YIELD"]["PROVIDERS"].keys():
|
|||
for provider in config["FITBIT_HEARTRATE_SUMMARY"]["PROVIDERS"].keys():
|
||||
if config["FITBIT_HEARTRATE_SUMMARY"]["PROVIDERS"][provider]["COMPUTE"]:
|
||||
files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_summary_raw.csv", pid=config["PIDS"]))
|
||||
files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_summary_parsed.csv", pid=config["PIDS"]))
|
||||
files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_summary_parsed_with_datetime.csv", pid=config["PIDS"]))
|
||||
files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_summary_with_datetime.csv", pid=config["PIDS"]))
|
||||
files_to_compute.extend(expand("data/interim/{pid}/fitbit_heartrate_summary_features/fitbit_heartrate_summary_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["FITBIT_HEARTRATE_SUMMARY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower()))
|
||||
files_to_compute.extend(expand("data/processed/features/{pid}/fitbit_heartrate_summary.csv", pid=config["PIDS"]))
|
||||
files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"]))
|
||||
|
|
|
|||
|
|
@ -32,20 +32,33 @@ If you want RAPIDS to process Fitbit sensor data using this stream, you will nee
|
|||
| HEARTRATE_DAILY_CALORIESFATBURN | FLAG_TO_MUTATE |
|
||||
| HEARTRATE_DAILY_CALORIESCARDIO | FLAG_TO_MUTATE |
|
||||
| HEARTRATE_DAILY_CALORIESPEAK | FLAG_TO_MUTATE |
|
||||
| FLAG_AS_EXTRA: | fitbit_data |
|
||||
|
||||
|
||||
**MUTATION_SCRIPTS**
|
||||
**MUTATION**
|
||||
|
||||
TODO list our parsing script
|
||||
- **COLUMN_MAPPINGS**
|
||||
|
||||
??? "Example of the raw data RAPIDS expects for this data stream"
|
||||
| Script column | Stream column |
|
||||
|-----------------|-----------------|
|
||||
| JSON_FITBIT_COLUMN | fitbit_data |
|
||||
|
||||
- **SCRIPTS**
|
||||
|
||||
```bash
|
||||
src/data/streams/mutations/fitbit/parse_heartrate_summary_json.py
|
||||
```
|
||||
|
||||
|device_id |fitbit_data |
|
||||
|---------------------------------------- |--------------------------------------------------------- |
|
||||
|a748ee1a-1d0b-4ae9-9074-279a2b6ba524 |{"activities-heart":[{"dateTime":"2020-10-07","value":{"customHeartRateZones":[],"heartRateZones":[{"caloriesOut":1200.6102,"max":88,"min":31,"minutes":1058,"name":"Out of Range"},{"caloriesOut":760.3020,"max":120,"min":86,"minutes":366,"name":"Fat Burn"},{"caloriesOut":15.2048,"max":146,"min":120,"minutes":2,"name":"Cardio"},{"caloriesOut":0,"max":221,"min":148,"minutes":0,"name":"Peak"}],"restingHeartRate":72}}],"activities-heart-intraday":{"dataset":[{"time":"00:00:00","value":68},{"time":"00:01:00","value":67},{"time":"00:02:00","value":67},...],"datasetInterval":1,"datasetType":"minute"}}
|
||||
|a748ee1a-1d0b-4ae9-9074-279a2b6ba524 |{"activities-heart":[{"dateTime":"2020-10-08","value":{"customHeartRateZones":[],"heartRateZones":[{"caloriesOut":1100.1120,"max":89,"min":30,"minutes":921,"name":"Out of Range"},{"caloriesOut":660.0012,"max":118,"min":82,"minutes":361,"name":"Fat Burn"},{"caloriesOut":23.7088,"max":142,"min":108,"minutes":3,"name":"Cardio"},{"caloriesOut":0,"max":221,"min":148,"minutes":0,"name":"Peak"}],"restingHeartRate":70}}],"activities-heart-intraday":{"dataset":[{"time":"00:00:00","value":77},{"time":"00:01:00","value":75},{"time":"00:02:00","value":73},...],"datasetInterval":1,"datasetType":"minute"}}
|
||||
|a748ee1a-1d0b-4ae9-9074-279a2b6ba524 |{"activities-heart":[{"dateTime":"2020-10-09","value":{"customHeartRateZones":[],"heartRateZones":[{"caloriesOut":750.3615,"max":77,"min":30,"minutes":851,"name":"Out of Range"},{"caloriesOut":734.1516,"max":107,"min":77,"minutes":550,"name":"Fat Burn"},{"caloriesOut":131.8579,"max":130,"min":107,"minutes":29,"name":"Cardio"},{"caloriesOut":0,"max":220,"min":130,"minutes":0,"name":"Peak"}],"restingHeartRate":69}}],"activities-heart-intraday":{"dataset":[{"time":"00:00:00","value":90},{"time":"00:01:00","value":89},{"time":"00:02:00","value":88},...],"datasetInterval":1,"datasetType":"minute"}}
|
||||
!!! note
|
||||
All columns except `DEVICE_ID` are parsed from `JSON_FITBIT_COLUMN`. `JSON_FITBIT_COLUMN` is a string column containing the JSON objects returned by Fitbit's API. See an example of the raw data RAPIDS expects for this data stream:
|
||||
|
||||
|
||||
??? "Example of the raw data RAPIDS expects for this data stream"
|
||||
|
||||
|device_id |fitbit_data |
|
||||
|---------------------------------------- |--------------------------------------------------------- |
|
||||
|a748ee1a-1d0b-4ae9-9074-279a2b6ba524 |{"activities-heart":[{"dateTime":"2020-10-07","value":{"customHeartRateZones":[],"heartRateZones":[{"caloriesOut":1200.6102,"max":88,"min":31,"minutes":1058,"name":"Out of Range"},{"caloriesOut":760.3020,"max":120,"min":86,"minutes":366,"name":"Fat Burn"},{"caloriesOut":15.2048,"max":146,"min":120,"minutes":2,"name":"Cardio"},{"caloriesOut":0,"max":221,"min":148,"minutes":0,"name":"Peak"}],"restingHeartRate":72}}],"activities-heart-intraday":{"dataset":[{"time":"00:00:00","value":68},{"time":"00:01:00","value":67},{"time":"00:02:00","value":67},...],"datasetInterval":1,"datasetType":"minute"}}
|
||||
|a748ee1a-1d0b-4ae9-9074-279a2b6ba524 |{"activities-heart":[{"dateTime":"2020-10-08","value":{"customHeartRateZones":[],"heartRateZones":[{"caloriesOut":1100.1120,"max":89,"min":30,"minutes":921,"name":"Out of Range"},{"caloriesOut":660.0012,"max":118,"min":82,"minutes":361,"name":"Fat Burn"},{"caloriesOut":23.7088,"max":142,"min":108,"minutes":3,"name":"Cardio"},{"caloriesOut":0,"max":221,"min":148,"minutes":0,"name":"Peak"}],"restingHeartRate":70}}],"activities-heart-intraday":{"dataset":[{"time":"00:00:00","value":77},{"time":"00:01:00","value":75},{"time":"00:02:00","value":73},...],"datasetInterval":1,"datasetType":"minute"}}
|
||||
|a748ee1a-1d0b-4ae9-9074-279a2b6ba524 |{"activities-heart":[{"dateTime":"2020-10-09","value":{"customHeartRateZones":[],"heartRateZones":[{"caloriesOut":750.3615,"max":77,"min":30,"minutes":851,"name":"Out of Range"},{"caloriesOut":734.1516,"max":107,"min":77,"minutes":550,"name":"Fat Burn"},{"caloriesOut":131.8579,"max":130,"min":107,"minutes":29,"name":"Cardio"},{"caloriesOut":0,"max":220,"min":130,"minutes":0,"name":"Peak"}],"restingHeartRate":69}}],"activities-heart-intraday":{"dataset":[{"time":"00:00:00","value":90},{"time":"00:01:00","value":89},{"time":"00:02:00","value":88},...],"datasetInterval":1,"datasetType":"minute"}}
|
||||
|
||||
??? info "FITBIT_STEPS_SUMMARY"
|
||||
|
||||
|
|
|
|||
|
|
@ -6,13 +6,14 @@ This is a description of the format RAPIDS needs to process data for the followi
|
|||
|
||||
| RAPIDS column | Description |
|
||||
|-----------------|-----------------|
|
||||
| TIMESTAMP | An UNIX timestamp (13 digits) when a row of data was logged |
|
||||
| LOCAL_DATE_TIME | Date time string with format `yyyy-mm-dd hh:mm:ss` |
|
||||
| DEVICE_ID | A string that uniquely identifies a device |
|
||||
| HEARTRATE_DAILY_RESTINGHR | TODO |
|
||||
| HEARTRATE_DAILY_CALORIESOUTOFRANGE | TODO |
|
||||
| HEARTRATE_DAILY_CALORIESFATBURN | TODO |
|
||||
| HEARTRATE_DAILY_CALORIESCARDIO | TODO |
|
||||
| HEARTRATE_DAILY_CALORIESPEAK | TODO |
|
||||
| HEARTRATE_DAILY_RESTINGHR | Daily resting heartrate |
|
||||
| HEARTRATE_DAILY_CALORIESOUTOFRANGE | Calories spent while heartrate was oustide a heartrate [zone](https://help.fitbit.com/articles/en_US/Help_article/1565.htm#) |
|
||||
| HEARTRATE_DAILY_CALORIESFATBURN | Calories spent while heartrate was inside the fat burn [zone](https://help.fitbit.com/articles/en_US/Help_article/1565.htm#) |
|
||||
| HEARTRATE_DAILY_CALORIESCARDIO | Calories spent while heartrate was inside the cardio [zone](https://help.fitbit.com/articles/en_US/Help_article/1565.htm#) |
|
||||
| HEARTRATE_DAILY_CALORIESPEAK | Calories spent while heartrate was inside the peak [zone](https://help.fitbit.com/articles/en_US/Help_article/1565.htm#) |
|
||||
|
||||
??? info "FITBIT_STEPS_SUMMARY"
|
||||
|
||||
|
|
|
|||
|
|
@ -532,7 +532,7 @@ rule fitbit_data_yield_r_features:
|
|||
|
||||
rule fitbit_heartrate_summary_python_features:
|
||||
input:
|
||||
sensor_data = "data/raw/{pid}/fitbit_heartrate_summary_parsed_with_datetime.csv",
|
||||
sensor_data = "data/raw/{pid}/fitbit_heartrate_summary_with_datetime.csv",
|
||||
time_segments_labels = "data/interim/time_segments/{pid}_time_segments_labels.csv"
|
||||
params:
|
||||
provider = lambda wildcards: config["FITBIT_HEARTRATE_SUMMARY"]["PROVIDERS"][wildcards.provider_key.upper()],
|
||||
|
|
@ -545,7 +545,7 @@ rule fitbit_heartrate_summary_python_features:
|
|||
|
||||
rule fitbit_heartrate_summary_r_features:
|
||||
input:
|
||||
sensor_data = "data/raw/{pid}/fitbit_heartrate_summary_parsed_with_datetime.csv",
|
||||
sensor_data = "data/raw/{pid}/fitbit_heartrate_summary_with_datetime.csv",
|
||||
time_segments_labels = "data/interim/time_segments/{pid}_time_segments_labels.csv"
|
||||
params:
|
||||
provider = lambda wildcards: config["FITBIT_HEARTRATE_SUMMARY"]["PROVIDERS"][wildcards.provider_key.upper()],
|
||||
|
|
|
|||
|
|
@ -1,3 +1,19 @@
|
|||
FITBIT_HEARTRATE_SUMMARY:
|
||||
RAPIDS_COLUMN_MAPPINGS:
|
||||
TIMESTAMP: FLAG_TO_MUTATE
|
||||
DEVICE_ID: device_id
|
||||
LOCAL_DATE_TIME: FLAG_TO_MUTATE
|
||||
HEARTRATE_DAILY_RESTINGHR: FLAG_TO_MUTATE
|
||||
HEARTRATE_DAILY_CALORIESOUTOFRANGE: FLAG_TO_MUTATE
|
||||
HEARTRATE_DAILY_CALORIESFATBURN: FLAG_TO_MUTATE
|
||||
HEARTRATE_DAILY_CALORIESCARDIO: FLAG_TO_MUTATE
|
||||
HEARTRATE_DAILY_CALORIESPEAK: FLAG_TO_MUTATE
|
||||
MUTATION:
|
||||
COLUMN_MAPPINGS:
|
||||
JSON_FITBIT_COLUMN: fitbit_data # text column with JSON objects
|
||||
SCRIPTS: # List any python or r scripts that mutate your raw data
|
||||
- src/data/streams/mutations/fitbit/parse_heartrate_summary_json.py
|
||||
|
||||
FITBIT_STEPS_SUMMARY:
|
||||
RAPIDS_COLUMN_MAPPINGS:
|
||||
TIMESTAMP: FLAG_TO_MUTATE
|
||||
|
|
|
|||
|
|
@ -0,0 +1,74 @@
|
|||
import yaml, json, sys
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from datetime import datetime, timezone
|
||||
from math import trunc
|
||||
|
||||
|
||||
HR_SUMMARY_COLUMNS = ("device_id",
|
||||
"local_date_time",
|
||||
"timestamp",
|
||||
"heartrate_daily_restinghr",
|
||||
"heartrate_daily_caloriesoutofrange",
|
||||
"heartrate_daily_caloriesfatburn",
|
||||
"heartrate_daily_caloriescardio",
|
||||
"heartrate_daily_caloriespeak")
|
||||
|
||||
|
||||
def parseHeartrateSummaryData(record_summary, device_id, curr_date):
|
||||
# API Version X: not sure the exact version
|
||||
if "heartRateZones" in record_summary:
|
||||
heartrate_zones = record_summary["heartRateZones"]
|
||||
d_resting_heartrate = record_summary["value"] if "value" in record_summary else None
|
||||
# API VERSION Y: not sure the exact version
|
||||
elif "value" in record_summary:
|
||||
heartrate_zones = record_summary["value"]["heartRateZones"]
|
||||
d_resting_heartrate = record_summary["value"]["restingHeartRate"] if "restingHeartRate" in record_summary["value"] else None
|
||||
else:
|
||||
ValueError("Heartrate zone are stored in an unkown format, this could mean Fitbit's heartrate API changed")
|
||||
|
||||
if "caloriesOut" in heartrate_zones[0]:
|
||||
d_calories_outofrange = heartrate_zones[0]["caloriesOut"]
|
||||
d_calories_fatburn = heartrate_zones[1]["caloriesOut"]
|
||||
d_calories_cardio = heartrate_zones[2]["caloriesOut"]
|
||||
d_calories_peak = heartrate_zones[3]["caloriesOut"]
|
||||
else:
|
||||
d_calories_outofrange, d_calories_fatburn, d_calories_cardio, d_calories_peak = None, None, None, None
|
||||
|
||||
row_summary = (device_id,
|
||||
curr_date,
|
||||
0,
|
||||
d_resting_heartrate,
|
||||
d_calories_outofrange,
|
||||
d_calories_fatburn,
|
||||
d_calories_cardio,
|
||||
d_calories_peak)
|
||||
return row_summary
|
||||
|
||||
def parseHeartrateData(heartrate_data):
|
||||
if heartrate_data.empty:
|
||||
return pd.DataFrame(columns=HR_SUMMARY_COLUMNS)
|
||||
|
||||
device_id = heartrate_data["device_id"].iloc[0]
|
||||
records_summary = []
|
||||
|
||||
|
||||
# Parse JSON into individual records
|
||||
for record in heartrate_data.json_fitbit_column:
|
||||
record = json.loads(record) # Parse text into JSON
|
||||
if "activities-heart" in record:
|
||||
curr_date = datetime.strptime(record["activities-heart"][0]["dateTime"], "%Y-%m-%d")
|
||||
|
||||
record_summary = record["activities-heart"][0]
|
||||
row_summary = parseHeartrateSummaryData(record_summary, device_id, curr_date)
|
||||
records_summary.append(row_summary)
|
||||
parsed_data = pd.DataFrame(data=records_summary, columns=HR_SUMMARY_COLUMNS)
|
||||
|
||||
return parsed_data
|
||||
|
||||
|
||||
def main(json_raw, stream_parameters):
|
||||
parsed_data = parseHeartrateData(json_raw)
|
||||
parsed_data["timestamp"] = 0 # this column is added at readable_datetime.R because we neeed to take into account multiple timezones
|
||||
parsed_data['local_date_time'] = parsed_data['local_date_time'].dt.strftime('%Y-%m-%d %H:%M:%S')
|
||||
return(parsed_data)
|
||||
|
|
@ -92,6 +92,16 @@ PHONE_WIFI_VISIBLE:
|
|||
- FREQUENCY
|
||||
- RSSI
|
||||
|
||||
FITBIT_HEARTRATE_SUMMARY:
|
||||
- TIMESTAMP
|
||||
- DEVICE_ID
|
||||
- LOCAL_DATE_TIME
|
||||
- HEARTRATE_DAILY_RESTINGHR
|
||||
- HEARTRATE_DAILY_CALORIESOUTOFRANGE
|
||||
- HEARTRATE_DAILY_CALORIESFATBURN
|
||||
- HEARTRATE_DAILY_CALORIESCARDIO
|
||||
- HEARTRATE_DAILY_CALORIESPEAK
|
||||
|
||||
FITBIT_STEPS_SUMMARY:
|
||||
- TIMESTAMP
|
||||
- DEVICE_ID
|
||||
|
|
|
|||
Loading…
Reference in New Issue