From e952e27350c7ae02703bd444e8f92979e37d9ba6 Mon Sep 17 00:00:00 2001 From: nikunjgoel95 Date: Wed, 27 May 2020 14:29:28 -0400 Subject: [PATCH] Adding the code to parse the FITBIT sleep data for API version 1.2 Co-authored-by: JulioV --- .../fitbit_parse_sleep.py | 107 +++++++++++++++++- 1 file changed, 103 insertions(+), 4 deletions(-) diff --git a/src/data/fitbit_parse_sensors/fitbit_parse_sleep.py b/src/data/fitbit_parse_sensors/fitbit_parse_sleep.py index 9ec8ea74..ad0f1e1a 100644 --- a/src/data/fitbit_parse_sensors/fitbit_parse_sleep.py +++ b/src/data/fitbit_parse_sensors/fitbit_parse_sleep.py @@ -1,7 +1,9 @@ import json import pandas as pd from datetime import datetime - +import numpy as np +import dateutil.parser +from datetime import timedelta SLEEP_CODE2LEVEL = ["asleep", "restless", "awake"] @@ -29,6 +31,37 @@ SLEEP_INTRADAY_COLUMNS = ("device_id", "local_day_of_week", "local_time", "local_hour", "local_minute", "local_day_segment") +def mergeLongAndShortData(data_summary): + longData = pd.DataFrame(columns=['dateTime', 'level', 'seconds']) + shortData = pd.DataFrame(columns=['dateTime','level', 'seconds']) + + windowLength = 30 + + for data in data_summary['data']: + origEntry = data + counter = 0 + numberOfSplits = origEntry['seconds']//windowLength + for times in range(numberOfSplits): + newRow = {'dateTime':dateutil.parser.parse(origEntry['dateTime'])+timedelta(seconds=counter*windowLength),'level':origEntry['level'],'seconds':windowLength} + longData = longData.append(newRow, ignore_index = True) + counter = counter + 1 + + for data in data_summary['shortData']: + origEntry = data + counter = 0 + numberOfSplits = origEntry['seconds']//windowLength + for times in range(numberOfSplits): + newRow = {'dateTime':dateutil.parser.parse(origEntry['dateTime'])+timedelta(seconds=counter*windowLength),'level':origEntry['level'],'seconds':windowLength} + shortData = shortData.append(newRow,ignore_index = True) + counter = counter + 1 + longData.set_index('dateTime',inplace=True) + shortData.set_index('dateTime',inplace=True) + longData['level'] = np.where(longData.index.isin(shortData.index) == True,'wake',longData['level']) + + longData.reset_index(inplace=True) + + return longData.values.tolist() + # Parse one record for sleep API version 1 def parseOneRecordForV1(record, device_id, d_is_main_sleep, records_summary, records_intraday, HOUR2EPOCH): @@ -81,8 +114,74 @@ def parseOneRecordForV1(record, device_id, d_is_main_sleep, records_summary, rec return records_summary, records_intraday # Parse one record for sleep API version 1.2 -def parseOneRecordForV12(record, d_is_main_sleep, records_summary, records_intraday): - return None +def parseOneRecordForV12(record, device_id, d_is_main_sleep, records_summary, records_intraday, HOUR2EPOCH): + + # Summary data + sleep_record_type = record['type'] + + d_start_datetime = datetime.strptime(record["startTime"][:18], "%Y-%m-%dT%H:%M:%S") + d_end_datetime = datetime.strptime(record["endTime"][:18], "%Y-%m-%dT%H:%M:%S") + + row_summary = (device_id, record["efficiency"], + record["minutesAfterWakeup"], record["minutesAsleep"], record["minutesAwake"], record["minutesToFallAsleep"], record["timeInBed"], + d_is_main_sleep, sleep_record_type, + d_start_datetime, d_end_datetime, + d_start_datetime.date(), d_end_datetime.date(), + HOUR2EPOCH[d_start_datetime.hour], HOUR2EPOCH[d_end_datetime.hour]) + + records_summary.append(row_summary) + if sleep_record_type == 'classic': + # Intraday data + start_date = d_start_datetime.date() + end_date = d_end_datetime.date() + is_before_midnight = True + curr_date = start_date + data_summary = record['levels'] + for data in data_summary['data']: + # For overnight episodes, use end_date once we are over midnight + d_time = dateutil.parser.parse(data["dateTime"]).time() + if is_before_midnight and d_time.hour == 0: + curr_date = end_date + d_datetime = datetime.combine(curr_date, d_time) + + d_original_level = data["level"] + + d_unified_level = 0 if d_original_level == "awake" or d_original_level == "restless" else 1 + + row_intraday = (device_id, + d_original_level, d_unified_level, d_is_main_sleep, sleep_record_type, + d_datetime, d_datetime.date(), d_datetime.month, d_datetime.day, + d_datetime.weekday(), d_datetime.time(), d_datetime.hour, d_datetime.minute, + HOUR2EPOCH[d_datetime.hour]) + records_intraday.append(row_intraday) + else: + ## for sleep type "stages" + start_date = d_start_datetime.date() + end_date = d_end_datetime.date() + is_before_midnight = True + curr_date = start_date + data_summary = record['levels'] + dataList = mergeLongAndShortData(data_summary) + for data in dataList: + + d_time = data[0].time() + if is_before_midnight and d_time.hour == 0: + curr_date = end_date + d_datetime = datetime.combine(curr_date, d_time) + + d_original_level = data[1] + + d_unified_level = 1 if d_original_level == "deep" or d_original_level == "light" or d_original_level == "rem" else 0 + + row_intraday = (device_id, + d_original_level, d_unified_level, d_is_main_sleep, sleep_record_type, + d_datetime, d_datetime.date(), d_datetime.month, d_datetime.day, + d_datetime.weekday(), d_datetime.time(), d_datetime.hour, d_datetime.minute, + HOUR2EPOCH[d_datetime.hour]) + + records_intraday.append(row_intraday) + + return records_summary, records_intraday @@ -104,6 +203,6 @@ def parseSleepData(sleep_data, HOUR2EPOCH): # For sleep API version 1.2 else: SLEEP_SUMMARY_COLUMNS = SLEEP_SUMMARY_COLUMNS_V1_2 - raise ValueError("Sleep data for API v1.2 is not supported yet.") + records_summary, records_intraday = parseOneRecordForV12(record, device_id, d_is_main_sleep, records_summary, records_intraday, HOUR2EPOCH) return pd.DataFrame(data=records_summary, columns=SLEEP_SUMMARY_COLUMNS), pd.DataFrame(data=records_intraday, columns=SLEEP_INTRADAY_COLUMNS)