junos
/
rapids
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'data_stream_fix' into develop

pull/143/head
Meng Li 2021-06-11 18:37:30 -04:00
parent 4b8698a4c6 f248b6c97d
commit 339781252a
7 changed files with 13 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/data/streams/mutations/fitbit/parse_calories_intraday_json.py
View File

@ -18,7 +18,7 @@ def parseCaloriesData(calories_data):
dataset = record["activities-calories-intraday"]["dataset"]
for data in dataset:
d_time = datetime.strptime(data["time"], '%H:%M:%S').time()
d_datetime = datetime.combine(curr_date, d_time)
d_datetime = datetime.combine(curr_date, d_time).strftime("%Y-%m-%d %H:%M:%S")
row_intraday = (device_id, data["level"], data["mets"], data["value"], d_datetime, 0)
records_intraday.append(row_intraday)

4
src/data/streams/mutations/fitbit/parse_heartrate_intraday_json.py
View File

@ -32,13 +32,13 @@ def parseHeartrateZones(heartrate_data):
def parseHeartrateIntradayData(records_intraday, dataset, device_id, curr_date, heartrate_zones_range):
for data in dataset:
d_time = datetime.strptime(data["time"], '%H:%M:%S').time()
d_datetime = datetime.combine(curr_date, d_time)
d_datetime = datetime.combine(curr_date, d_time).strftime("%Y-%m-%d %H:%M:%S")
d_hr = data["value"]
# Get heartrate zone by range: min <= heartrate < max
d_hrzone = None
for hrzone, hrrange in heartrate_zones_range.items():
if d_hr >= hrrange[0] and d_hr < hrrange[1]:
if d_hr >= hrrange[0] and d_hr <= hrrange[1]:
d_hrzone = hrzone
break

3
src/data/streams/mutations/fitbit/parse_heartrate_summary_json.py
View File

@ -1,6 +1,5 @@
import json
import pandas as pd
from datetime import datetime
HR_SUMMARY_COLUMNS = ("device_id",
@ -55,7 +54,7 @@ def parseHeartrateData(heartrate_data):
for record in heartrate_data.json_fitbit_column:
record = json.loads(record) # Parse text into JSON
if "activities-heart" in record:
curr_date = datetime.strptime(record["activities-heart"][0]["dateTime"], "%Y-%m-%d")
curr_date = record["activities-heart"][0]["dateTime"] + " 00:00:00"
record_summary = record["activities-heart"][0]
row_summary = parseHeartrateSummaryData(record_summary, device_id, curr_date)

7
src/data/streams/mutations/fitbit/parse_sleep_intraday_json.py
View File

@ -64,7 +64,7 @@ def parseOneRecordForV1(record, device_id, d_is_main_sleep, records_intraday, ty
d_time = datetime.strptime(data["dateTime"], '%H:%M:%S').time()
if is_before_midnight and d_time.hour == 0:
curr_date = end_date
d_datetime = datetime.combine(curr_date, d_time)
d_datetime = datetime.combine(curr_date, d_time).strftime("%Y-%m-%d %H:%M:%S")
# API 1.2 stores original_level as strings, so we convert original_levels of API 1 to strings too
# (1: "asleep", 2: "restless", 3: "awake")
@ -86,7 +86,7 @@ def parseOneRecordForV12(record, device_id, d_is_main_sleep, records_intraday, t
if sleep_record_type == "classic":
for data in record["levels"]["data"]:
d_datetime = dateutil.parser.parse(data["dateTime"])
d_datetime = data["dateTime"][:19].replace("T", " ")
row_intraday = (device_id, type_episode_id, data["seconds"],
data["level"], d_is_main_sleep, sleep_record_type,
@ -95,9 +95,10 @@ def parseOneRecordForV12(record, device_id, d_is_main_sleep, records_intraday, t
else:
# For sleep type "stages"
for data in mergeLongAndShortData(record["levels"]):
d_datetime = data[0].strftime("%Y-%m-%d %H:%M:%S")
row_intraday = (device_id, type_episode_id, 30,
data[1], d_is_main_sleep, sleep_record_type,
data[0], 0)
d_datetime, 0)
records_intraday.append(row_intraday)

9
src/data/streams/mutations/fitbit/parse_sleep_summary_json.py
View File

@ -1,8 +1,5 @@
import json, yaml
import json
import pandas as pd
import numpy as np
from datetime import datetime, timedelta
import dateutil.parser
SLEEP_SUMMARY_COLUMNS = ("device_id", "efficiency",
"minutes_after_wakeup", "minutes_asleep", "minutes_awake", "minutes_to_fall_asleep", "minutes_in_bed",
@ -16,8 +13,8 @@ def parseOneSleepRecord(record, device_id, d_is_main_sleep, records_summary, epi
sleep_record_type = episode_type
d_start_datetime = datetime.strptime(record["startTime"][:18], "%Y-%m-%dT%H:%M:%S")
d_end_datetime = datetime.strptime(record["endTime"][:18], "%Y-%m-%dT%H:%M:%S")
d_start_datetime = record["startTime"][:19].replace("T", " ")
d_end_datetime = record["endTime"][:19].replace("T", " ")
# Summary data
row_summary = (device_id, record["efficiency"],
record["minutesAfterWakeup"], record["minutesAsleep"], record["minutesAwake"], record["minutesToFallAsleep"], record["timeInBed"],

2
src/data/streams/mutations/fitbit/parse_steps_intraday_json.py
View File

@ -23,7 +23,7 @@ def parseStepsData(steps_data):
dataset = record["activities-steps-intraday"]["dataset"]
for data in dataset:
d_time = datetime.strptime(data["time"], '%H:%M:%S').time()
d_datetime = datetime.combine(curr_date, d_time)
d_datetime = datetime.combine(curr_date, d_time).strftime("%Y-%m-%d %H:%M:%S")
row_intraday = (device_id,
data["value"],

3
src/data/streams/mutations/fitbit/parse_steps_summary_json.py
View File

@ -1,6 +1,5 @@
import json
import pandas as pd
from datetime import datetime
STEPS_COLUMNS = ("device_id", "steps", "local_date_time", "timestamp")
@ -16,7 +15,7 @@ def parseStepsData(steps_data):
for record in steps_data.json_fitbit_column:
record = json.loads(record) # Parse text into JSON
if "activities-steps" in record.keys():
curr_date = datetime.strptime(record["activities-steps"][0]["dateTime"], "%Y-%m-%d")
curr_date = record["activities-steps"][0]["dateTime"] + " 00:00:00"
row_summary = (device_id,
record["activities-steps"][0]["value"],