80 lines
3.0 KiB
Python
80 lines
3.0 KiB
Python
import json, yaml
|
|
import pandas as pd
|
|
import numpy as np
|
|
from datetime import datetime, timezone
|
|
from math import trunc
|
|
|
|
STEPS_COLUMNS = ("device_id", "steps", "local_date_time", "timestamp")
|
|
|
|
|
|
def parseStepsData(steps_data, fitbit_data_type):
|
|
if steps_data.empty:
|
|
return pd.DataFrame(), pd.DataFrame(columns=STEPS_INTRADAY_COLUMNS)
|
|
device_id = steps_data["device_id"].iloc[0]
|
|
records_summary, records_intraday = [], []
|
|
|
|
# Parse JSON into individual records
|
|
for record in steps_data.fitbit_data:
|
|
record = json.loads(record) # Parse text into JSON
|
|
curr_date = datetime.strptime(record["activities-steps"][0]["dateTime"], "%Y-%m-%d")
|
|
|
|
# Parse summary data
|
|
if fitbit_data_type == "summary":
|
|
|
|
row_summary = (device_id,
|
|
record["activities-steps"][0]["value"],
|
|
curr_date,
|
|
0)
|
|
|
|
records_summary.append(row_summary)
|
|
|
|
# Parse intraday data
|
|
if fitbit_data_type == "intraday":
|
|
dataset = record["activities-steps-intraday"]["dataset"]
|
|
for data in dataset:
|
|
d_time = datetime.strptime(data["time"], '%H:%M:%S').time()
|
|
d_datetime = datetime.combine(curr_date, d_time)
|
|
|
|
row_intraday = (device_id,
|
|
data["value"],
|
|
d_datetime,
|
|
0)
|
|
|
|
records_intraday.append(row_intraday)
|
|
|
|
if fitbit_data_type == "summary":
|
|
parsed_data = pd.DataFrame(data=records_summary, columns=STEPS_COLUMNS)
|
|
elif fitbit_data_type == "intraday":
|
|
parsed_data = pd.DataFrame(data=records_intraday, columns=STEPS_COLUMNS)
|
|
else:
|
|
raise ValueError("fitbit_data_type can only be one of ['summary', 'intraday'].")
|
|
|
|
return parsed_data
|
|
|
|
|
|
|
|
timezone = snakemake.params["timezone"]
|
|
column_format = snakemake.params["column_format"]
|
|
fitbit_data_type = snakemake.params["fitbit_data_type"]
|
|
|
|
with open(snakemake.input["participant_file"], "r", encoding="utf-8") as f:
|
|
participant_file = yaml.safe_load(f)
|
|
local_start_date = pd.Timestamp(participant_file["FITBIT"]["START_DATE"])
|
|
local_end_date = pd.Timestamp(participant_file["FITBIT"]["END_DATE"]) + pd.DateOffset(1)
|
|
|
|
if column_format == "JSON":
|
|
json_raw = pd.read_csv(snakemake.input["raw_data"])
|
|
parsed_data = parseStepsData(json_raw, fitbit_data_type)
|
|
elif column_format == "PLAIN_TEXT":
|
|
parsed_data = pd.read_csv(snakemake.input["raw_data"], parse_dates=["local_date_time"], date_parser=lambda col: pd.to_datetime(col).tz_localize(None))
|
|
else:
|
|
raise ValueError("column_format can only be one of ['JSON', 'PLAIN_TEXT'].")
|
|
|
|
# Only keep dates in the range of [local_start_date, local_end_date)
|
|
parsed_data = parsed_data.loc[(parsed_data["local_date_time"] >= local_start_date) & (parsed_data["local_date_time"] < local_end_date)]
|
|
|
|
if parsed_data.shape[0] > 0:
|
|
parsed_data["timestamp"] = parsed_data["local_date_time"].dt.tz_localize(timezone).astype(np.int64) // 10**6
|
|
|
|
parsed_data.to_csv(snakemake.output[0], index=False)
|