From 13174b0c2ac42b5a780daca4c5ade9a9961e6c5c Mon Sep 17 00:00:00 2001 From: JulioV Date: Thu, 11 Mar 2021 14:51:16 -0500 Subject: [PATCH] Fix a bug when fitbit data is empty --- config.yaml | 4 ++-- docs/datastreams/empatica-zip.md | 2 +- .../mutations/fitbit/parse_heartrate_intraday_json.py | 3 ++- .../mutations/fitbit/parse_heartrate_summary_json.py | 3 ++- .../streams/mutations/fitbit/parse_sleep_intraday_json.py | 3 ++- .../streams/mutations/fitbit/parse_sleep_summary_json.py | 6 ++++-- .../streams/mutations/fitbit/parse_steps_intraday_json.py | 3 ++- .../streams/mutations/fitbit/parse_steps_summary_json.py | 3 ++- 8 files changed, 17 insertions(+), 10 deletions(-) diff --git a/config.yaml b/config.yaml index 77aa5d2c..5c3b4271 100644 --- a/config.yaml +++ b/config.yaml @@ -13,11 +13,11 @@ PIDS: [test01] CREATE_PARTICIPANT_FILES: CSV_FILE_PATH: "data/external/example_participants.csv" # see docs for required format PHONE_SECTION: - ADD: TRUE + ADD: True DEVICE_ID_COLUMN: device_id # column name IGNORED_DEVICE_IDS: [] FITBIT_SECTION: - ADD: FALSE + ADD: True DEVICE_ID_COLUMN: fitbit_id # column name IGNORED_DEVICE_IDS: [] EMPATICA_SECTION: diff --git a/docs/datastreams/empatica-zip.md b/docs/datastreams/empatica-zip.md index 1d7a54d2..401fdc4f 100644 --- a/docs/datastreams/empatica-zip.md +++ b/docs/datastreams/empatica-zip.md @@ -3,7 +3,7 @@ This [data stream](../../datastreams/data-streams-introduction) handles Empatica ## Container -You need to create a subfolder for every participant named after their `pid` in the path specified by `[EMPATICA_DATA_STREAMS][empatica_zipfiles][FOLDER]`. You can add one or more Empatica zip files to any subfolder. +You need to create a subfolder for every participant named after their `device id` inside the folder specified by `[EMPATICA_DATA_STREAMS][empatica_zipfiles][FOLDER]`. You can add one or more Empatica zip files to any subfolder. The script to connect and download data from this container is at: ```bash diff --git a/src/data/streams/mutations/fitbit/parse_heartrate_intraday_json.py b/src/data/streams/mutations/fitbit/parse_heartrate_intraday_json.py index 0ff3764d..1a571682 100644 --- a/src/data/streams/mutations/fitbit/parse_heartrate_intraday_json.py +++ b/src/data/streams/mutations/fitbit/parse_heartrate_intraday_json.py @@ -79,5 +79,6 @@ def parseHeartrateData(heartrate_data): def main(json_raw, stream_parameters): parsed_data = parseHeartrateData(json_raw) parsed_data["timestamp"] = 0 # this column is added at readable_datetime.R because we neeed to take into account multiple timezones - parsed_data['local_date_time'] = parsed_data['local_date_time'].dt.strftime('%Y-%m-%d %H:%M:%S') + if pd.api.types.is_datetime64_any_dtype( parsed_data['local_date_time']): + parsed_data['local_date_time'] = parsed_data['local_date_time'].dt.strftime('%Y-%m-%d %H:%M:%S') return(parsed_data) diff --git a/src/data/streams/mutations/fitbit/parse_heartrate_summary_json.py b/src/data/streams/mutations/fitbit/parse_heartrate_summary_json.py index e6bac1c8..c2d073c9 100644 --- a/src/data/streams/mutations/fitbit/parse_heartrate_summary_json.py +++ b/src/data/streams/mutations/fitbit/parse_heartrate_summary_json.py @@ -68,5 +68,6 @@ def parseHeartrateData(heartrate_data): def main(json_raw, stream_parameters): parsed_data = parseHeartrateData(json_raw) parsed_data["timestamp"] = 0 # this column is added at readable_datetime.R because we neeed to take into account multiple timezones - parsed_data['local_date_time'] = parsed_data['local_date_time'].dt.strftime('%Y-%m-%d %H:%M:%S') + if pd.api.types.is_datetime64_any_dtype( parsed_data['local_date_time']): + parsed_data['local_date_time'] = parsed_data['local_date_time'].dt.strftime('%Y-%m-%d %H:%M:%S') return(parsed_data) diff --git a/src/data/streams/mutations/fitbit/parse_sleep_intraday_json.py b/src/data/streams/mutations/fitbit/parse_sleep_intraday_json.py index 5f07c1eb..bc16b5e1 100644 --- a/src/data/streams/mutations/fitbit/parse_sleep_intraday_json.py +++ b/src/data/streams/mutations/fitbit/parse_sleep_intraday_json.py @@ -137,6 +137,7 @@ def parseSleepData(sleep_data): def main(json_raw, stream_parameters): parsed_data = parseSleepData(json_raw) parsed_data["timestamp"] = 0 # this column is added at readable_datetime.R because we neeed to take into account multiple timezones - parsed_data['local_date_time'] = parsed_data['local_date_time'].dt.strftime('%Y-%m-%d %H:%M:%S') + if pd.api.types.is_datetime64_any_dtype( parsed_data['local_date_time']): + parsed_data['local_date_time'] = parsed_data['local_date_time'].dt.strftime('%Y-%m-%d %H:%M:%S') return(parsed_data) diff --git a/src/data/streams/mutations/fitbit/parse_sleep_summary_json.py b/src/data/streams/mutations/fitbit/parse_sleep_summary_json.py index 121901d7..33d7ef50 100644 --- a/src/data/streams/mutations/fitbit/parse_sleep_summary_json.py +++ b/src/data/streams/mutations/fitbit/parse_sleep_summary_json.py @@ -59,8 +59,10 @@ def parseSleepData(sleep_data): def main(json_raw, stream_parameters): parsed_data = parseSleepData(json_raw) parsed_data["timestamp"] = 0 # this column is added at readable_datetime.R because we neeed to take into account multiple timezones - parsed_data['local_start_date_time'] = parsed_data['local_start_date_time'].dt.strftime('%Y-%m-%d %H:%M:%S') - parsed_data['local_end_date_time'] = parsed_data['local_end_date_time'].dt.strftime('%Y-%m-%d %H:%M:%S') + if pd.api.types.is_datetime64_any_dtype( parsed_data['local_start_date_time']): + parsed_data['local_start_date_time'] = parsed_data['local_start_date_time'].dt.strftime('%Y-%m-%d %H:%M:%S') + if pd.api.types.is_datetime64_any_dtype( parsed_data['local_end_date_time']): + parsed_data['local_end_date_time'] = parsed_data['local_end_date_time'].dt.strftime('%Y-%m-%d %H:%M:%S') if stream_parameters["SLEEP_SUMMARY_EPISODE_DAY_ANCHOR"] == "start": parsed_data["local_date_time"] = parsed_data['local_start_date_time'] diff --git a/src/data/streams/mutations/fitbit/parse_steps_intraday_json.py b/src/data/streams/mutations/fitbit/parse_steps_intraday_json.py index f9ea13a4..ae468878 100644 --- a/src/data/streams/mutations/fitbit/parse_steps_intraday_json.py +++ b/src/data/streams/mutations/fitbit/parse_steps_intraday_json.py @@ -40,5 +40,6 @@ def parseStepsData(steps_data): def main(json_raw, stream_parameters): parsed_data = parseStepsData(json_raw) parsed_data["timestamp"] = 0 # this column is added at readable_datetime.R because we neeed to take into account multiple timezones - parsed_data['local_date_time'] = parsed_data['local_date_time'].dt.strftime('%Y-%m-%d %H:%M:%S') + if pd.api.types.is_datetime64_any_dtype( parsed_data['local_date_time']): + parsed_data['local_date_time'] = parsed_data['local_date_time'].dt.strftime('%Y-%m-%d %H:%M:%S') return(parsed_data) diff --git a/src/data/streams/mutations/fitbit/parse_steps_summary_json.py b/src/data/streams/mutations/fitbit/parse_steps_summary_json.py index 932f2988..00b3745c 100644 --- a/src/data/streams/mutations/fitbit/parse_steps_summary_json.py +++ b/src/data/streams/mutations/fitbit/parse_steps_summary_json.py @@ -33,5 +33,6 @@ def parseStepsData(steps_data): def main(json_raw, stream_parameters): parsed_data = parseStepsData(json_raw) parsed_data["timestamp"] = 0 # this column is added at readable_datetime.R because we neeed to take into account multiple timezones - parsed_data['local_date_time'] = parsed_data['local_date_time'].dt.strftime('%Y-%m-%d %H:%M:%S') + if pd.api.types.is_datetime64_any_dtype( parsed_data['local_date_time']): + parsed_data['local_date_time'] = parsed_data['local_date_time'].dt.strftime('%Y-%m-%d %H:%M:%S') return(parsed_data)