diff --git a/tests/scripts/assign_test_timestamps.py b/tests/scripts/assign_test_timestamps.py index dd62249b..574d33d8 100644 --- a/tests/scripts/assign_test_timestamps.py +++ b/tests/scripts/assign_test_timestamps.py @@ -8,21 +8,28 @@ DayOfWeek2Date = {"Fri": ["2020-03-06", "2020-10-30"], "Mon": ["2020-03-09", "2020-11-02"]} def assign_test_timestamps(file_path): + + columns_to_delete = ["test_time", "day_of_week", "time"] + data = pd.read_csv(file_path) - data[["day_of_week","time"]] = data["test_time"].str.split(pat=" ", n=1, expand=True) + data[["day_of_week", "time"]] = data["test_time"].str.split(pat=" ", n=1, expand=True) data_with_timestamps = pd.DataFrame() # 0 is for March and 1 is for Nov for i in [0, 1]: - data["datetime"] = pd.to_datetime(data.apply(lambda row: DayOfWeek2Date[row["day_of_week"]][i] + " " + row["time"], axis=1)) + data["local_date_time"] = pd.to_datetime(data.apply(lambda row: DayOfWeek2Date[row["day_of_week"]][i] + " " + row["time"], axis=1)) data_with_timestamps = pd.concat([data_with_timestamps, data], axis=0) - - # Convert datetime with timezone to timestamp - data_with_timestamps.insert(0, "timestamp", data_with_timestamps["datetime"].dt.tz_localize(tz="America/New_York").astype(np.int64) // 10**6) - + + if "fitbit" in file_path: + data_with_timestamps.insert(0, "timestamp", 0) + else: + # Convert local_date_time with timezone to timestamp + data_with_timestamps.insert(0, "timestamp", data_with_timestamps["local_date_time"].dt.tz_localize(tz="America/New_York").astype(np.int64) // 10**6) + columns_to_delete.add("local_date_time") + # Discard useless columns - for col in ["test_time", "day_of_week", "time", "datetime"]: + for col in columns_to_delete: del data_with_timestamps[col] return data_with_timestamps diff --git a/tests/settings/mtz_event_config.yaml b/tests/settings/mtz_event_config.yaml index d6a0d779..5f69ef87 100644 --- a/tests/settings/mtz_event_config.yaml +++ b/tests/settings/mtz_event_config.yaml @@ -319,7 +319,7 @@ PHONE_WIFI_VISIBLE: # See https://www.rapids.science/latest/setup/configuration/#data-stream-configuration FITBIT_DATA_STREAMS: - USE: fitbitjson_mysql + USE: fitbitparsed_csv # AVAILABLE: fitbitjson_mysql: @@ -335,7 +335,7 @@ FITBIT_DATA_STREAMS: SLEEP_SUMMARY_EPISODE_DAY_ANCHOR: end # summary sleep episodes are considered as events based on either the start timestamp or end timestamp. fitbitparsed_csv: - FOLDER: data/external/fitbit_csv + FOLDER: tests/data/external/aware_csv SLEEP_SUMMARY_EPISODE_DAY_ANCHOR: end # summary sleep episodes are considered as events based on either the start timestamp or end timestamp. # Sensors ------ diff --git a/tests/settings/mtz_frequency_config.yaml b/tests/settings/mtz_frequency_config.yaml index a3b3e376..e30db7fe 100644 --- a/tests/settings/mtz_frequency_config.yaml +++ b/tests/settings/mtz_frequency_config.yaml @@ -319,7 +319,7 @@ PHONE_WIFI_VISIBLE: # See https://www.rapids.science/latest/setup/configuration/#data-stream-configuration FITBIT_DATA_STREAMS: - USE: fitbitjson_mysql + USE: fitbitparsed_csv # AVAILABLE: fitbitjson_mysql: @@ -335,7 +335,7 @@ FITBIT_DATA_STREAMS: SLEEP_SUMMARY_EPISODE_DAY_ANCHOR: end # summary sleep episodes are considered as events based on either the start timestamp or end timestamp. fitbitparsed_csv: - FOLDER: data/external/fitbit_csv + FOLDER: tests/data/external/aware_csv SLEEP_SUMMARY_EPISODE_DAY_ANCHOR: end # summary sleep episodes are considered as events based on either the start timestamp or end timestamp. # Sensors ------ diff --git a/tests/settings/mtz_periodic_config.yaml b/tests/settings/mtz_periodic_config.yaml index 3374b849..13a8462e 100644 --- a/tests/settings/mtz_periodic_config.yaml +++ b/tests/settings/mtz_periodic_config.yaml @@ -319,7 +319,7 @@ PHONE_WIFI_VISIBLE: # See https://www.rapids.science/latest/setup/configuration/#data-stream-configuration FITBIT_DATA_STREAMS: - USE: fitbitjson_mysql + USE: fitbitparsed_csv # AVAILABLE: fitbitjson_mysql: @@ -335,7 +335,7 @@ FITBIT_DATA_STREAMS: SLEEP_SUMMARY_EPISODE_DAY_ANCHOR: end # summary sleep episodes are considered as events based on either the start timestamp or end timestamp. fitbitparsed_csv: - FOLDER: data/external/fitbit_csv + FOLDER: tests/data/external/aware_csv SLEEP_SUMMARY_EPISODE_DAY_ANCHOR: end # summary sleep episodes are considered as events based on either the start timestamp or end timestamp. # Sensors ------ diff --git a/tests/settings/stz_event_config.yaml b/tests/settings/stz_event_config.yaml index 15eccb97..9d2a003c 100644 --- a/tests/settings/stz_event_config.yaml +++ b/tests/settings/stz_event_config.yaml @@ -319,7 +319,7 @@ PHONE_WIFI_VISIBLE: # See https://www.rapids.science/latest/setup/configuration/#data-stream-configuration FITBIT_DATA_STREAMS: - USE: fitbitjson_mysql + USE: fitbitparsed_csv # AVAILABLE: fitbitjson_mysql: @@ -335,7 +335,7 @@ FITBIT_DATA_STREAMS: SLEEP_SUMMARY_EPISODE_DAY_ANCHOR: end # summary sleep episodes are considered as events based on either the start timestamp or end timestamp. fitbitparsed_csv: - FOLDER: data/external/fitbit_csv + FOLDER: tests/data/external/aware_csv SLEEP_SUMMARY_EPISODE_DAY_ANCHOR: end # summary sleep episodes are considered as events based on either the start timestamp or end timestamp. # Sensors ------ diff --git a/tests/settings/stz_frequency_config.yaml b/tests/settings/stz_frequency_config.yaml index 4a8bf1f2..47051ead 100644 --- a/tests/settings/stz_frequency_config.yaml +++ b/tests/settings/stz_frequency_config.yaml @@ -319,7 +319,7 @@ PHONE_WIFI_VISIBLE: # See https://www.rapids.science/latest/setup/configuration/#data-stream-configuration FITBIT_DATA_STREAMS: - USE: fitbitjson_mysql + USE: fitbitparsed_csv # AVAILABLE: fitbitjson_mysql: @@ -335,7 +335,7 @@ FITBIT_DATA_STREAMS: SLEEP_SUMMARY_EPISODE_DAY_ANCHOR: end # summary sleep episodes are considered as events based on either the start timestamp or end timestamp. fitbitparsed_csv: - FOLDER: data/external/fitbit_csv + FOLDER: tests/data/external/aware_csv SLEEP_SUMMARY_EPISODE_DAY_ANCHOR: end # summary sleep episodes are considered as events based on either the start timestamp or end timestamp. # Sensors ------ diff --git a/tests/settings/stz_periodic_config.yaml b/tests/settings/stz_periodic_config.yaml index 5b4de209..2ec9f640 100644 --- a/tests/settings/stz_periodic_config.yaml +++ b/tests/settings/stz_periodic_config.yaml @@ -319,7 +319,7 @@ PHONE_WIFI_VISIBLE: # See https://www.rapids.science/latest/setup/configuration/#data-stream-configuration FITBIT_DATA_STREAMS: - USE: fitbitjson_mysql + USE: fitbitparsed_csv # AVAILABLE: fitbitjson_mysql: @@ -335,7 +335,7 @@ FITBIT_DATA_STREAMS: SLEEP_SUMMARY_EPISODE_DAY_ANCHOR: end # summary sleep episodes are considered as events based on either the start timestamp or end timestamp. fitbitparsed_csv: - FOLDER: data/external/fitbit_csv + FOLDER: tests/data/external/aware_csv SLEEP_SUMMARY_EPISODE_DAY_ANCHOR: end # summary sleep episodes are considered as events based on either the start timestamp or end timestamp. # Sensors ------