Update data streams mutation of fitbit data

docs/datastreams/mandatory-fitbit-format.md

@@ -30,7 +30,7 @@ This is a description of the format RAPIDS needs to process data for the followi
     | RAPIDS column   | Description   |
     |-----------------|-----------------|
     | TIMESTAMP       |  An UNIX timestamp (13 digits) when a row of data was logged (automatically created by RAPIDS) |
-    | LOCAL_DATE_TIME       |  Date time string with format `yyyy-mm-dd hh:mm:ss`, this either is a copy of LOCAL_START_DATE_TIME or LOCAL_END_DATE_TIME depending on which column is used to assign an episode to a specific day|
+    | LOCAL_DATE_TIME       |  Date time string with format `yyyy-mm-dd 00:00:00`, the date is the same as the start date of a daily sleep episode if its time is after SLEEP_SUMMARY_LAST_NIGHT_END, otherwise it is the day before the start date of that sleep episode |
     | LOCAL_START_DATE_TIME       |  Date time string with format `yyyy-mm-dd hh:mm:ss` representing the start of a daily sleep episode |
     | LOCAL_END_DATE_TIME       |  Date time string with format `yyyy-mm-dd hh:mm:ss`  representing the end of a daily sleep episode|
     | DEVICE_ID       |  A string that uniquely identifies a device |

docs/snippets/jsonfitbit_format.md

@@ -33,7 +33,8 @@ If you want RAPIDS to process Fitbit sensor data using this stream, you will nee
     - **SCRIPTS**
     
         ```bash
-        src/data/streams/mutations/fitbit/parse_heartrate_summary_json.py
+        - src/data/streams/mutations/fitbit/parse_heartrate_summary_json.py
+        - src/data/streams/mutations/fitbit/add_zero_timestamp.py
         ```
 
         !!! note
@@ -71,7 +72,8 @@ If you want RAPIDS to process Fitbit sensor data using this stream, you will nee
     - **SCRIPTS**
     
         ```bash
-        src/data/streams/mutations/fitbit/parse_heartrate_intraday_json.py
+        - src/data/streams/mutations/fitbit/parse_heartrate_intraday_json.py
+        - src/data/streams/mutations/fitbit/add_zero_timestamp.py
         ```
 
         !!! note
@@ -117,7 +119,9 @@ If you want RAPIDS to process Fitbit sensor data using this stream, you will nee
     - **SCRIPTS**
     
         ```bash
-        src/data/streams/mutations/fitbit/parse_sleep_summary_json.py
+        - src/data/streams/mutations/fitbit/parse_sleep_summary_json.py
+        - src/data/streams/mutations/fitbit/add_local_date_time.py
+        - src/data/streams/mutations/fitbit/add_zero_timestamp.py
         ```
 
         !!! note
@@ -160,7 +164,8 @@ If you want RAPIDS to process Fitbit sensor data using this stream, you will nee
     - **SCRIPTS**
     
         ```bash
-        src/data/streams/mutations/fitbit/parse_sleep_intraday_json.py
+        - src/data/streams/mutations/fitbit/parse_sleep_intraday_json.py
+        - src/data/streams/mutations/fitbit/add_zero_timestamp.py
         ```
 
         !!! note
@@ -199,7 +204,8 @@ If you want RAPIDS to process Fitbit sensor data using this stream, you will nee
     - **SCRIPTS**
     
         ```bash
-        src/data/streams/mutations/fitbit/parse_steps_summary_json.py
+        - src/data/streams/mutations/fitbit/parse_steps_summary_json.py
+        - src/data/streams/mutations/fitbit/add_zero_timestamp.py
         ```
 
         !!! note
@@ -235,7 +241,8 @@ If you want RAPIDS to process Fitbit sensor data using this stream, you will nee
     - **SCRIPTS**
     
         ```bash
-        src/data/streams/mutations/fitbit/parse_steps_intraday_json.py
+        - src/data/streams/mutations/fitbit/parse_steps_intraday_json.py
+        - src/data/streams/mutations/fitbit/add_zero_timestamp.py
         ```
 
         !!! note

docs/snippets/parsedfitbit_format.md

@@ -88,7 +88,7 @@ All columns are mandatory; however, all except `device_id` and `local_date_time`
     | RAPIDS column   | Stream column   |
     |-----------------|-----------------|
     | TIMESTAMP| FLAG_TO_MUTATE |
-    | LOCAL_DATE_TIME| local_date_time |
+    | LOCAL_DATE_TIME| FLAG_TO_MUTATE |
     | LOCAL_START_DATE_TIME| local_start_date_time |
     | LOCAL_END_DATE_TIME| local_end_date_time |
     | DEVICE_ID| device_id |
@@ -108,7 +108,8 @@ All columns are mandatory; however, all except `device_id` and `local_date_time`
     - **SCRIPTS** 
 
         ```bash
-        src/data/streams/mutations/fitbit/add_zero_timestamp.py
+        - src/data/streams/mutations/fitbit/add_local_date_time.py
+        - src/data/streams/mutations/fitbit/add_zero_timestamp.py
         ```
 
     !!! note

src/data/streams/fitbitjson_csv/format.yaml

@@ -13,6 +13,7 @@ FITBIT_HEARTRATE_SUMMARY:
       JSON_FITBIT_COLUMN: fitbit_data # string columnwith JSON objects
     SCRIPTS: # List any python or r scripts that mutate your raw data
       - src/data/streams/mutations/fitbit/parse_heartrate_summary_json.py
+      - src/data/streams/mutations/fitbit/add_zero_timestamp.py
 
 FITBIT_HEARTRATE_INTRADAY:
   RAPIDS_COLUMN_MAPPINGS:
@@ -26,6 +27,7 @@ FITBIT_HEARTRATE_INTRADAY:
       JSON_FITBIT_COLUMN: fitbit_data # string columnwith JSON objects
     SCRIPTS: # List any python or r scripts that mutate your raw data
       - src/data/streams/mutations/fitbit/parse_heartrate_intraday_json.py
+      - src/data/streams/mutations/fitbit/add_zero_timestamp.py
 
 FITBIT_SLEEP_SUMMARY:
   RAPIDS_COLUMN_MAPPINGS:
@@ -47,6 +49,8 @@ FITBIT_SLEEP_SUMMARY:
       JSON_FITBIT_COLUMN: fitbit_data # string columnwith JSON objects
     SCRIPTS: # List any python or r scripts that mutate your raw data
       - src/data/streams/mutations/fitbit/parse_sleep_summary_json.py
+      - src/data/streams/mutations/fitbit/add_local_date_time.py
+      - src/data/streams/mutations/fitbit/add_zero_timestamp.py
 
 FITBIT_SLEEP_INTRADAY:
   RAPIDS_COLUMN_MAPPINGS:
@@ -63,6 +67,7 @@ FITBIT_SLEEP_INTRADAY:
       JSON_FITBIT_COLUMN: fitbit_data # string columnwith JSON objects
     SCRIPTS: # List any python or r scripts that mutate your raw data
       - src/data/streams/mutations/fitbit/parse_sleep_intraday_json.py
+      - src/data/streams/mutations/fitbit/add_zero_timestamp.py
 
 FITBIT_STEPS_SUMMARY:
   RAPIDS_COLUMN_MAPPINGS:
@@ -75,6 +80,7 @@ FITBIT_STEPS_SUMMARY:
       JSON_FITBIT_COLUMN: fitbit_data # string columnwith JSON objects
     SCRIPTS: # List any python or r scripts that mutate your raw data
       - src/data/streams/mutations/fitbit/parse_steps_summary_json.py
+      - src/data/streams/mutations/fitbit/add_zero_timestamp.py
 
 FITBIT_STEPS_INTRADAY:
   RAPIDS_COLUMN_MAPPINGS:
@@ -87,6 +93,7 @@ FITBIT_STEPS_INTRADAY:
       JSON_FITBIT_COLUMN: fitbit_data # string columnwith JSON objects
     SCRIPTS: # List any python or r scripts that mutate your raw data
       - src/data/streams/mutations/fitbit/parse_steps_intraday_json.py
+      - src/data/streams/mutations/fitbit/add_zero_timestamp.py
 
 FITBIT_CALORIES_INTRADAY:
   RAPIDS_COLUMN_MAPPINGS:
@@ -101,3 +108,4 @@ FITBIT_CALORIES_INTRADAY:
       JSON_FITBIT_COLUMN: fitbit_data # string columnwith JSON objects
     SCRIPTS: # List any python or r scripts that mutate your raw data
       - src/data/streams/mutations/fitbit/parse_calories_intraday_json.py
+      - src/data/streams/mutations/fitbit/add_zero_timestamp.py

src/data/streams/fitbitjson_mysql/format.yaml

@@ -13,6 +13,7 @@ FITBIT_HEARTRATE_SUMMARY:
       JSON_FITBIT_COLUMN: fitbit_data # string columnwith JSON objects
     SCRIPTS: # List any python or r scripts that mutate your raw data
       - src/data/streams/mutations/fitbit/parse_heartrate_summary_json.py
+      - src/data/streams/mutations/fitbit/add_zero_timestamp.py
 
 FITBIT_HEARTRATE_INTRADAY:
   RAPIDS_COLUMN_MAPPINGS:
@@ -26,6 +27,7 @@ FITBIT_HEARTRATE_INTRADAY:
       JSON_FITBIT_COLUMN: fitbit_data # string columnwith JSON objects
     SCRIPTS: # List any python or r scripts that mutate your raw data
       - src/data/streams/mutations/fitbit/parse_heartrate_intraday_json.py
+      - src/data/streams/mutations/fitbit/add_zero_timestamp.py
 
 FITBIT_SLEEP_SUMMARY:
   RAPIDS_COLUMN_MAPPINGS:
@@ -47,6 +49,8 @@ FITBIT_SLEEP_SUMMARY:
       JSON_FITBIT_COLUMN: fitbit_data # string columnwith JSON objects
     SCRIPTS: # List any python or r scripts that mutate your raw data
       - src/data/streams/mutations/fitbit/parse_sleep_summary_json.py
+      - src/data/streams/mutations/fitbit/add_local_date_time.py
+      - src/data/streams/mutations/fitbit/add_zero_timestamp.py
 
 FITBIT_SLEEP_INTRADAY:
   RAPIDS_COLUMN_MAPPINGS:
@@ -63,6 +67,7 @@ FITBIT_SLEEP_INTRADAY:
       JSON_FITBIT_COLUMN: fitbit_data # string columnwith JSON objects
     SCRIPTS: # List any python or r scripts that mutate your raw data
       - src/data/streams/mutations/fitbit/parse_sleep_intraday_json.py
+      - src/data/streams/mutations/fitbit/add_zero_timestamp.py
 
 FITBIT_STEPS_SUMMARY:
   RAPIDS_COLUMN_MAPPINGS:
@@ -75,6 +80,7 @@ FITBIT_STEPS_SUMMARY:
       JSON_FITBIT_COLUMN: fitbit_data # string columnwith JSON objects
     SCRIPTS: # List any python or r scripts that mutate your raw data
       - src/data/streams/mutations/fitbit/parse_steps_summary_json.py
+      - src/data/streams/mutations/fitbit/add_zero_timestamp.py
 
 FITBIT_STEPS_INTRADAY:
   RAPIDS_COLUMN_MAPPINGS:
@@ -87,6 +93,7 @@ FITBIT_STEPS_INTRADAY:
       JSON_FITBIT_COLUMN: fitbit_data # string columnwith JSON objects
     SCRIPTS: # List any python or r scripts that mutate your raw data
       - src/data/streams/mutations/fitbit/parse_steps_intraday_json.py
+      - src/data/streams/mutations/fitbit/add_zero_timestamp.py
 
 FITBIT_CALORIES_INTRADAY:
   RAPIDS_COLUMN_MAPPINGS:
@@ -101,3 +108,4 @@ FITBIT_CALORIES_INTRADAY:
       JSON_FITBIT_COLUMN: fitbit_data # string columnwith JSON objects
     SCRIPTS: # List any python or r scripts that mutate your raw data
       - src/data/streams/mutations/fitbit/parse_calories_intraday_json.py
+      - src/data/streams/mutations/fitbit/add_zero_timestamp.py

src/data/streams/fitbitparsed_csv/format.yaml

@@ -29,7 +29,7 @@ FITBIT_SLEEP_SUMMARY:
   RAPIDS_COLUMN_MAPPINGS:
     TIMESTAMP: FLAG_TO_MUTATE
     DEVICE_ID: device_id
-    LOCAL_DATE_TIME: local_date_time
+    LOCAL_DATE_TIME: FLAG_TO_MUTATE
     LOCAL_START_DATE_TIME: local_start_date_time
     LOCAL_END_DATE_TIME: local_end_date_time
     EFFICIENCY: efficiency
@@ -43,6 +43,7 @@ FITBIT_SLEEP_SUMMARY:
   MUTATION:
     COLUMN_MAPPINGS:
     SCRIPTS: # List any python or r scripts that mutate your raw data
+      - src/data/streams/mutations/fitbit/add_local_date_time.py
       - src/data/streams/mutations/fitbit/add_zero_timestamp.py
 
 FITBIT_SLEEP_INTRADAY:

src/data/streams/fitbitparsed_mysql/format.yaml

@@ -29,7 +29,7 @@ FITBIT_SLEEP_SUMMARY:
   RAPIDS_COLUMN_MAPPINGS:
     TIMESTAMP: FLAG_TO_MUTATE
     DEVICE_ID: device_id
-    LOCAL_DATE_TIME: local_date_time
+    LOCAL_DATE_TIME: FLAG_TO_MUTATE
     LOCAL_START_DATE_TIME: local_start_date_time
     LOCAL_END_DATE_TIME: local_end_date_time
     EFFICIENCY: efficiency
@@ -43,6 +43,7 @@ FITBIT_SLEEP_SUMMARY:
   MUTATION:
     COLUMN_MAPPINGS:
     SCRIPTS: # List any python or r scripts that mutate your raw data
+      - src/data/streams/mutations/fitbit/add_local_date_time.py
       - src/data/streams/mutations/fitbit/add_zero_timestamp.py
 
 FITBIT_SLEEP_INTRADAY:

src/data/streams/mutations/fitbit/add_local_date_time.py

@@ -0,0 +1,13 @@
+import pandas as pd
+
+def main(parsed_data, stream_parameters):
+
+    parsed_data["local_date_time"] = (pd.to_datetime(parsed_data["local_start_date_time"]) - pd.Timedelta(minutes=stream_parameters["SLEEP_SUMMARY_LAST_NIGHT_END"])).dt.strftime('%Y-%m-%d 00:00:00')
+
+    # complete missing dates
+    missed_dates = list(set([x.strftime('%Y-%m-%d 00:00:00') for x in pd.date_range(parsed_data["local_date_time"].min(), parsed_data["local_date_time"].max()).to_pydatetime()]) - set(parsed_data["local_date_time"]))
+    parsed_data = pd.concat([parsed_data, pd.DataFrame({"local_date_time": missed_dates})], axis=0)
+    parsed_data.sort_values(by=["local_date_time", "local_start_date_time"], inplace=True)
+    parsed_data["device_id"] = parsed_data["device_id"].interpolate(method="pad")
+
+    return parsed_data

src/data/streams/mutations/fitbit/add_zero_timestamp.py

@@ -4,4 +4,4 @@ def main(parsed_data, stream_parameters):
     parsed_data["timestamp"] = 0 # this column is added at readable_datetime.R because we neeed to take into account multiple timezones
     if pd.api.types.is_datetime64_any_dtype(parsed_data['local_date_time']):
         parsed_data['local_date_time'] = parsed_data['local_date_time'].dt.strftime('%Y-%m-%d %H:%M:%S')
-    return(parsed_data)
+    return parsed_data

src/data/streams/mutations/fitbit/parse_calories_intraday_json.py

@@ -26,8 +26,5 @@ def parseCaloriesData(calories_data):
 
 def main(json_raw, stream_parameters):
     parsed_data = parseCaloriesData(json_raw)
-    parsed_data["timestamp"] = 0 # this column is added at readable_datetime.R because we neeed to take into account multiple timezones
     parsed_data["mets"] = parsed_data["mets"] / 10
-    if pd.api.types.is_datetime64_any_dtype( parsed_data['local_date_time']):
+    return parsed_data
-        parsed_data['local_date_time'] = parsed_data['local_date_time'].dt.strftime('%Y-%m-%d %H:%M:%S')
-    return(parsed_data)

src/data/streams/mutations/fitbit/parse_heartrate_intraday_json.py

@@ -78,7 +78,4 @@ def parseHeartrateData(heartrate_data):
 
 def main(json_raw, stream_parameters):
     parsed_data = parseHeartrateData(json_raw)
-    parsed_data["timestamp"] = 0 # this column is added at readable_datetime.R because we neeed to take into account multiple timezones
+    return parsed_data
-    if pd.api.types.is_datetime64_any_dtype( parsed_data['local_date_time']):
-        parsed_data['local_date_time'] = parsed_data['local_date_time'].dt.strftime('%Y-%m-%d %H:%M:%S')
-    return(parsed_data)

src/data/streams/mutations/fitbit/parse_heartrate_summary_json.py

@@ -67,7 +67,4 @@ def parseHeartrateData(heartrate_data):
 
 def main(json_raw, stream_parameters):
     parsed_data = parseHeartrateData(json_raw)
-    parsed_data["timestamp"] = 0 # this column is added at readable_datetime.R because we neeed to take into account multiple timezones
+    return parsed_data
-    if pd.api.types.is_datetime64_any_dtype( parsed_data['local_date_time']):
-        parsed_data['local_date_time'] = parsed_data['local_date_time'].dt.strftime('%Y-%m-%d %H:%M:%S')
-    return(parsed_data)

src/data/streams/mutations/fitbit/parse_sleep_intraday_json.py

@@ -136,8 +136,4 @@ def parseSleepData(sleep_data):
 
 def main(json_raw, stream_parameters):
     parsed_data = parseSleepData(json_raw)
-    parsed_data["timestamp"] = 0 # this column is added at readable_datetime.R because we neeed to take into account multiple timezones
+    return parsed_data
-    if pd.api.types.is_datetime64_any_dtype( parsed_data['local_date_time']):
-        parsed_data['local_date_time'] = parsed_data['local_date_time'].dt.strftime('%Y-%m-%d %H:%M:%S')
-
-    return(parsed_data)

src/data/streams/mutations/fitbit/parse_sleep_summary_json.py

@@ -56,20 +56,13 @@ def parseSleepData(sleep_data):
 
     return parsed_data
 
+
 def main(json_raw, stream_parameters):
     parsed_data = parseSleepData(json_raw)
-    parsed_data["local_date_time"] = (parsed_data["local_start_date_time"] - pd.Timedelta(minutes=stream_parameters["SLEEP_SUMMARY_LAST_NIGHT_END"])).dt.strftime('%Y-%m-%d 00:00:00')
 
-    # complete missing dates
-    missed_dates = list(set([x.strftime('%Y-%m-%d 00:00:00') for x in pd.date_range(parsed_data["local_date_time"].min(), parsed_data["local_date_time"].max()).to_pydatetime()]) - set(parsed_data["local_date_time"]))
-    parsed_data = pd.concat([parsed_data, pd.DataFrame({"local_date_time": missed_dates})], axis=0)
-    parsed_data.sort_values(by=["local_date_time", "local_start_date_time"], inplace=True)
-    parsed_data["device_id"] = parsed_data["device_id"].interpolate(method="pad")
-
-    parsed_data["timestamp"] = 0 # this column is added at readable_datetime.R because we neeed to take into account multiple timezones
     if pd.api.types.is_datetime64_any_dtype( parsed_data['local_start_date_time']):
         parsed_data['local_start_date_time'] = parsed_data['local_start_date_time'].dt.strftime('%Y-%m-%d %H:%M:%S')
     if pd.api.types.is_datetime64_any_dtype( parsed_data['local_end_date_time']):
         parsed_data['local_end_date_time'] = parsed_data['local_end_date_time'].dt.strftime('%Y-%m-%d %H:%M:%S')
 
-    return(parsed_data)
+    return parsed_data

src/data/streams/mutations/fitbit/parse_steps_intraday_json.py

@@ -39,7 +39,4 @@ def parseStepsData(steps_data):
 
 def main(json_raw, stream_parameters):
     parsed_data = parseStepsData(json_raw)
-    parsed_data["timestamp"] = 0 # this column is added at readable_datetime.R because we neeed to take into account multiple timezones
+    return parsed_data
-    if pd.api.types.is_datetime64_any_dtype( parsed_data['local_date_time']):
-        parsed_data['local_date_time'] = parsed_data['local_date_time'].dt.strftime('%Y-%m-%d %H:%M:%S')
-    return(parsed_data)

src/data/streams/mutations/fitbit/parse_steps_summary_json.py

@@ -32,7 +32,4 @@ def parseStepsData(steps_data):
 
 def main(json_raw, stream_parameters):
     parsed_data = parseStepsData(json_raw)
-    parsed_data["timestamp"] = 0 # this column is added at readable_datetime.R because we neeed to take into account multiple timezones
+    return parsed_data
-    if pd.api.types.is_datetime64_any_dtype( parsed_data['local_date_time']):
-        parsed_data['local_date_time'] = parsed_data['local_date_time'].dt.strftime('%Y-%m-%d %H:%M:%S')
-    return(parsed_data)