Test & fix bugs of sleep intraday features

config.yaml

@@ -321,19 +321,19 @@ FITBIT_DATA_STREAMS:
   # AVAILABLE:
   fitbitjson_mysql: 
     DATABASE_GROUP: MY_GROUP
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
   
   fitbitparsed_mysql: 
     DATABASE_GROUP: MY_GROUP
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
   
   fitbitjson_csv: 
     FOLDER: data/external/fitbit_csv
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
 
   fitbitparsed_csv: 
     FOLDER: data/external/fitbit_csv
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
 
 # Sensors ------
 
@@ -400,15 +400,12 @@ FITBIT_SLEEP_INTRADAY:
         LEVELS_AND_TYPES: [countepisode, sumduration, maxduration, minduration, avgduration, medianduration, stdduration]
         RATIOS_TYPE: [count, duration]
         RATIOS_SCOPE: [ACROSS_LEVELS, ACROSS_TYPES, WITHIN_LEVELS, WITHIN_TYPES]
-        ROUTINE: [starttimefirstmainsleep, endtimelastmainsleep, starttimefirstnap, endtimelastnap]
       SLEEP_LEVELS:
         INCLUDE_ALL_GROUPS: True
         CLASSIC: [awake, restless, asleep]
         STAGES: [wake, deep, light, rem]
         UNIFIED: [awake, asleep]
       SLEEP_TYPES: [main, nap, all]
-      LAST_NIGHT_END: 0 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight
-      ROUTINE_REFERENCE_TIME: MIDNIGHT # chosen from "MIDNIGHT" and "START_OF_THE_SEGMENT"
       SRC_SCRIPT: src/features/fitbit_sleep_intraday/rapids/main.py
   
     PRICE:
@@ -420,7 +417,7 @@ FITBIT_SLEEP_INTRADAY:
         STAGES: [wake, deep, light, rem]
         UNIFIED: [awake, asleep]
       DAY_TYPES: [WEEKEND, WEEK, ALL]
-      LAST_NIGHT_END: 660 # number of minutes after the midnight (11:00) 11*60
+      LAST_NIGHT_END: 660 # number of minutes after midnight (11:00) 11*60
       SRC_SCRIPT: src/features/fitbit_sleep_intraday/price/main.py
 
 # See https://www.rapids.science/latest/features/fitbit-steps-summary/

docs/developers/test-cases.md

@@ -24,10 +24,13 @@ The following is a list of the sensors that testing is currently available.
 | Phone Screen                  | RAPIDS   | Y        | N         | N     |
 | Phone WiFi Connected          | RAPIDS   | Y        | Y         | N     |
 | Phone WiFi Visible            | RAPIDS   | Y        | Y         | N     |
+| Fitbit Calories Intraday      | RAPIDS   | Y        | Y         | Y     |
 | Fitbit Data Yield             | RAPIDS   | N        | N         | N     |
 | Fitbit Heart Rate Summary     | RAPIDS   | N        | N         | N     |
 | Fitbit Heart Rate Intraday    | RAPIDS   | N        | N         | N     |
 | Fitbit Sleep Summary          | RAPIDS   | N        | N         | N     |
+| Fitbit Sleep Intraday         | RAPIDS   | Y        | Y         | Y     |
+| Fitbit Sleep Intraday         | PRICE    | Y        | Y         | Y     |
 | Fitbit Steps Summary          | RAPIDS   | N        | N         | N     |
 | Fitbit Steps Intraday         | RAPIDS   | N        | N         | N     |
 
@@ -242,3 +245,51 @@ Checklist
 |weekend|OK|OK|fitbit|
 |beforeMarchEvent|OK|OK|fitbit|
 |beforeNovemberEvent|OK|OK|fitbit|
+
+## Fitbit Sleep Summary
+
+Description
+
+- A main sleep episode that starts on Fri 20:00:00 and ends on Sat 02:00:00. This episode starts after 11am (Last Night End) which will be considered as today's (Fri) data.
+- A nap that starts on Sat 04:00:00 and ends on Sat 06:00:00. This episode starts before 11am (Last Night End) which will be considered as yesterday's (Fri) data.
+- A nap that starts on Sat 13:00:00 and ends on Sat 15:00:00. This episode starts after 11am (Last Night End) which will be considered as today's (Sat) data.
+- A main sleep that starts on Sun 01:00:00 and ends on Sun 12:00:00. This episode starts before 11am (Last Night End) which will be considered as yesterday's (Sat) data.
+- A main sleep that starts on Sun 23:00:00 and ends on Mon 07:00:00. This episode starts after 11am (Last Night End) which will be considered as today's (Sun) data.
+- Any segment shorter than one day will be ignored for sleep RAPIDS features.
+
+Checklist
+
+|time segment| single tz | multi tz|platform|
+|-|-|-|-|
+|30min|OK|OK|fitbit|
+|morning|OK|OK|fitbit|
+|daily|OK|OK|fitbit|
+|threeday|OK|OK|fitbit|
+|weekend|OK|OK|fitbit|
+|beforeMarchEvent|OK|OK|fitbit|
+|beforeNovemberEvent|OK|OK|fitbit|
+
+## Fitbit Sleep Intraday
+
+Description
+
+- A five-minute main sleep episode with asleep-classic level on Fri 11:00:00.
+- An eight-hour main sleep episode on Fri 17:00:00. It is split into 2 parts for daily segment: a seven-hour sleep episode on Fri 17:00:00 and an one-hour sleep episode on Sat 00:00:00.
+- A two-hour nap on Sat 01:00:00 that will be ignored for main sleep features.
+- An one-hour nap on Sat 13:00:00 that will be ignored for main sleep features.
+- An eight-hour main sleep episode on Sat 22:00:00. This episode ends on Sun 08:00:00 (NY) for March and Sun 06:00:00 (NY) for Novembers due to daylight savings. It will be considered for `beforeMarchEvent` segment and ignored for `beforeNovemberEvent` segment.
+- A nine-hour main sleep episode on Sun 11:00:00. Start time will be assigned as NY time zone and converted to 14:00:00.
+- A seven-hour main sleep episode on Mon 06:00:00. This episode will be split into two parts: a five-hour sleep episode on Mon 06:00:00 and a two-hour sleep episode on Mon 11:00:00. The first part will be discarded as it is before 11am (Last Night End)
+- Any segment shorter than one day will be ignored for sleep PRICE features.
+
+Checklist
+
+|time segment| single tz | multi tz|platform|
+|-|-|-|-|
+|30min|OK|OK|fitbit|
+|morning|OK|OK|fitbit|
+|daily|OK|OK|fitbit|
+|threeday|OK|OK|fitbit|
+|weekend|OK|OK|fitbit|
+|beforeMarchEvent|OK|OK|fitbit|
+|beforeNovemberEvent|OK|OK|fitbit|

docs/features/fitbit-sleep-intraday.md

@@ -9,7 +9,7 @@ Sensor parameters description for `[FITBIT_SLEEP_INTRADAY]`:
 ## RAPIDS provider
 
 !!! hint "Understanding RAPIDS features"
-    [This diagram](../../img/sleep_intraday_rapids.png) will help you understand how sleep episodes are chunked and grouped within time segments and `LNE-LNE` intervals for the RAPIDS provider.
+    [This diagram](../../img/sleep_intraday_rapids.png) will help you understand how sleep episodes are chunked and grouped within time segments for the RAPIDS provider.
 
 
 !!! info "Available time segments"
@@ -35,8 +35,6 @@ Parameters description for `[FITBIT_SLEEP_INTRADAY][PROVIDERS][RAPIDS]`:
 |`[FEATURES]`                                 |         Features to be computed from sleep intraday data, see table below           |
 |`[SLEEP_LEVELS]`                             | Fitbit’s sleep API Version 1 only provides `CLASSIC` records. However, Version 1.2 provides 2 types of records: `CLASSIC` and `STAGES`. `STAGES` is only available in devices with a heart rate sensor and even those devices will fail to report it if the battery is low or the device is not tight enough. While `CLASSIC` contains 3 sleep levels (`awake`, `restless`, and `asleep`), `STAGES` contains 4 sleep levels (`wake`, `deep`, `light`, `rem`). To make it consistent, RAPIDS groups them into 2 `UNIFIED` sleep levels: `awake` (`CLASSIC`: `awake` and `restless`; `STAGES`: `wake`) and `asleep` (`CLASSIC`: `asleep`; `STAGES`: `deep`, `light`, and `rem`). In this section, there is a boolean flag named `INCLUDE_ALL_GROUPS` that if set to TRUE, computes LEVELS_AND_TYPES features grouping all levels together in a single `all` category.
 |`[SLEEP_TYPES]`                              | Types of sleep to be included in the feature extraction computation. There are three sleep types: `main`, `nap`, and `all`. The `all` type means both main sleep and naps are considered.
-|`[LAST_NIGHT_END]`| All resampled sleep rows (bin interval: one minute) that started after this time will be included in the feature computation. It ranges from 0 (midnight) to 1439 (23:59) which denotes the number of minutes after midnight. If a segment is longer than one day, this value is applied every day.
-|`[ROUTINE_REFERENCE_TIME]`| The reference point from which the `[ROUTINE]` features are computed, it can be `MIDNIGHT` or `START_OF_THE_SEGMENT`, default is `MIDNIGHT`. If you have multiple time segments per day it might be more informative to set this flag to `START_OF_THE_SEGMENT`.
 
 
 Features description for `[FITBIT_SLEEP_INTRADAY][PROVIDERS][RAPIDS][LEVELS_AND_TYPES]`:
@@ -72,8 +70,8 @@ Features description for `[FITBIT_SLEEP_INTRADAY][PROVIDERS][RAPIDS]` RATIOS `[W
 
 |Feature                                                      |Units          |Description                                                  |
 |--------------------------------- |-------------- |-------------------------------------------------------------|
-|ratiocount`[TYPE]`within`[LEVEL]`    |-              |Ratio between the **count** of episodes of a single sleep `[LEVEL]` during `main` sleep divided by the **count** of episodes of a single sleep `[LEVEL]` during `main` **and** `nap`. This answers the question: are `rem` episodes more frequent during `main` than `nap` sleep? We do not provide the ratio for `nap` because is complementary. ($countepisode[remstages][main] / countepisode[remstages][all]$)
+|ratiocountmainwithin`[LEVEL]`    |-              |Ratio between the **count** of episodes of a single sleep `[LEVEL]` during `main` sleep divided by the **count** of episodes of a single sleep `[LEVEL]` during `main` **and** `nap`. This answers the question: are `rem` episodes more frequent during `main` than `nap` sleep? We do not provide the ratio for `nap` because is complementary. ($countepisode[remstages][main] / countepisode[remstages][all]$)
-|ratioduration`[TYPE]`within`[LEVEL]` |-              |Ratio between the **duration** of episodes of a single sleep `[LEVEL]` during `main` sleep divided by the **duration** of episodes of a single sleep `[LEVEL]` during `main` **and** `nap`. This answers the question: is `rem` time more frequent during `main` than `nap` sleep? We do not provide the ratio for `nap` because is complementary. ($countepisode[remstages][main] / countepisode[remstages][all]$)
+|ratiodurationmainwithin`[LEVEL]` |-              |Ratio between the **duration** of episodes of a single sleep `[LEVEL]` during `main` sleep divided by the **duration** of episodes of a single sleep `[LEVEL]` during `main` **and** `nap`. This answers the question: is `rem` time more frequent during `main` than `nap` sleep? We do not provide the ratio for `nap` because is complementary. ($countepisode[remstages][main] / countepisode[remstages][all]$)
 
 
 Features description for `[FITBIT_SLEEP_INTRADAY][PROVIDERS][RAPIDS]` RATIOS `[WITHIN_TYPES]`:
@@ -84,27 +82,15 @@ Features description for `[FITBIT_SLEEP_INTRADAY][PROVIDERS][RAPIDS]` RATIOS `[W
 |ratioduration`[LEVEL]`within`[TYPE]` |-|Ratio between the **duration** of episodes of a single sleep `[LEVEL]` and the **duration** of all episodes of all levels during either `main` or `nap` sleep types. This answers the question: what percentage of all `wake`, `deep`, `light`, and `rem` time was `rem` during `main`/`nap` sleep time? (e.g., $sumduration[remstages][main] / sumduration[all][main]$)
 
 
-Features description for `[FITBIT_SLEEP_INTRADAY][PROVIDERS][RAPIDS][ROUTINE]`:
-
-|Feature                           |Units          |Description                                                  |
-|--------------------------------- |-------------- |-------------------------------------------------------------|
-|starttimefirstmainsleep           |minutes        |Start time (in minutes since `ROUTINE_REFERENCE_TIME`) of the first main sleep episode after `INCLUDE_EPISODES_LATER_THAN`.
-|endtimelastmainsleep              |minutes        |End time (in minutes since `ROUTINE_REFERENCE_TIME`) of the last main sleep episode after `INCLUDE_EPISODES_LATER_THAN`.
-|starttimefirstnap                 |minutes        |Start time (in minutes since `ROUTINE_REFERENCE_TIME`) of the first nap episode after `INCLUDE_EPISODES_LATER_THAN`.
-|endtimelastnap                    |minutes        |End time (in minutes since `ROUTINE_REFERENCE_TIME`) of the last nap episode after `INCLUDE_EPISODES_LATER_THAN`.
-
-
-
-
 
 !!! note "Assumptions/Observations"
-    1. [This diagram](../../img/sleep_intraday_rapids.png) will help you understand how sleep episodes are chunked and grouped within time segments and `LNE-LNE` intervals for the RAPIDS provider.
+    1. [This diagram](../../img/sleep_intraday_rapids.png) will help you understand how sleep episodes are chunked and grouped within time segments for the RAPIDS provider.
     1. Features listed in `[LEVELS_AND_TYPES]` are computed for any levels and types listed in `[SLEEP_LEVELS]` or `[SLEEP_TYPES]`. For example if `STAGES` only contains `[rem, light]` you will not get `countepisode[wake|deep][TYPE]` or sum, max, min, avg, median, or std `duration`. Levels or types in these lists do not influence `RATIOS` or `ROUTINE` features.
     2. Any `[LEVEL]` grouping is done within the elements of each class `CLASSIC`, `STAGES`, and `UNIFIED`. That is, we never combine `CLASSIC` or `STAGES` types to compute features.
     3. The categories for `all` levels (when `INCLUDE_ALL_GROUPS` is `True`) and `all` `SLEEP_TYPES` are not considered for `RATIOS` features as they are always 1.
     3. These features can be computed in time segments of any length, but only the 1-minute sleep chunks within each segment instance will be used.
-    4. Within any time segment instance, any chunks with a local time before `LAST_NIGHT_END` will be discarded. The default `LNE` is 00:00 so no chunks are ignored.
+
-    5. `ROUTINE_REFERENCE_TIME` influences all the `[ROUTINE]` features. If `MIDNIGHT`, the reference for these times is 00:00, if `START_OF_THE_SEGMENT`, the reference time is the start of each segment instance.
+
 
 ## PRICE provider
 

src/features/fitbit_sleep_intraday/episodes/sleep_intraday_episodes.py

@@ -5,7 +5,7 @@ import numpy as np
 def mergeSleepEpisodes(sleep_data, cols_for_groupby):
     sleep_episodes = pd.DataFrame(columns=["device_id", "type_episode_id", "level_episode_id", "level", "unified_level", "is_main_sleep", "type", "timestamp", "duration"])
     if not sleep_data.empty:
-        sleep_data = sleep_data.groupby(by=cols_for_groupby)
+        sleep_data = sleep_data.groupby(by=cols_for_groupby, sort=False)
         sleep_episodes = sleep_data[["timestamp"]].first()
         sleep_episodes["duration"] = sleep_data["duration"].sum()
     

src/features/fitbit_sleep_intraday/price/main.py

@@ -34,7 +34,7 @@ def mergeSleepEpisodes(sleep_data, cols_for_groupby, base_sleep_levels):
     sleep_episodes = pd.DataFrame(columns=["local_segment", "durationinbed", "start_timestamp", "end_timestamp", "local_start_date_time", "local_end_date_time"] + ["duration" + x for x in sleep_level_with_group])
 
     if cols_for_groupby and (not sleep_data.empty):
-        sleep_data = sleep_data.groupby(by=cols_for_groupby)
+        sleep_data = sleep_data.groupby(by=cols_for_groupby, sort=False)
         sleep_episodes = sleep_data[["duration"]].sum().rename(columns={"duration": "durationinbed"})
 
         sleep_episodes["start_timestamp"] = sleep_data["start_timestamp"].first()
@@ -64,6 +64,9 @@ def extractDailyFeatures(sleep_data):
             daily_features["ratio" + col + "withinmain"] = daily_features[col + "main"] / daily_features["durationinbedmain"]
     daily_features.reset_index(inplace=True)
 
+    # Only keep one row per fake_date
+    daily_features.drop_duplicates(subset=["fake_date"], keep="first", inplace=True)
+
     # The day of the week with Monday=0, Sunday=6. Set Friday and Saturday as Weekend, others as Weekday.
     daily_features["is_weekend"] = pd.to_datetime(daily_features["fake_date"]).dt.dayofweek.apply(lambda x: 1 if (x == 4 or x == 5) else 0)
     
@@ -71,9 +74,9 @@ def extractDailyFeatures(sleep_data):
 
 def statsOfDailyFeatures(daily_features, day_type, sleep_levels, intraday_features_to_compute, sleep_intraday_features, levels_include_all_groups):
     if day_type == "WEEKEND":
-        daily_features = daily_features[daily_features["is_weekend"] == 0]
-    elif day_type == "WEEK":
         daily_features = daily_features[daily_features["is_weekend"] == 1]
+    elif day_type == "WEEK":
+        daily_features = daily_features[daily_features["is_weekend"] == 0]
     elif day_type == "ALL":
         pass
     else:
@@ -190,10 +193,11 @@ def price_features(sensor_data_files, time_segment, provider, filter_data_by_seg
         return sleep_intraday_features
 
     # Discard segments shorter than one day
-    sleep_intraday_data["segment_length"] = (sleep_intraday_data["segment_end_timestamp"] - sleep_intraday_data["segment_start_timestamp"]) / 1000 # in seconds
+    sleep_intraday_data[["segment_start_datetime", "segment_end_datetime"]] = sleep_intraday_data["local_segment"].str.split("#", expand=True)[1].str.split(",", expand=True).astype("datetime64[ns]")
+    sleep_intraday_data["segment_length"] = (sleep_intraday_data["segment_end_datetime"] - sleep_intraday_data["segment_start_datetime"]).dt.total_seconds()
     sleep_intraday_data = sleep_intraday_data[sleep_intraday_data["segment_length"] >= 24 * 60 * 60 - 1]
-    del sleep_intraday_data["segment_length"]
+    for col in ["segment_start_datetime", "segment_end_datetime", "segment_length"]:
-
+        del sleep_intraday_data[col]
     # Select main sleep records
     sleep_intraday_data = sleep_intraday_data[sleep_intraday_data["is_main_sleep"] == 1]
 

src/features/fitbit_sleep_intraday/rapids/main.py

@@ -19,11 +19,11 @@ def featuresFullNames(intraday_features_to_compute, sleep_levels_to_compute, sle
         features_fullname.extend(["ratio" + x[0] + x[1] for x in itertools.product(intraday_features_to_compute["RATIOS_TYPE"], sleep_level_with_group)])
     if "ACROSS_TYPES" in intraday_features_to_compute["RATIOS_SCOPE"] and "main" in sleep_types_to_compute:
         features_fullname.extend(["ratio" + x + "main" for x in intraday_features_to_compute["RATIOS_TYPE"]])
-    if "WITHIN_LEVELS" in intraday_features_to_compute["RATIOS_SCOPE"]:
+    if "WITHIN_LEVELS" in intraday_features_to_compute["RATIOS_SCOPE"] and "main" in sleep_types_to_compute:
-        features_fullname.extend(["ratio" + x[0] + x[1] + "within" + x[2] for x in itertools.product(intraday_features_to_compute["RATIOS_TYPE"], set(sleep_types_to_compute) & set(["main", "nap"]), sleep_level_with_group)])
+        features_fullname.extend(["ratio" + x[0] + "mainwithin" + x[1] for x in itertools.product(intraday_features_to_compute["RATIOS_TYPE"], sleep_level_with_group)])
     if "WITHIN_TYPES" in intraday_features_to_compute["RATIOS_SCOPE"]:
         features_fullname.extend(["ratio" + x[0] + x[1] + "within" + x[2] for x in itertools.product(intraday_features_to_compute["RATIOS_TYPE"], sleep_level_with_group,  set(sleep_types_to_compute) & set(["main", "nap"]))])
-    features_fullname.extend(intraday_features_to_compute["ROUTINE"])
+    
     return features_fullname
 
 def mergeSleepEpisodes(sleep_data, cols_for_groupby):
@@ -31,7 +31,7 @@ def mergeSleepEpisodes(sleep_data, cols_for_groupby):
     sleep_episodes = pd.DataFrame(columns=["local_segment", "duration", "start_timestamp", "end_timestamp", "local_start_date_time", "local_end_date_time"])
 
     if cols_for_groupby and (not sleep_data.empty):
-        sleep_data = sleep_data.groupby(by=cols_for_groupby)
+        sleep_data = sleep_data.groupby(by=cols_for_groupby, sort=False)
         sleep_episodes = sleep_data[["duration"]].sum()
         sleep_episodes["start_timestamp"] = sleep_data["start_timestamp"].first()
         sleep_episodes["end_timestamp"] = sleep_data["end_timestamp"].last()
@@ -93,6 +93,8 @@ def allStatsFeatures(sleep_data, base_sleep_levels, base_sleep_types, features,
         sleep_episodes_none = sleep_data[sleep_data["is_main_sleep"] == (1 if sleep_type == "main" else 0)] if sleep_type != "all" else sleep_data
         sleep_episodes_none = mergeSleepEpisodes(sleep_episodes_none, ["local_segment", "type_episode_id"])
         sleep_intraday_features = pd.concat([sleep_intraday_features, statsFeatures(sleep_episodes_none, features, "all" + sleep_type)], axis=1)
+    
+    sleep_intraday_features.fillna(0, inplace=True)
 
     return sleep_intraday_features
 
@@ -134,6 +136,7 @@ def ratiosFeatures(sleep_intraday_features, ratios_types, ratios_scopes, sleep_l
     if "ACROSS_TYPES" in ratios_scopes:
         for ratios_type in ratios_types:
             agg_func = "countepisode" if ratios_type == "count" else "sumduration"
+            # We do not provide the ratio for nap because is complementary.
             across_types = (sleep_intraday_features[agg_func + "allmain"] / sleep_intraday_features[agg_func + "allall"]).to_frame().rename(columns={0: "ratio" + ratios_type + "main"})
             sleep_intraday_features = pd.concat([sleep_intraday_features, across_types], axis=1)
     
@@ -162,7 +165,7 @@ def ratiosFeatures(sleep_intraday_features, ratios_types, ratios_scopes, sleep_l
         agg_func = "countepisode" if ratios_type == "count" else "sumduration"
 
         # WITHIN LEVELS
-        if "WITHIN_LEVELS" in ratios_scopes:
+        if ("WITHIN_LEVELS" in ratios_scopes) and (sleep_type == "main"): # We do not provide the ratio for nap because is complementary.
             within_levels = (sleep_intraday_features[agg_func + sleep_level + sleep_level_group + sleep_type] / sleep_intraday_features[agg_func + sleep_level + sleep_level_group + "all"]).to_frame().rename(columns={0: "ratio" + ratios_type + sleep_type + "within" + sleep_level + sleep_level_group})
             sleep_intraday_features = pd.concat([sleep_intraday_features, within_levels], axis=1)
 
@@ -174,47 +177,11 @@ def ratiosFeatures(sleep_intraday_features, ratios_types, ratios_scopes, sleep_l
     return sleep_intraday_features
 
 
-def singleSleepTypeRoutineFeatures(sleep_intraday_data, routine, routine_reference_time, sleep_type, sleep_intraday_features):
-
-    sleep_intraday_data = sleep_intraday_data[sleep_intraday_data["is_main_sleep"] == (1 if sleep_type == "mainsleep" else 0)]
-    if "starttimefirst" + sleep_type in routine:
-        grouped_first = sleep_intraday_data.groupby(["local_segment"]).first()
-        if routine_reference_time == "MIDNIGHT":
-            sleep_intraday_features["starttimefirst" + sleep_type] = grouped_first["local_start_date_time"].apply(lambda x: x.hour * 60 + x.minute + x.second / 60)
-        elif routine_reference_time == "START_OF_THE_SEGMENT":
-            sleep_intraday_features["starttimefirst" + sleep_type] = (grouped_first["start_timestamp"] - grouped_first["segment_start_timestamp"]) / (60 * 1000)
-        else:
-            raise ValueError("Please check FITBIT_SLEEP_INTRADAY section of config.yaml: ROUTINE_REFERENCE_TIME can only be MIDNIGHT or START_OF_THE_SEGMENT.")
-    
-    if "endtimelast" + sleep_type in routine:
-        grouped_last = sleep_intraday_data.groupby(["local_segment"]).last()
-        if routine_reference_time == "MIDNIGHT":
-            sleep_intraday_features["endtimelast" + sleep_type] = grouped_last["local_end_date_time"].apply(lambda x: x.hour * 60 + x.minute + x.second / 60)
-        elif routine_reference_time == "START_OF_THE_SEGMENT":
-            sleep_intraday_features["endtimelast" + sleep_type] = (grouped_last["end_timestamp"] - grouped_last["segment_start_timestamp"]) / (60 * 1000)
-        else:
-            raise ValueError("Please check FITBIT_SLEEP_INTRADAY section of config.yaml: ROUTINE_REFERENCE_TIME can only be MIDNIGHT or START_OF_THE_SEGMENT.")
-
-    return sleep_intraday_features
-
-def routineFeatures(sleep_intraday_data, routine, routine_reference_time, sleep_type, sleep_intraday_features):        
-    
-    if "starttimefirstmainsleep" in routine or "endtimelastmainsleep" in routine:
-        sleep_intraday_features = singleSleepTypeRoutineFeatures(sleep_intraday_data, routine, routine_reference_time, "mainsleep", sleep_intraday_features)
-    
-    if "starttimefirstnap" in routine or "endtimelastnap" in routine:
-        sleep_intraday_features = singleSleepTypeRoutineFeatures(sleep_intraday_data, routine, routine_reference_time, "nap", sleep_intraday_features)
-    
-    return sleep_intraday_features
 
 
 def rapids_features(sensor_data_files, time_segment, provider, filter_data_by_segment, *args, **kwargs):
-
+    
     sleep_intraday_data = pd.read_csv(sensor_data_files["sensor_data"])
-
-    last_night_end = provider["LAST_NIGHT_END"]
-    routine_reference_time = provider["ROUTINE_REFERENCE_TIME"]
-
     requested_intraday_features = provider["FEATURES"]
     levels_include_all_groups = provider["SLEEP_LEVELS"]["INCLUDE_ALL_GROUPS"]
     requested_sleep_levels = provider["SLEEP_LEVELS"]
@@ -223,8 +190,7 @@ def rapids_features(sensor_data_files, time_segment, provider, filter_data_by_se
     # Name of the features this function can compute
     base_intraday_features = {"LEVELS_AND_TYPES": ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"],
                                 "RATIOS_TYPE": ["count", "duration"],
-                                "RATIOS_SCOPE": ["ACROSS_LEVELS", "ACROSS_TYPES", "WITHIN_LEVELS", "WITHIN_TYPES"],
+                                "RATIOS_SCOPE": ["ACROSS_LEVELS", "ACROSS_TYPES", "WITHIN_LEVELS", "WITHIN_TYPES"]}
-                                "ROUTINE": ["starttimefirstmainsleep", "endtimelastmainsleep", "starttimefirstnap", "endtimelastnap"]}
     base_sleep_levels = {"CLASSIC": ["awake", "restless", "asleep"],
                             "STAGES": ["wake", "deep", "light", "rem"],
                             "UNIFIED": ["awake", "asleep"]}
@@ -238,12 +204,6 @@ def rapids_features(sensor_data_files, time_segment, provider, filter_data_by_se
     # Full names
     features_fullnames = featuresFullNames(intraday_features_to_compute, sleep_levels_to_compute, sleep_types_to_compute, levels_include_all_groups)
     sleep_intraday_features = pd.DataFrame(columns=features_fullnames)
-    
-    # Any 1-minute sleep chuncks with a local time before LAST_NIGHT_END will be discarded.
-    start_minutes = sleep_intraday_data.groupby("start_timestamp").first()["local_time"].apply(lambda x: int(x.split(":")[0]) * 60 + int(x.split(":")[1]) + int(x.split(":")[2]) / 60).to_frame().rename(columns={"local_time": "start_minutes"}).reset_index()
-    sleep_intraday_data = sleep_intraday_data.merge(start_minutes, on="start_timestamp", how="left")
-    sleep_intraday_data = sleep_intraday_data[sleep_intraday_data["start_minutes"] >= last_night_end]
-    del sleep_intraday_data["start_minutes"]
 
     sleep_intraday_data = filter_data_by_segment(sleep_intraday_data, time_segment)
 
@@ -260,9 +220,6 @@ def rapids_features(sensor_data_files, time_segment, provider, filter_data_by_se
         # RATIOS: only compute requested features
         sleep_intraday_features = ratiosFeatures(sleep_intraday_features, intraday_features_to_compute["RATIOS_TYPE"], intraday_features_to_compute["RATIOS_SCOPE"], sleep_levels_to_compute, sleep_types_to_compute)
         
-        # ROUTINE: only compute requested features
-        sleep_intraday_features = routineFeatures(sleep_intraday_data, intraday_features_to_compute["ROUTINE"], routine_reference_time, sleep_types_to_compute, sleep_intraday_features)
-
         # Reset index and discard features which are not requested by user
         sleep_intraday_features.index.name = "local_segment"
         sleep_intraday_features.reset_index(inplace=True)

src/features/utils/utils.py

@@ -71,7 +71,7 @@ def chunk_episodes(sensor_episodes):
     # Merge episodes
     cols_for_groupby = [col for col in sensor_episodes.columns if col not in ["timestamps_segment", "timestamp", "assigned_segments", "start_datetime", "end_datetime", "start_timestamp", "end_timestamp", "duration", "chunked_start_timestamp", "chunked_end_timestamp"]]
 
-    sensor_episodes_grouped = sensor_episodes.groupby(by=cols_for_groupby)
+    sensor_episodes_grouped = sensor_episodes.groupby(by=cols_for_groupby, sort=False)
     merged_sensor_episodes = sensor_episodes_grouped[["duration"]].sum()
 
     merged_sensor_episodes["start_timestamp"] = sensor_episodes_grouped["chunked_start_timestamp"].first()

tests/data/external/aware_csv/fitbit_sleep_intraday_raw.csv

@@ -0,0 +1,23 @@
+timestamp,device_id,type_episode_id,duration,level,is_main_sleep,type,local_date_time
+0,fitbit,0,300,asleep,1,classic,2020-03-06 11:00:00
+0,fitbit,1,14400,deep,1,stages,2020-03-06 17:00:00
+0,fitbit,1,7200,wake,1,stages,2020-03-06 21:00:00
+0,fitbit,1,7200,light,1,stages,2020-03-06 23:00:00
+0,fitbit,2,3600,restless,0,classic,2020-03-07 01:00:00
+0,fitbit,2,3600,asleep,0,classic,2020-03-07 02:00:00
+0,fitbit,3,3600,restless,0,classic,2020-03-07 13:00:00
+0,fitbit,4,3600,rem,1,stages,2020-03-07 22:00:00
+0,fitbit,4,28800,deep,1,stages,2020-03-07 23:00:00
+0,fitbit,5,32400,deep,1,stages,2020-03-08 11:00:00
+0,fitbit,6,25200,deep,1,stages,2020-03-09 06:00:00
+0,fitbit,0,300,asleep,1,classic,2020-10-30 11:00:00
+0,fitbit,1,14400,deep,1,stages,2020-10-30 17:00:00
+0,fitbit,1,7200,wake,1,stages,2020-10-30 21:00:00
+0,fitbit,1,7200,light,1,stages,2020-10-30 23:00:00
+0,fitbit,2,3600,restless,0,classic,2020-10-31 01:00:00
+0,fitbit,2,3600,asleep,0,classic,2020-10-31 02:00:00
+0,fitbit,3,3600,restless,0,classic,2020-10-31 13:00:00
+0,fitbit,4,3600,rem,1,stages,2020-10-31 22:00:00
+0,fitbit,4,28800,deep,1,stages,2020-10-31 23:00:00
+0,fitbit,5,32400,deep,1,stages,2020-11-01 11:00:00
+0,fitbit,6,25200,deep,1,stages,2020-11-02 06:00:00

tests/data/manual/aware_csv/fitbit_sleep_intraday_raw.csv

@@ -0,0 +1,12 @@
+test_time,device_id,type_episode_id,duration,level,is_main_sleep,type
+Fri 11:00:00,fitbit,0,300,asleep,1,classic
+Fri 17:00:00,fitbit,1,14400,deep,1,stages
+Fri 21:00:00,fitbit,1,7200,wake,1,stages
+Fri 23:00:00,fitbit,1,7200,light,1,stages
+Sat 01:00:00,fitbit,2,3600,restless,0,classic
+Sat 02:00:00,fitbit,2,3600,asleep,0,classic
+Sat 13:00:00,fitbit,3,3600,restless,0,classic
+Sat 22:00:00,fitbit,4,3600,rem,1,stages
+Sat 23:00:00,fitbit,4,28800,deep,1,stages
+Sun 11:00:00,fitbit,5,32400,deep,1,stages
+Mon 06:00:00,fitbit,6,25200,deep,1,stages

tests/settings/mtz_event_config.yaml

@@ -324,19 +324,19 @@ FITBIT_DATA_STREAMS:
   # AVAILABLE:
   fitbitjson_mysql: 
     DATABASE_GROUP: MY_GROUP
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
   
   fitbitparsed_mysql: 
     DATABASE_GROUP: MY_GROUP
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
 
   fitbitjson_csv: 
     FOLDER: data/external/fitbit_csv
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
 
   fitbitparsed_csv: 
     FOLDER: tests/data/external/aware_csv
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
 
 # Sensors ------
 FITBIT_CALORIES_INTRADAY:
@@ -393,27 +393,24 @@ FITBIT_SLEEP_SUMMARY:
 
 # See https://www.rapids.science/latest/features/fitbit-sleep-intraday/
 FITBIT_SLEEP_INTRADAY:
-  CONTAINER: sleep_intraday
+  CONTAINER: fitbit_sleep_intraday_raw.csv
   PROVIDERS:
     RAPIDS:
-      COMPUTE: False
+      COMPUTE: True
       FEATURES:
         LEVELS_AND_TYPES: [countepisode, sumduration, maxduration, minduration, avgduration, medianduration, stdduration]
         RATIOS_TYPE: [count, duration]
         RATIOS_SCOPE: [ACROSS_LEVELS, ACROSS_TYPES, WITHIN_LEVELS, WITHIN_TYPES]
-        ROUTINE: [starttimefirstmainsleep, endtimelastmainsleep, starttimefirstnap, endtimelastnap]
       SLEEP_LEVELS:
         INCLUDE_ALL_GROUPS: True
         CLASSIC: [awake, restless, asleep]
         STAGES: [wake, deep, light, rem]
         UNIFIED: [awake, asleep]
       SLEEP_TYPES: [main, nap, all]
-      LAST_NIGHT_END: 0 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight
-      ROUTINE_REFERENCE_TIME: MIDNIGHT # chosen from "MIDNIGHT" and "START_OF_THE_SEGMENT"
       SRC_SCRIPT: src/features/fitbit_sleep_intraday/rapids/main.py
   
     PRICE:
-      COMPUTE: False
+      COMPUTE: True
       FEATURES: [avgduration, avgratioduration, avgstarttimeofepisodemain, avgendtimeofepisodemain, avgmidpointofepisodemain, stdstarttimeofepisodemain, stdendtimeofepisodemain, stdmidpointofepisodemain, socialjetlag, rmssdmeanstarttimeofepisodemain, rmssdmeanendtimeofepisodemain, rmssdmeanmidpointofepisodemain, rmssdmedianstarttimeofepisodemain, rmssdmedianendtimeofepisodemain, rmssdmedianmidpointofepisodemain]
       SLEEP_LEVELS:
         INCLUDE_ALL_GROUPS: True
@@ -421,7 +418,7 @@ FITBIT_SLEEP_INTRADAY:
         STAGES: [wake, deep, light, rem]
         UNIFIED: [awake, asleep]
       DAY_TYPES: [WEEKEND, WEEK, ALL]
-      LAST_NIGHT_END: 660 # number of minutes after the midnight (11:00) 11*60
+      LAST_NIGHT_END: 660 # number of minutes after midnight (11:00) 11*60
       SRC_SCRIPT: src/features/fitbit_sleep_intraday/price/main.py
 
 # See https://www.rapids.science/latest/features/fitbit-steps-summary/

tests/settings/mtz_frequency_config.yaml

@@ -324,19 +324,19 @@ FITBIT_DATA_STREAMS:
   # AVAILABLE:
   fitbitjson_mysql: 
     DATABASE_GROUP: MY_GROUP
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
   
   fitbitparsed_mysql: 
     DATABASE_GROUP: MY_GROUP
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
 
   fitbitjson_csv: 
     FOLDER: data/external/fitbit_csv
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
 
   fitbitparsed_csv: 
     FOLDER: tests/data/external/aware_csv
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
 
 # Sensors ------
 
@@ -394,27 +394,24 @@ FITBIT_SLEEP_SUMMARY:
 
 # See https://www.rapids.science/latest/features/fitbit-sleep-intraday/
 FITBIT_SLEEP_INTRADAY:
-  CONTAINER: sleep_intraday
+  CONTAINER: fitbit_sleep_intraday_raw.csv
   PROVIDERS:
     RAPIDS:
-      COMPUTE: False
+      COMPUTE: True
       FEATURES:
         LEVELS_AND_TYPES: [countepisode, sumduration, maxduration, minduration, avgduration, medianduration, stdduration]
         RATIOS_TYPE: [count, duration]
         RATIOS_SCOPE: [ACROSS_LEVELS, ACROSS_TYPES, WITHIN_LEVELS, WITHIN_TYPES]
-        ROUTINE: [starttimefirstmainsleep, endtimelastmainsleep, starttimefirstnap, endtimelastnap]
       SLEEP_LEVELS:
         INCLUDE_ALL_GROUPS: True
         CLASSIC: [awake, restless, asleep]
         STAGES: [wake, deep, light, rem]
         UNIFIED: [awake, asleep]
       SLEEP_TYPES: [main, nap, all]
-      LAST_NIGHT_END: 0 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight
-      ROUTINE_REFERENCE_TIME: MIDNIGHT # chosen from "MIDNIGHT" and "START_OF_THE_SEGMENT"
       SRC_SCRIPT: src/features/fitbit_sleep_intraday/rapids/main.py
   
     PRICE:
-      COMPUTE: False
+      COMPUTE: True
       FEATURES: [avgduration, avgratioduration, avgstarttimeofepisodemain, avgendtimeofepisodemain, avgmidpointofepisodemain, stdstarttimeofepisodemain, stdendtimeofepisodemain, stdmidpointofepisodemain, socialjetlag, rmssdmeanstarttimeofepisodemain, rmssdmeanendtimeofepisodemain, rmssdmeanmidpointofepisodemain, rmssdmedianstarttimeofepisodemain, rmssdmedianendtimeofepisodemain, rmssdmedianmidpointofepisodemain]
       SLEEP_LEVELS:
         INCLUDE_ALL_GROUPS: True
@@ -422,7 +419,7 @@ FITBIT_SLEEP_INTRADAY:
         STAGES: [wake, deep, light, rem]
         UNIFIED: [awake, asleep]
       DAY_TYPES: [WEEKEND, WEEK, ALL]
-      LAST_NIGHT_END: 660 # number of minutes after the midnight (11:00) 11*60
+      LAST_NIGHT_END: 660 # number of minutes after midnight (11:00) 11*60
       SRC_SCRIPT: src/features/fitbit_sleep_intraday/price/main.py
 
 # See https://www.rapids.science/latest/features/fitbit-steps-summary/

tests/settings/mtz_periodic_config.yaml

@@ -324,19 +324,19 @@ FITBIT_DATA_STREAMS:
   # AVAILABLE:
   fitbitjson_mysql: 
     DATABASE_GROUP: MY_GROUP
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
   
   fitbitparsed_mysql: 
     DATABASE_GROUP: MY_GROUP
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
 
   fitbitjson_csv: 
     FOLDER: data/external/fitbit_csv
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
 
   fitbitparsed_csv: 
     FOLDER: tests/data/external/aware_csv
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
 
 # Sensors ------
 FITBIT_CALORIES_INTRADAY:
@@ -393,27 +393,24 @@ FITBIT_SLEEP_SUMMARY:
 
 # See https://www.rapids.science/latest/features/fitbit-sleep-intraday/
 FITBIT_SLEEP_INTRADAY:
-  CONTAINER: sleep_intraday
+  CONTAINER: fitbit_sleep_intraday_raw.csv
   PROVIDERS:
     RAPIDS:
-      COMPUTE: False
+      COMPUTE: True
       FEATURES:
         LEVELS_AND_TYPES: [countepisode, sumduration, maxduration, minduration, avgduration, medianduration, stdduration]
         RATIOS_TYPE: [count, duration]
         RATIOS_SCOPE: [ACROSS_LEVELS, ACROSS_TYPES, WITHIN_LEVELS, WITHIN_TYPES]
-        ROUTINE: [starttimefirstmainsleep, endtimelastmainsleep, starttimefirstnap, endtimelastnap]
       SLEEP_LEVELS:
         INCLUDE_ALL_GROUPS: True
         CLASSIC: [awake, restless, asleep]
         STAGES: [wake, deep, light, rem]
         UNIFIED: [awake, asleep]
       SLEEP_TYPES: [main, nap, all]
-      LAST_NIGHT_END: 0 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight
-      ROUTINE_REFERENCE_TIME: MIDNIGHT # chosen from "MIDNIGHT" and "START_OF_THE_SEGMENT"
       SRC_SCRIPT: src/features/fitbit_sleep_intraday/rapids/main.py
   
     PRICE:
-      COMPUTE: False
+      COMPUTE: True
       FEATURES: [avgduration, avgratioduration, avgstarttimeofepisodemain, avgendtimeofepisodemain, avgmidpointofepisodemain, stdstarttimeofepisodemain, stdendtimeofepisodemain, stdmidpointofepisodemain, socialjetlag, rmssdmeanstarttimeofepisodemain, rmssdmeanendtimeofepisodemain, rmssdmeanmidpointofepisodemain, rmssdmedianstarttimeofepisodemain, rmssdmedianendtimeofepisodemain, rmssdmedianmidpointofepisodemain]
       SLEEP_LEVELS:
         INCLUDE_ALL_GROUPS: True
@@ -421,7 +418,7 @@ FITBIT_SLEEP_INTRADAY:
         STAGES: [wake, deep, light, rem]
         UNIFIED: [awake, asleep]
       DAY_TYPES: [WEEKEND, WEEK, ALL]
-      LAST_NIGHT_END: 660 # number of minutes after the midnight (11:00) 11*60
+      LAST_NIGHT_END: 660 # number of minutes after midnight (11:00) 11*60
       SRC_SCRIPT: src/features/fitbit_sleep_intraday/price/main.py
 
 # See https://www.rapids.science/latest/features/fitbit-steps-summary/

tests/settings/stz_event_config.yaml

@@ -324,19 +324,19 @@ FITBIT_DATA_STREAMS:
   # AVAILABLE:
   fitbitjson_mysql: 
     DATABASE_GROUP: MY_GROUP
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
   
   fitbitparsed_mysql: 
     DATABASE_GROUP: MY_GROUP
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
 
   fitbitjson_csv: 
     FOLDER: data/external/fitbit_csv
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
 
   fitbitparsed_csv: 
     FOLDER: tests/data/external/aware_csv
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
 
 # Sensors ------
 FITBIT_CALORIES_INTRADAY:
@@ -393,27 +393,24 @@ FITBIT_SLEEP_SUMMARY:
 
 # See https://www.rapids.science/latest/features/fitbit-sleep-intraday/
 FITBIT_SLEEP_INTRADAY:
-  CONTAINER: sleep_intraday
+  CONTAINER: fitbit_sleep_intraday_raw.csv
   PROVIDERS:
     RAPIDS:
-      COMPUTE: False
+      COMPUTE: True
       FEATURES:
         LEVELS_AND_TYPES: [countepisode, sumduration, maxduration, minduration, avgduration, medianduration, stdduration]
         RATIOS_TYPE: [count, duration]
         RATIOS_SCOPE: [ACROSS_LEVELS, ACROSS_TYPES, WITHIN_LEVELS, WITHIN_TYPES]
-        ROUTINE: [starttimefirstmainsleep, endtimelastmainsleep, starttimefirstnap, endtimelastnap]
       SLEEP_LEVELS:
         INCLUDE_ALL_GROUPS: True
         CLASSIC: [awake, restless, asleep]
         STAGES: [wake, deep, light, rem]
         UNIFIED: [awake, asleep]
       SLEEP_TYPES: [main, nap, all]
-      LAST_NIGHT_END: 0 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight
-      ROUTINE_REFERENCE_TIME: MIDNIGHT # chosen from "MIDNIGHT" and "START_OF_THE_SEGMENT"
       SRC_SCRIPT: src/features/fitbit_sleep_intraday/rapids/main.py
   
     PRICE:
-      COMPUTE: False
+      COMPUTE: True
       FEATURES: [avgduration, avgratioduration, avgstarttimeofepisodemain, avgendtimeofepisodemain, avgmidpointofepisodemain, stdstarttimeofepisodemain, stdendtimeofepisodemain, stdmidpointofepisodemain, socialjetlag, rmssdmeanstarttimeofepisodemain, rmssdmeanendtimeofepisodemain, rmssdmeanmidpointofepisodemain, rmssdmedianstarttimeofepisodemain, rmssdmedianendtimeofepisodemain, rmssdmedianmidpointofepisodemain]
       SLEEP_LEVELS:
         INCLUDE_ALL_GROUPS: True
@@ -421,7 +418,7 @@ FITBIT_SLEEP_INTRADAY:
         STAGES: [wake, deep, light, rem]
         UNIFIED: [awake, asleep]
       DAY_TYPES: [WEEKEND, WEEK, ALL]
-      LAST_NIGHT_END: 660 # number of minutes after the midnight (11:00) 11*60
+      LAST_NIGHT_END: 660 # number of minutes after midnight (11:00) 11*60
       SRC_SCRIPT: src/features/fitbit_sleep_intraday/price/main.py
 
 # See https://www.rapids.science/latest/features/fitbit-steps-summary/

tests/settings/stz_frequency_config.yaml

@@ -324,19 +324,19 @@ FITBIT_DATA_STREAMS:
   # AVAILABLE:
   fitbitjson_mysql: 
     DATABASE_GROUP: MY_GROUP
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
   
   fitbitparsed_mysql: 
     DATABASE_GROUP: MY_GROUP
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
 
   fitbitjson_csv: 
     FOLDER: data/external/fitbit_csv
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
 
   fitbitparsed_csv: 
     FOLDER: tests/data/external/aware_csv
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
 
 # Sensors ------
 
@@ -394,27 +394,24 @@ FITBIT_SLEEP_SUMMARY:
 
 # See https://www.rapids.science/latest/features/fitbit-sleep-intraday/
 FITBIT_SLEEP_INTRADAY:
-  CONTAINER: sleep_intraday
+  CONTAINER: fitbit_sleep_intraday_raw.csv
   PROVIDERS:
     RAPIDS:
-      COMPUTE: False
+      COMPUTE: True
       FEATURES:
         LEVELS_AND_TYPES: [countepisode, sumduration, maxduration, minduration, avgduration, medianduration, stdduration]
         RATIOS_TYPE: [count, duration]
         RATIOS_SCOPE: [ACROSS_LEVELS, ACROSS_TYPES, WITHIN_LEVELS, WITHIN_TYPES]
-        ROUTINE: [starttimefirstmainsleep, endtimelastmainsleep, starttimefirstnap, endtimelastnap]
       SLEEP_LEVELS:
         INCLUDE_ALL_GROUPS: True
         CLASSIC: [awake, restless, asleep]
         STAGES: [wake, deep, light, rem]
         UNIFIED: [awake, asleep]
       SLEEP_TYPES: [main, nap, all]
-      LAST_NIGHT_END: 0 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight
-      ROUTINE_REFERENCE_TIME: MIDNIGHT # chosen from "MIDNIGHT" and "START_OF_THE_SEGMENT"
       SRC_SCRIPT: src/features/fitbit_sleep_intraday/rapids/main.py
   
     PRICE:
-      COMPUTE: False
+      COMPUTE: True
       FEATURES: [avgduration, avgratioduration, avgstarttimeofepisodemain, avgendtimeofepisodemain, avgmidpointofepisodemain, stdstarttimeofepisodemain, stdendtimeofepisodemain, stdmidpointofepisodemain, socialjetlag, rmssdmeanstarttimeofepisodemain, rmssdmeanendtimeofepisodemain, rmssdmeanmidpointofepisodemain, rmssdmedianstarttimeofepisodemain, rmssdmedianendtimeofepisodemain, rmssdmedianmidpointofepisodemain]
       SLEEP_LEVELS:
         INCLUDE_ALL_GROUPS: True
@@ -422,7 +419,7 @@ FITBIT_SLEEP_INTRADAY:
         STAGES: [wake, deep, light, rem]
         UNIFIED: [awake, asleep]
       DAY_TYPES: [WEEKEND, WEEK, ALL]
-      LAST_NIGHT_END: 660 # number of minutes after the midnight (11:00) 11*60
+      LAST_NIGHT_END: 660 # number of minutes after midnight (11:00) 11*60
       SRC_SCRIPT: src/features/fitbit_sleep_intraday/price/main.py
 
 # See https://www.rapids.science/latest/features/fitbit-steps-summary/

tests/settings/stz_periodic_config.yaml

@@ -324,19 +324,19 @@ FITBIT_DATA_STREAMS:
   # AVAILABLE:
   fitbitjson_mysql: 
     DATABASE_GROUP: MY_GROUP
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
   
   fitbitparsed_mysql: 
     DATABASE_GROUP: MY_GROUP
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
 
   fitbitjson_csv: 
     FOLDER: data/external/fitbit_csv
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
 
   fitbitparsed_csv: 
     FOLDER: tests/data/external/aware_csv
-    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight. By default, 660 (11:00).
+    SLEEP_SUMMARY_LAST_NIGHT_END: 660 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after midnight. By default, 660 (11:00).
 
 # Sensors ------
 FITBIT_CALORIES_INTRADAY:
@@ -393,27 +393,24 @@ FITBIT_SLEEP_SUMMARY:
 
 # See https://www.rapids.science/latest/features/fitbit-sleep-intraday/
 FITBIT_SLEEP_INTRADAY:
-  CONTAINER: sleep_intraday
+  CONTAINER: fitbit_sleep_intraday_raw.csv
   PROVIDERS:
     RAPIDS:
-      COMPUTE: False
+      COMPUTE: True
       FEATURES:
         LEVELS_AND_TYPES: [countepisode, sumduration, maxduration, minduration, avgduration, medianduration, stdduration]
         RATIOS_TYPE: [count, duration]
         RATIOS_SCOPE: [ACROSS_LEVELS, ACROSS_TYPES, WITHIN_LEVELS, WITHIN_TYPES]
-        ROUTINE: [starttimefirstmainsleep, endtimelastmainsleep, starttimefirstnap, endtimelastnap]
       SLEEP_LEVELS:
         INCLUDE_ALL_GROUPS: True
         CLASSIC: [awake, restless, asleep]
         STAGES: [wake, deep, light, rem]
         UNIFIED: [awake, asleep]
       SLEEP_TYPES: [main, nap, all]
-      LAST_NIGHT_END: 0 # a number ranged from 0 (midnight) to 1439 (23:59) which denotes number of minutes after the midnight
-      ROUTINE_REFERENCE_TIME: MIDNIGHT # chosen from "MIDNIGHT" and "START_OF_THE_SEGMENT"
       SRC_SCRIPT: src/features/fitbit_sleep_intraday/rapids/main.py
   
     PRICE:
-      COMPUTE: False
+      COMPUTE: True
       FEATURES: [avgduration, avgratioduration, avgstarttimeofepisodemain, avgendtimeofepisodemain, avgmidpointofepisodemain, stdstarttimeofepisodemain, stdendtimeofepisodemain, stdmidpointofepisodemain, socialjetlag, rmssdmeanstarttimeofepisodemain, rmssdmeanendtimeofepisodemain, rmssdmeanmidpointofepisodemain, rmssdmedianstarttimeofepisodemain, rmssdmedianendtimeofepisodemain, rmssdmedianmidpointofepisodemain]
       SLEEP_LEVELS:
         INCLUDE_ALL_GROUPS: True
@@ -421,7 +418,7 @@ FITBIT_SLEEP_INTRADAY:
         STAGES: [wake, deep, light, rem]
         UNIFIED: [awake, asleep]
       DAY_TYPES: [WEEKEND, WEEK, ALL]
-      LAST_NIGHT_END: 660 # number of minutes after the midnight (11:00) 11*60
+      LAST_NIGHT_END: 660 # number of minutes after midnight (11:00) 11*60
       SRC_SCRIPT: src/features/fitbit_sleep_intraday/price/main.py
 
 # See https://www.rapids.science/latest/features/fitbit-steps-summary/

tools/config.schema.yaml

@@ -973,7 +973,7 @@ properties:
               - properties:
                   FEATURES:
                     type: object
-                    required: [LEVELS_AND_TYPES, RATIOS_TYPE, RATIOS_SCOPE, ROUTINE]
+                    required: [LEVELS_AND_TYPES, RATIOS_TYPE, RATIOS_SCOPE]
                     properties:
                       LEVELS_AND_TYPES:
                         type: array
@@ -993,12 +993,6 @@ properties:
                         items:
                           type: string
                           enum: [ACROSS_LEVELS, ACROSS_TYPES, WITHIN_LEVELS, WITHIN_TYPES]
-                      ROUTINE:
-                        type: array
-                        uniqueItems: True
-                        items:
-                          type: string
-                          enum: [starttimefirstmainsleep, endtimelastmainsleep, starttimefirstnap, endtimelastnap]
                   SLEEP_LEVELS:
                     type: object
                     required: [INCLUDE_ALL_GROUPS, CLASSIC, STAGES, UNIFIED]
@@ -1029,13 +1023,6 @@ properties:
                     items:
                       type: string
                       enum: [main, nap, all]
-                  LAST_NIGHT_END:
-                      type: number
-                      minimum: 0
-                      maximum: 1439
-                  ROUTINE_REFERENCE_TIME:
-                    type: string
-                    enum: [MIDNIGHT, START_OF_THE_SEGMENT]
           PRICE:
             allOf:
               - $ref: "#/definitions/PROVIDER"