calls_sms_features() now returns all communication features.

Add another test.

data/example_proximity.csv

@@ -0,0 +1,63 @@
+id,timestamp,device_id,_id,double_proximity,accuracy,label,dateTime
+39017,1565802024310,f67354f7-d675-4b76-80c8-123cc4744a5b,2962,0,3,,2019-08-14T17:00:24Z
+39018,1565802051075,f67354f7-d675-4b76-80c8-123cc4744a5b,2963,0,3,,2019-08-14T17:00:51Z
+39019,1565802051354,f67354f7-d675-4b76-80c8-123cc4744a5b,2964,8,3,,2019-08-14T17:00:51Z
+39089,1565010418305,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,51,5,3,,2019-08-05T13:06:58Z
+39090,1565010772188,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,52,5,3,,2019-08-05T13:12:52Z
+39091,1565012334450,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,53,5,3,,2019-08-05T13:38:54Z
+39092,1565013000660,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,54,5,3,,2019-08-05T13:50:00Z
+39093,1565022742894,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,55,0,3,,2019-08-05T16:32:22Z
+39094,1565089295906,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,56,5,3,,2019-08-06T11:01:35Z
+39095,1565096030817,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,57,0,3,,2019-08-06T12:53:50Z
+39096,1565096367694,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,58,5,3,,2019-08-06T12:59:27Z
+39097,1565096408570,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,59,5,3,,2019-08-06T13:00:08Z
+39098,1565116821528,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,60,5,3,,2019-08-06T18:40:21Z
+39099,1565131345333,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,61,0,3,,2019-08-06T22:42:25Z
+39100,1565131375072,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,62,5,3,,2019-08-06T22:42:55Z
+39101,1565131386353,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,63,0,3,,2019-08-06T22:43:06Z
+39102,1565131389213,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,64,5,3,,2019-08-06T22:43:09Z
+39103,1565131448891,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,65,0,3,,2019-08-06T22:44:08Z
+39104,1565131454131,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,66,5,3,,2019-08-06T22:44:14Z
+39105,1565176143083,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,67,0,3,,2019-08-07T11:09:03Z
+39106,1565179569310,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,68,5,3,,2019-08-07T12:06:09Z
+39107,1565180699173,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,69,5,3,,2019-08-07T12:24:59Z
+39108,1565182538578,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,70,5,3,,2019-08-07T12:55:38Z
+39109,1565192592776,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,71,0,3,,2019-08-07T15:43:12Z
+39110,1565216023797,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,72,5,3,,2019-08-07T22:13:43Z
+39111,1565248358647,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,73,0,3,,2019-08-08T07:12:38Z
+39112,1565275859157,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,74,5,3,,2019-08-08T14:50:59Z
+39113,1565304201431,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,75,0,3,,2019-08-08T22:43:21Z
+39114,1565304229591,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,76,5,3,,2019-08-08T22:43:49Z
+39115,1565304262050,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,77,0,3,,2019-08-08T22:44:22Z
+39116,1565613142970,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,78,5,3,,2019-08-12T12:32:22Z
+39117,1565618266531,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,79,5,3,,2019-08-12T13:57:46Z
+39118,1565618410488,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,80,5,3,,2019-08-12T14:00:10Z
+39119,1565618704942,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,81,5,3,,2019-08-12T14:05:04Z
+39120,1565619005315,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,82,5,3,,2019-08-12T14:10:05Z
+39121,1565619405904,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,83,5,3,,2019-08-12T14:16:45Z
+39122,1565619678037,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,84,5,3,,2019-08-12T14:21:18Z
+39123,1565621206713,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,85,5,3,,2019-08-12T14:46:46Z
+39124,1565626622125,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,86,5,3,,2019-08-12T16:17:02Z
+39125,1565684876738,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,87,5,3,,2019-08-13T08:27:56Z
+39126,1565684956618,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,88,5,3,,2019-08-13T08:29:16Z
+39127,1565684965647,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,89,5,3,,2019-08-13T08:29:25Z
+39128,1565685092246,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,90,5,3,,2019-08-13T08:31:32Z
+39129,1565685136337,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,91,5,3,,2019-08-13T08:32:16Z
+39130,1565685147453,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,92,5,3,,2019-08-13T08:32:27Z
+39131,1565685212523,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,93,5,3,,2019-08-13T08:33:32Z
+39132,1565703397796,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,94,0,3,,2019-08-13T13:36:37Z
+39133,1565776203019,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,95,5,3,,2019-08-14T09:50:03Z
+39134,1565776434168,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,96,5,3,,2019-08-14T09:53:54Z
+39135,1565776435231,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,97,0,3,,2019-08-14T09:53:55Z
+39136,1565776443368,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,98,5,3,,2019-08-14T09:54:03Z
+39137,1565779277109,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,99,0,3,,2019-08-14T10:41:17Z
+39138,1565780016327,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,100,5,3,,2019-08-14T10:53:36Z
+39139,1565780027437,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,101,5,3,,2019-08-14T10:53:47Z
+39140,1565783470934,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,102,5,3,,2019-08-14T11:51:10Z
+39141,1565783801540,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,103,0,3,,2019-08-14T11:56:41Z
+39142,1565783802120,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,104,5,3,,2019-08-14T11:56:42Z
+39143,1565783861495,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,105,5,3,,2019-08-14T11:57:41Z
+39144,1565785318762,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,106,0,3,,2019-08-14T12:21:58Z
+39145,1565785319346,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,107,5,3,,2019-08-14T12:21:59Z
+39146,1565960121019,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,108,5,3,,2019-08-16T12:55:21Z
+39147,1565960226792,fdb06d4a-ee6e-4336-9a96-fc8d2715f243,109,5,3,,2019-08-16T12:57:06Z

exploration/ex_ml_pipeline.py

@@ -148,3 +148,28 @@ lin_reg_proximity.score(
     df_full_data_daily_means[["freq_prox_near", "prop_prox_near"]],
     df_full_data_daily_means["PA"],
 )
+
+# %% [markdown]
+# # Merging these into a pipeline
+
+# %%
+from machine_learning import pipeline
+
+# %%
+ml_pipeline = pipeline.MachineLearningPipeline(
+    labels_questionnaire="PANAS", data_types="proximity"
+)
+
+# %%
+ml_pipeline.get_labels()
+
+# %% tags=[]
+ml_pipeline.get_sensor_data()
+
+# %%
+ml_pipeline.aggregate_daily()
+
+# %%
+ml_pipeline.df_full_data_daily_means
+
+# %%

features/communication.py

@@ -8,6 +8,43 @@ from setup import db_engine, session
 call_types = {1: "incoming", 2: "outgoing", 3: "missed"}
 sms_types = {1: "received", 2: "sent"}
 
+FEATURES_CALLS = (
+    ["no_calls_all"]
+    + ["no_" + call_type for call_type in call_types.values()]
+    + ["duration_total_" + call_types.get(1), "duration_total_" + call_types.get(2)]
+    + ["duration_max_" + call_types.get(1), "duration_max_" + call_types.get(2)]
+    + ["no_" + call_types.get(1) + "_ratio", "no_" + call_types.get(2) + "_ratio"]
+    + ["no_contacts_calls"]
+)
+
+# FEATURES_CALLS =
+# ["no_calls_all",
+# "no_incoming", "no_outgoing", "no_missed",
+# "duration_total_incoming", "duration_total_outgoing",
+# "duration_max_incoming", "duration_max_outgoing",
+# "no_incoming_ratio", "no_outgoing_ratio",
+# "no_contacts"]
+
+FEATURES_SMS = (
+    ["no_sms_all"]
+    + ["no_" + sms_type for sms_type in sms_types.values()]
+    + ["no_" + sms_types.get(1) + "_ratio", "no_" + sms_types.get(2) + "_ratio"]
+    + ["no_contacts_sms"]
+)
+# FEATURES_SMS =
+# ["no_sms_all",
+#  "no_received", "no_sent",
+#  "no_received_ratio", "no_sent_ratio",
+#  "no_contacts"]
+
+FEATURES_CONTACT = [
+    "proportion_calls_all",
+    "proportion_calls_incoming",
+    "proportion_calls_outgoing",
+    "proportion_calls_contacts",
+    "proportion_calls_missed_sms_received",
+]
+
 
 def get_call_data(usernames: Collection) -> pd.DataFrame:
     """
@@ -114,10 +151,12 @@ def count_comms(comm_df: pd.DataFrame) -> pd.DataFrame:
         These are:
         * the number of calls by type (incoming, outgoing missed) and in total,
         * the ratio of incoming and outgoing calls to the total number of calls,
-        * the total and maximum duration of calls by type, and
-        * the number of messages by type (received, sent).
+        * the total and maximum duration of calls by type,
+        * the number of messages by type (received, sent), and
+        * the number of communication contacts by type. 
     """
     if "call_type" in comm_df:
+        data_type = "calls"
         comm_counts = (
             comm_df.value_counts(subset=["participant_id", "call_type"])
             .unstack()
@@ -125,11 +164,11 @@ def count_comms(comm_df: pd.DataFrame) -> pd.DataFrame:
             .add_prefix("no_")
         )
         # Count calls by type.
-        comm_counts["no_all"] = comm_counts.sum(axis=1)
+        comm_counts["no_calls_all"] = comm_counts.sum(axis=1)
         # Add a total count of calls.
         comm_counts = comm_counts.assign(
-            no_incoming_ratio=lambda x: x.no_incoming / x.no_all,
-            no_outgoing_ratio=lambda x: x.no_outgoing / x.no_all,
+            no_incoming_ratio=lambda x: x.no_incoming / x.no_calls_all,
+            no_outgoing_ratio=lambda x: x.no_outgoing / x.no_calls_all,
         )
         # Ratio of incoming and outgoing calls to all calls.
         comm_duration_total = (
@@ -159,44 +198,56 @@ def count_comms(comm_df: pd.DataFrame) -> pd.DataFrame:
         # If there were no missed calls, this exception is raised.
         # But we are dropping the column anyway, so no need to deal with the exception.
     elif "message_type" in comm_df:
+        data_type = "sms"
         comm_counts = (
             comm_df.value_counts(subset=["participant_id", "message_type"])
             .unstack()
             .rename(columns=sms_types)
             .add_prefix("no_")
         )
-        comm_counts["no_all"] = comm_counts.sum(axis=1)
+        comm_counts["no_sms_all"] = comm_counts.sum(axis=1)
         # Add a total count of messages.
         comm_features = comm_counts.assign(
-            no_received_ratio=lambda x: x.no_received / x.no_all,
-            no_sent_ratio=lambda x: x.no_sent / x.no_all,
+            no_received_ratio=lambda x: x.no_received / x.no_sms_all,
+            no_sent_ratio=lambda x: x.no_sent / x.no_sms_all,
         )
         # Ratio of incoming and outgoing messages to all messages.
     else:
         raise KeyError("The dataframe contains neither call_type or message_type")
+    comm_contacts_counts = (
+        enumerate_contacts(comm_df)
+        .groupby(["participant_id"])
+        .nunique()["contact_id"]
+        .rename("no_contacts_" + data_type)
+    )
+    # Number of communication contacts
+    comm_features = comm_features.join(comm_contacts_counts)
     return comm_features
 
 
-def contact_features(df_enumerated: pd.DataFrame) -> pd.DataFrame:
+def contact_features(comm_df: pd.DataFrame) -> pd.DataFrame:
     """
-    Counts the number of people contacted (for each participant) and, if
-    df_enumerated is a dataframe containing calls data, the total duration
-    of calls between a participant and each of her contacts. 
+    For each participant and for each of his contacts, this function
+    counts the number of communications (by type) between them. If the
+    argument passed is a dataframe with calls data, it additionally counts
+    the total duration of calls between every pair (participant, contact).
 
     Parameters
     ----------
-    df_enumerated: pd.DataFrame
-        A dataframe of calls or SMSes; return of function enumerate_contacts.
+    comm_df: pd.DataFrame
+        A dataframe of calls or SMSes.
 
     Returns
     -------
     comm_df: pd.DataFrame
-        The altered dataframe with the column no_contacts and, if df_enumerated
-        contains calls data, an additional column total_call_duration.
+        A new dataframe with a row for each pair (participant, contact).
     """
-
+    df_enumerated = enumerate_contacts(comm_df)
+    contacts_count = (
+        df_enumerated.groupby(["participant_id", "contact_id"]).size().reset_index()
+    )
     # Check whether df contains calls or SMS data since some
-    # features we want to calculate are type-specyfic
+    # features we want to calculate are type-specific
     if "call_duration" in df_enumerated:
         # Add a column with the total duration of calls between two people
         duration_count = (
@@ -207,33 +258,14 @@ def contact_features(df_enumerated: pd.DataFrame) -> pd.DataFrame:
             .reset_index()  # Make index (which is actually the participant id) a normal column
             .rename(columns={"call_duration": "total_call_duration"})
         )
-        # The new dataframe now contains columns containing information about
-        # participants, callers and the total duration of their calls. All that
-        # is now left to do is to merge the original df with the new one.
-        df_enumerated = df_enumerated.merge(
+        contacts_count = contacts_count.merge(
             duration_count, on=["participant_id", "contact_id"]
         )
-
-    contact_count = (
-        df_enumerated.groupby(["participant_id"])
-        .nunique()[
-            "contact_id"
-        ]  # For each participant, count the number of distinct contacts
-        .reset_index()  # Make index (which is actually the participant id) a normal column
-        .rename(columns={"contact_id": "no_contacts"})
-    )
-
-    df_enumerated = (
-        # Merge df with the newely created df containing info about number of contacts
-        df_enumerated.merge(contact_count, on="participant_id")
-        # Sort first by participant_id and then by contact_id and
-        # thereby restore the inital ordering of input dataframes.
-        .sort_values(["participant_id", "contact_id"])
-    )
-
+        contacts_count.rename(columns={0: "no_calls"}, inplace=True)
+    else:
+        contacts_count.rename(columns={0: "no_sms"}, inplace=True)
     # TODO:Determine work vs non-work contacts by work hours heuristics
-
-    return df_enumerated
+    return contacts_count
 
 
 def calls_sms_features(df_calls: pd.DataFrame, df_sms: pd.DataFrame) -> pd.DataFrame:
@@ -245,14 +277,14 @@ def calls_sms_features(df_calls: pd.DataFrame, df_sms: pd.DataFrame) -> pd.DataF
     df_calls: pd.DataFrame
         A dataframe of calls (return of get_call_data).
     df_sms: pd.DataFrame
-        A dataframe of calls (return of get_sms_data).
+        A dataframe of SMSes (return of get_sms_data).
 
     Returns
     -------
     df_calls_sms: pd.DataFrame
         The list of features relating calls and sms data for every participant.
         These are:
-        * proportion_calls:
+        * proportion_calls_all:
             proportion of calls in total number of communications
         * proportion_calls_incoming:
             proportion of incoming calls in total number of incoming/received communications
@@ -263,62 +295,24 @@ def calls_sms_features(df_calls: pd.DataFrame, df_sms: pd.DataFrame) -> pd.DataF
         * proportion_calls_contacts:
             proportion of calls contacts in total number of communication contacts
     """
-
     count_calls = count_comms(df_calls)
     count_sms = count_comms(df_sms)
-
-    count_joined = (
-        count_calls.merge(
-            count_sms, on="participant_id", suffixes=("_calls", "_sms")
-        )  # Merge calls and sms features
-        .reset_index()  # Make participant_id a regular column
-        .assign(
-            proportion_calls=(
-                lambda x: x.no_all_calls / (x.no_all_calls + x.no_all_sms)
-            ),
-            proportion_calls_incoming=(
-                lambda x: x.no_incoming / (x.no_incoming + x.no_received)
-            ),
-            proportion_calls_missed_sms_received=(
-                lambda x: x.no_missed / (x.no_missed + x.no_received)
-            ),
-            proportion_calls_outgoing=(
-                lambda x: x.no_outgoing / (x.no_outgoing + x.no_sent)
-            )
-            # Calculate new features and create additional columns
-        )[
-            [
-                "participant_id",
-                "proportion_calls",
-                "proportion_calls_incoming",
-                "proportion_calls_outgoing",
-                "proportion_calls_missed_sms_received",
-            ]
-        ]  # Filter out only the relevant features
+    count_joined = count_calls.join(count_sms).assign(
+        proportion_calls_all=(
+            lambda x: x.no_calls_all / (x.no_calls_all + x.no_sms_all)
+        ),
+        proportion_calls_incoming=(
+            lambda x: x.no_incoming / (x.no_incoming + x.no_received)
+        ),
+        proportion_calls_missed_sms_received=(
+            lambda x: x.no_missed / (x.no_missed + x.no_received)
+        ),
+        proportion_calls_outgoing=(
+            lambda x: x.no_outgoing / (x.no_outgoing + x.no_sent)
+        ),
+        proportion_calls_contacts=(
+            lambda x: x.no_contacts_calls / (x.no_contacts_calls + x.no_contacts_sms)
+        )
+        # Calculate new features and create additional columns
     )
-
-    features_calls = contact_features(enumerate_contacts(df_calls))
-    features_sms = contact_features(enumerate_contacts(df_sms))
-
-    features_joined = (
-        features_calls.merge(
-            features_sms, on="participant_id", suffixes=("_calls", "_sms")
-        )  # Merge calls and sms features
-        .reset_index()  # Make participant_id a regular column
-        .assign(
-            proportion_calls_contacts=(
-                lambda x: x.no_contacts_calls
-                / (x.no_contacts_calls + x.no_contacts_sms)
-            )  # Calculate new features and create additional columns
-        )[
-            ["participant_id", "proportion_calls_contacts"]
-        ]  # Filter out only the relevant features
-        # Since we are interested only in some features and ignored
-        # others, a lot of duplicate rows were created. Remove them.
-        .drop_duplicates()
-    )
-
-    # Join the newly created dataframes
-    df_calls_sms = count_joined.merge(features_joined, on="participant_id")
-
-    return df_calls_sms
+    return count_joined

features/proximity.py

@@ -5,6 +5,8 @@ import pandas as pd
 from config.models import Participant, Proximity
 from setup import db_engine, session
 
+FEATURES_PROXIMITY = ["freq_prox_near", "prop_prox_near"]
+
 
 def get_proximity_data(usernames: Collection) -> pd.DataFrame:
     """
@@ -56,7 +58,7 @@ def recode_proximity(df_proximity: pd.DataFrame) -> pd.DataFrame:
 
 
 def count_proximity(
-    df_proximity: pd.DataFrame, group_by: Collection = ["participant_id"]
+    df_proximity: pd.DataFrame, group_by: Collection = None
 ) -> pd.DataFrame:
     """
     The function counts how many times a "near" value occurs in proximity
@@ -75,6 +77,8 @@ def count_proximity(
     df_proximity_features: pd.DataFrame
         A dataframe with the count of "near" proximity values and their relative count.
     """
+    if group_by is None:
+        group_by = ["participant_id"]
     if "bool_prox_near" not in df_proximity:
         df_proximity = recode_proximity(df_proximity)
     df_proximity["bool_prox_far"] = ~df_proximity["bool_prox_near"]

features/screen.py

@@ -38,8 +38,10 @@ def identify_screen_sequence(df_screen: pd.DataFrame) -> pd.DataFrame:
     # - OFF -> ON -> unlocked (a true phone unlock)
     # - OFF -> ON -> OFF/locked (no unlocking, i.e. a screen status check)
     # Consider that screen data is sometimes unreliable as shown in expl_screen.ipynb:
-    # "I have also seen off -> on -> unlocked (with 2 - locked missing)
-    #   and off -> locked -> on -> off -> locked (*again*)."
+    # "I have also seen
+    # off -> on -> unlocked (with 2 - locked missing)
+    # and
+    # off -> locked -> on -> off -> locked (*again*)."
     # Either clean the data beforehand or deal with these inconsistencies in this function.
     pass
 

machine_learning/__init__.py

@@ -0,0 +1,7 @@
+QUESTIONNAIRE_IDS = {"PANAS": {"PA": 8.0, "NA": 9.0}}
+
+QUESTIONNAIRE_IDS_RENAME = {}
+
+for questionnaire in QUESTIONNAIRE_IDS.items():
+    for k, v in questionnaire[1].items():
+        QUESTIONNAIRE_IDS_RENAME[v] = k

machine_learning/pipeline.py

@@ -0,0 +1,125 @@
+import datetime
+
+import pandas as pd
+from sklearn.model_selection import cross_val_score
+
+import participants.query_db
+from features import esm, helper, proximity
+from machine_learning import QUESTIONNAIRE_IDS, QUESTIONNAIRE_IDS_RENAME
+
+
+class MachineLearningPipeline:
+    def __init__(
+        self,
+        labels_questionnaire,
+        labels_scale,
+        data_types,
+        participants_usernames=None,
+        feature_names=None,
+        grouping_variable=None,
+    ):
+        if participants_usernames is None:
+            participants_usernames = participants.query_db.get_usernames(
+                collection_start=datetime.date.fromisoformat("2020-08-01")
+            )
+        self.participants_usernames = participants_usernames
+        self.labels_questionnaire = labels_questionnaire
+        self.data_types = data_types
+
+        if feature_names is None:
+            self.feature_names = []
+        self.df_features = pd.DataFrame()
+        self.labels_scale = labels_scale
+        self.df_labels = pd.DataFrame()
+        self.grouping_variable = grouping_variable
+        self.df_groups = pd.DataFrame()
+
+        self.model = None
+        self.validation_method = None
+
+        self.df_esm = pd.DataFrame()
+        self.df_esm_preprocessed = pd.DataFrame()
+        self.df_esm_interest = pd.DataFrame()
+        self.df_esm_clean = pd.DataFrame()
+
+        self.df_proximity = pd.DataFrame()
+
+        self.df_full_data_daily_means = pd.DataFrame()
+        self.df_esm_daily_means = pd.DataFrame()
+        self.df_proximity_daily_counts = pd.DataFrame()
+
+    def get_labels(self):
+        self.df_esm = esm.get_esm_data(self.participants_usernames)
+        self.df_esm_preprocessed = esm.preprocess_esm(self.df_esm)
+        if self.labels_questionnaire == "PANAS":
+            self.df_esm_interest = self.df_esm_preprocessed[
+                (
+                    self.df_esm_preprocessed["questionnaire_id"]
+                    == QUESTIONNAIRE_IDS.get("PANAS").get("PA")
+                )
+                | (
+                    self.df_esm_preprocessed["questionnaire_id"]
+                    == QUESTIONNAIRE_IDS.get("PANAS").get("NA")
+                )
+            ]
+        self.df_esm_clean = esm.clean_up_esm(self.df_esm_interest)
+
+    def get_sensor_data(self):
+        if "proximity" in self.data_types:
+            self.df_proximity = proximity.get_proximity_data(
+                self.participants_usernames
+            )
+            self.df_proximity = helper.get_date_from_timestamp(self.df_proximity)
+            self.df_proximity = proximity.recode_proximity(self.df_proximity)
+
+    def aggregate_daily(self):
+        self.df_esm_daily_means = (
+            self.df_esm_clean.groupby(["participant_id", "date_lj", "questionnaire_id"])
+            .esm_user_answer_numeric.agg("mean")
+            .reset_index()
+            .rename(columns={"esm_user_answer_numeric": "esm_numeric_mean"})
+        )
+        self.df_esm_daily_means = (
+            self.df_esm_daily_means.pivot(
+                index=["participant_id", "date_lj"],
+                columns="questionnaire_id",
+                values="esm_numeric_mean",
+            )
+            .reset_index(col_level=1)
+            .rename(columns=QUESTIONNAIRE_IDS_RENAME)
+            .set_index(["participant_id", "date_lj"])
+        )
+        self.df_full_data_daily_means = self.df_esm_daily_means.copy()
+        if "proximity" in self.data_types:
+            self.df_proximity_daily_counts = proximity.count_proximity(
+                self.df_proximity, ["participant_id", "date_lj"]
+            )
+            self.df_full_data_daily_means = self.df_full_data_daily_means.join(
+                self.df_proximity_daily_counts
+            )
+
+    def assign_columns(self):
+        self.df_features = self.df_full_data_daily_means[self.feature_names]
+        self.df_labels = self.df_full_data_daily_means[self.labels_scale]
+        if self.grouping_variable:
+            self.df_groups = self.df_full_data_daily_means[self.grouping_variable]
+        else:
+            self.df_groups = None
+
+    def validate_model(self):
+        if self.model is None:
+            raise AttributeError(
+                "Please, specify a machine learning model first, by setting the .model attribute."
+            )
+        if self.validation_method is None:
+            raise AttributeError(
+                "Please, specify a cross validation method first, by setting the .validation_method attribute."
+            )
+        cross_val_score(
+            estimator=self.model,
+            X=self.df_features,
+            y=self.df_labels,
+            groups=self.df_groups,
+            cv=self.validation_method,
+            n_jobs=-1,
+        )

test/test_communication.py

@@ -5,7 +5,7 @@ import pandas as pd
 from numpy.random import default_rng
 from pandas.testing import assert_series_equal
 
-from features.communication import count_comms, enumerate_contacts, get_call_data
+from features.communication import *
 
 rng = default_rng()
 
@@ -76,10 +76,18 @@ class CallsFeatures(unittest.TestCase):
 
     def test_count_comms_calls(self):
         self.features = count_comms(self.calls)
-        print(self.features)
         self.assertIsInstance(self.features, pd.DataFrame)
+        self.assertCountEqual(self.features.columns.to_list(), FEATURES_CALLS)
 
     def test_count_comms_sms(self):
         self.features = count_comms(self.sms)
-        print(self.features)
         self.assertIsInstance(self.features, pd.DataFrame)
+        self.assertCountEqual(self.features.columns.to_list(), FEATURES_SMS)
+
+    def test_calls_sms_features(self):
+        self.features_call_sms = calls_sms_features(self.calls, self.sms)
+        self.assertIsInstance(self.features_call_sms, pd.DataFrame)
+        self.assertCountEqual(
+            self.features_call_sms.columns.to_list(),
+            FEATURES_CALLS + FEATURES_SMS + FEATURES_CONTACT,
+        )

test/test_proximity.py

@@ -0,0 +1,31 @@
+import unittest
+
+from features.proximity import *
+
+
+class ProximityFeatures(unittest.TestCase):
+    df_proximity = pd.DataFrame()
+    df_proximity_recoded = pd.DataFrame()
+    df_proximity_features = pd.DataFrame()
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        cls.df_proximity = pd.read_csv("../data/example_proximity.csv")
+        cls.df_proximity["participant_id"] = 99
+
+    def test_recode_proximity(self):
+        self.df_proximity_recoded = recode_proximity(self.df_proximity)
+        self.assertIn("bool_prox_near", self.df_proximity_recoded)
+        # Is the recoded column present?
+        self.assertIn(True, self.df_proximity_recoded.bool_prox_near)
+        # Are there "near" values in the data?
+        self.assertIn(False, self.df_proximity_recoded.bool_prox_near)
+        # Are there "far" values in the data?
+
+    def test_count_proximity(self):
+        self.df_proximity_recoded = recode_proximity(self.df_proximity)
+        self.df_proximity_features = count_proximity(self.df_proximity_recoded)
+        print(self.df_proximity_features.columns)
+        self.assertCountEqual(
+            self.df_proximity_features.columns.to_list(), FEATURES_PROXIMITY
+        )