exploration/expl_communication.py

@@ -13,15 +13,14 @@
 #     name: straw2analysis
 # ---
 
-# %%
-import importlib
-
 # %%
 # %matplotlib inline
 import os
 import sys
 
 import matplotlib.pyplot as plt
+
+# %%
 import seaborn as sns
 
 nb_dir = os.path.split(os.getcwd())[0]
@@ -29,29 +28,21 @@ if nb_dir not in sys.path:
     sys.path.append(nb_dir)
 
 # %%
-from features import communication, helper
-
-# %%
-importlib.reload(communication)
+from features.communication import *
 
 # %% [markdown]
 # # Example of communication data and feature calculation
 
 # %%
-df_calls = communication.get_call_data(["nokia_0000003"])
+df_calls = get_call_data(["nokia_0000003"])
 print(df_calls)
 
 # %%
-df_calls = helper.get_date_from_timestamp(df_calls)
-communication.count_comms(df_calls, ["date_lj"])
+count_comms(df_calls)
 
 # %%
-df_sms = communication.get_sms_data(["nokia_0000003"])
-df_sms = helper.get_date_from_timestamp(df_sms)
-communication.count_comms(df_sms, ["date_lj"])
-
-# %%
-communication.calls_sms_features(df_calls, df_sms, ["date_lj"])
+df_sms = get_sms_data(["nokia_0000003"])
+count_comms(df_sms)
 
 # %% [markdown]
 # # Call data

features/communication.py

@@ -135,7 +135,7 @@ def enumerate_contacts(comm_df: pd.DataFrame) -> pd.DataFrame:
     return comm_df
 
 
-def count_comms(comm_df: pd.DataFrame, group_by=None) -> pd.DataFrame:
+def count_comms(comm_df: pd.DataFrame) -> pd.DataFrame:
     """
     Calculate frequencies (and duration) of messages (or calls), grouped by their types.
 
@@ -143,9 +143,6 @@ def count_comms(comm_df: pd.DataFrame, group_by=None) -> pd.DataFrame:
     ----------
     comm_df: pd.DataFrame
         A dataframe of calls or SMSes.
-    group_by: list
-        A list of strings, specifying by which parameters to group.
-        By default, the features are calculated per participant, but could be "date_lj" etc.
 
     Returns
     -------
@@ -158,13 +155,11 @@ def count_comms(comm_df: pd.DataFrame, group_by=None) -> pd.DataFrame:
         * the number of messages by type (received, sent), and
         * the number of communication contacts by type. 
     """
-    if group_by is None:
-        group_by = []
     if "call_type" in comm_df:
         data_type = "calls"
         comm_counts = (
-            comm_df.value_counts(subset=group_by + ["participant_id", "call_type"])
-            .unstack(level="call_type", fill_value=0)
+            comm_df.value_counts(subset=["participant_id", "call_type"])
+            .unstack()
             .rename(columns=call_types)
             .add_prefix("no_")
         )
@@ -177,17 +172,17 @@ def count_comms(comm_df: pd.DataFrame, group_by=None) -> pd.DataFrame:
         )
         # Ratio of incoming and outgoing calls to all calls.
         comm_duration_total = (
-            comm_df.groupby(group_by + ["participant_id", "call_type"])
+            comm_df.groupby(["participant_id", "call_type"])
             .sum()["call_duration"]
-            .unstack(level="call_type", fill_value=0)
+            .unstack()
             .rename(columns=call_types)
             .add_prefix("duration_total_")
         )
         # Total call duration by type.
         comm_duration_max = (
-            comm_df.groupby(group_by + ["participant_id", "call_type"])
+            comm_df.groupby(["participant_id", "call_type"])
             .max()["call_duration"]
-            .unstack(level="call_type", fill_value=0)
+            .unstack()
             .rename(columns=call_types)
             .add_prefix("duration_max_")
         )
@@ -205,8 +200,8 @@ def count_comms(comm_df: pd.DataFrame, group_by=None) -> pd.DataFrame:
     elif "message_type" in comm_df:
         data_type = "sms"
         comm_counts = (
-            comm_df.value_counts(subset=group_by + ["participant_id", "message_type"])
-            .unstack(level="message_type", fill_value=0)
+            comm_df.value_counts(subset=["participant_id", "message_type"])
+            .unstack()
             .rename(columns=sms_types)
             .add_prefix("no_")
         )
@@ -221,7 +216,7 @@ def count_comms(comm_df: pd.DataFrame, group_by=None) -> pd.DataFrame:
         raise KeyError("The dataframe contains neither call_type or message_type")
     comm_contacts_counts = (
         enumerate_contacts(comm_df)
-        .groupby(group_by + ["participant_id"])
+        .groupby(["participant_id"])
         .nunique()["contact_id"]
         .rename("no_contacts_" + data_type)
     )
@@ -273,9 +268,7 @@ def contact_features(comm_df: pd.DataFrame) -> pd.DataFrame:
     return contacts_count
 
 
-def calls_sms_features(
-    df_calls: pd.DataFrame, df_sms: pd.DataFrame, group_by=None
-) -> pd.DataFrame:
+def calls_sms_features(df_calls: pd.DataFrame, df_sms: pd.DataFrame) -> pd.DataFrame:
     """
     Calculates additional features relating calls and sms data.
 
@@ -285,9 +278,6 @@ def calls_sms_features(
         A dataframe of calls (return of get_call_data).
     df_sms: pd.DataFrame
         A dataframe of SMSes (return of get_sms_data).
-    group_by: list
-        A list of strings, specifying by which parameters to group.
-        By default, the features are calculated per participant, but could be "date_lj" etc.
 
     Returns
     -------
@@ -305,38 +295,24 @@ def calls_sms_features(
         * proportion_calls_contacts:
             proportion of calls contacts in total number of communication contacts
     """
-    if group_by is None:
-        group_by = []
-    count_calls = count_comms(df_calls, group_by)
-    count_sms = count_comms(df_sms, group_by)
-    count_joined = (
-        count_calls.merge(
-            count_sms,
-            how="outer",
-            left_index=True,
-            right_index=True,
-            validate="one_to_one",
+    count_calls = count_comms(df_calls)
+    count_sms = count_comms(df_sms)
+    count_joined = count_calls.join(count_sms).assign(
+        proportion_calls_all=(
+            lambda x: x.no_calls_all / (x.no_calls_all + x.no_sms_all)
+        ),
+        proportion_calls_incoming=(
+            lambda x: x.no_incoming / (x.no_incoming + x.no_received)
+        ),
+        proportion_calls_missed_sms_received=(
+            lambda x: x.no_missed / (x.no_missed + x.no_received)
+        ),
+        proportion_calls_outgoing=(
+            lambda x: x.no_outgoing / (x.no_outgoing + x.no_sent)
+        ),
+        proportion_calls_contacts=(
+            lambda x: x.no_contacts_calls / (x.no_contacts_calls + x.no_contacts_sms)
         )
-        .fillna(0, downcast="infer")
-        .assign(
-            proportion_calls_all=(
-                lambda x: x.no_calls_all / (x.no_calls_all + x.no_sms_all)
-            ),
-            proportion_calls_incoming=(
-                lambda x: x.no_incoming / (x.no_incoming + x.no_received)
-            ),
-            proportion_calls_missed_sms_received=(
-                lambda x: x.no_missed / (x.no_missed + x.no_received)
-            ),
-            proportion_calls_outgoing=(
-                lambda x: x.no_outgoing / (x.no_outgoing + x.no_sent)
-            ),
-            proportion_calls_contacts=(
-                lambda x: x.no_contacts_calls
-                / (x.no_contacts_calls + x.no_contacts_sms)
-            )
-            # Calculate new features and create additional columns
-        )
-        .fillna(0.5, downcast="infer")
+        # Calculate new features and create additional columns
     )
     return count_joined