Merge branch 'communication' into 'master'

Communication

See merge request junoslukan/straw2analysis!1

exploration/expl_communication.py

@@ -6,7 +6,7 @@
 #       extension: .py
 #       format_name: percent
 #       format_version: '1.3'
-#       jupytext_version: 1.11.2
+#       jupytext_version: 1.11.4
 #   kernelspec:
 #     display_name: straw2analysis
 #     language: python
@@ -14,6 +14,7 @@
 # ---
 
 # %%
+# %matplotlib inline
 import os
 import sys
 
@@ -53,6 +54,15 @@ import participants.query_db
 participants_inactive_usernames = participants.query_db.get_usernames()
 df_calls_inactive = get_call_data(participants_inactive_usernames)
 
+# %%
+participants_inactive_usernames
+
+# %%
+df_calls_inactive.head()
+
+# %%
+enumerate_contacts(df_calls_inactive).head()
+
 # %%
 df_calls_features = count_comms(df_calls_inactive)
 df_calls_features.head()
@@ -70,6 +80,9 @@ calls_number = pd.wide_to_long(
     suffix="\D+",
 )
 
+# %%
+calls_number
+
 # %%
 sns.displot(calls_number, x="no", hue="call_type", binwidth=5, element="step", height=8)
 
@@ -126,3 +139,30 @@ sms_number = pd.wide_to_long(
 sns.displot(
     sms_number, x="no", hue="message_type", binwidth=5, element="step", height=8
 )
+
+# %% [markdown]
+# # Communication features
+
+# %%
+df_calls_enumerated = enumerate_contacts(df_calls)
+display(df_calls_enumerated)
+
+# %%
+df_calls_contact_features = contact_features(df_calls_enumerated)
+display(df_calls_contact_features)
+
+# %%
+df_sms_enumerated = enumerate_contacts(df_sms)
+df_sms_contact_features = contact_features(df_sms_enumerated)
+display(df_sms_contact_features)
+
+# %%
+display(count_comms(df_calls))
+
+# %%
+display(count_comms(df_sms))
+
+# %%
+display(calls_sms_features(df_calls, df_sms))
+
+# %%

features/communication.py

@@ -86,7 +86,8 @@ def enumerate_contacts(comm_df: pd.DataFrame) -> pd.DataFrame:
     # In other words, recode the contacts into integers from 0 to n_contacts,
     # so that the first one is contacted the most often.
     contact_ids = (
-        contact_counts.groupby("participant_id")  # Group again for enumeration.
+        # Group again for enumeration.
+        contact_counts.groupby("participant_id")
         .cumcount()  # Enumerate (count) rows *within* participants.
         .to_frame("contact_id")
     )
@@ -150,8 +151,10 @@ def count_comms(comm_df: pd.DataFrame) -> pd.DataFrame:
         comm_features = comm_counts.join(comm_duration_total)
         comm_features = comm_features.join(comm_duration_max)
         try:
-            comm_features.drop(columns="duration_total_" + call_types[3], inplace=True)
+            comm_features.drop(columns="duration_total_" +
-            comm_features.drop(columns="duration_max_" + call_types[3], inplace=True)
+                               call_types[3], inplace=True)
+            comm_features.drop(columns="duration_max_" +
+                               call_types[3], inplace=True)
         # The missed calls are always of 0 duration.
         except KeyError:
             pass
@@ -172,19 +175,145 @@ def count_comms(comm_df: pd.DataFrame) -> pd.DataFrame:
         )
         # Ratio of incoming and outgoing messages to all messages.
     else:
-        raise KeyError("The dataframe contains neither call_type or message_type")
+        raise KeyError(
+            "The dataframe contains neither call_type or message_type")
     return comm_features
 
 
-def contact_features():
+def contact_features(df_enumerated: pd.DataFrame) -> pd.DataFrame:
-    # TODO Implement a method that takes a DF with enumerated contacts as argument and calculates:
+    """
-    # * Duration of calls per caller (for most common callers)
+    Counts the number of people contacted (for each participant) and, if
-    # * Determine work vs non-work contacts by work hours heuristics
+    df_enumerated is a dataframe containing calls data, the total duration
-    # * Number of people contacted
+    of calls between a participant and each of her contacts. 
-    # And similarly for SMS.
+
-    pass
+    Parameters
+    ----------
+    df_enumerated: pd.DataFrame
+        A dataframe of calls or SMSes; return of function enumerate_contacts.
+
+    Returns
+    -------
+    comm_df: pd.DataFrame
+        The altered dataframe with the column no_contacts and, if df_enumerated
+        contains calls data, an additional column total_call_duration.
+    """
+
+    # Check whether df contains calls or SMS data since some
+    # features we want to calculate are type-specyfic
+    if "call_duration" in df_enumerated:
+        # Add a column with the total duration of calls between two people
+        duration_count = (
+            df_enumerated.groupby(
+                ["participant_id", "contact_id"]
+            )
+            # For each participant and for each caller, sum durations of their calls
+            ["call_duration"].sum()
+            .reset_index()  # Make index (which is actually the participant id) a normal column
+            .rename(columns={"call_duration": "total_call_duration"})
+        )
+        # The new dataframe now contains columns containing information about
+        # participants, callers and the total duration of their calls. All that
+        # is now left to do is to merge the original df with the new one.
+        df_enumerated = df_enumerated.merge(
+            duration_count,
+            on=["participant_id", "contact_id"]
+            )
+
+    contact_count = (
+        df_enumerated.groupby(["participant_id"])
+        .nunique()["contact_id"] # For each participant, count the number of distinct contacts
+        .reset_index()  # Make index (which is actually the participant id) a normal column
+        .rename(columns={"contact_id": "no_contacts"})
+    )
+    
+    df_enumerated = (
+        # Merge df with the newely created df containing info about number of contacts
+        df_enumerated.merge(contact_count, on="participant_id")
+        # Sort first by participant_id and then by contact_id and
+        # thereby restore the inital ordering of input dataframes.
+        .sort_values(["participant_id", "contact_id"])
+    )
+
+    # TODO:Determine work vs non-work contacts by work hours heuristics
+
+    return df_enumerated
 
 
-def calls_sms_features():
+def calls_sms_features(df_calls: pd.DataFrame, df_sms: pd.DataFrame) -> pd.DataFrame:
-    # TODO Relate the calls and sms data, such as comparing the number of (missed) calls and messages.
+    """
-    pass
+    Calculates additional features relating calls and sms data.
+
+    Parameters
+    ----------
+    df_calls: pd.DataFrame
+        A dataframe of calls (return of get_call_data).
+    df_sms: pd.DataFrame
+        A dataframe of calls (return of get_sms_data).
+
+    Returns
+    -------
+    df_calls_sms: pd.DataFrame
+        The list of features relating calls and sms data for every participant.
+        These are:
+        * no_calls_no_sms_ratio:
+            proportion of calls in total number of communications
+        * no_incoming_calls_no_recieved_sms_ratio:
+            proportion of incoming calls in total number of incoming/recieved communications
+        * no_outgoing_calls_no_sent_sms_ratio:
+            proportion of outgoing calls in total number of outgoing/sent communications
+        * no_calls_contacts_no_sms_contacts_ratio:
+            proportion of calls contacts in total number of communication contacts 
+    """
+
+    count_calls = count_comms(df_calls)
+    count_sms = count_comms(df_sms)
+
+    count_joined = (
+        count_calls.merge(
+            count_sms, on="participant_id", suffixes=("_calls", "_sms")
+        )  # Merge calls and sms features
+        .reset_index()  # Make participant_id a regular column
+        .assign(
+            no_calls_no_sms_ratio=(
+                lambda x: x.no_all_calls / (x.no_all_calls + x.no_all_sms)
+            ),
+            no_incoming_calls_no_recieved_sms_ratio=(
+                lambda x: x.no_received / (x.no_incoming + x.no_received)
+            ),
+            no_outgoing_calls_no_sent_sms_ratio=(
+                lambda x: x.no_outgoing / (x.no_outgoing + x.no_sent)
+            )  # Calculate new features and create additional columns
+        )[
+            ["participant_id",
+             "no_calls_no_sms_ratio",
+             "no_incoming_calls_no_recieved_sms_ratio",
+             "no_outgoing_calls_no_sent_sms_ratio"]
+        ]  # Filter out only the relevant feautres
+    )
+
+    features_calls = contact_features(enumerate_contacts(df_calls))
+    features_sms = contact_features(enumerate_contacts(df_sms))
+
+    features_joined = (
+        features_calls.merge(
+            features_sms, on="participant_id", suffixes=("_calls", "_sms")
+        )  # Merge calls and sms features
+        .reset_index()  # Make participand_id a regular column
+        .assign(
+            no_calls_contacts_no_sms_contacts_ratio=(
+                lambda x: x.no_contacts_calls /
+                    (x.no_contacts_calls + x.no_contacts_sms)
+            )  # Calculate new features and create additional columns
+        )[
+            ["participant_id",
+             "no_calls_contacts_no_sms_contacts_ratio"]
+        ]  # Filter out only the relevant feautres
+        # Since we are interested only in some features and ignored
+        # others, a lot of duplicate rows were created. Remove them.
+        .drop_duplicates()
+    )
+
+    # Join the newely created dataframes
+    df_calls_sms = count_joined.merge(features_joined, on="participant_id")
+
+    return df_calls_sms