Merge branch 'master' into ml_pipeline

rapids
junos 2021-08-20 17:43:53 +02:00
commit d6337e82ac
2 changed files with 40 additions and 17 deletions

View File

@ -13,14 +13,15 @@
# name: straw2analysis # name: straw2analysis
# --- # ---
# %%
import importlib
# %% # %%
# %matplotlib inline # %matplotlib inline
import os import os
import sys import sys
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
# %%
import seaborn as sns import seaborn as sns
nb_dir = os.path.split(os.getcwd())[0] nb_dir = os.path.split(os.getcwd())[0]
@ -28,21 +29,29 @@ if nb_dir not in sys.path:
sys.path.append(nb_dir) sys.path.append(nb_dir)
# %% # %%
from features.communication import * from features import communication, helper
# %%
importlib.reload(communication)
# %% [markdown] # %% [markdown]
# # Example of communication data and feature calculation # # Example of communication data and feature calculation
# %% # %%
df_calls = get_call_data(["nokia_0000003"]) df_calls = communication.get_call_data(["nokia_0000003"])
print(df_calls) print(df_calls)
# %% # %%
count_comms(df_calls) df_calls = helper.get_date_from_timestamp(df_calls)
communication.count_comms(df_calls, ["date_lj"])
# %% # %%
df_sms = get_sms_data(["nokia_0000003"]) df_sms = communication.get_sms_data(["nokia_0000003"])
count_comms(df_sms) df_sms = helper.get_date_from_timestamp(df_sms)
communication.count_comms(df_sms, ["date_lj"])
# %%
communication.calls_sms_features(df_calls, df_sms, ["date_lj"])
# %% [markdown] # %% [markdown]
# # Call data # # Call data

View File

@ -137,7 +137,7 @@ def enumerate_contacts(comm_df: pd.DataFrame) -> pd.DataFrame:
return comm_df return comm_df
def count_comms(comm_df: pd.DataFrame) -> pd.DataFrame: def count_comms(comm_df: pd.DataFrame, group_by=None) -> pd.DataFrame:
""" """
Calculate frequencies (and duration) of messages (or calls), grouped by their types. Calculate frequencies (and duration) of messages (or calls), grouped by their types.
@ -145,6 +145,9 @@ def count_comms(comm_df: pd.DataFrame) -> pd.DataFrame:
---------- ----------
comm_df: pd.DataFrame comm_df: pd.DataFrame
A dataframe of calls or SMSes. A dataframe of calls or SMSes.
group_by: list
A list of strings, specifying by which parameters to group.
By default, the features are calculated per participant, but could be "date_lj" etc.
Returns Returns
------- -------
@ -157,10 +160,12 @@ def count_comms(comm_df: pd.DataFrame) -> pd.DataFrame:
* the number of messages by type (received, sent), and * the number of messages by type (received, sent), and
* the number of communication contacts by type. * the number of communication contacts by type.
""" """
if group_by is None:
group_by = []
if "call_type" in comm_df: if "call_type" in comm_df:
data_type = "calls" data_type = "calls"
comm_counts = ( comm_counts = (
comm_df.value_counts(subset=["participant_id", "call_type"]) comm_df.value_counts(subset=group_by + ["participant_id", "call_type"])
.unstack() .unstack()
.rename(columns=call_types) .rename(columns=call_types)
.add_prefix("no_") .add_prefix("no_")
@ -174,7 +179,7 @@ def count_comms(comm_df: pd.DataFrame) -> pd.DataFrame:
) )
# Ratio of incoming and outgoing calls to all calls. # Ratio of incoming and outgoing calls to all calls.
comm_duration_total = ( comm_duration_total = (
comm_df.groupby(["participant_id", "call_type"]) comm_df.groupby(group_by + ["participant_id", "call_type"])
.sum()["call_duration"] .sum()["call_duration"]
.unstack() .unstack()
.rename(columns=call_types) .rename(columns=call_types)
@ -182,7 +187,7 @@ def count_comms(comm_df: pd.DataFrame) -> pd.DataFrame:
) )
# Total call duration by type. # Total call duration by type.
comm_duration_max = ( comm_duration_max = (
comm_df.groupby(["participant_id", "call_type"]) comm_df.groupby(group_by + ["participant_id", "call_type"])
.max()["call_duration"] .max()["call_duration"]
.unstack() .unstack()
.rename(columns=call_types) .rename(columns=call_types)
@ -202,7 +207,7 @@ def count_comms(comm_df: pd.DataFrame) -> pd.DataFrame:
elif "message_type" in comm_df: elif "message_type" in comm_df:
data_type = "sms" data_type = "sms"
comm_counts = ( comm_counts = (
comm_df.value_counts(subset=["participant_id", "message_type"]) comm_df.value_counts(subset=group_by + ["participant_id", "message_type"])
.unstack() .unstack()
.rename(columns=sms_types) .rename(columns=sms_types)
.add_prefix("no_") .add_prefix("no_")
@ -218,7 +223,7 @@ def count_comms(comm_df: pd.DataFrame) -> pd.DataFrame:
raise KeyError("The dataframe contains neither call_type or message_type") raise KeyError("The dataframe contains neither call_type or message_type")
comm_contacts_counts = ( comm_contacts_counts = (
enumerate_contacts(comm_df) enumerate_contacts(comm_df)
.groupby(["participant_id"]) .groupby(group_by + ["participant_id"])
.nunique()["contact_id"] .nunique()["contact_id"]
.rename("no_contacts_" + data_type) .rename("no_contacts_" + data_type)
) )
@ -270,7 +275,9 @@ def contact_features(comm_df: pd.DataFrame) -> pd.DataFrame:
return contacts_count return contacts_count
def calls_sms_features(df_calls: pd.DataFrame, df_sms: pd.DataFrame) -> pd.DataFrame: def calls_sms_features(
df_calls: pd.DataFrame, df_sms: pd.DataFrame, group_by=None
) -> pd.DataFrame:
""" """
Calculates additional features relating calls and sms data. Calculates additional features relating calls and sms data.
@ -280,6 +287,9 @@ def calls_sms_features(df_calls: pd.DataFrame, df_sms: pd.DataFrame) -> pd.DataF
A dataframe of calls (return of get_call_data). A dataframe of calls (return of get_call_data).
df_sms: pd.DataFrame df_sms: pd.DataFrame
A dataframe of SMSes (return of get_sms_data). A dataframe of SMSes (return of get_sms_data).
group_by: list
A list of strings, specifying by which parameters to group.
By default, the features are calculated per participant, but could be "date_lj" etc.
Returns Returns
------- -------
@ -297,9 +307,13 @@ def calls_sms_features(df_calls: pd.DataFrame, df_sms: pd.DataFrame) -> pd.DataF
* proportion_calls_contacts: * proportion_calls_contacts:
proportion of calls contacts in total number of communication contacts proportion of calls contacts in total number of communication contacts
""" """
count_calls = count_comms(df_calls) if group_by is None:
count_sms = count_comms(df_sms) group_by = []
count_joined = count_calls.join(count_sms).assign( count_calls = count_comms(df_calls, group_by)
count_sms = count_comms(df_sms, group_by)
count_joined = count_calls.merge(
count_sms, how="outer", left_index=True, right_index=True, validate="one_to_one"
).assign(
proportion_calls_all=( proportion_calls_all=(
lambda x: x.no_calls_all / (x.no_calls_all + x.no_sms_all) lambda x: x.no_calls_all / (x.no_calls_all + x.no_sms_all)
), ),