Compare commits
No commits in common. "e2e268148d06cf97d87cf23efdabe0b7938ee239" and "777e6f0a583df6503fbf62a03813f9241f4c567f" have entirely different histories.
e2e268148d
...
777e6f0a58
|
@ -13,15 +13,14 @@
|
|||
# name: straw2analysis
|
||||
# ---
|
||||
|
||||
# %%
|
||||
import importlib
|
||||
|
||||
# %%
|
||||
# %matplotlib inline
|
||||
import os
|
||||
import sys
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
# %%
|
||||
import seaborn as sns
|
||||
|
||||
nb_dir = os.path.split(os.getcwd())[0]
|
||||
|
@ -29,29 +28,21 @@ if nb_dir not in sys.path:
|
|||
sys.path.append(nb_dir)
|
||||
|
||||
# %%
|
||||
from features import communication, helper
|
||||
|
||||
# %%
|
||||
importlib.reload(communication)
|
||||
from features.communication import *
|
||||
|
||||
# %% [markdown]
|
||||
# # Example of communication data and feature calculation
|
||||
|
||||
# %%
|
||||
df_calls = communication.get_call_data(["nokia_0000003"])
|
||||
df_calls = get_call_data(["nokia_0000003"])
|
||||
print(df_calls)
|
||||
|
||||
# %%
|
||||
df_calls = helper.get_date_from_timestamp(df_calls)
|
||||
communication.count_comms(df_calls, ["date_lj"])
|
||||
count_comms(df_calls)
|
||||
|
||||
# %%
|
||||
df_sms = communication.get_sms_data(["nokia_0000003"])
|
||||
df_sms = helper.get_date_from_timestamp(df_sms)
|
||||
communication.count_comms(df_sms, ["date_lj"])
|
||||
|
||||
# %%
|
||||
communication.calls_sms_features(df_calls, df_sms, ["date_lj"])
|
||||
df_sms = get_sms_data(["nokia_0000003"])
|
||||
count_comms(df_sms)
|
||||
|
||||
# %% [markdown]
|
||||
# # Call data
|
||||
|
|
|
@ -135,7 +135,7 @@ def enumerate_contacts(comm_df: pd.DataFrame) -> pd.DataFrame:
|
|||
return comm_df
|
||||
|
||||
|
||||
def count_comms(comm_df: pd.DataFrame, group_by=None) -> pd.DataFrame:
|
||||
def count_comms(comm_df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
Calculate frequencies (and duration) of messages (or calls), grouped by their types.
|
||||
|
||||
|
@ -143,9 +143,6 @@ def count_comms(comm_df: pd.DataFrame, group_by=None) -> pd.DataFrame:
|
|||
----------
|
||||
comm_df: pd.DataFrame
|
||||
A dataframe of calls or SMSes.
|
||||
group_by: list
|
||||
A list of strings, specifying by which parameters to group.
|
||||
By default, the features are calculated per participant, but could be "date_lj" etc.
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
@ -158,13 +155,11 @@ def count_comms(comm_df: pd.DataFrame, group_by=None) -> pd.DataFrame:
|
|||
* the number of messages by type (received, sent), and
|
||||
* the number of communication contacts by type.
|
||||
"""
|
||||
if group_by is None:
|
||||
group_by = []
|
||||
if "call_type" in comm_df:
|
||||
data_type = "calls"
|
||||
comm_counts = (
|
||||
comm_df.value_counts(subset=group_by + ["participant_id", "call_type"])
|
||||
.unstack(level="call_type", fill_value=0)
|
||||
comm_df.value_counts(subset=["participant_id", "call_type"])
|
||||
.unstack()
|
||||
.rename(columns=call_types)
|
||||
.add_prefix("no_")
|
||||
)
|
||||
|
@ -177,17 +172,17 @@ def count_comms(comm_df: pd.DataFrame, group_by=None) -> pd.DataFrame:
|
|||
)
|
||||
# Ratio of incoming and outgoing calls to all calls.
|
||||
comm_duration_total = (
|
||||
comm_df.groupby(group_by + ["participant_id", "call_type"])
|
||||
comm_df.groupby(["participant_id", "call_type"])
|
||||
.sum()["call_duration"]
|
||||
.unstack(level="call_type", fill_value=0)
|
||||
.unstack()
|
||||
.rename(columns=call_types)
|
||||
.add_prefix("duration_total_")
|
||||
)
|
||||
# Total call duration by type.
|
||||
comm_duration_max = (
|
||||
comm_df.groupby(group_by + ["participant_id", "call_type"])
|
||||
comm_df.groupby(["participant_id", "call_type"])
|
||||
.max()["call_duration"]
|
||||
.unstack(level="call_type", fill_value=0)
|
||||
.unstack()
|
||||
.rename(columns=call_types)
|
||||
.add_prefix("duration_max_")
|
||||
)
|
||||
|
@ -205,8 +200,8 @@ def count_comms(comm_df: pd.DataFrame, group_by=None) -> pd.DataFrame:
|
|||
elif "message_type" in comm_df:
|
||||
data_type = "sms"
|
||||
comm_counts = (
|
||||
comm_df.value_counts(subset=group_by + ["participant_id", "message_type"])
|
||||
.unstack(level="message_type", fill_value=0)
|
||||
comm_df.value_counts(subset=["participant_id", "message_type"])
|
||||
.unstack()
|
||||
.rename(columns=sms_types)
|
||||
.add_prefix("no_")
|
||||
)
|
||||
|
@ -221,7 +216,7 @@ def count_comms(comm_df: pd.DataFrame, group_by=None) -> pd.DataFrame:
|
|||
raise KeyError("The dataframe contains neither call_type or message_type")
|
||||
comm_contacts_counts = (
|
||||
enumerate_contacts(comm_df)
|
||||
.groupby(group_by + ["participant_id"])
|
||||
.groupby(["participant_id"])
|
||||
.nunique()["contact_id"]
|
||||
.rename("no_contacts_" + data_type)
|
||||
)
|
||||
|
@ -273,9 +268,7 @@ def contact_features(comm_df: pd.DataFrame) -> pd.DataFrame:
|
|||
return contacts_count
|
||||
|
||||
|
||||
def calls_sms_features(
|
||||
df_calls: pd.DataFrame, df_sms: pd.DataFrame, group_by=None
|
||||
) -> pd.DataFrame:
|
||||
def calls_sms_features(df_calls: pd.DataFrame, df_sms: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
Calculates additional features relating calls and sms data.
|
||||
|
||||
|
@ -285,9 +278,6 @@ def calls_sms_features(
|
|||
A dataframe of calls (return of get_call_data).
|
||||
df_sms: pd.DataFrame
|
||||
A dataframe of SMSes (return of get_sms_data).
|
||||
group_by: list
|
||||
A list of strings, specifying by which parameters to group.
|
||||
By default, the features are calculated per participant, but could be "date_lj" etc.
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
@ -305,20 +295,9 @@ def calls_sms_features(
|
|||
* proportion_calls_contacts:
|
||||
proportion of calls contacts in total number of communication contacts
|
||||
"""
|
||||
if group_by is None:
|
||||
group_by = []
|
||||
count_calls = count_comms(df_calls, group_by)
|
||||
count_sms = count_comms(df_sms, group_by)
|
||||
count_joined = (
|
||||
count_calls.merge(
|
||||
count_sms,
|
||||
how="outer",
|
||||
left_index=True,
|
||||
right_index=True,
|
||||
validate="one_to_one",
|
||||
)
|
||||
.fillna(0, downcast="infer")
|
||||
.assign(
|
||||
count_calls = count_comms(df_calls)
|
||||
count_sms = count_comms(df_sms)
|
||||
count_joined = count_calls.join(count_sms).assign(
|
||||
proportion_calls_all=(
|
||||
lambda x: x.no_calls_all / (x.no_calls_all + x.no_sms_all)
|
||||
),
|
||||
|
@ -332,11 +311,8 @@ def calls_sms_features(
|
|||
lambda x: x.no_outgoing / (x.no_outgoing + x.no_sent)
|
||||
),
|
||||
proportion_calls_contacts=(
|
||||
lambda x: x.no_contacts_calls
|
||||
/ (x.no_contacts_calls + x.no_contacts_sms)
|
||||
lambda x: x.no_contacts_calls / (x.no_contacts_calls + x.no_contacts_sms)
|
||||
)
|
||||
# Calculate new features and create additional columns
|
||||
)
|
||||
.fillna(0.5, downcast="infer")
|
||||
)
|
||||
return count_joined
|
||||
|
|
Loading…
Reference in New Issue