from collections.abc import Collection import pandas as pd from config.models import SMS, Call, Participant from setup import db_engine, session call_types = {1: "incoming", 2: "outgoing", 3: "missed"} sms_types = {1: "received", 2: "sent"} def get_call_data(usernames: Collection) -> pd.DataFrame: """ Read the data from the calls table and return it in a dataframe. Parameters ---------- usernames: Collection A list of usernames to put into the WHERE condition. Returns ------- df_calls: pd.DataFrame A dataframe of call data. """ query_calls = ( session.query(Call, Participant.username) .filter(Participant.id == Call.participant_id) .filter(Participant.username.in_(usernames)) ) with db_engine.connect() as connection: df_calls = pd.read_sql(query_calls.statement, connection) return df_calls def get_sms_data(usernames: Collection) -> pd.DataFrame: """ Read the data from the sms table and return it in a dataframe. Parameters ---------- usernames: Collection A list of usernames to put into the WHERE condition. Returns ------- df_sms: pd.DataFrame A dataframe of call data. """ query_sms = ( session.query(SMS, Participant.username) .filter(Participant.id == SMS.participant_id) .filter(Participant.username.in_(usernames)) ) with db_engine.connect() as connection: df_sms = pd.read_sql(query_sms.statement, connection) return df_sms def enumerate_contacts(comm_df: pd.DataFrame) -> pd.DataFrame: """ Count contacts (callers, senders) and enumerate them by their frequency. Parameters ---------- comm_df: pd.DataFrame A dataframe of calls or SMSes. Returns ------- comm_df: pd.DataFrame The altered dataframe with the column contact_id, arranged by frequency. """ contact_counts = ( comm_df.groupby( ["participant_id", "trace"] ) # We want to count rows by participant_id and trace .size() # Count rows .reset_index() # Make participant_id a regular column. .rename(columns={0: "freq"}) .sort_values(["participant_id", "freq"], ascending=False) # First sort by participant_id and then by call frequency. ) # We now have a frequency table of different traces (contacts) *within* each participant_id. # Next, enumerate these contacts. # In other words, recode the contacts into integers from 0 to n_contacts, # so that the first one is contacted the most often. contact_ids = ( contact_counts.groupby("participant_id") # Group again for enumeration. .cumcount() # Enumerate (count) rows *within* participants. .to_frame("contact_id") ) contact_counts = contact_counts.join(contact_ids) # Add these contact_ids to the temporary (grouped) data frame. comm_df = comm_df.merge(contact_counts, on=["participant_id", "trace"]) # Add these contact_ids to the original data frame. return comm_df def count_comms(comm_df: pd.DataFrame) -> pd.DataFrame: """ Calculate frequencies (and duration) of messages (or calls), grouped by their types. Parameters ---------- comm_df: pd.DataFrame A dataframe of calls or SMSes. Returns ------- comm_features: pd.DataFrame A list of communication features for every participant. These are: * the number of calls by type (incoming, outgoing missed) and in total, * the ratio of incoming and outgoing calls to the total number of calls, * the total and maximum duration of calls by type, and * the number of messages by type (received, sent). """ if "call_type" in comm_df: comm_counts = ( comm_df.value_counts(subset=["participant_id", "call_type"]) .unstack() .rename(columns=call_types) .add_prefix("no_") ) # Count calls by type. comm_counts["no_all"] = comm_counts.sum(axis=1) # Add a total count of calls. comm_counts = comm_counts.assign( no_incoming_ratio=lambda x: x.no_incoming / x.no_all, no_outgoing_ratio=lambda x: x.no_outgoing / x.no_all, ) # Ratio of incoming and outgoing calls to all calls. comm_duration_total = ( comm_df.groupby(["participant_id", "call_type"]) .sum()["call_duration"] .unstack() .rename(columns=call_types) .add_prefix("duration_total_") ) # Total call duration by type. comm_duration_max = ( comm_df.groupby(["participant_id", "call_type"]) .max()["call_duration"] .unstack() .rename(columns=call_types) .add_prefix("duration_max_") ) # Max call duration by type comm_features = comm_counts.join(comm_duration_total) comm_features = comm_features.join(comm_duration_max) try: comm_features.drop(columns="duration_total_" + call_types[3], inplace=True) comm_features.drop(columns="duration_max_" + call_types[3], inplace=True) # The missed calls are always of 0 duration. except KeyError: pass # If there were no missed calls, this exception is raised. # But we are dropping the column anyway, so no need to deal with the exception. elif "message_type" in comm_df: comm_counts = ( comm_df.value_counts(subset=["participant_id", "message_type"]) .unstack() .rename(columns=sms_types) .add_prefix("no_") ) comm_counts["no_all"] = comm_counts.sum(axis=1) # Add a total count of messages. comm_features = comm_counts.assign( no_received_ratio=lambda x: x.no_received / x.no_all, no_sent_ratio=lambda x: x.no_sent / x.no_all, ) # Ratio of incoming and outgoing messages to all messages. else: raise KeyError("The dataframe contains neither call_type or message_type") return comm_features def contact_features(): # TODO Implement a method that takes a DF with enumerated contacts as argument and calculates: # * Duration of calls per caller (for most common callers) # * Determine work vs non-work contacts by work hours heuristics # * Number of people contacted # And similarly for SMS. pass def calls_sms_features(): # TODO Relate the calls and sms data, such as comparing the number of (missed) calls and messages. pass