stress_at_work_analysis/features/esm_SAM.py

import numpy as np
import pandas as pd

import features.esm

QUESTIONNAIRE_ID_SAM = {
    "event_stress": 87,
    "event_threat": 88,
    "event_challenge": 89,
    "event_time": 90,
    "event_duration": 91,
    "event_work_related": 92,
    "period_stress": 93,
}
QUESTIONNAIRE_ID_SAM_LOW = min(QUESTIONNAIRE_ID_SAM.values())
QUESTIONNAIRE_ID_SAM_HIGH = max(QUESTIONNAIRE_ID_SAM.values())

GROUP_QUESTIONNAIRES_BY = [
    "participant_id",
    "device_id",
    "esm_session",
]
# Each questionnaire occurs only once within each esm_session on the same device
# within the same participant.


def extract_stressful_events(df_esm: pd.DataFrame) -> pd.DataFrame:
    """
    Extract information about stressful events.

    Participants were asked: "Was there a particular event that created tension in you?"
    Then a subset of questions related to this event followed.
    This function goes through the follow-up questions one by one
        and preprocesses them, so that it adds new columns to the dataframe.

    Parameters
    ----------
    df_esm: pd.DataFrame
        A raw dataframe of all ESM data.

    Returns
    -------
    df_esm_events: pd.DataFrame
        A cleaned up df of Stress Appraisal Measure items with additional columns.

    """
    # 0. Select only questions from Stress Appraisal Measure.
    df_esm_preprocessed = features.esm.preprocess_esm(df_esm)
    df_esm_sam = df_esm_preprocessed[
        (df_esm_preprocessed["questionnaire_id"] >= QUESTIONNAIRE_ID_SAM_LOW)
        & (df_esm_preprocessed["questionnaire_id"] <= QUESTIONNAIRE_ID_SAM_HIGH)
    ]

    df_esm_sam_clean = features.esm.clean_up_esm(df_esm_sam)
    # 1.
    df_esm_event_threat_challenge_mean_wide = calculate_threat_challenge_means(
        df_esm_sam_clean
    )
    # 2.
    df_esm_event_stress = detect_stressful_event(df_esm_sam_clean)

    # Join to the previously calculated features related to the events.
    df_esm_events = df_esm_event_threat_challenge_mean_wide.join(
        df_esm_event_stress[
            GROUP_QUESTIONNAIRES_BY + ["event_present", "event_stressfulness"]
        ].set_index(GROUP_QUESTIONNAIRES_BY)
    )

    # 3.
    df_esm_event_work_related = detect_event_work_related(df_esm_sam_clean)

    df_esm_events = df_esm_events.join(
        df_esm_event_work_related[
            GROUP_QUESTIONNAIRES_BY + ["event_work_related"]
        ].set_index(GROUP_QUESTIONNAIRES_BY)
    )

    # 4.
    df_esm_event_time = convert_event_time(df_esm_sam_clean)

    df_esm_events = df_esm_events.join(
        df_esm_event_time[GROUP_QUESTIONNAIRES_BY + ["event_time"]].set_index(
            GROUP_QUESTIONNAIRES_BY
        )
    )

    # 5.
    df_esm_event_duration = extract_event_duration(df_esm_sam_clean)

    df_esm_events = df_esm_events.join(
        df_esm_event_duration[
            GROUP_QUESTIONNAIRES_BY + ["event_duration", "event_duration_info"]
        ].set_index(GROUP_QUESTIONNAIRES_BY)
    )

    return df_esm_events


def calculate_threat_challenge_means(df_esm_sam_clean: pd.DataFrame) -> pd.DataFrame:
    """
    This function calculates challenge and threat
        (two Stress Appraisal Measure subscales) means,
        for each ESM session (within participants and devices).
    It creates a grouped dataframe with means in two columns.

    Parameters
    ----------
    df_esm_sam_clean: pd.DataFrame
        A cleaned up dataframe of Stress Appraisal Measure items.

    Returns
    -------
    df_esm_event_threat_challenge_mean_wide: pd.DataFrame
        A dataframe of unique ESM sessions (by participants and devices)
        with threat and challenge means.
    """
    # Select only threat and challenge assessments for events
    df_esm_event_threat_challenge = df_esm_sam_clean[
        (
            df_esm_sam_clean["questionnaire_id"]
            == QUESTIONNAIRE_ID_SAM.get("event_threat")
        )
        | (
            df_esm_sam_clean["questionnaire_id"]
            == QUESTIONNAIRE_ID_SAM.get("event_challenge")
        )
    ]
    # Calculate mean of threat and challenge subscales for each ESM session.
    df_esm_event_threat_challenge_mean_wide = pd.pivot_table(
        df_esm_event_threat_challenge,
        index=["participant_id", "device_id", "esm_session"],
        columns=["questionnaire_id"],
        values=["esm_user_answer_numeric"],
        aggfunc="mean",
    )
    # Drop unnecessary column values.
    df_esm_event_threat_challenge_mean_wide.columns = (
        df_esm_event_threat_challenge_mean_wide.columns.get_level_values(1)
    )
    df_esm_event_threat_challenge_mean_wide.columns.name = None
    df_esm_event_threat_challenge_mean_wide.rename(
        columns={
            QUESTIONNAIRE_ID_SAM.get("event_threat"): "threat_mean",
            QUESTIONNAIRE_ID_SAM.get("event_challenge"): "challenge_mean",
        },
        inplace=True,
    )
    return df_esm_event_threat_challenge_mean_wide


def detect_stressful_event(df_esm_sam_clean: pd.DataFrame) -> pd.DataFrame:
    """
    Participants were asked: "Was there a particular event that created tension in you?"
    The following options were available:
        0 - No,
        1 - Yes, slightly,
        2 - Yes, moderately,
        3 - Yes, considerably,
        4 - Yes, extremely.
    This function indicates whether there was a stressful event (True/False)
        and how stressful it was on a scale of 1 to 4.

    Parameters
    ----------
    df_esm_sam_clean: pd.DataFrame
        A cleaned up dataframe of Stress Appraisal Measure items.

    Returns
    -------
    df_esm_event_stress: pd.DataFrame
        The same dataframe with two new columns:
            - event_present, indicating whether there was a stressful event at all,
            - event_stressfulness, a numeric answer (1-4) to the single item question.

    """
    df_esm_event_stress = df_esm_sam_clean[
        df_esm_sam_clean["questionnaire_id"] == QUESTIONNAIRE_ID_SAM.get("event_stress")
    ]
    df_esm_event_stress = df_esm_event_stress.assign(
        event_present=lambda x: x.esm_user_answer_numeric > 0,
        event_stressfulness=lambda x: x.esm_user_answer_numeric,
    )
    return df_esm_event_stress


def detect_event_work_related(df_esm_sam_clean: pd.DataFrame) -> pd.DataFrame:
    """
    This function simply adds a column indicating the answer to the question:
        "Was/is this event work-related?"

    Parameters
    ----------
    df_esm_sam_clean: pd.DataFrame
        A cleaned up dataframe of Stress Appraisal Measure items.

    Returns
    -------
    df_esm_event_stress: pd.DataFrame
        The same dataframe with a new column event_work_related (True/False).

    """
    df_esm_event_stress = df_esm_sam_clean[
        df_esm_sam_clean["questionnaire_id"]
        == QUESTIONNAIRE_ID_SAM.get("event_work_related")
    ]
    df_esm_event_stress = df_esm_event_stress.assign(
        event_work_related=lambda x: x.esm_user_answer_numeric > 0
    )
    return df_esm_event_stress


def convert_event_time(df_esm_sam_clean: pd.DataFrame) -> pd.DataFrame:
    """
    This function only serves to convert the string datetime answer
        into a real datetime type.
    Errors during this conversion are coerced, meaning that non-datetime answers
        are assigned Not a Time (NaT).
    NOTE: Since the only available non-datetime answer to this question was
        "0 - I do not remember", the NaTs can be interpreted to mean this.

    Parameters
    ----------
    df_esm_sam_clean: pd.DataFrame
        A cleaned up dataframe of Stress Appraisal Measure items.

    Returns
    -------
    df_esm_event_time: pd.DataFrame
        The same dataframe with a new column event_time of datetime type.
    """
    df_esm_event_time = df_esm_sam_clean[
        df_esm_sam_clean["questionnaire_id"] == QUESTIONNAIRE_ID_SAM.get("event_time")
    ].assign(
        event_time=lambda x: pd.to_datetime(
            x.esm_user_answer,
            errors="coerce",
            format="%Y-%m-%d %H:%M:%S %z",
            exact=True,
        )
    )
    # Example answer: 2020-09-29 00:05:00 +0200
    return df_esm_event_time


def extract_event_duration(df_esm_sam_clean: pd.DataFrame) -> pd.DataFrame:
    """
    If participants indicated a stressful events, they were asked:
        "How long did this event last? (Answer in hours and minutes)"
    This function extracts this duration time and saves additional answers:
        0 - I do not remember,
        1 - It is still going on.

    Parameters
    ----------
    df_esm_sam_clean: pd.DataFrame
        A cleaned up dataframe of Stress Appraisal Measure items.

    Returns
    -------
    df_esm_event_duration: pd.DataFrame
        The same dataframe with two new columns:
            - event_duration, a time part of a datetime,
            - event_duration_info, giving other options to this question:
                0 - I do not remember,
                1 - It is still going on
    """
    df_esm_event_duration = df_esm_sam_clean[
        df_esm_sam_clean["questionnaire_id"]
        == QUESTIONNAIRE_ID_SAM.get("event_duration")
    ].assign(
        event_duration=lambda x: pd.to_datetime(
            x.esm_user_answer.str.slice(start=0, stop=-6),
            errors="coerce",
            format="%Y-%m-%d %H:%M:%S",
        ).dt.time
    )
    # Example answer: 2020-09-29 00:05:00 +0200
    # TODO Explore the values recorded in event_duration and possibly fix mistakes.
    # For example, participants reported setting 23:50:00 instead of 00:50:00.

    # For the events that no duration was found (i.e. event_duration = NaT),
    # we can determine whether:
    #   - this event is still going on ("1 - It is still going on")
    #   - the participant couldn't remember it's duration ("0 - I do not remember")
    # Generally, these answers were converted to esm_user_answer_numeric in clean_up_esm
    # but only the numeric types of questions and answers.
    # Since this was of "datetime" type, convert these specific answers here again.
    df_esm_event_duration["event_duration_info"] = np.nan
    df_esm_event_duration[
        df_esm_event_duration.event_duration.isna()
    ] = df_esm_event_duration[df_esm_event_duration.event_duration.isna()].assign(
        event_duration_info=lambda x: x.esm_user_answer.str.slice(stop=1).astype(int)
    )

    return df_esm_event_duration


# TODO: How many questions about the stressfulness of the period were asked
#  and how does this relate to events?