stress_at_work_analysis/exploration/expl_esm_labels.py

# -*- coding: utf-8 -*-
# ---
# jupyter:
#   jupytext:
#     formats: ipynb,py:percent
#     text_representation:
#       extension: .py
#       format_name: percent
#       format_version: '1.3'
#       jupytext_version: 1.14.5
#   kernelspec:
#     display_name: straw2analysis
#     language: python
#     name: straw2analysis
# ---

# %%
import datetime

import seaborn as sns

import participants.query_db
from features.esm import (
    QUESTIONNAIRE_IDS,
    clean_up_esm,
    get_esm_data,
    increment_answers,
    preprocess_esm,
)
from features.esm_COPE import reassign_question_ids
from features.esm_JCQ import reverse_jcq_demand_control_scoring
from features.esm_SAM import extract_stressful_events

# import os
# import sys
# nb_dir = os.path.split(os.getcwd())[0]
# if nb_dir not in sys.path:
#     sys.path.append(nb_dir)


# %%
save_figs = False
export_data = True

# %%
participants_inactive_usernames = participants.query_db.get_usernames(
    collection_start=datetime.date.fromisoformat("2020-08-01")
)
df_esm_inactive = get_esm_data(participants_inactive_usernames)

# %%
df_esm_preprocessed = preprocess_esm(df_esm_inactive)

# %% [markdown]
# # PANAS

# %%
df_esm_PANAS = df_esm_preprocessed[
    (
        df_esm_preprocessed["questionnaire_id"]
        == QUESTIONNAIRE_IDS["PANAS_positive_affect"]
    )
    | (
        df_esm_preprocessed["questionnaire_id"]
        == QUESTIONNAIRE_IDS["PANAS_negative_affect"]
    )
]
df_esm_PANAS_clean = clean_up_esm(df_esm_PANAS)

# %% [markdown]
# Group by participants, date, and subscale and calculate daily means.

# %%
df_esm_PANAS_daily_means = (
    df_esm_PANAS_clean.groupby(["participant_id", "date_lj", "questionnaire_id"])
    .esm_user_answer_numeric.agg("mean")
    .reset_index()
    .rename(columns={"esm_user_answer_numeric": "esm_numeric_mean"})
)

# %% [markdown]
# Next, calculate mean, median, and standard deviation across all days for each participant.

# %%
df_esm_PANAS_summary_participant = (
    df_esm_PANAS_daily_means.groupby(["participant_id", "questionnaire_id"])
    .esm_numeric_mean.agg(["mean", "median", "std"])
    .reset_index(col_level=1)
)
df_esm_PANAS_summary_participant[
    "PANAS subscale"
] = df_esm_PANAS_daily_means.questionnaire_id.astype("category").cat.rename_categories(
    {8.0: "positive affect", 9.0: "negative affect"}
)

# %%
df_esm_PANAS_summary_participant.groupby("PANAS subscale").describe()["mean"]

# %%
df_esm_PANAS_summary_participant.groupby("PANAS subscale").describe()["std"]

# %%
df_esm_PANAS_summary_participant.query("std == 0")

# %%
fig1 = sns.displot(
    data=df_esm_PANAS_summary_participant, x="mean", hue="PANAS subscale", binwidth=0.2
)
fig1.set_axis_labels(x_var="participant mean", y_var="frequency")
if save_figs:
    fig1.figure.savefig("PANAS_mean_participant.pdf", dpi=300)

# %%
sns.displot(
    data=df_esm_PANAS_summary_participant,
    x="median",
    hue="PANAS subscale",
    binwidth=0.2,
)

# %%
fig2 = sns.displot(
    data=df_esm_PANAS_summary_participant, x="std", hue="PANAS subscale", binwidth=0.05
)
fig2.set_axis_labels(x_var="participant standard deviation", y_var="frequency")
if save_figs:
    fig2.figure.savefig("PANAS_std_participant.pdf", dpi=300)

# %%
df_esm_PANAS_summary_participant[df_esm_PANAS_summary_participant["std"] < 0.1]

# %% [markdown]
# # Stress appraisal measure

# %%
df_SAM_all = extract_stressful_events(df_esm_inactive)

# %%
df_SAM_all.head()

# %%
df_esm_SAM = df_esm_preprocessed[
    (
        df_esm_preprocessed["questionnaire_id"]
        >= QUESTIONNAIRE_IDS["appraisal_stressfulness_event"]
    )
    & (
        df_esm_preprocessed["questionnaire_id"]
        <= QUESTIONNAIRE_IDS["appraisal_stressfulness_period"]
    )
]
df_esm_SAM_clean = clean_up_esm(df_esm_SAM)

# %% [markdown]
# ## Stressful events

# %%
df_esm_SAM_event = df_esm_SAM_clean[
    df_esm_SAM_clean["questionnaire_id"]
    == QUESTIONNAIRE_IDS["appraisal_stressfulness_event"]
].assign(stressful_event=lambda x: (x.esm_user_answer_numeric > 0))

# %%
df_esm_SAM_daily_events = (
    df_esm_SAM_event.groupby(["participant_id", "date_lj"])
    .stressful_event.agg("mean")
    .reset_index()
    .rename(columns={"stressful_event": "SAM_event_ratio"})
)

# %% [markdown]
# Calculate the daily mean of YES (1) or NO (0) answers to the question about stressful events. This is then the daily ratio of EMA sessions that included a stressful event.

# %%
df_esm_SAM_event_summary_participant = (
    df_esm_SAM_daily_events.groupby(["participant_id"])
    .SAM_event_ratio.agg(["mean", "median", "std"])
    .reset_index(col_level=1)
)

# %%
fig6 = sns.displot(data=df_esm_SAM_event_summary_participant, x="mean", binwidth=0.1)
fig6.set_axis_labels(
    x_var="participant proportion of stressful events", y_var="frequency"
)
if save_figs:
    fig6.figure.savefig("SAM_events_mean_participant.pdf", dpi=300)

# %%
sns.displot(data=df_esm_SAM_event_summary_participant, x="std", binwidth=0.05)

# %% [markdown]
# ### Threat and challenge

# %% [markdown]
# * Example of threat: "Did this event make you feel anxious?"
# * Example of challenge: "How eager are you to tackle this event?"
# * Possible answers:
#   0 - Not at all,
#   1 - Slightly,
#   2 - Moderately,
#   3 - Considerably,
#   4 - Extremely

# %%
df_esm_SAM_daily = (
    df_esm_SAM_clean.groupby(["participant_id", "date_lj", "questionnaire_id"])
    .esm_user_answer_numeric.agg("mean")
    .reset_index()
    .rename(columns={"esm_user_answer_numeric": "esm_numeric_mean"})
)

# %%
df_esm_SAM_daily_threat_challenge = df_esm_SAM_daily[
    (df_esm_SAM_daily["questionnaire_id"] == QUESTIONNAIRE_IDS["appraisal_threat"])
    | (df_esm_SAM_daily["questionnaire_id"] == QUESTIONNAIRE_IDS["appraisal_challenge"])
]

# %%
df_esm_SAM_summary_participant = (
    df_esm_SAM_daily.groupby(["participant_id", "questionnaire_id"])
    .esm_numeric_mean.agg(["mean", "median", "std"])
    .reset_index(col_level=1)
)

# %%
df_esm_SAM_event_stressfulness_summary_participant = df_esm_SAM_summary_participant[
    df_esm_SAM_summary_participant["questionnaire_id"]
    == QUESTIONNAIRE_IDS["appraisal_stressfulness_event"]
]
df_esm_SAM_event_stressfulness_summary_participant.describe()["mean"]

# %%
df_esm_SAM_event_stressfulness_summary_participant.describe()["std"]

# %%
sns.displot(
    data=df_esm_SAM_event_stressfulness_summary_participant, x="mean", binwidth=0.2
)

# %%
df_esm_SAM_threat_challenge_summary_participant = df_esm_SAM_summary_participant[
    (
        df_esm_SAM_summary_participant["questionnaire_id"]
        == QUESTIONNAIRE_IDS["appraisal_threat"]
    )
    | (
        df_esm_SAM_summary_participant["questionnaire_id"]
        == QUESTIONNAIRE_IDS["appraisal_challenge"]
    )
]
df_esm_SAM_threat_challenge_summary_participant[
    "event subscale"
] = df_esm_SAM_threat_challenge_summary_participant.questionnaire_id.astype(
    "category"
).cat.rename_categories(
    {88: "threat", 89: "challenge"}
)

# %%
sns.displot(
    data=df_esm_SAM_threat_challenge_summary_participant,
    x="mean",
    hue="event subscale",
    binwidth=0.2,
)

# %%
fig3 = sns.displot(
    data=df_esm_SAM_threat_challenge_summary_participant,
    x="std",
    hue="event subscale",
    binwidth=0.1,
)
fig3.set_axis_labels(x_var="participant standard deviation", y_var="frequency")
if save_figs:
    fig3.figure.savefig("SAM_std_participant.pdf", dpi=300)

# %%
df_esm_SAM_threat_challenge_summary_participant.groupby("event subscale").describe()[
    "mean"
]

# %%
df_esm_SAM_threat_challenge_summary_participant.groupby("event subscale").describe()[
    "std"
]

# %%
df_esm_SAM_clean.columns

# %%
df_esm_SAM_clean.esm_status.value_counts()

# %%
if export_data:
    df_esm_SAM_for_export = df_esm_SAM_clean[
        [
            "participant_id",
            "username",
            "device_id",
            "_id",
            "esm_trigger",
            "esm_session",
            "esm_notification_id",
            "question_id",
            "questionnaire_id",
            "esm_instructions",
            "double_esm_user_answer_timestamp",
            "datetime_lj",
            "date_lj",
            "time",
            "esm_user_answer",
            "esm_user_answer_numeric",
        ]
    ]
    df_esm_SAM_for_export = increment_answers(df_esm_SAM_for_export)
    df_esm_SAM_for_export.sort_values(
        by=["participant_id", "device_id", "_id"], ignore_index=True, inplace=True
    )
    print(df_esm_SAM_for_export.head())
    df_esm_SAM_for_export.to_csv(
        "../data/raw/df_esm_SAM_threat_challenge.csv", index=False
    )

# %% [markdown]
# ## Stressfulness of period

# %%
df_esm_SAM_period_summary_participant = df_esm_SAM_summary_participant[
    df_esm_SAM_summary_participant["questionnaire_id"]
    == QUESTIONNAIRE_IDS["appraisal_stressfulness_period"]
]

# %%
df_esm_SAM_period_summary_participant.describe()["mean"]

# %%
df_esm_SAM_period_summary_participant.describe()["std"]

# %%
sns.displot(data=df_esm_SAM_period_summary_participant, x="mean", binwidth=0.2)

# %%
sns.displot(data=df_esm_SAM_period_summary_participant, x="std", binwidth=0.1)

# %% [markdown]
# # Job demand and control

# %%
df_esm_JCQ_demand_control = df_esm_preprocessed[
    (df_esm_preprocessed["questionnaire_id"] >= QUESTIONNAIRE_IDS["JCQ_job_demand"])
    & (df_esm_preprocessed["questionnaire_id"] <= QUESTIONNAIRE_IDS["JCQ_job_control"])
]
df_esm_JCQ_demand_control_clean = clean_up_esm(df_esm_JCQ_demand_control)

# %%
df_esm_JCQ_demand_control_reversed = reverse_jcq_demand_control_scoring(
    df_esm_JCQ_demand_control_clean
)

# %%
df_esm_JCQ_daily = (
    df_esm_JCQ_demand_control_reversed.groupby(
        ["participant_id", "date_lj", "questionnaire_id"]
    )
    .esm_user_score.agg("mean")
    .reset_index()
    .rename(columns={"esm_user_score": "esm_score_mean"})
)
df_esm_JCQ_summary_participant = (
    df_esm_JCQ_daily.groupby(["participant_id", "questionnaire_id"])
    .esm_score_mean.agg(["mean", "median", "std"])
    .reset_index(col_level=1)
)
df_esm_JCQ_summary_participant[
    "JCQ subscale"
] = df_esm_JCQ_summary_participant.questionnaire_id.astype(
    "category"
).cat.rename_categories(
    {10: "job demand", 11: "job control"}
)

# %%
df_esm_JCQ_summary_participant.groupby("JCQ subscale").describe()["mean"]

# %%
df_esm_JCQ_summary_participant.groupby("JCQ subscale").describe()["std"]

# %%
fig4 = sns.displot(
    data=df_esm_JCQ_summary_participant,
    x="mean",
    hue="JCQ subscale",
    binwidth=0.1,
)
fig4.set_axis_labels(x_var="participant mean", y_var="frequency")
if save_figs:
    fig4.figure.savefig("JCQ_mean_participant.pdf", dpi=300)

# %%
fig5 = sns.displot(
    data=df_esm_JCQ_summary_participant,
    x="std",
    hue="JCQ subscale",
    binwidth=0.05,
)
fig6.set_axis_labels(x_var="participant standard deviation", y_var="frequency")
if save_figs:
    fig5.figure.savefig("JCQ_std_participant.pdf", dpi=300)

# %% [markdown]
# # COPE Inventory

# %%
df_esm_COPE = df_esm_preprocessed[
    (df_esm_preprocessed["questionnaire_id"] >= QUESTIONNAIRE_IDS["COPE_active"])
    & (df_esm_preprocessed["questionnaire_id"] <= QUESTIONNAIRE_IDS["COPE_emotions"])
]

# %%
df_esm_COPE_clean = clean_up_esm(df_esm_COPE)
df_esm_COPE_clean = increment_answers(df_esm_COPE_clean)
df_esm_COPE_fixed = reassign_question_ids(df_esm_COPE_clean)

# %%
if export_data:
    df_esm_COPE_for_export = df_esm_COPE_fixed[
        [
            "participant_id",
            "username",
            "device_id",
            "_id",
            "esm_trigger",
            "esm_session",
            "esm_notification_id",
            "question_id",
            "questionnaire_id",
            "esm_instructions",
            "double_esm_user_answer_timestamp",
            "datetime_lj",
            "date_lj",
            "time",
            "esm_user_answer",
            "esm_user_answer_numeric",
        ]
    ]
    df_esm_COPE_for_export.sort_values(
        by=["participant_id", "device_id", "_id"], ignore_index=True, inplace=True
    )
    print(df_esm_COPE_for_export.head())
    df_esm_COPE_for_export.to_csv("../data/raw/df_esm_COPE.csv", index=False)