From 9199b53ded1c5d858882d9826aa15e5e2102ab08 Mon Sep 17 00:00:00 2001 From: Primoz Date: Wed, 9 Nov 2022 15:11:51 +0000 Subject: [PATCH] Get, join and start processing required ERS stress event data. --- .../process_user_event_related_segments.py | 56 ++++++++++++++++--- 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/src/features/phone_esm/straw/process_user_event_related_segments.py b/src/features/phone_esm/straw/process_user_event_related_segments.py index c2569a3b..59a72104 100644 --- a/src/features/phone_esm/straw/process_user_event_related_segments.py +++ b/src/features/phone_esm/straw/process_user_event_related_segments.py @@ -22,6 +22,7 @@ def format_timestamp(x): return tstring + def extract_ers_from_file(esm_df, device_id): pd.set_option("display.max_rows", 20) @@ -31,7 +32,7 @@ def extract_ers_from_file(esm_df, device_id): config = yaml.load(stream, Loader=yaml.FullLoader) - pd.DataFrame().to_csv(snakemake.output[1]) # Create an empty stress event file either way + pd.DataFrame().to_csv(snakemake.output[1]) # Create an empty stress event file either way TODO esm_preprocessed = clean_up_esm(preprocess_esm(esm_df)) @@ -45,7 +46,7 @@ def extract_ers_from_file(esm_df, device_id): targets_method = config["TIME_SEGMENTS"]["TAILORED_EVENTS"]["TARGETS_METHOD"] if targets_method in ["30_before", "90_before"]: # takes 30-minute peroid before the questionnaire + the duration of the questionnaire # Extract time-relevant information - extracted_ers = esm_df.groupby(["device_id", "esm_session"])['timestamp'].apply(lambda x: math.ceil((x.max() - x.min()) / 1000)).reset_index() # is rounded up in seconds + extracted_ers = esm_df.groupby(["device_id", "esm_session"])['timestamp'].apply(lambda x: math.ceil((x.max() - x.min()) / 1000)).reset_index() # questionnaire length extracted_ers["label"] = f"straw_event_{targets_method}_" + snakemake.params["pid"] + "_" + extracted_ers.index.astype(str).str.zfill(3) extracted_ers[['event_timestamp', 'device_id']] = esm_df.groupby(["device_id", "esm_session"])['timestamp'].min().reset_index()[['timestamp', 'device_id']] extracted_ers = extracted_ers[extracted_ers["timestamp"] <= 15 * 60].reset_index(drop=True) # ensure that the longest duration of the questionnaire anwsering is 15 min @@ -72,18 +73,55 @@ def extract_ers_from_file(esm_df, device_id): extracted_ers["shift"] = extracted_ers["diffs"].apply(lambda x: format_timestamp(x)) elif targets_method == "stress_event": - - pd.DataFrame().to_csv(snakemake.output[1]) - # TODO: generiranje ERS datoteke za stress_events + + # Get and join required data + extracted_ers = esm_df.groupby(["device_id", "esm_session"])['timestamp'].apply(lambda x: math.ceil((x.max() - x.min()) / 1000)).reset_index().rename(columns={'timestamp': 'session_length'}) # questionnaire end timestamp + session_end_timestamp = esm_df.groupby(['device_id', 'esm_session'])['timestamp'].max().to_frame().rename(columns={'timestamp': 'session_end_timestamp'}) # questionnaire end timestamp + se_time = esm_df[esm_df.questionnaire_id == 90.].set_index(['device_id', 'esm_session'])['esm_user_answer'].to_frame().rename(columns={'esm_user_answer': 'se_time'}) + se_duration = esm_df[esm_df.questionnaire_id == 91.].set_index(['device_id', 'esm_session'])['esm_user_answer'].to_frame().rename(columns={'esm_user_answer': 'se_duration'}) + se_intensity = esm_df[esm_df.questionnaire_id == 87.].set_index(['device_id', 'esm_session'])['esm_user_answer_numeric'].to_frame().rename(columns={'esm_user_answer_numeric': 'se_intensity'}) + + extracted_ers = extracted_ers.join(session_end_timestamp, on=['device_id', 'esm_session'], how='inner') \ + .join(se_time, on=['device_id', 'esm_session'], how='inner') \ + .join(se_duration, on=['device_id', 'esm_session'], how='inner') \ + .join(se_intensity, on=['device_id', 'esm_session'], how='inner') + + # Filter sessions that are not useful + extracted_ers = extracted_ers[(extracted_ers.se_time != "0 - Ne spomnim se")] + + # Transform data into its final form, ready for the extraction + extracted_ers.reset_index(inplace=True) + extracted_ers["label"] = f"straw_event_{targets_method}_" + snakemake.params["pid"] + "_" + extracted_ers.index.astype(str).str.zfill(3) + + # Convert to unix timestamp + + time_before_event = 90 * 60 # in seconds (10 minutes) + extracted_ers['event_timestamp'] = pd.to_datetime(extracted_ers['se_time']).apply(lambda x: x.timestamp() * 1000).astype('int64') + extracted_ers['shift'] = time_before_event + extracted_ers['shift_direction'] = -1 + + print(extracted_ers[['session_end_timestamp', 'event_timestamp']]) + + extracted_ers['se_duration'] = \ + np.where(extracted_ers['se_duration'] == "1 - Še vedno traja", + extracted_ers['session_end_timestamp'] - extracted_ers['event_timestamp'], + extracted_ers['se_duration']) + + extracted_ers['se_duration'] = \ + extracted_ers['se_duration'].apply(lambda x: math.ceil(x / 1000) if isinstance(x, int) else (pd.to_datetime(x).hour * 60 + pd.to_datetime(x).minute) * 60) + + sys.exit() # VV Testiranje različnih povpraševanj za VV - # print(esm_df[esm_df.questionnaire_id == 87]) - # filter_esm = esm_df[(esm_df.esm_type == 7) & ((esm_df.questionnaire_id == 90.) | (esm_df.questionnaire_id == 91.))][['questionnaire_id', 'esm_user_answer', 'esm_session']] - # print(filter_esm[filter_esm.esm_user_answer == "1 - Še vedno traja"].shape) - # print(filter_esm.shape) + filter_esm = esm_df[(esm_df.esm_type == 7) & ((esm_df.questionnaire_id == 90.) | (esm_df.questionnaire_id == 91.))][['questionnaire_id', 'esm_user_answer', 'esm_session']] + print(filter_esm[filter_esm.esm_user_answer == "1 - Še vedno traja"].shape) + print(filter_esm.shape) # TODO: generiranje stress_events_targets datoteke (dodaj tudi stolpec s pid) + dodati moraš merge metodo, ki bo združila te datoteke # TODO: na koncu se mora v čistilni skripti ustrezno odstraniti vse targete in prilepiti nove targete zraven ustreznih segmentov (zna se zgoditi, da bodo overlap) + + pd.DataFrame().to_csv(snakemake.output[1]) + sys.exit() else: raise Exception("Please select correct target method for the event-related segments.")