junos
/
rapids
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Changes in the cleaning script and preparation of empatica data yield method.

notes
Primoz 2022-09-23 13:24:50 +00:00
parent 19aa8707c0
commit d9a574c550
2 changed files with 61 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
src/features/all_cleaning_individual/straw/main.py
View File

@ -25,9 +25,6 @@ def straw_cleaning(sensor_data_files, provider):
target = config['PARAMS_FOR_ANALYSIS']['TARGET']['LABEL'] # get target label from config
features = features[features['phone_esm_straw_' + target].notna()].reset_index(drop=True)
# TODO: add conditions that differentiates cleaning steps for standardized and nonstandardized features, for this
# the snakemake rules will also have to come with additional parameter (in rules/features.smk)
# TODO: imputate the rows where the participants have at least 2 rows (2 time segments) - error prevention (has to be tested)
# TODO: because of different imputation logic (e.g., the phone_data_yield parameter for phone features) the imputation has to
# be planned accordingly. Should the phone features first be imputated with 0 and only then general kNN imputation is executed
@ -38,12 +35,29 @@ def straw_cleaning(sensor_data_files, provider):
# values still exist.
# (2) REMOVE COLS IF THEIR NAN THRESHOLD IS PASSED (should be <= if even all NaN columns must be preserved - this solution now drops columns with all NaN rows)
# TODO: determine the threshold at which the column should be removed because of too many Nans.
features = features.loc[:, features.isna().sum() < provider["COLS_NAN_THRESHOLD"] * features.shape[0]]
# (3.1) QUALITY CHECK (DATA YIELD COLUMN) which determines if the row stays or not (if either E4 or phone is low quality the row is useless - TODO: determine threshold)
# Here, the imputation is still not executed - only quality check
# ??? Drop rows with the value of phone_data_yield_column less than data_yield_ratio_threshold ???
phone_data_yield_unit = provider["PHONE_DATA_YIELD_FEATURE"].split("_")[3].lower()
phone_data_yield_column = "phone_data_yield_rapids_ratiovalidyielded" + phone_data_yield_unit
empatica_data_yield_column = "????????????"
if not phone_data_yield_column in features.columns and not empatica_data_yield_column in features.columns:
raise KeyError(f"RAPIDS provider needs to clean the selected event features based on {phone_data_yield_column} column, please set config[PHONE_DATA_YIELD][PROVIDERS][RAPIDS][COMPUTE] to True and include 'ratiovalidyielded{data_yield_unit}' in [FEATURES].")
if provider["PHONE_DATA_YIELD_RATIO_THRESHOLD"]:
features = features[features[phone_data_yield_column] >= provider["PHONE_DATA_YIELD_RATIO_THRESHOLD"]]
# Potrebno premisliti točno kako bi izgledal data_yield za E4: bo se ustvaril dodaten stolpec; bodo različne spremenljivke, podobno kot hour in minute pri phone?
if provider["EMPATICA_DATA_YIELD_RATIO_THRESHOLD"]:
features = features[features['???????????????'] >= provider["EMPATICA_DATA_YIELD_RATIO_THRESHOLD"]]
# ---> imputation ??
impute_phone_features = provider["IMPUTE_PHONE_SELECTED_EVENT_FEATURES"]
if True: #impute_phone_features["COMPUTE"]:
@ -64,22 +78,20 @@ def straw_cleaning(sensor_data_files, provider):
features.loc[mask, phone_cols] = impute(features[mask][phone_cols], method=impute_phone_features["TYPE"].lower())
print(features[features['phone_data_yield_rapids_ratiovalidyieldedminutes'] > impute_phone_features['MIN_DATA_YIELDED_MINUTES_TO_IMPUTE']][phone_cols])
# ??? Drop rows with the value of data_yield_column less than data_yield_ratio_threshold ???
data_yield_unit = provider["DATA_YIELD_FEATURE"].split("_")[3].lower()
data_yield_column = "phone_data_yield_rapids_ratiovalidyielded" + data_yield_unit
if not data_yield_column in features.columns:
raise KeyError(f"RAPIDS provider needs to impute the selected event features based on {data_yield_column} column, please set config[PHONE_DATA_YIELD][PROVIDERS][RAPIDS][COMPUTE] to True and include 'ratiovalidyielded{data_yield_unit}' in [FEATURES].")
if provider["DATA_YIELD_RATIO_THRESHOLD"]:
features = features[features[data_yield_column] >= provider["DATA_YIELD_RATIO_THRESHOLD"]]
# (3.2) (optional) DOES ROW CONSIST OF ENOUGH NON-NAN VALUES? Possible some of these examples could still pass previous condition but not this one?
# () Remove rows if threshold of NaN values is passed
min_count = math.ceil((1 - provider["ROWS_NAN_THRESHOLD"]) * features.shape[1]) # minimal not nan values in row
features.dropna(axis=0, thresh=min_count, inplace=True)
# (4) IMPUTATION: IMPUTE DATA WITH KNN METHOD
# (4) IMPUTATION: IMPUTE DATA WITH KNN METHOD (TODO: for now only kNN)
# - no other input restriction for this method except that rows are full enough and have reasonably high quality as assessed by data yield
graph_bf_af(features, "before_knn")
impute_cols = [col for col in features.columns if col not in excluded_columns]
features[impute_cols] = impute(features[impute_cols], method="knn")
graph_bf_af(features, "after_knn")
# (5) REMOVE COLS WHERE VARIANCE IS 0
if provider["COLS_VAR_THRESHOLD"]:
@ -105,25 +117,12 @@ def straw_cleaning(sensor_data_files, provider):
features.drop(to_drop, axis=1, inplace=True)
# (7) Remove rows if threshold of NaN values is passed
min_count = math.ceil((1 - provider["ROWS_NAN_THRESHOLD"]) * features.shape[1]) # minimal not nan values in row
features.dropna(axis=0, thresh=min_count, inplace=True)
sns.set(rc={"figure.figsize":(16, 8)})
sns.heatmap(features.isna(), cbar=False)
plt.savefig(f'features_nans_bf_knn.png', bbox_inches='tight')
## (8) STANDARDIZATION
if provider["STANDARDIZATION"]:
features.loc[:, ~features.columns.isin(excluded_columns)] = StandardScaler().fit_transform(features.loc[:, ~features.columns.isin(excluded_columns)])
sns.set(rc={"figure.figsize":(16, 8)})
sns.heatmap(features.isna(), cbar=False)
plt.savefig(f'features_nans_af_knn.png', bbox_inches='tight')
# (9) VERIFY IF THERE ARE ANY NANS LEFT IN THE DATAFRAME
if features.isna().any().any():
raise ValueError
@ -132,6 +131,11 @@ def straw_cleaning(sensor_data_files, provider):
return features
def graph_bf_af(features, phase_name):
sns.set(rc={"figure.figsize":(16, 8)})
sns.heatmap(features.isna(), cbar=False)
plt.savefig(f'features_nans_{phase_name}.png', bbox_inches='tight')
def impute(df, method='zero'):
def k_nearest(df):

29
src/features/empatica_data_yield.py 100644
View File

@ -0,0 +1,29 @@
import pandas as pd
import numpy as np
from datetime import datetime
import sys
def calculate_empatica_data_yield(features):
# Get time segment duration in seconds from dataframe
datetime_start = datetime.strptime(df.loc[0, 'local_segment_start_datetime'], '%y-%m-%d %H:%M:%S')
datetime_end = datetime.strptime(df.loc[0, 'local_segment_end_datetime'], '%y-%m-%d %H:%M:%S')
tseg_duration = (datetime_end - datetime_start).total_seconds()
acc_data_yield = (features['empatica_accelerometer_cr_SO_windowsCount'] * 15) / tseg_duration
temp_data_yield = (features['empatica_temperature_cr_SO_windowsCount'] * 300) / tseg_duration
acc_data_yield = (features['empatica_electrodermal_activity_cr_SO_windowsCount'] * 60) / tseg_duration
ibi_data_yield = (features['empatica_inter_beat_interval_cr_SO_windowsCount'] * 300) / tseg_duration
# TODO: morda smisleno obdelovati različne senzorje ločeno -> lahko da ibi ne bo dobre kvalitete, ostali pa bodo okej. Zakaj bi samo zaradi IBI zavrgli celotno vrstico ...
# lahko se tudi naredi overall kvaliteta empatice npr. povprečje vseh data_yield rezultatov? Oz. povprečje z utežmi glede na število stolpcev, ki jih senzor vsebuje
# ... čeprav št. stolpcev ni najboljše, saj je pomembnost nekaterih (npr. EDA) značilk zelo vprašljiva.
# TODO: bolja nastavitev delovnih ur sedaj je od 4 do 4... to povzroči veliko manjkajočih podatkov in posledično nizek (telefonski in E4) data_yield ...
data_yield_features = [col for col in features.columns if "SO_windowsCount" in col and "a"]