From 2d5d23b615830b32fedfc342aa56b6601f1fda65 Mon Sep 17 00:00:00 2001 From: Primoz Date: Wed, 6 Jul 2022 07:35:39 +0000 Subject: [PATCH] Testing files change and remove standardization from hrv sensors main files. --- .../empatica_blood_volume_pulse/cr/main.py | 6 --- .../empatica_inter_beat_interval/cr/main.py | 9 +--- tests/scripts/missing_vals.py | 49 ++++++++++++------- tests/scripts/zero_vals.py | 23 ++++++--- 4 files changed, 48 insertions(+), 39 deletions(-) diff --git a/src/features/empatica_blood_volume_pulse/cr/main.py b/src/features/empatica_blood_volume_pulse/cr/main.py index 243f7e72..1e7df9b2 100644 --- a/src/features/empatica_blood_volume_pulse/cr/main.py +++ b/src/features/empatica_blood_volume_pulse/cr/main.py @@ -66,14 +66,8 @@ def cr_features(sensor_data_files, time_segment, provider, filter_data_by_segmen requested_window_length, time_segment, filter_data_by_segment) if calc_windows: - if provider["WINDOWS"].get("STANDARDIZE_FEATURES", False): - fo_columns = bvp_intraday_features.columns.values[2:] - fo_columns_z_score = [col + "_zscore" for col in fo_columns] - bvp_intraday_features[fo_columns_z_score] = StandardScaler().fit_transform(bvp_intraday_features[fo_columns]) - so_features_names = provider["WINDOWS"]["SECOND_ORDER_FEATURES"] bvp_second_order_features = extract_second_order_features(bvp_intraday_features, so_features_names) - return bvp_intraday_features, bvp_second_order_features return bvp_intraday_features \ No newline at end of file diff --git a/src/features/empatica_inter_beat_interval/cr/main.py b/src/features/empatica_inter_beat_interval/cr/main.py index 29d00b23..803bf3a8 100644 --- a/src/features/empatica_inter_beat_interval/cr/main.py +++ b/src/features/empatica_inter_beat_interval/cr/main.py @@ -23,7 +23,7 @@ def extract_ibi_features_from_intraday_data(ibi_intraday_data, features, window_ if not ibi_intraday_data.empty: ibi_intraday_features = pd.DataFrame() - + # apply methods from calculate features module if window_length is None: ibi_intraday_features = \ @@ -70,12 +70,7 @@ def cr_features(sensor_data_files, time_segment, provider, filter_data_by_segmen ibi_intraday_features = extract_ibi_features_from_intraday_data(ibi_intraday_data, intraday_features_to_compute, requested_window_length, time_segment, filter_data_by_segment) - if calc_windows: - if provider["WINDOWS"].get("STANDARDIZE_FEATURES", False): - fo_columns = ibi_intraday_features.columns.values[2:] - fo_columns_z_score = [col + "_zscore" for col in fo_columns] - ibi_intraday_features[fo_columns_z_score] = StandardScaler().fit_transform(ibi_intraday_features[fo_columns]) - + if calc_windows: so_features_names = provider["WINDOWS"]["SECOND_ORDER_FEATURES"] ibi_second_order_features = extract_second_order_features(ibi_intraday_features, so_features_names) diff --git a/tests/scripts/missing_vals.py b/tests/scripts/missing_vals.py index db8eb193..9965bec3 100644 --- a/tests/scripts/missing_vals.py +++ b/tests/scripts/missing_vals.py @@ -2,27 +2,38 @@ import pandas as pd import seaborn as sns import matplotlib.pyplot as plt + participant = "p02" +all_sensors = ["ibi", "bvp"]#["eda", "bvp", "ibi", "temp", "acc"] -# path = "/rapids/data/processed/features/all_participants/all_sensor_features.csv" # all features all participants -# path = "/rapids/data/interim/p03/empatica_accelerometer_features/empatica_accelerometer_python_cr_windows.csv" -path = f"/rapids/data/interim/{participant}/empatica_electrodermal_activity_features/empatica_electrodermal_activity_python_cr_windows.csv" -# path = "/rapids/data/interim/p02/empatica_inter_beat_interval_features/empatica_inter_beat_interval_python_cr_windows.csv" -# path = "/rapids/data/interim/p02/empatica_blood_volume_pulse_features/empatica_blood_volume_pulse_python_cr_windows.csv" -# path = "/rapids/data/interim/p02/empatica_temperature_features/empatica_temperature_python_cr_windows.csv" +for sensor in all_sensors: -df = pd.read_csv(path) -print(df) -is_NaN = df.isnull() -row_has_NaN = is_NaN.any(axis=1) -rows_with_NaN = df[row_has_NaN] - -print("All rows:", len(df.index)) -print("\nCount NaN vals:", rows_with_NaN.size) -print("\nDf mean:") -print(df.mean()) - -sns.heatmap(df.isna(), cbar=False) -plt.savefig(f'eda_{participant}_windows_NaN.png', bbox_inches='tight') + if sensor == "eda": + path = f"/rapids/data/interim/{participant}/empatica_electrodermal_activity_features/empatica_electrodermal_activity_python_cr_windows.csv" + elif sensor == "bvp": + path = f"/rapids/data/interim/{participant}/empatica_blood_volume_pulse_features/empatica_blood_volume_pulse_python_cr_windows.csv" + elif sensor == "ibi": + path = f"/rapids/data/interim/{participant}/empatica_inter_beat_interval_features/empatica_inter_beat_interval_python_cr_windows.csv" + elif sensor == "acc": + path = f"/rapids/data/interim/{participant}/empatica_accelerometer_features/empatica_accelerometer_python_cr_windows.csv" + elif sensor == "temp": + path = f"/rapids/data/interim/{participant}/empatica_temperature_features/empatica_temperature_python_cr_windows.csv" + else: + path = "/rapids/data/processed/features/all_participants/all_sensor_features.csv" # all features all participants + + + df = pd.read_csv(path) + print(df) + is_NaN = df.isnull() + row_has_NaN = is_NaN.any(axis=1) + rows_with_NaN = df[row_has_NaN] + + print("All rows:", len(df.index)) + print("\nCount NaN vals:", rows_with_NaN.size) + print("\nDf mean:") + print(df.mean()) + + sns.heatmap(df.isna(), cbar=False) + plt.savefig(f'{sensor}_{participant}_windows_NaN.png', bbox_inches='tight') diff --git a/tests/scripts/zero_vals.py b/tests/scripts/zero_vals.py index d657673c..0b02a07c 100644 --- a/tests/scripts/zero_vals.py +++ b/tests/scripts/zero_vals.py @@ -3,14 +3,23 @@ import seaborn as sns import matplotlib.pyplot as plt from itertools import compress -participant = "p02" -# path = "/rapids/data/processed/features/all_participants/all_sensor_features.csv" # all features all participants -# path = "/rapids/data/interim/p03/empatica_accelerometer_features/empatica_accelerometer_python_cr_windows.csv" -path = f"/rapids/data/interim/{participant}/empatica_electrodermal_activity_features/empatica_electrodermal_activity_python_cr_windows.csv" -# path = "/rapids/data/interim/p02/empatica_inter_beat_interval_features/empatica_inter_beat_interval_python_cr_windows.csv" -# path = "/rapids/data/interim/p02/empatica_blood_volume_pulse_features/empatica_blood_volume_pulse_python_cr_windows.csv" -# path = "/rapids/data/interim/p02/empatica_temperature_features/empatica_temperature_python_cr_windows.csv" +participant = "p031" +sensor = "eda" + +if sensor == "eda": + path = f"/rapids/data/interim/{participant}/empatica_electrodermal_activity_features/empatica_electrodermal_activity_python_cr_windows.csv" +elif sensor == "bvp": + path = f"/rapids/data/interim/{participant}/empatica_blood_volume_pulse_features/empatica_blood_volume_pulse_python_cr_windows.csv" +elif sensor == "ibi": + path = f"/rapids/data/interim/{participant}/empatica_inter_beat_interval_features/empatica_inter_beat_interval_python_cr_windows.csv" +elif sensor == "acc": + path = f"/rapids/data/interim/{participant}/empatica_accelerometer_features/empatica_accelerometer_python_cr_windows.csv" +elif sensor == "temp": + path = f"/rapids/data/interim/{participant}/empatica_temperature_features/empatica_temperature_python_cr_windows.csv" +else: + path = "/rapids/data/processed/features/all_participants/all_sensor_features.csv" # all features all participants" + df = pd.read_csv(path) df_num_peaks_zero = df[df["empatica_electrodermal_activity_cr_numPeaks"] == 0]