diff --git a/config.yaml b/config.yaml index 248342e1..eafe27fe 100644 --- a/config.yaml +++ b/config.yaml @@ -3,7 +3,7 @@ ######################################################################################################################## # See https://www.rapids.science/latest/setup/configuration/#participant-files -PIDS: [p03] #p01, p02, p03] +PIDS: [p02] #p01, p02, p03] # See https://www.rapids.science/latest/setup/configuration/#automatic-creation-of-participant-files CREATE_PARTICIPANT_FILES: @@ -484,7 +484,7 @@ EMPATICA_ACCELEROMETER: FEATURES: ["maxmagnitude", "minmagnitude", "avgmagnitude", "medianmagnitude", "stdmagnitude"] SRC_SCRIPT: src/features/empatica_accelerometer/dbdp/main.py CR: - COMPUTE: True + COMPUTE: False FEATURES: ["totalMagnitudeBand", "absoluteMeanBand", "varianceBand"] # Acc features WINDOWS: COMPUTE: True @@ -511,7 +511,7 @@ EMPATICA_TEMPERATURE: FEATURES: ["maxtemp", "mintemp", "avgtemp", "mediantemp", "modetemp", "stdtemp", "diffmaxmodetemp", "diffminmodetemp", "entropytemp"] SRC_SCRIPT: src/features/empatica_temperature/dbdp/main.py CR: - COMPUTE: True + COMPUTE: False FEATURES: ["countAboveMean", "countBelowMean", "maximum", "minimum", "meanAbsChange", "longestStrikeAboveMean", "longestStrikeBelowMean", "stdDev", "median", "meanChange", "sumSquared", "squareSumOfComponent", "sumOfSquareComponents"] WINDOWS: @@ -529,7 +529,7 @@ EMPATICA_ELECTRODERMAL_ACTIVITY: FEATURES: ["maxeda", "mineda", "avgeda", "medianeda", "modeeda", "stdeda", "diffmaxmodeeda", "diffminmodeeda", "entropyeda"] SRC_SCRIPT: src/features/empatica_electrodermal_activity/dbdp/main.py CR: - COMPUTE: True + COMPUTE: False FEATURES: ['mean', 'std', 'q25', 'q75', 'qd', 'deriv', 'power', 'numPeaks', 'ratePeaks', 'powerPeaks', 'sumPosDeriv', 'propPosDeriv', 'derivTonic', 'sigTonicDifference', 'freqFeats','maxPeakAmplitudeChangeBefore', 'maxPeakAmplitudeChangeAfter', 'avgPeakAmplitudeChangeBefore', 'avgPeakAmplitudeChangeAfter', 'avgPeakChangeRatio', 'maxPeakIncreaseTime', 'maxPeakDecreaseTime', 'maxPeakDuration', 'maxPeakChangeRatio', diff --git a/src/data/streams/empatica_zip/container.py b/src/data/streams/empatica_zip/container.py index f1fe9bcd..93a6e5d8 100644 --- a/src/data/streams/empatica_zip/container.py +++ b/src/data/streams/empatica_zip/container.py @@ -129,19 +129,18 @@ def patch_ibi_with_bvp(ibi_data, bvp_data): ibi_data_file = BytesIO(ibi_data).getvalue().decode('utf-8') ibi_data_file = StringIO(ibi_data_file) - - # Begin with the cr-features part try: ibi_data, ibi_start_timestamp = empatica2d_to_array(ibi_data_file) - except IndexError: + except IndexError as e: # Checks whether IBI.csv is empty df_test = pd.read_csv(ibi_data_file, names=['timings', 'inter_beat_interval'], header=None) - print(df_test) if df_test.empty: df_test['timestamp'] = df_test['timings'] df_test = df_test.set_index('timestamp') return df_test + else: + raise IndexError("Something went wrong with indices. Error that was previously caught:\n", repr(e)) bvp_data_file = BytesIO(bvp_data).getvalue().decode('utf-8') bvp_data_file = StringIO(bvp_data_file) @@ -154,7 +153,7 @@ def patch_ibi_with_bvp(ibi_data, bvp_data): winsorize_value=25, hampel_fiter=False, median_filter=False, mod_z_score_filter=True, sampling=64, feature_names=['meanHr']) - ibi_timings, ibi_rr = get_patched_ibi_with_bvp(ibi_data[0], ibi_data[1], bvp_timings, bvp_rr, min_length=None) + ibi_timings, ibi_rr = get_patched_ibi_with_bvp(ibi_data[0], ibi_data[1], bvp_timings, bvp_rr) df = \ pd.DataFrame(np.array([ibi_timings, ibi_rr]).transpose(), columns=['timestamp', 'inter_beat_interval']) diff --git a/src/features/empatica_blood_volume_pulse/cr/main.py b/src/features/empatica_blood_volume_pulse/cr/main.py index 562c5a46..aa7a4186 100644 --- a/src/features/empatica_blood_volume_pulse/cr/main.py +++ b/src/features/empatica_blood_volume_pulse/cr/main.py @@ -1,7 +1,7 @@ import pandas as pd from scipy.stats import entropy -from cr_features.helper_functions import convert_to2d, hrv_features, hrv_freq_features +from cr_features.helper_functions import convert_to2d, hrv_features from cr_features.hrv import extract_hrv_features_2d_wrapper from cr_features_helper_methods import extract_second_order_features @@ -55,7 +55,7 @@ def cr_features(sensor_data_files, time_segment, provider, filter_data_by_segmen requested_window_length = None # name of the features this function can compute - base_intraday_features_names = hrv_features + hrv_freq_features + base_intraday_features_names = hrv_features # the subset of requested features this function can compute intraday_features_to_compute = list(set(requested_intraday_features) & set(base_intraday_features_names)) diff --git a/src/features/empatica_electrodermal_activity/cr/main.py b/src/features/empatica_electrodermal_activity/cr/main.py index 9499deec..25ecba59 100644 --- a/src/features/empatica_electrodermal_activity/cr/main.py +++ b/src/features/empatica_electrodermal_activity/cr/main.py @@ -4,6 +4,7 @@ from scipy.stats import entropy from cr_features.helper_functions import convert_to2d, gsr_features from cr_features.calculate_features import calculate_features +from cr_features.gsr import extractGsrFeatures2D from cr_features_helper_methods import extract_second_order_features import sys @@ -29,11 +30,11 @@ def extract_eda_features_from_intraday_data(eda_intraday_data, features, window_ if window_length is None: eda_intraday_features = \ eda_intraday_data.groupby('local_segment').apply(\ - lambda x: calculate_features(convert_to2d(x['electrodermal_activity'], x.shape[0]), fs=sample_rate, feature_names=features)) + lambda x: extractGsrFeatures2D(convert_to2d(x['electrodermal_activity'], x.shape[0]), sampleRate=sample_rate, threshold=0, featureNames=features)) else: eda_intraday_features = \ eda_intraday_data.groupby('local_segment').apply(\ - lambda x: calculate_features(convert_to2d(x['electrodermal_activity'], window_length*sample_rate), fs=sample_rate, feature_names=features)) + lambda x: extractGsrFeatures2D(convert_to2d(x['electrodermal_activity'], window_length*sample_rate), sampleRate=sample_rate, threshold=0, featureNames=features)) eda_intraday_features.reset_index(inplace=True) diff --git a/src/features/empatica_inter_beat_interval/cr/main.py b/src/features/empatica_inter_beat_interval/cr/main.py index 8b4cab5d..0e27a79f 100644 --- a/src/features/empatica_inter_beat_interval/cr/main.py +++ b/src/features/empatica_inter_beat_interval/cr/main.py @@ -1,7 +1,7 @@ import pandas as pd import numpy as np -from cr_features.helper_functions import convert_ibi_to2d_time, hrv_features, hrv_freq_features +from cr_features.helper_functions import convert_ibi_to2d_time, hrv_features from cr_features.hrv import extract_hrv_features_2d_wrapper, get_HRV_features from cr_features_helper_methods import extract_second_order_features @@ -61,7 +61,7 @@ def cr_features(sensor_data_files, time_segment, provider, filter_data_by_segmen requested_window_length = None # name of the features this function can compute - base_intraday_features_names = hrv_features + hrv_freq_features + base_intraday_features_names = hrv_features # the subset of requested features this function can compute intraday_features_to_compute = list(set(requested_intraday_features) & set(base_intraday_features_names)) diff --git a/tests/scripts/missing_values_heatmap.py b/tests/scripts/missing_values_heatmap.py index 827343dd..2f2364f2 100644 --- a/tests/scripts/missing_values_heatmap.py +++ b/tests/scripts/missing_values_heatmap.py @@ -5,14 +5,19 @@ import matplotlib.pyplot as plt # path = "/rapids/data/processed/features/all_participants/all_sensor_features.csv" # all features all participants # path = "/rapids/data/interim/p03/empatica_accelerometer_features/empatica_accelerometer_python_cr_windows.csv" -path = "/rapids/data/interim/p02/empatica_electrodermal_activity_features/empatica_electrodermal_activity_python_cr_windows.csv" +path = "/rapids/data/interim/p03/empatica_electrodermal_activity_features/empatica_electrodermal_activity_python_cr_windows.csv" # path = "/rapids/data/interim/p02/empatica_inter_beat_interval_features/empatica_inter_beat_interval_python_cr_windows.csv" # path = "/rapids/data/interim/p02/empatica_blood_volume_pulse_features/empatica_blood_volume_pulse_python_cr_windows.csv" # path = "/rapids/data/interim/p02/empatica_temperature_features/empatica_temperature_python_cr_windows.csv" df = pd.read_csv(path) print(df) +is_NaN = df. isnull() +row_has_NaN = is_NaN. any(axis=1) +rows_with_NaN = df[row_has_NaN] +print(rows_with_NaN.size) + sns.heatmap(df.isna(), cbar=False) -plt.savefig('eda_windows_p02_window_60_more_peaks.png', bbox_inches='tight') +plt.savefig('eda_windows_p03_window_60_thresh_default.png', bbox_inches='tight')