From 2a8f58f5c86aadbb08a9869127e8e9a140fcf7a1 Mon Sep 17 00:00:00 2001 From: Primoz Date: Fri, 20 May 2022 13:18:45 +0000 Subject: [PATCH] Patching IBI with BVP. WIP --- Snakefile | 2 ++ config.yaml | 25 ++++++++++--------- data/external/timesegments_periodic.csv | 2 +- rules/features.smk | 1 + .../cr/main.py | 11 ++++++-- .../empatica_inter_beat_interval/cr/main.py | 16 ++++++++++-- src/features/entry.py | 3 ++- src/features/utils/utils.py | 4 +++ 8 files changed, 46 insertions(+), 18 deletions(-) diff --git a/Snakefile b/Snakefile index 7dc5ad3a..d060e9f7 100644 --- a/Snakefile +++ b/Snakefile @@ -359,6 +359,8 @@ for provider in config["EMPATICA_INTER_BEAT_INTERVAL"]["PROVIDERS"].keys(): if config["EMPATICA_INTER_BEAT_INTERVAL"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/empatica_inter_beat_interval_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/empatica_inter_beat_interval_with_datetime.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/raw/{pid}/empatica_blood_volume_pulse_raw.csv", pid=config["PIDS"])) + files_to_compute.extend(expand("data/raw/{pid}/empatica_blood_volume_pulse_with_datetime.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/empatica_inter_beat_interval_features/empatica_inter_beat_interval_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["EMPATICA_INTER_BEAT_INTERVAL"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/empatica_inter_beat_interval.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) diff --git a/config.yaml b/config.yaml index 0bca22c5..69e14921 100644 --- a/config.yaml +++ b/config.yaml @@ -3,7 +3,7 @@ ######################################################################################################################## # See https://www.rapids.science/latest/setup/configuration/#participant-files -PIDS: [p01, p02] +PIDS: [p01] #p02, p03] # See https://www.rapids.science/latest/setup/configuration/#automatic-creation-of-participant-files CREATE_PARTICIPANT_FILES: @@ -477,11 +477,11 @@ EMPATICA_ACCELEROMETER: CONTAINER: ACC PROVIDERS: DBDP: - COMPUTE: True + COMPUTE: False FEATURES: ["maxmagnitude", "minmagnitude", "avgmagnitude", "medianmagnitude", "stdmagnitude"] SRC_SCRIPT: src/features/empatica_accelerometer/dbdp/main.py CR: - COMPUTE: True + COMPUTE: False FEATURES: ["fqHighestPeakFreqs", "fqHighestPeaks", "fqEnergyFeat", "fqEntropyFeat", "fqHistogramBins","fqAbsMean", "fqSkewness", "fqKurtosis", "fqInterquart", # Freq features "meanLow", "areaLow", "totalAbsoluteAreaBand", "totalMagnitudeBand", "entropyBand", "skewnessBand", "kurtosisBand", "postureDistanceLow", "absoluteMeanBand", "absoluteAreaBand", "quartilesBand", "interQuartileRangeBand", "varianceBand", @@ -502,7 +502,7 @@ EMPATICA_HEARTRATE: CONTAINER: HR PROVIDERS: DBDP: - COMPUTE: True + COMPUTE: False FEATURES: ["maxhr", "minhr", "avghr", "medianhr", "modehr", "stdhr", "diffmaxmodehr", "diffminmodehr", "entropyhr"] SRC_SCRIPT: src/features/empatica_heartrate/dbdp/main.py @@ -511,11 +511,11 @@ EMPATICA_TEMPERATURE: CONTAINER: TEMP PROVIDERS: DBDP: - COMPUTE: True + COMPUTE: False FEATURES: ["maxtemp", "mintemp", "avgtemp", "mediantemp", "modetemp", "stdtemp", "diffmaxmodetemp", "diffminmodetemp", "entropytemp"] SRC_SCRIPT: src/features/empatica_temperature/dbdp/main.py CR: - COMPUTE: True + COMPUTE: False FEATURES: ["autocorrelations", "countAboveMean", "countBelowMean", "maximum", "minimum", "meanAbsChange", "longestStrikeAboveMean", "longestStrikeBelowMean", "stdDev", "median", "meanChange", "numberOfZeroCrossings", "absEnergy", "linearTrendSlope", "ratioBeyondRSigma", "binnedEntropy", "numOfPeaksAutocorr", "numberOfZeroCrossingsAutocorr", "areaAutocorr", @@ -532,11 +532,11 @@ EMPATICA_ELECTRODERMAL_ACTIVITY: CONTAINER: EDA PROVIDERS: DBDP: - COMPUTE: True + COMPUTE: False FEATURES: ["maxeda", "mineda", "avgeda", "medianeda", "modeeda", "stdeda", "diffmaxmodeeda", "diffminmodeeda", "entropyeda"] SRC_SCRIPT: src/features/empatica_electrodermal_activity/dbdp/main.py CR: - COMPUTE: True + COMPUTE: False FEATURES: ['mean', 'std', 'q25', 'q75', 'qd', 'deriv', 'power', 'numPeaks', 'ratePeaks', 'powerPeaks', 'sumPosDeriv', 'propPosDeriv', 'derivTonic', 'sigTonicDifference', 'freqFeats','maxPeakAmplitudeChangeBefore', 'maxPeakAmplitudeChangeAfter', 'avgPeakAmplitudeChangeBefore', 'avgPeakAmplitudeChangeAfter', 'avgPeakChangeRatio', 'maxPeakIncreaseTime', 'maxPeakDecreaseTime', 'maxPeakDuration', 'maxPeakChangeRatio', @@ -553,11 +553,11 @@ EMPATICA_BLOOD_VOLUME_PULSE: CONTAINER: BVP PROVIDERS: DBDP: - COMPUTE: True + COMPUTE: False FEATURES: ["maxbvp", "minbvp", "avgbvp", "medianbvp", "modebvp", "stdbvp", "diffmaxmodebvp", "diffminmodebvp", "entropybvp"] SRC_SCRIPT: src/features/empatica_blood_volume_pulse/dbdp/main.py CR: - COMPUTE: True + COMPUTE: False FEATURES: ['meanHr', 'ibi', 'sdnn', 'sdsd', 'rmssd', 'pnn20', 'pnn50', 'sd', 'sd2', 'sd1/sd2', 'numRR', # Time features 'VLF', 'LF', 'LFnorm', 'HF', 'HFnorm', 'LF/HF', 'fullIntegral'] # Freq features WINDOWS: @@ -571,13 +571,14 @@ EMPATICA_INTER_BEAT_INTERVAL: CONTAINER: IBI PROVIDERS: DBDP: - COMPUTE: True + COMPUTE: False FEATURES: ["maxibi", "minibi", "avgibi", "medianibi", "modeibi", "stdibi", "diffmaxmodeibi", "diffminmodeibi", "entropyibi"] SRC_SCRIPT: src/features/empatica_inter_beat_interval/dbdp/main.py CR: COMPUTE: True FEATURES: ['meanHr', 'ibi', 'sdnn', 'sdsd', 'rmssd', 'pnn20', 'pnn50', 'sd', 'sd2', 'sd1/sd2', 'numRR', # Time features - 'VLF', 'LF', 'LFnorm', 'HF', 'HFnorm', 'LF/HF', 'fullIntegral'] # Freq features + 'VLF', 'LF', 'LFnorm', 'HF', 'HFnorm', 'LF/HF', 'fullIntegral'] # Freq features + PATCH_WITH_BVP: True WINDOWS: COMPUTE: True WINDOW_LENGTH: 300 # specify window length in seconds diff --git a/data/external/timesegments_periodic.csv b/data/external/timesegments_periodic.csv index 0a26eb6b..a2c4d9ac 100644 --- a/data/external/timesegments_periodic.csv +++ b/data/external/timesegments_periodic.csv @@ -1,3 +1,3 @@ label,start_time,length,repeats_on,repeats_value daily,00:00:00,23H 59M 59S,every_day,0 -night,00:00:00,5H 59M 59S,every_day,0 +E4baseline,01:00:00,3H,every_day,0 diff --git a/rules/features.smk b/rules/features.smk index 1b6e0ad8..76b2e94c 100644 --- a/rules/features.smk +++ b/rules/features.smk @@ -899,6 +899,7 @@ rule empatica_blood_volume_pulse_r_features: rule empatica_inter_beat_interval_python_features: input: sensor_data = "data/raw/{pid}/empatica_inter_beat_interval_with_datetime.csv", + bvp_sensor_data = "data/raw/{pid}/empatica_blood_volume_pulse_with_datetime.csv", time_segments_labels = "data/interim/time_segments/{pid}_time_segments_labels.csv" params: provider = lambda wildcards: config["EMPATICA_INTER_BEAT_INTERVAL"]["PROVIDERS"][wildcards.provider_key.upper()], diff --git a/src/features/empatica_electrodermal_activity/cr/main.py b/src/features/empatica_electrodermal_activity/cr/main.py index dee4869f..9499deec 100644 --- a/src/features/empatica_electrodermal_activity/cr/main.py +++ b/src/features/empatica_electrodermal_activity/cr/main.py @@ -1,10 +1,17 @@ import pandas as pd +import numpy as np from scipy.stats import entropy from cr_features.helper_functions import convert_to2d, gsr_features from cr_features.calculate_features import calculate_features from cr_features_helper_methods import extract_second_order_features +import sys + +#pd.set_option('display.max_columns', None) +#pd.set_option('display.max_rows', None) +#np.seterr(invalid='ignore') + def extract_eda_features_from_intraday_data(eda_intraday_data, features, window_length, time_segment, filter_data_by_segment): eda_intraday_features = pd.DataFrame(columns=["local_segment"] + features) @@ -14,7 +21,7 @@ def extract_eda_features_from_intraday_data(eda_intraday_data, features, window_ eda_intraday_data = filter_data_by_segment(eda_intraday_data, time_segment) - if not eda_intraday_data.empty: + if not eda_intraday_data.empty: eda_intraday_features = pd.DataFrame() @@ -28,7 +35,6 @@ def extract_eda_features_from_intraday_data(eda_intraday_data, features, window_ eda_intraday_data.groupby('local_segment').apply(\ lambda x: calculate_features(convert_to2d(x['electrodermal_activity'], window_length*sample_rate), fs=sample_rate, feature_names=features)) - eda_intraday_features.reset_index(inplace=True) return eda_intraday_features @@ -58,6 +64,7 @@ def cr_features(sensor_data_files, time_segment, provider, filter_data_by_segmen if calc_windows: so_features_names = provider["WINDOWS"]["SECOND_ORDER_FEATURES"] eda_second_order_features = extract_second_order_features(eda_intraday_features, so_features_names) + return eda_intraday_features, eda_second_order_features return eda_intraday_features \ No newline at end of file diff --git a/src/features/empatica_inter_beat_interval/cr/main.py b/src/features/empatica_inter_beat_interval/cr/main.py index 4d8c4fd1..607e5638 100644 --- a/src/features/empatica_inter_beat_interval/cr/main.py +++ b/src/features/empatica_inter_beat_interval/cr/main.py @@ -2,14 +2,14 @@ import pandas as pd import numpy as np from cr_features.helper_functions import convert_ibi_to2d_time, hrv_features, hrv_freq_features -from cr_features.hrv import extract_hrv_features_2d_wrapper +from cr_features.hrv import extract_hrv_features_2d_wrapper, get_HRV_features from cr_features_helper_methods import extract_second_order_features import math import sys pd.set_option('display.max_rows', 1000) -pd.set_option('display.max_columns', None) +#pd.set_option('display.max_columns', None) def extract_ibi_features_from_intraday_data(ibi_intraday_data, features, window_length, time_segment, filter_data_by_segment): @@ -48,9 +48,21 @@ def extract_ibi_features_from_intraday_data(ibi_intraday_data, features, window_ return ibi_intraday_features +def patch_IBI_with_BVP(bvp_intraday_data): + # get features method is used because + hrv_time_and_freq_features, sample, rr, timings, peak_indx = \ + get_HRV_features(bvp_intraday_data['blood_volume_pulse'].to_numpy(), hampel_fiter=False, median_filter=False, mod_z_score_filter=True, sampling=64, feature_names=['meanHr']) + + def cr_features(sensor_data_files, time_segment, provider, filter_data_by_segment, *args, **kwargs): + print(sensor_data_files) ibi_intraday_data = pd.read_csv(sensor_data_files["sensor_data"]) + if provider["PATCH_WITH_BVP"]: + bvp_intraday_data = pd.read_csv(sensor_data_files["bvp_sensor_data"]) + patch_IBI_with_BVP(bvp_intraday_data) + # sys.exit() + requested_intraday_features = provider["FEATURES"] calc_windows = kwargs.get('calc_windows', False) diff --git a/src/features/entry.py b/src/features/entry.py index 5517cdcc..d8c7e843 100644 --- a/src/features/entry.py +++ b/src/features/entry.py @@ -1,6 +1,8 @@ import pandas as pd from utils.utils import fetch_provider_features, run_provider_cleaning_script +import sys + sensor_data_files = dict(snakemake.input) provider = snakemake.params["provider"] @@ -29,7 +31,6 @@ else: elif "empatica" in sensor_key: pd.DataFrame().to_csv(snakemake.output[1], index=False) - sensor_features = fetch_provider_features(provider, provider_key, sensor_key, sensor_data_files, time_segments_file, calc_windows=False) if not calc_windows: diff --git a/src/features/utils/utils.py b/src/features/utils/utils.py index 90608ab1..a7bf9fe3 100644 --- a/src/features/utils/utils.py +++ b/src/features/utils/utils.py @@ -169,3 +169,7 @@ def run_provider_cleaning_script(provider, provider_key, sensor_key, sensor_data sensor_features = cleaning_function(sensor_data_files, provider) return sensor_features + + +def empatica_patch_IBI_with_BVP(bvp_data): + pass