junos
/
rapids
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Changes needed for testing and starting of the Event-Related Segments.

notes
Primoz 2022-10-12 15:51:23 +00:00
parent 0f21273508
commit 9baff159cd
3 changed files with 39 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/features/all_cleaning_individual/straw/main.py
View File

@ -100,7 +100,7 @@ def straw_cleaning(sensor_data_files, provider):
col.startswith('phone_screen_rapids_') or col.startswith('phone_screen_rapids_') or
col.startswith('phone_wifi_visible')] col.startswith('phone_wifi_visible')]
features[impute_zero] = features[impute_zero].fillna(0) features[impute_zero+list(esm_cols.columns)] = features[impute_zero+list(esm_cols.columns)].fillna(0)
## (5) STANDARDIZATION ## (5) STANDARDIZATION
if provider["STANDARDIZATION"]: if provider["STANDARDIZATION"]:

46
src/features/all_cleaning_overall/straw/main.py
View File

@ -14,6 +14,9 @@ def straw_cleaning(sensor_data_files, provider, target):
features = pd.read_csv(sensor_data_files["sensor_data"][0]) features = pd.read_csv(sensor_data_files["sensor_data"][0])
# print(features)
# sys.exit()
esm_cols = features.loc[:, features.columns.str.startswith('phone_esm_straw')] # Get target (esm) columns esm_cols = features.loc[:, features.columns.str.startswith('phone_esm_straw')] # Get target (esm) columns
with open('config.yaml', 'r') as stream: with open('config.yaml', 'r') as stream:
@ -26,8 +29,12 @@ def straw_cleaning(sensor_data_files, provider, target):
# (1) FILTER_OUT THE ROWS THAT DO NOT HAVE THE TARGET COLUMN AVAILABLE # (1) FILTER_OUT THE ROWS THAT DO NOT HAVE THE TARGET COLUMN AVAILABLE
if config['PARAMS_FOR_ANALYSIS']['TARGET']['COMPUTE']: if config['PARAMS_FOR_ANALYSIS']['TARGET']['COMPUTE']:
features = features[features['phone_esm_straw_' + target].notna()].reset_index(drop=True) features = features[features['phone_esm_straw_' + target].notna()].reset_index(drop=True)
if features.empty:
return pd.DataFrame(columns=excluded_columns)
graph_bf_af(features, "2target_rows_after") graph_bf_af(features, "2target_rows_after")
print("HERE1", target, features["pid"])
# (2) QUALITY CHECK (DATA YIELD COLUMN) drops the rows where E4 or phone data is low quality # (2) QUALITY CHECK (DATA YIELD COLUMN) drops the rows where E4 or phone data is low quality
phone_data_yield_unit = provider["PHONE_DATA_YIELD_FEATURE"].split("_")[3].lower() phone_data_yield_unit = provider["PHONE_DATA_YIELD_FEATURE"].split("_")[3].lower()
@ -39,27 +46,30 @@ def straw_cleaning(sensor_data_files, provider, target):
raise KeyError(f"RAPIDS provider needs to clean the selected event features based on {phone_data_yield_column} and empatica_data_yield columns. For phone data yield, please set config[PHONE_DATA_YIELD][PROVIDERS][RAPIDS][COMPUTE] to True and include 'ratiovalidyielded{data_yield_unit}' in [FEATURES].") raise KeyError(f"RAPIDS provider needs to clean the selected event features based on {phone_data_yield_column} and empatica_data_yield columns. For phone data yield, please set config[PHONE_DATA_YIELD][PROVIDERS][RAPIDS][COMPUTE] to True and include 'ratiovalidyielded{data_yield_unit}' in [FEATURES].")
hist = features[["empatica_data_yield", phone_data_yield_column]].hist() hist = features[["empatica_data_yield", phone_data_yield_column]].hist()
plt.legend()
plt.savefig(f'phone_E4_histogram.png', bbox_inches='tight') plt.savefig(f'phone_E4_histogram.png', bbox_inches='tight')
# Drop rows where phone data yield is less then given threshold # Drop rows where phone data yield is less then given threshold
if provider["PHONE_DATA_YIELD_RATIO_THRESHOLD"]: if provider["PHONE_DATA_YIELD_RATIO_THRESHOLD"]:
print("\nThreshold:", provider["PHONE_DATA_YIELD_RATIO_THRESHOLD"]) # print("\nThreshold:", provider["PHONE_DATA_YIELD_RATIO_THRESHOLD"])
print("Phone features data yield stats:", features[phone_data_yield_column].describe(), "\n") # print("Phone features data yield stats:", features[phone_data_yield_column].describe(), "\n")
# print(features[phone_data_yield_column].sort_values()) # print(features[phone_data_yield_column].sort_values())
hist = features[phone_data_yield_column].hist(bins=5) hist = features[phone_data_yield_column].hist(bins=5)
plt.close() plt.close()
features = features[features[phone_data_yield_column] >= provider["PHONE_DATA_YIELD_RATIO_THRESHOLD"]].reset_index(drop=True) features = features[features[phone_data_yield_column] >= provider["PHONE_DATA_YIELD_RATIO_THRESHOLD"]].reset_index(drop=True)
# Drop rows where empatica data yield is less then given threshold # Drop rows where empatica data yield is less then given threshold
if provider["EMPATICA_DATA_YIELD_RATIO_THRESHOLD"]: if provider["EMPATICA_DATA_YIELD_RATIO_THRESHOLD"]:
print("\nThreshold:", provider["EMPATICA_DATA_YIELD_RATIO_THRESHOLD"]) # print("\nThreshold:", provider["EMPATICA_DATA_YIELD_RATIO_THRESHOLD"])
print("E4 features data yield stats:", features["empatica_data_yield"].describe(), "\n") # print("E4 features data yield stats:", features["empatica_data_yield"].describe(), "\n")
# print(features["empatica_data_yield"].sort_values()) # print(features["empatica_data_yield"].sort_values())
features = features[features["empatica_data_yield"] >= provider["EMPATICA_DATA_YIELD_RATIO_THRESHOLD"]].reset_index(drop=True) features = features[features["empatica_data_yield"] >= provider["EMPATICA_DATA_YIELD_RATIO_THRESHOLD"]].reset_index(drop=True)
graph_bf_af(features, "3data_yield_drop_rows") graph_bf_af(features, "3data_yield_drop_rows")
if features.empty:
return pd.DataFrame(columns=excluded_columns)
# (3) CONTEXTUAL IMPUTATION # (3) CONTEXTUAL IMPUTATION
# Impute selected phone features with a high number # Impute selected phone features with a high number
@ -83,7 +93,7 @@ def straw_cleaning(sensor_data_files, provider, target):
impute_w_sn3 = [col for col in features.columns if "loglocationvariance" in col] impute_w_sn3 = [col for col in features.columns if "loglocationvariance" in col]
features[impute_w_sn2] = features[impute_w_sn2].fillna(-1000000) # Special case of imputation - loglocation features[impute_w_sn2] = features[impute_w_sn2].fillna(-1000000) # Special case of imputation - loglocation
# Impute selected phone features with 0 # Impute selected phone features with 0 + impute ESM features with 0
impute_zero = [col for col in features if \ impute_zero = [col for col in features if \
col.startswith('phone_applications_foreground_rapids_') or col.startswith('phone_applications_foreground_rapids_') or
col.startswith('phone_battery_rapids_') or col.startswith('phone_battery_rapids_') or
@ -94,23 +104,22 @@ def straw_cleaning(sensor_data_files, provider, target):
col.startswith('phone_screen_rapids_') or col.startswith('phone_screen_rapids_') or
col.startswith('phone_wifi_visible')] col.startswith('phone_wifi_visible')]
features[impute_zero] = features[impute_zero].fillna(0) features[impute_zero+list(esm_cols.columns)] = features[impute_zero+list(esm_cols.columns)].fillna(0)
graph_bf_af(features, "5zero_imp") graph_bf_af(features, "4context_imp")
# (4) REMOVE COLS IF THEIR NAN THRESHOLD IS PASSED (should be <= if even all NaN columns must be preserved - this solution now drops columns with all NaN rows) # (4) REMOVE COLS IF THEIR NAN THRESHOLD IS PASSED (should be <= if even all NaN columns must be preserved - this solution now drops columns with all NaN rows)
esm_cols = features.loc[:, features.columns.str.startswith('phone_esm_straw')] # Get target (esm) columns esm_cols = features.loc[:, features.columns.str.startswith('phone_esm_straw')] # Get target (esm) columns
features = features.loc[:, features.isna().sum() < provider["COLS_NAN_THRESHOLD"] * features.shape[0]] features = features.loc[:, features.isna().sum() < provider["COLS_NAN_THRESHOLD"] * features.shape[0]]
graph_bf_af(features, "6too_much_nans_cols") graph_bf_af(features, "5too_much_nans_cols")
# (5) REMOVE COLS WHERE VARIANCE IS 0 # (5) REMOVE COLS WHERE VARIANCE IS 0
if provider["COLS_VAR_THRESHOLD"]: if provider["COLS_VAR_THRESHOLD"]:
features.drop(features.std()[features.std() == 0].index.values, axis=1, inplace=True) features.drop(features.std()[features.std() == 0].index.values, axis=1, inplace=True)
graph_bf_af(features, "7variance_drop") graph_bf_af(features, "6variance_drop")
# Preserve esm cols if deleted (has to come after drop cols operations) # Preserve esm cols if deleted (has to come after drop cols operations)
for esm in esm_cols: for esm in esm_cols:
@ -121,9 +130,13 @@ def straw_cleaning(sensor_data_files, provider, target):
min_count = math.ceil((1 - provider["ROWS_NAN_THRESHOLD"]) * features.shape[1]) # minimal not nan values in row min_count = math.ceil((1 - provider["ROWS_NAN_THRESHOLD"]) * features.shape[1]) # minimal not nan values in row
features.dropna(axis=0, thresh=min_count, inplace=True) # Thresh => at least this many not-nans features.dropna(axis=0, thresh=min_count, inplace=True) # Thresh => at least this many not-nans
graph_bf_af(features, "8too_much_nans_rows") graph_bf_af(features, "7too_much_nans_rows")
# (7) STANDARDIZATION if features.empty:
return pd.DataFrame(columns=excluded_columns)
# (7) STANDARDIZATION TODO: exclude nominal features from standardization
if provider["STANDARDIZATION"]: if provider["STANDARDIZATION"]:
# Expected warning within this code block # Expected warning within this code block
@ -132,14 +145,15 @@ def straw_cleaning(sensor_data_files, provider, target):
features.loc[:, ~features.columns.isin(excluded_columns + ["pid"])] = \ features.loc[:, ~features.columns.isin(excluded_columns + ["pid"])] = \
features.loc[:, ~features.columns.isin(excluded_columns)].groupby('pid').transform(lambda x: StandardScaler().fit_transform(x.values[:,np.newaxis]).ravel()) features.loc[:, ~features.columns.isin(excluded_columns)].groupby('pid').transform(lambda x: StandardScaler().fit_transform(x.values[:,np.newaxis]).ravel())
graph_bf_af(features, "9standardization") graph_bf_af(features, "8standardization")
# (8) IMPUTATION: IMPUTE DATA WITH KNN METHOD # (8) IMPUTATION: IMPUTE DATA WITH KNN METHOD
features.reset_index(drop=True, inplace=True) features.reset_index(drop=True, inplace=True)
impute_cols = [col for col in features.columns if col not in excluded_columns and col != "pid"] impute_cols = [col for col in features.columns if col not in excluded_columns and col != "pid"]
features[impute_cols] = impute(features[impute_cols], method="knn") features[impute_cols] = impute(features[impute_cols], method="knn")
graph_bf_af(features, "10knn_after") graph_bf_af(features, "9knn_after")
# (9) DROP HIGHLY CORRELATED FEATURES # (9) DROP HIGHLY CORRELATED FEATURES

12
src/models/merge_features_and_targets_for_population_model.py
View File

@ -12,9 +12,13 @@ for baseline_features_path in snakemake.input["demographic_features"]:
all_baseline_features = pd.concat([all_baseline_features, baseline_features], axis=0) all_baseline_features = pd.concat([all_baseline_features, baseline_features], axis=0)
# merge sensor features and baseline features # merge sensor features and baseline features
features = sensor_features.merge(all_baseline_features, on="pid", how="left") if not sensor_features.empty:
features = sensor_features.merge(all_baseline_features, on="pid", how="left")
target_variable_name = snakemake.params["target_variable"] target_variable_name = snakemake.params["target_variable"]
model_input = retain_target_column(features, target_variable_name) model_input = retain_target_column(features, target_variable_name)
model_input.to_csv(snakemake.output[0], index=False) model_input.to_csv(snakemake.output[0], index=False)
else:
sensor_features.to_csv(snakemake.output[0], index=False)