junos
/
rapids
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Last changes before processing on the server.

notes
Primoz 2022-10-03 12:53:31 +00:00
parent 44531c6d94
commit bbeabeee6f
2 changed files with 65 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
src/features/all_cleaning_individual/straw/main.py
View File

@ -73,9 +73,12 @@ def straw_cleaning(sensor_data_files, provider):
"timelastmessages" in col] "timelastmessages" in col]
features[impute_w_hn] = impute(features[impute_w_hn], method="high_number") features[impute_w_hn] = impute(features[impute_w_hn], method="high_number")
# Impute special case (mostcommonactivity) # Impute special case (mostcommonactivity) and (homelabel)
impute_w_sn = [col for col in features.columns if "mostcommonactivity" in col] impute_w_sn = [col for col in features.columns if "mostcommonactivity" in col]
features[impute_w_sn] = features[impute_w_sn].fillna(4) # Special case of imputation features[impute_w_sn] = features[impute_w_sn].fillna(4) # Special case of imputation - nominal/ordinal value
impute_w_sn2 = [col for col in features.columns if "homelabel" in col]
features[impute_w_sn2] = features[impute_w_sn2].fillna(1) # Special case of imputation - nominal/ordinal value
# Impute selected phone features with 0 # Impute selected phone features with 0
impute_zero = [col for col in features if \ impute_zero = [col for col in features if \
@ -87,20 +90,16 @@ def straw_cleaning(sensor_data_files, provider):
col.startswith('phone_messages_rapids_') or col.startswith('phone_messages_rapids_') or
col.startswith('phone_screen_rapids_') or col.startswith('phone_screen_rapids_') or
col.startswith('phone_wifi_visible')] col.startswith('phone_wifi_visible')]
features[impute_locations] = impute(features[impute_locations], method="zero") features[impute_zero] = impute(features[impute_zero], method="zero")
## (5) STANDARDIZATION ## (5) STANDARDIZATION
if provider["STANDARDIZATION"]: if provider["STANDARDIZATION"]:
features.loc[:, ~features.columns.isin(excluded_columns)] = StandardScaler().fit_transform(features.loc[:, ~features.columns.isin(excluded_columns)]) features.loc[:, ~features.columns.isin(excluded_columns)] = StandardScaler().fit_transform(features.loc[:, ~features.columns.isin(excluded_columns)])
graph_bf_af(features[impute_locations], "knn_before")
# (6) IMPUTATION: IMPUTE DATA WITH KNN METHOD # (6) IMPUTATION: IMPUTE DATA WITH KNN METHOD
impute_cols = [col for col in features.columns if col not in excluded_columns] impute_cols = [col for col in features.columns if col not in excluded_columns]
features[impute_cols] = impute(features[impute_cols], method="knn") features[impute_cols] = impute(features[impute_cols], method="knn")
graph_bf_af(features[impute_locations], "knn_after")
# (7) REMOVE COLS WHERE VARIANCE IS 0 # (7) REMOVE COLS WHERE VARIANCE IS 0
esm_cols = features.loc[:, features.columns.str.startswith('phone_esm_straw')] esm_cols = features.loc[:, features.columns.str.startswith('phone_esm_straw')]
@ -131,8 +130,6 @@ def straw_cleaning(sensor_data_files, provider):
if features.isna().any().any(): if features.isna().any().any():
raise ValueError raise ValueError
sys.exit()
return features return features
def impute(df, method='zero'): def impute(df, method='zero'):
@ -149,9 +146,13 @@ def impute(df, method='zero'):
'knn': k_nearest(df) 'knn': k_nearest(df)
}[method] }[method]
def graph_bf_af(features, phase_name): def graph_bf_af(features, phase_name, plt_flag=False):
sns.set(rc={"figure.figsize":(16, 8)}) if plt_flag:
print(features) sns.set(rc={"figure.figsize":(16, 8)})
sns.heatmap(features.isna(), cbar=False) #features.select_dtypes(include=np.number) sns.heatmap(features.isna(), cbar=False) #features.select_dtypes(include=np.number)
plt.savefig(f'features_individual_nans_{phase_name}.png', bbox_inches='tight') plt.savefig(f'features_overall_nans_{phase_name}.png', bbox_inches='tight')
print(f"\n-------------{phase_name}-------------")
print("Rows number:", features.shape[0])
print("Columns number:", len(features.columns))
print("---------------------------------------------\n")

63
src/features/all_cleaning_overall/straw/main.py
View File

@ -11,8 +11,6 @@ import seaborn as sns
sys.path.append('/rapids/') sys.path.append('/rapids/')
from src.features import empatica_data_yield as edy from src.features import empatica_data_yield as edy
pd.set_option('display.max_columns', 20)
def straw_cleaning(sensor_data_files, provider): def straw_cleaning(sensor_data_files, provider):
features = pd.read_csv(sensor_data_files["sensor_data"][0]) features = pd.read_csv(sensor_data_files["sensor_data"][0])
@ -24,11 +22,15 @@ def straw_cleaning(sensor_data_files, provider):
excluded_columns = ['local_segment', 'local_segment_label', 'local_segment_start_datetime', 'local_segment_end_datetime'] excluded_columns = ['local_segment', 'local_segment_label', 'local_segment_start_datetime', 'local_segment_end_datetime']
graph_bf_af(features, "1target_rows_before")
# (1) FILTER_OUT THE ROWS THAT DO NOT HAVE THE TARGET COLUMN AVAILABLE # (1) FILTER_OUT THE ROWS THAT DO NOT HAVE THE TARGET COLUMN AVAILABLE
if config['PARAMS_FOR_ANALYSIS']['TARGET']['COMPUTE']: if config['PARAMS_FOR_ANALYSIS']['TARGET']['COMPUTE']:
target = config['PARAMS_FOR_ANALYSIS']['TARGET']['LABEL'] # get target label from config target = config['PARAMS_FOR_ANALYSIS']['TARGET']['LABEL'] # get target label from config
features = features[features['phone_esm_straw_' + target].notna()].reset_index(drop=True) features = features[features['phone_esm_straw_' + target].notna()].reset_index(drop=True)
graph_bf_af(features, "2target_rows_after")
# (2.1) QUALITY CHECK (DATA YIELD COLUMN) deletes the rows where E4 or phone data is low quality # (2.1) QUALITY CHECK (DATA YIELD COLUMN) deletes the rows where E4 or phone data is low quality
phone_data_yield_unit = provider["PHONE_DATA_YIELD_FEATURE"].split("_")[3].lower() phone_data_yield_unit = provider["PHONE_DATA_YIELD_FEATURE"].split("_")[3].lower()
phone_data_yield_column = "phone_data_yield_rapids_ratiovalidyielded" + phone_data_yield_unit phone_data_yield_column = "phone_data_yield_rapids_ratiovalidyielded" + phone_data_yield_unit
@ -38,23 +40,41 @@ def straw_cleaning(sensor_data_files, provider):
if not phone_data_yield_column in features.columns and not "empatica_data_yield" in features.columns: if not phone_data_yield_column in features.columns and not "empatica_data_yield" in features.columns:
raise KeyError(f"RAPIDS provider needs to clean the selected event features based on {phone_data_yield_column} and empatica_data_yield columns. For phone data yield, please set config[PHONE_DATA_YIELD][PROVIDERS][RAPIDS][COMPUTE] to True and include 'ratiovalidyielded{data_yield_unit}' in [FEATURES].") raise KeyError(f"RAPIDS provider needs to clean the selected event features based on {phone_data_yield_column} and empatica_data_yield columns. For phone data yield, please set config[PHONE_DATA_YIELD][PROVIDERS][RAPIDS][COMPUTE] to True and include 'ratiovalidyielded{data_yield_unit}' in [FEATURES].")
hist = features[["empatica_data_yield", phone_data_yield_column]].hist()
plt.legend()
plt.savefig(f'phone_E4_histogram.png', bbox_inches='tight')
# Drop rows where phone data yield is less then given threshold # Drop rows where phone data yield is less then given threshold
if provider["PHONE_DATA_YIELD_RATIO_THRESHOLD"]: if provider["PHONE_DATA_YIELD_RATIO_THRESHOLD"]:
print("\nThreshold:", provider["PHONE_DATA_YIELD_RATIO_THRESHOLD"])
print("Phone features data yield stats:", features[phone_data_yield_column].describe(), "\n")
print(features[phone_data_yield_column].sort_values())
hist = features[phone_data_yield_column].hist(bins=5)
features = features[features[phone_data_yield_column] >= provider["PHONE_DATA_YIELD_RATIO_THRESHOLD"]].reset_index(drop=True) features = features[features[phone_data_yield_column] >= provider["PHONE_DATA_YIELD_RATIO_THRESHOLD"]].reset_index(drop=True)
# Drop rows where empatica data yield is less then given threshold # Drop rows where empatica data yield is less then given threshold
if provider["EMPATICA_DATA_YIELD_RATIO_THRESHOLD"]: if provider["EMPATICA_DATA_YIELD_RATIO_THRESHOLD"]:
print("\nThreshold:", provider["EMPATICA_DATA_YIELD_RATIO_THRESHOLD"])
print("E4 features data yield stats:", features["empatica_data_yield"].describe(), "\n")
print(features["empatica_data_yield"].sort_values())
features = features[features["empatica_data_yield"] >= provider["EMPATICA_DATA_YIELD_RATIO_THRESHOLD"]].reset_index(drop=True) features = features[features["empatica_data_yield"] >= provider["EMPATICA_DATA_YIELD_RATIO_THRESHOLD"]].reset_index(drop=True)
sys.exit()
graph_bf_af(features, "3data_yield_drop_rows")
# (2.2) DO THE ROWS CONSIST OF ENOUGH NON-NAN VALUES? # (2.2) DO THE ROWS CONSIST OF ENOUGH NON-NAN VALUES?
min_count = math.ceil((1 - provider["ROWS_NAN_THRESHOLD"]) * features.shape[1]) # minimal not nan values in row min_count = math.ceil((1 - provider["ROWS_NAN_THRESHOLD"]) * features.shape[1]) # minimal not nan values in row
features.dropna(axis=0, thresh=min_count, inplace=True) # Thresh => at least this many not-nans features.dropna(axis=0, thresh=min_count, inplace=True) # Thresh => at least this many not-nans
graph_bf_af(features, "4too_much_nans_rows")
# (3) REMOVE COLS IF THEIR NAN THRESHOLD IS PASSED (should be <= if even all NaN columns must be preserved - this solution now drops columns with all NaN rows) # (3) REMOVE COLS IF THEIR NAN THRESHOLD IS PASSED (should be <= if even all NaN columns must be preserved - this solution now drops columns with all NaN rows)
esm_cols = features.loc[:, features.columns.str.startswith('phone_esm_straw')] # Get target (esm) columns esm_cols = features.loc[:, features.columns.str.startswith('phone_esm_straw')] # Get target (esm) columns
features = features.loc[:, features.isna().sum() < provider["COLS_NAN_THRESHOLD"] * features.shape[0]] features = features.loc[:, features.isna().sum() < provider["COLS_NAN_THRESHOLD"] * features.shape[0]]
graph_bf_af(features, "5too_much_nans_cols")
# Preserve esm cols if deleted (has to come after drop cols operations) # Preserve esm cols if deleted (has to come after drop cols operations)
for esm in esm_cols: for esm in esm_cols:
if esm not in features: if esm not in features:
@ -73,9 +93,14 @@ def straw_cleaning(sensor_data_files, provider):
"timelastmessages" in col] "timelastmessages" in col]
features[impute_w_hn] = impute(features[impute_w_hn], method="high_number") features[impute_w_hn] = impute(features[impute_w_hn], method="high_number")
# Impute special case (mostcommonactivity) graph_bf_af(features, "6high_number_imp")
# Impute special case (mostcommonactivity) and (homelabel)
impute_w_sn = [col for col in features.columns if "mostcommonactivity" in col] impute_w_sn = [col for col in features.columns if "mostcommonactivity" in col]
features[impute_w_sn] = features[impute_w_sn].fillna(4) # Special case of imputation features[impute_w_sn] = features[impute_w_sn].fillna(4) # Special case of imputation - nominal/ordinal value
impute_w_sn2 = [col for col in features.columns if "homelabel" in col]
features[impute_w_sn2] = features[impute_w_sn2].fillna(1) # Special case of imputation - nominal/ordinal value
# Impute selected phone features with 0 # Impute selected phone features with 0
impute_zero = [col for col in features if \ impute_zero = [col for col in features if \
@ -87,7 +112,9 @@ def straw_cleaning(sensor_data_files, provider):
col.startswith('phone_messages_rapids_') or col.startswith('phone_messages_rapids_') or
col.startswith('phone_screen_rapids_') or col.startswith('phone_screen_rapids_') or
col.startswith('phone_wifi_visible')] col.startswith('phone_wifi_visible')]
features[impute_locations] = impute(features[impute_locations], method="zero") features[impute_zero] = impute(features[impute_zero], method="zero")
graph_bf_af(features, "7zero_imp")
# Impute phone locations with median - should this rather be imputed at kNN step?? # Impute phone locations with median - should this rather be imputed at kNN step??
# impute_locations = [col for col in features.columns if "phone_locations_" in col] # impute_locations = [col for col in features.columns if "phone_locations_" in col]
@ -103,18 +130,19 @@ def straw_cleaning(sensor_data_files, provider):
# features[impute_locations] = features[impute_locations + ["pid"]].groupby("pid").transform(lambda x: x.fillna(x.median()))[impute_locations] # features[impute_locations] = features[impute_locations + ["pid"]].groupby("pid").transform(lambda x: x.fillna(x.median()))[impute_locations]
# (5) STANDARDIZATION # (5) STANDARDIZATION
if provider["STANDARDIZATION"]: if provider["STANDARDIZATION"]:
features.loc[:, ~features.columns.isin(excluded_columns + ["pid"])] = \ features.loc[:, ~features.columns.isin(excluded_columns + ["pid"])] = \
features.loc[:, ~features.columns.isin(excluded_columns)].groupby('pid').transform(lambda x: 0 if (x.std() == 0) else (x - x.mean()) / x.std()) features.loc[:, ~features.columns.isin(excluded_columns)].groupby('pid').transform(lambda x: 0 if (x.std() == 0) else (x - x.mean()) / x.std())
graph_bf_af(features[impute_locations], "knn_before") graph_bf_af(features, "8standardization")
# (6) IMPUTATION: IMPUTE DATA WITH KNN METHOD # (6) IMPUTATION: IMPUTE DATA WITH KNN METHOD
impute_cols = [col for col in features.columns if col not in excluded_columns and col != "pid"] impute_cols = [col for col in features.columns if col not in excluded_columns and col != "pid"]
features[impute_cols] = impute(features[impute_cols], method="knn") features[impute_cols] = impute(features[impute_cols], method="knn")
graph_bf_af(features[impute_locations], "knn_after") graph_bf_af(features, "9knn_after")
# (7) REMOVE COLS WHERE VARIANCE IS 0 # (7) REMOVE COLS WHERE VARIANCE IS 0
esm_cols = features.loc[:, features.columns.str.startswith('phone_esm_straw')] esm_cols = features.loc[:, features.columns.str.startswith('phone_esm_straw')]
@ -122,6 +150,8 @@ def straw_cleaning(sensor_data_files, provider):
if provider["COLS_VAR_THRESHOLD"]: if provider["COLS_VAR_THRESHOLD"]:
features.drop(features.std()[features.std() == 0].index.values, axis=1, inplace=True) features.drop(features.std()[features.std() == 0].index.values, axis=1, inplace=True)
graph_bf_af(features, "10variance_drop")
# (8) DROP HIGHLY CORRELATED FEATURES # (8) DROP HIGHLY CORRELATED FEATURES
drop_corr_features = provider["DROP_HIGHLY_CORRELATED_FEATURES"] drop_corr_features = provider["DROP_HIGHLY_CORRELATED_FEATURES"]
if drop_corr_features["COMPUTE"] and features.shape[0] > 5: # If small amount of segments (rows) is present, do not execute correlation check if drop_corr_features["COMPUTE"] and features.shape[0] > 5: # If small amount of segments (rows) is present, do not execute correlation check
@ -142,12 +172,12 @@ def straw_cleaning(sensor_data_files, provider):
if esm not in features: if esm not in features:
features[esm] = esm_cols[esm] features[esm] = esm_cols[esm]
graph_bf_af(features, "11correlation_drop")
# (9) VERIFY IF THERE ARE ANY NANS LEFT IN THE DATAFRAME # (9) VERIFY IF THERE ARE ANY NANS LEFT IN THE DATAFRAME
if features.isna().any().any(): if features.isna().any().any():
raise ValueError raise ValueError
sys.exit()
return features return features
def impute(df, method='zero'): def impute(df, method='zero'):
@ -164,9 +194,14 @@ def impute(df, method='zero'):
'knn': k_nearest(df) 'knn': k_nearest(df)
}[method] }[method]
def graph_bf_af(features, phase_name): def graph_bf_af(features, phase_name, plt_flag=False):
sns.set(rc={"figure.figsize":(16, 8)}) if plt_flag:
print(features) sns.set(rc={"figure.figsize":(16, 8)})
sns.heatmap(features.isna(), cbar=False) #features.select_dtypes(include=np.number) sns.heatmap(features.isna(), cbar=False) #features.select_dtypes(include=np.number)
plt.savefig(f'features_overall_nans_{phase_name}.png', bbox_inches='tight') plt.savefig(f'features_overall_nans_{phase_name}.png', bbox_inches='tight')
print(f"\n-------------{phase_name}-------------")
print("Rows number:", features.shape[0])
print("Columns number:", len(features.columns))
print("---------------------------------------------\n")