From 183758cd37681eafee174c602f056ba08c227581 Mon Sep 17 00:00:00 2001 From: Primoz Date: Tue, 22 Nov 2022 14:31:49 +0100 Subject: [PATCH] Improve general ml classification pipeline script. --- exploration/ml_pipeline_classification.py | 104 ++++++++++++++-------- 1 file changed, 67 insertions(+), 37 deletions(-) diff --git a/exploration/ml_pipeline_classification.py b/exploration/ml_pipeline_classification.py index 140464b..fc2fb81 100644 --- a/exploration/ml_pipeline_classification.py +++ b/exploration/ml_pipeline_classification.py @@ -52,25 +52,20 @@ import machine_learning.model model_input = pd.read_csv("../data/intradaily_30_min_all_targets/input_JCQ_job_demand_mean.csv") # %% jupyter={"source_hidden": true} -bins = [-4, -1, 1, 4] # bins for z-scored targets -model_input['target'], edges = pd.cut(model_input.target, bins=bins, labels=['low', 'medium', 'high'], retbins=True, right=False) +index_columns = ["local_segment", "local_segment_label", "local_segment_start_datetime", "local_segment_end_datetime"] +model_input.set_index(index_columns, inplace=True) + +# %% jupyter={"source_hidden": true} +bins = [-10, -1, 1, 10] # bins for z-scored targets +model_input['target'], edges = pd.cut(model_input.target, bins=bins, labels=['low', 'medium', 'high'], retbins=True, right=False) #['low', 'medium', 'high'] model_input['target'].value_counts(), edges model_input = model_input[model_input['target'] != "medium"] model_input['target'] = model_input['target'].astype(str).apply(lambda x: 0 if x == "low" else 1) model_input['target'].value_counts() -# %% jupyter={"source_hidden": true} -index_columns = ["local_segment", "local_segment_label", "local_segment_start_datetime", "local_segment_end_datetime"] -#if "pid" in model_input.columns: -# index_columns.append("pid") -model_input.set_index(index_columns, inplace=True) - -# %% jupyter={"source_hidden": true} -cv_method = '5kfold' -if cv_method == 'logo': - data_x, data_y, data_groups = model_input.drop(["target", "pid"], axis=1), model_input["target"], model_input["pid"] -else: +cv_method_str = 'logo' # logo, halflogo, 5kfold +if cv_method_str == 'halflogo': model_input['pid_index'] = model_input.groupby('pid').cumcount() model_input['pid_count'] = model_input.groupby('pid')['pid'].transform('count') @@ -78,6 +73,9 @@ else: model_input["pid_half"] = model_input["pid"] + "_" + model_input["pid_index"].astype(int).astype(str) data_x, data_y, data_groups = model_input.drop(["target", "pid", "pid_index", "pid_half"], axis=1), model_input["target"], model_input["pid_half"] +else: + data_x, data_y, data_groups = model_input.drop(["target", "pid"], axis=1), model_input["target"], model_input["pid"] + # %% jupyter={"source_hidden": true} categorical_feature_colnames = ["gender", "startlanguage"] @@ -100,19 +98,21 @@ train_x = pd.concat([numerical_features, categorical_features], axis=1) train_x.dtypes # %% jupyter={"source_hidden": true} -logo = LeaveOneGroupOut() -logo.get_n_splits( - train_x, - data_y, - groups=data_groups, -) - -# Defaults to 5 k-folds in cross_validate method -if cv_method != 'logo' and cv_method != 'half_logo': - logo = None +cv_method = None # Defaults to 5 k-folds in cross_validate method +if cv_method_str == 'logo' or cv_method_str == 'half_logo': + cv_method = LeaveOneGroupOut() + cv_method.get_n_splits( + train_x, + data_y, + groups=data_groups, + ) +# %% jupyter={"source_hidden": true} +# %% [markdown] +# ### Set n for nlargest and nsmallest +n = 5 # %% jupyter={"source_hidden": true} -imputer = SimpleImputer(missing_values=np.nan, strategy='mean') +imputer = SimpleImputer(missing_values=np.nan, strategy='median') # %% [markdown] # ### Baseline: Dummy Classifier (most frequent) @@ -124,8 +124,9 @@ dummy_classifier = cross_validate( X=imputer.fit_transform(train_x), y=data_y, groups=data_groups, - cv=logo, + cv=cv_method, n_jobs=-1, + error_score='raise', scoring=('accuracy', 'average_precision', 'recall', 'f1') ) # %% jupyter={"source_hidden": true} @@ -133,6 +134,8 @@ print("Acc", np.median(dummy_classifier['test_accuracy'])) print("Precision", np.median(dummy_classifier['test_average_precision'])) print("Recall", np.median(dummy_classifier['test_recall'])) print("F1", np.median(dummy_classifier['test_f1'])) +print("Largest 5 ACC:", np.sort(-np.partition(-dummy_classifier['test_accuracy'], n)[:n])[::-1]) +print("Smallest 5 ACC:", np.sort(np.partition(dummy_classifier['test_accuracy'], n)[:n])) # %% [markdown] # ### Logistic Regression @@ -146,7 +149,7 @@ log_reg_scores = cross_validate( X=imputer.fit_transform(train_x), y=data_y, groups=data_groups, - cv=logo, + cv=cv_method, n_jobs=-1, scoring=('accuracy', 'precision', 'recall', 'f1') ) @@ -155,6 +158,8 @@ print("Acc", np.median(log_reg_scores['test_accuracy'])) print("Precision", np.median(log_reg_scores['test_precision'])) print("Recall", np.median(log_reg_scores['test_recall'])) print("F1", np.median(log_reg_scores['test_f1'])) +print("Largest 5 ACC:", np.sort(-np.partition(-log_reg_scores['test_accuracy'], n)[:n])[::-1]) +print("Smallest 5 ACC:", np.sort(np.partition(log_reg_scores['test_accuracy'], n)[:n])) # %% [markdown] # ### Support Vector Machine @@ -168,7 +173,7 @@ svc_scores = cross_validate( X=imputer.fit_transform(train_x), y=data_y, groups=data_groups, - cv=logo, + cv=cv_method, n_jobs=-1, scoring=('accuracy', 'precision', 'recall', 'f1') ) @@ -177,6 +182,8 @@ print("Acc", np.median(svc_scores['test_accuracy'])) print("Precision", np.median(svc_scores['test_precision'])) print("Recall", np.median(svc_scores['test_recall'])) print("F1", np.median(svc_scores['test_f1'])) +print("Largest 5 ACC:", np.sort(-np.partition(-svc_scores['test_accuracy'], n)[:n])[::-1]) +print("Smallest 5 ACC:", np.sort(np.partition(svc_scores['test_accuracy'], n)[:n])) # %% [markdown] # ### Gaussian Naive Bayes @@ -190,8 +197,9 @@ gaussian_nb_scores = cross_validate( X=imputer.fit_transform(train_x), y=data_y, groups=data_groups, - cv=logo, + cv=cv_method, n_jobs=-1, + error_score='raise', scoring=('accuracy', 'precision', 'recall', 'f1') ) # %% jupyter={"source_hidden": true} @@ -199,6 +207,8 @@ print("Acc", np.median(gaussian_nb_scores['test_accuracy'])) print("Precision", np.median(gaussian_nb_scores['test_precision'])) print("Recall", np.median(gaussian_nb_scores['test_recall'])) print("F1", np.median(gaussian_nb_scores['test_f1'])) +print("Largest 5 ACC:", np.sort(-np.partition(-gaussian_nb_scores['test_accuracy'], n)[:n])[::-1]) +print("Smallest 5 ACC:", np.sort(np.partition(gaussian_nb_scores['test_accuracy'], n)[:n])) # %% [markdown] # ### Stochastic Gradient Descent Classifier @@ -212,8 +222,9 @@ sgdc_scores = cross_validate( X=imputer.fit_transform(train_x), y=data_y, groups=data_groups, - cv=logo, + cv=cv_method, n_jobs=-1, + error_score='raise', scoring=('accuracy', 'precision', 'recall', 'f1') ) # %% jupyter={"source_hidden": true} @@ -221,6 +232,8 @@ print("Acc", np.median(sgdc_scores['test_accuracy'])) print("Precision", np.median(sgdc_scores['test_precision'])) print("Recall", np.median(sgdc_scores['test_recall'])) print("F1", np.median(sgdc_scores['test_f1'])) +print("Largest 5 ACC:", np.sort(-np.partition(-sgdc_scores['test_accuracy'], n)[:n])[::-1]) +print("Smallest 5 ACC:", np.sort(np.partition(sgdc_scores['test_accuracy'], n)[:n])) # %% [markdown] # ### K-nearest neighbors @@ -229,21 +242,23 @@ print("F1", np.median(sgdc_scores['test_f1'])) knn = neighbors.KNeighborsClassifier() # %% jupyter={"source_hidden": true} -knn_scores = cross_validate( # Nekaj ne funkcionira pravilno +knn_scores = cross_validate( knn, X=imputer.fit_transform(train_x), y=data_y, groups=data_groups, - cv=logo, + cv=cv_method, n_jobs=-1, + error_score='raise', scoring=('accuracy', 'precision', 'recall', 'f1') - # error_score='raise' ) # %% jupyter={"source_hidden": true} print("Acc", np.median(knn_scores['test_accuracy'])) print("Precision", np.median(knn_scores['test_precision'])) print("Recall", np.median(knn_scores['test_recall'])) print("F1", np.median(knn_scores['test_f1'])) +print("Largest 5 ACC:", np.sort(-np.partition(-knn_scores['test_accuracy'], n)[:n])[::-1]) +print("Smallest 5 ACC:", np.sort(np.partition(knn_scores['test_accuracy'], n)[:n])) # %% [markdown] # ### Decision Tree @@ -257,8 +272,9 @@ dtree_scores = cross_validate( X=imputer.fit_transform(train_x), y=data_y, groups=data_groups, - cv=logo, + cv=cv_method, n_jobs=-1, + error_score='raise', scoring=('accuracy', 'precision', 'recall', 'f1') ) # %% jupyter={"source_hidden": true} @@ -266,6 +282,8 @@ print("Acc", np.median(dtree_scores['test_accuracy'])) print("Precision", np.median(dtree_scores['test_precision'])) print("Recall", np.median(dtree_scores['test_recall'])) print("F1", np.median(dtree_scores['test_f1'])) +print("Largest 5 ACC:", np.sort(-np.partition(-dtree_scores['test_accuracy'], n)[:n])[::-1]) +print("Smallest 5 ACC:", np.sort(np.partition(dtree_scores['test_accuracy'], n)[:n])) # %% [markdown] # ### Random Forest Classifier @@ -279,8 +297,9 @@ rfc_scores = cross_validate( X=imputer.fit_transform(train_x), y=data_y, groups=data_groups, - cv=logo, + cv=cv_method, n_jobs=-1, + error_score='raise', scoring=('accuracy', 'precision', 'recall', 'f1') ) # %% jupyter={"source_hidden": true} @@ -288,6 +307,8 @@ print("Acc", np.median(rfc_scores['test_accuracy'])) print("Precision", np.median(rfc_scores['test_precision'])) print("Recall", np.median(rfc_scores['test_recall'])) print("F1", np.median(rfc_scores['test_f1'])) +print("Largest 5 ACC:", np.sort(-np.partition(-rfc_scores['test_accuracy'], n)[:n])[::-1]) +print("Smallest 5 ACC:", np.sort(np.partition(rfc_scores['test_accuracy'], n)[:n])) # %% [markdown] # ### Gradient Boosting Classifier @@ -301,8 +322,9 @@ gbc_scores = cross_validate( X=imputer.fit_transform(train_x), y=data_y, groups=data_groups, - cv=logo, + cv=cv_method, n_jobs=-1, + error_score='raise', scoring=('accuracy', 'precision', 'recall', 'f1') ) # %% jupyter={"source_hidden": true} @@ -310,6 +332,8 @@ print("Acc", np.median(gbc_scores['test_accuracy'])) print("Precision", np.median(gbc_scores['test_precision'])) print("Recall", np.median(gbc_scores['test_recall'])) print("F1", np.median(gbc_scores['test_f1'])) +print("Largest 5 ACC:", np.sort(-np.partition(-gbc_scores['test_accuracy'], n)[:n])[::-1]) +print("Smallest 5 ACC:", np.sort(np.partition(gbc_scores['test_accuracy'], n)[:n])) # %% [markdown] # ### LGBM Classifier @@ -323,8 +347,9 @@ lgbm_scores = cross_validate( X=imputer.fit_transform(train_x), y=data_y, groups=data_groups, - cv=logo, + cv=cv_method, n_jobs=-1, + error_score='raise', scoring=('accuracy', 'precision', 'recall', 'f1') ) # %% jupyter={"source_hidden": true} @@ -332,6 +357,8 @@ print("Acc", np.median(lgbm_scores['test_accuracy'])) print("Precision", np.median(lgbm_scores['test_precision'])) print("Recall", np.median(lgbm_scores['test_recall'])) print("F1", np.median(lgbm_scores['test_f1'])) +print("Largest 5 ACC:", np.sort(-np.partition(-lgbm_scores['test_accuracy'], n)[:n])[::-1]) +print("Smallest 5 ACC:", np.sort(np.partition(lgbm_scores['test_accuracy'], n)[:n])) # %% [markdown] # ### XGBoost Classifier @@ -345,8 +372,9 @@ xgb_classifier_scores = cross_validate( X=imputer.fit_transform(train_x), y=data_y, groups=data_groups, - cv=logo, + cv=cv_method, n_jobs=-1, + error_score='raise', scoring=('accuracy', 'precision', 'recall', 'f1') ) # %% jupyter={"source_hidden": true} @@ -354,3 +382,5 @@ print("Acc", np.median(xgb_classifier_scores['test_accuracy'])) print("Precision", np.median(xgb_classifier_scores['test_precision'])) print("Recall", np.median(xgb_classifier_scores['test_recall'])) print("F1", np.median(xgb_classifier_scores['test_f1'])) +print("Largest 5 ACC:", np.sort(-np.partition(-xgb_classifier_scores['test_accuracy'], n)[:n])[::-1]) +print("Smallest 5 ACC:", np.sort(np.partition(xgb_classifier_scores['test_accuracy'], n)[:n]))