EgzonP/assignment_4_supervised_learing

1 2#Supervised Learning.ipynb3 4import pandas as pd5import numpy as np6from sklearn.model_selection import train_test_split7from sklearn.preprocessing import StandardScaler, OneHotEncoder8from sklearn.impute import KNNImputer9from sklearn.metrics import mean_squared_error10from imblearn.under_sampling import RandomUnderSampler11from xgboost import XGBRegressor12import shap13import joblib14 15path = "micro_world_139countries.csv"16df = pd.read_csv(path, encoding="latin-1")17 18df.columns19 20"""Our hypothesis is whether we can predict how worried a person is of medical costs based on age, education and income."""21 22ndf = df[["fin44b", "age", "educ", "inc_q", "emp_in"]]23ndf24 25ndf.info()26 27ndf.isnull().sum()28 29ndf = ndf.dropna()30ndf = ndf[(ndf["fin44b"] < 4) & (ndf["educ"] < 4)]31ndf32 33from scipy.stats import zscore34zage = zscore(ndf["age"])35ndf["outlier_age"] = (zage > 3) | (zage < -3)36ndf37 38ndf_cage = ndf[~ndf["outlier_age"]]39ndfc = ndf_cage[["fin44b", "age", "educ", "inc_q", "emp_in"]]40ndfc41 42ndfc.info()43 44import seaborn as sns45sns.countplot(x="fin44b", data=ndfc)46 47rus = RandomUnderSampler(sampling_strategy='auto', random_state=42)48X_resampled, _ = rus.fit_resample(ndfc, ndfc['fin44b'])49ndfc = X_resampled50ndfc51 52ndfc.value_counts("educ")53 54X = ndfc.drop('fin44b', axis=1)55y = ndfc['fin44b']56 57scaler = StandardScaler()58num_features = ["age", "educ", "inc_q", "emp_in"]59X_num = pd.DataFrame(scaler.fit_transform(X[num_features]), columns=num_features)60X_pro = X_num61X_pro62 63X_train, X_test, y_train, y_test = train_test_split(X_pro, y, test_size=0.2)64 65model_xgb = XGBRegressor(random_state=42)66model_xgb.fit(X_train, y_train)67 68from sklearn.linear_model import LinearRegression, ElasticNet69from sklearn.ensemble import RandomForestRegressor70 71model_ols = LinearRegression()72model_el = ElasticNet()73model_rf = RandomForestRegressor(n_estimators=25)74 75model_ols.fit(X_train, y_train)76model_el.fit(X_train, y_train)77model_rf.fit(X_train, y_train)78 79print('Model OLS' + ' ' + str(model_ols.score(X_test, y_test)))80print('Model EL' + ' ' + str(model_el.score(X_test, y_test)))81print('Model RF' + ' ' + str(model_rf.score(X_test, y_test)))82 83from sklearn.model_selection import GridSearchCV84from sklearn.metrics import make_scorer85from sklearn.metrics import mean_squared_error86from sklearn.metrics import r2_score87 88scorer = make_scorer(mean_squared_error)89 90parameters_rf = {'bootstrap': [True, False],91 'max_depth': [10, 20, None],92 'min_samples_split': [2, 5, 10],93 'n_estimators': [25, 50]}94 95grid_obj = GridSearchCV(model_rf, parameters_rf, scoring=scorer)96 97grid_fit = grid_obj.fit(X, y)98 99# Get the estimator.100best_reg = grid_fit.best_estimator_101 102# Fit the new model.103best_reg.fit(X_train, y_train)104 105best_reg.score(X_train, y_train)106 107best_reg.score(X_test, y_test)108 109from sklearn.model_selection import RandomizedSearchCV, train_test_split110from sklearn.metrics import accuracy_score111from scipy.stats import uniform, randint112 113param_distributions = {114    'n_estimators': randint(100, 1000),       # Random integer values between 100 and 1000115    'max_depth': randint(3, 10),              # Random integer values between 3 and 10116    'learning_rate': uniform(0.01, 0.2),      # Uniformly distributed real values between 0.01 and 0.2117    'subsample': uniform(0.5, 0.5),           # Uniform values between 0.5 and 1.0118    'colsample_bytree': uniform(0.5, 0.5),    # Uniform values between 0.5 and 1.0119    'gamma': uniform(0, 5),                   # Uniform values between 0 and 5120    'min_child_weight': randint(1, 10),       # Random integer values between 1 and 10121    'reg_alpha': uniform(0, 1),               # Uniform values between 0 and 1 (L1 regularization)122    'reg_lambda': uniform(1, 5)               # Uniform values between 1 and 5 (L2 regularization)123}124 125random_search = RandomizedSearchCV(126    estimator=model_xgb, param_distributions=param_distributions,127    n_iter=50, scoring='neg_mean_squared_error', cv=5)128 129random_search.fit(X_train, y_train)130 131print("Best Parameters:", random_search.best_params_)132print("Best Score:", random_search.best_score_)133 134best_model = random_search.best_estimator_135best_model.fit(X_train, y_train)136best_model.score(X_train, y_train)137 138best_model.score(X_test, y_test)139 140from sklearn.ensemble import RandomForestClassifier141model = RandomForestClassifier()142 143from yellowbrick.features import FeatureImportances144 145viz = FeatureImportances(best_reg)146viz.fit(X, y)147viz.show()148 149from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score150import numpy as np151 152# Predict on training and test data153y_train_pred = model_xgb.predict(X_train)  # Predictions on training data154y_test_pred = model_xgb.predict(X_test)    # Predictions on test data155 156# Mean Squared Error (MSE) and Root Mean Squared Error (RMSE)157train_mse = mean_squared_error(y_train, y_train_pred)158test_mse = mean_squared_error(y_test, y_test_pred)159train_rmse = np.sqrt(train_mse)160test_rmse = np.sqrt(test_mse)161 162print(f"Train RMSE: {train_rmse:.2f}")163print(f"Test RMSE: {test_rmse:.2f}")164 165# Mean Absolute Error (MAE)166train_mae = mean_absolute_error(y_train, y_train_pred)167test_mae = mean_absolute_error(y_test, y_test_pred)168print(f"Train MAE: {train_mae:.2f}")169print(f"Test MAE: {test_mae:.2f}")170 171# R-squared (R²)172train_r2 = r2_score(y_train, y_train_pred)173test_r2 = r2_score(y_test, y_test_pred)174print(f"Train R²: {train_r2:.2f}")175print(f"Test R²: {test_r2:.2f}")176 177# Feature importance178from yellowbrick.features import FeatureImportances179 180viz = FeatureImportances(best_model)181viz.fit(X_test, y_test)182viz.show()183 184 185import shap186explainer = shap.TreeExplainer(best_model)187shap_values = explainer.shap_values(X_test)188 189shap.summary_plot(shap_values, X_test, plot_type="bar")190 191shap.summary_plot(shap_values, X_test)192 193most_important_feature = X_test.columns[np.argmax(np.abs(shap_values).mean(0))]194shap.dependence_plot(most_important_feature, shap_values, X_test)195 196joblib.dump(model_xgb, 'model_xgb.joblib')197joblib.dump(best_model, 'model_best.joblib')198joblib.dump(model, 'model.joblib')199joblib.dump(best_reg, 'best.reg.joblib')200joblib.dump(scaler, 'scaler.joblib')