AhmedKhaled1122/Data-Analysis-and-Machine-Learning

1import numpy as np2import pandas as pd3import missingno as msno4import matplotlib.pyplot as plt5import seaborn as sns6from sklearn.impute import SimpleImputer, KNNImputer7from sklearn.experimental import enable_iterative_imputer8from sklearn.impute import IterativeImputer9from scipy.stats.mstats import winsorize10 11def show_missing_value_persentage(df):12    missing = df.isnull().sum() / df.shape[0] * 10013    missing = missing[missing > 0].sort_values(ascending=False)14    return None if missing.empty else missing15 16def missing_matrix(df):17    fig, ax = plt.subplots(figsize=(10, 6))18    msno.matrix(df[df.columns[df.isnull().any()]], ax=ax, color=(0, 0.5, 0.5))19    ax.set_title("Missing Values Matrix")20    return fig21 22def missing_heatmap(df):23    fig, ax = plt.subplots(figsize=(10, 6))24    msno.heatmap(df[df.columns[df.isnull().any()]], ax=ax, cmap='viridis')25    ax.set_title("Missing Values Heatmap")26    return fig27 28def missing_bar(df):29    fig, ax = plt.subplots(figsize=(10, 4))30    msno.bar(df[df.columns[df.isnull().any()]], ax=ax, color='teal')31    ax.set_title("Missing Values Bar Chart")32    return fig33 34 35def simple_imputer(df, simple_numeric):36    for col, strategy, fill_value in simple_numeric:37        if strategy == "constant":38                imputer = SimpleImputer(strategy=strategy, fill_value=fill_value)39        else:40            imputer = SimpleImputer(strategy=strategy)41        df[[col]] = imputer.fit_transform(df[[col]])42    return df43 44 45def KNN_imputer(df, col, imputation_settings):46    n_neighbors = imputation_settings[col[0]]["n_neighbors"]47    knn_imputer = KNNImputer(n_neighbors=n_neighbors)48    df[col] = knn_imputer.fit_transform(df[col])49    return df50 51def iterative__imputer(df, iterative_cols, imputation_settings):52    max_iter = imputation_settings[iterative_cols[0]]["max_iter"]53    imputer = IterativeImputer(max_iter=max_iter, random_state=0)54    df[iterative_cols] = imputer.fit_transform(df[iterative_cols])55    return df56 57def fill_value(df, col, value):58    df[col] = df[col].fillna(value)59    return df60 61 62def show_duplicate(df):63    duplicate = df.duplicated().sum()64    return duplicate65 66def drop_duplicate(df):67    df = df.drop_duplicates()68    return df69 70def calculate_variance(df):71    return df.var(numeric_only=True).sort_values()72 73def drop_columns(df, select_drop):74    df = df.drop(select_drop, axis=1)75    return df76 77def box_plot(df, col):78    fig, ax = plt.subplots(figsize=(7, 3))79    sns.boxplot(x=df[col], ax=ax, color='teal')80    ax.set_xlabel(None)81    ax.set_title(f"Boxplot for {col}")82    return fig83 84def detect_outlier_columns(df):85    outlier_cols = []86    for col in df.select_dtypes(include=['int64', 'float64']).columns:87        Q1 = df[col].quantile(0.25)88        Q3 = df[col].quantile(0.75)89        IQR = Q3 - Q190        lower_bound = Q1 - 1.5 * IQR91        upper_bound = Q3 + 1.5 * IQR92        93        if ((df[col] < lower_bound) | (df[col] > upper_bound)).any():94            outlier_cols.append(col)95    return outlier_cols96 97 98def upper_lower_IQR(df, col):99    q1 = df[col].quantile(0.25)100    q3 = df[col].quantile(0.75)101    iqr = q3 - q1102    lower_bound = q1 - 1.5 * iqr103    upper_bound = q3 + 1.5 * iqr104    return lower_bound, upper_bound105 106def upper_lower_Zscore(df, col):107    lower_bound = df[col].mean() - 3 * df[col].std()108    upper_bound = df[col].mean() + 3 * df[col].std()109    return lower_bound, upper_bound110 111def remove_outliers(df, col, lower_bound, upper_bound):112    df = df[(df[col] >= lower_bound) & (df[col] <= upper_bound)]113    return df114 115def winsorize_outliers(df, col):116    df[col] = winsorize(df[col], limits=[0.05, 0.05])117    return df118 119def clip_outliers(df, col, lower_bound, upper_bound):120    df[col] = df[col].clip(lower=lower_bound, upper=upper_bound)121    return df122