NauRaa/Gene_Expression_Clustering_App

1"""2Unsupervised Learning on Gene Expression Data3 4"""5 6# =============================7#  Import Libraries8# =============================9import numpy as np10import pandas as pd11import matplotlib.pyplot as plt12import seaborn as sns13from sklearn.preprocessing import RobustScaler14from sklearn.feature_selection import VarianceThreshold15from sklearn.decomposition import PCA16from sklearn.cluster import KMeans, DBSCAN, AgglomerativeClustering17from sklearn.ensemble import IsolationForest18from sklearn.metrics import silhouette_score19import joblib20 21# =============================22#  Load Data23# =============================24df = pd.read_csv(r"E:\Rev-DataScience\AI-ML\genes.csv", encoding='latin')25 26# Drop text columns if they exist27df.drop(columns=['NAME', 'Gene_symbol'], errors='ignore', inplace=True)28 29# =============================30#  Memory Optimization31# =============================32for col in df.columns:33    if df[col].dtype == 'float64':34        df[col] = df[col].astype('float32')35 36# =============================37#  Outlier Detection 38# =============================39 40iso = IsolationForest(contamination=0.02, random_state=42)41outlier_labels = iso.fit_predict(df)42mask = outlier_labels != -1  43df = df[mask]44#=================================45# EDA & Visualization46#=================================47 48for col in df.columns:49    if np.issubdtype(df[col].dtype, np.number):50        skewed = df[col].skew()51        if skewed > 0.5:52            plt.figure()53            sns.histplot(df[col], kde=True, color='blue')54            plt.title(f"{col} Before Log Transform (Skew={skewed:.2f})")55            plt.show()56 57            df[col] = np.log1p(df[col])  # log transform58 59            sns.histplot(df[col], kde=True, color='red')60            plt.title(f"{col} After Log Transform")61            plt.show()62 63 64corr = df.corr().abs()65upper = corr.where(np.triu(np.ones(corr.shape), k=1).astype(bool))66high_corr = [column for column in upper.columns if any(upper[column] > 0.9)]67print(f"Highly correlated features (> 0.9): {len(high_corr)}")68 69plt.figure(figsize=(10,8))70sns.heatmap(corr, cmap='Blues')71plt.title('Correlation Heatmap')72plt.show()73 74# =============================75#  Feature Selection76# =============================77selector = VarianceThreshold(threshold=0.1)78X_reduced = selector.fit_transform(df)79selected_features = df.columns[selector.get_support()]80 81# =============================82# Scaling83# =============================84scaler = RobustScaler()85X_scaled = scaler.fit_transform(X_reduced)86 87# =============================88# Dimensionality Reduction89# =============================90pca = PCA(n_components=2, random_state=42)91X_pca = pca.fit_transform(X_scaled)92 93plt.figure(figsize=(7,5))94plt.scatter(X_pca[:,0], X_pca[:,1], s=10)95plt.title("PCA Projection (Before Clustering)")96plt.show()97 98explained = pca.explained_variance_ratio_99print(f"Explained variance by PCA components: {explained}")100print(f"Total variance retained: {explained.sum():.2f}")101 102# =============================103# 8. Clustering Models104# =============================105models = {106    'KMeans': KMeans(n_clusters=3, random_state=42),107    'DBSCAN': DBSCAN(eps=0.7, min_samples=5),108    'Agglomerative': AgglomerativeClustering(n_clusters=3, linkage='ward')109}110 111results = {}112 113for name, model in models.items():114    labels = model.fit_predict(X_pca)115    if len(set(labels)) > 1:116        score = silhouette_score(X_pca, labels)117        results[name] = score118        print(f"{name}: Silhouette Score = {score:.3f}")119    else:120        print(f"{name}: Only one cluster detected, skipping.")121 122# =============================123# Best Model124# =============================125best_model = max(results, key=results.get)126print(f"\n Best Model: {best_model} with score {results[best_model]:.3f}")127 128# Refit the best model129final_model = models[best_model]130final_labels = final_model.fit_predict(X_pca)131 132df['Cluster'] = final_labels133 134# =============================135#  Visualization136# =============================137plt.figure(figsize=(7,5))138sns.scatterplot(x=X_pca[:,0], y=X_pca[:,1], hue=final_labels, palette='tab10')139plt.title(f"Clusters Visualization ({best_model})")140plt.show()141 142# =============================143#  Save Results144# =============================145df.to_csv("clustered_genes.csv", index=False)146print("Clustered dataset saved as clustered_genes.csv ")147 148joblib.dump(final_model, "best_cluster_model.joblib")149joblib.dump(pca,"pca_transform.joblib")150joblib.dump(scaler,"scaler.joblib")151joblib.dump(selector,"selector.joblib")152