Osele1/sonic-clusters

1import pandas as pd2import numpy as np3import joblib4import json5import warnings6from sklearn.compose import ColumnTransformer7from sklearn.preprocessing import StandardScaler, OneHotEncoder8from sklearn.cluster import KMeans, AgglomerativeClustering, DBSCAN9from sklearn.decomposition import PCA10import umap11 12# Suppress warnings13warnings.filterwarnings('ignore')14 15print("Loading data...")16old_df = pd.read_csv('backend/data/songs_with_clusters.csv')17new_df = pd.read_csv('new_spotify_songs.csv')18 19# Drop cluster columns and umap from old_df to match new_df20drop_cols = ['kmeans_cluster', 'hierarchical_cluster', 'dbscan_cluster', 'umap_x', 'umap_y', 'umap_z']21for col in drop_cols:22    if col in old_df.columns:23        old_df = old_df.drop(columns=[col])24 25# Combine datasets26combined_df = pd.concat([old_df, new_df], ignore_index=True)27print(f"Combined dataset size: {len(combined_df):,} tracks")28 29# Load feature info30with open('backend/data/feature_info.json') as f:31    feature_info = json.load(f)32 33numeric_features = feature_info['numeric']34categorical_features = feature_info['categorical']35 36print("Fitting Preprocessor (adapting to modern EQs)...")37preprocessor = ColumnTransformer(38    transformers=[39        ('num', StandardScaler(), numeric_features),40        ('cat', OneHotEncoder(handle_unknown='ignore', sparse_output=False), categorical_features)41    ])42 43X_scaled = preprocessor.fit_transform(combined_df)44print(f"Feature matrix shape: {X_scaled.shape}")45 46print("Fitting K-Means (k=6)...")47kmeans = KMeans(n_clusters=6, random_state=42)48combined_df['kmeans_cluster'] = kmeans.fit_predict(X_scaled)49 50print("Fitting Hierarchical (k=6)...")51hierarchical = AgglomerativeClustering(n_clusters=6)52combined_df['hierarchical_cluster'] = hierarchical.fit_predict(X_scaled)53 54print("Fitting DBSCAN (eps=1.44, min_samples=3)...")55dbscan = DBSCAN(eps=1.44, min_samples=3)56combined_df['dbscan_cluster'] = dbscan.fit_predict(X_scaled)57 58print("Fitting PCA (10 components)...")59pca = PCA(n_components=10, random_state=42)60pca.fit(X_scaled)61 62print("Generating 3D UMAP Coordinates (this will take a minute)...")63reducer = umap.UMAP(n_components=3, random_state=42)64embeddings = reducer.fit_transform(X_scaled)65combined_df['umap_x'] = embeddings[:, 0]66combined_df['umap_y'] = embeddings[:, 1]67combined_df['umap_z'] = embeddings[:, 2]68 69print("Saving updated models (This also fixes old scikit-learn warnings!)...")70joblib.dump(preprocessor, 'backend/models/preprocessor.pkl')71joblib.dump(kmeans, 'backend/models/kmeans_model.pkl')72joblib.dump(hierarchical, 'backend/models/hierarchical_model.pkl')73joblib.dump(dbscan, 'backend/models/dbscan_model.pkl')74joblib.dump(pca, 'backend/models/pca_transformer.pkl')75 76print("Saving updated data...")77combined_df.to_csv('backend/data/songs_with_clusters.csv', index=False)78np.save('backend/data/feature_matrix.npy', X_scaled)79np.save('backend/data/kmeans_centroids.npy', kmeans.cluster_centers_)80 81print("\nAll done! The AI has been successfully retrained on the massive 12,000 track dataset.")82print("You can now restart your backend server with `python main.py`!")83