Haseeb949/fluenta-backend
0
1# -*- coding: utf-8 -*-2"""model.ipynb3 4Automatically generated by Colab.5 6Original file is located at7 https://colab.research.google.com/drive/1q9IXU6bH7Dj20JHiBnyWe_rB57X7VGdX8"""9 10import pandas as pd11import numpy as np12import os13import librosa14import soundfile as sf15!pip install noisereduce==2.0.116import noisereduce as nr17from sklearn.model_selection import train_test_split, GridSearchCV18from sklearn.preprocessing import RobustScaler19from sklearn.neighbors import KNeighborsClassifier20from sklearn.ensemble import RandomForestClassifier21from sklearn.metrics import accuracy_score, classification_report, confusion_matrix, f1_score22import joblib23import matplotlib.pyplot as plt24import seaborn as sns25import warnings26warnings.filterwarnings('ignore')27 28# PREPROCESSING29def preprocess_audio(audio, sr):30 # 1. Remove silence31 audio, _ = librosa.effects.trim(audio, top_db=20)32 33 # 2. Normalize amplitude34 audio = librosa.util.normalize(audio)35 36 # 3. Noise reduction37 try:38 audio = nr.reduce_noise(y=audio, sr=sr, prop_decrease=0.8)39 except:40 pass41 42 # 4. Resample to 16kHz43 if sr != 16000:44 audio = librosa.resample(audio, orig_sr=sr, target_sr=16000)45 sr = 1600046 47 # 5. Ensure minimum length48 min_length = int(0.5 * sr)49 if len(audio) < min_length:50 audio = np.pad(audio, (0, min_length - len(audio)), mode='constant')51 52 return audio, sr53 54def extract_features(file_path):55 try:56 audio, sr = librosa.load(file_path, sr=None)57 audio, sr = preprocess_audio(audio, sr)58 59 features = []60 61 # MFCCs (20 coefficients)62 mfccs = librosa.feature.mfcc(y=audio, sr=sr, n_mfcc=20)63 features.extend(np.mean(mfccs.T, axis=0))64 features.extend(np.std(mfccs.T, axis=0))65 66 # Spectral features67 spectral_centroids = librosa.feature.spectral_centroid(y=audio, sr=sr)[0]68 features.append(np.mean(spectral_centroids))69 features.append(np.std(spectral_centroids))70 71 spectral_rolloff = librosa.feature.spectral_rolloff(y=audio, sr=sr)[0]72 features.append(np.mean(spectral_rolloff))73 features.append(np.std(spectral_rolloff))74 75 # Zero-crossing rate76 zcr = librosa.feature.zero_crossing_rate(audio)[0]77 features.append(np.mean(zcr))78 features.append(np.std(zcr))79 80 # Chroma features81 chroma = librosa.feature.chroma_stft(y=audio, sr=sr)82 features.extend(np.mean(chroma.T, axis=0))83 84 # RMS Energy85 rms = librosa.feature.rms(y=audio)[0]86 features.append(np.mean(rms))87 features.append(np.std(rms))88 89 return np.array(features)90 91 except Exception as e:92 print(f"Error processing {file_path}: {e}")93 return None94 95# CUSTOM VOICE INTEGRATION96def load_custom_recordings(custom_path, verbose=True):97 features_list = []98 labels_list = []99 file_details = []100 101 if not os.path.exists(custom_path):102 print(f"Custom recordings path not found: {custom_path}")103 return features_list, labels_list, file_details104 105 # Define keywords for each class106 STUTTER_KEYWORDS = ['stutter', 'stuttering', 'disfluent', 'dysfluent', 'blocked', 'repetition']107 FLUENT_KEYWORDS = ['fluent', 'normal', 'nonstutter', 'non-stutter', 'clear']108 109 print(f"\n{'='*60}")110 print(f"Loading custom recordings from: {custom_path}")111 print(f"{'='*60}")112 113 files = [f for f in os.listdir(custom_path) if f.endswith('.wav')]114 115 if len(files) == 0:116 print(" No .wav files found in custom recordings folder!")117 return features_list, labels_list, file_details118 119 for file_name in sorted(files):120 file_path = os.path.join(custom_path, file_name)121 file_lower = file_name.lower()122 123 # Determine label based on filename124 label = None125 keyword_found = None126 127 # Check for stutter keywords128 for keyword in STUTTER_KEYWORDS:129 if keyword in file_lower:130 label = 1131 keyword_found = keyword132 break133 134 # Check for fluent keywords if not already labeled135 if label is None:136 for keyword in FLUENT_KEYWORDS:137 if keyword in file_lower:138 label = 0139 keyword_found = keyword140 break141 142 # If still no label, warn and skip143 if label is None:144 print(f"SKIPPED: {file_name} (no recognizable label keyword)")145 print(f" → Use keywords: {STUTTER_KEYWORDS + FLUENT_KEYWORDS}")146 continue147 148 # Extract features149 features = extract_features(file_path)150 151 if features is not None:152 features_list.append(features)153 labels_list.append(label)154 file_details.append({155 'filename': file_name,156 'label': 'Stutter' if label == 1 else 'Non-Stutter',157 'keyword': keyword_found158 })159 160 if verbose:161 label_str = "✓ Stutter" if label == 1 else "✓ Non-Stutter"162 print(f"{label_str:15} | {file_name:40} | keyword: '{keyword_found}'")163 else:164 print(f" FAILED: {file_name} (feature extraction error)")165 166 print(f"\n{'='*60}")167 print(f"Loaded {len(features_list)} custom recordings")168 print(f" - Stutter: {sum(labels_list)}")169 print(f" - Non-Stutter: {len(labels_list) - sum(labels_list)}")170 print(f"{'='*60}\n")171 172 return features_list, labels_list, file_details173 174# MAIN TRAINING PIPELINE175 176 177print("\n" + "="*60)178print("STUTTERING DETECTION MODEL TRAINING")179print("="*60)180 181# STEP 1: Load original dataset labels182print("\n[1/8] Loading original dataset labels...")183labels_df = pd.read_csv("/content/drive/MyDrive/fyp dataset/clips/labels.csv")184labels_df["label"] = labels_df[["Block", "Prolongation", "SoundRep", "WordRep", "Interjection"]].sum(axis=1)185labels_df["label"] = np.where(labels_df["label"] > 0, 1, 0)186labels_df["filepath"] = labels_df["filepath"].apply(lambda x: os.path.basename(x))187 188print(f"Loaded {len(labels_df)} label entries")189print(f" Original dataset distribution:\n{labels_df['label'].value_counts()}")190 191# STEP 2: Extract features from original dataset192print("\n[2/8] Extracting features from original dataset...")193dataset_path = "/content/drive/MyDrive/fyp dataset/clips/clips"194features_list = []195labels_list = []196dataset_source = [] # Track which dataset each sample comes from197 198for idx, file_name in enumerate(os.listdir(dataset_path)):199 if file_name.endswith(".wav"):200 if idx % 100 == 0:201 print(f" Processing file {idx}...")202 203 file_path = os.path.join(dataset_path, file_name)204 label_row = labels_df[labels_df["filepath"] == file_name]205 206 if label_row.empty:207 continue208 209 features = extract_features(file_path)210 if features is not None:211 features_list.append(features)212 labels_list.append(int(label_row["label"].values[0]))213 dataset_source.append("original")214 215print(f"Extracted {len(features_list)} samples from original dataset")216 217# STEP 3: Load and integrate custom recordings218print("\n[3/8] Loading custom voice recordings...")219custom_path = "/content/drive/MyDrive/myrecordings"220custom_features, custom_labels, custom_details = load_custom_recordings(custom_path, verbose=True)221 222# Add custom recordings to main dataset223if len(custom_features) > 0:224 features_list.extend(custom_features)225 labels_list.extend(custom_labels)226 dataset_source.extend(["custom"] * len(custom_features))227 228 print(f"Added {len(custom_features)} custom recordings to training set")229else:230 print(" No custom recordings loaded!")231 232# STEP 4: Create DataFrame and analyze233print("\n[4/8] Creating training dataset...")234df = pd.DataFrame(features_list)235df["label"] = labels_list236df["source"] = dataset_source237 238print(f"\n Dataset Statistics:")239print(f" Total samples: {len(df)}")240print(f"\n By source:")241print(df['source'].value_counts())242print(f"\n By label:")243print(df['label'].value_counts())244print(f"\n Custom recordings breakdown:")245custom_df = df[df['source'] == 'custom']246if len(custom_df) > 0:247 print(f" - Total custom: {len(custom_df)}")248 print(f" - Stutter: {(custom_df['label'] == 1).sum()}")249 print(f" - Non-Stutter: {(custom_df['label'] == 0).sum()}")250else:251 print(" - No custom recordings in dataset")252 253# Save for inspection254df.to_csv("training_data_with_custom.csv", index=False)255print(f"\n Saved to: training_data_with_custom.csv")256 257# CRITICAL: Check for class imbalance258stutter_ratio = df['label'].sum() / len(df)259print(f"\n Class Balance Check:")260print(f" Stutter ratio: {stutter_ratio:.2%}")261if stutter_ratio < 0.1 or stutter_ratio > 0.9:262 print(f" WARNING: Highly imbalanced dataset!")263 print(f" → Consider collecting more samples of minority class")264 265# STEP 5: Train-Test Split266print("\n[5/8] Splitting into train/test sets...")267X = df.drop(["label", "source"], axis=1).values268y = df["label"].values269 270# Stratified split to maintain class distribution271X_train, X_test, y_train, y_test = train_test_split(272 X, y, test_size=0.2, random_state=42, stratify=y273)274 275print(f" Split complete:")276print(f" Training: {len(X_train)} samples")277print(f" Testing: {len(X_test)} samples")278print(f" Training labels: {np.bincount(y_train)}")279print(f" Testing labels: {np.bincount(y_test)}")280# STEP 6: Feature Scaling (CRITICAL: Use RobustScaler)281print("\n[6/8] Scaling features...")282scaler = RobustScaler() # Better for outliers than StandardScaler283X_train_scaled = scaler.fit_transform(X_train)284X_test_scaled = scaler.transform(X_test)285print(" Features scaled using RobustScaler")286 287# STEP 7: Train models with grid search288print("\n[7/8] Training models...")289 290# KNN Model291print("\n Training KNN...")292knn_params = {293 'n_neighbors': [3, 5, 7, 9],294 'weights': ['uniform', 'distance'],295 'metric': ['euclidean', 'manhattan']296}297knn = KNeighborsClassifier()298knn_grid = GridSearchCV(knn, knn_params, cv=5, scoring='f1', n_jobs=-1)299knn_grid.fit(X_train_scaled, y_train)300best_knn = knn_grid.best_estimator_301 302y_pred_knn = best_knn.predict(X_test_scaled)303knn_f1 = f1_score(y_test, y_pred_knn)304print(f" KNN - F1 Score: {knn_f1:.4f}, Best params: {knn_grid.best_params_}")305 306# Random Forest Model307print("\n Training Random Forest...")308rf_params = {309 'n_estimators': [50, 100, 200],310 'max_depth': [10, 20, None],311 'min_samples_split': [2, 5],312 'class_weight': ['balanced', None] # Handle imbalance313}314rf = RandomForestClassifier(random_state=42)315rf_grid = GridSearchCV(rf, rf_params, cv=5, scoring='f1', n_jobs=-1)316rf_grid.fit(X_train_scaled, y_train)317best_rf = rf_grid.best_estimator_318 319y_pred_rf = best_rf.predict(X_test_scaled)320rf_f1 = f1_score(y_test, y_pred_rf)321print(f" Random Forest - F1 Score: {rf_f1:.4f}, Best params: {rf_grid.best_params_}")322 323# Select best model324if rf_f1 > knn_f1:325 best_model = best_rf326 y_pred = y_pred_rf327 model_name = "Random Forest"328else:329 best_model = best_knn330 y_pred = y_pred_knn331 model_name = "KNN"332 333print(f"\n Best model: {model_name}")334 335# STEP 8: Evaluate336print("\n[8/8] Model Evaluation...")337print(f"\n{'='*60}")338print(f" FINAL RESULTS")339print(f"{'='*60}")340print(f"\nAccuracy: {accuracy_score(y_test, y_pred):.4f}")341print(f"F1-Score: {f1_score(y_test, y_pred):.4f}")342print(f"\nClassification Report:")343print(classification_report(y_test, y_pred, target_names=['Non-Stutter', 'Stutter']))344 345# Confusion Matrix346cm = confusion_matrix(y_test, y_pred)347plt.figure(figsize=(8, 6))348sns.heatmap(cm, annot=True, fmt="d", cmap="Blues",349 xticklabels=['Non-Stutter', 'Stutter'],350 yticklabels=['Non-Stutter', 'Stutter'])351plt.xlabel("Predicted")352plt.ylabel("Actual")353plt.title(f"Confusion Matrix - {model_name}")354plt.show()355 356# STEP 10: Save models357print(f"\n{'='*60}")358print("Saving models...")359joblib.dump(best_model, "stutter_model_final.pkl")360joblib.dump(scaler, "scaler_final.pkl")361joblib.dump(X_train.shape[1], "feature_count.pkl")362 363print(" Saved:")364print(" - stutter_model_final.pkl")365print(" - scaler_final.pkl")366print(" - feature_count.pkl")367print(f"{'='*60}\n")368 369print("Training complete!")