hareshchander/Smart-Grid-Anomaly-Detector
2
1"""2Smart Grid Anomaly Detection System3Complete Gradio Web Application for Hugging Face Deployment4"""5 6import gradio as gr7import pandas as pd8import numpy as np9import matplotlib.pyplot as plt10import seaborn as sns11from datetime import datetime, timedelta12import warnings13import io14from PIL import Image15 16warnings.filterwarnings('ignore')17 18from sklearn.preprocessing import StandardScaler19from sklearn.ensemble import IsolationForest20from sklearn.svm import OneClassSVM21from sklearn.covariance import EllipticEnvelope22from sklearn.decomposition import PCA23from sklearn.feature_selection import VarianceThreshold24from scipy import stats25 26import plotly.graph_objects as go27from plotly.subplots import make_subplots28 29plt.style.use('seaborn-v0_8-darkgrid')30sns.set_palette("husl")31 32 33def generate_sample_data(duration_days=30):34 """Generate realistic sample smart grid data"""35 n_samples = duration_days * 9636 start_date = datetime(2024, 1, 1)37 timestamps = [start_date + timedelta(minutes=15*i) for i in range(n_samples)]38 39 t = np.arange(n_samples)40 daily_pattern = 50 + 30 * np.sin(2 * np.pi * t / 96)41 weekly_pattern = 10 * np.sin(2 * np.pi * t / (7 * 96))42 random_variation = np.random.normal(0, 5, n_samples)43 power = daily_pattern + weekly_pattern + random_variation44 45 df = pd.DataFrame({46 'timestamp': timestamps,47 'voltage': 230 + np.random.normal(0, 2, n_samples),48 'current': power / 230 * 1000 + np.random.normal(0, 10, n_samples),49 'frequency': 50 + np.random.normal(0, 0.1, n_samples),50 'power_consumption': power,51 'power_factor': 0.9 + np.random.normal(0, 0.03, n_samples)52 })53 54 n_anomalies = int(0.08 * n_samples)55 anomaly_idx = np.random.choice(df.index, n_anomalies, replace=False)56 df.loc[anomaly_idx[:n_anomalies//4], 'voltage'] += np.random.choice([-35, 35], n_anomalies//4)57 df.loc[anomaly_idx[n_anomalies//4:n_anomalies//2], 'power_consumption'] *= 2.558 df.loc[anomaly_idx[n_anomalies//2:3*n_anomalies//4], 'frequency'] += np.random.choice([-2, 2], n_anomalies//4)59 df.loc[anomaly_idx[3*n_anomalies//4:], 'power_factor'] = 0.660 61 return df62 63 64def validate_and_preprocess_data(df):65 """Validate and preprocess input data"""66 column_mapping = {67 'timestamp': ['timestamp', 'time', 'datetime', 'date_time', 'date'],68 'voltage': ['voltage', 'voltage_v', 'volt', 'v'],69 'current': ['current', 'current_a', 'amp', 'i', 'a'],70 'frequency': ['frequency', 'frequency_hz', 'freq', 'hz', 'f']71 }72 73 for standard_name, possible_names in column_mapping.items():74 for col in df.columns:75 if col.lower().strip() in possible_names:76 df.rename(columns={col: standard_name}, inplace=True)77 break78 79 required_cols = ['timestamp', 'voltage', 'current', 'frequency']80 missing_cols = [col for col in required_cols if col not in df.columns]81 82 if missing_cols:83 raise ValueError(f"Missing columns: {missing_cols}")84 85 df['timestamp'] = pd.to_datetime(df['timestamp'])86 df.set_index('timestamp', inplace=True)87 df.sort_index(inplace=True)88 89 if 'power_consumption' not in df.columns:90 df['power_consumption'] = (df['voltage'] * df['current']) / 100091 if 'power_factor' not in df.columns:92 df['power_factor'] = 0.993 94 initial_missing = df.isnull().sum().sum()95 if initial_missing > 0:96 df = df.interpolate(method='linear').fillna(method='bfill').fillna(method='ffill')97 98 initial_duplicates = df.index.duplicated().sum()99 if initial_duplicates > 0:100 df = df[~df.index.duplicated(keep='first')]101 102 return df, initial_missing, initial_duplicates103 104 105def engineer_features(df):106 """Create features for anomaly detection"""107 sensors = ['voltage', 'current', 'frequency', 'power_consumption', 'power_factor']108 window_size = max(12, min(48, len(df) // 240))109 110 for sensor in sensors:111 df[f'{sensor}_rolling_mean'] = df[sensor].rolling(window=window_size).mean()112 df[f'{sensor}_rolling_std'] = df[sensor].rolling(window=window_size).std()113 df[f'{sensor}_diff'] = df[sensor].diff()114 115 df['hour'] = df.index.hour116 df['day_of_week'] = df.index.dayofweek117 df['is_weekend'] = (df['day_of_week'] >= 5).astype(int)118 119 df_features = df.dropna()120 return df_features, sensors121 122 123def detect_anomalies(df_features, contamination=0.05):124 """Detect anomalies using ensemble of ML models"""125 numeric_cols = df_features.select_dtypes(include=[np.number]).columns.tolist()126 X = df_features[numeric_cols]127 128 n_samples, n_features = X.shape129 max_features = min(n_samples // 3, 30)130 131 if n_features > max_features:132 selector = VarianceThreshold(threshold=0.01)133 X_temp = selector.fit_transform(X)134 selected_cols = X.columns[selector.get_support()].tolist()135 136 if len(selected_cols) > max_features:137 pca = PCA(n_components=max_features, random_state=42)138 X_reduced = pca.fit_transform(X[selected_cols])139 X = pd.DataFrame(X_reduced, index=df_features.index)140 else:141 X = df_features[selected_cols]142 143 scaler = StandardScaler()144 X_scaled = scaler.fit_transform(X)145 146 models = {147 'Isolation Forest': IsolationForest(contamination=contamination, random_state=42, n_jobs=-1),148 'One-Class SVM': OneClassSVM(nu=contamination, kernel='rbf', gamma='auto')149 }150 151 if n_samples > X.shape[1] * 2:152 models['Elliptic Envelope'] = EllipticEnvelope(contamination=contamination, random_state=42)153 154 results = {}155 model_info = {}156 157 for name, model in models.items():158 try:159 model.fit(X_scaled)160 predictions = model.predict(X_scaled)161 predictions = (predictions == -1).astype(int)162 results[name] = predictions163 df_features[f'{name}_prediction'] = predictions164 n_detected = predictions.sum()165 model_info[name] = {'detected': n_detected, 'rate': (n_detected / len(predictions)) * 100}166 except:167 df_features[f'{name}_prediction'] = 0168 results[name] = np.zeros(len(X_scaled), dtype=int)169 model_info[name] = {'detected': 0, 'rate': 0.0}170 171 sensors = ['voltage', 'current', 'frequency', 'power_consumption', 'power_factor']172 df_features['zscore_anomaly'] = 0173 for sensor in sensors:174 z_scores = np.abs(stats.zscore(df_features[sensor]))175 df_features['zscore_anomaly'] |= (z_scores > 3).astype(int)176 177 n_zscore = df_features['zscore_anomaly'].sum()178 model_info['Z-Score'] = {'detected': n_zscore, 'rate': (n_zscore / len(df_features)) * 100}179 180 successful_models = list(results.keys())181 ensemble_sum = df_features['zscore_anomaly'].copy()182 for name in successful_models:183 ensemble_sum += df_features[f'{name}_prediction']184 185 threshold = (len(successful_models) + 1) // 2 + 1186 df_features['ensemble_anomaly'] = (ensemble_sum >= threshold).astype(int)187 188 n_ensemble = df_features['ensemble_anomaly'].sum()189 model_info['Ensemble'] = {'detected': n_ensemble, 'rate': (n_ensemble / len(df_features)) * 100}190 191 return df_features, results, sensors, model_info192 193 194def create_overview_visualization(df_features, sensors):195 """Create time series overview"""196 anomaly_mask = df_features['ensemble_anomaly'] == 1197 normal_mask = df_features['ensemble_anomaly'] == 0198 199 fig, axes = plt.subplots(4, 1, figsize=(16, 12))200 colors = ['#FF6B6B', '#4ECDC4', '#45B7D1', '#FFA07A']201 units = ['V', 'A', 'Hz', 'kW']202 203 for idx, (sensor, color, unit) in enumerate(zip(sensors[:4], colors, units)):204 ax = axes[idx]205 normal_data = df_features[normal_mask]206 ax.plot(normal_data.index, normal_data[sensor], color=color, alpha=0.6, linewidth=1)207 ax.fill_between(normal_data.index, normal_data[sensor], alpha=0.15, color=color)208 209 anomaly_data = df_features[anomaly_mask]210 if len(anomaly_data) > 0:211 ax.scatter(anomaly_data.index, anomaly_data[sensor], color='red', s=50, 212 alpha=0.8, marker='X', edgecolors='darkred', linewidths=2, zorder=5)213 214 ax.set_ylabel(f'{sensor.title()} ({unit})', fontsize=12, fontweight='bold')215 ax.set_title(f'{sensor.replace("_", " ").title()}', fontsize=14, fontweight='bold', color=color)216 ax.grid(True, alpha=0.3)217 ax.legend(['Normal', 'Anomaly'], loc='upper right')218 219 axes[-1].set_xlabel('Timestamp', fontsize=12, fontweight='bold')220 fig.suptitle('Smart Grid Anomaly Detection - Time Series', fontsize=16, fontweight='bold')221 plt.tight_layout()222 223 buf = io.BytesIO()224 plt.savefig(buf, format='png', dpi=150, bbox_inches='tight')225 buf.seek(0)226 img = Image.open(buf)227 plt.close()228 return img229 230 231def create_model_comparison(model_info, df_features):232 """Create model comparison chart"""233 fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 6))234 235 model_names = list(model_info.keys())236 detection_counts = [info['detected'] for info in model_info.values()]237 colors_bar = plt.cm.viridis(np.linspace(0.2, 0.9, len(model_names)))238 239 bars = ax1.bar(model_names, detection_counts, color=colors_bar, edgecolor='black', linewidth=2)240 for bar, count in zip(bars, detection_counts):241 height = bar.get_height()242 ax1.text(bar.get_x() + bar.get_width()/2., height, f'{int(count)}',243 ha='center', va='bottom', fontsize=10, fontweight='bold')244 245 ax1.set_ylabel('Anomalies Detected', fontsize=12, fontweight='bold')246 ax1.set_title('Model Comparison', fontsize=14, fontweight='bold')247 ax1.grid(axis='y', alpha=0.3)248 plt.setp(ax1.xaxis.get_majorticklabels(), rotation=20, ha='right')249 250 n_anomalies = df_features['ensemble_anomaly'].sum()251 n_normal = len(df_features) - n_anomalies252 ax2.pie([n_normal, n_anomalies], labels=['Normal', 'Anomalies'], 253 colors=['#4ECDC4', '#FF6B6B'], autopct='%1.1f%%', startangle=90)254 ax2.set_title('Data Distribution', fontsize=14, fontweight='bold')255 256 plt.tight_layout()257 buf = io.BytesIO()258 plt.savefig(buf, format='png', dpi=150, bbox_inches='tight')259 buf.seek(0)260 img = Image.open(buf)261 plt.close()262 return img263 264 265def create_interactive_timeline(df_features, sensors):266 """Create interactive timeline"""267 anomaly_mask = df_features['ensemble_anomaly'] == 1268 normal_mask = df_features['ensemble_anomaly'] == 0269 270 fig = make_subplots(rows=len(sensors), cols=1, shared_xaxes=True,271 subplot_titles=[s.title() for s in sensors])272 273 colors = ['#FF6B6B', '#4ECDC4', '#45B7D1', '#FFA07A', '#95E1D3']274 275 for idx, (sensor, color) in enumerate(zip(sensors, colors)):276 normal_data = df_features[normal_mask]277 fig.add_trace(go.Scatter(x=normal_data.index, y=normal_data[sensor],278 mode='lines', name='Normal' if idx == 0 else None,279 line=dict(color=color, width=2), showlegend=(idx == 0)),280 row=idx+1, col=1)281 282 anomaly_data = df_features[anomaly_mask]283 if len(anomaly_data) > 0:284 fig.add_trace(go.Scatter(x=anomaly_data.index, y=anomaly_data[sensor],285 mode='markers', name='Anomaly' if idx == 0 else None,286 marker=dict(color='red', size=8, symbol='x'),287 showlegend=(idx == 0)),288 row=idx+1, col=1)289 290 fig.update_layout(height=250*len(sensors), title_text="Interactive Timeline",291 showlegend=True, hovermode='x unified')292 return fig293 294 295def process_data(csv_file, use_sample, contamination):296 """Main processing function"""297 try:298 if use_sample or csv_file is None:299 df = generate_sample_data(30)300 status = "✅ Generated sample data (30 days)\n"301 else:302 df = pd.read_csv(csv_file.name)303 status = f"✅ Loaded: {csv_file.name}\n"304 305 df, missing, duplicates = validate_and_preprocess_data(df)306 status += f"✅ Preprocessed {len(df):,} records\n"307 status += f"📅 Range: {df.index.min()} to {df.index.max()}\n\n"308 309 df_features, sensors = engineer_features(df)310 status += f"✅ Features: {df_features.shape}\n\n"311 312 df_features, results, sensors, model_info = detect_anomalies(df_features, contamination)313 314 n_anomalies = df_features['ensemble_anomaly'].sum()315 status += f"🎯 RESULTS:\n"316 status += f" Total: {len(df_features):,}\n"317 status += f" Anomalies: {n_anomalies:,} ({n_anomalies/len(df_features)*100:.2f}%)\n\n"318 319 for name, info in model_info.items():320 status += f" {name}: {info['detected']} ({info['rate']:.1f}%)\n"321 322 img1 = create_overview_visualization(df_features, sensors)323 img2 = create_model_comparison(model_info, df_features)324 fig3 = create_interactive_timeline(df_features, sensors)325 326 output_columns = sensors + ['ensemble_anomaly']327 output_df = df_features[output_columns].copy()328 output_df.to_csv('results.csv')329 330 if n_anomalies > 0:331 summary = df_features[df_features['ensemble_anomaly'] == 1][sensors].head(20)332 summary_html = summary.to_html()333 else:334 summary_html = "<p>No anomalies detected</p>"335 336 return status, img1, img2, fig3, 'results.csv', summary_html337 338 except Exception as e:339 error_msg = f"❌ Error: {str(e)}"340 return error_msg, None, None, None, None, None341 342 343# Create Gradio Interface344with gr.Blocks(theme=gr.themes.Soft(), title="Smart Grid Anomaly Detection") as demo:345 346 gr.HTML("""347 <div style='text-align: center; padding: 30px; background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); 348 border-radius: 15px; margin-bottom: 20px;'>349 <h1 style='color: white; margin: 0; font-size: 36px;'>⚡ Smart Grid Anomaly Detection</h1>350 <p style='color: white; margin-top: 10px; font-size: 18px;'>351 Advanced ML-Powered Time Series Analysis352 </p>353 </div>354 """)355 356 with gr.Row():357 with gr.Column(scale=1):358 gr.Markdown("### 📁 Input Configuration")359 csv_file = gr.File(label="Upload CSV", file_types=[".csv"])360 use_sample = gr.Checkbox(label="Use Sample Data", value=True)361 contamination = gr.Slider(0.01, 0.20, value=0.05, step=0.01, 362 label="Expected Anomaly Rate")363 analyze_btn = gr.Button("🚀 Analyze", variant="primary", size="lg")364 365 gr.Markdown("""366 ### 📋 CSV Format367 **Required:**368 - timestamp369 - voltage370 - current 371 - frequency372 """)373 374 with gr.Column(scale=2):375 gr.Markdown("### 📊 Status")376 status_box = gr.Textbox(label="Analysis Status", lines=12)377 378 gr.Markdown("---")379 gr.Markdown("## 📈 Results")380 381 with gr.Tabs():382 with gr.Tab("📊 Time Series"):383 img_timeseries = gr.Image(label="Anomaly Detection")384 385 with gr.Tab("📊 Comparison"):386 img_comparison = gr.Image(label="Model Performance")387 388 with gr.Tab("🌐 Interactive"):389 plot_interactive = gr.Plot(label="Timeline")390 391 gr.Markdown("---")392 393 with gr.Row():394 with gr.Column():395 gr.Markdown("### 📄 Anomaly Summary")396 summary_html = gr.HTML()397 398 with gr.Column():399 gr.Markdown("### 💾 Download")400 output_csv = gr.File(label="Results CSV")401 402 analyze_btn.click(403 fn=process_data,404 inputs=[csv_file, use_sample, contamination],405 outputs=[status_box, img_timeseries, img_comparison, plot_interactive, output_csv, summary_html]406 )407 408 gr.HTML("""409 <div style='text-align: center; padding: 20px; margin-top: 20px; background: #f8f9fa; border-radius: 10px;'>410 <p style='color: #666;'>🔬 Powered by Machine Learning | Built with Gradio</p>411 </div>412 """)413 414if __name__ == "__main__":415 demo.launch()