astra08/data_visualization
0
1# ultimate_data_app.py
2import streamlit as st
3import pandas as pd
4import numpy as np
5import matplotlib.pyplot as plt
6import seaborn as sns
7import plotly.express as px
8import plotly.graph_objects as go
9import plotly.figure_factory as ff
10from plotly.subplots import make_subplots
11import io
12import base64
13import joblib
14import warnings
15from datetime import datetime
16import scipy.stats as stats
17from scipy import stats as scipy_stats
18
19# Machine Learning imports
20from sklearn.model_selection import train_test_split, cross_val_score
21from sklearn.preprocessing import StandardScaler, LabelEncoder
22from sklearn.linear_model import LinearRegression, LogisticRegression
23from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier, IsolationForest
24from sklearn.metrics import (mean_squared_error, r2_score, accuracy_score,
25 classification_report, confusion_matrix, f1_score)
26from sklearn.decomposition import PCA
27from sklearn.cluster import KMeans
28from sklearn.impute import SimpleImputer
29
30# Optional XGBoost with fallback
31try:
32 import xgboost as xgb
33 XGBOOST_AVAILABLE = True
34except ImportError:
35 XGBOOST_AVAILABLE = False
36 st.sidebar.warning("XGBoost not installed. Using alternative models.")
37
38warnings.filterwarnings('ignore')
39
40# Page configuration
41st.set_page_config(
42 page_title="Ultimate Data Analytics App",
43 page_icon="📊",
44 layout="wide",
45 initial_sidebar_state="expanded"
46)
47
48# Custom CSS for better styling
49st.markdown("""
50<style>
51 .main-header {
52 font-size: 2.5rem;
53 color: #1f77b4;
54 text-align: center;
55 margin-bottom: 2rem;
56 }
57 .section-header {
58 font-size: 1.5rem;
59 color: #2e86ab;
60 margin-top: 2rem;
61 margin-bottom: 1rem;
62 }
63 .metric-card {
64 background-color: #f8f9fa;
65 padding: 1rem;
66 border-radius: 0.5rem;
67 border-left: 4px solid #1f77b4;
68 margin-bottom: 1rem;
69 }
70 .success-box {
71 background-color: #d4edda;
72 border: 1px solid #c3e6cb;
73 border-radius: 0.5rem;
74 padding: 1rem;
75 margin: 1rem 0;
76 }
77 .warning-box {
78 background-color: #fff3cd;
79 border: 1px solid #ffeaa7;
80 border-radius: 0.5rem;
81 padding: 1rem;
82 margin: 1rem 0;
83 }
84</style>
85""", unsafe_allow_html=True)
86
87class DataAnalyzer:
88 """Class to handle data analysis operations"""
89
90 def __init__(self):
91 self.df = None
92 self.cleaned_df = None
93
94 def load_data(self, uploaded_file):
95 """Load data from uploaded file with automatic encoding detection"""
96 try:
97 if uploaded_file.name.endswith('.csv'):
98 # Try different encodings
99 encodings = ['utf-8', 'latin-1', 'iso-8859-1', 'cp1252']
100 for encoding in encodings:
101 try:
102 uploaded_file.seek(0)
103 self.df = pd.read_csv(uploaded_file, encoding=encoding)
104 break
105 except UnicodeDecodeError:
106 continue
107 else:
108 # If all encodings fail, try with error handling
109 try:
110 uploaded_file.seek(0)
111 self.df = pd.read_csv(uploaded_file, encoding='utf-8', errors='replace')
112 st.warning("File loaded with some character encoding issues")
113 except Exception as e:
114 st.error(f"Could not decode the file: {str(e)}")
115 return False
116
117 elif uploaded_file.name.endswith(('.xlsx', '.xls')):
118 self.df = pd.read_excel(uploaded_file)
119 elif uploaded_file.name.endswith('.json'):
120 self.df = pd.read_json(uploaded_file)
121 else:
122 st.error("Unsupported file format")
123 return False
124
125 self.cleaned_df = self.df.copy()
126 return True
127
128 except Exception as e:
129 st.error(f"Error loading file: {str(e)}")
130 return False
131
132 def get_data_info(self):
133 """Get comprehensive data information"""
134 info = {
135 'shape': self.df.shape,
136 'columns': list(self.df.columns),
137 'dtypes': self.df.dtypes.to_dict(),
138 'missing_values': self.df.isnull().sum().to_dict(),
139 'missing_percentage': (self.df.isnull().sum() / len(self.df) * 100).to_dict(),
140 'duplicates': self.df.duplicated().sum(),
141 'numeric_columns': self.df.select_dtypes(include=[np.number]).columns.tolist(),
142 'categorical_columns': self.df.select_dtypes(include=['object']).columns.tolist()
143 }
144 return info
145
146 def clean_data(self, operations):
147 """Perform data cleaning operations"""
148 df_copy = self.df.copy()
149
150 if operations.get('drop_na'):
151 df_copy = df_copy.dropna()
152 elif operations.get('fill_na'):
153 numeric_cols = df_copy.select_dtypes(include=[np.number]).columns
154 categorical_cols = df_copy.select_dtypes(include=['object']).columns
155
156 for col in numeric_cols:
157 if df_copy[col].isnull().any():
158 df_copy[col].fillna(df_copy[col].median(), inplace=True)
159
160 for col in categorical_cols:
161 if df_copy[col].isnull().any():
162 df_copy[col].fillna(df_copy[col].mode()[0] if not df_copy[col].mode().empty else 'Unknown', inplace=True)
163
164 if operations.get('drop_duplicates'):
165 df_copy = df_copy.drop_duplicates()
166
167 if operations.get('type_conversions'):
168 for col, new_type in operations['type_conversions'].items():
169 try:
170 if new_type == 'numeric':
171 df_copy[col] = pd.to_numeric(df_copy[col], errors='coerce')
172 elif new_type == 'datetime':
173 df_copy[col] = pd.to_datetime(df_copy[col], errors='coerce')
174 elif new_type == 'category':
175 df_copy[col] = df_copy[col].astype('category')
176 except Exception as e:
177 st.warning(f"Could not convert {col} to {new_type}: {str(e)}")
178
179 self.cleaned_df = df_copy
180 return df_copy
181
182def create_download_link(object_to_download, download_filename, download_link_text):
183 """Generate download link for various objects"""
184 if isinstance(object_to_download, pd.DataFrame):
185 object_to_download = object_to_download.to_csv(index=False)
186
187 b64 = base64.b64encode(object_to_download.encode()).decode()
188 return f'<a href="data:file/txt;base64,{b64}" download="{download_filename}">{download_link_text}</a>'
189
190def detect_task_type(df, target_column):
191 """Detect if the task is classification or regression"""
192 unique_values = df[target_column].nunique()
193
194 if df[target_column].dtype in ['object', 'category'] or unique_values < 10:
195 return 'classification'
196 else:
197 return 'regression'
198
199def main():
200 st.markdown('<h1 class="main-header">📊 Ultimate Data Analytics & ML Platform</h1>', unsafe_allow_html=True)
201
202 # Initialize session state
203 if 'analyzer' not in st.session_state:
204 st.session_state.analyzer = DataAnalyzer()
205 if 'data_loaded' not in st.session_state:
206 st.session_state.data_loaded = False
207 if 'theme' not in st.session_state:
208 st.session_state.theme = 'light'
209
210 # Sidebar
211 with st.sidebar:
212 st.title("⚙ Configuration")
213
214 # Theme selector
215 theme = st.selectbox("Theme", ["Light", "Dark"], index=0)
216 st.session_state.theme = theme.lower()
217
218 st.markdown("---")
219 st.header("Data Upload")
220
221 uploaded_file = st.file_uploader(
222 "Choose a file",
223 type=['csv', 'xlsx', 'xls', 'json'],
224 help="Upload CSV, Excel, or JSON files"
225 )
226
227 if uploaded_file is not None:
228 with st.spinner("Loading data..."):
229 if st.session_state.analyzer.load_data(uploaded_file):
230 st.session_state.data_loaded = True
231 st.success("Data loaded successfully!")
232 else:
233 st.session_state.data_loaded = False
234
235 # Main content area with tabs
236 if st.session_state.data_loaded:
237 tab1, tab2, tab3, tab4, tab5, tab6, tab7 = st.tabs([
238 "🏠 Home", "🧹 Data Cleaning", "🔍 EDA",
239 "📈 Visualization", "🤖 ML Models", "🔬 Advanced", "📤 Export"
240 ])
241
242 analyzer = st.session_state.analyzer
243 df = analyzer.df
244 cleaned_df = analyzer.cleaned_df
245
246 with tab1:
247 st.markdown('<h2 class="section-header">Dataset Overview</h2>', unsafe_allow_html=True)
248
249 # Basic info cards
250 col1, col2, col3, col4 = st.columns(4)
251 with col1:
252 st.metric("Rows", df.shape[0])
253 with col2:
254 st.metric("Columns", df.shape[1])
255 with col3:
256 st.metric("Missing Values", df.isnull().sum().sum())
257 with col4:
258 st.metric("Duplicate Rows", df.duplicated().sum())
259
260 # Data preview
261 st.subheader("Data Preview")
262 st.dataframe(df.head(), use_container_width=True)
263
264 # Data information
265 col1, col2 = st.columns(2)
266
267 with col1:
268 st.subheader("Data Types")
269 dtype_info = pd.DataFrame({
270 'Column': df.columns,
271 'Data Type': df.dtypes.values,
272 'Non-Null Count': df.notnull().sum().values
273 })
274 st.dataframe(dtype_info, use_container_width=True)
275
276 with col2:
277 st.subheader("Missing Values")
278 missing_info = pd.DataFrame({
279 'Column': df.columns,
280 'Missing Count': df.isnull().sum().values,
281 'Missing %': (df.isnull().sum() / len(df) * 100).round(2)
282 })
283 st.dataframe(missing_info, use_container_width=True)
284
285 # Summary statistics
286 st.subheader("Summary Statistics")
287 st.dataframe(df.describe(), use_container_width=True)
288
289 with tab2:
290 st.markdown('<h2 class="section-header">Data Cleaning & Preprocessing</h2>', unsafe_allow_html=True)
291
292 cleaning_operations = {}
293
294 col1, col2 = st.columns(2)
295
296 with col1:
297 st.subheader("Missing Values")
298 missing_option = st.radio(
299 "Handle missing values:",
300 ["Keep as is", "Drop rows with NA", "Fill missing values"]
301 )
302
303 if missing_option == "Drop rows with NA":
304 cleaning_operations['drop_na'] = True
305 elif missing_option == "Fill missing values":
306 cleaning_operations['fill_na'] = True
307
308 with col2:
309 st.subheader("Duplicates")
310 dup_option = st.radio(
311 "Handle duplicates:",
312 ["Keep duplicates", "Remove duplicates"]
313 )
314 if dup_option == "Remove duplicates":
315 cleaning_operations['drop_duplicates'] = True
316
317 # Data type conversions
318 st.subheader("Data Type Conversions")
319 type_conversions = {}
320 for col in df.columns:
321 col1, col2 = st.columns([2, 1])
322 with col1:
323 st.text(f"{col} (current: {df[col].dtype})")
324 with col2:
325 new_type = st.selectbox(
326 f"Type for {col}",
327 ["Keep original", "numeric", "datetime", "category"],
328 key=f"type_{col}"
329 )
330 if new_type != "Keep original":
331 type_conversions[col] = new_type
332
333 if type_conversions:
334 cleaning_operations['type_conversions'] = type_conversions
335
336 if st.button("Apply Cleaning Operations", type="primary"):
337 with st.spinner("Cleaning data..."):
338 cleaned_df = analyzer.clean_data(cleaning_operations)
339
340 # Show before/after comparison
341 col1, col2 = st.columns(2)
342
343 with col1:
344 st.subheader("Before Cleaning")
345 st.metric("Total Rows", df.shape[0])
346 st.metric("Missing Values", df.isnull().sum().sum())
347 st.metric("Duplicates", df.duplicated().sum())
348
349 with col2:
350 st.subheader("After Cleaning")
351 st.metric("Total Rows", cleaned_df.shape[0])
352 st.metric("Missing Values", cleaned_df.isnull().sum().sum())
353 st.metric("Duplicates", cleaned_df.duplicated().sum())
354
355 st.success("Data cleaning completed!")
356
357 with tab3:
358 st.markdown('<h2 class="section-header">Exploratory Data Analysis</h2>', unsafe_allow_html=True)
359
360 # Auto EDA configuration
361 st.subheader("Auto EDA Configuration")
362 target_col = st.selectbox("Select target column (optional)", [""] + list(cleaned_df.columns))
363
364 col1, col2 = st.columns(2)
365
366 with col1:
367 numeric_cols = cleaned_df.select_dtypes(include=[np.number]).columns.tolist()
368 selected_numeric = st.multiselect(
369 "Select numeric columns for analysis",
370 numeric_cols,
371 default=numeric_cols[:min(5, len(numeric_cols))]
372 )
373
374 with col2:
375 categorical_cols = cleaned_df.select_dtypes(include=['object', 'category']).columns.tolist()
376 selected_categorical = st.multiselect(
377 "Select categorical columns for analysis",
378 categorical_cols,
379 default=categorical_cols[:min(3, len(categorical_cols))]
380 )
381
382 if st.button("Generate EDA Report"):
383 with st.spinner("Generating comprehensive EDA..."):
384
385 # Distribution plots
386 if selected_numeric:
387 st.subheader("Distribution Analysis")
388
389 for col in selected_numeric[:4]: # Limit to first 4 columns
390 fig = px.histogram(
391 cleaned_df, x=col,
392 title=f"Distribution of {col}",
393 marginal="box"
394 )
395 st.plotly_chart(fig, use_container_width=True)
396
397 # Categorical analysis
398 if selected_categorical:
399 st.subheader("Categorical Analysis")
400
401 for col in selected_categorical[:3]:
402 value_counts = cleaned_df[col].value_counts().head(10)
403 fig = px.bar(
404 x=value_counts.index,
405 y=value_counts.values,
406 title=f"Top 10 values in {col}",
407 labels={'x': col, 'y': 'Count'}
408 )
409 st.plotly_chart(fig, use_container_width=True)
410
411 # Correlation matrix
412 if len(selected_numeric) > 1:
413 st.subheader("Correlation Matrix")
414 corr_matrix = cleaned_df[selected_numeric].corr()
415
416 fig = px.imshow(
417 corr_matrix,
418 title="Correlation Heatmap",
419 aspect="auto",
420 color_continuous_scale='RdBu_r'
421 )
422 st.plotly_chart(fig, use_container_width=True)
423
424 # Outlier detection
425 if selected_numeric:
426 st.subheader("Outlier Detection")
427
428 for col in selected_numeric[:3]:
429 fig = px.box(cleaned_df, y=col, title=f"Boxplot of {col}")
430 st.plotly_chart(fig, use_container_width=True)
431
432 with tab4:
433 st.markdown('<h2 class="section-header">Interactive Visualization</h2>', unsafe_allow_html=True)
434
435 # Chart type selection
436 chart_type = st.selectbox(
437 "Select Chart Type",
438 ["Scatter Plot", "Line Chart", "Bar Chart", "Histogram",
439 "Box Plot", "Violin Plot", "Pie Chart", "Area Chart", "Heatmap"]
440 )
441
442 col1, col2 = st.columns(2)
443
444 with col1:
445 x_axis = st.selectbox("X-axis", [""] + list(cleaned_df.columns))
446 y_axis = st.selectbox("Y-axis", [""] + list(cleaned_df.columns))
447
448 with col2:
449 color_by = st.selectbox("Color by", [""] + list(cleaned_df.columns))
450 facet_col = st.selectbox("Facet by", [""] + list(cleaned_df.columns))
451
452 # Advanced options
453 with st.expander("Advanced Options"):
454 col1, col2 = st.columns(2)
455 with col1:
456 theme = st.selectbox("Plot Theme", ["plotly", "plotly_white", "plotly_dark"])
457 show_grid = st.checkbox("Show grid", True)
458 with col2:
459 width = st.slider("Plot width", 600, 1200, 800)
460 height = st.slider("Plot height", 400, 800, 500)
461
462 if st.button("Generate Visualization"):
463 if chart_type == "Scatter Plot" and x_axis and y_axis:
464 fig = px.scatter(
465 cleaned_df, x=x_axis, y=y_axis, color=color_by,
466 title=f"Scatter Plot: {x_axis} vs {y_axis}",
467 template=theme,
468 width=width,
469 height=height
470 )
471 if show_grid:
472 fig.update_xaxis(showgrid=True)
473 fig.update_yaxis(showgrid=True)
474 st.plotly_chart(fig, use_container_width=True)
475
476 elif chart_type == "Line Chart" and x_axis and y_axis:
477 fig = px.line(
478 cleaned_df, x=x_axis, y=y_axis, color=color_by,
479 title=f"Line Chart: {y_axis} over {x_axis}",
480 template=theme
481 )
482 st.plotly_chart(fig, use_container_width=True)
483
484 elif chart_type == "Bar Chart" and x_axis:
485 if y_axis:
486 fig = px.bar(cleaned_df, x=x_axis, y=y_axis, color=color_by)
487 else:
488 value_counts = cleaned_df[x_axis].value_counts().head(20)
489 fig = px.bar(x=value_counts.index, y=value_counts.values)
490 st.plotly_chart(fig, use_container_width=True)
491
492 elif chart_type == "Histogram" and x_axis:
493 fig = px.histogram(cleaned_df, x=x_axis, color=color_by)
494 st.plotly_chart(fig, use_container_width=True)
495
496 elif chart_type == "Box Plot" and y_axis:
497 fig = px.box(cleaned_df, x=x_axis, y=y_axis, color=color_by)
498 st.plotly_chart(fig, use_container_width=True)
499
500 elif chart_type == "Violin Plot" and y_axis:
501 fig = px.violin(cleaned_df, x=x_axis, y=y_axis, color=color_by)
502 st.plotly_chart(fig, use_container_width=True)
503
504 elif chart_type == "Pie Chart" and x_axis:
505 value_counts = cleaned_df[x_axis].value_counts().head(10)
506 fig = px.pie(values=value_counts.values, names=value_counts.index)
507 st.plotly_chart(fig, use_container_width=True)
508
509 elif chart_type == "Heatmap":
510 numeric_cols = cleaned_df.select_dtypes(include=[np.number]).columns
511 if len(numeric_cols) > 1:
512 corr_matrix = cleaned_df[numeric_cols].corr()
513 fig = px.imshow(corr_matrix, color_continuous_scale='RdBu_r')
514 st.plotly_chart(fig, use_container_width=True)
515
516 with tab5:
517 st.markdown('<h2 class="section-header">Machine Learning Models</h2>', unsafe_allow_html=True)
518
519 if len(cleaned_df.columns) < 2:
520 st.warning("Need at least 2 columns for ML modeling")
521 else:
522 # Model configuration
523 col1, col2 = st.columns(2)
524
525 with col1:
526 target_column = st.selectbox(
527 "Select target variable",
528 cleaned_df.columns,
529 key="ml_target"
530 )
531
532 with col2:
533 test_size = st.slider("Test set size (%)", 10, 40, 20)
534
535 # Feature selection
536 feature_columns = st.multiselect(
537 "Select features for modeling",
538 [col for col in cleaned_df.columns if col != target_column],
539 default=[col for col in cleaned_df.columns if col != target_column][:min(5, len(cleaned_df.columns)-1)]
540 )
541
542 if target_column and feature_columns:
543 # Prepare data
544 X = cleaned_df[feature_columns]
545 y = cleaned_df[target_column]
546
547 # Handle categorical variables
548 categorical_cols = X.select_dtypes(include=['object', 'category']).columns
549 if len(categorical_cols) > 0:
550 X = pd.get_dummies(X, columns=categorical_cols, drop_first=True)
551
552 # Handle missing values
553 imputer = SimpleImputer(strategy='median')
554 X = pd.DataFrame(imputer.fit_transform(X), columns=X.columns)
555
556 # Detect task type
557 task_type = detect_task_type(cleaned_df, target_column)
558 st.info(f"Detected task type: {task_type.upper()}")
559
560 # Model selection
561 if task_type == 'classification':
562 models = {
563 'Logistic Regression': LogisticRegression(),
564 'Random Forest': RandomForestClassifier(n_estimators=100, random_state=42)
565 }
566 if XGBOOST_AVAILABLE:
567 models['XGBoost'] = xgb.XGBClassifier(random_state=42)
568 else:
569 models = {
570 'Linear Regression': LinearRegression(),
571 'Random Forest': RandomForestRegressor(n_estimators=100, random_state=42)
572 }
573 if XGBOOST_AVAILABLE:
574 models['XGBoost'] = xgb.XGBRegressor(random_state=42)
575
576 selected_model_name = st.selectbox("Select Model", list(models.keys()))
577
578 if st.button("Train Model"):
579 with st.spinner("Training model..."):
580 # Split data
581 X_train, X_test, y_train, y_test = train_test_split(
582 X, y, test_size=test_size/100, random_state=42
583 )
584
585 # Scale features for linear models
586 if selected_model_name in ['Logistic Regression', 'Linear Regression']:
587 scaler = StandardScaler()
588 X_train = scaler.fit_transform(X_train)
589 X_test = scaler.transform(X_test)
590
591 # Train model
592 model = models[selected_model_name]
593 model.fit(X_train, y_train)
594 y_pred = model.predict(X_test)
595
596 # Evaluate model
597 st.subheader("Model Evaluation")
598
599 if task_type == 'classification':
600 accuracy = accuracy_score(y_test, y_pred)
601 f1 = f1_score(y_test, y_pred, average='weighted')
602
603 col1, col2, col3 = st.columns(3)
604 with col1:
605 st.metric("Accuracy", f"{accuracy:.3f}")
606 with col2:
607 st.metric("F1 Score", f"{f1:.3f}")
608
609 # Confusion matrix
610 st.subheader("Confusion Matrix")
611 cm = confusion_matrix(y_test, y_pred)
612 fig = px.imshow(
613 cm,
614 text_auto=True,
615 title="Confusion Matrix"
616 )
617 st.plotly_chart(fig, use_container_width=True)
618
619 else: # regression
620 mse = mean_squared_error(y_test, y_pred)
621 r2 = r2_score(y_test, y_pred)
622
623 col1, col2, col3 = st.columns(3)
624 with col1:
625 st.metric("MSE", f"{mse:.3f}")
626 with col2:
627 st.metric("R² Score", f"{r2:.3f}")
628
629 # Prediction vs Actual plot
630 st.subheader("Prediction vs Actual")
631 fig = px.scatter(
632 x=y_test, y=y_pred,
633 labels={'x': 'Actual', 'y': 'Predicted'},
634 title="Predicted vs Actual Values"
635 )
636 fig.add_trace(go.Scatter(
637 x=[y_test.min(), y_test.max()],
638 y=[y_test.min(), y_test.max()],
639 mode='lines',
640 line=dict(dash='dash', color='red'),
641 name='Perfect Prediction'
642 ))
643 st.plotly_chart(fig, use_container_width=True)
644
645 # Feature importance
646 if hasattr(model, 'feature_importances_'):
647 st.subheader("Feature Importance")
648 importance_df = pd.DataFrame({
649 'feature': X.columns,
650 'importance': model.feature_importances_
651 }).sort_values('importance', ascending=True)
652
653 fig = px.bar(
654 importance_df.tail(15),
655 x='importance',
656 y='feature',
657 orientation='h',
658 title="Top 15 Feature Importances"
659 )
660 st.plotly_chart(fig, use_container_width=True)
661
662 # Save model
663 model_bytes = io.BytesIO()
664 joblib.dump(model, model_bytes)
665 model_bytes.seek(0)
666
667 st.download_button(
668 label="Download Trained Model",
669 data=model_bytes,
670 file_name=f"{selected_model_name}_{task_type}.joblib",
671 mime="application/octet-stream"
672 )
673
674 with tab6:
675 st.markdown('<h2 class="section-header">Advanced Analytics</h2>', unsafe_allow_html=True)
676
677 advanced_method = st.selectbox(
678 "Select Advanced Method",
679 ["PCA - Dimensionality Reduction", "KMeans Clustering", "Outlier Detection", "Statistical Tests"]
680 )
681
682 if advanced_method == "PCA - Dimensionality Reduction":
683 st.subheader("Principal Component Analysis")
684
685 numeric_cols = cleaned_df.select_dtypes(include=[np.number]).columns.tolist()
686 if len(numeric_cols) < 2:
687 st.warning("Need at least 2 numeric columns for PCA")
688 else:
689 selected_features = st.multiselect(
690 "Select features for PCA",
691 numeric_cols,
692 default=numeric_cols[:min(10, len(numeric_cols))]
693 )
694
695 n_components = st.slider("Number of components", 2, 10, 3)
696
697 if st.button("Run PCA"):
698 with st.spinner("Performing PCA..."):
699 X_pca = cleaned_df[selected_features].dropna()
700
701 # Standardize features
702 scaler = StandardScaler()
703 X_scaled = scaler.fit_transform(X_pca)
704
705 # Perform PCA
706 pca = PCA(n_components=n_components)
707 principal_components = pca.fit_transform(X_scaled)
708
709 # Create results DataFrame
710 pca_df = pd.DataFrame(
711 data=principal_components,
712 columns=[f'PC{i+1}' for i in range(n_components)]
713 )
714
715 # Explained variance
716 st.subheader("Explained Variance")
717 explained_var = pca.explained_variance_ratio_
718
719 fig = px.bar(
720 x=[f'PC{i+1}' for i in range(n_components)],
721 y=explained_var,
722 title="Explained Variance by Principal Components"
723 )
724 st.plotly_chart(fig, use_container_width=True)
725
726 # PCA plot
727 st.subheader("PCA Visualization")
728
729 if n_components >= 2:
730 color_by = st.selectbox(
731 "Color points by",
732 [""] + list(cleaned_df.columns),
733 key="pca_color"
734 )
735
736 if n_components >= 3:
737 dim_choice = st.radio(
738 "Plot dimensions",
739 ["2D", "3D"]
740 )
741 else:
742 dim_choice = "2D"
743
744 if dim_choice == "2D":
745 fig = px.scatter(
746 pca_df, x='PC1', y='PC2',
747 color=cleaned_df[color_by] if color_by else None,
748 title="PCA - First Two Components"
749 )
750 else:
751 fig = px.scatter_3d(
752 pca_df, x='PC1', y='PC2', z='PC3',
753 color=cleaned_df[color_by] if color_by else None,
754 title="PCA - First Three Components"
755 )
756 st.plotly_chart(fig, use_container_width=True)
757
758 elif advanced_method == "KMeans Clustering":
759 st.subheader("K-Means Clustering")
760
761 numeric_cols = cleaned_df.select_dtypes(include=[np.number]).columns.tolist()
762 if len(numeric_cols) < 2:
763 st.warning("Need at least 2 numeric columns for clustering")
764 else:
765 selected_features = st.multiselect(
766 "Select features for clustering",
767 numeric_cols,
768 default=numeric_cols[:2],
769 key="cluster_features"
770 )
771
772 n_clusters = st.slider("Number of clusters", 2, 10, 3)
773
774 if st.button("Perform Clustering"):
775 with st.spinner("Clustering data..."):
776 X_cluster = cleaned_df[selected_features].dropna()
777
778 # Standardize features
779 scaler = StandardScaler()
780 X_scaled = scaler.fit_transform(X_cluster)
781
782 # Perform KMeans
783 kmeans = KMeans(n_clusters=n_clusters, random_state=42)
784 clusters = kmeans.fit_predict(X_scaled)
785
786 # Add clusters to dataframe
787 cluster_df = X_cluster.copy()
788 cluster_df['Cluster'] = clusters
789
790 # Plot clusters
791 if len(selected_features) >= 2:
792 fig = px.scatter(
793 cluster_df,
794 x=selected_features[0],
795 y=selected_features[1],
796 color='Cluster',
797 title=f"K-Means Clustering (k={n_clusters})"
798 )
799 st.plotly_chart(fig, use_container_width=True)
800
801 elif advanced_method == "Outlier Detection":
802 st.subheader("Outlier Detection using Isolation Forest")
803
804 numeric_cols = cleaned_df.select_dtypes(include=[np.number]).columns.tolist()
805 if len(numeric_cols) < 1:
806 st.warning("Need numeric columns for outlier detection")
807 else:
808 selected_features = st.multiselect(
809 "Select features for outlier detection",
810 numeric_cols,
811 default=numeric_cols[:min(5, len(numeric_cols))],
812 key="outlier_features"
813 )
814
815 contamination = st.slider("Expected outlier fraction", 0.01, 0.5, 0.1)
816
817 if st.button("Detect Outliers"):
818 with st.spinner("Detecting outliers..."):
819 X_outlier = cleaned_df[selected_features].dropna()
820
821 # Fit Isolation Forest
822 iso_forest = IsolationForest(contamination=contamination, random_state=42)
823 outliers = iso_forest.fit_predict(X_outlier)
824
825 # Create results
826 outlier_df = X_outlier.copy()
827 outlier_df['Is_Outlier'] = outliers
828 outlier_df['Is_Outlier'] = outlier_df['Is_Outlier'].map({1: 'Normal', -1: 'Outlier'})
829
830 st.metric("Outliers detected", f"{(outliers == -1).sum()} ({(outliers == -1).mean()*100:.1f}%)")
831
832 # Plot outliers
833 if len(selected_features) >= 2:
834 fig = px.scatter(
835 outlier_df,
836 x=selected_features[0],
837 y=selected_features[1],
838 color='Is_Outlier',
839 title="Outlier Detection Results"
840 )
841 st.plotly_chart(fig, use_container_width=True)
842
843 with tab7:
844 st.markdown('<h2 class="section-header">Export Results</h2>', unsafe_allow_html=True)
845
846 col1, col2 = st.columns(2)
847
848 with col1:
849 st.subheader("Export Data")
850
851 # Download cleaned data
852 csv = cleaned_df.to_csv(index=False)
853 st.download_button(
854 label="Download Cleaned Data as CSV",
855 data=csv,
856 file_name="cleaned_data.csv",
857 mime="text/csv"
858 )
859
860 # Download summary report
861 report_text = f"""
862 Data Analysis Report
863 Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
864
865 Original Data Shape: {df.shape}
866 Cleaned Data Shape: {cleaned_df.shape}
867
868 Missing Values Handled: {df.isnull().sum().sum() - cleaned_df.isnull().sum().sum()}
869 Duplicates Removed: {df.duplicated().sum() - cleaned_df.duplicated().sum()}
870
871 Columns:
872 {', '.join(cleaned_df.columns.tolist())}
873 """
874
875 st.download_button(
876 label="Download Summary Report",
877 data=report_text,
878 file_name="data_analysis_report.txt",
879 mime="text/plain"
880 )
881
882 with col2:
883 st.subheader("ML Model Results")
884 st.info("Trained models can be downloaded from the ML Models tab")
885
886 st.subheader("Generate Comprehensive EDA Report")
887 if st.button("Create Full EDA Report"):
888 with st.spinner("Generating comprehensive report..."):
889 # This would typically generate a more comprehensive HTML report
890 # For simplicity, we'll create a detailed text report
891
892 report_content = f"""
893 COMPREHENSIVE EDA REPORT
894 ========================
895
896 Dataset: {uploaded_file.name}
897 Analysis Date: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
898
899 1. DATA OVERVIEW
900 ----------------
901 - Original shape: {df.shape}
902 - Cleaned shape: {cleaned_df.shape}
903 - Total missing values: {df.isnull().sum().sum()}
904 - Total duplicates: {df.duplicated().sum()}
905
906 2. COLUMN SUMMARY
907 ----------------
908 """
909
910 for col in cleaned_df.columns:
911 report_content += f"""
912 {col}:
913 - Data type: {cleaned_df[col].dtype}
914 - Missing values: {cleaned_df[col].isnull().sum()}
915 - Unique values: {cleaned_df[col].nunique()}
916 """
917
918 if cleaned_df[col].dtype in ['int64', 'float64']:
919 report_content += f"""
920 - Mean: {cleaned_df[col].mean():.2f}
921 - Std: {cleaned_df[col].std():.2f}
922 - Min: {cleaned_df[col].min():.2f}
923 - Max: {cleaned_df[col].max():.2f}
924 """
925
926 st.download_button(
927 label="Download Full EDA Report",
928 data=report_content,
929 file_name="full_eda_report.txt",
930 mime="text/plain"
931 )
932
933 else:
934 # Welcome screen when no data is loaded
935 st.markdown("""
936 <div style='text-align: center; padding: 5rem 0;'>
937 <h1>🚀 Welcome to Ultimate Data Analytics</h1>
938 <p style='font-size: 1.2rem; color: #666; margin-bottom: 3rem;'>
939 Upload your dataset to unlock powerful analytics and machine learning capabilities
940 </p>
941 </div>
942 """, unsafe_allow_html=True)
943
944 col1, col2, col3 = st.columns(3)
945
946 with col1:
947 st.markdown("""
948 ### 📁 Data Upload
949 - Support for CSV, Excel, JSON
950 - Automatic encoding detection
951 - Handle large datasets efficiently
952 """)
953
954 with col2:
955 st.markdown("""
956 ### 🔍 Advanced Analytics
957 - Interactive visualizations
958 - Statistical analysis
959 - Machine learning models
960 - Clustering & PCA
961 """)
962
963 with col3:
964 st.markdown("""
965 ### 📊 Visualization
966 - Multiple chart types
967 - Customizable themes
968 - Export capabilities
969 - Real-time updates
970 """)
971
972 st.markdown("---")
973
974 # Sample data option
975 st.subheader("Quick Start with Sample Data")
976 if st.button("Load Sample Dataset"):
977 # Create sample data
978 np.random.seed(42)
979 sample_data = pd.DataFrame({
980 'Age': np.random.randint(18, 65, 1000),
981 'Income': np.random.normal(50000, 15000, 1000),
982 'Education': np.random.choice(['High School', 'Bachelor', 'Master', 'PhD'], 1000),
983 'Score': np.random.normal(75, 15, 1000),
984 'Category': np.random.choice(['A', 'B', 'C'], 1000),
985 'Satisfaction': np.random.randint(1, 6, 1000)
986 })
987
988 # Save sample data to buffer and load
989 buffer = io.BytesIO()
990 sample_data.to_csv(buffer, index=False)
991 buffer.seek(0)
992
993 if st.session_state.analyzer.load_data(buffer):
994 st.session_state.data_loaded = True
995 st.rerun()
996
997if __name__ == "__main__":
998 main()