nsrawat/data-cleaning-visualization
1
1"""
2Gradio App for Google Analytics Data Cleaning & Visualization
3Mirrors the functionality of streamlit_app.py
4Deploy on Hugging Face Spaces
5"""
6
7import gradio as gr
8import pandas as pd
9import numpy as np
10import matplotlib
11matplotlib.use('Agg')
12import matplotlib.pyplot as plt
13import seaborn as sns
14import tempfile
15import os
16from datetime import datetime
17
18# --- DataCleaningAnalysis class (embedded for HF Spaces) ---
19class DataCleaningAnalysis:
20 def __init__(self, data_path='data/raw/google_analytics_export.csv'):
21 self.data_path = data_path
22 self.df = None
23 self.cleaned_df = None
24
25 def load_data(self):
26 self.df = pd.read_csv(self.data_path)
27 return self.df
28
29 def initial_assessment(self):
30 pass
31
32 def remove_duplicates(self):
33 self.df = self.df.drop_duplicates()
34 return self.df
35
36 def handle_missing_values(self):
37 for col in self.df.columns:
38 if self.df[col].isnull().sum() > 0:
39 if self.df[col].dtype in ['float64', 'int64']:
40 self.df[col].fillna(self.df[col].median(), inplace=True)
41 else:
42 self.df[col].fillna('Unknown', inplace=True)
43 return self.df
44
45 def remove_bot_traffic(self):
46 user_agent_col = None
47 for col in self.df.columns:
48 if 'user_agent' in col.lower() or 'useragent' in col.lower():
49 user_agent_col = col
50 break
51
52 if user_agent_col:
53 bot_keywords = ['bot', 'crawler', 'spider', 'googlebot']
54 mask = ~self.df[user_agent_col].astype(str).str.lower().str.contains('|'.join(bot_keywords), na=False)
55 self.df = self.df[mask]
56 return self.df
57
58 def normalize_timezones(self):
59 if 'timestamp' in self.df.columns:
60 self.df['timestamp'] = pd.to_datetime(self.df['timestamp'], utc=True)
61 return self.df
62
63 def standardize_columns(self):
64 self.df.columns = self.df.columns.str.lower().str.replace(' ', '_')
65 for col in self.df.select_dtypes(include='object').columns:
66 self.df[col] = self.df[col].str.strip().str.title()
67 return self.df
68
69 def quality_report(self):
70 return self.df
71
72 def generate_quality_report_csv(self, output_path='quality_report.csv'):
73 quality_metrics = []
74 total_rows = len(self.df)
75 total_cols = len(self.df.columns)
76 missing_total = self.df.isnull().sum().sum()
77 duplicates = self.df.duplicated().sum()
78
79 quality_metrics.append({'metric': 'Total Records', 'value': total_rows, 'percentage': 100.0, 'status': 'PASS'})
80 quality_metrics.append({'metric': 'Total Columns', 'value': total_cols, 'percentage': 100.0, 'status': 'PASS'})
81 quality_metrics.append({'metric': 'Missing Values', 'value': missing_total, 'percentage': (missing_total / (total_rows * total_cols)) * 100 if total_rows > 0 else 0, 'status': 'PASS' if (missing_total / (total_rows * total_cols)) * 100 < 1 else 'WARNING'})
82 quality_metrics.append({'metric': 'Duplicate Rows', 'value': duplicates, 'percentage': (duplicates / total_rows) * 100 if total_rows > 0 else 0, 'status': 'PASS' if duplicates == 0 else 'FAIL'})
83
84 for col in self.df.columns:
85 missing_count = self.df[col].isnull().sum()
86 missing_pct = (missing_count / total_rows) * 100 if total_rows > 0 else 0
87 quality_metrics.append({'metric': f'Column: {col}', 'value': f'Missing: {missing_count}', 'percentage': missing_pct, 'status': 'PASS' if missing_pct < 5 else 'WARNING' if missing_pct < 20 else 'FAIL'})
88
89 passed = sum(1 for m in quality_metrics if m['status'] == 'PASS')
90 total_checks = len(quality_metrics)
91 quality_score = (passed / total_checks) * 100 if total_checks > 0 else 0
92 quality_metrics.append({'metric': 'Overall Quality Score', 'value': f'{quality_score:.2f}%', 'percentage': quality_score, 'status': 'PASS' if quality_score >= 95 else 'WARNING' if quality_score >= 80 else 'FAIL'})
93
94 report_df = pd.DataFrame(quality_metrics)
95 report_df.to_csv(output_path, index=False)
96 return report_df
97
98# --- Home Tab ---
99def home_content():
100 return """
101# 📊 Google Analytics Data Cleaning & Visualization
102
103Professional data wrangling & business intelligence project
104
105### Key Features:
106- ✅ Data cleaning pipeline
107- ✅ Duplicate removal
108- ✅ Missing value handling
109- ✅ Bot traffic detection
110- ✅ Interactive visualizations
111- ✅ Data quality reports
112
113### Dataset:
114- **150,000+** user session events
115- **85 MB** raw data
116- **99%+** data accuracy
117"""
118
119# --- Data Overview Tab ---
120def data_overview(file):
121 if file is None:
122 return None, "", "", "", ""
123
124 df = pd.read_csv(file.name)
125
126 total_records = f"{len(df):,}"
127 total_columns = str(len(df.columns))
128 memory_usage = f"{df.memory_usage(deep=True).sum() / 1024**2:.2f} MB"
129 missing_values = f"{df.isnull().sum().sum():,}"
130
131 preview = df.head(10)
132
133 return preview, total_records, total_columns, memory_usage, missing_values
134
135def data_types_info(file):
136 if file is None:
137 return None
138
139 df = pd.read_csv(file.name)
140 info_df = pd.DataFrame({
141 'Column': df.columns,
142 'Type': df.dtypes.astype(str),
143 'Non-Null Count': df.count().values,
144 'Null Count': df.isnull().sum().values
145 })
146 return info_df
147
148# --- Data Cleaning Tab ---
149def run_cleaning_pipeline(file):
150 if file is None:
151 return None, None, "", ""
152
153 with tempfile.NamedTemporaryFile(delete=False, suffix='.csv') as tmp_input:
154 with open(file.name, 'rb') as f:
155 tmp_input.write(f.read())
156 tmp_input_path = tmp_input.name
157
158 tmp_cleaned_path = tempfile.mktemp(suffix='_cleaned.csv')
159 tmp_quality_path = tempfile.mktemp(suffix='_quality_report.csv')
160
161 try:
162 cleaner = DataCleaningAnalysis(data_path=tmp_input_path)
163 cleaner.load_data()
164 original_count = len(cleaner.df)
165
166 cleaner.initial_assessment()
167 cleaner.remove_duplicates()
168 cleaner.handle_missing_values()
169 cleaner.remove_bot_traffic()
170 cleaner.normalize_timezones()
171 cleaner.standardize_columns()
172 cleaner.quality_report()
173
174 cleaner.cleaned_df = cleaner.df
175 cleaner.cleaned_df.to_csv(tmp_cleaned_path, index=False)
176 cleaner.generate_quality_report_csv(output_path=tmp_quality_path)
177
178 cleaned_count = len(cleaner.cleaned_df)
179
180 return tmp_cleaned_path, tmp_quality_path, f"{original_count:,}", f"{cleaned_count:,}"
181 except Exception as e:
182 return None, None, f"Error: {str(e)}", ""
183 finally:
184 if os.path.exists(tmp_input_path):
185 os.unlink(tmp_input_path)
186
187# --- Visualizations Tab ---
188def get_numeric_columns(file):
189 if file is None:
190 return gr.update(choices=[], value=None)
191
192 df = pd.read_csv(file.name)
193 numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
194
195 if len(numeric_cols) > 0:
196 return gr.update(choices=numeric_cols, value=numeric_cols[0])
197 return gr.update(choices=[], value=None)
198
199def visualize_column(file, column):
200 if file is None or column is None:
201 return None, "", "", "", ""
202
203 df = pd.read_csv(file.name)
204
205 if column not in df.columns:
206 return None, "", "", "", ""
207
208 fig, ax = plt.subplots(figsize=(10, 6))
209 df[column].hist(bins=50, ax=ax, edgecolor='black', color='#1f77b4')
210 ax.set_title(f'Distribution of {column}', fontsize=14)
211 ax.set_xlabel(column)
212 ax.set_ylabel('Frequency')
213 plt.tight_layout()
214
215 mean_val = f"{df[column].mean():.2f}"
216 median_val = f"{df[column].median():.2f}"
217 std_val = f"{df[column].std():.2f}"
218 min_max_val = f"{df[column].min():.2f} / {df[column].max():.2f}"
219
220 return fig, mean_val, median_val, std_val, min_max_val
221
222# --- Quality Report Tab ---
223def display_quality_report(file):
224 if file is None:
225 return None, "", "", ""
226
227 df = pd.read_csv(file.name)
228
229 passed = len(df[df['status'] == 'PASS'])
230 warnings = len(df[df['status'] == 'WARNING'])
231 failed = len(df[df['status'] == 'FAIL'])
232
233 return df, str(passed), str(warnings), str(failed)
234
235# --- Footer ---
236footer_html = """
237<div style="text-align: center; padding: 20px; margin-top: 20px; border-top: 1px solid #ddd;">
238 <h4>💰 You can help me by Donating</h4>
239 <div style="display: flex; justify-content: center; gap: 20px; margin: 15px 0;">
240 <a href="https://www.buymeacoffee.com/nsrawat?ref=HuggingFace" target="_blank" style="background-color: #FFDD00; color: #000; padding: 12px 24px; border-radius: 5px; text-decoration: none; font-weight: bold;">☕ Buy Me a Coffee</a>
241 <a href="https://paypal.me/NRawat710?ref=HuggingFace" target="_blank" style="background-color: #00457C; color: #fff; padding: 12px 24px; border-radius: 5px; text-decoration: none; font-weight: bold;">💳 PayPal</a>
242 <a href="https://withupi.com/@nsrawat?ref=HuggingFace" target="_blank" style="background-color: #4CAF50; color: #fff; padding: 12px 24px; border-radius: 5px; text-decoration: none; font-weight: bold;">₹ UPI</a>
243 </div>
244 <hr style="margin: 20px 0;">
245 <p>Made with ❤️ by <a href="https://nsrawat.in" target="_blank" style="color: #1f77b4;">N S Rawat</a> | <a href="https://github.com/iNSRawat" target="_blank" style="color: #1f77b4;">GitHub</a> | <a href="https://github.com/iNSRawat/data-cleaning-visualization" target="_blank" style="color: #1f77b4;">Project Repo</a></p>
246</div>
247"""
248
249
250
251# --- Build Gradio App ---
252with gr.Blocks(title="Google Analytics Data Cleaning & Visualization", theme=gr.themes.Soft()) as demo:
253 gr.Markdown("# 📊 Google Analytics Data Cleaning & Visualization")
254 gr.Markdown("Professional data wrangling & business intelligence project")
255
256 with gr.Tabs():
257 with gr.Tab("🏠 Home"):
258 gr.Markdown(home_content())
259
260 with gr.Tab("📈 Data Overview"):
261 with gr.Row():
262 overview_file = gr.File(label="Upload Google Analytics CSV file", file_types=[".csv"])
263
264 with gr.Row():
265 total_records = gr.Textbox(label="Total Records", interactive=False)
266 total_columns = gr.Textbox(label="Total Columns", interactive=False)
267 memory_usage = gr.Textbox(label="Memory Usage", interactive=False)
268 missing_values = gr.Textbox(label="Missing Values", interactive=False)
269
270 gr.Markdown("### Data Preview")
271 overview_preview = gr.Dataframe(label="First 10 Rows")
272
273 gr.Markdown("### Data Types")
274 overview_types = gr.Dataframe(label="Column Information")
275
276 overview_file.change(fn=data_overview, inputs=[overview_file], outputs=[overview_preview, total_records, total_columns, memory_usage, missing_values], api_name=False)
277 overview_file.change(fn=data_types_info, inputs=[overview_file], outputs=[overview_types], api_name=False)
278
279 with gr.Tab("🧹 Data Cleaning"):
280 with gr.Row():
281 cleaning_file = gr.File(label="Upload CSV file for cleaning", file_types=[".csv"])
282
283 run_btn = gr.Button("🚀 Run Cleaning Pipeline", variant="primary")
284
285 with gr.Row():
286 original_records = gr.Textbox(label="Original Records", interactive=False)
287 cleaned_records = gr.Textbox(label="Cleaned Records", interactive=False)
288
289 with gr.Row():
290 cleaned_file_output = gr.File(label="Download Cleaned Data")
291 quality_file_output = gr.File(label="Download Quality Report")
292
293 run_btn.click(fn=run_cleaning_pipeline, inputs=[cleaning_file], outputs=[cleaned_file_output, quality_file_output, original_records, cleaned_records], api_name="run_cleaning")
294
295 with gr.Tab("📊 Visualizations"):
296 with gr.Row():
297 viz_file = gr.File(label="Upload cleaned CSV file", file_types=[".csv"])
298
299 column_dropdown = gr.Dropdown(label="Select column to visualize", choices=[], interactive=True)
300 viz_plot = gr.Plot(label="Distribution")
301
302 with gr.Row():
303 mean_box = gr.Textbox(label="Mean", interactive=False)
304 median_box = gr.Textbox(label="Median", interactive=False)
305 std_box = gr.Textbox(label="Std Dev", interactive=False)
306 minmax_box = gr.Textbox(label="Min / Max", interactive=False)
307
308 viz_file.change(fn=get_numeric_columns, inputs=[viz_file], outputs=[column_dropdown], api_name=False)
309 column_dropdown.change(fn=visualize_column, inputs=[viz_file, column_dropdown], outputs=[viz_plot, mean_box, median_box, std_box, minmax_box], api_name=False)
310
311 with gr.Tab("📋 Quality Report"):
312 with gr.Row():
313 quality_file = gr.File(label="Upload data quality report CSV", file_types=[".csv"])
314
315 quality_df = gr.Dataframe(label="Quality Report")
316
317 with gr.Row():
318 passed_box = gr.Textbox(label="✅ Passed", interactive=False)
319 warnings_box = gr.Textbox(label="⚠️ Warnings", interactive=False)
320 failed_box = gr.Textbox(label="❌ Failed", interactive=False)
321
322 quality_file.change(fn=display_quality_report, inputs=[quality_file], outputs=[quality_df, passed_box, warnings_box, failed_box], api_name=False)
323
324 gr.HTML(footer_html)
325
326if __name__ == "__main__":
327 demo.queue().launch(ssr_mode=False)
328