SafeVixAI/SafeVixAI-Dataset-Hub

1"""2Enterprise Accident Data Pipeline3Generates:4  1. accidents_summary.json  -> frontend/public/ + backend/data/5  2. blackspot_seed.csv      -> backend/datasets/accidents/6from the 1M-row Kaggle India road accidents CSV.7"""8from __future__ import annotations9 10import json11import sys12import io13from pathlib import Path14 15# Windows-safe UTF-8 output16sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")17 18try:19    import pandas as pd20except ImportError:21    sys.exit("pandas not installed. Run: pip install pandas")22 23# ── Paths ─────────────────────────────────────────────────────────────────────24REPO_ROOT = Path(__file__).resolve().parents[3]  # IITM/ repo root25CSV_PATH = REPO_ROOT / "backend" / "datasets" / "accidents" / "kaggle" / "india_road_accident_coords.csv"26OUT_SUMMARY = REPO_ROOT / "frontend" / "public" / "accidents_summary.json"27OUT_SUMMARY_BACKEND = REPO_ROOT / "backend" / "data" / "accidents_summary.json"28OUT_BLACKSPOT = REPO_ROOT / "backend" / "datasets" / "accidents" / "blackspot_seed.csv"29OUT_BLACKSPOT_OFFLINE = REPO_ROOT / "frontend" / "public" / "offline-data" / "blackspot_seed.csv"30 31if not CSV_PATH.exists():32    sys.exit(f"CSV not found: {CSV_PATH}")33 34# ── Load ──────────────────────────────────────────────────────────────────────35print("Loading 1M accident records...")36df = pd.read_csv(CSV_PATH, low_memory=False)37df.columns = df.columns.str.strip().str.lower().str.replace(" ", "_")38print(f"Loaded {len(df):,} rows | columns: {list(df.columns[:10])}")39 40# ── Fix columns ───────────────────────────────────────────────────────────────41lat_col  = next((c for c in df.columns if "lat" in c), None)42lon_col  = next((c for c in df.columns if "lon" in c or "lng" in c), None)43sev_col  = next((c for c in df.columns if "severity" in c), None)44cas_col  = next((c for c in df.columns if "casual" in c), None)45 46print(f"lat={lat_col} lon={lon_col} severity={sev_col} casualties={cas_col}")47 48# coerce to numeric49for col in [lat_col, lon_col, sev_col, cas_col]:50    if col:51        df[col] = pd.to_numeric(df[col], errors="coerce")52 53# Drop rows with no GPS54df_geo = df.dropna(subset=[lat_col, lon_col]).copy()55print(f"Rows with GPS: {len(df_geo):,}")56 57# ── 1. National Summary JSON ──────────────────────────────────────────────────58total_accidents = len(df)59total_casualties = int(df[cas_col].sum()) if cas_col else 060 61# Severity breakdown (1=Fatal, 2=Serious, 3=Slight — UK STATS19 encoding)62severity_map = {1: "fatal", 2: "serious", 3: "slight"}63severity_counts: dict = {}64if sev_col:65    for sev_val, label in severity_map.items():66        count = int((df[sev_col] == sev_val).sum())67        severity_counts[label] = count68 69# Day-of-week analysis70dow_col = next((c for c in df.columns if "day" in c and "week" in c), None)71day_names = {1:"Sunday",2:"Monday",3:"Tuesday",4:"Wednesday",5:"Thursday",6:"Friday",7:"Saturday"}72dow_stats: list = []73if dow_col:74    df[dow_col] = pd.to_numeric(df[dow_col], errors="coerce")75    dow = df.groupby(dow_col).size().sort_values(ascending=False)76    dow_stats = [{"day": day_names.get(int(k), str(k)), "accidents": int(v)} for k, v in dow.items()]77 78# Speed analysis79speed_col = next((c for c in df.columns if "speed" in c), None)80speed_stats: dict = {}81if speed_col:82    df[speed_col] = pd.to_numeric(df[speed_col], errors="coerce")83    speed_stats = {84        "mean_speed_limit": round(float(df[speed_col].mean()), 1),85        "high_speed_gt80": int((df[speed_col] > 80).sum()),86    }87 88summary = {89    "generated_at": "2026-04-27",90    "source": "Kaggle India Road Accidents Dataset (UK STATS19 encoding)",91    "total_accidents": total_accidents,92    "total_casualties": total_casualties,93    "accidents_with_gps": len(df_geo),94    "severity_breakdown": severity_counts,95    "accidents_by_day_of_week": dow_stats,96    "speed_analysis": speed_stats,97    "data_note": "Dataset uses UK STATS19 police-recorded format. Severity: 1=Fatal, 2=Serious, 3=Slight.",98}99 100OUT_SUMMARY.parent.mkdir(parents=True, exist_ok=True)101OUT_SUMMARY_BACKEND.parent.mkdir(parents=True, exist_ok=True)102 103with open(OUT_SUMMARY, "w", encoding="utf-8") as f:104    json.dump(summary, f, indent=2, ensure_ascii=False)105with open(OUT_SUMMARY_BACKEND, "w", encoding="utf-8") as f:106    json.dump(summary, f, indent=2, ensure_ascii=False)107 108print(f"accidents_summary.json written ({OUT_SUMMARY.stat().st_size//1024} KB)")109 110# ── 2. Blackspot Seed CSV ─────────────────────────────────────────────────────111print("Generating GPS blackspot clusters (1km grid)...")112 113df_geo["lat_r"] = df_geo[lat_col].round(2)114df_geo["lon_r"] = df_geo[lon_col].round(2)115 116agg = {lat_col: "mean", lon_col: "mean", "lat_r": "count"}117if cas_col:118    agg[cas_col] = "sum"119if sev_col:120    agg[sev_col] = "mean"121 122hotspots = (123    df_geo.groupby(["lat_r", "lon_r"])124    .agg(125        accident_count=(lat_col, "count"),126        latitude=(lat_col, "mean"),127        longitude=(lon_col, "mean"),128        **({f"total_casualties": (cas_col, "sum")} if cas_col else {}),129        **({f"avg_severity": (sev_col, "mean")} if sev_col else {}),130    )131    .reset_index()132)133 134# Only keep clusters with at least 2 accidents (removes noise)135hotspots = hotspots[hotspots["accident_count"] >= 2].copy()136 137# Risk score = accident_count * (1 + casualties / 10) 138if "total_casualties" in hotspots.columns:139    hotspots["risk_score"] = (140        hotspots["accident_count"] * (1 + hotspots["total_casualties"] / 10)141    ).round(2)142else:143    hotspots["risk_score"] = hotspots["accident_count"].astype(float)144 145hotspots = hotspots.sort_values("risk_score", ascending=False)146 147OUT_BLACKSPOT.parent.mkdir(parents=True, exist_ok=True)148OUT_BLACKSPOT_OFFLINE.parent.mkdir(parents=True, exist_ok=True)149 150hotspots.to_csv(OUT_BLACKSPOT, index=False)151hotspots.to_csv(OUT_BLACKSPOT_OFFLINE, index=False)152 153print(f"blackspot_seed.csv: {len(hotspots):,} clusters | top risk_score={hotspots['risk_score'].iloc[0]:.1f}")154print(f"Written to: {OUT_BLACKSPOT}")155print(f"Written to: {OUT_BLACKSPOT_OFFLINE}")156print("\nDONE - Enterprise accident data pipeline complete.")157