SafeVixAI/SafeVixAI-Dataset-Hub

1from pathlib import Path2import json, csv3 4ROOT = Path(".")5DATA = ROOT / "chatbot_service/data"6FRONTEND = ROOT / "frontend/public/offline-data"7 8checks = []9 10def check(label, path, min_bytes=100, check_fn=None):11    p = Path(path)12    if not p.exists():13        checks.append(("FAIL", label, "FILE MISSING"))14        return15    size = p.stat().st_size16    if size < min_bytes:17        checks.append(("FAIL", label, f"Too small: {size} bytes"))18        return19    if check_fn:20        try:21            result = check_fn(p)22            checks.append(("PASS", label, result))23        except Exception as e:24            checks.append(("WARN", label, str(e)))25    else:26        checks.append(("PASS", label, f"{size:,} bytes"))27 28def count_csv(p):29    with p.open(encoding="utf-8-sig") as f:30        return f"{sum(1 for _ in csv.DictReader(f))} rows"31 32def check_pdf(p):33    data = p.read_bytes()34    if data[:4] != b"%PDF":35        preview = data[:30].decode("latin-1", errors="replace")36        return f"NOT REAL PDF -- {preview}"37    return f"{p.stat().st_size:,} bytes (valid PDF)"38 39def count_json(p):40    data = json.loads(p.read_text(encoding="utf-8"))41    if isinstance(data, list):42        return f"{len(data)} items"43    if isinstance(data, dict):44        return f"{len(data)} keys"45    return "JSON ok"46 47def geojson_features(p):48    data = json.loads(p.read_text(encoding="utf-8"))49    return f"{len(data['features']):,} features"50 51# PDFs52check("MVA 1988 PDF", DATA/"legal/motor_vehicles_act_1988.pdf", 100000, check_pdf)53check("MVA Amendment 2019 PDF", DATA/"legal/mv_amendment_act_2019.pdf", 500, check_pdf)54check("WHO Trauma Guidelines", DATA/"medical/who_trauma_care_guidelines.pdf", 500, check_pdf)55check("MVA 1988 TXT summary", DATA/"legal/motor_vehicles_act_1988_summary.txt", 10000)56 57# CSVs58check("violations_seed.csv", DATA/"violations_seed.csv", 500, count_csv)59check("state_overrides.csv", DATA/"state_overrides.csv", 200, count_csv)60check("toll_plazas.csv", DATA/"roads/toll_plazas.csv", 50000, count_csv)61check("hospital_directory.csv", DATA/"hospitals/hospital_directory.csv", 1000000)62check("nin_facilities.csv", DATA/"hospitals/nin_facilities.csv", 5000000)63check("police_stations.csv", DATA/"emergency/police_stations.csv", 50000, count_csv)64check("fire_stations.csv", DATA/"emergency/fire_stations.csv", 10000, count_csv)65 66# Backend challan CSVs67check("backend violations.csv", "backend/datasets/challan/violations.csv", 500, count_csv)68check("backend state_overrides.csv", "backend/datasets/challan/state_overrides.csv", 200, count_csv)69 70# Frontend JSONs71check("first-aid.json (frontend)", FRONTEND/"first-aid.json", 5000, count_json)72check("first_aid.json (chatbot)", DATA/"first_aid.json", 5000, count_json)73check("india-emergency.geojson", FRONTEND/"india-emergency.geojson", 1000000, geojson_features)74check("violations.csv (frontend)", FRONTEND/"violations.csv", 200, count_csv)75 76# Large files77check("pmgsy_roads.geojson", DATA/"roads/pmgsy_roads.geojson", 50_000_000)78check("kaggle_india_accidents.csv", DATA/"accidents/kaggle_india_accidents.csv", 10000000)79 80# morth_2022 extracted81morth_dir = DATA / "accidents/morth_2022"82extracted = list(morth_dir.glob("extracted_*.csv"))83check("morth_2022 extracted tables", morth_dir, 10000,84      lambda p: f"{len(extracted)} extracted CSVs")85 86print()87print("=" * 70)88print(f"  DATA PIPELINE FINAL VERIFICATION -- {len(checks)} checks")89print("=" * 70)90fail = warn = 091for status, label, detail in checks:92    icon = "[PASS]" if status == "PASS" else ("[FAIL]" if status == "FAIL" else "[WARN]")93    print(f"  {icon} {label:<45} {detail}")94    if status == "FAIL": fail += 195    if status == "WARN": warn += 196print("=" * 70)97print(f"  Result: {len(checks)-fail-warn} PASS | {warn} WARN | {fail} FAIL")98print("=" * 70)99