SafeVixAI/SafeVixAI-Dataset-Hub

1"""2Enterprise ChromaDB Legal + Medical Ingestion3==============================================4Ingests ALL sources into ChromaDB:5  1. MV Act 1988 key sections (hardcoded enterprise-grade text)6  2. MV Amendment 2019 — from downloaded PDF (if available) + hardcoded7  3. State overrides CSV8  4. WHO Trauma Care Guidelines — from downloaded PDF (if available)9  5. First Aid JSON (20 WHO articles)10 11Run: python backend/scripts/ingest_legal_chromadb.py12"""13from __future__ import annotations14 15import csv16import json17import sys18import io19from pathlib import Path20 21sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")22 23try:24    import chromadb25    from chromadb.utils import embedding_functions26except ImportError:27    print("Install chromadb: pip install chromadb")28    sys.exit(1)29 30# ── Paths ─────────────────────────────────────────────────────────────────────31SCRIPT_DIR    = Path(__file__).parent          # scripts/data/32BACKEND_DIR   = Path(__file__).resolve().parents[2]  # backend/33CHROMA_PATH   = BACKEND_DIR / "chroma_db"34CHALLAN_CSV   = BACKEND_DIR / "datasets" / "challan" / "state_overrides.csv"35FIRST_AID_JSON = BACKEND_DIR.parent / "frontend" / "public" / "offline-data" / "first-aid.json"36 37# Dataset Hub paths for downloaded PDFs38HUB_ROOT      = BACKEND_DIR.parent.parent / "SafeVixAI-Dataset-Hub"39HUB_LEGAL_DIR = HUB_ROOT / "scripts" / "scripts" / "chatbot_service" / "data" / "legal"40HUB_MED_DIR   = HUB_ROOT / "scripts" / "scripts" / "chatbot_service" / "data" / "medical"41MV_ACT_PDF    = HUB_LEGAL_DIR / "mv_act_1988_full.pdf"42MV_AMEND_PDF  = HUB_LEGAL_DIR / "mv_amendment_act_2019.pdf"43WHO_PDF       = HUB_MED_DIR   / "who_trauma_care_guidelines.pdf"44 45# ── MV Act 1988 Key Sections ──────────────────────────────────────────────────46MV_ACT_1988_SECTIONS = [47    {48        "id": "mva-1988-s112",49        "section": "Section 112 — Limits of Speed",50        "content": (51            "Section 112 of the Motor Vehicles Act 1988 sets maximum speed limits. "52            "Urban area roads: 50 km/h for LMV; 40 km/h for HMV. "53            "National and State highways: 100 km/h for LMV; 65 km/h for HMV; 60 km/h for medium goods. "54            "State governments may fix lower speeds for specific routes. "55            "Fine: Rs.1,000-Rs.2,000 for first offense; Rs.2,000-Rs.4,000 for repeat."56        ),57        "act": "Motor Vehicles Act 1988", "category": "speeding",58    },59    {60        "id": "mva-1988-s129",61        "section": "Section 129 — Wearing of Protective Headgear",62        "content": (63            "Section 129 mandates every person driving or riding a motorcycle on any public road "64            "shall wear a protective helmet conforming to BIS standards. Helmet must be securely fastened. "65            "Fine: Rs.1,000 for non-wearing. Pillion passenger without helmet: Rs.1,000. "66            "Disqualification from driving for 3 months may be imposed on repeat offense."67        ),68        "act": "Motor Vehicles Act 1988", "category": "helmet",69    },70    {71        "id": "mva-1988-s138",72        "section": "Section 138 — Regulation of Traffic",73        "content": (74            "Section 138 empowers state governments to make traffic rules. "75            "Red light jumping: Rs.5,000 fine under MV Amendment 2019. "76            "Wrong side driving: Rs.5,000. "77            "No seatbelt: Rs.1,000. "78            "Mobile phone while driving: Rs.5,000 (repeat: Rs.10,000)."79        ),80        "act": "Motor Vehicles Act 1988", "category": "traffic_signal",81    },82    {83        "id": "mva-1988-s185",84        "section": "Section 185 — Driving by a Drunken Person",85        "content": (86            "Section 185 prohibits driving under influence of alcohol or drugs. "87            "BAC exceeding 30mg per 100ml of blood is an offense. "88            "First offense: imprisonment up to 6 months OR fine up to Rs.10,000 or both. "89            "Second offense within 3 years: imprisonment up to 2 years AND fine up to Rs.15,000. "90            "Driving license suspension for 6 months minimum on first conviction."91        ),92        "act": "Motor Vehicles Act 1988", "category": "drunk_driving",93    },94    {95        "id": "mva-1988-s194",96        "section": "Section 194 — Using Vehicle Exceeding Permissible Weight",97        "content": (98            "Section 194 addresses overloaded vehicles. "99            "Fine: Rs.20,000 for first offense, plus Rs.2,000 per additional tonne. "100            "Repeat offense: Rs.25,000 + per-tonne rate. "101            "State government may detain vehicle until excess load is unloaded."102        ),103        "act": "Motor Vehicles Act 1988", "category": "overloading",104    },105    {106        "id": "mva-1988-s196",107        "section": "Section 196 — Driving Without Insurance",108        "content": (109            "Section 196 mandates third-party insurance for all motor vehicles. "110            "Driving without valid third-party insurance: "111            "Fine Rs.2,000 and/or imprisonment up to 3 months for first offense. "112            "Repeat: Rs.4,000 and/or 3 months. Cognizable offense — police can arrest without warrant."113        ),114        "act": "Motor Vehicles Act 1988", "category": "no_insurance",115    },116    {117        "id": "mva-1988-s177",118        "section": "Section 177 — General Provisions for Punishment",119        "content": (120            "Section 177 general punishment for traffic violations not covered by specific sections. "121            "Fine: Rs.500 for first offense; Rs.1,500 for subsequent offenses. "122            "Driving license in violation of conditions: Rs.5,000."123        ),124        "act": "Motor Vehicles Act 1988", "category": "general",125    },126    {127        "id": "mva-1988-s134",128        "section": "Section 134 — Duty of Driver in Case of Accident",129        "content": (130            "Section 134: driver involved in accident must secure medical attention for injured. "131            "Driver must not flee the scene. Must report to nearest police station within 24 hours "132            "if any person was killed or injured. "133            "Failure to report: imprisonment up to 3 months or fine up to Rs.500. "134            "Hit and run cases: victim compensation from Solatium Fund."135        ),136        "act": "Motor Vehicles Act 1988", "category": "accident_duty",137    },138    {139        "id": "mva-1988-s181",140        "section": "Section 181 — Driving Without Licence",141        "content": (142            "Section 181: driving without a valid driving licence is an offense. "143            "Fine: Rs.5,000 (MV Amendment 2019 — was Rs.500). "144            "Unlicensed minor driving: Guardian or owner liable — Rs.25,000 fine, "145            "3 years imprisonment, minor treated as adult under JJ Act for this purpose."146        ),147        "act": "Motor Vehicles Act 1988", "category": "no_licence",148    },149    {150        "id": "mva-1988-s184",151        "section": "Section 184 — Dangerous Driving",152        "content": (153            "Section 184: dangerous driving which endangers public safety. "154            "First offense: imprisonment up to 1 year OR fine Rs.1,000-Rs.5,000. "155            "Repeat within 3 years: imprisonment up to 2 years. "156            "Racing on public roads: imprisonment up to 1 year OR fine up to Rs.5,000."157        ),158        "act": "Motor Vehicles Act 1988", "category": "dangerous_driving",159    },160    # ── MV Amendment Act 2019 ──────────────────────────────────────────────────161    {162        "id": "mva-2019-s119",163        "section": "MV Amendment 2019 — Complete Updated Fines Schedule",164        "content": (165            "Motor Vehicles (Amendment) Act 2019 significantly increased fines: "166            "Drunk driving: Rs.10,000 first offense, Rs.15,000 repeat (was Rs.2,000); "167            "Speeding: Rs.1,000-Rs.2,000 (was Rs.400); "168            "Red light jumping: Rs.5,000 (was Rs.1,000); "169            "No helmet: Rs.1,000 + 3-month license suspension (was Rs.100); "170            "No seatbelt: Rs.1,000 (was Rs.100); "171            "Dangerous driving: Rs.5,000 (was Rs.1,000); "172            "Mobile phone while driving: Rs.5,000 first, Rs.10,000 repeat (was Rs.1,000); "173            "No licence: Rs.5,000 (was Rs.500); "174            "No insurance: Rs.2,000 first, Rs.4,000 repeat (was Rs.1,000); "175            "Overloading 2-wheelers: Rs.2,000 + license disqualification 3 months; "176            "Juvenile driving: Guardian liable Rs.25,000, 3 years jail."177        ),178        "act": "MV Amendment Act 2019", "category": "general",179    },180    {181        "id": "mva-2019-golden-hour",182        "section": "MV Amendment 2019 — Good Samaritan Protection",183        "content": (184            "Good Samaritan provisions under MV Amendment 2019: "185            "Person who voluntarily helps accident victim in good faith cannot be subject to "186            "civil or criminal liability. Cannot be detained at hospital or police station. "187            "Police cannot compel Good Samaritan to be a witness. "188            "Hospital cannot demand payment before emergency treatment within first 24 hours. "189            "Cashless treatment for road accident victims within golden hour."190        ),191        "act": "MV Amendment Act 2019", "category": "good_samaritan",192    },193    {194        "id": "mva-2019-compensation",195        "section": "MV Amendment 2019 — Hit and Run Compensation",196        "content": (197            "Hit and run compensation under MV Amendment Act 2019: "198            "Death in hit and run: Rs.2,00,000 (was Rs.25,000). "199            "Grievous hurt in hit and run: Rs.50,000 (was Rs.12,500). "200            "Paid from Motor Vehicle Accident Fund maintained by Government of India. "201            "Claim to be filed within 6 months to Claim Enquiry Officer."202        ),203        "act": "MV Amendment Act 2019", "category": "compensation",204    },205    # ── State Overrides (inline) ───────────────────────────────────────────────206    {207        "id": "state-delhi-helmet",208        "content": (209            "Delhi: Helmet fine Rs.1,000 per Central Act. No separate state override. "210            "E-challan via automated CCTV cameras in Delhi NCR. "211            "Delhi traffic police issues challan via Parivahan portal."212        ),213        "act": "Delhi Motor Vehicle Rules", "category": "helmet",214    },215    {216        "id": "state-tamil-nadu-speed",217        "content": (218            "Tamil Nadu: Speed limits on National Highways: 80 km/h LMV, 60 km/h HMV. "219            "Urban area speed limit: 50 km/h. "220            "Speeding fine: Rs.1,000-Rs.2,000 per central act."221        ),222        "act": "Tamil Nadu Motor Vehicles Rules", "category": "speeding",223    },224    {225        "id": "state-maharashtra-drunk",226        "content": (227            "Maharashtra: Drunk driving fine Rs.10,000 + license suspension 6 months first offense. "228            "Repeat within 3 years: license cancellation + imprisonment up to 2 years. "229            "Breathalyzer test mandatory on NH and expressways."230        ),231        "act": "Maharashtra Motor Vehicles Rules", "category": "drunk_driving",232    },233    {234        "id": "state-karnataka-mobile",235        "content": (236            "Karnataka: Mobile phone use while driving Rs.5,000 per MV Amendment 2019. "237            "Bangalore Traffic Police operates automated challan system via CCTV. "238            "Challan sent to vehicle owner via SMS within 48 hours."239        ),240        "act": "Karnataka Motor Vehicles Rules", "category": "mobile_phone",241    },242    {243        "id": "state-up-overload",244        "content": (245            "Uttar Pradesh: Overloading fine Rs.20,000 + Rs.2,000 per extra tonne. "246            "Frequent night raids on NH-19, NH-58, NH-24 for overloaded trucks. "247            "Vehicle detained at nearest weighbridge until excess load removed."248        ),249        "act": "Uttar Pradesh Motor Vehicles Rules", "category": "overloading",250    },251]252 253# ── PDF Text Extraction Helper ────────────────────────────────────────────────254def extract_pdf_chunks(pdf_path: Path, tag: str, chunk_size: int = 800) -> list[dict]:255    """Extract text from PDF and split into chunks for ChromaDB."""256    try:257        import pdfplumber258    except ImportError:259        print(f"  [SKIP PDF] pdfplumber not installed. Skipping {pdf_path.name}")260        return []261 262    if not pdf_path.exists() or pdf_path.stat().st_size < 10000:263        print(f"  [SKIP PDF] Not found or too small ({pdf_path.stat().st_size if pdf_path.exists() else 0}B): {pdf_path.name}")264        return []265 266    chunks = []267    try:268        with pdfplumber.open(str(pdf_path)) as pdf:269            full_text = ""270            for page in pdf.pages:271                text = page.extract_text() or ""272                full_text += text + "\n"273 274        # Split into chunks275        words = full_text.split()276        chunk_words = chunk_size // 6  # ~6 chars per word average277        for i in range(0, len(words), chunk_words):278            chunk = " ".join(words[i : i + chunk_words])279            if len(chunk) > 100:  # skip tiny chunks280                chunks.append({281                    "id": f"{tag}-chunk-{i}",282                    "content": chunk,283                    "act": tag,284                    "category": "full_pdf",285                })286 287        print(f"  [PDF] Extracted {len(chunks)} chunks from {pdf_path.name}")288    except Exception as e:289        print(f"  [WARN PDF] Could not parse {pdf_path.name}: {e} — skipping")290 291    return chunks292 293 294# ── First Aid JSON Loader ────────────────────────────────────────────────────295def load_first_aid_docs() -> list[dict]:296    """Load all 20 WHO-based first aid articles from first-aid.json."""297    if not FIRST_AID_JSON.exists():298        print(f"  [SKIP] first-aid.json not found at {FIRST_AID_JSON}")299        return []300 301    with open(FIRST_AID_JSON, encoding="utf-8") as f:302        articles = json.load(f)303 304    docs = []305    for article in articles:306        title = article.get("title", "First Aid")307        steps = article.get("steps", [])308 309        def step_text(s):310            if isinstance(s, str):311                return s312            if isinstance(s, dict):313                return s.get("instruction") or s.get("text") or str(s)314            return str(s)315 316        content = f"{title}: " + " | ".join(step_text(s) for s in steps)317        docs.append({318            "id": f"firstaid-{article.get('id', len(docs))}",319            "content": content,320            "act": "WHO First Aid Guidelines",321            "category": "first_aid",322        })323 324    print(f"  [OK] Loaded {len(docs)} first-aid articles from first-aid.json")325    return docs326 327 328# ── Main Ingest ───────────────────────────────────────────────────────────────329def ingest_to_chromadb() -> None:330    CHROMA_PATH.mkdir(exist_ok=True)331 332    print(f"[CHROMA] Connecting to ChromaDB at: {CHROMA_PATH}")333    client = chromadb.PersistentClient(path=str(CHROMA_PATH))334 335    try:336        from chromadb.utils.embedding_functions import SentenceTransformerEmbeddingFunction337        ef = SentenceTransformerEmbeddingFunction(model_name="all-MiniLM-L6-v2")338        print("[OK] Using SentenceTransformer all-MiniLM-L6-v2 embeddings")339    except Exception:340        ef = embedding_functions.DefaultEmbeddingFunction()341        print("[WARN] Using default embeddings (install sentence-transformers for better accuracy)")342 343    # ── Legal collection ───────────────────────────────────────────────────────344    legal_col = client.get_or_create_collection(345        name="legal_knowledge",346        embedding_function=ef,347        metadata={"description": "India Motor Vehicles Act + Amendment 2019 + State Rules"},348    )349 350    all_legal = list(MV_ACT_1988_SECTIONS)  # start with hardcoded351 352    # Add CSV state overrides353    if CHALLAN_CSV.exists():354        with open(CHALLAN_CSV, encoding="utf-8") as f:355            reader = csv.DictReader(f)356            for i, row in enumerate(reader):357                all_legal.append({358                    "id": f"csv-state-{i}",359                    "content": (360                        f"State: {row.get('state','?')} | "361                        f"Offense: {row.get('offense_type','?')} | "362                        f"Fine: Rs.{row.get('fine_amount','?')} | "363                        f"Section: {row.get('mv_act_section','N/A')}"364                    ),365                    "act": "State Override CSV",366                    "category": row.get("offense_type", "general"),367                })368        print(f"  [OK] Loaded {i+1} state override rows from CSV")369 370    # Add PDF chunks for MV Act 1988 full text (if downloaded)371    all_legal += extract_pdf_chunks(MV_ACT_PDF, "MV Act 1988 Full PDF")372 373    # Add PDF chunks for MV Amendment 2019 (if downloaded)374    all_legal += extract_pdf_chunks(MV_AMEND_PDF, "MV Amendment Act 2019 PDF")375 376    legal_col.upsert(377        ids=[d["id"] for d in all_legal],378        documents=[d["content"] for d in all_legal],379        metadatas=[{"act": d.get("act",""), "category": d.get("category",""), "section": d.get("section","")} for d in all_legal],380    )381    print(f"[OK] Legal collection: {len(all_legal)} documents ingested")382 383    # ── Medical / First Aid collection ─────────────────────────────────────────384    medical_col = client.get_or_create_collection(385        name="medical_knowledge",386        embedding_function=ef,387        metadata={"description": "WHO First Aid Guidelines + Trauma Care"},388    )389 390    all_medical = load_first_aid_docs()391    all_medical += extract_pdf_chunks(WHO_PDF, "WHO Trauma Care Guidelines PDF")392 393    if all_medical:394        medical_col.upsert(395            ids=[d["id"] for d in all_medical],396            documents=[d["content"] for d in all_medical],397            metadatas=[{"act": d.get("act",""), "category": d.get("category","")} for d in all_medical],398        )399        print(f"[OK] Medical collection: {len(all_medical)} documents ingested")400 401    # ── Verification ──────────────────────────────────────────────────────────402    print("\n[TEST] Verification queries:")403    q1 = legal_col.query(query_texts=["drunk driving fine india"], n_results=2)404    print(f"  'drunk driving': {q1['documents'][0][0][:80]}...")405    q2 = legal_col.query(query_texts=["helmet not wearing penalty"], n_results=1)406    print(f"  'helmet penalty': {q2['documents'][0][0][:80]}...")407    if all_medical:408        q3 = medical_col.query(query_texts=["how to do CPR"], n_results=1)409        print(f"  'CPR steps': {q3['documents'][0][0][:80]}...")410 411    print(f"\n[DONE] ChromaDB enterprise ingestion complete.")412    print(f"  Legal documents: {legal_col.count()}")413    print(f"  Medical documents: {medical_col.count() if all_medical else 0}")414 415 416if __name__ == "__main__":417    ingest_to_chromadb()418