SafeVixAI/SafeVixAI-Dataset-Hub

1"""Download the 3 legal/medical PDFs using URLs found by browser subagent."""2import sys, io, time, requests3from pathlib import Path4 5sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")6 7LEGAL_DIR   = Path(r"C:\Hackathons\IITM\SafeVixAI-Dataset-Hub\scripts\scripts\chatbot_service\data\legal")8MEDICAL_DIR = Path(r"C:\Hackathons\IITM\SafeVixAI-Dataset-Hub\scripts\scripts\chatbot_service\data\medical")9LEGAL_DIR.mkdir(parents=True, exist_ok=True)10MEDICAL_DIR.mkdir(parents=True, exist_ok=True)11 12# Working URLs confirmed by browser subagent13DOWNLOADS = [14    {15        "filename": "mv_act_1988_full.pdf",16        "dest": LEGAL_DIR,17        "url": "https://www.indiacode.nic.in/bitstream/123456789/19318/1/the_motor_vehicle_act_1988.pdf",18    },19    {20        "filename": "mv_amendment_act_2019.pdf",21        "dest": LEGAL_DIR,22        "url": "https://prsindia.org/files/bills_acts/bills_parliament/2019/Motor%20Vehicles%20(Amendment)%20Act,%202019.pdf",23    },24    {25        "filename": "who_trauma_care_guidelines.pdf",26        "dest": MEDICAL_DIR,27        "url": "https://iris.who.int/bitstreams/ea9f1bd6-3eb8-4726-a3c5-d8d4d1bcb83a/download",28    },29]30 31HEADERS = {32    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",33    "Accept": "application/pdf,*/*",34    "Accept-Language": "en-US,en;q=0.9",35    "Referer": "https://www.google.com/",36}37 38results = {}39for item in DOWNLOADS:40    out = item["dest"] / item["filename"]41    print(f"\nDownloading: {item['filename']}")42    print(f"  URL: {item['url'][:80]}")43 44    # Skip if already valid45    if out.exists() and out.stat().st_size > 50000:46        with open(out, "rb") as f:47            magic = f.read(4)48        if magic == b"%PDF":49            print(f"  ALREADY EXISTS ({out.stat().st_size//1024}KB) -- skip")50            results[item["filename"]] = True51            continue52 53    try:54        r = requests.get(item["url"], headers=HEADERS, timeout=90, allow_redirects=True, stream=True)55        print(f"  HTTP {r.status_code} | Content-Type: {r.headers.get('Content-Type','?')}")56 57        if r.status_code == 200:58            data = b"".join(r.iter_content(65536))59            sz = len(data)60            magic = data[:4]61            print(f"  Size: {sz//1024}KB | Magic: {magic}")62 63            if sz > 50000 and magic == b"%PDF":64                out.write_bytes(data)65                print(f"  SAVED: {out.name} ({sz//1024}KB)")66                results[item["filename"]] = True67            else:68                print(f"  INVALID: too small or not PDF (magic={magic})")69                results[item["filename"]] = False70        else:71            print(f"  FAIL: HTTP {r.status_code}")72            results[item["filename"]] = False73 74    except Exception as e:75        print(f"  ERROR: {e}")76        results[item["filename"]] = False77 78    time.sleep(2)79 80print("\n" + "=" * 60)81print("  FINAL RESULTS")82print("=" * 60)83for fname, ok in results.items():84    p_legal = LEGAL_DIR / fname85    p_med   = MEDICAL_DIR / fname86    p = p_legal if p_legal.exists() else p_med87    size = f"({p.stat().st_size//1024}KB)" if p.exists() else ""88    status = "DOWNLOADED" if ok else "FAILED"89    print(f"  {'OK' if ok else 'XX'}  {fname:45s} {status} {size}")90 91ok_count = sum(results.values())92print(f"\n  {ok_count}/{len(results)} PDFs ready")93