AbdulRahmanAzam/taxpulse-corpus
TaxPulse: Pakistani Tax Law Corpus Law is current to the Finance Act, 2026 (effective 1 July 2026, Tax Year 2027). A document corpus of Pakistani tax law, assembled for the TaxPulse final-year project (an AI tax consultation and FBR filing assistant). Sources are public publications of the Federal Board of Revenue (FBR) and provincial revenue authorities, plus reported case law. Contents Directory What it holds 01_primary_law/ Income Tax Ordinance 2001… See the full description on the dataset page: https://huggingface.co/datasets/AbdulRahmanAzam/taxpulse-corpus.
0409
1import os,sys,json,re,ssl,hashlib,time,urllib.request,urllib.parse as up2from concurrent.futures import ThreadPoolExecutor3UA="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"4CTX=ssl.create_default_context(); CTX.check_hostname=False; CTX.verify_mode=ssl.CERT_NONE5ROOT=sys.argv[1]; MAN=sys.argv[2] # corpus root, manifest json in6OUTMAN=sys.argv[3]7items=json.load(open(MAN,encoding="utf-8"))8def safe(n):9 n=up.unquote(n); n=re.sub(r'[<>:"/\|?*\x00-\x1f]','_',n); n=re.sub(r'\s+','_',n).strip('._')10 return (n[:120] or "file")11def one(it):12 url=it["url"]; cat=it["cat"]; name=it.get("name")13 ext=(re.search(r'\.(pdf|xlsx|xls|docx|doc|zip|csv|pptx)(?:\?|$)',url,re.I) or [None,"pdf"])[1].lower()14 base=safe(name) if name else safe(os.path.basename(up.urlparse(url).path))15 if not base.lower().endswith("."+ext): base=f"{base}.{ext}"16 d=os.path.join(ROOT,cat); os.makedirs(d,exist_ok=True); p=os.path.join(d,base)17 if os.path.exists(p) and os.path.getsize(p)>1024:18 b=open(p,'rb').read()19 return dict(it,path=os.path.relpath(p,ROOT).replace("\\","/"),bytes=len(b),sha256=hashlib.sha256(b).hexdigest(),status="cached")20 for attempt in range(3):21 try:22 r=urllib.request.urlopen(urllib.request.Request(url,headers={"User-Agent":UA,"Accept":"*/*"}),timeout=120,context=CTX)23 b=r.read()24 if len(b)<600: raise ValueError(f"tiny {len(b)}")25 open(p,"wb").write(b)26 return dict(it,path=os.path.relpath(p,ROOT).replace("\\","/"),bytes=len(b),sha256=hashlib.sha256(b).hexdigest(),27 content_type=r.headers.get("Content-Type",""),status="ok")28 except Exception as e:29 err=f"{type(e).__name__}: {str(e)[:120]}"; time.sleep(1.5*(attempt+1))30 return dict(it,status="FAILED",error=err)31res=[]32with ThreadPoolExecutor(max_workers=16) as ex:33 for i,r in enumerate(ex.map(one,items),1):34 res.append(r)35 if i%25==0: print(f" {i}/{len(items)}",flush=True)36json.dump(res,open(OUTMAN,"w",encoding="utf-8"),indent=1,ensure_ascii=False)37ok=sum(1 for r in res if r["status"] in("ok","cached")); tot=sum(r.get("bytes",0) for r in res)38print(f"DONE {ok}/{len(res)} ok, {tot/1e6:.1f} MB",flush=True)39for r in res:40 if r["status"]=="FAILED": print(" FAIL",r["url"],r.get("error"),flush=True)41