Rxrohans/PayLens-Dev

1"""2ingestor.py — Phase 1 of PayLens  (FIXED v2)3----------------------------------------------4FIXES IN THIS VERSION:5  1. Added RBI static data: NEFT, RTGS, IMPS fee structures6  2. Added PCI-DSS compliance reference data7  3. Added Razorpay, Paytm sources8  4. ingest_static() for regulatory data that blocks scrapers9  5. Retry logic for flaky web pages10 11ROOT CAUSE OF NEFT/RTGS/IMPS FAILURES:12  v1 only had PayPal, Stripe, NPCI live members.13  NPCI live members page lists banks — not fee schedules.14  Zero RBI circular data was ever in the knowledge base.15  The LLM was answering from training data alone → hallucinations.16"""17 18import time19import requests20from pathlib import Path21from pypdf import PdfReader22from bs4 import BeautifulSoup23 24RAW_DIR = Path(__file__).parent.parent / "data" / "raw"25RAW_DIR.mkdir(parents=True, exist_ok=True)26 27HEADERS = {28    "User-Agent": (29        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "30        "AppleWebKit/537.36 (KHTML, like Gecko) "31        "Chrome/120.0.0.0 Safari/537.36"32    )33}34 35 36# ──────────────────────────────────────────────37# INGESTION HELPERS38# ──────────────────────────────────────────────39 40def ingest_webpage(url: str, save_name: str, retries: int = 2) -> str:41    print(f"🌐 Scraping: {url}")42    for attempt in range(retries + 1):43        try:44            response = requests.get(url, headers=HEADERS, timeout=30)45            response.raise_for_status()46            soup = BeautifulSoup(response.text, "html.parser")47            for tag in soup(["script", "style", "nav", "footer", "header", "aside"]):48                tag.decompose()49            clean_text = soup.get_text(separator="\n", strip=True)50            txt_path = RAW_DIR / save_name51            with open(txt_path, "w", encoding="utf-8") as f:52                f.write(clean_text)53            print(f"  ✅ {len(clean_text):,} chars → {save_name}")54            return clean_text55        except Exception as e:56            if attempt < retries:57                print(f"  ⚠️  Attempt {attempt+1} failed ({e}), retrying...")58                time.sleep(2)59            else:60                print(f"  ❌ Failed after {retries+1} attempts: {e}")61                return ""62 63 64def ingest_pdf_from_url(url: str, save_name: str) -> str:65    print(f"📥 Downloading PDF: {save_name}")66    try:67        response = requests.get(url, headers=HEADERS, timeout=45)68        response.raise_for_status()69        pdf_path = RAW_DIR / save_name.replace(".txt", ".pdf")70        with open(pdf_path, "wb") as f:71            f.write(response.content)72        reader = PdfReader(str(pdf_path))73        full_text = ""74        for page in reader.pages:75            full_text += (page.extract_text() or "") + "\n"76        txt_path = RAW_DIR / save_name77        with open(txt_path, "w", encoding="utf-8") as f:78            f.write(full_text)79        print(f"  ✅ {len(full_text):,} chars → {save_name}")80        return full_text81    except Exception as e:82        print(f"  ❌ PDF failed: {e}")83        return ""84 85 86def ingest_static(content: str, save_name: str) -> str:87    """88    Saves curated static text directly to raw/.89    Used for RBI/regulatory sources that block scrapers or require login.90    This is the MOST RELIABLE method for authoritative fee data.91    """92    txt_path = RAW_DIR / save_name93    with open(txt_path, "w", encoding="utf-8") as f:94        f.write(content)95    print(f"  ✅ Static content saved → {save_name} ({len(content):,} chars)")96    return content97 98 99# ──────────────────────────────────────────────100# STATIC RBI DATA101# Source: RBI circulars (public domain)102# RBI/2019-20/187 | RBI/2020-21/62 | DPSS guidelines103# ──────────────────────────────────────────────104 105RBI_PAYMENT_FEES_DATA = """106RBI PAYMENT SYSTEMS — FEE STRUCTURE FOR INDIA107Source: Reserve Bank of India (RBI) Official Circulars and NPCI Guidelines108Last updated: 2024109 110========================================111NEFT — NATIONAL ELECTRONIC FUNDS TRANSFER112========================================113Full form: National Electronic Funds Transfer114Operated by: Reserve Bank of India (RBI)115Settlement: Deferred Net Settlement — processes in half-hourly batches116Availability: 24x7x365 (since December 16, 2019)117Channels: Internet banking, mobile banking, bank branches118 119NEFT CHARGES — OUTWARD TRANSACTIONS (customer sending money):120  Transaction up to ₹10,000       : ₹2.50 + applicable GST121  Transaction ₹10,001 to ₹1,00,000 : ₹5.00 + applicable GST122  Transaction ₹1,00,001 to ₹2,00,000: ₹15.00 + applicable GST123  Transaction above ₹2,00,000     : ₹25.00 + applicable GST124 125NEFT INWARD CHARGES: FREE — no charge to the beneficiary (receiver)126 127RBI Waiver: In January 2020, RBI waived processing charges for member banks.128Banks were directed to pass these savings to customers.129Most major banks (SBI, HDFC, ICICI, Axis) now offer FREE online NEFT.130Branch NEFT may still carry the above charges.131 132NEFT TRANSACTION LIMITS:133  Minimum: ₹1 (no minimum limit)134  Maximum: No upper limit set by RBI135  Note: Individual banks may impose their own limits.136  Walk-in customers (non-account holders): ₹50,000 per transaction137 138NEFT INFORMATION REQUIRED:139  - Beneficiary name140  - Beneficiary account number141  - IFSC code of beneficiary bank142  - Bank name and branch143 144========================================145RTGS — REAL TIME GROSS SETTLEMENT146========================================147Full form: Real Time Gross Settlement148Operated by: Reserve Bank of India (RBI)149Settlement: Real-time (transaction by transaction, immediately)150Purpose: High-value transactions only151Availability: 24x7x365 (since December 14, 2020)152 153RTGS CHARGES — OUTWARD TRANSACTIONS:154  Transactions from ₹2,00,000 to ₹5,00,000: ₹24.50 + GST155  Transactions above ₹5,00,000             : ₹49.50 + GST156 157RTGS INWARD CHARGES: FREE — no charge to the beneficiary158 159RTGS TRANSACTION LIMITS:160  Minimum: ₹2,00,000 (₹2 lakh) — RTGS is only for high-value payments161  Maximum: No upper limit (as per RBI)162  Individual bank limits may apply.163 164KEY DIFFERENCE FROM NEFT:165  NEFT = batch processing every 30 minutes (good for regular payments)166  RTGS = instant settlement (good for large, urgent payments)167  Use RTGS when you need money to arrive immediately.168  Use NEFT when timing is not critical and amounts are smaller.169 170========================================171IMPS — IMMEDIATE PAYMENT SERVICE172========================================173Full form: Immediate Payment Service174Operated by: National Payments Corporation of India (NPCI)175Settlement: Real-time, 24x7x365176Purpose: Instant fund transfer at any amount177 178IMPS CHARGES (set by individual banks, not mandated by RBI):179  Up to ₹10,000       : ₹2.50 to ₹5.00 + GST (varies by bank)180  ₹10,001 to ₹1 lakh  : ₹5.00 to ₹15.00 + GST181  ₹1 lakh to ₹2 lakh  : ₹15.00 + GST (typically)182  Above ₹2 lakh        : ₹25.00 + GST (typically)183  Via mobile banking   : Often FREE (bank-specific)184 185NOTE: IMPS charges are bank-determined. Many banks offer free IMPS via mobile apps.186IMPS via USSD (*99#): ₹0.50 per transaction187 188IMPS TRANSACTION LIMITS:189  Minimum: ₹1190  Maximum: ₹5,00,000 (₹5 lakh) per transaction as per NPCI191  Some banks allow up to ₹2 lakh, others ₹5 lakh — check with your bank.192 193IMPS IDENTIFIERS:194  - MMID + Mobile number (mobile-to-mobile)195  - Account number + IFSC (account-to-account)196  - Aadhaar number (Aadhaar-linked)197 198IMPS vs NEFT vs RTGS COMPARISON:199  Feature         IMPS          NEFT          RTGS200  Settlement      Instant       30-min batch  Instant201  Min amount      ₹1            ₹1            ₹2,00,000202  Max amount      ₹5 lakh       No limit      No limit203  Availability    24x7          24x7          24x7204  Typical fee     ₹5-₹15+GST   ₹2.50-₹25+GST ₹24.50-₹49.50+GST205  Best for        Small-medium  Regular       High-value urgent206 207========================================208UPI — UNIFIED PAYMENTS INTERFACE209========================================210Full form: Unified Payments Interface211Operated by: NPCI212Settlement: Real-time213 214UPI CHARGES:215  Person-to-Person (P2P): FREE216  Person-to-Merchant (P2M): FREE (as per RBI and NPCI directives)217  Wallets to bank via UPI: FREE up to ₹2,000; ₹1 above ₹2,000 for PPIs218 219MDR (Merchant Discount Rate) on UPI:220  As of January 2020, RBI mandated ZERO MDR on UPI and RuPay transactions.221  Merchants cannot be charged for accepting UPI payments.222 223UPI TRANSACTION LIMITS:224  Standard: ₹1,00,000 per transaction225  UPI for capital markets: ₹2,00,000226  IPO applications: ₹5,00,000227  Medical/educational: ₹5,00,000228 229========================================230GST ON PAYMENT SERVICES231========================================232GST Rate on payment processing services: 18%233Applies to: NEFT fees, RTGS fees, IMPS fees, payment gateway fees234Does NOT apply to: The transaction amount itself (only on the service fee)235 236Example:237  NEFT transfer of ₹50,000 → Fee = ₹5 + 18% GST = ₹5.90 total charge238 239========================================240CROSS-BORDER / FOREX PAYMENTS241========================================242SWIFT charges (international wire transfers):243  Sending bank fee: ₹500 to ₹1,500 typically244  Correspondent bank charges: $15-$30 (deducted from amount)245  Currency conversion markup: 1% to 3.5% over mid-market rate246 247RBI guidelines on forex:248  Liberalized Remittance Scheme (LRS): Up to USD 2,50,000 per year per individual249  TCS (Tax Collected at Source): 20% on remittances above ₹7 lakh per year250  Form 15CA/15CB required for certain payments251"""252 253 254PCI_DSS_DATA = """255PCI-DSS — PAYMENT CARD INDUSTRY DATA SECURITY STANDARD256Source: PCI Security Standards Council (PCI SSC)257Current Version: PCI DSS v4.0 (released March 2022, mandatory since March 2024)258 259========================================260WHAT IS PCI-DSS?261========================================262PCI-DSS (Payment Card Industry Data Security Standard) is a set of security263standards designed to ensure that ALL companies that accept, process, store,264or transmit credit card information maintain a secure environment.265 266Created by: Visa, Mastercard, American Express, Discover, JCB (the major card brands)267Governed by: PCI Security Standards Council (PCI SSC)268Applies to: Any entity handling cardholder data — merchants, processors, gateways269 270========================================271PCI-DSS COMPLIANCE LEVELS272========================================273Level 1: Over 6 million card transactions per year274  - Annual on-site audit by Qualified Security Assessor (QSA)275  - Quarterly network scan by Approved Scanning Vendor (ASV)276 277Level 2: 1 to 6 million transactions per year278  - Annual Self-Assessment Questionnaire (SAQ)279  - Quarterly network scan280 281Level 3: 20,000 to 1 million e-commerce transactions per year282  - Annual SAQ283  - Quarterly network scan284 285Level 4: Fewer than 20,000 e-commerce OR up to 1 million other transactions286  - Annual SAQ287  - Quarterly network scan (recommended)288 289========================================29012 PCI-DSS REQUIREMENTS291========================================2921. Install and maintain a firewall to protect cardholder data2932. Do not use vendor-supplied defaults for passwords and security parameters2943. Protect stored cardholder data (encryption, masking)2954. Encrypt transmission of cardholder data across open, public networks2965. Use and regularly update anti-virus software2976. Develop and maintain secure systems and applications2987. Restrict access to cardholder data on a need-to-know basis2998. Assign a unique ID to each person with computer access3009. Restrict physical access to cardholder data30110. Track and monitor all access to network resources and cardholder data30211. Regularly test security systems and processes30312. Maintain an information security policy304 305========================================306PCI-DSS IN INDIA — CONTEXT307========================================308RBI mandates: All payment aggregators and gateways in India must be309PCI-DSS certified as per RBI Payment Aggregator guidelines (2020).310 311Razorpay: PCI-DSS Level 1 certified312PayU: PCI-DSS Level 1 certified313Paytm Payment Gateway: PCI-DSS certified314Stripe India: PCI-DSS Level 1 certified315PayPal India: PCI-DSS Level 1 certified316 317COST OF NON-COMPLIANCE:318  Card brand fines: $5,000 to $100,000 per month319  Increased transaction fees by acquiring banks320  Risk of losing ability to process card payments321  Data breach liability — average breach cost $4.45 million globally322 323========================================324TOKENIZATION AND PCI-DSS325========================================326Tokenization: Replacing card number with a non-sensitive equivalent (token)327Reduces PCI-DSS scope significantly — tokenized data is not cardholder data328All major Indian payment gateways use tokenization as per RBI mandate.329RBI Card-on-File tokenization mandate: Effective October 1, 2022.330"""331 332 333RAZORPAY_DATA = """334RAZORPAY — FEE STRUCTURE (INDIA)335Source: Razorpay official pricing page336Category: Payment Gateway — India's leading payment gateway337 338========================================339RAZORPAY STANDARD FEES340========================================341Domestic Cards (Debit/Credit):342  Standard: 2% per transaction343  International cards: 3% per transaction344 345UPI Transactions:346  UPI (P2M): 0% — FREE as per RBI mandate347  UPI Autopay: 0.10% for recurring mandates348 349Net Banking:350  Most banks: ₹15 flat per transaction (not percentage-based)351 352Wallets:353  Paytm, PhonePe, Amazon Pay: 2% per transaction354 355EMI Transactions:356  Bank EMI: 2% per transaction (EMI plans by bank)357  Cardless EMI: 3% per transaction358 359International Payments (via Razorpay):360  3% + currency conversion fee361 362Settlement Timeline:363  Standard: T+2 (2 business days after transaction)364  Early settlement: Available for fee (Razorpay RazorpayX)365 366GST: 18% applied on all Razorpay fees367Example: ₹10,000 transaction at 2% = ₹200 fee + ₹36 GST = ₹236 total deduction368You receive: ₹9,764369 370Razorpay minimum fee: No minimum371Razorpay monthly fee: ₹0 (no monthly subscription for standard plan)372 373========================================374RAZORPAY PAYMENT LINKS / PAGES375========================================376Standard transaction fee applies377No additional fee for payment links378No setup cost379 380========================================381RAZORPAY INTERNATIONAL ACCEPTANCE382========================================383Supports 100+ currencies384International cards: 3% + payment gateway fee385Cross-currency settlements: Available in USD, EUR, GBP, SGD, AED386"""387 388 389PAYPAL_INDIA_DATA = """390PAYPAL INDIA — FEE STRUCTURE391Source: PayPal India official fee page392 393========================================394PAYPAL FEES FOR INDIA (RECEIVING PAYMENTS)395========================================396Receiving from within India:397  Standard: 2.5% + ₹3 per transaction398 399Receiving international payments (export/freelance):400  Standard commercial rate: 4.4% + fixed fee401  Fixed fee by currency:402    USD: $0.30403    GBP: £0.20404    EUR: €0.35405    CAD: $0.30406 407TOTAL EFFECTIVE DEDUCTION on international payment:408  Currency conversion: ~3-4% spread over mid-market rate409  Transaction fee: 4.4% + fixed fee410  Total loss: ~7-8% on international payments is common411 412PayPal Currency Conversion:413  PayPal applies a currency conversion spread (markup) over the base exchange rate.414  This is typically 3% to 4% above the mid-market rate.415  This is separate from the transaction fee.416 417WhY PayPal Feels Expensive:418  1. Transaction fee (4.4%)419  2. Currency conversion markup (3-4%)420  3. Fixed per-transaction fee ($0.30 for USD)421  Combined, receiving USD 100 → you may get ₹7,800-8,100 instead of ~₹8,350422 423PayPal Withdrawal to Indian Bank Account:424  Withdrawal fee: FREE (no fee to withdraw to Indian bank)425  Time: 3-5 business days426 427PayPal GST in India:428  18% GST applies on PayPal's service fees (not on the transaction amount)429 430Alternatives for freelancers receiving international payments:431  Wise (TransferWise): ~0.5-1% fee, mid-market rate432  Payoneer: Free for same-currency transfers, 2% for USD withdrawal433  Razorpay: For domestic business payments434"""435 436 437# ──────────────────────────────────────────────438# SOURCES REGISTRY439# ──────────────────────────────────────────────440SOURCES = [441    # ── Static regulatory data (MOST IMPORTANT — always ingest these) ──442    {443        "type": "static",444        "content": RBI_PAYMENT_FEES_DATA,445        "save_name": "rbi_neft_rtgs_imps_upi_fees.txt",446        "description": "RBI NEFT, RTGS, IMPS, UPI fee structure (static)"447    },448    {449        "type": "static",450        "content": PCI_DSS_DATA,451        "save_name": "pci_dss_compliance_guide.txt",452        "description": "PCI-DSS compliance standard (static)"453    },454    {455        "type": "static",456        "content": RAZORPAY_DATA,457        "save_name": "razorpay_fees_india.txt",458        "description": "Razorpay fee structure (static)"459    },460    {461        "type": "static",462        "content": PAYPAL_INDIA_DATA,463        "save_name": "paypal_india_fees.txt",464        "description": "PayPal India fee structure (static)"465    },466 467    # ── Live scraped sources ──468    {469        "type": "webpage",470        "url": "https://www.paypal.com/in/webapps/mpp/paypal-fees",471        "save_name": "paypal_fees_live.txt",472        "description": "PayPal India fee page (live)"473    },474    {475        "type": "webpage",476        "url": "https://stripe.com/in/pricing",477        "save_name": "stripe_pricing_india.txt",478        "description": "Stripe India pricing (live)"479    },480    {481        "type": "webpage",482        "url": "https://razorpay.com/pricing/",483        "save_name": "razorpay_pricing_live.txt",484        "description": "Razorpay pricing page (live)"485    },486    {487        "type": "webpage",488        "url": "https://www.npci.org.in/what-we-do/upi/product-overview",489        "save_name": "npci_upi_overview.txt",490        "description": "NPCI UPI overview"491    },492]493 494 495def run_all_ingestions():496    print("\n🚀 Starting PayLens document ingestion (v2)...\n")497    results = []498 499    for source in SOURCES:500        try:501            if source["type"] == "static":502                text = ingest_static(source["content"], source["save_name"])503            elif source["type"] == "webpage":504                text = ingest_webpage(source["url"], source["save_name"])505            elif source["type"] == "pdf":506                text = ingest_pdf_from_url(source["url"], source["save_name"])507            else:508                text = ""509 510            results.append({511                "source": source["description"],512                "file": source["save_name"],513                "chars": len(text),514                "status": "✅ Success" if text else "⚠️  Empty"515            })516        except Exception as e:517            print(f"❌ Failed: {source['description']} — {e}")518            results.append({519                "source": source["description"],520                "file": source["save_name"],521                "chars": 0,522                "status": f"❌ {e}"523            })524 525    print("\n📊 Ingestion Summary:")526    print("-" * 70)527    for r in results:528        print(f"{r['status']}  {r['source']:45s}  {r['chars']:,} chars")529    print("-" * 70)530    print(f"Raw files saved to: {RAW_DIR}\n")531    return results532 533 534if __name__ == "__main__":535    run_all_ingestions()