Mansi-Yelkar/Reconcile-Razorpay

1"""2run_experiment.py3 4Batch Evaluation Framework for Reconcile.5Runs a 10,000+ event synthetic benchmark comparing:6  - Blind Retry7  - Rule-Based8  - Reconcile Orchestrator9 10Usage:11  python evaluation/run_experiment.py --events 1000012"""13 14import argparse15import sys16import os17 18# Add backend and project root directories to sys.path19PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))20BACKEND_DIR = os.path.join(PROJECT_ROOT, "backend")21if PROJECT_ROOT not in sys.path:22    sys.path.insert(0, PROJECT_ROOT)23if BACKEND_DIR not in sys.path:24    sys.path.insert(0, BACKEND_DIR)25 26from simulation.simulator import SimulationGenerator27from evaluation.baselines import BlindRetryStrategy, RuleBasedStrategy, ReconcileStrategy28 29 30def run_benchmark(n_events: int = 10000):31    events = SimulationGenerator.generate_events(n_events)32 33    total_revenue_processed = sum(e["amount"] for e in events) * 1.534    revenue_at_risk = sum(e["amount"] for e in events)35    eligible_revenue = sum(e["amount"] for e in events if e["ground_truth_recoverable"] and not e["is_duplicate_suspected"])36 37    # Blind Retry run38    blind_recovered = 0.039    blind_unsafe = 040 41    # Rule-Based run42    rule_recovered = 0.043 44    # Reconcile run45    reconcile_attempted = 0.046    reconcile_recovered = 0.047    reconcile_unsafe = 048    blocked_duplicates = 049    human_escalations = 050 51    for e in events:52        # Blind53        b_att, b_rec, b_uns, b_dup = BlindRetryStrategy.evaluate(e)54        blind_recovered += b_rec55        if b_uns:56            blind_unsafe += 157 58        # Rule59        r_att, r_rec, r_uns, r_dup = RuleBasedStrategy.evaluate(e)60        rule_recovered += r_rec61 62        # Reconcile63        x_att, x_rec, x_uns, x_dup = ReconcileStrategy.evaluate(e)64        if x_att:65            reconcile_attempted += e["amount"]66        reconcile_recovered += x_rec67        if x_uns:68            reconcile_unsafe += 169        if x_dup:70            blocked_duplicates += 171        if e["amount"] > 50000 or e["is_duplicate_suspected"]:72            human_escalations += 173 74    recovery_rate = (reconcile_recovered / revenue_at_risk * 100.0) if revenue_at_risk > 0 else 0.075    improvement_vs_blind = ((reconcile_recovered - blind_recovered) / blind_recovered * 100.0) if blind_recovered > 0 else 0.076    improvement_vs_rules = ((reconcile_recovered - rule_recovered) / rule_recovered * 100.0) if rule_recovered > 0 else 0.077 78    results = {79        "n_events": n_events,80        "total_revenue_processed": total_revenue_processed,81        "revenue_at_risk": revenue_at_risk,82        "eligible_revenue": eligible_revenue,83        "reconcilex_attempted": reconcile_attempted,84        "reconcilex_recovered": reconcile_recovered,85        "reconcile_attempted": reconcile_attempted,86        "reconcile_recovered": reconcile_recovered,87        "recovery_rate_pct": recovery_rate,88        "reconcilex_unsafe": reconcile_unsafe,89        "reconcile_unsafe": reconcile_unsafe,90        "blocked_duplicates": blocked_duplicates,91        "human_escalations": human_escalations,92        "blind_recovered": blind_recovered,93        "blind_unsafe": blind_unsafe,94        "rule_recovered": rule_recovered,95        "improvement_vs_blind_pct": improvement_vs_blind,96        "improvement_vs_rules_pct": improvement_vs_rules,97    }98 99    print("========================================")100    print("REVENUE RECOVERY EVALUATION BENCHMARK")101    print("========================================")102    print(f"Transactions:              {n_events:,}")103    print(f"Revenue processed:         INR {total_revenue_processed:,.2f}")104    print(f"Revenue at risk:           INR {revenue_at_risk:,.2f}")105    print(f"Recovery eligible:         INR {eligible_revenue:,.2f}")106    print(f"Recovery attempted:        INR {reconcile_attempted:,.2f}")107    print(f"Revenue recovered:         INR {reconcile_recovered:,.2f}")108    print(f"Recovery rate:              {recovery_rate:.2f}%")109    print("----------------------------------------")110    print(f"Unsafe recoveries:          {reconcile_unsafe} (Target: 0)")111    print(f"Duplicate actions blocked:  {blocked_duplicates}")112    print(f"Human escalations:          {human_escalations}")113    print("----------------------------------------")114    print("BASELINE COMPARISON")115    print("----------------------------------------")116    print(f"Blind Retry:                INR {blind_recovered:,.2f} ({blind_unsafe} unsafe debits)")117    print(f"Rule-Based:                 INR {rule_recovered:,.2f}")118    print(f"Reconcile:                  INR {reconcile_recovered:,.2f}")119    print(f"Improvement vs Blind Retry: {improvement_vs_blind:+.2f}%")120    print(f"Improvement vs Rules:       {improvement_vs_rules:+.2f}%")121    print("========================================")122 123    return results124 125 126if __name__ == "__main__":127    parser = argparse.ArgumentParser(description="Reconcile Batch Evaluation Benchmark")128    parser.add_argument("--events", type=int, default=10000, help="Number of synthetic payment events")129    args = parser.parse_args()130    run_benchmark(args.events)131