BHaritha/msme-openenv

1"""2validate.py - Pre-submission validator3Run this before submitting to catch any issues early.4Usage: python validate.py [ENV_URL]5"""6import sys7import json8import requests9 10ENV_URL = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:7860"11 12PASS = "[PASS]"13FAIL = "[FAIL]"14WARN = "[WARN]"15 16results = []17 18def check(name, fn):19    try:20        ok, msg = fn()21        status = PASS if ok else FAIL22        results.append((status, name, msg))23        print(f"{status} {name}: {msg}")24        return ok25    except Exception as e:26        results.append((FAIL, name, str(e)))27        print(f"{FAIL} {name}: {e}")28        return False29 30print(f"\nValidating environment at: {ENV_URL}\n{'='*50}")31 32# 1. Root returns 20033def test_root():34    r = requests.get(f"{ENV_URL}/", timeout=10)35    return r.status_code == 200, f"status={r.status_code}"36check("Root endpoint returns 200", test_root)37 38# 2. Health endpoint39def test_health():40    r = requests.get(f"{ENV_URL}/health", timeout=10)41    return r.status_code == 200, f"status={r.status_code}"42check("Health endpoint returns 200", test_health)43 44# 3. Tasks endpoint lists 3 tasks45def test_tasks():46    r = requests.get(f"{ENV_URL}/tasks", timeout=10)47    data = r.json()48    n = len(data.get("tasks", []))49    return n >= 3, f"found {n} tasks (need >= 3)"50check("Tasks endpoint lists 3+ tasks", test_tasks)51 52# 4-6. reset() works for all 3 task IDs53for tid in [1, 2, 3]:54    def test_reset(t=tid):55        r = requests.post(f"{ENV_URL}/reset", json={"task_id": t, "seed": 42}, timeout=15)56        data = r.json()57        has_obs = "observation" in data58        return r.status_code == 200 and has_obs, f"status={r.status_code}, has_observation={has_obs}"59    check(f"reset(task_id={tid}) returns observation", test_reset)60 61# 7-9. step() returns reward strictly in (0,1) for all tasks62dummy_actions = {63    1: {"label": "delayed_payment"},64    2: {"claimant": "Test Co", "opponent": "Other Co", "amount": 50000, "due_date": "31st March 2024", "days_overdue": 30},65    3: {"letter": "This is a formal demand letter from Test Company to Defendant Company. Invoice #001 for Rs. 50,000 is overdue. We demand payment under the MSME Act. Legal action and interest will apply if not paid within 15 days. We have all documentation to support this claim and will pursue all legal remedies available."}66}67for tid in [1, 2, 3]:68    def test_step(t=tid):69        requests.post(f"{ENV_URL}/reset", json={"task_id": t, "seed": 42}, timeout=15)70        r = requests.post(f"{ENV_URL}/step", json={"action": dummy_actions[t]}, timeout=30)71        data = r.json()72        reward = data.get("reward", -1)73        in_range = 0.001 < float(reward) < 0.99974        return r.status_code == 200 and in_range, f"reward={reward:.3f} (valid={in_range})"75    check(f"step() task {tid} returns reward in (0,1)", test_step)76 77# 10. state() endpoint78def test_state():79    r = requests.get(f"{ENV_URL}/state", timeout=10)80    return r.status_code == 200, f"status={r.status_code}"81check("state() endpoint works", test_state)82 83# 11. Reproducibility: same seed = same scenario84def test_reproducibility():85    r1 = requests.post(f"{ENV_URL}/reset", json={"task_id": 1, "seed": 99}, timeout=15).json()86    r2 = requests.post(f"{ENV_URL}/reset", json={"task_id": 1, "seed": 99}, timeout=15).json()87    s1 = r1["observation"]["email"]["subject"]88    s2 = r2["observation"]["email"]["subject"]89    return s1 == s2, f"same_subject={s1 == s2}"90check("Same seed produces same scenario (reproducible)", test_reproducibility)91 92# Summary93print(f"\n{'='*50}")94passed = sum(1 for s,_,_ in results if s == PASS)95total  = len(results)96print(f"Result: {passed}/{total} checks passed")97if passed == total:98    print("\nAll checks passed! Safe to submit.")99else:100    failed = [name for s,name,_ in results if s == FAIL]101    print(f"\nFailed checks:")102    for f in failed:103        print(f"  - {f}")104    print("\nFix the above before submitting.")105sys.exit(0 if passed == total else 1)106