pavan1910/ticket-triage-openenv
0
1#!/usr/bin/env python32"""3Deterministic test suite for the Ticket Triage OpenEnv environment.4Tests the server endpoints, reward logic, grader scoring, and edge cases5WITHOUT needing any LLM API key.6"""7 8import sys9import json10import requests11 12BASE = "http://localhost:7860"13PASS = 014FAIL = 015TOTAL = 016 17 18def log(msg, ok=True):19 global PASS, FAIL, TOTAL20 TOTAL += 121 if ok:22 PASS += 123 print(f" ✅ {msg}")24 else:25 FAIL += 126 print(f" ❌ {msg}")27 28 29def reset(task_id):30 r = requests.post(f"{BASE}/stateful/reset", json={"task_id": task_id}, timeout=10)31 r.raise_for_status()32 return r.json()33 34 35def step(action_type, ticket_id="", payload=""):36 r = requests.post(f"{BASE}/stateful/step", json={37 "action": {"action_type": action_type, "ticket_id": ticket_id, "payload": payload}38 }, timeout=10)39 r.raise_for_status()40 return r.json()41 42 43def state():44 r = requests.get(f"{BASE}/stateful/state", timeout=10)45 r.raise_for_status()46 return r.json()47 48 49# ═══════════════════════════════════════════════════════════════════════════════50# TEST 1: Health & Root Endpoints51# ═══════════════════════════════════════════════════════════════════════════════52 53def test_health_endpoints():54 print("\n🔬 Test 1: Health & Root Endpoints")55 56 r = requests.get(f"{BASE}/health")57 log(f"GET /health → {r.status_code}", r.status_code == 200 and r.json()["status"] == "healthy")58 59 r = requests.get(f"{BASE}/")60 data = r.json()61 log(f"GET / → {data.get('environment')}", data.get("environment") == "ticket-triage")62 63 64# ═══════════════════════════════════════════════════════════════════════════════65# TEST 2: Reset Endpoint (OpenEnv format: observation + reward + done)66# ═══════════════════════════════════════════════════════════════════════════════67 68def test_reset():69 print("\n🔬 Test 2: Reset Endpoint")70 71 for task_id, expected_count in [("task_easy", 2), ("task_medium", 3), ("task_hard", 4)]:72 data = reset(task_id)73 obs = data["observation"]74 75 # Verify OpenEnv response format76 has_reward = "reward" in data77 has_done = "done" in data78 log(79 f"Reset {task_id}: {len(obs['tickets'])} tickets, step={obs['step_number']}, score={obs['current_score']}, format_ok={has_reward and has_done}",80 len(obs["tickets"]) == expected_count81 and obs["step_number"] == 082 and obs["current_score"] == 0.083 and obs["pending_count"] == expected_count84 and has_reward85 and has_done86 )87 88 # Invalid task falls back to task_easy89 data = reset("invalid_task")90 log(f"Invalid task fallback: {len(data['observation']['tickets'])} tickets", len(data["observation"]["tickets"]) == 2)91 92 93# ═══════════════════════════════════════════════════════════════════════════════94# TEST 3: Perfect Run — task_easy95# ═══════════════════════════════════════════════════════════════════════════════96 97def test_perfect_easy():98 print("\n🔬 Test 3: Perfect Run — task_easy")99 reset("task_easy")100 101 total_reward = 0.0102 103 # T1: technical, medium, respond, close104 r = step("categorize", "T1", "technical")105 log(f"T1 categorize=technical → reward={r['reward']}", r["reward"] > 0)106 total_reward += r["reward"]107 108 r = step("set_priority", "T1", "medium")109 log(f"T1 priority=medium → reward={r['reward']}", r["reward"] > 0)110 total_reward += r["reward"]111 112 r = step("respond", "T1", "We're sorry about the login issue. Please try resetting your password using the 'Forgot Password' link on the login page. If you still cannot access your account, we can reset it for you.")113 log(f"T1 respond → reward={r['reward']}", r["reward"] > 0)114 total_reward += r["reward"]115 116 r = step("close", "T1", "")117 log(f"T1 close → reward={r['reward']}", r["reward"] > 0)118 total_reward += r["reward"]119 120 # T2: billing, medium, respond, close121 r = step("categorize", "T2", "billing")122 log(f"T2 categorize=billing → reward={r['reward']}", r["reward"] > 0)123 total_reward += r["reward"]124 125 r = step("set_priority", "T2", "medium")126 log(f"T2 priority=medium → reward={r['reward']}", r["reward"] > 0)127 total_reward += r["reward"]128 129 r = step("respond", "T2", "We apologize for the damage to your order #9921. We will process a full refund immediately. No need to return the damaged item.")130 log(f"T2 respond → reward={r['reward']}", r["reward"] > 0)131 total_reward += r["reward"]132 133 r = step("close", "T2", "")134 log(f"T2 close → reward={r['reward']}", r["reward"] > 0)135 total_reward += r["reward"]136 log(f"T2 close done={r['done']}", r["done"] is True)137 138 final_score = r["observation"]["current_score"]139 log(f"Final score: {final_score} (expected ≥ 0.90)", final_score >= 0.90)140 log(f"Total reward: {round(total_reward, 4)}", total_reward > 0.5)141 142 143# ═══════════════════════════════════════════════════════════════════════════════144# TEST 4: Perfect Run — task_medium (with escalation on T5)145# ═══════════════════════════════════════════════════════════════════════════════146 147def test_perfect_medium():148 print("\n🔬 Test 4: Perfect Run — task_medium")149 reset("task_medium")150 151 total_reward = 0.0152 153 # T3: technical, low, respond, close154 for atype, tid, payload, label in [155 ("categorize", "T3", "technical", "T3 cat"),156 ("set_priority", "T3", "low", "T3 pri"),157 ("respond", "T3", "To reset your password, go to the login page and click the 'Forgot Password' link. Enter your email and follow the reset instructions.", "T3 resp"),158 ("close", "T3", "", "T3 close"),159 ]:160 r = step(atype, tid, payload)161 total_reward += r["reward"]162 log(f"{label} → reward={r['reward']}", r["reward"] >= 0)163 164 # T4: shipping, high, respond, close165 for atype, tid, payload, label in [166 ("categorize", "T4", "shipping", "T4 cat"),167 ("set_priority", "T4", "high", "T4 pri"),168 ("respond", "T4", "We apologize about your order #443. We're tracking delivery with tracking number TRACK-8812 and will escalate with the carrier to ensure fast delivery.", "T4 resp"),169 ("close", "T4", "", "T4 close"),170 ]:171 r = step(atype, tid, payload)172 total_reward += r["reward"]173 log(f"{label} → reward={r['reward']}", r["reward"] >= 0)174 175 # T5: billing, high, respond, ESCALATE, close176 for atype, tid, payload, label in [177 ("categorize", "T5", "billing", "T5 cat"),178 ("set_priority", "T5", "high", "T5 pri"),179 ("respond", "T5", "We sincerely apologize for the duplicate charge. We will reverse the extra charge and issue a refund for transaction TXN-001123 immediately.", "T5 resp"),180 ("escalate", "T5", "", "T5 escalate"),181 ("close", "T5", "", "T5 close"),182 ]:183 r = step(atype, tid, payload)184 total_reward += r["reward"]185 log(f"{label} → reward={r['reward']}", r["reward"] >= 0)186 187 log(f"Episode done={r['done']}", r["done"] is True)188 final_score = r["observation"]["current_score"]189 log(f"Final score: {final_score} (expected ≥ 0.85)", final_score >= 0.85)190 log(f"Total reward: {round(total_reward, 4)}", total_reward > 0.5)191 192 193# ═══════════════════════════════════════════════════════════════════════════════194# TEST 5: Perfect Run — task_hard (security no-response, angry empathy)195# ═══════════════════════════════════════════════════════════════════════════════196 197def test_perfect_hard():198 print("\n🔬 Test 5: Perfect Run — task_hard")199 reset("task_hard")200 201 total_reward = 0.0202 203 # T6: security, critical, escalate ONLY (no respond!), then close204 for atype, tid, payload, label in [205 ("categorize", "T6", "security", "T6 cat"),206 ("set_priority", "T6", "critical", "T6 pri"),207 ("escalate", "T6", "", "T6 escalate"),208 ("close", "T6", "", "T6 close"),209 ]:210 r = step(atype, tid, payload)211 total_reward += r["reward"]212 log(f"{label} → reward={r['reward']}", r["reward"] >= 0)213 214 # T7: technical, critical, respond, escalate, close215 for atype, tid, payload, label in [216 ("categorize", "T7", "technical", "T7 cat"),217 ("set_priority", "T7", "critical", "T7 pri"),218 ("respond", "T7", "We're aware of the API v2 error and our engineering team is actively investigating the 500 errors on the /api/v2/orders endpoint. This is our top priority and we'll provide an update shortly.", "T7 resp"),219 ("escalate", "T7", "", "T7 escalate"),220 ("close", "T7", "", "T7 close"),221 ]:222 r = step(atype, tid, payload)223 total_reward += r["reward"]224 log(f"{label} → reward={r['reward']}", r["reward"] >= 0)225 226 # T8: billing, low, respond (upgrade inquiry), close — NO escalation227 for atype, tid, payload, label in [228 ("categorize", "T8", "billing", "T8 cat"),229 ("set_priority", "T8", "low", "T8 pri"),230 ("respond", "T8", "Thank you for your interest in upgrading your plan! Our Enterprise plan includes advanced features like priority support, custom integrations, and dedicated account management. I'd be happy to help you with the upgrade process.", "T8 resp"),231 ("close", "T8", "", "T8 close"),232 ]:233 r = step(atype, tid, payload)234 total_reward += r["reward"]235 log(f"{label} → reward={r['reward']}", r["reward"] >= 0)236 237 # T9: security, critical, escalate ONLY (legal data breach — no respond!), close238 for atype, tid, payload, label in [239 ("categorize", "T9", "security", "T9 cat"),240 ("set_priority", "T9", "critical", "T9 pri"),241 ("escalate", "T9", "", "T9 escalate"),242 ("close", "T9", "", "T9 close"),243 ]:244 r = step(atype, tid, payload)245 total_reward += r["reward"]246 log(f"{label} → reward={r['reward']}", r["reward"] >= 0)247 248 log(f"Episode done={r['done']}", r["done"] is True)249 final_score = r["observation"]["current_score"]250 log(f"Final score: {final_score} (expected ≥ 0.80)", final_score >= 0.80)251 log(f"Total reward: {round(total_reward, 4)}", total_reward > 1.0)252 253 254# ═══════════════════════════════════════════════════════════════════════════════255# TEST 6: Edge Cases256# ═══════════════════════════════════════════════════════════════════════════════257 258def test_edge_cases():259 print("\n🔬 Test 6: Edge Cases")260 reset("task_easy")261 262 # Invalid ticket ID263 r = step("categorize", "TX99", "technical")264 log(f"Invalid ticket ID → reward={r['reward']}", r["reward"] < 0)265 266 # Invalid category267 r = step("categorize", "T1", "unknown_category")268 log(f"Invalid category → reward={r['reward']}", r["reward"] < 0)269 270 # Invalid priority271 r = step("set_priority", "T1", "extreme")272 log(f"Invalid priority → reward={r['reward']}", r["reward"] < 0)273 274 # Response too short275 r = step("respond", "T1", "ok")276 log(f"Short response → reward={r['reward']}", r["reward"] < 0)277 278 # Note (any length is accepted)279 r = step("add_note", "T1", "hi")280 log(f"Short note → reward={r['reward']}", r["reward"] >= 0)281 282 # Noop penalty283 r = step("noop", "", "")284 log(f"Noop → reward={r['reward']}", r["reward"] < 0)285 286 # Unknown action287 r = step("fly_ticket", "T1", "")288 log(f"Unknown action → reward={r['reward']}", r["reward"] < 0)289 290 # Wrong category gives partial credit (not negative)291 r = step("categorize", "T1", "shipping")292 log(f"Wrong category → reward={r['reward']}", r["reward"] > 0)293 294 # Now fix and close properly295 r = step("categorize", "T1", "technical")296 log(f"Correct re-categorize (no double reward) → reward={r['reward']}", r["reward"] >= 0)297 298 299# ═══════════════════════════════════════════════════════════════════════════════300# TEST 7: State Endpoint301# ═══════════════════════════════════════════════════════════════════════════════302 303def test_state_endpoint():304 print("\n🔬 Test 7: State Endpoint")305 reset("task_easy")306 s = state()307 log(f"State: task={s['task_id']}, step={s['step_count']}, done={s['done']}",308 s["task_id"] == "task_easy" and s["step_count"] == 0 and s["done"] is False)309 310 step("categorize", "T1", "technical")311 s = state()312 log(f"After action: step={s['step_count']}, score={s['score']}", s["step_count"] == 1)313 314 315# ═══════════════════════════════════════════════════════════════════════════════316# TEST 8: Episode termination at step limit317# ═══════════════════════════════════════════════════════════════════════════════318 319def test_step_limit():320 print("\n🔬 Test 8: Step Limit Termination")321 reset("task_easy") # max_steps = 20322 323 for i in range(21):324 r = step("noop", "", "")325 if r["done"]:326 break327 328 log(f"Episode ended at step limit: done={r['done']}", r["done"] is True)329 330 331# ═══════════════════════════════════════════════════════════════════════════════332# TEST 9: Double close / actions on closed ticket333# ═══════════════════════════════════════════════════════════════════════════════334 335def test_double_close():336 print("\n🔬 Test 9: Double Close & Actions After Close")337 reset("task_easy")338 339 step("categorize", "T1", "technical")340 step("set_priority", "T1", "medium")341 step("respond", "T1", "Please reset your password via the forgot password link on your account page.")342 step("close", "T1", "")343 344 r = step("close", "T1", "")345 log(f"Double close → reward={r['reward']}", r["reward"] < 0)346 347 348# ═══════════════════════════════════════════════════════════════════════════════349# TEST 10: OpenEnv Response Format Validation350# ═══════════════════════════════════════════════════════════════════════════════351 352def test_openenv_response_format():353 print("\n🔬 Test 10: OpenEnv Response Format")354 355 # Test reset response format356 # Test the standard OpenEnv /reset endpoint (from create_fastapi_app)357 r = requests.post(f"{BASE}/reset", json={}, timeout=10)358 data = r.json()359 has_observation = "observation" in data360 has_reward = "reward" in data361 has_done = "done" in data362 log(f"OpenEnv /reset response has observation={has_observation}, reward={has_reward}, done={has_done}",363 has_observation and has_reward and has_done)364 365 # Test stateful reset format366 data = reset("task_easy")367 has_observation = "observation" in data368 has_reward = "reward" in data369 has_done = "done" in data370 no_info = "info" not in data371 log(f"Stateful reset has observation={has_observation}, reward={has_reward}, done={has_done}, no_info={no_info}",372 has_observation and has_reward and has_done and no_info)373 log(f"Reset reward is None", data["reward"] is None)374 log(f"Reset done is False", data["done"] is False)375 376 # Test step response format377 r = step("categorize", "T1", "technical")378 has_observation = "observation" in r379 has_reward = "reward" in r380 has_done = "done" in r381 no_info = "info" not in r382 log(f"Step response has observation={has_observation}, reward={has_reward}, done={has_done}, no_info={no_info}",383 has_observation and has_reward and has_done and no_info)384 log(f"Step reward is float", isinstance(r["reward"], (int, float)))385 log(f"Step done is bool", isinstance(r["done"], bool))386 387 # Test health response format388 h = requests.get(f"{BASE}/health").json()389 log(f"Health status is 'healthy'", h["status"] == "healthy")390 391 392# ═══════════════════════════════════════════════════════════════════════════════393# MAIN394# ═══════════════════════════════════════════════════════════════════════════════395 396if __name__ == "__main__":397 print("=" * 60)398 print(" 🎫 Ticket Triage OpenEnv — Test Suite")399 print("=" * 60)400 401 try:402 requests.get(f"{BASE}/health", timeout=3)403 except Exception:404 print(f"\n❌ Cannot reach server at {BASE}. Is it running?")405 sys.exit(1)406 407 test_health_endpoints()408 test_reset()409 test_perfect_easy()410 test_perfect_medium()411 test_perfect_hard()412 test_edge_cases()413 test_state_endpoint()414 test_step_limit()415 test_double_close()416 test_openenv_response_format()417 418 print("\n" + "=" * 60)419 print(f" Results: {PASS} passed / {FAIL} failed / {TOTAL} total")420 if FAIL == 0:421 print(" 🎉 ALL TESTS PASSED!")422 else:423 print(f" ⚠️ {FAIL} test(s) failed")424 print("=" * 60)425 426 sys.exit(0 if FAIL == 0 else 1)427