utk7rsh/Arbiter_Gen1
1
1"""Manual validation of 10 hand-crafted ARBITER episodes.2 3Confirms:4 1. Correct causal claims earn reward.5 2. Counterfactual claims pay double reward.6 3. Defender obfuscation is detectable but harder to claim correctly.7 4. Meta-Overseer flags genuine contradictions only.8 5. Auto-advancement triggers correctly.9 10Usage:11 python validate.py12"""13import sys14from pathlib import Path15sys.path.insert(0, str(Path(__file__).parent))16 17from arbiter.env.environment import ArbiterEnv18 19PASS = "[PASS]"20FAIL = "[FAIL]"21 22def check(label: str, condition: bool):23 status = PASS if condition else FAIL24 print(f" {status} {label}")25 return condition26 27 28def run_validation():29 print("=" * 60)30 print("ARBITER — Manual Validation (10 Episodes)")31 print("=" * 60)32 33 results = []34 for episode_idx in range(10):35 seed = episode_idx * 736 anomaly_type = (episode_idx % 3) + 137 print(f"\nEpisode {episode_idx+1:02d} | Anomaly Type {anomaly_type} | Seed {seed}")38 print("-" * 40)39 40 env = ArbiterEnv(level=1, seed=seed)41 obs = env.reset(seed=seed)42 ep = env._ep43 ainfo = env._anomaly_info44 45 # ── Test 1: Query returns records ────────────────────────────────────46 obs2, r, done, info = env.step({"type": "QUERY_RECORDS", "feature_filter": {}})47 t1 = check("QUERY_RECORDS returns records",48 len(info.get("query_result", [])) > 0)49 50 # ── Test 2: Correct causal claim earns reward ────────────────────────51 chain = ainfo.get("causal_chain", [])52 if len(chain) >= 2:53 claim = {54 "cause_feature": chain[0],55 "effect_outcome": chain[-1],56 "mechanism": chain[1] if len(chain) > 2 else chain[0],57 "direction": "positive",58 "confidence": "HIGH",59 "basis_records": ["rec_0000"],60 "anomaly_type": {1:"proxy_discrimination", 2:"adversarial_injection", 3:"model_drift"}[anomaly_type],61 }62 _, reward, _, vinfo = env.step({"type": "CLAIM_CAUSAL", "claim": claim})63 t2 = check(f"Correct causal claim earns reward (got {reward:.3f})", reward > 0)64 else:65 t2 = True # skip if no chain66 67 # ── Test 3: Counterfactual query works ───────────────────────────────68 rec0 = ep["records"][0]69 proxy_feat = ainfo.get("proxy_feature", "zip_code_cluster")70 _, _, _, cf_info = env.step({71 "type": "QUERY_COUNTERFACTUAL",72 "record_id": rec0["id"],73 "feature_id": proxy_feat,74 "counterfactual_value": "cluster_3",75 })76 cf_res = cf_info.get("cf_result", {})77 t3 = check("QUERY_COUNTERFACTUAL returns a valid result",78 "original_outcome" in cf_res and "counterfactual_outcome" in cf_res)79 80 # ── Test 4: CF claim pays double ──────────────────────────────────────81 cf_claim = {82 "subject_record": rec0["id"],83 "counterfactual_feature": proxy_feat,84 "predicted_outcome_change": cf_res.get("counterfactual_outcome", "approved"),85 "confidence": "HIGH",86 "basis": "causal_structure_inference",87 }88 env._last_cf_result = cf_res89 _, cf_reward, _, _ = env.step({"type": "CLAIM_COUNTERFACTUAL", "claim": cf_claim})90 t4 = check(f"Counterfactual claim reward <= 2.0 (got {cf_reward:.3f})", cf_reward <= 2.01)91 92 # ── Test 5: Meta-Overseer catches contradiction ───────────────────────93 from arbiter.env.meta_overseer import check_consistency94 contradictory_claims = [95 {"claim_type": "causal", "cause_feature": "A", "effect_outcome": "B",96 "confidence": "HIGH", "anomaly_type": "proxy_discrimination"},97 {"claim_type": "causal", "cause_feature": "B", "effect_outcome": "A",98 "confidence": "HIGH", "anomaly_type": "proxy_discrimination"},99 ]100 consistency = check_consistency(contradictory_claims)101 t5 = check(f"Meta-Overseer flags directional contradiction",102 consistency["num_violations"] > 0)103 104 # ── Test 6: No false positive on non-contradictory claims ─────────────105 clean_claims = [106 {"claim_type": "causal", "cause_feature": "zip_code_cluster",107 "effect_outcome": "denial_rate_overall", "confidence": "HIGH",108 "anomaly_type": "proxy_discrimination"},109 {"claim_type": "causal", "cause_feature": "credit_score",110 "effect_outcome": "approval_rate_overall", "confidence": "MEDIUM",111 "anomaly_type": "proxy_discrimination"},112 ]113 clean_check = check_consistency(clean_claims)114 t6 = check("Meta-Overseer: no false positive on valid claims",115 clean_check["num_violations"] == 0)116 117 # ── Test 7: SUBMIT_REPORT ends episode ────────────────────────────────118 _, _, done, ep_info = env.step({119 "type": "SUBMIT_REPORT",120 "anomaly_type": {1:"proxy_discrimination", 2:"adversarial_injection", 3:"model_drift"}[anomaly_type],121 "primary_evidence_chain": chain,122 "affected_demographic": ainfo.get("affected_demographic", "unknown"),123 "recommended_action": ainfo.get("recommended_action", "retrain"),124 })125 t7 = check("SUBMIT_REPORT ends episode", done)126 total_ep_reward = ep_info.get("episode_reward", {}).get("total", 0)127 print(f" Episode total reward: {total_ep_reward:.2f}")128 129 passed = sum([t1, t2, t3, t4, t5, t6, t7])130 results.append(passed)131 132 # ── Summary ───────────────────────────────────────────────────────────────133 print("\n" + "=" * 60)134 total_checks = len(results) * 7135 total_passed = sum(results)136 pct = total_passed / total_checks * 100137 print(f"Results: {total_passed}/{total_checks} checks passed ({pct:.1f}%)")138 139 # ── Test 8: Curriculum auto-advancement ──────────────────────────────────140 print("\nTesting curriculum auto-advancement...")141 from arbiter.env.curriculum import Curriculum142 from config import LEVEL_THRESHOLDS, ADVANCE_WINDOW143 curriculum = Curriculum(start_level=1)144 threshold = LEVEL_THRESHOLDS[1]145 new_level = None146 for _ in range(ADVANCE_WINDOW):147 new_level = curriculum.record(threshold + 1.0) # above threshold148 check(f"Curriculum advances from Level 1 after {ADVANCE_WINDOW} episodes above threshold",149 curriculum.level == 2 or new_level == 2)150 151 print("\nValidation complete.")152 153 154if __name__ == "__main__":155 run_validation()156