Pandago/graphstrike-model-training
0
1"""Pre-submission validator for the Fake Gang Detection OpenEnv environment.2 3Checks all submission requirements and prints pass/fail for each.4Exits 0 if all checks pass, 1 if any fail.5 6Usage:7 python validate.py # server must be running on :80008 python validate.py --url http://host:80019 python validate.py --local # skip HTTP checks, test locally10"""11 12from __future__ import annotations13 14import argparse15import importlib16import json17import sys18import time19import urllib.error20import urllib.request21from pathlib import Path22from typing import Any, Dict, List, Optional, Tuple23 24_ROOT = Path(__file__).parent25sys.path.insert(0, str(_ROOT))26sys.path.insert(0, str(_ROOT / "server"))27 28# ---------------------------------------------------------------------------29# Helpers30# ---------------------------------------------------------------------------31 32_PASS = "[PASS]"33_FAIL = "[FAIL]"34_results: List[Tuple[bool, str]] = []35 36 37def check(name: str, ok: bool, detail: str = "") -> bool:38 tag = _PASS if ok else _FAIL39 line = f" {tag} {name}"40 if detail and not ok:41 line += f"\n → {detail}"42 print(line)43 _results.append((ok, name))44 return ok45 46 47def _get(url: str) -> Tuple[Optional[dict], Optional[str]]:48 try:49 with urllib.request.urlopen(url, timeout=10) as r:50 return json.loads(r.read()), None51 except Exception as exc:52 return None, str(exc)53 54 55def _post(url: str, body: Any = None) -> Tuple[Optional[dict], Optional[str]]:56 try:57 data = json.dumps(body or {}).encode()58 req = urllib.request.Request(59 url, data=data, headers={"Content-Type": "application/json"}, method="POST"60 )61 with urllib.request.urlopen(req, timeout=30) as r:62 return json.loads(r.read()), None63 except urllib.error.HTTPError as exc:64 try:65 body_bytes = exc.read()66 return None, f"HTTP {exc.code}: {body_bytes.decode()}"67 except Exception:68 return None, f"HTTP {exc.code}"69 except Exception as exc:70 return None, str(exc)71 72 73# ---------------------------------------------------------------------------74# HTTP checks75# ---------------------------------------------------------------------------76 77def run_http_checks(base_url: str) -> None:78 print(f"\nHTTP checks against {base_url}\n")79 80 # /health81 data, err = _get(f"{base_url}/health")82 check("/health reachable", data is not None and data.get("status") == "healthy", err or "")83 84 # /tasks — must have action_schema and 3 tasks85 data, err = _get(f"{base_url}/tasks")86 if check("/tasks reachable", data is not None, err or ""):87 has_schema = isinstance(data.get("action_schema"), dict)88 has_3_tasks = isinstance(data.get("tasks"), list) and len(data["tasks"]) == 389 has_score_range = "score_range" in data90 check("/tasks has action_schema", has_schema, str(data))91 check("/tasks has 3 tasks", has_3_tasks, str(data))92 check("/tasks has score_range", has_score_range, str(data))93 94 # /reset for each task95 for task in ["easy", "medium", "hard"]:96 data, err = _post(f"{base_url}/reset", {"task": task, "seed": 0})97 check(f"/reset task={task}", data is not None and "observation" in data, err or "")98 99 # /step — INSPECT, FLAG, SUBMIT cycle100 _post(f"{base_url}/reset", {"task": "easy", "seed": 0})101 obs_resp, err = _post(f"{base_url}/step", {"action_type": "inspect",102 "account_id": "acc_0000"})103 check("/step INSPECT", obs_resp is not None, err or "")104 105 # Get a visible account ID from the observation to flag106 acc_to_flag = None107 if obs_resp:108 vis_ids = obs_resp.get("observation", {}).get("visible_account_ids", [])109 if vis_ids:110 acc_to_flag = vis_ids[0]111 112 if acc_to_flag:113 flag_resp, err = _post(f"{base_url}/step", {"action_type": "flag",114 "account_id": acc_to_flag})115 check("/step FLAG", flag_resp is not None, err or "")116 117 sub_resp, err = _post(f"{base_url}/step", {"action_type": "submit"})118 check("/step SUBMIT", sub_resp is not None and sub_resp.get("done") is True, err or "")119 120 # /grader — must return float in [0, 1]121 data, err = _get(f"{base_url}/grader")122 if check("/grader reachable", data is not None, err or ""):123 score = data.get("score")124 check("/grader returns [0,1] float",125 isinstance(score, (int, float)) and 0.0 <= score <= 1.0,126 f"score={score}")127 128 # /baseline — must return 3 task scores in [0, 1]129 data, err = _post(f"{base_url}/baseline")130 if check("/baseline reachable", data is not None, err or ""):131 scores = data.get("scores", {})132 all_valid = (133 set(scores.keys()) == {"easy", "medium", "hard"}134 and all(isinstance(v, (int, float)) and 0.0 <= v <= 1.0135 for v in scores.values())136 )137 check("/baseline returns 3 valid scores", all_valid,138 f"got: {scores}")139 140 141# ---------------------------------------------------------------------------142# Local checks (no server needed)143# ---------------------------------------------------------------------------144 145def run_local_checks() -> None:146 print("\nLocal checks\n")147 148 # scoring.py importable and correct149 try:150 from scoring import ( # type: ignore[import]151 compute_fake_risk, compute_hub_legitimacy, grader_score152 )153 gang_risk = compute_fake_risk(0.75, 0.65, 0.85, 0.10)154 hub = compute_hub_legitimacy(2_000_000, 200, 2000, 0.05)155 celeb_risk = compute_fake_risk(0.02, 0.02, 0.10, hub)156 # Perfect score: 10 TP, 0 FP, 0 FN, 0 steps used → efficiency=1.0 → score=1.0157 perfect = grader_score(10, 0, 0, 0, 30)158 ok = (gang_risk >= 0.60 and celeb_risk < 0.20 and perfect == 1.0)159 check("scoring.py math correct", ok,160 f"gang_risk={gang_risk} celeb_risk={celeb_risk} perfect={perfect}")161 except Exception as exc:162 check("scoring.py importable", False, str(exc))163 164 # models.py has AccountStatus + new fields165 try:166 from models import AccountStatus, AccountProfile, FakeGangObservation # type: ignore[import]167 p = AccountProfile(168 account_id="acc_0001", follower_count=100, following_count=50,169 post_count=10, avg_post_hour=14.0, photo_reuse_score=0.8,170 bio_template_score=0.7, account_age_days=60,171 )172 check("models.py AccountProfile has fake_risk_score",173 hasattr(p, "fake_risk_score"), "")174 check("models.py FakeGangObservation has suspect_ids",175 hasattr(FakeGangObservation(), "suspect_ids"), "")176 check("models.py AccountStatus enum exists",177 AccountStatus.SUSPECT == "suspect", "")178 except Exception as exc:179 check("models.py new fields", False, str(exc))180 181 # environment.py runs episode + status cascade182 try:183 from environment import FakeGangEnvironment # type: ignore[import]184 from models import FakeGangAction, ActionType # type: ignore[import]185 env = FakeGangEnvironment()186 obs = env.reset(task="easy", seed=0)187 ep_path = _ROOT / "episodes" / "easy_000.json"188 if ep_path.exists():189 gang_id = json.loads(ep_path.read_text())["gang_member_ids"][0]190 obs = env.step(FakeGangAction(action_type=ActionType.INSPECT, account_id=gang_id))191 obs = env.step(FakeGangAction(action_type=ActionType.FLAG, account_id=gang_id))192 cascade_ok = len(obs.suspect_ids) > 0193 check("environment.py SUSPECT cascade works", cascade_ok,194 f"suspect_ids={obs.suspect_ids[:3]}")195 p_flagged = next((p for p in obs.visible_accounts if p.account_id == gang_id), None)196 check("environment.py fake_risk_score computed",197 p_flagged is not None and p_flagged.fake_risk_score > 0, "")198 else:199 check("episode file exists for cascade test", False,200 f"run python server/generator.py first")201 except Exception as exc:202 check("environment.py status cascade", False, str(exc))203 204 # inference.py importable + runs one episode locally205 try:206 from inference import run_rule_based_episode # type: ignore[import]207 from environment import FakeGangEnvironment # type: ignore[import]208 env2 = FakeGangEnvironment()209 score = run_rule_based_episode(env2, task="easy", seed=1)210 check("inference.py runs locally",211 isinstance(score, float) and 0.0 <= score <= 1.0,212 f"score={score}")213 except Exception as exc:214 check("inference.py importable", False, str(exc))215 216 # Episodes have new features217 ep_path = _ROOT / "episodes" / "easy_000.json"218 if ep_path.exists():219 ep = json.loads(ep_path.read_text())220 accounts = ep["network"]["accounts"]221 first = accounts[0]["features"]222 has_features = "comment_repeat_score" in first and "shared_ip_count" in first223 check("episodes have new features (comment_repeat_score, shared_ip_count)",224 has_features, f"keys: {list(first.keys())}")225 has_celebs = "celeb_ids" in ep226 check("episodes have celeb_ids field", has_celebs, "")227 else:228 check("episodes directory has files", False, "run python server/generator.py")229 230 231# ---------------------------------------------------------------------------232# Main233# ---------------------------------------------------------------------------234 235if __name__ == "__main__":236 parser = argparse.ArgumentParser()237 parser.add_argument("--url", default="http://localhost:8000")238 parser.add_argument("--local", action="store_true",239 help="Run local checks only (no server needed)")240 args = parser.parse_args()241 242 run_local_checks()243 244 if not args.local:245 run_http_checks(args.url)246 247 total = len(_results)248 passed = sum(1 for ok, _ in _results if ok)249 failed = total - passed250 251 print(f"\n{'='*50}")252 print(f"Results: {passed}/{total} passed", end="")253 if failed:254 print(f" ({failed} FAILED)")255 failed_names = [name for ok, name in _results if not ok]256 for name in failed_names:257 print(f" - {name}")258 print()259 sys.exit(1)260 else:261 print(" — all OK")262 print()263 sys.exit(0)264