The-Myth/DeepThinkers
0
1#!/usr/bin/env python32"""3validate.py — Pre-submission validation script for OpenEnv.4Checks all items from the pre-submission checklist:5 1. openenv.yaml exists and is well-formed6 2. Typed Pydantic models exist7 3. step() / reset() / state() work correctly8 4. 3+ tasks with graders, scores in [0.0, 1.0]9 5. Dockerfile exists and is parseable10 6. inference.py exists at root11 7. Environment variables documented12 8. Stdout log format compliance13"""14 15import json16import os17import sys18import time19import subprocess20import importlib21 22# ── Utilities ────────────────────────────────────────────────────────────────23 24PASS = 025FAIL = 026WARN = 027 28 29def check(name: str, condition: bool, got=None, expected=None, critical: bool = True):30 global PASS, FAIL, WARN31 if condition:32 print(f" ✓ {name}")33 PASS += 134 elif critical:35 msg = f" ✗ {name}"36 if got is not None:37 msg += f" (got={got!r}"38 if expected is not None:39 msg += f", expected={expected!r}"40 msg += ")"41 print(msg)42 FAIL += 143 else:44 print(f" ⚠ {name} (warning)")45 WARN += 146 47 48ROOT = os.path.dirname(os.path.abspath(__file__))49 50 51def main():52 global PASS, FAIL, WARN53 54 # ─────────────────────────────────────────────55 # 1. File structure56 # ─────────────────────────────────────────────57 print("\n=== 1. File Structure ===")58 59 required_files = [60 "openenv.yaml",61 "Dockerfile",62 "README.md",63 "requirements.txt",64 "inference.py",65 "server/app.py",66 "server/environment.py",67 "server/models.py",68 "server/graders.py",69 "server/__init__.py",70 "data/emails.py",71 "data/__init__.py",72 ]73 74 for f in required_files:75 path = os.path.join(ROOT, f)76 check(f"File exists: {f}", os.path.exists(path))77 78 # ─────────────────────────────────────────────79 # 2. openenv.yaml validation80 # ─────────────────────────────────────────────81 print("\n=== 2. openenv.yaml ===")82 83 yaml_path = os.path.join(ROOT, "openenv.yaml")84 try:85 # Use simple YAML parsing (no PyYAML dependency needed)86 with open(yaml_path, encoding="utf-8") as f:87 yaml_content = f.read()88 check("openenv.yaml is readable", True)89 90 required_fields = ["name:", "version:", "tasks:", "observation_space:", "action_space:", "reward:", "endpoint:"]91 for field in required_fields:92 check(f"openenv.yaml has '{field}'", field in yaml_content)93 94 # Check for openenv tag95 check("openenv.yaml has 'openenv' tag", "openenv" in yaml_content)96 97 # Count tasks98 task_count = yaml_content.count("- name:")99 check(f"openenv.yaml defines 3+ tasks", task_count >= 3, got=task_count, expected=">=3")100 101 except FileNotFoundError:102 check("openenv.yaml exists", False)103 104 # ─────────────────────────────────────────────105 # 3. Pydantic typed models106 # ─────────────────────────────────────────────107 print("\n=== 3. Typed Models ===")108 109 sys.path.insert(0, ROOT)110 111 try:112 from server.models import (113 EmailObservation, TriageAction, TriageReward,114 StepResult, EpisodeState,115 Priority, Category, Sentiment,116 )117 check("Import EmailObservation", True)118 check("Import TriageAction", True)119 check("Import TriageReward", True)120 check("Import StepResult", True)121 check("Import EpisodeState", True)122 check("Import Priority enum", True)123 check("Import Category enum", True)124 check("Import Sentiment enum", True)125 126 # Verify they're proper Pydantic models127 obs = EmailObservation(email_id="test", subject="Test", sender="a@b.com")128 check("EmailObservation instantiable", obs is not None)129 check("EmailObservation has model_dump", hasattr(obs, "model_dump"))130 131 action = TriageAction(priority="critical", category="engineering")132 check("TriageAction instantiable", action is not None)133 check("TriageAction has model_dump", hasattr(action, "model_dump"))134 135 reward = TriageReward(total=0.75)136 check("TriageReward instantiable", reward is not None)137 138 except Exception as e:139 check(f"Import models: {e}", False)140 141 # ─────────────────────────────────────────────142 # 4. Environment: reset() / step() / state()143 # ─────────────────────────────────────────────144 print("\n=== 4. Environment Interface ===")145 146 try:147 from server.environment import EmailTriageEnv, TASK_NAMES148 from server.graders import TASK_MAX_STEPS149 150 check("3+ task names defined", len(TASK_NAMES) >= 3, got=len(TASK_NAMES))151 152 for task in TASK_NAMES:153 env = EmailTriageEnv(task_name=task, seed=42)154 155 # reset()156 obs = env.reset()157 check(f"{task}: reset() returns observation", obs is not None)158 check(f"{task}: obs has email_id", hasattr(obs, "email_id") and obs.email_id)159 check(f"{task}: obs.step == 0", obs.step == 0)160 161 # state()162 state = env.state()163 check(f"{task}: state() returns EpisodeState", state is not None)164 check(f"{task}: state.done == False after reset", not state.done)165 check(f"{task}: state.task_name correct", state.task_name == task)166 167 # step()168 action = {169 "priority": "high",170 "category": "engineering",171 "route_to": "engineering-oncall",172 "action_items": ["review logs"],173 "sla_hours": 4,174 "sentiment": "urgent",175 "flags": ["escalate"],176 "reasoning": "Test action.",177 }178 result = env.step(action)179 check(f"{task}: step() returns result", result is not None)180 check(f"{task}: reward in [0,1]",181 0.0 <= result.reward <= 1.0, got=result.reward)182 check(f"{task}: done is bool", isinstance(result.done, bool))183 check(f"{task}: result has observation", result.observation is not None)184 185 # state after step186 state2 = env.state()187 check(f"{task}: state.step==1 after step", state2.step == 1)188 189 except Exception as e:190 check(f"Environment interface: {e}", False)191 192 # ─────────────────────────────────────────────193 # 5. Graders: 3+ tasks, scores in [0.0, 1.0]194 # ─────────────────────────────────────────────195 print("\n=== 5. Task Graders ===")196 197 try:198 from server.graders import grade, TASK_GRADERS199 from data.emails import get_emails_for_task200 201 check("3+ graders defined", len(TASK_GRADERS) >= 3, got=len(TASK_GRADERS))202 203 for task in TASK_NAMES:204 emails = get_emails_for_task(task)205 check(f"{task}: has emails for grading", len(emails) >= 3, got=len(emails))206 207 for email in emails:208 gt = email["ground_truth"]209 210 # Perfect action211 perfect = {212 "priority": gt.get("priority"),213 "category": gt.get("category"),214 "route_to": gt.get("route_to"),215 "action_items": gt.get("action_items", []),216 "sla_hours": gt.get("sla_hours"),217 "sentiment": gt.get("sentiment"),218 "flags": gt.get("flags", []),219 }220 scores = grade(task, perfect, gt)221 check(f"{task}/{email['email_id']}: perfect score in [0,1]",222 0.0 <= scores["total"] <= 1.0, got=scores["total"])223 check(f"{task}/{email['email_id']}: perfect score >= 0.9",224 scores["total"] >= 0.9, got=scores["total"])225 226 # Worst action227 worst = {228 "priority": "low" if gt["priority"] == "critical" else "critical",229 "category": "spam",230 "route_to": "trash",231 "action_items": [],232 "sla_hours": 999,233 "sentiment": "positive" if gt.get("sentiment") == "urgent" else "urgent",234 "flags": [],235 }236 worst_scores = grade(task, worst, gt)237 check(f"{task}/{email['email_id']}: worst score in [0,1]",238 0.0 <= worst_scores["total"] <= 1.0, got=worst_scores["total"])239 240 except Exception as e:241 check(f"Grader validation: {e}", False)242 243 # ─────────────────────────────────────────────244 # 6. Dockerfile validation245 # ─────────────────────────────────────────────246 print("\n=== 6. Dockerfile ===")247 248 dockerfile_path = os.path.join(ROOT, "Dockerfile")249 try:250 with open(dockerfile_path, encoding="utf-8") as f:251 dockerfile = f.read()252 check("Dockerfile is readable", True)253 check("Dockerfile has FROM", "FROM" in dockerfile)254 check("Dockerfile has EXPOSE 7860", "EXPOSE 7860" in dockerfile)255 check("Dockerfile has CMD", "CMD" in dockerfile)256 check("Dockerfile copies requirements", "requirements.txt" in dockerfile)257 check("Dockerfile has HEALTHCHECK", "HEALTHCHECK" in dockerfile)258 except FileNotFoundError:259 check("Dockerfile exists", False)260 261 # ─────────────────────────────────────────────262 # 7. inference.py validation263 # ─────────────────────────────────────────────264 print("\n=== 7. inference.py ===")265 266 inference_path = os.path.join(ROOT, "inference.py")267 try:268 with open(inference_path, encoding="utf-8") as f:269 inf_content = f.read()270 check("inference.py is readable", True)271 check("inference.py uses OpenAI client", "OpenAI" in inf_content or "openai" in inf_content)272 check("inference.py reads HF_TOKEN", "HF_TOKEN" in inf_content)273 check("inference.py reads API_BASE_URL", "API_BASE_URL" in inf_content)274 check("inference.py reads MODEL_NAME", "MODEL_NAME" in inf_content)275 check("inference.py has [START] log", "[START]" in inf_content)276 check("inference.py has [STEP] log", "[STEP]" in inf_content)277 check("inference.py has [END] log", "[END]" in inf_content)278 check("inference.py has main()", "def main(" in inf_content)279 except FileNotFoundError:280 check("inference.py exists at root", False)281 282 # ─────────────────────────────────────────────283 # 8. Stdout format compliance284 # ─────────────────────────────────────────────285 print("\n=== 8. Stdout Log Format ===")286 287 import io288 import contextlib289 290 def capture_print(fn, *args, **kwargs):291 buf = io.StringIO()292 with contextlib.redirect_stdout(buf):293 fn(*args, **kwargs)294 return buf.getvalue().strip()295 296 def log_start(task, env, model):297 print(f"[START] task={task} env={env} model={model}", flush=True)298 299 def log_step(step, action, reward, done, error):300 error_val = error if error else "null"301 done_val = str(done).lower()302 action_clean = str(action).replace("\n", " ")[:200]303 print(f"[STEP] step={step} action={action_clean} reward={reward:.2f} done={done_val} error={error_val}", flush=True)304 305 def log_end(success, steps, score, rewards):306 rewards_str = ",".join(f"{r:.2f}" for r in rewards)307 print(f"[END] success={str(success).lower()} steps={steps} score={score:.3f} rewards={rewards_str}", flush=True)308 309 start_line = capture_print(log_start, "priority-classification", "email-triage-env", "TestModel")310 check("START: begins with [START]", start_line.startswith("[START]"))311 check("START: has task=", "task=" in start_line)312 check("START: has env=", "env=" in start_line)313 check("START: has model=", "model=" in start_line)314 315 step_line = capture_print(log_step, 1, '{"priority":"high"}', 0.75, False, None)316 check("STEP: begins with [STEP]", step_line.startswith("[STEP]"))317 check("STEP: has step=", "step=" in step_line)318 check("STEP: has reward=0.75", "reward=0.75" in step_line)319 check("STEP: has done=false", "done=false" in step_line)320 check("STEP: has error=null", "error=null" in step_line)321 322 end_line = capture_print(log_end, True, 5, 0.623, [0.5, 0.7, 0.6, 0.55, 0.7])323 check("END: begins with [END]", end_line.startswith("[END]"))324 check("END: has success=true", "success=true" in end_line)325 check("END: has steps=5", "steps=5" in end_line)326 check("END: has score=0.623", "score=0.623" in end_line)327 check("END: has rewards list", "rewards=" in end_line)328 check("END: no newlines in line", "\n" not in end_line)329 330 # ─────────────────────────────────────────────331 # 9. README checks332 # ─────────────────────────────────────────────333 print("\n=== 9. README.md ===")334 335 readme_path = os.path.join(ROOT, "README.md")336 try:337 with open(readme_path, encoding="utf-8") as f:338 readme = f.read()339 check("README.md is readable", True)340 check("README has environment description", "environment" in readme.lower() or "description" in readme.lower())341 check("README has action space", "action" in readme.lower() and "space" in readme.lower())342 check("README has observation space", "observation" in readme.lower() and "space" in readme.lower())343 check("README has task descriptions", "task" in readme.lower())344 check("README has setup instructions", "setup" in readme.lower() or "usage" in readme.lower())345 check("README has baseline scores", "baseline" in readme.lower() and "score" in readme.lower())346 347 # HF Space frontmatter348 check("README has HF Space frontmatter", readme.startswith("---"))349 check("README has sdk: docker", "sdk: docker" in readme)350 check("README has app_port", "app_port:" in readme)351 check("README has openenv tag in frontmatter", "openenv" in readme.split("---")[1] if readme.startswith("---") and readme.count("---") >= 2 else False)352 except FileNotFoundError:353 check("README.md exists", False)354 355 # ─────────────────────────────────────────────356 # 10. Environment variable config357 # ─────────────────────────────────────────────358 print("\n=== 10. Environment Variables ===")359 360 check("API_BASE_URL documented", "API_BASE_URL" in readme if 'readme' in dir() else False)361 check("MODEL_NAME documented", "MODEL_NAME" in readme if 'readme' in dir() else False)362 check("HF_TOKEN documented", "HF_TOKEN" in readme if 'readme' in dir() else False)363 364 # ─────────────────────────────────────────────365 # Summary366 # ─────────────────────────────────────────────367 total = PASS + FAIL + WARN368 print(f"\n{'='*60}")369 print(f"VALIDATION RESULTS: {PASS}/{total} passed", end="")370 if WARN:371 print(f" ({WARN} warnings)", end="")372 if FAIL:373 print(f" ({FAIL} FAILED)")374 print(f"\n❌ VALIDATION FAILED — fix {FAIL} issue(s) before submitting.")375 sys.exit(1)376 else:377 print(f"\n\n✅ ALL CHECKS PASSED — ready to submit!")378 sys.exit(0)379 380 381if __name__ == "__main__":382 main()383