b4rty/torchdebug-env

1#!/usr/bin/env python32"""3TorchDebug pre-submission validator.4 5Runs a practical local checklist aligned to the hackathon pass/fail gate:6- openenv validate7- local environment smoke test (reset + step)8- optional docker build/run/reset smoke test9- optional baseline inference run (requires API env vars)10"""11 12from __future__ import annotations13 14import argparse15import json16import os17import shlex18import subprocess19import sys20from datetime import datetime, timezone21from pathlib import Path22 23 24ROOT = Path(__file__).resolve().parent25 26 27def run_cmd(cmd: str, *, cwd: Path | None = None, timeout: int = 1200) -> None:28    print(f"\n$ {cmd}")29    proc = subprocess.run(30        cmd,31        cwd=str(cwd or ROOT),32        shell=True,33        text=True,34        timeout=timeout,35    )36    if proc.returncode != 0:37        raise RuntimeError(f"Command failed (exit {proc.returncode}): {cmd}")38 39 40def check_openenv_validate() -> None:41    run_cmd("openenv validate .", cwd=ROOT, timeout=300)42 43 44def check_local_smoke() -> None:45    script = r'''46try:47    from torchdebug_env.server.torchdebug_environment import TorchDebugEnvironment48    from torchdebug_env.models import TorchDebugAction49except ImportError:50    from server.torchdebug_environment import TorchDebugEnvironment51    from models import TorchDebugAction52 53env = TorchDebugEnvironment()54obs = env.reset(task_id='basic_failures', scenario_id='easy_lr_too_high')55assert obs.done is False56o1 = env._process_action(TorchDebugAction(action_type='analyze_logs'))57assert o1.step_number == 158o2 = env._process_action(TorchDebugAction(action_type='diagnose', diagnosis='learning rate too high'))59assert o2.step_number == 260print('local-smoke-ok')61'''62    py = shlex.quote(sys.executable)63    run_cmd(f"{py} -c {shlex.quote(script)}", cwd=ROOT, timeout=120)64 65 66def check_tests() -> bool:67    py = shlex.quote(sys.executable)68    try:69        run_cmd(70            f"{py} -m pytest -q tests/test_reward.py tests/test_environment_flow.py",71            cwd=ROOT,72            timeout=240,73        )74        return True75    except RuntimeError:76        print("⚠️  pytest not available in current interpreter; skipping tests. "77              "Install with: python -m pip install pytest")78        return False79 80 81def write_report(report: dict) -> Path:82    out_dir = ROOT / "outputs" / "evals"83    out_dir.mkdir(parents=True, exist_ok=True)84    out_file = out_dir / "submission_report.json"85    with out_file.open("w", encoding="utf-8") as f:86        json.dump(report, f, indent=2)87    return out_file88 89 90def check_docker_smoke(image: str, keep_running: bool = False) -> bool:91    run_cmd(f"docker build -f server/Dockerfile -t {image} .", cwd=ROOT, timeout=1800)92    run_cmd(f"docker rm -f {image}-ctr >/dev/null 2>&1 || true", cwd=ROOT, timeout=30)93    run_cmd(f"docker run --rm -d -p 8000:8000 --name {image}-ctr {image}", cwd=ROOT, timeout=60)94    try:95        run_cmd("sleep 5 && curl -sSf http://localhost:8000/health", cwd=ROOT, timeout=30)96        run_cmd(97            "curl -sSf -X POST http://localhost:8000/reset "98            "-H 'content-type: application/json' "99            "-d '{\"task_id\":\"basic_failures\",\"scenario_id\":\"easy_lr_too_high\"}' "100            "| python -c \"import sys, json; o=json.load(sys.stdin); assert 'observation' in o; print('docker-reset-ok')\"",101            cwd=ROOT,102            timeout=30,103        )104        if keep_running:105            return True106    finally:107        if not keep_running:108            run_cmd(f"docker stop {image}-ctr >/dev/null 2>&1 || true", cwd=ROOT, timeout=30)109 110    return False111 112 113def stop_docker_container(image: str) -> None:114    run_cmd(f"docker stop {image}-ctr >/dev/null 2>&1 || true", cwd=ROOT, timeout=30)115 116 117def check_baseline(timeout_s: int) -> None:118    required = ["API_BASE_URL", "MODEL_NAME", "HF_TOKEN"]119    missing = [k for k in required if not os.environ.get(k)]120    if missing:121        raise RuntimeError(122            f"Missing required env vars for baseline: {', '.join(missing)}"123        )124 125    py = shlex.quote(sys.executable)126    run_cmd(f"{py} inference.py", cwd=ROOT, timeout=timeout_s)127 128    out_file = ROOT / "outputs" / "evals" / "baseline_results.json"129    if not out_file.exists():130        raise RuntimeError(f"Expected output not found: {out_file}")131    print(f"baseline-output-ok: {out_file}")132 133 134def main() -> int:135    parser = argparse.ArgumentParser(description="TorchDebug pre-submission checks")136    parser.add_argument("--docker", action="store_true", help="Include docker build/run smoke checks")137    parser.add_argument("--baseline", action="store_true", help="Run inference baseline (requires API env vars)")138    parser.add_argument("--skip-tests", action="store_true", help="Skip local grader tests")139    parser.add_argument("--baseline-timeout", type=int, default=1200, help="Timeout seconds for baseline run")140    parser.add_argument("--docker-image", default="torchdebug-env-local", help="Docker image tag to use")141    args = parser.parse_args()142 143    print("== TorchDebug pre-submission checks ==")144    checks = {145        "openenv_validate": False,146        "local_smoke": False,147        "tests": None if args.skip_tests else "pending",148        "docker_smoke": None if not args.docker else False,149        "baseline": None if not args.baseline else False,150    }151    docker_kept_running = False152 153    try:154        check_openenv_validate()155        checks["openenv_validate"] = True156 157        check_local_smoke()158        checks["local_smoke"] = True159 160        if not args.skip_tests:161            tests_ok = check_tests()162            checks["tests"] = True if tests_ok else "skipped"163 164        if args.docker:165            keep_running = args.baseline and not os.environ.get("ENV_BASE_URL")166            docker_kept_running = check_docker_smoke(args.docker_image, keep_running=keep_running)167            checks["docker_smoke"] = True168 169        if args.baseline:170            if docker_kept_running and not os.environ.get("ENV_BASE_URL"):171                os.environ["ENV_BASE_URL"] = "http://localhost:8000"172            check_baseline(args.baseline_timeout)173            checks["baseline"] = True174 175        status = "passed"176        exit_code = 0177    except Exception as e:178        print(f"\n❌ Presubmit failed: {e}")179        status = "failed"180        exit_code = 1181    finally:182        if docker_kept_running:183            try:184                stop_docker_container(args.docker_image)185            except Exception:186                pass187 188    report = {189        "generated_at": datetime.now(timezone.utc).isoformat(),190        "status": status,191        "checks": checks,192        "options": {193            "docker": args.docker,194            "baseline": args.baseline,195            "skip_tests": args.skip_tests,196            "baseline_timeout": args.baseline_timeout,197            "docker_image": args.docker_image,198        },199        "environment": {200            "python": sys.executable,201            "cwd": str(ROOT),202        },203    }204    report_file = write_report(report)205    print(f"\nReport written: {report_file}")206 207    if exit_code == 0:208        print("\n✅ All selected checks passed")209 210    return exit_code211 212 213if __name__ == "__main__":214    raise SystemExit(main())215