RohanExploit/Meta-hackathon

1"""Run inference twice and compare outputs for reproducibility evidence.2 3Requires API_BASE_URL, MODEL_NAME, HF_TOKEN and a running env server.4"""5 6from __future__ import annotations7 8import json9import os10import subprocess11import sys12import time13from pathlib import Path14 15 16ROOT = Path(__file__).resolve().parent.parent17 18 19def _run_once(index: int) -> tuple[bool, float, dict]:20    start = time.time()21    proc = subprocess.run(22        [sys.executable, "inference.py"],23        cwd=ROOT,24        capture_output=True,25        text=True,26    )27    duration = time.time() - start28 29    payload = {"returncode": proc.returncode, "stdout": proc.stdout, "stderr": proc.stderr}30 31    report = ROOT / f"inference_run_{index}.log"32    report.write_text(proc.stdout + "\n\n[stderr]\n" + proc.stderr, encoding="utf-8")33 34    mean_score = None35    for line in proc.stdout.splitlines()[::-1]:36        stripped = line.strip()37        if stripped.startswith("{") and '"mean_score"' in stripped:38            try:39                mean_score = json.loads(stripped).get("mean_score")40            except json.JSONDecodeError:41                mean_score = None42            break43 44    return proc.returncode == 0, duration, {"mean_score": mean_score, **payload}45 46 47def main() -> int:48    required = ["API_BASE_URL", "MODEL_NAME", "HF_TOKEN"]49    missing = [k for k in required if not os.getenv(k)]50    if missing:51        print(f"Missing required env vars: {', '.join(missing)}")52        return 153 54    ok1, t1, r1 = _run_once(1)55    ok2, t2, r2 = _run_once(2)56 57    print("Benchmark summary:")58    print(f"run1_ok={ok1} runtime_sec={t1:.2f} mean_score={r1.get('mean_score')}")59    print(f"run2_ok={ok2} runtime_sec={t2:.2f} mean_score={r2.get('mean_score')}")60    print(f"runtime_under_20min={t1 < 1200 and t2 < 1200}")61    print(f"score_match={r1.get('mean_score') == r2.get('mean_score')}")62    print("Logs: inference_run_1.log, inference_run_2.log")63 64    if not (ok1 and ok2):65        return 166    if not (t1 < 1200 and t2 < 1200):67        return 168    return 069 70 71if __name__ == "__main__":72    raise SystemExit(main())73