Nomearod/agentbench
0
1"""Repeat harness runs k times per configuration with injected provenance.2 3This script can make real, PAID API calls. The free path is narrow: a *custom*4entry run with a mock config (provider.default: mock). It is the only path5exercised in CI. langchain entries always build a real ChatOpenAI/ChatAnthropic6from their --provider regardless of any config, so a mock config does NOT make7them free; and a custom entry without a mock config uses its real provider.8run_config_epochs therefore refuses any paid run unless allow_paid is set9(--allow-paid on the CLI), so invoking this script directly cannot silently10spend money -- the Makefile epochs target is not the only guard (guardrail 2).11Provenance is injected by post-processing each entry point's --output JSON into12an envelope file (design spec section 6); harness internals are never edited.13"""14 15import argparse16import datetime17import hashlib18import json19import os20import subprocess21import sys22from pathlib import Path23 24import yaml25 26# Repo root on path so the preflight can import agent_bench when this file is27# run as a script (python scripts/run_epochs.py puts scripts/ on sys.path[0],28# not the repo root). The heavy work still happens in subprocesses.29sys.path.insert(0, str(Path(__file__).resolve().parent.parent))30 31CROCKFORD = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"32 33PROVIDER_KEY_ENV = {"openai": "OPENAI_API_KEY", "anthropic": "ANTHROPIC_API_KEY"}34 35# Provider -> model recorded in langchain config_id (audit #7) AND imported by36# run_langchain_eval to build the client, so the recorded provenance and the37# model that actually bills are the same constant. Do not duplicate it.38MODEL_DEFAULTS = {"openai": "gpt-4o-mini", "anthropic": "claude-haiku-4-5-20251001"}39 40 41def _is_mock_config(path: Path | None) -> bool:42 """True only if the config actually sets provider.default: mock.43 44 The free-path guard must verify config CONTENT, not just that --mock-config45 was passed: a real config handed to --mock-config would otherwise be46 classified free and bill silently (paid-path audit finding #6).47 """48 if path is None:49 return False50 data = yaml.safe_load(Path(path).read_text()) or {}51 return bool(data.get("provider", {}).get("default") == "mock")52 53 54def new_ulid() -> str:55 ts = int(datetime.datetime.now(tz=datetime.timezone.utc).timestamp() * 1000)56 time_part = ""57 for _ in range(10):58 time_part = CROCKFORD[ts % 32] + time_part59 ts //= 3260 rand = int.from_bytes(os.urandom(10), "big")61 rand_part = ""62 for _ in range(16):63 rand_part = CROCKFORD[rand % 32] + rand_part64 rand //= 3265 return time_part + rand_part66 67 68def write_envelope(69 raw_output: Path,70 dest_dir: Path,71 run_id: str,72 config_id: str,73 epoch: int,74 code_version: str,75 dataset_version: str,76 timestamp: str,77) -> Path:78 dest_dir.mkdir(parents=True, exist_ok=True)79 envelope = {80 "run_id": run_id,81 "timestamp": timestamp,82 "config_id": config_id,83 "code_version": code_version,84 "dataset_version": dataset_version,85 "epoch": epoch,86 "results": json.loads(raw_output.read_text()),87 }88 out = dest_dir / f"{config_id.split('+')[0]}_e{epoch}.json"89 out.write_text(json.dumps(envelope, indent=1))90 return out91 92 93def _config_hash(path: Path | None) -> str:94 if path is None:95 return "00000000"96 return hashlib.sha256(path.read_bytes()).hexdigest()[:8]97 98 99def _dataset_version(golden_path: Path) -> str:100 return "sha-" + hashlib.sha256(golden_path.read_bytes()).hexdigest()[:8]101 102 103def _config_id(name: str, spec: dict, mock_config: Path | None) -> str:104 if spec["entry"] == "langchain":105 # Encode the real model so langchain-openai/-anthropic carry meaningful106 # provenance instead of the constant +00000000 sentinel (audit #7); the107 # model is what actually bills. config_id is free-form, schema untouched.108 return f"{name}+{MODEL_DEFAULTS[spec['provider']]}"109 return f"{name}+{_config_hash(mock_config or spec.get('config'))}"110 111 112def _code_version() -> str:113 sha = subprocess.run(114 ["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True, check=True115 ).stdout.strip()116 dirty = subprocess.run(117 ["git", "status", "--porcelain"], capture_output=True, text=True, check=True118 ).stdout.strip()119 return sha + ("-dirty" if dirty else "")120 121 122# name -> (entry, config yaml or None, provider flag or None, corpus, golden path)123REGISTRY: dict[str, dict] = {124 "custom-openai": {125 "entry": "custom",126 "config": Path("configs/default.yaml"),127 "corpus": "fastapi",128 "golden": Path("agent_bench/evaluation/datasets/tech_docs_golden.json"),129 },130 "custom-anthropic": {131 "entry": "custom",132 "config": Path("configs/anthropic.yaml"),133 "corpus": "fastapi",134 "golden": Path("agent_bench/evaluation/datasets/tech_docs_golden.json"),135 },136 "langchain-openai": {137 "entry": "langchain",138 "provider": "openai",139 "golden": Path("agent_bench/evaluation/datasets/tech_docs_golden.json"),140 },141 "langchain-anthropic": {142 "entry": "langchain",143 "provider": "anthropic",144 "golden": Path("agent_bench/evaluation/datasets/tech_docs_golden.json"),145 },146 # --- k8s corpus (campaign executed 2026-06-22, paid) --------------------147 # The full 25-question k8s campaign has run: K=5 per config, one run_id each.148 # Envelopes live under results/epochs/ (gitignored, force-added like the149 # fastapi run); tidy rows in results/long/k8s/; the report's k8s section150 # regenerates with `make evaluate-stats`. langchain has no --corpus flag (see151 # _entry_cmd), so k8s is custom-only and the framework-equivalence (TOST)152 # section is empty by construction, not a regression. Re-run with:153 # make epochs K=5 CONFIGS=custom-openai-k8s,custom-anthropic-k8s CONFIRM_PAID=1154 "custom-openai-k8s": {155 "entry": "custom",156 "config": Path("configs/default.yaml"),157 "corpus": "k8s",158 "golden": Path("agent_bench/evaluation/datasets/k8s_golden.json"),159 },160 "custom-anthropic-k8s": {161 "entry": "custom",162 "config": Path("configs/anthropic.yaml"),163 "corpus": "k8s",164 "golden": Path("agent_bench/evaluation/datasets/k8s_golden.json"),165 },166}167 168 169def _entry_cmd(spec: dict, raw_out: Path, mock_config: Path | None) -> list[str]:170 config = mock_config or spec.get("config")171 if spec["entry"] == "custom":172 cmd = [173 sys.executable,174 "scripts/evaluate.py",175 "--mode",176 "deterministic",177 "--output",178 str(raw_out),179 ]180 if config:181 cmd += ["--config", str(config)]182 if spec.get("corpus"):183 cmd += ["--corpus", spec["corpus"]]184 return cmd185 cmd = [186 sys.executable,187 "scripts/run_langchain_eval.py",188 "--provider",189 spec["provider"],190 "--output",191 str(raw_out),192 ]193 if config:194 cmd += ["--config", str(config)]195 return cmd196 197 198def run_config_epochs(199 name: str,200 k: int,201 dest_root: Path,202 mock_config: Path | None = None,203 golden_override: Path | None = None,204 allow_paid: bool = False,205) -> list[Path]:206 spec = REGISTRY[name]207 # Guardrail 2 (no silent paid calls): the only free path is a custom entry208 # with a mock config. langchain entries always use a real LLM; a custom209 # entry without a mock config uses its real provider. Refuse otherwise so210 # direct script invocation cannot bill around the Makefile's CONFIRM_PAID.211 if mock_config is not None and not _is_mock_config(mock_config):212 raise SystemExit(213 f"refusing: --mock-config {mock_config} does not set provider.default: mock, "214 "so it would make real (paid) API calls. Pass an actual mock config, "215 "or --allow-paid to confirm you intend to spend money."216 )217 is_free = spec["entry"] == "custom" and mock_config is not None218 if not is_free and not allow_paid:219 raise SystemExit(220 f"refusing: config {name!r} would make real (paid) API calls "221 f"(entry={spec['entry']}, mock_config={'set' if mock_config else 'none'}); "222 "pass --allow-paid to confirm you intend to spend money"223 )224 golden = golden_override or spec["golden"]225 config_id = _config_id(name, spec, mock_config)226 run_id = new_ulid()227 written = []228 for epoch in range(1, k + 1):229 # Raw harness output lives UNDER the run dir, not a sibling results/epochs/raw/,230 # so results/epochs/ contains only run_id dirs. The WP5 convert loop globs231 # results/epochs/*/ and would otherwise feed raw EvalResult lists (no envelope232 # wrapper) to convert_envelopes and crash (audit #4).233 raw_out = dest_root / run_id / "raw" / f"{name}_e{epoch}.json"234 raw_out.parent.mkdir(parents=True, exist_ok=True)235 subprocess.run(_entry_cmd(spec, raw_out, mock_config), check=True)236 written.append(237 write_envelope(238 raw_output=raw_out,239 dest_dir=dest_root / run_id,240 run_id=run_id,241 config_id=config_id,242 epoch=epoch,243 code_version=_code_version(),244 dataset_version=_dataset_version(golden),245 timestamp=datetime.datetime.now(tz=datetime.timezone.utc).isoformat(),246 )247 )248 return written249 250 251def _preflight(names: list[str], mock_config: Path | None) -> None:252 """Validate every requested config BEFORE any subprocess runs, so a broken253 config fails at zero cost instead of after earlier configs have already254 billed (paid-path audit finding #2). Checks the registry name, that a custom255 entry's corpus resolves in its config, that the golden file and corpus store256 exist, and that the API key for any paying entry is present in os.environ257 (load_dotenv has already run in main()).258 """259 from agent_bench.core.config import load_config260 261 problems: list[str] = []262 for name in names:263 spec = REGISTRY.get(name)264 if spec is None:265 problems.append(f"{name}: unknown config (known: {sorted(REGISTRY)})")266 continue267 if not Path(spec["golden"]).exists():268 problems.append(f"{name}: golden dataset missing: {spec['golden']}")269 if spec["entry"] == "custom":270 cfg_path = mock_config or spec.get("config")271 try:272 cfg = load_config(Path(cfg_path) if cfg_path else None)273 except Exception as exc:274 problems.append(f"{name}: config {cfg_path} failed to load ({exc})")275 continue276 provider = cfg.provider.default277 corpus = spec.get("corpus")278 if corpus and corpus not in cfg.corpora:279 problems.append(280 f"{name}: corpus {corpus!r} not in {cfg_path} corpora "281 f"{sorted(cfg.corpora)}; evaluate.py would exit 1 mid-run"282 )283 elif corpus and not Path(cfg.corpora[corpus].store_path).exists():284 problems.append(285 f"{name}: corpus {corpus!r} store {cfg.corpora[corpus].store_path} "286 "missing; build it (make ingest) before the campaign"287 )288 else:289 provider = spec["provider"]290 free = spec["entry"] == "custom" and _is_mock_config(mock_config)291 if not free and provider != "mock":292 env = PROVIDER_KEY_ENV.get(provider)293 if env and not os.environ.get(env):294 problems.append(f"{name}: {env} not set; a real {provider} run needs it")295 if problems:296 raise SystemExit(297 "preflight failed (no API calls made); fix before spending:\n - "298 + "\n - ".join(problems)299 )300 301 302def main() -> None:303 from dotenv import load_dotenv304 305 # Load the gitignored .env so the provider key reads (os.environ, in the306 # subprocesses) and the preflight key check below see the keys (audit #3).307 load_dotenv()308 parser = argparse.ArgumentParser(description=__doc__)309 parser.add_argument("--k", type=int, required=True)310 parser.add_argument("--configs", required=True, help="comma-separated registry names")311 parser.add_argument("--dest", default="results/epochs")312 parser.add_argument(313 "--mock-config", default=None, help="config YAML forcing provider mock (free)"314 )315 parser.add_argument("--golden", default=None, help="override golden path (tests only)")316 parser.add_argument(317 "--allow-paid",318 action="store_true",319 help="confirm real (paid) API calls for non-mock runs (langchain or no mock config)",320 )321 parser.add_argument(322 "--dry-run",323 action="store_true",324 help="run preflight only: validate every config, make no API calls, spend nothing",325 )326 args = parser.parse_args()327 names = args.configs.split(",")328 mock_config = Path(args.mock_config) if args.mock_config else None329 _preflight(names, mock_config=mock_config)330 if args.dry_run:331 print(f"preflight OK for {names}; --dry-run, no API calls made")332 return333 for name in names:334 files = run_config_epochs(335 name,336 args.k,337 Path(args.dest),338 mock_config=mock_config,339 golden_override=Path(args.golden) if args.golden else None,340 allow_paid=args.allow_paid,341 )342 print(f"{name}: wrote {len(files)} epoch envelopes")343 344 345if __name__ == "__main__":346 main()347