Nomearod/agentbench
0
1"""Generate (and freshness-check) the README's statistics plots.2 3A figure is a measurement too: an unpinned plot is an unpinned number. Every4plot here is generated from the SAME pinned source the README cells read --5``docs/_generated/stats_report.md`` -- and carries a ``source-hash`` of the6exact values it drew. ``check`` recomputes that hash from the current report and7fails loudly if a committed SVG drifted, the same discipline8``scripts/check_readme_stats.py`` applies to the table cells.9 10Usage (matplotlib only needed for ``generate``, lazy-imported there so ``check``11and the tests run without the ``[plots]`` extra)::12 13 python scripts/make_plots.py generate # rebuild SVGs from the report14 python scripts/make_plots.py check # assert committed SVGs are fresh15"""16 17import argparse18import hashlib19import json20import re21import sys22from pathlib import Path23 24ROOT = Path(__file__).resolve().parents[1]25REPORT = ROOT / "docs" / "_generated" / "stats_report.md"26# The judge-unfolding demonstration lives in the hand-maintained design doc, not27# the auto-generated report, so its one plot pins to this file instead.28JUDGE_DESIGN = ROOT / "docs" / "judge-design.md"29PLOTS_DIR = ROOT / "docs" / "_generated" / "plots"30# The 1280x640 GitHub social-preview card. It lives outside PLOTS_DIR because it31# is uploaded in repo settings rather than embedded in the README, so the32# freshness check (which walks PLOTS_DIR) does not gate it; it still carries the33# same source-hash pin and regenerates with `make plots`.34ASSETS_DIR = ROOT / "assets"35 36# Same KEY = value block the README markers pin to (scripts/check_readme_stats.py).37VALUE_RE = re.compile(r"^- ([a-z0-9_]+) = (.+)$", re.MULTILINE)38# Exactly 16 hex chars: source_hash() returns sha256()[:16], and bounding the39# length stops the match from running into a PNG tEXt chunk's trailing CRC byte40# when that byte happens to be an ASCII hex digit (seen on paired_diff).41HASH_RE = re.compile(r"source-hash:\s*([0-9a-f]{16})")42 43# (value-key fragment, display label, framework) in table-column order.44_CONFIGS = (45 ("custom_openai", "Custom OpenAI", "custom"),46 ("custom_anthropic", "Custom Anthropic", "custom"),47 ("langchain_openai", "LangChain OpenAI", "langchain"),48 ("langchain_anthropic", "LangChain Anthropic", "langchain"),49)50_METRICS = (("p_at_5", "P@5"), ("r_at_5", "R@5"))51 52 53def read_values(report_text: str) -> dict[str, str]:54 """Parse the report's README-values block into {KEY: value}."""55 return dict(VALUE_RE.findall(report_text))56 57 58def _ci(raw: str) -> tuple[float, float]:59 lo, hi = raw.strip().strip("[]").split(",")60 return float(lo), float(hi)61 62 63def forest_rows(values: dict[str, str], corpus: str = "fastapi") -> list[dict]:64 """The eight (config x metric) headline points with their 95 percent CIs."""65 rows = []66 for key, label, framework in _CONFIGS:67 for metric, mlabel in _METRICS:68 lo, hi = _ci(values[f"{corpus}_{key}_{metric}_ci"])69 rows.append(70 {71 "config": key,72 "label": label,73 "framework": framework,74 "metric": metric,75 "metric_label": mlabel,76 "mean": float(values[f"{corpus}_{key}_{metric}_mean"]),77 "lo": lo,78 "hi": hi,79 }80 )81 return rows82 83 84def significant_points(values: dict[str, str], corpus: str = "fastapi") -> set:85 """The (config, metric) points belonging to a 95 percent significant pair."""86 raw = values.get(f"{corpus}_significant_pairs_95", "none")87 points = set()88 for clause in raw.split(";"):89 m = re.match(r"\s*(\w+) vs (\w+) (p_at_5|r_at_5)\s*$", clause)90 if m:91 a, b, metric = m.groups()92 points.update({(a, metric), (b, metric)})93 return points94 95 96def forest_source(values: dict[str, str], corpus: str = "fastapi") -> dict:97 """The exact value set the forest plot draws -- hashed for provenance."""98 return {99 "rows": forest_rows(values, corpus),100 "significant": sorted(f"{c}:{m}" for c, m in significant_points(values, corpus)),101 "mde": values.get(f"{corpus}_mde_p_at_5_80"),102 }103 104 105# (custom-key, langchain-key, display label) for the framework difference plot.106_PAIRS = (107 ("custom_openai", "langchain_openai", "Custom OpenAI - LC OpenAI"),108 ("custom_anthropic", "langchain_anthropic", "Custom Anthropic - LC Anthropic"),109 ("custom_openai", "langchain_anthropic", "Custom OpenAI - LC Anthropic"),110 ("custom_anthropic", "langchain_openai", "Custom Anthropic - LC OpenAI"),111)112 113 114def significant_pairs(values: dict[str, str], corpus: str = "fastapi") -> set:115 """The (custom, langchain, metric) tuples flagged significant at 95 percent."""116 raw = values.get(f"{corpus}_significant_pairs_95", "none")117 out = set()118 for clause in raw.split(";"):119 m = re.match(r"\s*(\w+) vs (\w+) (p_at_5|r_at_5)\s*$", clause)120 if m:121 out.add((m.group(1), m.group(2), m.group(3)))122 return out123 124 125def paired_rows(values: dict[str, str], corpus: str = "fastapi") -> list[dict]:126 """One row per framework comparison: the mean paired difference with its127 nested 90/95 percent CIs and the pinned TOST verdict. The 90 percent CI reads128 against the +/-margin band (equivalence), the 95 percent against zero."""129 sig = significant_pairs(values, corpus)130 rows = []131 for custom, lc, label in _PAIRS:132 for metric, mlabel in _METRICS:133 stem = f"{corpus}_{custom}_vs_{lc}_{metric}"134 if f"{stem}_diff" not in values:135 continue136 rows.append(137 {138 "label": label,139 "metric": metric,140 "metric_label": mlabel,141 "diff": float(values[f"{stem}_diff"]),142 "ci90": _ci(values[f"{stem}_ci90"]),143 "ci95": _ci(values[f"{stem}_ci95"]),144 "tost": values.get(f"{stem}_tost", ""),145 "significant": (custom, lc, metric) in sig,146 "same_provider": custom.rsplit("_", 1)[1] == lc.rsplit("_", 1)[1],147 }148 )149 return rows150 151 152def paired_source(values: dict[str, str], corpus: str = "fastapi") -> dict:153 """The exact value set the paired-difference plot draws -- hashed for provenance."""154 return {"rows": paired_rows(values, corpus)}155 156 157# (corpus-key, display label) for the variance-decomposition contrast.158_VAR_CORPORA = (("fastapi", "FastAPI"), ("k8s", "Kubernetes"))159 160 161def variance_rows(values: dict[str, str]) -> list[dict]:162 """Per corpus: the P@5 variance split into between-question (stable difficulty)163 and within-question (epoch noise a single run hides), plus the ICC."""164 rows = []165 for key, label in _VAR_CORPORA:166 if f"{key}_icc_p_at_5" not in values:167 continue168 between = float(values[f"{key}_between_question_var_p_at_5"])169 within = float(values[f"{key}_within_question_var_p_at_5"])170 total = between + within171 rows.append(172 {173 "label": label,174 "between": between,175 "within": within,176 "icc": float(values[f"{key}_icc_p_at_5"]),177 "within_frac": within / total if total else 0.0,178 }179 )180 return rows181 182 183def variance_source(values: dict[str, str]) -> dict:184 """The exact value set the ICC-contrast plot draws -- hashed for provenance."""185 return {"rows": variance_rows(values)}186 187 188def mde_source(values: dict[str, str], corpus: str = "fastapi") -> dict:189 """The P@5 minimum detectable effect (80 percent power) and the four190 framework-pair |gaps| measured against it -- the detectability/power view.191 ``detectable`` is the resolution criterion |gap| >= MDE (NOT 95 percent192 significance); the two coincide on current data but are different questions."""193 mde = float(values[f"{corpus}_mde_p_at_5_80"])194 gaps = [195 {196 "label": r["label"],197 "abs_diff": abs(r["diff"]),198 "detectable": abs(r["diff"]) >= mde,199 "same_provider": r["same_provider"],200 }201 for r in paired_rows(values, corpus)202 if r["metric"] == "p_at_5"203 ]204 return {"mde": mde, "gaps": gaps}205 206 207def unfolding_source() -> dict:208 """Parse the judge-unfolding table from judge-design.md section 1.9. Pinned to209 that hand-maintained doc (NOT the auto report); editing the table fails the210 freshness check. Returns observed/truth pass-rate and the two corrected211 estimators (regularized D'Agostini, naive matrix inversion) with their CIs."""212 text = JUDGE_DESIGN.read_text()213 214 def row(label: str) -> tuple[float, float, float]:215 # point and bounds all sign-tolerant: the matrix-inversion estimator is the216 # unstable one and a negative point/bound is a valid table value, not a parse error.217 m = re.search(rf"{re.escape(label)} \| (-?[\d.]+), 95% CI \[(-?[\d.]+), (-?[\d.]+)\]", text)218 if not m:219 raise ValueError(f"judge-design.md section 1.9: row {label!r} not found/parseable")220 return float(m.group(1)), float(m.group(2)), float(m.group(3))221 222 def scalar(label: str) -> float:223 m = re.search(rf"{re.escape(label)} \| [\d/]+ = (-?[\d.]+)", text)224 if not m:225 raise ValueError(f"judge-design.md section 1.9: row {label!r} not found")226 return float(m.group(1))227 228 dago = row("Corrected, D'Agostini")229 matinv = row("Corrected, matrix inversion")230 # Pin the KNOWN-TRUTH row, not just observed: the figure's whole claim is that231 # the corrected CI contains a separately-known truth, so a drift in that row232 # must invalidate the plot even though the truth line currently equals observed.233 return {234 "observed": scalar("Observed (jury) pass-rate"),235 "known_true": scalar("Known true pass-rate"),236 "dagostini": {"point": dago[0], "lo": dago[1], "hi": dago[2]},237 "matrix_inversion": {"point": matinv[0], "lo": matinv[1], "hi": matinv[2]},238 }239 240 241def source_hash(obj) -> str:242 return hashlib.sha256(json.dumps(obj, sort_keys=True).encode()).hexdigest()[:16]243 244 245def embedded_hash(svg_text: str) -> str | None:246 m = HASH_RE.search(svg_text)247 return m.group(1) if m else None248 249 250# name -> source-set builder. One entry per committed figure. PNG (not SVG):251# GitHub's Markdown sanitizer does not reliably render relative-path SVGs inline,252# so the figure ships as PNG and the source-hash lives in a PNG tEXt chunk.253EXPECTED_PLOTS = {254 "forest_fastapi.png": lambda v: forest_source(v, "fastapi"),255 "paired_diff_fastapi.png": lambda v: paired_source(v, "fastapi"),256 "icc_contrast.png": variance_source,257 "mde_resolution.png": lambda v: mde_source(v, "fastapi"),258 "unfolding_shift.png": lambda v: unfolding_source(),259}260 261 262def check(report_text: str, plots_dir: Path) -> list[str]:263 """One failure string per missing or stale plot (empty == all fresh)."""264 values = read_values(report_text)265 failures = []266 for name, builder in EXPECTED_PLOTS.items():267 svg = plots_dir / name268 if not svg.exists():269 failures.append(f"{name} missing; run `make plots`")270 continue271 want = source_hash(builder(values))272 # latin-1 maps every byte, so this never raises on a binary PNG and keeps273 # the ASCII source-hash substring intact (a PNG tEXt chunk is plain text).274 got = embedded_hash(svg.read_bytes().decode("latin-1", "ignore"))275 if got != want:276 failures.append(277 f"{name} stale: embedded source-hash {got} != report {want}; run `make plots`"278 )279 return failures280 281 282def _save_with_hash(fig, out_path: Path, h: str, dpi: int = 200) -> None:283 """Write the figure and embed ``source-hash:<h>`` -- an XML comment for SVG,284 a PNG tEXt chunk for raster. ``check`` reads it back to detect drift."""285 import matplotlib.pyplot as plt286 287 if out_path.suffix == ".svg":288 fig.savefig(out_path, metadata={"Date": None})289 plt.close(fig)290 svg = out_path.read_text().replace("</svg>", f"<!-- source-hash: {h} -->\n</svg>", 1)291 out_path.write_text(svg)292 else: # raster (png): the hash rides in a PNG tEXt chunk via savefig metadata293 fig.savefig(out_path, dpi=dpi, metadata={"Description": f"source-hash:{h}"})294 plt.close(fig)295 296 297def _render_forest(298 values: dict[str, str],299 out_path: Path,300 figsize: tuple[float, float] = (7.6, 4.4),301 title: str = "FastAPI retrieval: framework comparison (overlapping CIs)",302 dpi: int = 200,303) -> None:304 import matplotlib305 306 matplotlib.use("Agg")307 matplotlib.rcParams["svg.hashsalt"] = "agent-bench" # deterministic element ids308 import matplotlib.pyplot as plt309 from matplotlib.lines import Line2D310 311 rows = forest_rows(values)312 sig = significant_points(values)313 colors = {"custom": "#2b6cb0", "langchain": "#dd6b20"}314 315 # P@5 group on top, R@5 below; four configs each in table order. y descends so316 # the first config sits highest in its group, with a one-row gap between groups.317 groups = [[r for r in rows if r["metric"] == metric] for metric, _ in _METRICS]318 fig, ax = plt.subplots(figsize=figsize)319 y = float(len(rows) + 1)320 group_tops = []321 for group in groups:322 group_tops.append(y)323 for r in group:324 highlighted = (r["config"], r["metric"]) in sig325 fw = r["framework"]326 ax.errorbar(327 r["mean"],328 y,329 xerr=[[r["mean"] - r["lo"]], [r["hi"] - r["mean"]]],330 fmt="o",331 color=colors[fw],332 ecolor=colors[fw],333 elinewidth=2,334 capsize=4,335 markersize=10 if highlighted else 7,336 markeredgecolor="#b7791f" if highlighted else "white",337 markeredgewidth=2.2 if highlighted else 0.8,338 zorder=3 if highlighted else 2,339 )340 ax.text(r["lo"] - 0.012, y, r["label"], ha="right", va="center", fontsize=8.5)341 y -= 1342 y -= 1 # gap between metric groups343 344 ax.set_yticks([])345 ax.set_xlim(0.40, 1.02)346 ax.set_xlabel("score (95% CI, cluster bootstrap)")347 for (_, mlabel), top in zip(_METRICS, group_tops):348 ax.text(0.40, top + 0.5, mlabel, fontsize=11, fontweight="bold", va="bottom")349 ax.set_title(title, fontsize=11)350 351 handles = [352 Line2D(353 [0],354 [0],355 marker="o",356 color="w",357 markerfacecolor=colors["custom"],358 markersize=8,359 label="custom",360 ),361 Line2D(362 [0],363 [0],364 marker="o",365 color="w",366 markerfacecolor=colors["langchain"],367 markersize=8,368 label="LangChain",369 ),370 Line2D(371 [0],372 [0],373 marker="o",374 color="w",375 markerfacecolor="#cbd5e0",376 markeredgecolor="#b7791f",377 markeredgewidth=2,378 markersize=8,379 label="only significant pair (95%)",380 ),381 ]382 ax.legend(383 handles=handles,384 loc="upper center",385 bbox_to_anchor=(0.5, -0.12),386 ncol=3,387 fontsize=8,388 frameon=False,389 )390 ax.spines[["top", "right"]].set_visible(False)391 fig.tight_layout()392 _save_with_hash(fig, out_path, source_hash(forest_source(values)), dpi=dpi)393 394 395def _render_paired(values: dict[str, str], out_path: Path) -> None:396 import matplotlib397 398 matplotlib.use("Agg")399 matplotlib.rcParams["svg.hashsalt"] = "agent-bench"400 import matplotlib.pyplot as plt401 from matplotlib.lines import Line2D402 from matplotlib.patches import Patch403 404 rows = paired_rows(values)405 margin = 0.10406 sig_color, base_color, tie_color = "#b7791f", "#2b6cb0", "#718096"407 408 # group by metric (P@5 top, R@5 below); same-provider before cross within each409 def order(r: dict) -> tuple:410 return (0 if r["same_provider"] else 1, r["label"])411 412 groups = [413 (mlabel, sorted((r for r in rows if r["metric"] == metric), key=order))414 for metric, mlabel in _METRICS415 ]416 417 fig, ax = plt.subplots(figsize=(8.4, 5.6))418 ax.axvspan(-margin, margin, color="#e2e8f0", zorder=0) # equivalence band, for the 90% bar419 ax.axvline(0.0, color="#1a202c", lw=1.3, zorder=1) # zero rule, for the 95% caps420 421 y = float(sum(len(g) for _, g in groups) + len(groups))422 group_tops = []423 for mlabel, group in groups:424 group_tops.append((mlabel, y))425 for r in group:426 color = sig_color if r["significant"] else base_color427 lo95, hi95 = r["ci95"]428 lo90, hi90 = r["ci90"]429 if lo95 == hi95 == 0.0: # identical recall: a 0-width bar would read as a render bug430 ax.plot(0.0, y, marker="D", color=tie_color, markersize=8, zorder=6)431 ax.text(0.024, y, "exact tie (Δ=0)", va="center", fontsize=7.5, color=tie_color)432 else:433 ax.plot([lo95, hi95], [y, y], color=color, lw=1.3, zorder=3) # 95% thin434 for x in (lo95, hi95): # 95% caps435 ax.plot([x, x], [y - 0.14, y + 0.14], color=color, lw=1.3, zorder=3)436 ax.plot([lo90, hi90], [y, y], color=color, lw=5.5, solid_capstyle="butt", zorder=4)437 ax.plot(438 r["diff"],439 y,440 marker="o",441 color=color,442 markersize=9 if r["significant"] else 6,443 markeredgecolor="white",444 markeredgewidth=0.8,445 zorder=6,446 )447 ax.text(-0.215, y, r["label"], ha="right", va="center", fontsize=8)448 y -= 1449 y -= 1450 451 ax.set_yticks([])452 ax.set_xlim(-0.22, 0.40)453 # Headroom above the top group so the title clears the bold "P@5" label.454 ax.set_ylim(1.2, max(t for _, t in group_tops) + 1.6)455 ax.set_xlabel("paired difference: custom − LangChain (per-question, cluster bootstrap)")456 for mlabel, top in group_tops:457 ax.text(-0.215, top + 0.5, mlabel, fontsize=11, fontweight="bold", va="bottom")458 ax.set_title(459 "Framework difference (paired): equivalence vs the ±0.10 band, significance vs zero",460 fontsize=10.5,461 pad=12,462 )463 464 handles = [465 Line2D([0], [0], color=base_color, lw=5.5, label="90% CI — equivalence (vs ±0.10 band)"),466 Line2D([0], [0], color=base_color, lw=1.3, label="95% CI — significance (vs zero)"),467 Line2D(468 [0],469 [0],470 marker="o",471 color="w",472 markerfacecolor=sig_color,473 markersize=8,474 label="significant pair (95%)",475 ),476 Patch(facecolor="#e2e8f0", label="±0.10 TOST margin"),477 ]478 ax.legend(479 handles=handles,480 loc="upper center",481 bbox_to_anchor=(0.5, -0.11),482 ncol=2,483 fontsize=8,484 frameon=False,485 )486 ax.spines[["top", "right", "left"]].set_visible(False)487 fig.tight_layout()488 _save_with_hash(fig, out_path, source_hash(paired_source(values)))489 490 491def _render_icc(values: dict[str, str], out_path: Path) -> None:492 import matplotlib493 494 matplotlib.use("Agg")495 matplotlib.rcParams["svg.hashsalt"] = "agent-bench"496 import matplotlib.pyplot as plt497 from matplotlib.patches import Patch498 499 rows = variance_rows(values)500 between_color, within_color = "#cbd5e0", "#dd6b20"501 502 fig, ax = plt.subplots(figsize=(8.2, 2.9))503 ys = list(range(len(rows)))[::-1] # first corpus on top504 for r, y in zip(rows, ys):505 bf = 1.0 - r["within_frac"] # between-question fraction == ICC506 ax.barh(y, bf, height=0.5, color=between_color)507 ax.barh(y, r["within_frac"], left=bf, height=0.5, color=within_color)508 ax.text(1.015, y, f"ICC {r['icc']:.2f}", va="center", fontsize=10, fontweight="bold")509 ax.text(510 0.0,511 y - 0.42,512 f"within-question (epoch noise): {r['within_frac'] * 100:.1f}% of P@5 variance",513 va="top",514 ha="left",515 fontsize=8,516 color="#4a5568",517 )518 ax.set_yticks(ys)519 ax.set_yticklabels([r["label"] for r in rows], fontsize=12, fontweight="bold")520 ax.set_xlim(0, 1.0)521 ax.set_xticks([0, 0.25, 0.5, 0.75, 1.0])522 ax.set_xticklabels(["0", "25%", "50%", "75%", "100%"])523 ax.set_ylim(-0.85, max(ys) + 0.75)524 ax.set_xlabel("share of P@5 variance")525 ax.set_title(526 "A single run hides a distribution — and how much depends on the corpus",527 fontsize=11,528 pad=10,529 )530 ax.legend(531 handles=[532 Patch(facecolor=between_color, label="between-question (stable difficulty)"),533 Patch(facecolor=within_color, label="within-question (epoch noise, hidden by one run)"),534 ],535 loc="upper center",536 bbox_to_anchor=(0.5, -0.32),537 ncol=2,538 fontsize=8,539 frameon=False,540 )541 ax.spines[["top", "right", "left"]].set_visible(False)542 fig.tight_layout()543 _save_with_hash(fig, out_path, source_hash(variance_source(values)))544 545 546def _render_mde(values: dict[str, str], out_path: Path) -> None:547 import matplotlib548 549 matplotlib.use("Agg")550 matplotlib.rcParams["svg.hashsalt"] = "agent-bench"551 import matplotlib.pyplot as plt552 553 src = mde_source(values)554 mde = src["mde"]555 sig_color, base_color = "#b7791f", "#2b6cb0"556 557 fig, ax = plt.subplots(figsize=(8.2, 2.4))558 ax.axvspan(0, mde, color="#e2e8f0", zorder=0) # below-resolution zone559 ax.axvline(mde, color="#718096", lw=1.3, ls="--", zorder=1)560 ax.text(561 mde + 0.004,562 0.82,563 f"resolution floor\nMDE {mde:.3f} (80% power)",564 fontsize=8.5,565 color="#4a5568",566 va="top",567 )568 detectable = [g for g in src["gaps"] if g["detectable"]]569 below = len(src["gaps"]) - len(detectable)570 for g in src["gaps"]:571 color = sig_color if g["detectable"] else base_color # past the floor, not 95% significance572 ax.plot(573 g["abs_diff"],574 0,575 marker="o",576 markersize=11,577 color=color,578 markeredgecolor="white",579 markeredgewidth=0.8,580 zorder=3,581 )582 if len(detectable) == 1:583 g = detectable[0]584 prov = "cross-provider" if not g["same_provider"] else "same-provider"585 ax.annotate(586 f"{g['label']} +{g['abs_diff']:.3f}\n({prov}: the one gap above the floor)",587 (g["abs_diff"], 0),588 xytext=(g["abs_diff"], -0.9),589 ha="center",590 fontsize=8,591 color=sig_color,592 arrowprops=dict(arrowstyle="-", color=sig_color, lw=0.8),593 )594 else: # 0 -> no annotation; >1 -> label each gap that clears the floor595 for g in detectable:596 ax.annotate(597 g["label"],598 (g["abs_diff"], 0),599 xytext=(g["abs_diff"], -0.9),600 ha="center",601 fontsize=7.5,602 color=sig_color,603 arrowprops=dict(arrowstyle="-", color=sig_color, lw=0.8),604 )605 ax.text(606 mde / 2,607 -0.9,608 f"{below} of {len(src['gaps'])} P@5 gaps fall below\nthe floor (within the noise)",609 ha="center",610 va="center",611 fontsize=8,612 color=base_color,613 )614 ax.set_ylim(-1.5, 1.4)615 ax.set_yticks([])616 ax.set_xlim(0, 0.20)617 ax.set_xlabel("|P@5 difference|, custom vs LangChain")618 ax.set_title("What the benchmark can resolve at this sample size", fontsize=11)619 ax.spines[["top", "right", "left"]].set_visible(False)620 fig.tight_layout()621 _save_with_hash(fig, out_path, source_hash(mde_source(values)))622 623 624def _render_unfolding(out_path: Path) -> None:625 import matplotlib626 627 matplotlib.use("Agg")628 matplotlib.rcParams["svg.hashsalt"] = "agent-bench"629 import matplotlib.pyplot as plt630 631 src = unfolding_source()632 obs, true = src["observed"], src["known_true"]633 dago, matinv = src["dagostini"], src["matrix_inversion"]634 reg_color, naive_color = "#2b6cb0", "#a0aec0"635 636 fig, ax = plt.subplots(figsize=(8.4, 3.0))637 # The reference line is the KNOWN TRUTH (what the corrected CI must contain),638 # drawn from the pinned known_true row, not observed. They coincide in 1.9639 # (the canary confusion is the identity), so say so only when they actually do.640 ax.axvline(true, color="#1a202c", lw=1.3, zorder=1)641 same = abs(obs - true) < 1e-9642 label = f"observed = known truth {true:.3f}" if same else f"known truth {true:.3f}"643 ax.text(true + 0.008, 1.62, label, fontsize=8.5, va="bottom")644 645 # D'Agostini (regularized): point + wide CI, entirely inside [0,1]646 ax.plot(647 [dago["lo"], dago["hi"]], [1, 1], color=reg_color, lw=4, solid_capstyle="butt", zorder=3648 )649 for x in (dago["lo"], dago["hi"]):650 ax.plot([x, x], [0.88, 1.12], color=reg_color, lw=2, zorder=3)651 ax.plot(dago["point"], 1, "o", ms=10, color=reg_color, mec="white", mew=0.8, zorder=4)652 ax.text(653 dago["hi"] + 0.012,654 1,655 f"{dago['point']:.3f} [{dago['lo']:.3f}, {dago['hi']:.3f}]",656 va="center",657 fontsize=8,658 color=reg_color,659 )660 661 # matrix inversion (naive): CI leaves [0,1] -> bar across with off-axis arrows662 ax.plot([0.0, 1.0], [0, 0], color=naive_color, lw=4, solid_capstyle="butt", zorder=2)663 ax.annotate(664 "",665 xy=(-0.025, 0),666 xytext=(0.05, 0),667 arrowprops=dict(arrowstyle="->", color=naive_color, lw=2.5),668 )669 ax.annotate(670 "",671 xy=(1.025, 0),672 xytext=(0.95, 0),673 arrowprops=dict(arrowstyle="->", color=naive_color, lw=2.5),674 )675 ax.plot(matinv["point"], 0, "o", ms=10, color=naive_color, mec="white", mew=0.8, zorder=4)676 ax.text(677 0.5,678 -0.36,679 f"95% CI [{matinv['lo']:.3f}, {matinv['hi']:.3f}] — leaves [0,1]: unidentified at n≈20",680 ha="center",681 va="top",682 fontsize=8,683 color="#718096",684 )685 686 ax.set_yticks([0, 1])687 ax.set_yticklabels(["matrix inversion\n(naive)", "D'Agostini\n(regularized)"], fontsize=9)688 ax.set_ylim(-0.75, 1.9)689 ax.set_xlim(-0.04, 1.04)690 ax.set_xticks([0, 0.25, 0.5, 0.75, 1.0])691 ax.set_xlabel("completeness pass-rate (corrected through the judge confusion matrix)")692 ax.set_title(693 "Judge unfolding: the correction moves the rate and widens the honest uncertainty",694 fontsize=10.5,695 pad=8,696 )697 ax.spines[["top", "right", "left"]].set_visible(False)698 fig.tight_layout()699 _save_with_hash(fig, out_path, source_hash(unfolding_source()))700 701 702def generate() -> None:703 PLOTS_DIR.mkdir(parents=True, exist_ok=True)704 values = read_values(REPORT.read_text())705 _render_forest(values, PLOTS_DIR / "forest_fastapi.png")706 _render_paired(values, PLOTS_DIR / "paired_diff_fastapi.png")707 _render_icc(values, PLOTS_DIR / "icc_contrast.png")708 _render_mde(values, PLOTS_DIR / "mde_resolution.png")709 _render_unfolding(PLOTS_DIR / "unfolding_shift.png")710 ASSETS_DIR.mkdir(parents=True, exist_ok=True)711 # 8.0x4.0 in at 160 dpi = exactly 1280x640 px, close to the README forest's712 # committed 7.6x4.4 proportions so the layout survives the aspect change.713 _render_forest(714 values,715 ASSETS_DIR / "social_preview.png",716 figsize=(8.0, 4.0),717 title="agent-bench · FastAPI retrieval: framework comparison (overlapping CIs)",718 dpi=160,719 )720 print(f"wrote {len(EXPECTED_PLOTS)} plot(s) to {PLOTS_DIR} + social card to {ASSETS_DIR}")721 722 723def main() -> int:724 parser = argparse.ArgumentParser(description=__doc__)725 parser.add_argument("mode", choices=("generate", "check"), nargs="?", default="generate")726 args = parser.parse_args()727 if args.mode == "generate":728 generate()729 return 0730 failures = check(REPORT.read_text(), PLOTS_DIR)731 for line in failures:732 print(f"FAIL: {line}")733 if not failures:734 print(f"OK: all {len(EXPECTED_PLOTS)} plot(s) fresh against the report")735 return 1 if failures else 0736 737 738if __name__ == "__main__":739 sys.exit(main())740