ALchemt/llm-eval

1"""Produce a markdown report and summary CSV from per-run scores.2 3Reads everything in `runs/scores_*.csv`, aggregates via metrics.build_summary(),4writes `runs/summary_<ts>.csv` + `runs/report_<ts>.md`.5 6Usage:7    python -m src.report8"""9 10from __future__ import annotations11 12import sys13import time14from pathlib import Path15 16import pandas as pd17 18from src.metrics import RUNS_DIR, build_summary, diff_runs19 20ROOT = Path(__file__).resolve().parent.parent21 22 23def _to_md_table(df: pd.DataFrame, float_fmt: str = ".3f") -> str:24    """Minimal markdown table formatter — avoids tabulate dependency."""25    if df.empty:26        return "_empty_"27    cols = list(df.columns)28    header = "| " + " | ".join(cols) + " |"29    sep = "|" + "|".join(["---"] * len(cols)) + "|"30    rows = []31    for _, row in df.iterrows():32        cells = []33        for c in cols:34            v = row[c]35            if isinstance(v, float):36                cells.append(f"{v:{float_fmt}}" if pd.notna(v) else "NaN")37            else:38                cells.append(str(v))39        rows.append("| " + " | ".join(cells) + " |")40    return "\n".join([header, sep, *rows])41 42 43def render_md(summary) -> str:44    if summary.empty:45        return "# LLM Eval Report\n\nNo score files found. Run runner + judge first.\n"46 47    lines = [48        "# LLM Eval Report",49        "",50        f"_Generated: {time.strftime('%Y-%m-%d %H:%M:%S')}_",51        "",52        "## Per-run × suite summary",53        "",54        _to_md_table(summary),55        "",56    ]57 58    run_ids = sorted(summary["run_id"].unique())59    if len(run_ids) >= 2:60        a, b = run_ids[0], run_ids[1]61        delta = diff_runs(summary, a, b)62        if not delta.empty:63            lines += [64                f"## Accuracy delta: `{b}` vs `{a}`",65                "",66                _to_md_table(delta),67                "",68            ]69 70    lines += [71        "## Notes",72        "",73        "- `accuracy` = share of samples passing the rubric (exact/contains/judge).",74        "- `agreement_vs_human` = share of samples where judge verdict matches `human_score` "75        "in the suite file. NaN means no human labels yet.",76        "- `est_cost_usd` uses public pricing snapshot in `src/metrics.py`.",77        "- For scaffold / `--dry-run` outputs, responses come from a mock LLM and",78        "  accuracy numbers are not meaningful — the pipeline is exercised end-to-end only.",79    ]80    return "\n".join(lines) + "\n"81 82 83def main() -> int:84    summary = build_summary()85    ts = int(time.time())86 87    summary_csv = RUNS_DIR / f"summary_{ts}.csv"88    report_md = RUNS_DIR / f"report_{ts}.md"89 90    if summary.empty:91        print("No scores found in runs/. Run runner + judge first.")92        return 193 94    summary.to_csv(summary_csv, index=False)95    report_md.write_text(render_md(summary))96    print(f"Summary: {summary_csv.relative_to(ROOT)}")97    print(f"Report:  {report_md.relative_to(ROOT)}")98    return 099 100 101if __name__ == "__main__":102    sys.exit(main())103