build-small-hackathon/puck

1"""Eval harness: the bar a molt must clear before an adapter swap.2 3Programmatic, judge-free metrics on the held-out set (data/eval.jsonl):4- fact_retention  — numbers, filenames, proper nouns from the event survive5- no_banned       — zero corporate-assistant phrases6- length_ok       — speakable: ≤ 280 chars7- clean_shape     — one utterance: no preamble, no JSON, no paragraph breaks8 9Also reports mean output length per mischief level — the register10calibration curve (plain should be shorter than mythic) at a glance.11 12Usage:13  PUCK_BRAIN_URL=http://127.0.0.1:11434/v1 \14  PUCK_BRAIN_MODEL=hf.co/mradermacher/Holo-3.1-4B-GGUF:Q8_0 \15  uv run eval.py [--n 999] [--tag baseline]16 17Writes data/eval_report_<tag>.json with per-sample outputs for human reading —18the metrics gate the swap, but read the samples; charm isn't programmatic.19"""20 21import argparse22import json23import re24import statistics25import sys26from pathlib import Path27 28HERE = Path(__file__).resolve().parent29ROOT = HERE.parent30sys.path.insert(0, str(ROOT / "server"))31from brain import BRAIN_MODEL, BRAIN_URL, _chat_completion  # noqa: E40232 33BANNED = [34    "i'd be happy",35    "i would be happy",36    "as an ai",37    "great question",38    "certainly!",39    "i cannot assist",40    "how can i help",41    "feel free to",42    "let me know if",43]44 45MAX_CHARS = 28046 47# the mythic register's vocabulary — fine at high mischief, a calibration48# failure at low. Derived from the deck's own tier lines.49MYTHIC_WORDS = ["goblin", "bog", "spell", "scroll", "mist", "ritual", "omen", "spirit", "forge", "realm", "bloom", "nest", "whisper"]50 51 52def facts(title: str) -> list[str]:53    """The concrete tokens whimsy must not eat: numbers, filenames, proper nouns."""54    out = set(re.findall(r"\d+(?:m \d+s)?", title))  # counts, durations55    out |= set(re.findall(r"[\w-]+\.[a-z]{1,4}(?:\.[a-z]{1,4})?", title))  # file.ts, auth.test.ts56    out |= {w for w in re.findall(r"\b[A-Z][a-z]{2,}\b", title) if w not in {"The", "New", "Your"}}57    return sorted(out)58 59 60def score_one(user: str, output: str, mischief: int | None) -> dict:61    title = next((ln.removeprefix("What happened: ") for ln in user.splitlines() if ln.startswith("What happened: ")), "")62    fs = facts(title)63    kept = [f for f in fs if f.lower() in output.lower()]64    low = output.lower()65    # Puck speaks AS himself; naming himself = narrator voice break.66    # Strip title-derived tokens first — retaining "#puck-build" is a fact, not a voice break.67    voice_text = low68    for tok in title.lower().split():69        if "puck" in tok:70            voice_text = voice_text.replace(tok, "")71    first_person = "puck" not in voice_text72    # low mischief must be sober: no exclamations, no mythic lexicon73    register_ok = True74    if mischief is not None and mischief <= 20:75        register_ok = "!" not in output and not any(w in low for w in MYTHIC_WORDS)76    return {77        "fact_retention": len(kept) / len(fs) if fs else 1.0,78        "facts_missing": [f for f in fs if f.lower() not in low],79        "no_banned": not any(b in low for b in BANNED),80        "length_ok": len(output) <= MAX_CHARS,81        "clean_shape": "\n\n" not in output and not low.startswith(("here", "sure", "okay,", "{")) and "{" not in output,82        "first_person": first_person,83        "register_ok": register_ok,84        "chars": len(output),85    }86 87 88def main() -> None:89    ap = argparse.ArgumentParser()90    ap.add_argument("--n", type=int, default=999, help="max samples")91    ap.add_argument("--k", type=int, default=3, help="generations per sample — n=6 at temp 0.7 is noise; k repeats make the gate stable")92    ap.add_argument("--tag", default="baseline", help="report filename tag")93    ap.add_argument("--set", default="eval", choices=["eval", "sft"], help="which split to run")94    args = ap.parse_args()95 96    rows = [json.loads(line) for line in (HERE / "data" / f"{args.set}.jsonl").read_text().splitlines()]97    rows = rows[: args.n] * args.k98    print(f"evaluating {len(rows)} generations ({args.k} per sample) against {BRAIN_URL} ({BRAIN_MODEL})\n")99 100    samples = []101    for i, ex in enumerate(rows):102        system, user, gold = (m["content"] for m in ex["messages"])103        output = _chat_completion(system, user, temperature=0.7)104        s = score_one(user, output, ex["meta"].get("mischief"))105        samples.append({"meta": ex["meta"], "user": user, "gold": gold, "output": output, **s})106        flag = "" if all((s["fact_retention"] == 1, s["no_banned"], s["length_ok"], s["clean_shape"], s["first_person"], s["register_ok"])) else "  ⚠"107        print(f"[{i + 1}/{len(rows)}] {ex['meta'].get('event', ex['meta']['kind'])} m={ex['meta'].get('mischief')}{flag}")108        print(f"   {output[:160]}")109 110    agg = {111        "n": len(samples),112        "fact_retention": round(statistics.mean(s["fact_retention"] for s in samples), 3),113        "no_banned": sum(s["no_banned"] for s in samples) / len(samples),114        "length_ok": sum(s["length_ok"] for s in samples) / len(samples),115        "clean_shape": sum(s["clean_shape"] for s in samples) / len(samples),116        "first_person": sum(s["first_person"] for s in samples) / len(samples),117        "register_ok": sum(s["register_ok"] for s in samples) / len(samples),118        "mean_chars_by_mischief": {119            str(m): round(statistics.mean(s["chars"] for s in samples if s["meta"].get("mischief") == m))120            for m in sorted({s["meta"].get("mischief") for s in samples if s["meta"].get("mischief") is not None})121        },122    }123    report = {"brain": {"url": BRAIN_URL, "model": BRAIN_MODEL}, "aggregate": agg, "samples": samples}124    out = HERE / "data" / f"eval_report_{args.tag}.json"125    out.write_text(json.dumps(report, indent=2, ensure_ascii=False))126    print(f"\n=== aggregate ({args.tag}) ===")127    print(json.dumps(agg, indent=2))128    print(f"\nreport: {out}")129 130 131if __name__ == "__main__":132    main()133