build-small-hackathon/microfactory-lab

1"""The QA Inspector — a SEPARATE reviewer persona (the hybrid evaluator).2 3Integrity, restated: the Chief Engineer never grades its own work. The4deterministic simulated world (`sim/outcome.py`) produces the ground-truth5pass/fail. The Inspector is a *distinct* voice — skeptical, conservative —6that reads what the Engineer claimed and what actually happened and writes a7verdict. The grade is therefore "hybrid": deterministic physics + a second LLM8opinion, never the proposer marking its own homework.9 10One persona, three sets of rules depending on where it stands in the workflow:11  • second_opinion(...)  — BUILD: critique the PLAN before any print runs.12  • grade_outcome(...)   — PRINT: grade one finished (simulated) print vs the13                           Engineer's prediction — did the called risk hold?14  • summarize_run(...)   — REVIEW: one verdict across a whole iteration run.15 16LLM-backed via `llm.chat_json` with a distinct system prompt; each mode has a17deterministic fallback so the verdict is always present offline.18"""19 20from __future__ import annotations21 22from dataclasses import dataclass23 24from . import llm25from .models import Advice, Environment, Job, PrintSettings26from sim.outcome import SimResult27 28PERSONA = """You are La Forge, the QA Inspector: a skeptical, conservative print-shop \29inspector. You did NOT propose these settings — Chief Engineer O'Brien did, and \30O'Brien is an optimist. Your job is to second-guess, not to please. You are \31terse and physical. You never flatter. You call out optimism, thin margins, and \32unflagged risks, and you give credit only when the evidence earns it."""33 34# predicted-risk vocabulary → simulated failure_mode it corresponds to35_RISK_TO_MODE = {36    "sag": "sag", "stringing": "stringing", "adhesion": "adhesion",37    "warping": "warp", "warp": "warp", "delamination": "under_extrusion",38}39_MODE_HUMAN = {40    "sag": "sagging", "stringing": "stringing", "adhesion": "first-layer adhesion",41    "warp": "warping", "under_extrusion": "under-extrusion", "none": "no failure",42}43 44 45@dataclass46class InspectorVerdict:47    stance: str                 # short label, e.g. "concur" / "caution" / "held" / "missed"48    headline: str               # one-line verdict49    detail: str                 # 1-2 lines of rationale50    agreement: bool | None = None   # outcome modes: did the Engineer's prediction match reality?51 52    @property53    def color(self) -> str:54        s = self.stance.lower()55        if s in ("dispute", "missed", "fail"):56            return "var(--ao-red, #d9534f)"57        if s in ("caution", "overcautious", "watch"):58            return "var(--ao-amber, #e0a458)"59        return "var(--ao-green)"60 61 62def _predicted_modes(advice: Advice) -> set[str]:63    out: set[str] = set()64    for r in advice.risks:65        key = (r.risk or "").strip().lower()66        out.add(_RISK_TO_MODE.get(key, key))67    return out68 69 70def _settings_line(s: PrintSettings) -> str:71    return (f"nozzle {s.nozzle_temp:.0f}°C, bed {s.bed_temp:.0f}°C, fan {s.fan_pct:.0f}%, "72            f"first-layer fan {s.first_layer_fan_pct:.0f}%, retraction {s.retraction_mm:.1f}mm")73 74 75# ── BUILD: a second opinion on the plan, before anything prints ───────────────76def second_opinion(job: Job, env: Environment, settings: PrintSettings, advice: Advice) -> InspectorVerdict:77    raw = llm.chat_json(78        PERSONA + "\n\nRespond ONLY with JSON: "79        '{"stance":"concur|caution|dispute","headline":"<one line>","detail":"<1-2 lines>"}',80        "Review this PLAN before it prints — do not re-propose, just critique.\n"81        f"JOB: {job.material}/{job.geometry_type}, bed position {job.bed_position}, "82        f"room {env.temp:.0f}°C/{env.humidity:.0f}%RH on a {env.printer}.\n"83        f"ENGINEER PROPOSED: {_settings_line(settings)}.\n"84        f"ENGINEER REASONING: {advice.reasoning}\n"85        f"ENGINEER FLAGGED RISKS: {[r.risk for r in advice.risks] or 'none'}.\n"86        "Where is the Engineer being optimistic? What would you watch?",87    )88    if raw and {"stance", "headline", "detail"} <= set(raw):89        return InspectorVerdict(str(raw["stance"]), str(raw["headline"]), str(raw["detail"]))90    return _second_opinion_fallback(job, env, settings, advice)91 92 93def _second_opinion_fallback(job: Job, env: Environment, settings: PrintSettings, advice: Advice) -> InspectorVerdict:94    geo, mat = job.geometry_type, job.material.upper()95    flags: list[str] = []96    if geo in ("overhang", "bridge") and settings.fan_pct < 60:97        flags.append(f"fan {settings.fan_pct:.0f}% is thin for a {geo} — sagging risk the Engineer may be underweighting")98    if mat == "ABS" and job.bed_position in ("edge", "corner"):99        flags.append(f"ABS off-center ({job.bed_position}) will pull at the edges — I'd second a warp watch and a brim")100    if mat == "ABS" and settings.fan_pct > 40:101        flags.append(f"fan {settings.fan_pct:.0f}% on ABS invites cracking/warp")102    if env.humidity > 55 and mat in ("PETG", "TPU", "ABS") and settings.retraction_mm < 3:103        flags.append(f"humid air ({env.humidity:.0f}%RH) + {settings.retraction_mm:.1f}mm retraction → expect stringing")104    if not advice.risks:105        flags.append("Engineer flagged NO failure regions — verify that's confidence, not optimism")106 107    if not flags:108        return InspectorVerdict("concur", "No red flags from a second look.",109                                "Plan sits inside sane bounds for this material and room. Cleared to print.")110    stance = "dispute" if len(flags) >= 2 else "caution"111    return InspectorVerdict(stance, f"Second opinion: {flags[0]}.",112                            " · ".join(flags[1:]) or "Print it, but watch that region.")113 114 115# ── PRINT: grade one finished (simulated) print against the prediction ────────116def grade_outcome(job: Job, env: Environment, settings: PrintSettings,117                  advice: Advice, result: SimResult) -> InspectorVerdict:118    predicted = _predicted_modes(advice)119    raw = llm.chat_json(120        PERSONA + "\n\nRespond ONLY with JSON: "121        '{"stance":"held|missed|overcautious|confirmed","headline":"<one line>","detail":"<1-2 lines>"}',122        "Grade this finished print. The outcome below came from the deterministic "123        "world, not from the Engineer — you are checking the Engineer's CALL against it.\n"124        f"JOB: {job.material}/{job.geometry_type} @ {env.temp:.0f}°C/{env.humidity:.0f}%RH.\n"125        f"ENGINEER PREDICTED RISKS: {[r.risk for r in advice.risks] or 'none'}.\n"126        f"ACTUAL OUTCOME: {result.outcome} — {result.detail} "127        f"(failure mode: {result.failure_mode}).\n"128        "Did the Engineer's prediction hold? Be blunt.",129    )130    agreement = _agreement(predicted, result)131    if raw and {"stance", "headline", "detail"} <= set(raw):132        return InspectorVerdict(str(raw["stance"]), str(raw["headline"]), str(raw["detail"]), agreement)133    return _grade_fallback(predicted, result, agreement)134 135 136def _agreement(predicted: set[str], result: SimResult) -> bool:137    if result.failure_mode == "none":138        return True            # clean print — nothing to have missed139    return result.failure_mode in predicted140 141 142def _grade_fallback(predicted: set[str], result: SimResult, agreement: bool) -> InspectorVerdict:143    mode = _MODE_HUMAN.get(result.failure_mode, result.failure_mode)144    if result.failure_mode == "none":145        if predicted:146            return InspectorVerdict("overcautious", f"Print held (q={result.quality:.2f}).",147                                    f"Engineer flagged {', '.join(sorted(predicted))}; the settings covered it. "148                                    "Credit the call — or it was conservative.", True)149        return InspectorVerdict("held", f"Clean print (q={result.quality:.2f}).",150                                "No failure flagged, none occurred. Plan and reality agree.", True)151    if agreement:152        return InspectorVerdict("confirmed", f"Failed on {mode} — exactly as called.",153                                f"Quality {result.quality:.2f}. The Engineer's risk flag was right; "154                                "the loop now has the lesson.", True)155    return InspectorVerdict("missed", f"Failed on {mode} — and it wasn't flagged.",156                            f"Quality {result.quality:.2f}. The Engineer didn't predict this mode. "157                            "That gap is what the next iteration has to close.", False)158 159 160# canonical failure mode each geometry is expected to risk (the loop's implicit161# prediction — the deterministic policy loop carries no LLM Advice per iteration)162_GEO_EXPECT = {"overhang": "sag", "bridge": "sag", "stringing": "stringing",163               "adhesion": "adhesion", "vase": "warp"}164 165 166def grade_iteration(geometry_type: str, result: SimResult) -> InspectorVerdict:167    """Deterministic-only grade for one loop iteration (no LLM — the loop runs168    many fast, reproducible iterations). Checks the outcome against the failure169    mode this geometry is expected to risk."""170    expected = {_GEO_EXPECT.get(geometry_type, "sag")}171    return _grade_fallback(expected, result, _agreement(expected, result))172 173 174# ── REVIEW: one verdict across a whole iteration run ──────────────────────────175def summarize_run(records: list, *, material: str, geometry: str) -> InspectorVerdict:176    if not records:177        return InspectorVerdict("watch", "No run to review yet.", "Run the Print loop first.")178    qualities = [r.result.quality for r in records]179    first_clean = next((r.n for r in records if r.result.outcome == "success"), None)180    start, end = qualities[0], qualities[-1]181    climbed = end - start182    if first_clean:183        stance, head = "concur", f"Converged to clean by iteration {first_clean}."184        detail = (f"{material}/{geometry} climbed {start:.2f} → {end:.2f}. The compounding is real: "185                  "each simulated outcome tightened the policy and the next run was better-informed.")186    elif climbed > 0.05:187        stance, head = "caution", f"Improving but not yet clean (best {max(qualities):.2f})."188        detail = (f"Quality rose {start:.2f} → {end:.2f} over {len(records)} runs but never crossed the "189                  "bar. More iterations or a different lever needed — the loop is learning, slowly.")190    else:191        stance, head = "dispute", "No real progress this run."192        detail = (f"Quality stuck around {start:.2f}. Either the job is mis-specified or the policy is "193                  "saturated for these conditions — worth a human look before trusting the trend.")194    return InspectorVerdict(stance, head, detail)195