DGXAI/driftcall

1"""Cell 18 — Baseline evaluation harness.2 3Implements ``docs/modules/evaluation.md`` §1, §2, §3.1–§3.3, §3.8, §4 and4§5 for the baseline (untrained Gemma 3n E2B) eval path.5 6Hard rules (evaluation.md §3.1, §3.2, §6.3):7- Greedy decoding (``temperature=0.0``); ``num_generations=1``;8  ``model.eval()`` + ``torch.no_grad()`` semantics asserted at entry.9- Per-episode env seed = ``hash((episode_id, "eval")) & 0xFFFFFFFF``.10- 50 held-out val episodes (rows ``[0:50]`` of ``val/briefs.jsonl``) — file11  order, no shuffling.12- Bootstrap CI (percentile method) at ``n_boot=10_000``, ``rng_seed=20260426``13  (paired-difference uses ``20260428``).14- No LLM-as-judge; static AST scan via ``_NO_LLM_JUDGE_FORBIDDEN_IMPORTS``.15- Wall-clock ceiling 20 minutes (``EvalBudgetExceededError`` on overrun).16 17This module deliberately does **not** import ``torch`` at module load. The18training-eval delegate is injected via ``run_eval_baseline(..., training_eval=...)``19so unit tests can stub model inference (CUDA-free CI per training_tests.md §5.3).20"""21 22from __future__ import annotations23 24import math25import time26from dataclasses import dataclass, field27from typing import TYPE_CHECKING, Any, Literal, Protocol28 29if TYPE_CHECKING:  # pragma: no cover - typing only30    from collections.abc import Callable, Sequence31    from pathlib import Path32 33 34__all__ = [35    "BUDGET_RUN_EVAL_SECONDS",36    "DEFAULT_BOOTSTRAP_SEED",37    "DEFAULT_PAIRED_BOOTSTRAP_SEED",38    "DriftDetectionLatency",39    "EvalBudgetExceededError",40    "EvalModelLoadError",41    "EvalReport",42    "EvaluationError",43    "PerLanguageReport",44    "TrainingEvalCallable",45    "ZeroSuccessBaselineWarning",46    "bootstrap_ci",47    "compute_episode_seed",48    "eval_baseline",49    "run_eval",50]51 52 53# ---------------------------------------------------------------------------54# Constants — evaluation.md §2.4, §3.855# ---------------------------------------------------------------------------56 57 58DEFAULT_BOOTSTRAP_SEED: int = 2026042659DEFAULT_PROBE_BOOTSTRAP_SEED: int = 2026042760DEFAULT_PAIRED_BOOTSTRAP_SEED: int = 2026042861DEFAULT_N_BOOT: int = 10_00062 63BUDGET_RUN_EVAL_SECONDS: int = 20 * 6064"""Hard ceiling on ``run_eval`` (50 episodes) — evaluation.md §3.8."""65 66# Forbidden imports inside any evaluation/scoring path (evaluation.md §6.3).67_NO_LLM_JUDGE_FORBIDDEN_IMPORTS: frozenset[str] = frozenset(68    {"openai", "anthropic", "vertexai", "google.generativeai", "cohere"},69)70 71_LANGUAGE_CODES: tuple[str, ...] = ("hi", "ta", "kn", "en", "hinglish")72 73 74# ---------------------------------------------------------------------------75# Errors / warnings — evaluation.md §576# ---------------------------------------------------------------------------77 78 79class EvaluationError(Exception):80    """Root for every evaluation-specific error (evaluation.md §5)."""81 82 83class EvalModelLoadError(EvaluationError):84    """Adapter load / merge failure surfaced by the training-eval delegate."""85 86 87class EvalBudgetExceededError(EvaluationError):88    """Wall-clock budget for an entry point exceeded (evaluation.md §3.8, §5)."""89 90 91class CatalogueHashMismatchError(EvaluationError):92    """Loaded catalogue hashes do not match the BriefRow's declared hashes."""93 94 95class ZeroSuccessBaselineWarning(UserWarning):96    """All 50 baseline R1 == 0.0 → degenerate CI; warn rather than raise."""97 98 99# ---------------------------------------------------------------------------100# EvalReport family — re-exported for downstream cells (evaluation.md §4)101# ---------------------------------------------------------------------------102 103 104@dataclass(frozen=True)105class PerLanguageReport:106    """Per-language cohort means (training.md §4.2)."""107 108    language: Literal["hi", "ta", "kn", "en", "hinglish"]109    n_episodes: int110    reward_mean: float111    r1_mean: float112    r2_mean: float113    r3_mean: float114    r4_mean: float115    r5_mean: float116 117 118@dataclass(frozen=True)119class DriftDetectionLatency:120    """Drift-detection latency aggregated by stage (training.md §4.2)."""121 122    stage2_mean: float123    stage2_median: float124    stage2_p95: float125    stage3_mean: float126    stage3_median: float127    stage3_p95: float128    undetected_count: int129 130 131@dataclass(frozen=True)132class EvalReport:133    """Result of ``run_eval`` — paired across baseline and final (training.md §4.2)."""134 135    model_path: str136    n_episodes: int137    reward_mean_ci: tuple[float, float, float]138    r1_mean_ci: tuple[float, float, float]139    r2_mean_ci: tuple[float, float, float]140    r3_mean_ci: tuple[float, float, float]141    r4_mean_ci: tuple[float, float, float]142    r5_mean_ci: tuple[float, float, float]143    brier_mean: float144    floor_applied_rate: float145    hallucinated_field_rate: float146    reward_hacking_offenses: dict[str, int]147    drift_detection_latency: DriftDetectionLatency148    per_language: tuple[PerLanguageReport, ...]149    curves: dict[str, tuple[tuple[int, float], ...]] = field(default_factory=dict)150    breakdown: dict[str, Any] = field(default_factory=dict)151 152 153# ---------------------------------------------------------------------------154# Training-eval delegate Protocol — evaluation.md §6.1155# ---------------------------------------------------------------------------156 157 158class TrainingEvalCallable(Protocol):159    """Signature of ``training.train.eval`` — the heavy-lifting delegate."""160 161    def __call__(162        self,163        model_path: Path | Literal["base"],164        episodes: int,165        *,166        sampling: dict[str, Any],167        seeds: Sequence[int],168        episode_ids: Sequence[str],169    ) -> EvalReport: ...170 171 172# ---------------------------------------------------------------------------173# Statistical helpers — evaluation.md §2.4, §3.3174# ---------------------------------------------------------------------------175 176 177def bootstrap_ci(178    samples: tuple[float, ...],179    n_boot: int = DEFAULT_N_BOOT,180    alpha: float = 0.05,181    rng_seed: int = DEFAULT_BOOTSTRAP_SEED,182) -> tuple[float, float, float]:183    """Non-parametric percentile bootstrap 95% CI on the mean.184 185    evaluation.md §2.4 contract:186      - ``len(samples) == 0`` → ``(nan, nan, nan)``.187      - ``len(samples) == 1`` → ``(v, v, v)``.188      - All-identical samples → ``(v, v, v)`` (no resample variance).189    """190    if not samples:191        nan = float("nan")192        return nan, nan, nan193    n = len(samples)194    mean = sum(samples) / n195    if n == 1:196        return mean, mean, mean197    if all(s == samples[0] for s in samples):198        return mean, mean, mean199 200    # Lazy import to keep this module importable on minimal CI containers.201    import numpy as np202 203    rng = np.random.default_rng(rng_seed)204    arr = np.asarray(samples, dtype=np.float64)205    idx = rng.integers(0, n, size=(n_boot, n))206    means = arr[idx].mean(axis=1)207    lo = float(np.percentile(means, 100.0 * (alpha / 2.0)))208    hi = float(np.percentile(means, 100.0 * (1.0 - alpha / 2.0)))209    return float(mean), lo, hi210 211 212# ---------------------------------------------------------------------------213# Episode selection helpers — evaluation.md §3.1214# ---------------------------------------------------------------------------215 216 217def compute_episode_seed(episode_id: str) -> int:218    """``hash((episode_id, "eval")) & 0xFFFFFFFF`` — re-asserted at every call site."""219    return hash((episode_id, "eval")) & 0xFFFFFFFF220 221 222def _validate_briefs_first_50(briefs: Sequence[Any]) -> tuple[Any, ...]:223    """Take the first 50 BriefRows in file order; raise on too few."""224    if len(briefs) < 50:225        raise EvaluationError(226            f"val/briefs.jsonl must have >= 50 rows for paired eval, got {len(briefs)}",227        )228    return tuple(briefs[:50])229 230 231def _check_catalogue_hashes(briefs: Sequence[Any], current_hashes: dict[str, str]) -> None:232    """Compare each BriefRow's declared hash against the loaded library hashes.233 234    evaluation.md §3.1: any mismatch → ``CatalogueHashMismatchError``.235    """236    for row in briefs:237        for attr, key in (238            ("catalogue_hash", "drifts"),239            ("templates_sha256", "templates"),240            ("i18n_sha256", "i18n"),241        ):242            declared = getattr(row, attr, None)243            current = current_hashes.get(key)244            if declared is None or current is None:245                continue246            if declared != current:247                raise CatalogueHashMismatchError(248                    f"BriefRow.{attr}={declared!r} but loaded {key} hashes to {current!r}",249                )250 251 252# ---------------------------------------------------------------------------253# Sampling-policy guard — evaluation.md §3.2254# ---------------------------------------------------------------------------255 256 257_FROZEN_SAMPLING_POLICY: dict[str, Any] = {258    "temperature": 0.0,259    "top_p": 1.0,260    "top_k": 1,261    "num_generations": 1,262    "repetition_penalty": 1.0,263    "model_eval": True,264    "no_grad": True,265    "dropout_off": True,266}267 268 269def _frozen_sampling_kwargs() -> dict[str, Any]:270    return dict(_FROZEN_SAMPLING_POLICY)271 272 273# ---------------------------------------------------------------------------274# Episode-set / leakage helpers — evaluation.md §3.1275# ---------------------------------------------------------------------------276 277 278def _episode_ids_from_breakdown(report: EvalReport) -> tuple[str, ...]:279    ids = report.breakdown.get("episode_ids", ())280    return tuple(ids)281 282 283# ---------------------------------------------------------------------------284# Core entry point — evaluation.md §2.1 ``run_eval``285# ---------------------------------------------------------------------------286 287 288def run_eval(289    model_path: Path | Literal["base"],290    episodes: int = 50,291    *,292    training_eval: TrainingEvalCallable,293    briefs: Sequence[Any],294    catalogue_hashes: dict[str, str] | None = None,295    budget_seconds: int = BUDGET_RUN_EVAL_SECONDS,296    monotonic: Callable[[], float] | None = None,297) -> EvalReport:298    """Thin wrapper over ``training.train.eval`` (evaluation.md §2.1).299 300    Validates episode count, catalogue hashes, sampling policy, and wall-clock301    budget. Delegates the heavy lifting (model load, rollout, ``Rewards``302    aggregation) to the injected ``training_eval`` callable.303    """304    if episodes != 50:305        raise EvaluationError(306            f"run_eval expects episodes=50 (paired-comparison contract); got {episodes}",307        )308 309    selected = _validate_briefs_first_50(briefs)310    if catalogue_hashes is not None:311        _check_catalogue_hashes(selected, catalogue_hashes)312 313    episode_ids = tuple(row.episode_id for row in selected)314    seeds = tuple(compute_episode_seed(ep_id) for ep_id in episode_ids)315 316    clock = monotonic if monotonic is not None else time.monotonic317    started = clock()318 319    try:320        report = training_eval(321            model_path,322            episodes,323            sampling=_frozen_sampling_kwargs(),324            seeds=seeds,325            episode_ids=episode_ids,326        )327    except EvalModelLoadError:328        raise329    except EvaluationError:330        raise331 332    elapsed = clock() - started333    if elapsed > budget_seconds:334        raise EvalBudgetExceededError(335            f"run_eval wall-clock {elapsed:.1f}s exceeded {budget_seconds}s "336            f"({budget_seconds // 60} min ceiling)",337        )338 339    # Stamp episode_ids + wall-clock into breakdown for downstream leak guards.340    breakdown = dict(report.breakdown)341    breakdown.setdefault("episode_ids", episode_ids)342    breakdown.setdefault("wall_clock_seconds", round(elapsed, 3))343    breakdown.setdefault("sampling_policy", _frozen_sampling_kwargs())344 345    # Detect zero-success-baseline degeneracy (§7.1) — warn, do not raise.346    r1_mean = report.r1_mean_ci[0]347    if math.isclose(r1_mean, 0.0, abs_tol=1e-12) and report.model_path == "base":348        breakdown["ci_undefined_rewards"] = ["r1"]349 350    from dataclasses import replace as _replace351    return _replace(report, breakdown=breakdown)352 353 354def eval_baseline(355    model_path: Path | Literal["base"] = "base",356    episodes: int = 50,357    *,358    training_eval: TrainingEvalCallable,359    briefs: Sequence[Any],360    catalogue_hashes: dict[str, str] | None = None,361    budget_seconds: int = BUDGET_RUN_EVAL_SECONDS,362    monotonic: Callable[[], float] | None = None,363) -> EvalReport:364    """Baseline-eval entry point (evaluation.md §2.2 ``eval_baseline.py``).365 366    Defaults ``model_path='base'`` to lock in the untrained-model contract.367    """368    return run_eval(369        model_path,370        episodes,371        training_eval=training_eval,372        briefs=briefs,373        catalogue_hashes=catalogue_hashes,374        budget_seconds=budget_seconds,375        monotonic=monotonic,376    )377