srr84/agent-data-layer

1"""Unit tests for M-report: the offline dual-score harness + the 5 metrics + the single write.2 3Drives run_acceptance with the deterministic $0 OracleProvider (the same test double the bench4uses — no Ollama). The oracle is NOT a correctness oracle, but it derives a valid ToolCall for5each in-coverage gold item and abstains on the OOC probes, so the harness exercises BOTH axes:6the correctness scorer AND the model-free resolver re-derivation. Because the oracle maps each7gold question to the CORRECT tool by its structured shape, the in-coverage items resolve and8score correct here — which lets these tests assert the harness wiring, the dual-score roll-up,9and the metric derivations deterministically (the GATE-a *model-quality* number is the separate10live run; these tests assert the SCORING MACHINERY, not the model).11 12```text13spec traceability (audit map — safe to skip)14LLD §1 (M-report offline single-writer; dual-score + metrics), §3.4 (per item: correctness axis +15        provenance axis; pass iff BOTH)16Validation §2 gate (a)/(b)/(e), §3 (gold categories), TechSpec §6 (the 5 metrics + derivations)17"""18 19from __future__ import annotations20 21import json22from pathlib import Path23 24from agent_data_layer.report.report import (25    AcceptanceReport,26    load_and_run,27    render_report,28    write_report,29)30from tests.bench_tool_calling import OracleProvider31 32_FIVE_METRICS = {33    "request_count",34    "latency_p95",35    "error_rate",36    "provenance_resolution_rate",37    "abstention_rate",38}39 40 41def _report() -> AcceptanceReport:42    return load_and_run(OracleProvider())43 44 45def test_runs_full_gold_set() -> None:46    report = _report()47    assert report.total_items == 4848    # 45 NL-correctness items carry GATE-a; 3 OOC probes carry GATE-e action.49    assert report.gate_a_total == 4550    assert report.gate_e_action_total == 351 52 53def test_oracle_drives_both_axes() -> None:54    """The harness runs BOTH axes (correctness scorer + model-free resolver) and rolls them up.55 56    Asserts the WIRING, not a model number: the oracle is a test double (it answers the items its57    structured-shape parse recognises and abstains on the rest), so it is NOT perfect on GATE-a.58    What IS load-bearing here: every item the oracle ANSWERED with the correct tool resolves on59    the provenance axis (100% — the resolver re-derivation agreed), the abstention ACTION on the60    3 OOC probes is correct, and CONTAINMENT is structurally clean. If the harness silently61    skipped the provenance axis, provenance would not be 100% over a non-trivial answered subset.62    """63    report = _report()64    # Provenance (GATE-b): every answered/absence item resolved (resolver re-derivation agreed).65    assert report.provenance_ok is True66    assert report.provenance_resolved == report.provenance_total67    assert report.provenance_total >= 40  # a non-trivial answered subset actually ran the axis68    # The oracle answers MOST in-coverage items correctly (it is a deterministic shape-parser);69    # whatever it answered AND scored correct must be consistent with its answered subset.70    assert report.gate_a_passed >= 4071    # Abstention ACTION: the 3 OOC probes abstained.72    assert report.gate_e_action_passed == 373    # Containment is structurally clean: no OOC probe produced a resolvable fact.74    assert report.gate_e_containment_ok is True75 76 77def test_item_passes_requires_both_axes() -> None:78    """An item passes iff BOTH the correctness AND the provenance axis pass (LLD §3.4)."""79    report = _report()80    for i in report.items:81        if not i.is_out_of_coverage:82            assert i.item_passes == (i.correctness.correct and i.provenance_axis_ok)83 84 85def test_five_metrics_present_and_derived() -> None:86    report = _report()87    m = report.metrics88    assert m.request_count == 4889    # abstention_rate = abstained / total. The oracle abstains on the 3 OOC probes PLUS the90    # in-coverage items its shape-parse does not recognise — count them from the rolled-up items91    # rather than hard-coding the oracle's parse coverage.92    abstained = sum(1 for i in report.items if i.outcome == "abstained")93    assert abs(m.abstention_rate - (abstained / 48)) < 1e-994    # provenance_resolution_rate = resolved / (answered+absence) = 1.0 (all answered resolved).95    assert m.provenance_resolution_rate == 1.096    # error_rate = (rejected+error)/total = 0 under the oracle (no rejections).97    assert m.error_rate == 0.098    assert m.latency_p95 >= 0.099 100 101def test_render_exposes_the_five_metrics_and_gates() -> None:102    rendered = render_report(_report())103    metrics = rendered["metrics"]104    gates = rendered["gates"]105    assert isinstance(metrics, dict) and set(metrics.keys()) == _FIVE_METRICS106    assert isinstance(gates, dict)107    assert "gate_a_nl_correctness" in gates108    assert "gate_b_provenance" in gates109    assert "gate_e_abstention_action" in gates110    assert "gate_e_containment" in gates111 112 113def test_write_report_is_the_single_stateful_write(tmp_path: Path) -> None:114    report = _report()115    target = tmp_path / "sub" / "acceptance.json"116    written = write_report(report, target)117    assert written == target118    assert target.exists()119    payload = json.loads(target.read_text(encoding="utf-8"))120    assert set(payload["metrics"].keys()) == _FIVE_METRICS121    assert payload["dataset_version"] == "v1.0.0"122    assert len(payload["items"]) == 48123