srr84/agent-data-layer
0
1"""Tests for M-gold_loader (M1) + CHECK-M1-gold (the gold spot-audit, docs/09 §4).2 3Two kinds of test live here:4 1. Loader contract: load() happy-path + the load-time invariants (exact 6-field5 shape, N>=48, dataset_version lockstep, id/category/provenance validity,6 duplicate-id + ungrounded-positive rejection) with a negative case for each.7 2. CHECK-M1-gold: for EVERY gold item, independently re-derive8 expected_provenance_event_ids over store_loader.all_events using the REAL9 matches() predicate (a DIFFERENT data path than the executor's per-tool10 index) and assert it equals the hand-authored set. matches() being proven11 independently (SUITE-matches-truthtable, M0) means agreement here is12 CORRECTNESS, not shared-bug agreement. This catches a hand-authoring slip in13 gold.json BEFORE GATE-a / GATE-b consume it.14 15The re-derivation here is deliberately NOT the same code that authored gold.json:16gold was authored from the raw store via an inline boundary rule; this re-derives17from the LOADED store via the real contracts.matching.matches() + the per-tool18selection rule (LLD §1 tool semantics). Independence is the whole point (D-VP-3).19"""20 21from __future__ import annotations22 23import json24import re25from collections.abc import Sequence26from pathlib import Path27from typing import Any28 29import pytest30 31from agent_data_layer.contracts.data_model import ScanEvent32from agent_data_layer.contracts.enums import ShelfState33from agent_data_layer.contracts.filters import TypedFilter, Window34from agent_data_layer.contracts.ids import ProductId, StoreId, Timestamp35from agent_data_layer.contracts.matching import matches36from agent_data_layer.contracts.settings import Settings37from agent_data_layer.store.gold_loader import (38 MIN_GOLD_ITEMS,39 GoldItem,40 GoldLoadError,41 GoldVersionMismatch,42 LoadedGold,43 load_gold,44)45from agent_data_layer.store.store_loader import LoadedStore, load_store46 47# T4 condition param key -> how it is exhibited on an event (LLD §1 T4 rollup):48# stock conditions match event.state; compliance conditions match compliance_flag.49_STOCK_CONDITIONS = {"OUT_OF_STOCK", "LOW_STOCK"}50_COMPLIANCE_CONDITIONS = {"PRICE_MISMATCH", "EXPIRED", "PLANOGRAM_VIOLATION"}51 52# The bundled gold artifact (for negative tests that tamper with a copy).53_BUNDLED_GOLD = (54 Path(__file__).resolve().parent.parent55 / "src" / "agent_data_layer" / "data" / "gold" / "gold.json"56)57 58 59# =====================================================================60# 1. Loader contract — happy path + invariants61# =====================================================================62def test_load_gold_roundtrips() -> None:63 gold = load_gold()64 assert isinstance(gold, LoadedGold)65 assert gold.dataset_version == Settings().DATASET_VERSION66 assert len(gold.items) >= MIN_GOLD_ITEMS67 assert all(isinstance(i, GoldItem) for i in gold.items)68 69 70def test_gold_has_at_least_48_items() -> None:71 assert len(load_gold().items) >= 4872 73 74def test_gold_dataset_version_lockstep_with_settings_and_store() -> None:75 gold = load_gold()76 store = load_store()77 # Lockstep triple: gold == Settings == store (D-TSP-3).78 assert gold.dataset_version == Settings().DATASET_VERSION79 assert gold.dataset_version == store.dataset_version80 81 82def test_gold_ids_unique_and_well_formed() -> None:83 items = load_gold().items84 ids = [i.id for i in items]85 assert len(set(ids)) == len(ids)86 assert all(re.match(r"^gold-\d{3}$", i.id) for i in items)87 88 89def test_gold_category_distribution_matches_validation_plan() -> None:90 # Validation §3 per-category counts: scalar 12, aggregate 10, list 8, window 8,91 # must_answer 7, out_of_coverage 3 (total 48).92 items = load_gold().items93 counts: dict[str, int] = {}94 for i in items:95 counts[i.category] = counts.get(i.category, 0) + 196 assert counts == {97 "scalar": 12,98 "aggregate": 10,99 "list": 8,100 "window": 8,101 "must_answer": 7,102 "out_of_coverage": 3,103 }104 105 106def test_gold_out_of_coverage_items_abstain_with_empty_provenance() -> None:107 for i in load_gold().items:108 if i.category == "out_of_coverage":109 assert i.is_out_of_coverage is True110 assert i.expected_answer == {"kind": "abstain"}111 assert i.expected_provenance_event_ids == ()112 113 114def test_gold_scalar_items_have_exactly_one_provenance() -> None:115 for i in load_gold().items:116 if i.expected_answer.get("kind") in ("stock", "price", "compliance"):117 assert len(i.expected_provenance_event_ids) == 1118 119 120# ---- helpers for negative tests: load a tampered artifact from tmp_path ----121def _gold_payload() -> dict[str, Any]:122 """The real bundled gold payload as a mutable dict (deep-copied via json)."""123 payload: dict[str, Any] = json.loads(_BUNDLED_GOLD.read_text(encoding="utf-8"))124 return payload125 126 127def _write(tmp_path: Path, payload: dict[str, Any]) -> str:128 p = tmp_path / "gold.json"129 p.write_text(json.dumps(payload), encoding="utf-8")130 return str(p)131 132 133def test_version_mismatch_rejected(tmp_path: Path) -> None:134 payload = _gold_payload()135 payload["dataset_version"] = "v9.9.9"136 with pytest.raises(GoldVersionMismatch):137 load_gold(_write(tmp_path, payload))138 139 140def test_too_few_items_rejected(tmp_path: Path) -> None:141 payload = _gold_payload()142 payload["items"] = payload["items"][:10] # below the 48 floor143 with pytest.raises(GoldLoadError):144 load_gold(_write(tmp_path, payload))145 146 147def test_missing_field_rejected(tmp_path: Path) -> None:148 payload = _gold_payload()149 del payload["items"][0]["category"] # drop a required field150 with pytest.raises(GoldLoadError):151 load_gold(_write(tmp_path, payload))152 153 154def test_extra_field_rejected(tmp_path: Path) -> None:155 payload = _gold_payload()156 payload["items"][0]["unexpected"] = "x" # an extra key157 with pytest.raises(GoldLoadError):158 load_gold(_write(tmp_path, payload))159 160 161def test_duplicate_id_rejected(tmp_path: Path) -> None:162 payload = _gold_payload()163 payload["items"][1]["id"] = payload["items"][0]["id"] # collide ids164 with pytest.raises(GoldLoadError):165 load_gold(_write(tmp_path, payload))166 167 168def test_bad_id_pattern_rejected(tmp_path: Path) -> None:169 payload = _gold_payload()170 payload["items"][0]["id"] = "G1" # not gold-NNN171 with pytest.raises(GoldLoadError):172 load_gold(_write(tmp_path, payload))173 174 175def test_unknown_category_rejected(tmp_path: Path) -> None:176 payload = _gold_payload()177 payload["items"][0]["category"] = "not_a_category"178 with pytest.raises(GoldLoadError):179 load_gold(_write(tmp_path, payload))180 181 182def test_ooc_flag_disagreeing_with_category_rejected(tmp_path: Path) -> None:183 payload = _gold_payload()184 # A scalar item flagged out_of_coverage — the coupling check must reject it.185 payload["items"][0]["is_out_of_coverage"] = True186 with pytest.raises(GoldLoadError):187 load_gold(_write(tmp_path, payload))188 189 190def test_positive_item_with_empty_provenance_rejected(tmp_path: Path) -> None:191 payload = _gold_payload()192 # Strip provenance off a positive scalar item -> ungrounded positive (INV-C2).193 payload["items"][0]["expected_provenance_event_ids"] = []194 with pytest.raises(GoldLoadError):195 load_gold(_write(tmp_path, payload))196 197 198def test_dangling_provenance_id_rejected(tmp_path: Path) -> None:199 payload = _gold_payload()200 payload["items"][0]["expected_provenance_event_ids"] = ["bad-id"]201 with pytest.raises(GoldLoadError):202 load_gold(_write(tmp_path, payload))203 204 205def test_missing_file_raises_typed_error(tmp_path: Path) -> None:206 with pytest.raises(GoldLoadError):207 load_gold(str(tmp_path / "does_not_exist.json"))208 209 210# =====================================================================211# 2. CHECK-M1-gold — independent re-derivation over all_events + matches()212# =====================================================================213#214# These helpers re-derive each item's provenance from the LOADED store using the215# REAL matches() predicate over all_events, then apply the per-tool selection rule216# (LLD §1). This path shares ONLY the proven matches() leaf with gold authorship;217# the data path (all_events enumeration) is independent of both the authoring218# inline rule and the executor's per-tool index (D-VP-3 / INV-A7).219 220def _filter(store: str | None, product: str | None, state: str | None,221 start: str | None, end: str) -> TypedFilter:222 return TypedFilter(223 store_id=StoreId(store) if store is not None else None,224 product_id=ProductId(product) if product is not None else None,225 state=ShelfState(state) if state is not None else None,226 condition=None,227 window=Window(228 start=Timestamp(start) if start is not None else None,229 end=Timestamp(end),230 ),231 )232 233 234def _latest_le(events: tuple[ScanEvent, ...], store: str, product: str, as_of: str) -> ScanEvent | None:235 """The single latest event <= as_of for (store,product), via matches() over all_events."""236 f = _filter(store, product, None, None, as_of) # as_of: start=None -> timestamp<=end237 cand = [e for e in events if matches(e, f)]238 if not cand:239 return None240 cand.sort(key=lambda e: (e.timestamp, e.event_id))241 return cand[-1]242 243 244def _extract_tokens(question: str) -> tuple[str, str, str]:245 """Extract (store_id, product_id, as_of_ts) tokens from a question string."""246 store = re.search(r"st-\d{3}", question)247 product = re.search(r"pr-\d{3}", question)248 ts = re.search(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", question)249 assert store and product and ts, f"could not extract tokens from: {question}"250 return store.group(0), product.group(0), ts.group(0)251 252 253def _rederive_scalar(item: GoldItem, store_obj: LoadedStore) -> list[str]:254 s, p, as_of = _extract_tokens(item.question)255 e = _latest_le(store_obj.all_events, s, p, as_of)256 return [] if e is None else [e.event_id]257 258 259def _rederive_t4(item: GoldItem, store_obj: LoadedStore, all_store_ids: Sequence[str],260 all_product_ids: Sequence[str]) -> list[str]:261 """T4: per qualifying store, the contributing per-store-product latest<=as_of events."""262 cond_match = re.search(r"condition ([A-Z_]+)", item.question)263 ts_match = re.search(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", item.question)264 assert cond_match and ts_match, item.question265 cond, as_of = cond_match.group(1), ts_match.group(0)266 prov: list[str] = []267 for sid in all_store_ids:268 for pid in all_product_ids:269 e = _latest_le(store_obj.all_events, sid, pid, as_of)270 if e is None:271 continue272 if cond in _STOCK_CONDITIONS and e.state.value == cond:273 prov.append(e.event_id)274 elif cond in _COMPLIANCE_CONDITIONS and e.compliance_flag.value == cond:275 prov.append(e.event_id)276 return prov277 278 279def _rederive_t5(item: GoldItem, store_obj: LoadedStore, all_product_ids: Sequence[str]) -> list[str]:280 """T5: products whose latest<=as_of event at the store is in the given state."""281 s = re.search(r"st-\d{3}", item.question)282 state = re.search(r"state ([A-Z_]+)", item.question)283 ts = re.search(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", item.question)284 assert s and state and ts, item.question285 store, want_state, as_of = s.group(0), state.group(1), ts.group(0)286 prov: list[str] = []287 for pid in all_product_ids:288 e = _latest_le(store_obj.all_events, store, pid, as_of)289 if e is not None and e.state.value == want_state:290 prov.append(e.event_id)291 return prov292 293 294def _rederive_window(item: GoldItem, store_obj: LoadedStore) -> list[str]:295 """T6: all events at (store,state) in [start,end) via matches() over all_events."""296 s = re.search(r"st-\d{3}", item.question)297 state = re.search(r"(ON_SHELF|OUT_OF_STOCK|LOW_STOCK|MISPLACED)", item.question)298 bounds = re.search(r"\[(\S+), (\S+)\)", item.question)299 assert s and state and bounds, item.question300 store, want_state = s.group(0), state.group(1)301 start, end = bounds.group(1), bounds.group(2)302 f = _filter(store, None, want_state, start, end)303 cand = [e for e in store_obj.all_events if matches(e, f)]304 cand.sort(key=lambda e: (e.timestamp, e.event_id))305 return [e.event_id for e in cand]306 307 308def _rederive_provenance(item: GoldItem, store_obj: LoadedStore,309 store_ids: Sequence[str], product_ids: Sequence[str]) -> list[str]:310 """Dispatch re-derivation by the item's expected_answer kind (the tool it exercises)."""311 kind = item.expected_answer.get("kind")312 if kind in ("stock", "price", "compliance"):313 return _rederive_scalar(item, store_obj)314 if kind == "condition_count":315 return _rederive_t4(item, store_obj, store_ids, product_ids)316 if kind == "product_list":317 return _rederive_t5(item, store_obj, product_ids)318 if kind == "window_count":319 return _rederive_window(item, store_obj)320 if kind == "abstain":321 return []322 raise AssertionError(f"unknown answer kind for {item.id}: {kind!r}")323 324 325def _t4_store_ids(item: GoldItem, store_obj: LoadedStore, store_ids: Sequence[str],326 product_ids: Sequence[str]) -> set[str]:327 """The set of stores qualifying for a T4 item (independent re-derivation)."""328 cond = re.search(r"condition ([A-Z_]+)", item.question)329 ts = re.search(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", item.question)330 assert cond and ts, item.question331 c, as_of = cond.group(1), ts.group(0)332 qualifying: set[str] = set()333 for sid in store_ids:334 for pid in product_ids:335 e = _latest_le(store_obj.all_events, sid, pid, as_of)336 if e is None:337 continue338 if (c in _STOCK_CONDITIONS and e.state.value == c) or (339 c in _COMPLIANCE_CONDITIONS and e.compliance_flag.value == c340 ):341 qualifying.add(sid)342 break343 return qualifying344 345 346@pytest.fixture347def _store() -> LoadedStore:348 # Function-scoped (not module-scoped) so the autouse DATASET_VERSION env pin in349 # conftest is active when load_store() reads Settings() — a module-scoped fixture350 # would resolve before the function-scoped env patch and hit the no-default field.351 return load_store()352 353 354def test_check_m1_gold_provenance_rederives_for_every_item(_store: LoadedStore) -> None:355 """CHECK-M1-gold: re-derive EVERY item's provenance independently and compare.356 357 A mismatch means a hand-authoring slip in gold.json (per docs/09 §4 this is the358 mitigation that catches it before the acceptance gates consume gold).359 """360 gold = load_gold()361 store_ids = [s.store_id for s in _store.stores]362 product_ids = [p.product_id for p in _store.products]363 364 mismatches: list[str] = []365 for item in gold.items:366 rederived = set(_rederive_provenance(item, _store, store_ids, product_ids))367 authored = set(item.expected_provenance_event_ids)368 if rederived != authored:369 mismatches.append(370 f"{item.id} ({item.category}): authored={sorted(authored)} "371 f"rederived={sorted(rederived)}"372 )373 assert not mismatches, "CHECK-M1-gold provenance mismatch:\n" + "\n".join(mismatches)374 375 376def test_check_m1_gold_aggregate_counts_match_provenance(_store: LoadedStore) -> None:377 """Cross-check the typed answer cardinality against the COMPLETE provenance set.378 379 For aggregate/window/list items the answer's count / list length must equal the380 provenance cardinality the independent re-derivation produced (INV-C2381 completeness — an under-count would be a wrong gold item).382 """383 gold = load_gold()384 store_ids = [s.store_id for s in _store.stores]385 product_ids = [p.product_id for p in _store.products]386 387 problems: list[str] = []388 for item in gold.items:389 kind = item.expected_answer.get("kind")390 rederived = _rederive_provenance(item, _store, store_ids, product_ids)391 if kind == "window_count":392 if item.expected_answer["count"] != len(rederived):393 problems.append(f"{item.id}: window count {item.expected_answer['count']} != {len(rederived)}")394 elif kind == "condition_count":395 # count is the number of qualifying STORES; provenance is the contributing396 # per-store-product events (>= count). store_ids set-equality is the axis.397 derived_store_ids = _t4_store_ids(item, _store, store_ids, product_ids)398 if set(item.expected_answer["store_ids"]) != derived_store_ids:399 problems.append(400 f"{item.id}: store_ids {item.expected_answer['store_ids']} != {sorted(derived_store_ids)}"401 )402 if item.expected_answer["count"] != len(derived_store_ids):403 problems.append(f"{item.id}: condition count != #store_ids")404 elif kind == "product_list":405 if len(item.expected_answer["product_ids"]) != len(rederived):406 problems.append(f"{item.id}: product_list len != provenance len")407 assert not problems, "aggregate cardinality mismatch:\n" + "\n".join(problems)408