srr84/agent-data-layer

1"""SUITE-load (M7, F-NFR-1): the deterministic-span load/NFR gate.2 3Plain-language lead: this drives >=10 CONCURRENT requests through the real request path4(`answer_question`) using a PINNED $0 provider (the bench `OracleProvider` — no model, no5Ollama, no network), so the measured span is the DETERMINISTIC C-TDL path. It then asserts the6deterministic-span p95 (the `latency_ms` log field, NOT end-to-end model time) is <= 300ms7(`DETERMINISTIC_P95_BUDGET_MS`). It is fully CI-runnable: the pinned provider removes the model8from the measured loop, exactly as the Validation Plan §7 load test requires ("stub or pin the9model so the measured span is the deterministic path").10 11The concurrency itself is the race exercise: >=10 threads append to one shared `MetricsCounter`,12so the test also proves logging under concurrency is append-only with no interleave corruption13(one recorded row per request, INV-A2). The p95 is derived BY AGGREGATING the per-request14`latency_ms` log values via the same `metrics_from_lines` the offline run-log roll-up uses15(F-NFR-3 / D-TS-8).16 17New reader -> WALKTHROUGH.md §10 (observability / SUITE-load).18 19```text20spec traceability (audit map — safe to skip)21Validation §7 (SUITE-load: tool=stdlib asyncio/locust/k6 — all $0; >=10 concurrent users; stub or22        PIN the model so the measured span is the deterministic path; latency_p95 = p95 of latency_ms23        deterministic span; bar p95 <= 300ms = DETERMINISTIC_P95_BUDGET_MS)24TechSpec §6 (latency_ms = DETERMINISTIC C-TDL span only, excl. inference; the 5 metrics), §125        (DETERMINISTIC_P95_BUDGET_MS=300)26docs/09 §1/§4 M7 (SUITE-load >=10 concurrent, deterministic p95 <= 300ms, F-NFR-1)27Charter F-NFR-1 (load: >=10 concurrent, deterministic-span p95 <= 300ms), INV-C6 ($0: pinned28        provider, no Ollama), INV-A2 (logging append-only side-effect)29"""30 31from __future__ import annotations32 33from concurrent.futures import ThreadPoolExecutor34 35from agent_data_layer.agent.request_pipeline import answer_question36from agent_data_layer.contracts.settings import Settings37from agent_data_layer.observability.app_surfaces import MetricsCounter38from agent_data_layer.observability.log_line import metrics_from_lines39from agent_data_layer.store.event_index import build_index40from agent_data_layer.store.gold_loader import load_gold41from agent_data_layer.store.store_loader import load_store42from tests.bench_tool_calling import OracleProvider43 44# >=10 concurrent users (F-NFR-1). We drive more than the floor to make the p95 meaningful and to45# stress the shared-counter append path under real contention.46_CONCURRENCY = 1647_REQUESTS = 200  # enough samples that the p95 rank is well-defined (>= a few per worker).48 49 50def test_suite_load_deterministic_p95_under_budget() -> None:51    """>=10 concurrent requests on the pinned $0 provider; deterministic-span p95 <= 300ms.52 53    The measured span is the deterministic C-TDL path only — the OracleProvider's select_tool /54    compose_nl are pinned (no inference), so `latency_ms` reflects validate->execute->attach->55    result-envelope->staple, exactly what the F-NFR-1 budget governs.56    """57    store = load_store()58    index = build_index(store)59    gold = load_gold()60    provider = OracleProvider()61    counter = MetricsCounter()62 63    # The in-coverage gold questions (the OOC probes abstain before the deterministic span, so they64    # contribute a 0ms span and would not exercise the C-TDL path). Cycle them to fill _REQUESTS.65    in_cov = [item.question for item in gold.items if not item.is_out_of_coverage]66    questions = [in_cov[i % len(in_cov)] for i in range(_REQUESTS)]67 68    def _drive(question: str) -> None:69        # all_events from the loaded store (R-B-7); feed the shared counter (the live /metrics path).70        answer_question(question, index, store.all_events, provider, counter=counter)71 72    with ThreadPoolExecutor(max_workers=_CONCURRENCY) as pool:73        list(pool.map(_drive, questions))74 75    rows = list(counter.snapshot())76 77    # Concurrency-safety: every request recorded EXACTLY one row — no lost/duplicated append under78    # >=16-way contention (append-only, INV-A2).79    assert len(rows) == _REQUESTS80 81    metrics = metrics_from_lines(rows)82    budget = Settings(DATASET_VERSION=gold.dataset_version).DETERMINISTIC_P95_BUDGET_MS83 84    # The load-bearing F-NFR-1 assertion: deterministic-span p95 <= 300ms.85    assert metrics["latency_p95"] <= budget, (86        f"deterministic-span p95 {metrics['latency_p95']:.3f}ms exceeds budget {budget}ms"87    )88    # Sanity: the span is positive (the C-TDL path actually ran and was timed, not a 0 no-op).89    assert metrics["latency_p95"] > 0.090 91 92def test_suite_load_concurrency_floor_is_at_least_ten() -> None:93    """Guard the F-NFR-1 concurrency floor in-code so a future edit cannot silently weaken it."""94    assert _CONCURRENCY >= 1095