lankasailendra/BMCTESTMAIN
0
1#!/usr/bin/env python32"""3coverage_audit.py — Phase 1 of the codegen re-architecture (see project memory4`codegen-assembler-handoff.md`).5 6Question this answers, deterministically and with NO LLM in the loop:7 8 What percentage of the ITSM regression catalog's steps can be ASSEMBLED from9 the existing harness-proven conftest helper library, rather than generated10 as fresh code by a model?11 12The number produced here is the go/no-go input for the step->helper assembler.13This script makes NO decision and edits NOTHING; it only measures.14 15Inputs (all read-only), resolved relative to --reference:16 regression_catalog.json the 407-case catalog17 conftest_manifest.json the published helper vocabulary18 conftest_files/conftest_*.py helper definitions (AST-harvested)19 test_*.py reference test modules (real call-sites = evidence)20 21Usage:22 python coverage_audit.py --reference <path to reference/itsm> \23 [--json out.json] [--markdown out.md] [--samples N]24 25Design notes / deliberate conservatism (see MATCHER NOTES in the report):26 * Matching is verb-class x domain-object, both drawn from lexicons below.27 There is no fuzzy string similarity and no learned model: a step matches a28 helper only if it names the same OBJECT the helper name names, and applies a29 VERB in the same class as the helper's leading verb token.30 * A helper only yields the strongest verdict if it is PROVEN -- actually called31 by name in at least one reference test. Helpers that exist but have never run32 are demoted to PARTIAL, because an unexercised helper is not evidence.33 * Where the algorithm was underspecified, the stricter reading was taken and34 recorded. Ambiguity resolves DOWNWARD (ASSEMBLABLE -> PARTIAL -> GAP).35"""36 37from __future__ import annotations38 39import argparse40import ast41import collections42import json43import os44import re45import sys46from typing import Dict, List, Optional, Sequence, Set, Tuple47 48# --------------------------------------------------------------------------49# Verdicts50# --------------------------------------------------------------------------51# Five fine-grained labels. They roll up two ways (see rollup()) because the52# right rollup depends on what the assembler's emitter is taken to include.53HELPER = "HELPER" # a proven helper performs this operation54FIXTURE = "FIXTURE" # login / navigation, already carried by a pytest fixture55ASSERT = "ASSERT" # pure verification; emitter writes an assert from step.expected56FIELD = "FIELD" # a form-field entry: a PARAMETER of a neighbouring helper57 # call, not a call of its own. Needs step-grouping.58PARTIAL = "PARTIAL" # a helper touches this object but verb/params are unclear59GAP = "GAP" # nothing in the library covers this operation60 61VERDICTS = (HELPER, FIXTURE, ASSERT, FIELD, PARTIAL, GAP)62 63# How many ranked candidate helpers a step verdict carries. Only affects the64# reported candidate lists and the recall measure, never the verdict itself.65MAX_HELPERS = int(os.environ.get("COVERAGE_AUDIT_TOPN", "8"))66 67# Form fields. A step that only puts a value into one of these is not a separate68# operation -- it is an argument to the create/update helper for the record.69FIELD_TRIGGERS = (70 "summary", "description", "detailed description", "notes", "note",71 "impact", "urgency", "priority", "severity", "status", "reason",72 "status reason", "company", "site", "location", "region", "department",73 "organization", "organisation", "first name", "last name", "phone",74 "email", "category", "categorization", "categorisation", "tier",75 "class", "type", "risk", "risk level", "lead time", "start date",76 "end date", "due date", "target date", "scheduled start", "scheduled end",77 "scope", "timing", "service", "product", "manufacturer", "model",78 "serial", "serial number", "tag", "asset tag", "cost", "vendor",79 "supplier", "template", "source", "reported source", "resolution",80 "workaround", "root cause", "title", "keywords", "author", "reviewer",81 "expiry", "expiration", "environment", "version", "customer",82 "requester", "requested for", "assignee", "support group", "owner",83 "coordinator", "manager", "operational", "field",84)85 86# --------------------------------------------------------------------------87# Lexicons88# --------------------------------------------------------------------------89# Step-side verb triggers -> canonical verb class.90# Order matters only in that all triggers are scanned; a step may carry several.91VERB_TRIGGERS: Dict[str, Sequence[str]] = {92 "LOGIN": ("log in", "login", "log into", "sign in", "signin", "sign-in",93 "authenticate", "logged in as", "log on"),94 "NAV": ("open", "navigate", "go to", "browse", "launch", "access",95 "return to", "switch to", "back to", "land on", "visit"),96 "CREATE": ("create", "raise", "submit a new", "new record", "log a",97 "log an", "register", "generate a", "add a new", "make a new",98 "start a new", "initiate", "add", "post", "write", "document",99 "record a", "capture a"),100 "SET": ("set", "enter", "type", "fill", "select", "choose", "specify",101 "populate", "provide", "input", "pick", "assign a value",102 "update the", "modify", "edit", "change the", "supply", "flag",103 "tick", "check the box", "untick", "toggle"),104 "SAVE": ("save", "click save", "submit the form", "commit"),105 "SEARCH": ("search", "find", "look up", "lookup", "filter", "query",106 "locate", "identify", "list all", "retrieve"),107 "RELATE": ("relate", "associate", "link", "related item", "association",108 "attach the", "connect", "tie to"),109 "ASSIGN": ("assign", "reassign", "allocate", "assign to me", "take ownership"),110 "TRANSITION": ("move", "advance", "transition", "progress", "promote",111 "resolve", "close", "cancel", "complete", "reopen",112 "approve", "reject", "acknowledge", "escalate", "reroute",113 "mark as", "set status", "status to", "put the", "revert"),114 "ATTACH": ("attach", "upload", "add an attachment", "add attachment"),115 "CLEAR": ("delete", "remove", "clear", "detach", "unlink", "discard"),116 "SEND": ("send", "issue", "post", "invoke", "call the api", "rest call",117 "api call", "get request", "put request", "patch request"),118 "CLICK": ("click", "press", "tap", "hit"),119 "WAIT": ("wait", "pause", "poll until", "allow time"),120 "ASSERT": ("verify", "confirm", "check", "observe", "note", "read",121 "review", "validate", "inspect", "compare", "ensure that",122 "make sure", "should show", "should be", "record the",123 "capture the", "screenshot"),124}125 126# Helper-name leading-token -> verb class it satisfies.127HELPER_VERB_TOKENS: Dict[str, Sequence[str]] = {128 "CREATE": ("create", "new", "build", "make", "add", "raise", "submit",129 "generate", "provision", "seed"),130 "SET": ("set", "fill", "select", "pick", "patch", "update", "apply",131 "choose", "populate", "assign", "edit", "modify", "put",132 "normalize", "format", "rebuild", "toggle"),133 "SAVE": ("save", "click", "submit", "commit", "finalize"),134 "SEARCH": ("get", "find", "fetch", "search", "lookup", "query", "list",135 "resolve", "read", "extract", "load", "current", "collect",136 "discover", "scan"),137 "RELATE": ("relate", "associate", "link", "add", "attach", "association",138 "associations"),139 "ASSIGN": ("assign", "set", "fill", "pick", "select"),140 "TRANSITION": ("advance", "move", "patch", "close", "resolve", "complete",141 "reopen", "approve", "reject", "cancel", "transition",142 "set", "escalate", "ensure", "progress"),143 "ATTACH": ("attach", "upload", "add"),144 "CLEAR": ("clear", "remove", "delete", "detach", "reset", "cleanup"),145 "SEND": ("post", "put", "patch", "send", "call", "request", "api",146 "create", "get", "delete"),147 "CLICK": ("click", "open", "expand", "menu", "press", "select"),148 "WAIT": ("wait", "poll", "ensure"),149 "ASSERT": ("assert", "verify", "is", "has", "check", "get", "expect",150 "match", "compare", "validate", "confirm"),151 "NAV": ("open", "goto", "navigate", "smartit", "console", "page"),152 "LOGIN": ("login", "signin", "persona", "session", "authenticate"),153}154 155# Domain object triggers -> helper-name tokens that denote the same object.156# A step and a helper must agree on OBJECT for the helper to be a candidate.157OBJECT_LEXICON: Dict[str, Tuple[Tuple[str, ...], Tuple[str, ...]]] = {158 # canonical: (step-text triggers, helper-name tokens)159 "incident": (("incident", "hpd"),160 ("incident", "inc", "hpd")),161 "work_info": (("work info", "work note", "worklog", "work log",162 "work detail", "public note", "internal note"),163 ("work", "info", "note", "notes", "worklog")),164 "activity": (("activity", "timeline", "audit trail", "audit history",165 "history", "feed"),166 ("activity", "timeline", "audit", "history", "feed")),167 "recordtype": (("class", "type", "kind"),168 ("type", "class", "kind")),169 "tab": (("tab", "section", "panel", "accordion"),170 ("tab", "section", "panel")),171 "change": (("change request", "change", "crq", "rfc", "chg"),172 ("change", "chg", "crq")),173 "problem": (("problem investigation", "problem", "pbi", "pbm"),174 ("problem", "pbm", "pbi")),175 "known_error": (("known error", "known-error", "ke record", "pke"),176 ("known", "error", "ke", "pke")),177 "work_order": (("work order", "work-order", "woi", "wo "),178 ("work", "order", "wo", "woi")),179 "task": (("task", "tms", "sub-task", "subtask"),180 ("task", "tasks", "tms")),181 "knowledge": (("knowledge", "article", "kba", "rkm", "how to", "how-to"),182 ("knowledge", "article", "kba", "rkm", "km")),183 "release": (("release", "rms", "release manifest"),184 ("release", "rms", "rel")),185 "asset": (("asset", "ast:", "hardware", "inventory"),186 ("asset", "ast")),187 "ci": (("configuration item", "ci", "cis", "cmdb", "bmc.core"),188 ("ci", "cmdb", "config")),189 "approval": (("approval", "approver", "approve", "sfa", "signature",190 "authoriz"),191 ("approval", "approve", "approver", "sfa", "signature",192 "auth")),193 "category": (("categorization", "categorisation", "category", "categories",194 "operational cat", "product cat", "tier 1", "tier 2"),195 ("category", "categories", "cat", "cats", "op", "prod",196 "tier")),197 "status": (("status", "state", "lifecycle", "stage bar", "stage"),198 ("status", "state", "stage", "lifecycle")),199 "assignment": (("assign", "assignee", "support group", "assigned group",200 "owner group", "coordinator", "manager", "ownership"),201 ("assign", "assignment", "assignee", "group", "support",202 "coordinator", "manager", "owner")),203 "attachment": (("attachment", "attach a file", "upload"),204 ("attachment", "attach", "upload", "file")),205 "console": (("console", "ticket console", "dashboard", "list view",206 "grid", "search results"),207 ("console", "filter", "filters", "grid", "list")),208 "template": (("template",), ("template", "templates", "tmpl")),209 # Raw REST-transport steps ("Send POST {base}/api/jwt/login ...", "Resend210 # the GET request without the Authorization header"). Added 2026-08-17211 # with send_rest_request / jwt_login_capture_token; deliberately does NOT212 # include the bare word "request" ("change request" would false-positive) —213 # "get/post/put request" phrasings are covered as two-word triggers.214 "rest": (("rest", "api", "endpoint", "jwt", "token", "http", "https",215 "curl", "postman", "payload", "json", "authorization header",216 "get request", "post request", "put request", "patch request",217 "delete request", "rest client", "response code", "status code"),218 ("rest", "api", "jwt", "token", "request", "http", "endpoint")),219 "sla": (("sla", "service target", "svt", "milestone"),220 ("sla", "svt", "target", "milestone")),221 "freeze": (("freeze", "blackout", "calendar"),222 ("freeze", "cal", "calendar", "blackout")),223 "customer": (("customer", "requester", "requested for", "affected user",224 "contact"),225 ("customer", "requester", "requested", "contact", "people",226 "ctm")),227 "notification": (("notification", "email", "notify", "alert"),228 ("notification", "email", "notify", "alert")),229 "relationship": (("related item", "relationship", "association",230 "relate"),231 ("relate", "association", "associations", "item",232 "items", "link")),233 "date": (("scheduled start", "scheduled end", "target date", "due date",234 "date/time", "schedule"),235 ("date", "dates", "schedule", "sched", "start", "end", "due")),236 "priority": (("priority", "impact", "urgency", "severity", "weight"),237 ("priority", "impact", "urgency", "severity")),238 "risk": (("risk", "risk level"), ("risk",)),239 "resolution": (("resolution", "resolve", "root cause", "workaround",240 "fix note", "closure"),241 ("resolution", "resolve", "cause", "workaround", "close",242 "closure")),243 "person": (("persona", "user account", "login as", "test account",244 "agent", "requester login"),245 ("persona", "user", "login", "session", "account")),246 "company": (("company", "organization", "organisation", "site",247 "location", "region"),248 ("company", "org", "site", "location", "region")),249 "service": (("service", "business service", "service ci"),250 ("service", "svc")),251 "cost": (("cost", "price", "financial", "budget", "invoice"),252 ("cost", "price", "financial", "budget")),253 "report": (("report", "metric", "kpi", "chart", "export"),254 ("report", "metric", "kpi", "export")),255 "helixgpt": (("helixgpt", "helix gpt", "ai agent", "chatbot", "copilot",256 "collaborator", "sparkle", "generative"),257 ("helixgpt", "gpt", "ai", "chat", "agent")),258}259 260# Steps whose whole content is environment setup already carried by a fixture.261FIXTURE_PATTERNS = (262 r"\blog ?in\b", r"\bsign ?in\b", r"\blogged in\b",263 r"\bopen the smart ?it url\b", r"\bopen smart ?it\b",264 r"\bnavigate to (the )?smart ?it\b",265 r"\bopen (the )?(mid ?tier|browser)\b",266 r"\blaunch the (application|browser|url)\b",267)268 269# Module ownership tokens. If a helper's NAME carries one of these tokens, the270# helper belongs to that module and may only serve a step that actually names271# that module's object (or a step in that module's own cases). This is the272# deterministic cross-module veto -- the recurring leak class in this codebase273# (see project memory `module-inference-catalog-leak`, `cross-module-audit`).274MODULE_TOKENS: Dict[str, Tuple[str, ...]] = {275 "incident": ("incident", "inc", "hpd"),276 "change": ("change", "chg", "crq"),277 "problem": ("problem", "pbm", "pbi"),278 "known_error": ("pke", "ke"),279 "work_order": ("wo", "woi"),280 "task": ("task", "tasks", "tms"),281 "knowledge": ("knowledge", "kba", "rkm", "km"),282 "release": ("release", "rms"),283 "asset": ("asset", "ast"),284 "ci": ("cmdb",),285}286# Catalog module name -> canonical object key, for the veto's "own module" test.287CASE_MODULE_CANON = {288 "Incident": "incident", "Change": "change", "Problem": "problem",289 "Known Error": "known_error", "Work Order": "work_order",290 "Knowledge": "knowledge", "Release": "release", "Asset": "asset",291 "CMDB": "ci", "HelixGPT": "helixgpt",292}293 294# Quoted test data and record ids leak object words ("Create infrastructure295# change request") into steps that are not about that object. Strip them before296# object detection.297QUOTED_RE = re.compile(r"'[^']*'|\"[^\"]*\"|‘[^’]*’|“[^”]*”")298IDLIKE_RE = re.compile(r"\b(?:ITSM|TC|UAT|CRQ|INC|PBI|WO|RLM|TAS|AST|KBA)[-_A-Z0-9]*\d[-_A-Z0-9]*\b",299 re.IGNORECASE)300 301STOPWORDS = {302 "the", "a", "an", "and", "or", "of", "to", "in", "on", "for", "with",303 "is", "are", "be", "it", "that", "this", "as", "at", "by", "from", "if",304 "then", "when", "your", "you", "its", "any", "all", "new", "test",305}306 307 308# --------------------------------------------------------------------------309# Vocabulary harvesting310# --------------------------------------------------------------------------311class Helper:312 __slots__ = ("name", "source", "tokens", "lead", "doc", "is_fixture",313 "is_private", "proven", "call_count")314 315 def __init__(self, name: str, source: str, doc: str = "",316 is_fixture: bool = False):317 self.name = name318 self.source = source319 self.doc = (doc or "").strip()320 self.is_fixture = is_fixture321 self.is_private = name.startswith("_")322 parts = [p for p in name.strip("_").lower().split("_") if p]323 self.tokens = set(parts)324 self.lead = parts[0] if parts else ""325 self.proven = False326 self.call_count = 0327 328 329def harvest_vocabulary(ref: str) -> Dict[str, Helper]:330 """manifest names UNION AST defs across conftest*.py."""331 helpers: Dict[str, Helper] = {}332 333 manifest_path = os.path.join(ref, "conftest_manifest.json")334 if os.path.exists(manifest_path):335 with open(manifest_path, encoding="utf-8") as fh:336 manifest = json.load(fh)337 for fn in manifest.get("functions", []):338 nm = fn.get("name")339 if not nm:340 continue341 helpers[nm] = Helper(nm, "manifest:" + str(fn.get("module", "")),342 fn.get("docstring") or "")343 344 conftests = [os.path.join(ref, "conftest.py")]345 conftests += sorted(346 os.path.join(ref, "conftest_files", f)347 for f in os.listdir(os.path.join(ref, "conftest_files"))348 if f.startswith("conftest_") and f.endswith(".py")349 ) if os.path.isdir(os.path.join(ref, "conftest_files")) else []350 351 for path in conftests:352 if not os.path.exists(path):353 continue354 try:355 tree = ast.parse(_read(path))356 except SyntaxError as exc: # never fatal; record and continue357 print(f" ! could not parse {os.path.basename(path)}: {exc}",358 file=sys.stderr)359 continue360 for node in ast.walk(tree):361 if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):362 continue363 is_fix = any(364 (isinstance(d, ast.Name) and d.id == "fixture")365 or (isinstance(d, ast.Attribute) and d.attr == "fixture")366 or (isinstance(d, ast.Call) and _dec_name(d.func) == "fixture")367 for d in node.decorator_list368 )369 existing = helpers.get(node.name)370 if existing is None:371 helpers[node.name] = Helper(372 node.name, os.path.basename(path),373 ast.get_docstring(node) or "", is_fix)374 else:375 existing.source = os.path.basename(path)376 existing.is_fixture = existing.is_fixture or is_fix377 if not existing.doc:378 existing.doc = ast.get_docstring(node) or ""379 return helpers380 381 382def _dec_name(node) -> str:383 if isinstance(node, ast.Attribute):384 return node.attr385 if isinstance(node, ast.Name):386 return node.id387 return ""388 389 390def _read(path: str) -> str:391 with open(path, encoding="utf-8", errors="replace") as fh:392 return fh.read()393 394 395# --------------------------------------------------------------------------396# Evidence: real call-sites in the reference test modules397# --------------------------------------------------------------------------398TC_RE = re.compile(r"(TC-[A-Z]+-[A-Z]+-\d+)")399STEP_COMMENT_RE = re.compile(r"^\s*#\s*Steps?\s*(\d+)(?:\s*[-–]\s*(\d+))?\s*[:—-]?\s*(.*)$",400 re.IGNORECASE)401 402 403class Evidence:404 def __init__(self):405 self.by_case: Dict[str, dict] = {}406 self.step_level: List[dict] = [] # only where '# Step N' comments exist407 self.test_count = 0408 409 410def harvest_evidence(ref: str, helpers: Dict[str, Helper]) -> Evidence:411 """For every reference test: which helpers it calls BY NAME, which fixtures412 it takes, and how much raw inline UI/REST it still contains."""413 ev = Evidence()414 test_files = sorted(f for f in os.listdir(ref)415 if f.startswith("test_") and f.endswith(".py"))416 for fname in test_files:417 path = os.path.join(ref, fname)418 src = _read(path)419 lines = src.splitlines()420 try:421 tree = ast.parse(src)422 except SyntaxError as exc:423 print(f" ! could not parse {fname}: {exc}", file=sys.stderr)424 continue425 for node in tree.body:426 if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):427 continue428 if not node.name.startswith("test_"):429 continue430 ev.test_count += 1431 doc = ast.get_docstring(node) or ""432 m = TC_RE.search(doc) or TC_RE.search(node.name.upper().replace("_", "-"))433 tc_id = m.group(1) if m else None434 435 called: Set[str] = set()436 raw_rest = 0437 raw_ui = 0438 for sub in ast.walk(node):439 if not isinstance(sub, ast.Call):440 continue441 fn = sub.func442 # Only bare Name() calls count as helper usage. page.click()443 # and similar attribute calls are Playwright, not our library --444 # counting them would inflate "proven" with false positives.445 if isinstance(fn, ast.Name):446 if fn.id in helpers:447 called.add(fn.id)448 helpers[fn.id].call_count += 1449 helpers[fn.id].proven = True450 elif isinstance(fn, ast.Attribute):451 root = _attr_root(fn)452 if root in ("requests", "session") and fn.attr in (453 "post", "put", "patch", "delete"):454 raw_rest += 1455 elif fn.attr in ("click", "fill", "type", "press",456 "select_option", "set_input_files",457 "check", "uncheck", "goto"):458 raw_ui += 1459 for arg in node.args.args:460 if arg.arg in helpers:461 helpers[arg.arg].proven = True462 helpers[arg.arg].call_count += 1463 called.add(arg.arg)464 465 rec = {466 "test": node.name, "file": fname, "tc_id": tc_id,467 "helpers": sorted(called), "raw_rest": raw_rest,468 "raw_ui": raw_ui,469 }470 if tc_id:471 ev.by_case[tc_id] = rec472 473 # step-level evidence where the author left '# Step N' markers474 start = node.lineno - 1475 end = getattr(node, "end_lineno", len(lines))476 markers: List[Tuple[int, int, str]] = []477 for i in range(start, min(end, len(lines))):478 sm = STEP_COMMENT_RE.match(lines[i])479 if sm:480 lo = int(sm.group(1))481 hi = int(sm.group(2)) if sm.group(2) else lo482 markers.append((i, lo, hi))483 for idx, (line_i, lo, hi) in enumerate(markers):484 seg_end = markers[idx + 1][0] if idx + 1 < len(markers) else end485 seg = "\n".join(lines[line_i:seg_end])486 try:487 seg_tree = ast.parse(_dedent(seg))488 except SyntaxError:489 continue490 seg_helpers = {491 n.func.id for n in ast.walk(seg_tree)492 if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)493 and n.func.id in helpers494 }495 ev.step_level.append({496 "tc_id": tc_id, "step_lo": lo, "step_hi": hi,497 "helpers": sorted(seg_helpers),498 })499 return ev500 501 502def _attr_root(node) -> str:503 cur = node504 while isinstance(cur, ast.Attribute):505 cur = cur.value506 return cur.id if isinstance(cur, ast.Name) else ""507 508 509def _dedent(block: str) -> str:510 lines = [l for l in block.splitlines() if l.strip()]511 if not lines:512 return block513 pad = min(len(l) - len(l.lstrip()) for l in lines)514 return "\n".join(l[pad:] if len(l) >= pad else l515 for l in block.splitlines())516 517 518# --------------------------------------------------------------------------519# Matcher520# --------------------------------------------------------------------------521def _trigger_re(triggers: Sequence[str]) -> "re.Pattern":522 """Word-boundary alternation. Substring matching produced false positives523 (' ci ' inside 'specific', 'ke' inside 'make'), so every trigger is524 anchored on word boundaries."""525 parts = sorted((re.escape(t) for t in triggers), key=len, reverse=True)526 # trailing (e)s? so plurals match: 'changes', 'assets', 'CIs', 'tasks'527 return re.compile(r"(?<![a-z0-9])(?:" + "|".join(parts) + r")(?:e?s)?(?![a-z0-9])",528 re.IGNORECASE)529 530 531_VERB_RE = {c: _trigger_re(t) for c, t in VERB_TRIGGERS.items()}532_OBJ_RE = {c: _trigger_re(t[0]) for c, t in OBJECT_LEXICON.items()}533_FIELD_RE = _trigger_re(FIELD_TRIGGERS)534 535# Relative clauses describe a PRECONDITION, not the step's own action:536# "log in with an account that can create changes" is a login step.537SUBORDINATE_RE = re.compile(538 r"\b(that|who|which)\s+(can|may|is|are|has|have)\b.*$|"539 r"\bwith (permission|rights|access|the role)\b.*$|"540 r"\b(able|allowed|authoriz\w+|entitled|permitted) to\b.*$",541 re.IGNORECASE)542 543 544def strip_literals(text: str) -> str:545 """Remove quoted test data and record ids so their words do not register546 as domain objects for the surrounding step."""547 return IDLIKE_RE.sub(" ", QUOTED_RE.sub(" ", text))548 549 550def step_verbs(text: str) -> Set[str]:551 return {c for c, rx in _VERB_RE.items() if rx.search(text)}552 553 554def step_objects(text: str) -> Set[str]:555 clean = strip_literals(text)556 return {c for c, rx in _OBJ_RE.items() if rx.search(clean)}557 558 559def module_veto(h: Helper, objs: Set[str], case_module: str) -> bool:560 """True => reject this helper for this step (cross-module leak)."""561 owned = {mod for mod, toks in MODULE_TOKENS.items() if h.tokens & set(toks)}562 if not owned:563 return False # module-neutral helper, always allowed564 allowed = set(objs)565 own = CASE_MODULE_CANON.get(case_module)566 if own:567 allowed.add(own)568 return not (owned & allowed)569 570 571def is_fixture_step(text: str) -> bool:572 low = text.lower()573 return any(re.search(p, low) for p in FIXTURE_PATTERNS)574 575 576def classify_step(action: str, module: str, helpers: Dict[str, Helper],577 obj_index: Dict[str, List[Helper]]) -> dict:578 verbs = step_verbs(action)579 objs = step_objects(action)580 581 # 1. Fixture-carried environment setup (login / open the app).582 # Conservative note: this is genuinely covered today by the *_smartit_page583 # and persona_session fixtures, but it is NOT a helper mapping, so it is584 # reported in its own bucket rather than folded into HELPER.585 main_clause = SUBORDINATE_RE.sub(" ", action)586 if is_fixture_step(action) and not (587 step_verbs(main_clause) &588 {"CREATE", "SET", "SAVE", "RELATE", "ASSIGN", "TRANSITION",589 "ATTACH", "CLEAR"}):590 return {"verdict": FIXTURE, "helpers": [], "verbs": sorted(verbs),591 "objects": sorted(objs)}592 593 mutating = verbs & {"CREATE", "SET", "SAVE", "RELATE", "ASSIGN",594 "TRANSITION", "ATTACH", "CLEAR", "SEND"}595 596 # 2. Pure verification -- no mutation verb anywhere in the sentence.597 if not mutating and (verbs & {"ASSERT"}):598 # The emitter can write an assert from step.expected, but only if some599 # proven reader/asserter exists for the object (or it is a plain600 # on-screen text check, which assert_pwa_text covers generically).601 cands = _candidates(objs, obj_index, module)602 proven_readers = [h for h in cands603 if h.proven and h.lead in HELPER_VERB_TOKENS["ASSERT"]]604 return {"verdict": ASSERT,605 "helpers": [h.name for h in proven_readers[:MAX_HELPERS]],606 "verbs": sorted(verbs), "objects": sorted(objs)}607 608 # 3. Operational steps: need an object to anchor on.609 cands = _candidates(objs, obj_index, module)610 if not cands:611 # Object-less but unambiguous UI commit ("Click Save.")612 if verbs & {"SAVE"}:613 savers = [h for h in helpers.values()614 if h.proven and "save" in h.tokens]615 if savers:616 return {"verdict": HELPER,617 "helpers": [h.name for h in savers[:3]],618 "verbs": sorted(verbs), "objects": []}619 fld = _field_step(action, verbs)620 if fld:621 return {"verdict": FIELD, "helpers": [], "field": fld,622 "verbs": sorted(verbs), "objects": sorted(objs)}623 return {"verdict": GAP, "helpers": [], "verbs": sorted(verbs),624 "objects": sorted(objs)}625 626 want = set()627 for v in (mutating or verbs):628 want |= set(HELPER_VERB_TOKENS.get(v, ()))629 630 verb_ok_proven = [h for h in cands if h.proven and h.lead in want]631 verb_ok_any = [h for h in cands if h.lead in want]632 633 if verb_ok_proven:634 return {"verdict": HELPER,635 "helpers": [h.name for h in _rank(verb_ok_proven)[:MAX_HELPERS]],636 "verbs": sorted(verbs), "objects": sorted(objs)}637 if verb_ok_any:638 # helper exists for exactly this operation but has never been exercised639 return {"verdict": PARTIAL, "reason": "helper exists but unproven",640 "helpers": [h.name for h in _rank(verb_ok_any)[:MAX_HELPERS]],641 "verbs": sorted(verbs), "objects": sorted(objs)}642 fld = _field_step(action, verbs)643 if fld:644 return {"verdict": FIELD, "helpers": [], "field": fld,645 "verbs": sorted(verbs), "objects": sorted(objs)}646 proven_obj = [h for h in cands if h.proven]647 if proven_obj:648 return {"verdict": PARTIAL, "reason": "object covered, verb unmatched",649 "helpers": [h.name for h in _rank(proven_obj)[:MAX_HELPERS]],650 "verbs": sorted(verbs), "objects": sorted(objs)}651 return {"verdict": GAP, "helpers": [], "verbs": sorted(verbs),652 "objects": sorted(objs)}653 654 655def _field_step(action: str, verbs: Set[str]) -> Optional[str]:656 """A value-entry step: 'Enter Summary X', 'Set Impact = 3-Moderate'.657 These are ARGUMENTS to a create/update helper, not operations of their own,658 so the assembler must group them onto the neighbouring call rather than659 emit one line each. Only fires for pure entry verbs -- a step that also660 transitions, relates or assigns is a real operation."""661 if verbs & {"TRANSITION", "RELATE", "ASSIGN", "ATTACH", "CLEAR", "SEND",662 "CREATE", "SEARCH"}:663 return None664 if not (verbs & {"SET", "CLICK", "SAVE"}):665 return None666 m = _FIELD_RE.search(strip_literals(action))667 return m.group(0).lower() if m else None668 669 670def _candidates(objs: Set[str], obj_index: Dict[str, List[Helper]],671 case_module: str) -> List[Helper]:672 out: Dict[str, Helper] = {}673 for o in objs:674 for h in obj_index.get(o, ()):675 if module_veto(h, objs, case_module):676 continue677 out[h.name] = h678 return list(out.values())679 680 681def _rank(hs: List[Helper]) -> List[Helper]:682 return sorted(hs, key=lambda h: (-h.call_count, h.is_private, len(h.name)))683 684 685def build_object_index(helpers: Dict[str, Helper]) -> Dict[str, List[Helper]]:686 idx: Dict[str, List[Helper]] = collections.defaultdict(list)687 for h in helpers.values():688 for canon, (_, htokens) in OBJECT_LEXICON.items():689 if h.tokens & set(htokens):690 idx[canon].append(h)691 return idx692 693 694# --------------------------------------------------------------------------695# Gap clustering (same style as src/helper_gap_report.py: signature, count, 20)696# --------------------------------------------------------------------------697def gap_signature(action: str, verdict_rec: dict) -> str:698 verbs = verdict_rec.get("verbs") or ["?"]699 objs = verdict_rec.get("objects") or ["?"]700 return f"{'+'.join(sorted(verbs))} :: {'+'.join(sorted(objs))}"701 702 703def cluster_gaps(rows: List[dict], cap: int = 20) -> List[dict]:704 buckets: Dict[str, dict] = {}705 for r in rows:706 if r["verdict"] not in (GAP, PARTIAL):707 continue708 sig = gap_signature(r["action"], r)709 b = buckets.setdefault(sig, {710 "signature": sig, "verdict": r["verdict"], "count": 0,711 "modules": collections.Counter(), "examples": [],712 "gap_count": 0, "partial_count": 0,713 })714 b["count"] += 1715 b["gap_count"] += 1 if r["verdict"] == GAP else 0716 b["partial_count"] += 1 if r["verdict"] == PARTIAL else 0717 b["modules"][r["module"]] += 1718 if len(b["examples"]) < 3:719 b["examples"].append(f'{r["tc_id"]}: {r["action"][:110]}')720 out = sorted(buckets.values(), key=lambda b: -b["gap_count"])721 for b in out:722 b["modules"] = b["modules"].most_common()723 return out[:cap]724 725 726# --------------------------------------------------------------------------727# Rollups728# --------------------------------------------------------------------------729def rollup(counter: collections.Counter) -> dict:730 total = sum(counter.values()) or 1731 strict_ok = counter[HELPER]732 design_ok = counter[HELPER] + counter[FIXTURE] + counter[ASSERT]733 grouped_ok = design_ok + counter[FIELD]734 return {735 "total": sum(counter.values()),736 "by_verdict": {v: counter[v] for v in VERDICTS},737 "pct": {v: round(100.0 * counter[v] / total, 1) for v in VERDICTS},738 # helper calls only -- the narrowest reading739 "strict_assemblable_pct": round(100.0 * strict_ok / total, 1),740 # + fixture-carried setup and expected-text asserts (the emitter's job741 # per the Phase 2 design sketch)742 "design_assemblable_pct": round(100.0 * design_ok / total, 1),743 # + field-entry steps folded into a neighbouring call (needs grouping)744 "grouped_assemblable_pct": round(100.0 * grouped_ok / total, 1),745 "grouped_plus_partial_pct": round(746 100.0 * (grouped_ok + counter[PARTIAL]) / total, 1),747 }748 749 750# --------------------------------------------------------------------------751# Main752# --------------------------------------------------------------------------753def precision_check(ev: Evidence, catalog: List[dict],754 helpers: Dict[str, Helper],755 obj_index: Dict[str, List[Helper]]) -> dict:756 """The one OBJECTIVE accuracy measure available.757 758 Where a reference test author left '# Step N' markers, we know which helpers759 really served that catalog step. Run the matcher on the same step text and760 ask whether its predicted helper set intersects the truth. Small n, but it761 is a real held-out check rather than self-assessment."""762 by_tc = {c["tc_id"]: c for c in catalog if c.get("tc_id")}763 hits = misses = no_pred = 0764 detail = []765 for rec in ev.step_level:766 case = by_tc.get(rec["tc_id"])767 if not case or not rec["helpers"]:768 continue # no ground truth for this segment769 steps = case.get("steps") or []770 idxs = range(rec["step_lo"] - 1, min(rec["step_hi"], len(steps)))771 actions = [steps[i].get("action", "") for i in idxs if 0 <= i < len(steps)]772 if not actions:773 continue774 pred: Set[str] = set()775 for a in actions:776 pred |= set(classify_step(a, case.get("module", "?"), helpers,777 obj_index).get("helpers") or [])778 truth = set(rec["helpers"])779 if not pred:780 no_pred += 1781 outcome = "no-prediction"782 elif pred & truth:783 hits += 1784 outcome = "hit"785 else:786 misses += 1787 outcome = "miss"788 detail.append({789 "tc_id": rec["tc_id"], "steps": f'{rec["step_lo"]}-{rec["step_hi"]}',790 "outcome": outcome, "predicted": sorted(pred)[:4],791 "actual": sorted(truth)[:4],792 "action": (actions[0] or "")[:100],793 })794 n = hits + misses + no_pred795 return {796 "n_segments_with_truth": n,797 "hit": hits, "miss": misses, "no_prediction": no_pred,798 "hit_rate_pct": round(100.0 * hits / n, 1) if n else None,799 "detail": detail,800 }801 802 803def case_level_recall(ev: Evidence, catalog: List[dict],804 helpers: Dict[str, Helper],805 obj_index: Dict[str, List[Helper]]) -> dict:806 """Better-powered accuracy measure than the 52 step markers.807 808 For every catalog case that HAS a reference test (n=298), compare the union809 of helpers the matcher proposes across the case's steps against the helpers810 the reference test actually calls. Recall = how much of the real solution811 the matcher found. This is the number to trust when judging the matcher,812 because segment-level '# Step N' truth is misaligned (authors do data setup813 for later steps inside the first segment).814 815 Setup/fixture helpers are excluded from truth: they are carried by fixtures,816 not proposed per step."""817 ignore = {"persona_session", "session", "page"}818 by_tc = {c["tc_id"]: c for c in catalog if c.get("tc_id")}819 recalls, precisions = [], []820 good = 0821 worst = []822 for tc, rec in ev.by_case.items():823 case = by_tc.get(tc)824 if not case:825 continue826 truth = {h for h in rec["helpers"]827 if h not in ignore and not helpers[h].is_fixture}828 if not truth:829 continue830 pred: Set[str] = set()831 for st in case.get("steps") or []:832 r = classify_step(st.get("action", ""), case.get("module", "?"),833 helpers, obj_index)834 pred |= set(r.get("helpers") or [])835 inter = pred & truth836 rc = len(inter) / len(truth)837 pr = len(inter) / len(pred) if pred else 0.0838 recalls.append(rc)839 precisions.append(pr)840 if rc >= 0.5:841 good += 1842 else:843 worst.append({"tc_id": tc, "recall": round(rc, 2),844 "missed": sorted(truth - pred)[:6]})845 n = len(recalls) or 1846 return {847 "cases_evaluated": len(recalls),848 "mean_recall_pct": round(100.0 * sum(recalls) / n, 1),849 "mean_precision_pct": round(100.0 * sum(precisions) / n, 1),850 "cases_recall_ge_50pct": good,851 "cases_recall_ge_50pct_share": round(100.0 * good / n, 1),852 "worst": sorted(worst, key=lambda w: w["recall"])[:12],853 }854 855 856def run(ref: str, samples: int = 6) -> dict:857 print(f"reference: {ref}")858 helpers = harvest_vocabulary(ref)859 print(f" helper vocabulary: {len(helpers)}")860 ev = harvest_evidence(ref, helpers)861 proven = [h for h in helpers.values() if h.proven]862 print(f" reference tests parsed: {ev.test_count}"863 f" ({len(ev.by_case)} carry a TC id)")864 print(f" proven helpers (called by name in >=1 test): {len(proven)}")865 print(f" step-level '# Step N' evidence records: {len(ev.step_level)}")866 867 obj_index = build_object_index(helpers)868 869 with open(os.path.join(ref, "regression_catalog.json"), encoding="utf-8") as fh:870 catalog = json.load(fh)["test_cases"]871 print(f" catalog cases: {len(catalog)}")872 873 rows: List[dict] = []874 case_rows: List[dict] = []875 for case in catalog:876 tc = case.get("tc_id")877 mod = case.get("module", "?")878 has_ref = tc in ev.by_case879 verdicts = collections.Counter()880 step_recs = []881 for i, st in enumerate(case.get("steps") or []):882 action = (st.get("action") or "").strip()883 rec = classify_step(action, mod, helpers, obj_index)884 rec.update({"tc_id": tc, "module": mod, "step_no": i + 1,885 "action": action, "has_ref_test": has_ref})886 rows.append(rec)887 step_recs.append(rec)888 verdicts[rec["verdict"]] += 1889 worst = (GAP if verdicts[GAP] else890 PARTIAL if verdicts[PARTIAL] else HELPER)891 case_rows.append({892 "tc_id": tc, "module": mod, "has_ref_test": has_ref,893 "steps": len(step_recs), "verdicts": dict(verdicts),894 "case_verdict": worst,895 "raw_ui": ev.by_case.get(tc, {}).get("raw_ui", 0),896 "raw_rest": ev.by_case.get(tc, {}).get("raw_rest", 0),897 })898 899 def cnt(pred) -> collections.Counter:900 c = collections.Counter()901 for r in rows:902 if pred(r):903 c[r["verdict"]] += 1904 return c905 906 overall = rollup(cnt(lambda r: True))907 reuse = rollup(cnt(lambda r: r["has_ref_test"]))908 nonreuse = rollup(cnt(lambda r: not r["has_ref_test"]))909 910 by_module = {}911 for mod in sorted({r["module"] for r in rows}):912 by_module[mod] = rollup(cnt(lambda r, m=mod: r["module"] == m))913 914 # case-level: a case is assemblable only if EVERY step is915 case_total = len(case_rows)916 case_strict = sum(1 for c in case_rows917 if not c["verdicts"].get(GAP)918 and not c["verdicts"].get(PARTIAL))919 case_tolerant = sum(1 for c in case_rows if not c["verdicts"].get(GAP))920 case_nonreuse = [c for c in case_rows if not c["has_ref_test"]]921 case_reuse = [c for c in case_rows if c["has_ref_test"]]922 923 result = {924 "inputs": {925 "reference_dir": ref,926 "helper_vocabulary": len(helpers),927 "proven_helpers": len(proven),928 "reference_tests": ev.test_count,929 "reference_tests_with_tc_id": len(ev.by_case),930 "step_level_evidence_records": len(ev.step_level),931 "catalog_cases": len(catalog),932 "catalog_steps": len(rows),933 },934 "steps": {935 "overall": overall,936 "reuse_path": reuse,937 "non_reuse_path": nonreuse,938 "by_module": by_module,939 },940 "cases": {941 "total": case_total,942 "all_steps_helper_or_fixture_or_assert": case_strict,943 "no_gap_steps": case_tolerant,944 "strict_pct": round(100.0 * case_strict / max(case_total, 1), 1),945 "tolerant_pct": round(100.0 * case_tolerant / max(case_total, 1), 1),946 "reuse": {947 "total": len(case_reuse),948 "no_gap": sum(1 for c in case_reuse if not c["verdicts"].get(GAP)),949 },950 "non_reuse": {951 "total": len(case_nonreuse),952 "no_gap": sum(1 for c in case_nonreuse if not c["verdicts"].get(GAP)),953 },954 },955 "precision_check": precision_check(ev, catalog, helpers, obj_index),956 "case_level_recall": case_level_recall(ev, catalog, helpers, obj_index),957 "gap_clusters": cluster_gaps(rows),958 "worst_cases": sorted(959 [c for c in case_rows if c["verdicts"].get(GAP)],960 key=lambda c: -c["verdicts"].get(GAP, 0))[:15],961 "samples": {962 v: [f'[{r["module"]}/{r["tc_id"]}#{r["step_no"]}] {r["action"][:130]}'963 f' -> {",".join(r["helpers"][:3]) or "-"}'964 for r in rows if r["verdict"] == v][:samples]965 for v in VERDICTS966 },967 "unproven_helper_count": len(helpers) - len(proven),968 "_rows": rows,969 "_cases": case_rows,970 }971 return result972 973 974def main() -> int:975 ap = argparse.ArgumentParser()976 ap.add_argument("--reference", required=True,977 help="path to reference/itsm")978 ap.add_argument("--json", help="write full results as JSON")979 ap.add_argument("--samples", type=int, default=6)980 args = ap.parse_args()981 982 res = run(args.reference, args.samples)983 984 s = res["steps"]985 print("\n=== STEP COVERAGE ===")986 for label, blk in (("overall", s["overall"]),987 ("reuse path (has ref test)", s["reuse_path"]),988 ("NON-REUSE path (the decision)", s["non_reuse_path"])):989 print(f'{label:32s} n={blk["total"]:5d} '990 f'helper={blk["strict_assemblable_pct"]:5.1f}% '991 f'+fix/assert={blk["design_assemblable_pct"]:5.1f}% '992 f'+field={blk["grouped_assemblable_pct"]:5.1f}% '993 f'+partial={blk["grouped_plus_partial_pct"]:5.1f}% '994 f'gap={blk["pct"][GAP]:5.1f}%')995 print("\n=== BY MODULE (grouped-assemblable) ===")996 for mod, blk in sorted(s["by_module"].items(),997 key=lambda kv: -kv[1]["total"]):998 print(f'{mod:14s} n={blk["total"]:5d} helper={blk["strict_assemblable_pct"]:5.1f}%'999 f' grouped={blk["grouped_assemblable_pct"]:5.1f}%'1000 f' partial={blk["pct"][PARTIAL]:5.1f}% gap={blk["pct"][GAP]:5.1f}%')1001 print("\n=== MATCHER PRECISION (held-out, '# Step N' evidence) ===")1002 pc = res["precision_check"]1003 print(f' n={pc["n_segments_with_truth"]} hit={pc["hit"]} '1004 f'miss={pc["miss"]} no-prediction={pc["no_prediction"]} '1005 f'hit-rate={pc["hit_rate_pct"]}%')1006 cl = res["case_level_recall"]1007 print("\n=== MATCHER RECALL (case level, n=%d grounded cases) ==="1008 % cl["cases_evaluated"])1009 print(f' mean recall={cl["mean_recall_pct"]}% '1010 f'mean precision={cl["mean_precision_pct"]}% '1011 f'cases with >=50%% recall: {cl["cases_recall_ge_50pct"]} '1012 f'({cl["cases_recall_ge_50pct_share"]}%)')1013 for w in cl["worst"][:6]:1014 print(f' worst {w["tc_id"]} recall={w["recall"]} missed={w["missed"][:4]}')1015 for d in pc["detail"][:10]:1016 print(f' [{d["outcome"]:14s}] {d["tc_id"]} s{d["steps"]}: {d["action"][:70]}')1017 print(f' pred={d["predicted"]} actual={d["actual"]}')1018 print("\n=== CASES ===")1019 print(json.dumps(res["cases"], indent=1))1020 print("\n=== TOP GAP CLUSTERS ===")1021 for c in res["gap_clusters"][:12]:1022 print(f'{c["gap_count"]:4d} gap /{c["partial_count"]:4d} partial {c["signature"]}')1023 print(f' e.g. {c["examples"][0][:120]}')1024 print("\n=== SAMPLES ===")1025 for v, items in res["samples"].items():1026 print(f'-- {v} --')1027 for it in items:1028 print(" ", it)1029 1030 if args.json:1031 out = {k: v for k, v in res.items() if not k.startswith("_")}1032 with open(args.json, "w", encoding="utf-8") as fh:1033 json.dump(out, fh, indent=1)1034 print(f"\nwrote {args.json}")1035 return 01036 1037 1038if __name__ == "__main__":1039 raise SystemExit(main())1040