lankasailendra/BMCTESTMAIN
0
1"""Preflight: requirement-driven doc page picking (src/doc_pages.py).2 3 python scripts/preflight_doc_pages.py4 5Offline. Live fetches are stubbed, so it checks what gets picked and how it6reaches the release context, not docs.helixops.ai itself.7"""8 9import os10import re11import sys12import time13 14ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))15sys.path.insert(0, os.path.join(ROOT, "src"))16 17import doc_pages # noqa: E40218 19failures: list = []20 21 22def check(cond, msg):23 if not cond:24 failures.append(msg)25 26 27def titles(req, product):28 _b, info = doc_pages.relevant_doc_pages(req, product, live=False)29 return [i["title"].lower() for i in info], info30 31 32def main() -> int:33 # 0. the committed index alone must be enough - docs/ is gitignored and34 # absent on HF, so hide it for every check below35 from pathlib import Path36 37 files = sorted(doc_pages._INDEX_DIR.glob("*.json"))38 check(files, "reference/doc_pages/ has no index files")39 doc_pages._DOCS_DIR = Path("/nonexistent-docs-dir")40 doc_pages.reset_cache()41 corpus, _df = doc_pages._load_corpus()42 check(len(corpus) >= 300, f"index loaded only {len(corpus)} pages")43 for f in files:44 check(45 f.stat().st_size < 9_000_000,46 f"{f.name} is over 9 MB - HF rejects large non-LFS files",47 )48 49 # 1. the case that started it: PSR finds the Proactive Service Resolution pages50 # (on BHOM, BHOM's own ITSM-integration pages)51 for product in ("BMC Helix Operations Management", "BMC Helix ITSM"):52 block, info = doc_pages.relevant_doc_pages(53 "Want to test PSR", product, live=False54 )55 check(info, f"PSR on {product}: no page picked")56 check(57 "proactive service resolution" in block.lower(),58 f"PSR on {product}: picked pages never mention Proactive Service Resolution",59 )60 61 # 2. same-product pages for ordinary requirements62 cases = [63 (64 "Case approval by the business owner",65 "BMC Helix Business Workflows",66 "approval",67 "bwf263",68 ),69 (70 "Test CMDB reconciliation merges duplicate computer systems",71 "BMC Helix CMDB",72 "reconcil",73 "cmdb",74 ),75 (76 "Verify situation groups related events and shows probable cause",77 "BMC Helix AIOps",78 "situation",79 "aiops263",80 ),81 # Indexed for the first time on 2026-09-24; before that these two82 # products grounded against nothing.83 (84 "Assign a field service work order to a technician and track it",85 "BMC Helix Field Service Management (FSM)",86 "work order",87 "fsm263",88 ),89 (90 "Create a record definition with a field and a process in the application",91 "Helix Innovation Studio",92 "record definition",93 "is263",94 ),95 ]96 for req, product, word, key in cases:97 t, info = titles(req, product)98 check(info, f"{req!r}: no page picked")99 check(any(word in x for x in t), f"{req!r}: top pages {t} miss {word!r}")100 check(101 all(i["product"] == key for i in info),102 f"{req!r}: another product's page leaked in: {info}",103 )104 105 # 3. plain words and common acronyms never pull ANOTHER product's pages106 # (the product's own doc spaces are fine - ITSM has several)107 for req, product in (108 ("Incident assignment to a support group", "BMC Helix ITSM"),109 ("Monitor policies with thresholds on BHOM", "BMC Helix Operations Management"),110 ("Verify SLA breach escalation for incidents", "BMC Helix ITSM"),111 ("Verify CI relationships in Asset Management console", "BMC Helix ITSM"),112 ("Test RSSO login and MFA on BHOM", "BMC Helix Operations Management"),113 ("Verify work order assignment and SLM targets", "BMC Remedy"),114 ):115 _t, info = titles(req, product)116 check(117 all(doc_pages._same_product(i["product"], product) for i in info),118 f"{req!r} on {product}: foreign pages "119 f"{[(i['product'], i['title']) for i in info]}",120 )121 122 # 3a. ITSM modules and BHOM find their own doc spaces123 for req, product, space, word in (124 (125 "Incident reassignment and resolution",126 "BMC Helix ITSM",127 "servicedesk",128 "incident",129 ),130 (131 "Change request approval by the change advisory board",132 "BMC Helix ITSM",133 "change",134 "approv",135 ),136 (137 "Create a knowledge article and publish it",138 "BMC Helix ITSM",139 "km",140 "knowledge",141 ),142 (143 "Problem investigation linked to a known error",144 "BMC Helix ITSM",145 "servicedesk",146 "problem",147 ),148 (149 "Asset lifecycle - receive and deploy a computer system",150 "BMC Helix ITSM",151 "asset",152 "asset",153 ),154 (155 "Create a work order and plan its dates",156 "BMC Helix ITSM",157 "srm",158 "work order",159 ),160 (161 "Close a work order after entering work information",162 "BMC Helix ITSM",163 "srm",164 "work order",165 ),166 (167 "Event deduplication and suppression",168 "BMC Helix Operations Management",169 "bhom",170 "dedup",171 ),172 (173 "Blackout policy for a device",174 "BMC Helix Operations Management",175 "bhom",176 "blackout",177 ),178 ):179 t, info = titles(req, product)180 check(info, f"{req!r}: no page picked")181 check(182 any(i["product"].startswith(space) for i in info),183 f"{req!r}: no {space} page in {[(i['product'], i['title']) for i in info]}",184 )185 check(any(word in x for x in t), f"{req!r}: top pages {t} miss {word!r}")186 187 # 3e. Fable round 3: CMDB space for ITSM jobs, no sign-in pages, one copy188 # of a Discovery page, no Remedy swap into Helix ITSM spaces189 _t, info = titles(190 "Verify CMDB reconciliation identifies and merges duplicate CIs",191 "BMC Helix ITSM",192 )193 check(194 any(i["product"] == "cmdb" for i in info), f"ITSM job missed CMDB space: {info}"195 )196 corpus, _df = doc_pages._load_corpus()197 check(198 not any(doc_pages._is_sign_in(p[0], p[1] + "\n" + p[2]) for p in corpus),199 "sign-in pages in the corpus",200 )201 _t, info = titles(202 "Verify Discovery scans a subnet and creates host CIs synced to CMDB",203 "BMC Helix Discovery",204 )205 keys = [doc_pages._page_key(i["url"]) for i in info]206 check(len(keys) == len(set(keys)), f"same Discovery page picked twice: {info}")207 orig = doc_pages._live_text208 try:209 asked = []210 doc_pages._live_text = lambda url: (asked.append(url), "")[1]211 doc_pages.relevant_doc_pages(212 "Incident reassignment and resolution", "BMC Remedy", bmc_version="21.05"213 )214 check(asked and not any("2105" in u for u in asked), f"Remedy swapped: {asked}")215 doc_pages._live_text = lambda url: "BMCHelix - Sign In\n\n" + "x " * 400216 _b, info = doc_pages.relevant_doc_pages(217 "Event deduplication and suppression",218 "BMC Helix Operations Management",219 bmc_version="26.4",220 )221 check(222 info and all(i["source"] == "bundled" for i in info),223 f"sign-in page accepted as live text: {info}",224 )225 finally:226 doc_pages._live_text = orig227 228 # 3b. upgrade / migration pages do not outrank how the feature works229 t, info = titles(230 "Monitor policies with thresholds on BHOM", "BMC Helix Operations Management"231 )232 check(233 info and not any(w in t[0] for w in ("migrat", "upgrad")),234 f"lifecycle page ranked first: {t}",235 )236 t, _info = titles(237 "Migrating monitor policies after upgrade", "BMC Helix Operations Management"238 )239 check(any("migrat" in x for x in t), f"asked about migration, got {t}")240 241 # 3c. unknown words in a pasted requirement do not crowd out real ones (Fable #2)242 jargon = " ".join(f"zqx{i}field" for i in range(40))243 _t, info = titles(244 "Verify situation groups related events and shows probable cause " + jargon,245 "BMC Helix AIOps",246 )247 check(info, "tenant jargon crowded out the real words - no page picked")248 249 # 3d. weak matches are not promoted as primary (Fable #3)250 _t, info = titles("Verify change request approval flow", "BMC Helix ITSM")251 check(252 all(i["score"] >= doc_pages._MIN_SCORE for i in info),253 f"weak page picked: {info}",254 )255 256 # 4. nothing to go on -> nothing picked, no error257 for req in ("", " ", "Want to check how many admins are configured"):258 b, info = doc_pages.relevant_doc_pages(259 req, "BMC Helix Operations Management", live=False260 )261 check(not b and not info, f"{req!r}: expected no pages, got {info}")262 263 # 5. a pasted BRD stays fast264 brd = " ".join(265 ["The service desk agent resolves the situation and the incident"] * 400266 )267 t0 = time.time()268 doc_pages.relevant_doc_pages(brd + " PSR", "BMC Helix AIOps", live=False)269 check(time.time() - t0 < 5, f"long requirement took {time.time() - t0:.1f}s")270 271 # 6. live text wins when it fetches; bundled copy when it does not272 orig = doc_pages._live_text273 try:274 doc_pages._live_text = (275 lambda url: "LIVE PAGE Proactive Service Resolution\n\n" + "x " * 400276 )277 _b, info = doc_pages.relevant_doc_pages("Want to test PSR", "BMC Helix AIOps")278 check(279 info and all(i["source"] == "live" for i in info),280 f"live text not used: {info}",281 )282 doc_pages._live_text = lambda url: ""283 _b, info = doc_pages.relevant_doc_pages("Want to test PSR", "BMC Helix AIOps")284 check(285 info and all(i["source"] == "bundled" for i in info),286 f"bundled fallback not used: {info}",287 )288 finally:289 doc_pages._live_text = orig290 291 # 6b. own-product pages are fetched in the JOB's release (26.4 support)292 orig = doc_pages._live_text293 req, product = (294 "Test CMDB reconciliation merges duplicate computer systems",295 "BMC Helix CMDB",296 )297 body = "Reconciliation merges duplicate CIs\n\n" + "text " * 200298 try:299 asked = []300 doc_pages._live_text = lambda url: (301 asked.append(url),302 body if "/ac263/" in url else "",303 )[1]304 block, info = doc_pages.relevant_doc_pages(req, product, bmc_version="26.3")305 check(306 info and all("/ac263/" in i["url"] for i in info),307 f"26.3 page not used: {info}",308 )309 check(310 all(i["version"] == "26.3" for i in info), f"version not reported: {info}"311 )312 check("may differ" not in block, "current-release page wrongly labelled older")313 314 doc_pages._live_text = lambda url: body if "/ac263/" not in url else ""315 block, info = doc_pages.relevant_doc_pages(req, product, bmc_version="26.3")316 check(317 info318 and all(i["source"] == "live" and "/ac263/" not in i["url"] for i in info),319 f"fallback to indexed release failed: {info}",320 )321 check("may differ in 26.3" in block, "older-release page is not labelled")322 323 doc_pages._live_text = lambda url: ""324 block, info = doc_pages.relevant_doc_pages(req, product, bmc_version="26.3")325 check(326 info and all(i["source"] == "bundled" for i in info),327 f"bundled fallback failed: {info}",328 )329 check(330 "may differ in 26.3" in block, "bundled older-release page is not labelled"331 )332 333 # another product's page keeps its own release334 asked.clear()335 doc_pages._live_text = lambda url: (asked.append(url), "")[1]336 doc_pages.relevant_doc_pages(337 "Want to test PSR", "BMC Helix Operations Management", bmc_version="26.4"338 )339 check(340 asked and all("aiops264" not in u for u in asked),341 f"AIOps page was version-swapped: {asked}",342 )343 finally:344 doc_pages._live_text = orig345 346 # 6c. a document of many requirements: pages picked PER requirement347 from requirement_units import requirement_units348 349 doc = (350 "1. Create a work order and plan its dates.\n"351 "2. Change request approval by the change advisory board.\n"352 "3. Create a knowledge article and publish it.\n"353 "4. Event deduplication and suppression for incoming events.\n"354 )355 units = requirement_units(doc)356 check(len(units) == 4, f"splitter gave {len(units)} units")357 block, info = doc_pages.pages_for_units(358 units, "BMC Helix ITSM", bmc_version="26.3", live=False359 )360 fors = {i.get("for", "")[:2] for i in info}361 check({"1:", "2:", "3:"} <= fors, f"a requirement got no page: {info}")362 check(363 any(i["product"].startswith("srm") for i in info)364 and any(i["product"].startswith("change") for i in info)365 and any(i["product"].startswith("km") for i in info),366 f"per-requirement spaces missing: {[(i['product'], i.get('for')) for i in info]}",367 )368 check(len(info) <= doc_pages._DOC_PAGES_MAX, f"page budget exceeded: {len(info)}")369 keys = [doc_pages._page_key(i["url"]) for i in info]370 check(371 len(keys) == len(set(keys)),372 "a page shared by two requirements was included twice",373 )374 check(" - for: 1: " in block, "pages are not labelled with their requirement")375 check(376 block.startswith(doc_pages.PAGES_HEADER)377 and block.endswith(doc_pages.PAGES_END),378 "pages block is not delimited",379 )380 381 # 6d. a long document stays fast (budget filled -> later requirements skipped)382 many = "\n".join(f"{n + 1}. {units[n % 4][3:]} case {n}" for n in range(60))383 t0 = time.time()384 doc_pages.pages_for_units(requirement_units(many), "BMC Helix ITSM", live=False)385 check(time.time() - t0 < 6, f"60 requirements took {time.time() - t0:.1f}s")386 387 # 6k. EVERY requirement gets a page (was: only the first ~9 topics)388 topics = [389 "Create a work order and plan its dates",390 "Reassign an incident to another support group",391 "Approve a change request in the CAB workspace",392 "Publish a knowledge article after review",393 "Receive and deploy a computer system asset",394 "Create a problem investigation from an incident",395 "Link a known error to a problem",396 "Create a release with a manifest",397 "Add tasks to a work order",398 "Close a work order after work information",399 "Resolve an incident with a resolution note",400 "Schedule a change in the change calendar",401 "Flag a knowledge article",402 "Manage software licenses for assets",403 "Create a purchase requisition",404 "Incident priority from impact and urgency",405 ]406 brd = "\n".join(407 f"{n + 1}. {t} with SLA target and notification to the manager."408 for n, t in enumerate(topics)409 )410 block, info = doc_pages.pages_for_units(411 requirement_units(brd), "BMC Helix ITSM", bmc_version="26.3", live=False412 )413 check(414 info and info[0].get("requirements_with_page") == len(topics),415 f"not every requirement got a page: {info[:1]}",416 )417 labelled = set()418 for i in info:419 labelled.update(re.findall(r"(?:^|; )([A-Z]*-?\d+): ", i.get("for", "")))420 check(421 len(labelled) >= len(topics) - 1,422 f"only {len(labelled)} of {len(topics)} requirements labelled on a page",423 )424 check(425 len(block) < doc_pages._DOC_CHARS + 300 * len(info),426 f"pages block over budget: {len(block)} chars",427 )428 429 # 6m. a dense chunk: 90 distinct topics, all but the unmatchable covered430 import random as _rnd431 432 corpus, _df = doc_pages._load_corpus()433 _rnd.seed(3)434 itsm_titles = [435 p[1]436 for p in corpus437 if p[3] in ("servicedesk263", "change263", "srm263", "km263", "asset263")438 and len(p[1].split()) >= 3439 ]440 dense = "\n".join(441 f"{n + 1}. Verify {t.lower()}."442 for n, t in enumerate(_rnd.sample(itsm_titles, 90))443 )444 _b, info = doc_pages.pages_for_units(445 requirement_units(dense), "BMC Helix ITSM", bmc_version="26.3", live=False446 )447 h = info[0] if info else {}448 check(449 h.get("requirements_with_page", 0) + h.get("requirements_matching_no_page", 0)450 >= 88,451 f"dense chunk left requirements without a page: {h}",452 )453 454 # 6n. formal BRD lines sharing a long prefix keep distinct labels and455 # are counted one by one (Fable round 7)456 formal = "\n".join(457 f"REQ-{n + 1:03d} The system shall provide the ability for the authorised "458 f"service desk agent to {t.lower()} from the console."459 for n, t in enumerate(topics[:8])460 )461 fu = requirement_units(formal)462 _b, info = doc_pages.pages_for_units(fu, "BMC Helix ITSM", live=False)463 labels = {x for i in info for x in re.findall(r"REQ-\d{3}", i.get("for", ""))}464 check(len(fu) == 8, f"formal BRD split into {len(fu)} units")465 check(len(labels) >= 7, f"labels collapsed: {labels}")466 check(467 info and info[0].get("requirements_with_page", 0) >= 7,468 f"formal BRD coverage miscounted: {info[:1]}",469 )470 check(471 not any("shall provide the ability" in i.get("for", "") for i in info),472 "shared prefix not dropped from labels",473 )474 475 # 6o. one differently phrased line does not cancel the shared prefix476 mixed = formal + "\nREQ-009 Agents must be able to reopen a closed incident."477 _b, info = doc_pages.pages_for_units(478 requirement_units(mixed), "BMC Helix ITSM", live=False479 )480 check(481 not any("shall provide the ability" in i.get("for", "") for i in info),482 f"majority prefix not dropped: {[i.get('for') for i in info][:3]}",483 )484 485 # 6l. boilerplate on every line does not decide the pages486 six = "\n".join(487 f"{n + 1}. {t} with SLA target and notification to the manager."488 for n, t in enumerate(topics[:6])489 )490 _b, info = doc_pages.pages_for_units(491 requirement_units(six), "BMC Helix ITSM", bmc_version="26.3", live=False492 )493 check(494 not any(495 ("sla" in i["title"].lower() or "notification" in i["title"].lower())496 and i.get("for_count", 1) > 1497 for i in info498 ),499 f"boilerplate page shared across requirements: "500 f"{[(i['title'], i.get('for_count')) for i in info]}",501 )502 check(503 any("work order" in i["title"].lower() for i in info)504 and any("incident" in i["title"].lower() for i in info)505 and any("cab" in i["title"].lower() for i in info),506 f"real topics lost to boilerplate: {[i['title'] for i in info]}",507 )508 509 # 6e. strip_pages_block keeps the general docs, drops only the pages510 ctx = block + "\n\nGENERAL DOCS TEXT"511 check(512 doc_pages.strip_pages_block(ctx) == "GENERAL DOCS TEXT",513 "strip_pages_block damaged the general docs",514 )515 check(516 doc_pages.strip_pages_block("plain") == "plain",517 "strip changed a context without pages",518 )519 520 # 6f. chunked generation: each chunk carries ITS OWN pages521 import test_generator as tg522 523 seen_docs = {}524 saved_gen = tg.generate_test_cases525 orig = doc_pages._live_text526 try:527 doc_pages._live_text = lambda url: ""528 tg.generate_test_cases = lambda chunk, *a, **k: (529 seen_docs.__setitem__(chunk[:12], a[5]),530 [],531 )[1]532 wo = "1. Create a work order and plan its dates.\n" + (533 "Work order detail. " * 900534 )535 chg = "2. Change request approval by the change advisory board.\n" + (536 "Change detail. " * 900537 )538 shared = block + "\n\nGENERAL DOCS TEXT"539 tg.generate_test_cases_chunked(540 wo + "\n\n" + chg, "BMC Helix ITSM", "26.3", docs_context=shared541 )542 check(len(seen_docs) >= 2, f"expected 2+ chunks, got {len(seen_docs)}")543 wo_docs = next((v for k, v in seen_docs.items() if k.startswith("1.")), "")544 chg_docs = next((v for k, v in seen_docs.items() if k.startswith("2.")), "")545 check(546 "work order" in wo_docs.split(doc_pages.PAGES_END)[0].lower(),547 "WO chunk lacks WO pages",548 )549 check(550 "CAB" in chg_docs551 or "change" in chg_docs.split(doc_pages.PAGES_END)[0].lower(),552 "Change chunk lacks change pages",553 )554 check(555 wo_docs.count(doc_pages.PAGES_HEADER) == 1556 and chg_docs.count(doc_pages.PAGES_HEADER) == 1,557 "document-level pages were not replaced in the chunks",558 )559 check(560 "GENERAL DOCS TEXT" in wo_docs and "GENERAL DOCS TEXT" in chg_docs,561 "general docs lost in chunks",562 )563 check(564 {c["chunk"] for c in tg.last_chunk_doc_pages()} >= {1, 2},565 f"per-chunk pages not recorded: {tg.last_chunk_doc_pages()}",566 )567 finally:568 tg.generate_test_cases = saved_gen569 doc_pages._live_text = orig570 571 # 6g. formal requirement ids split into units (Fable round 4 #1)572 got = requirement_units(573 "REQ-001 Create a work order.\nREQ-002: Approve a change.\nFR3 Publish an article."574 )575 check(len(got) == 3, f"REQ-id lines gave {len(got)} unit(s): {got}")576 577 # 6h. per-chunk page log is per thread (two jobs at once)578 import threading as _th579 580 tg._CHUNK_PAGES_TLS.pages = [{"chunk": 99, "pages": []}]581 other = []582 t = _th.Thread(target=lambda: other.append(tg.last_chunk_doc_pages()))583 t.start()584 t.join()585 check(other == [[]], f"another thread saw this thread's page log: {other}")586 587 # 6i. a single-chunk document still gets its own pages588 seen_docs.clear()589 orig = doc_pages._live_text590 saved_gen = tg.generate_test_cases591 try:592 doc_pages._live_text = lambda url: ""593 tg.generate_test_cases = lambda chunk, *a, **k: (594 seen_docs.__setitem__("one", a[5]),595 [],596 )[1]597 tg.generate_test_cases_chunked(598 "1. Create a work order and plan its dates.",599 "BMC Helix ITSM",600 "26.3",601 docs_context="GENERAL DOCS TEXT",602 )603 check(604 doc_pages.PAGES_HEADER in seen_docs.get("one", ""),605 "single-chunk document got no requirement pages",606 )607 finally:608 tg.generate_test_cases = saved_gen609 doc_pages._live_text = orig610 611 # 6j. the live-fetch cache: a page is fetched once per process612 calls = []613 import helpers as _hp614 615 saved_fetch = _hp.fetch_single_page616 try:617 doc_pages.reset_cache()618 _hp.fetch_single_page = lambda url, max_chars=0: (calls.append(url), "")[1]619 doc_pages._live_text("https://docs.helixops.ai/x/")620 doc_pages._live_text("https://docs.helixops.ai/x/")621 check(len(calls) == 1, f"a failed fetch was retried at once: {calls}")622 finally:623 _hp.fetch_single_page = saved_fetch624 doc_pages.reset_cache()625 626 # 7. release context puts the pages FIRST and reports them627 import release_context as rc628 629 saved = (630 rc.fetch_core_docs,631 rc.fetch_release_notes_text,632 rc.summarize_release_delta,633 )634 orig = doc_pages._live_text635 try:636 doc_pages._live_text = lambda url: ""637 rc.fetch_core_docs = lambda url, bmc_product="": "HOME PAGE TEXT " * 30638 rc.fetch_release_notes_text = lambda *a, **k: ("", False)639 rc.summarize_release_delta = lambda *a, **k: ""640 ctx = rc.build_release_context(641 "BMC Helix Operations Management",642 "26.2",643 requirement_text="Want to test PSR",644 )645 check(ctx.get("doc_pages"), "release context: doc_pages missing")646 check(647 ctx["docs_context"].startswith("=== BMC DOCUMENTATION PAGES"),648 "release context: requirement pages are not first",649 )650 check(651 "HOME PAGE TEXT" in ctx["docs_context"],652 "release context: general docs dropped",653 )654 ctx = rc.build_release_context(655 "BMC Helix ITSM",656 "26.3",657 requirement_text=doc,658 requirement_units=units,659 )660 check(661 len({i.get("for", "")[:2] for i in ctx.get("doc_pages") or []}) >= 3,662 f"release context did not pick per requirement: {ctx.get('doc_pages')}",663 )664 ctx = rc.build_release_context("BMC Helix Operations Management", "26.2")665 check(666 ctx.get("doc_pages") == [],667 "release context without a requirement picked pages",668 )669 finally:670 rc.fetch_core_docs, rc.fetch_release_notes_text, rc.summarize_release_delta = (671 saved672 )673 doc_pages._live_text = orig674 675 if failures:676 print(f"FAIL ({len(failures)}):")677 for f in failures:678 print(f" x {f}")679 return 1680 print("ALL PASS")681 return 0682 683 684if __name__ == "__main__":685 sys.exit(main())686