internationalscholarsprogram/handbook-engine
0
1"""Normalization layer — converts raw MySQL handbook content into typed render blocks.2 3Each section_json from the database is parsed into a list of RenderBlock4objects. Every block has a `block_type` that maps 1-to-1 to a Jinja5partial and a CSS class. This prevents ad-hoc interpretation of raw6JSON throughout the rendering pipeline.7 8Block types (from theme.BLOCK_TYPES):9 heading_1, heading_2, paragraph, bullet_list, note, table,10 enrollment_steps, school_profile, university_summary, toc,11 cover, full_page_image12"""13 14from __future__ import annotations15 16import re17from urllib.parse import quote_plus18from dataclasses import dataclass, field19from typing import Any20 21from app.services.renderers import _extract_university_funding22from app.services.utils import (23 ensure_program_options_pair,24 emphasize_keywords,25 format_money_figures,26 get_any,27 h,28 hb_slug,29 is_assoc,30 is_truthy,31 linkify_urls,32)33from app.services.renderers import fetch_image_data_uri34 35 36# ───────────────────────────────────────────────────────────────37# Block data-classes38# ───────────────────────────────────────────────────────────────39 40@dataclass41class RenderBlock:42 """Base typed render block."""43 block_type: str44 css_class: str = ""45 data: dict[str, Any] = field(default_factory=dict)46 47 48# ───────────────────────────────────────────────────────────────49# Section → blocks50# ───────────────────────────────────────────────────────────────51 52def normalize_section(53 section_key: str,54 section_title: str,55 section_json: dict | list,56 *,57 universities: list[dict] | None = None,58 debug: bool = False,59) -> list[RenderBlock]:60 """Convert a single global section payload into a list of RenderBlocks.61 62 This is the single translation point between the database schema63 and the rendering layer.64 """65 blocks: list[RenderBlock] = []66 key_norm = section_key.lower().strip()67 68 if not isinstance(section_json, dict):69 section_json = {}70 71 layout_norm = str(section_json.get("layout", "")).lower().strip()72 73 # ── Section heading ──74 # Prefer the JSON-level title (display-ready) over the DB section_title75 json_title = str(section_json.get("title", "")).strip() if isinstance(section_json, dict) else ""76 title = json_title or section_title.strip()77 if title and key_norm != "table_of_contents":78 blocks.append(RenderBlock(79 block_type="heading_1",80 css_class="hb-heading-1",81 data={"text": title},82 ))83 84 # ── Steps → enrollment_steps ──85 steps = section_json.get("steps")86 if isinstance(steps, list):87 blocks.append(RenderBlock(88 block_type="enrollment_steps",89 css_class="hb-enrollment-steps",90 data={"steps": _normalize_steps(steps)},91 ))92 return blocks93 94 # ── Bullets ──95 has_bullets = isinstance(section_json.get("bullets"), list)96 has_items = isinstance(section_json.get("items"), list)97 if has_bullets or (layout_norm == "bullets_with_note" and has_items):98 from markupsafe import Markup99 lst = section_json.get("items") if has_items else section_json.get("bullets")100 items = [_normalize_text_content(str(b).strip()) for b in lst if str(b).strip()]101 html_items = [Markup(emphasize_keywords(it)) for it in items]102 blocks.append(RenderBlock(103 block_type="bullet_list",104 css_class="hb-bullet-list",105 data={"entries": html_items, "html_entries": True},106 ))107 note = _normalize_text_content(108 str(section_json.get("note", section_json.get("footnote", ""))).strip()109 )110 if note:111 blocks.append(RenderBlock(112 block_type="note",113 css_class="hb-note",114 data={"text": note},115 ))116 return blocks117 118 # ── Basic table ──119 cols = section_json.get("columns")120 rows = section_json.get("rows")121 if isinstance(cols, list) and isinstance(rows, list):122 blocks.append(_normalize_basic_table(cols, rows))123 return blocks124 125 # ── table_v2 ──126 if layout_norm == "table_v2":127 blocks.append(_normalize_table_v2(section_json))128 return blocks129 130 # ── doc_v1 ──131 if layout_norm == "doc_v1" and isinstance(section_json.get("blocks"), list):132 blocks.extend(_normalize_doc_v1(section_json["blocks"], skip_title=title))133 # Post-process breakdown section for Relocation Cost layout134 if key_norm == "program_features_breakdown":135 blocks = _postprocess_breakdown(blocks, section_json["blocks"])136 # Post-process Tier 2 section for sub-bullet styling137 if key_norm == "summary_of_universities_cosigner":138 blocks = _postprocess_tier2(blocks)139 return blocks140 141 # ── Fallback ──142 if "text" in section_json:143 text = _normalize_text_content(str(section_json["text"]))144 if text.strip():145 from markupsafe import Markup146 blocks.append(RenderBlock(147 block_type="paragraph",148 css_class="hb-paragraph",149 data={150 "text": text,151 "html": Markup(emphasize_keywords(text)),152 },153 ))154 155 return blocks156 157 158def _normalize_text_content(text: str) -> str:159 """Apply global handbook text normalization in a single place."""160 return ensure_program_options_pair(format_money_figures(text))161 162 163# ───────────────────────────────────────────────────────────────164# University profile normalisation165# ───────────────────────────────────────────────────────────────166 167def normalize_university(168 uni_raw: dict[str, Any],169 allow_remote: bool,170 include_inactive_programs: bool,171 debug: bool,172 stats: dict[str, Any],173) -> RenderBlock:174 """Convert raw university data into a school_profile RenderBlock."""175 uni_name = uni_raw["name"]176 sections = uni_raw.get("sections", [])177 is_first = uni_raw.get("_is_first", False)178 179 stats["universities"] = stats.get("universities", 0) + 1180 181 # Build section map; merge duplicate "programs" sections182 sec_map: dict[str, dict] = {}183 for s in sections:184 if not isinstance(s, dict):185 continue186 k = str(s.get("section_key", ""))187 if not k:188 continue189 if k == "programs" and k in sec_map:190 existing = sec_map["programs"].get("section_json", {})191 incoming = s.get("section_json", {})192 if not isinstance(existing, dict):193 existing = {}194 if not isinstance(incoming, dict):195 incoming = {}196 a = existing.get("programs", [])197 b = incoming.get("programs", [])198 if not isinstance(a, list):199 a = []200 if not isinstance(b, list):201 b = []202 existing["programs"] = a + b203 sec_map["programs"]["section_json"] = existing204 continue205 sec_map[k] = s206 207 # Campus image208 img_section = sec_map.get("campus_image") or sec_map.get("image")209 campus_image = ""210 campus_caption = ""211 if img_section:212 j = img_section.get("section_json", {})213 if isinstance(j, dict):214 campus_url = str(j.get("image_url", "")).strip()215 campus_caption = str(j.get("caption", "")).strip()216 if allow_remote and campus_url:217 embedded = fetch_image_data_uri(campus_url)218 if embedded:219 campus_image = embedded220 stats["images_embedded"] = stats.get("images_embedded", 0) + 1221 else:222 stats["images_placeholder"] = stats.get("images_placeholder", 0) + 1223 else:224 stats["images_placeholder"] = stats.get("images_placeholder", 0) + 1225 226 # Overview and website227 resolved_website = (uni_raw.get("website") or "").strip()228 overview_data = None229 230 if "overview" in sec_map:231 overview_json = sec_map["overview"].get("section_json", {})232 if not isinstance(overview_json, dict):233 overview_json = {}234 235 site_from_overview = get_any(236 overview_json,237 ["university_website", "university_website_url", "website",238 "site", "url", "homepage", "web_url"],239 )240 if not resolved_website and site_from_overview:241 resolved_website = site_from_overview242 243 overview_data = {244 "founded": get_any(overview_json, ["founded", "Founded"]),245 "total_students": get_any(overview_json, ["total_students", "Total Students"]),246 "undergraduates": get_any(overview_json, [247 "undergraduates", "Undergraduate Students", "undergraduate_students",248 ]),249 "postgraduates": get_any(overview_json, [250 "postgraduate_students", "Postgraduate Students",251 ]),252 "acceptance_rate": get_any(overview_json, ["acceptance_rate", "Acceptance Rate"]),253 "location": get_any(overview_json, ["location", "Location"]),254 "tuition": format_money_figures(str(get_any(overview_json, [255 "tuition_out_of_state_yearly",256 "Yearly Out of State Tuition Fees",257 "Yearly Out-of-State Tuition Fees",258 "Yearly Tuition Fees",259 "Yearly Out-of-State Tuition Fees:",260 ]) or "")) or None,261 }262 263 if resolved_website:264 stats["university_links"] = stats.get("university_links", 0) + 1265 stats["website_rows"] = stats.get("website_rows", 0) + 1266 267 # Benefits + Funding268 benefits: list[str] | None = []269 funding_heading = "Funding Available"270 funding_items: list[str] = []271 if "benefits" in sec_map:272 j = sec_map["benefits"].get("section_json", {})273 if not isinstance(j, dict):274 j = {}275 raw_benefits = j.get("benefits", [])276 if isinstance(raw_benefits, list):277 benefits = [278 _normalize_text_content(str(b).strip())279 for b in raw_benefits280 if str(b).strip()281 ]282 else:283 benefits = []284 285 funding_heading, funding_items = _extract_university_funding(286 j,287 {288 "school_category": uni_raw.get("school_category"),289 "status": "in" if is_truthy(uni_raw.get("is_active", True)) else "out",290 },291 )292 # Normalize money formatting in funding items293 funding_items = [_normalize_text_content(item) for item in funding_items]294 295 # Programs296 programs = None297 if "programs" in sec_map:298 j = sec_map["programs"].get("section_json", {})299 if not isinstance(j, dict):300 j = {}301 programs_raw = j.get("programs", [])302 if not isinstance(programs_raw, list):303 programs_raw = []304 305 if not include_inactive_programs:306 programs_raw = [307 p for p in programs_raw308 if isinstance(p, dict) and is_truthy(309 p.get("program_active", p.get("is_active", p.get("active", 1)))310 )311 ]312 313 programs = []314 seen_names = set()315 for p in programs_raw:316 if not isinstance(p, dict):317 continue318 program_name = _normalize_text_content(str(p.get("program_name", "")).strip())319 # Deduplicate by lowercase program name320 key = program_name.lower()321 if key in seen_names:322 continue323 seen_names.add(key)324 link = str(p.get("program_link", "")).strip()325 if not link and isinstance(p.get("program_links"), dict):326 link = str(p["program_links"].get("web_link", "")).strip()327 328 programs.append({329 "name": program_name,330 "link": link,331 "designation": _normalize_text_content(str(p.get("designation", ""))),332 "entrance": _normalize_text_content(str(p.get("entrance_exam", p.get("entrance_examination", "")))),333 })334 335 # Extra sections (rendered via global blocks normalizer)336 skip_keys = {"campus_image", "image", "overview", "benefits", "programs"}337 extra_blocks: list[list[RenderBlock]] = []338 for s in sections:339 if not isinstance(s, dict):340 continue341 k = str(s.get("section_key", ""))342 if not k or k in skip_keys:343 continue344 title = str(s.get("section_title", ""))345 j = s.get("section_json", {})346 if not isinstance(j, dict):347 j = {}348 extra_blocks.append(normalize_section(k, title, j, debug=debug))349 350 classes = ["hb-school-profile", "page-break"]351 352 return RenderBlock(353 block_type="school_profile",354 css_class=" ".join(classes),355 data={356 "name": uni_name,357 "anchor": uni_raw.get("anchor"),358 "sort_order": uni_raw.get("sort_order"),359 "website": resolved_website,360 "overview": overview_data,361 "campus_image": campus_image,362 "campus_caption": campus_caption,363 "benefits": benefits,364 "funding_heading": funding_heading,365 "funding_items": funding_items,366 "programs": programs,367 "extra_blocks": extra_blocks,368 },369 )370 371 372# ───────────────────────────────────────────────────────────────373# Internal helpers374# ───────────────────────────────────────────────────────────────375 376def _normalize_steps(steps: list) -> list[dict]:377 """Normalise enrollment steps into structured dicts."""378 result = []379 step_num = 0380 for s in steps:381 if not isinstance(s, dict):382 continue383 step_num += 1384 step_title = str(s.get("title", s.get("step_title", ""))).strip()385 body = _normalize_text_content(str(s.get("body", s.get("description", ""))).strip())386 387 # Pre-format body with bold emphasis on REGULAR, PRIME, $ amounts388 from markupsafe import Markup389 body_html = Markup(emphasize_keywords(body)) if body else ""390 391 links = []392 plain_links = []393 raw_links = s.get("links", [])394 if isinstance(raw_links, list):395 for lnk in raw_links:396 if not isinstance(lnk, dict):397 continue398 label = str(lnk.get("label", "Link")).strip()399 url = str(lnk.get("url", "")).strip()400 if url:401 low_label = label.lower()402 low_url = url.lower()403 is_telegram = "telegram" in low_label or "t.me" in low_url404 if step_num == 2 and "internationalscholarsprogram.com" in low_url and not re.match(r"^https?://", url, flags=re.IGNORECASE):405 url = "https://" + url406 # All links (including Telegram) are rendered as clickable anchors.407 # For Telegram use the full URL as visible label so readers can see/type it.408 link_label = url if is_telegram else label409 links.append({"label": link_label, "url": url})410 411 if step_num == 2 and not any(412 "internationalscholarsprogram.com" in str(l.get("url", "")).lower()413 for l in links414 ):415 links.append({416 "label": "www.internationalscholarsprogram.com",417 "url": "https://www.internationalscholarsprogram.com",418 })419 420 qr = str(s.get("qr_url", s.get("qr_image", ""))).strip()421 telegram_url = ""422 if step_num == 1:423 telegram_ref = ""424 if plain_links:425 telegram_ref = plain_links[0]426 elif isinstance(body, str):427 m = re.search(r"(https?://(?:t\.me|telegram\.me)/[^\s<)]+)", body, flags=re.IGNORECASE)428 if m:429 telegram_ref = m.group(1)430 if telegram_ref:431 telegram_url = telegram_ref432 if not qr:433 qr = (434 "https://api.qrserver.com/v1/create-qr-code/?size=160x160&data="435 + quote_plus(telegram_ref)436 )437 # Strip the raw telegram URL and the follow-up description from body438 body = re.sub(r"https?://(?:t\.me|telegram\.me)/[^\s<)]+", "", body, flags=re.IGNORECASE)439 body = re.sub(r"This telegram group will help you interact with program administrators and other prospective students where you can ask any questions you may have about the program\.?", "", body, flags=re.IGNORECASE)440 body = re.sub(r"\n{2,}", "\n", body).strip()441 body_html = Markup(emphasize_keywords(body)) if body else ""442 443 result.append({444 "number": step_num,445 "title": step_title,446 "body": body,447 "body_html": body_html,448 "links": links,449 "plain_links": plain_links,450 "qr_url": qr,451 "telegram_url": telegram_url,452 })453 return result454 455 456def _normalize_basic_table(cols: list, rows: list) -> RenderBlock:457 """Normalise a basic table (columns + rows)."""458 norm_rows = []459 for r in rows:460 if not isinstance(r, (list, dict)):461 continue462 if isinstance(r, dict):463 row = []464 for col_label in cols:465 key_guess = re.sub(r"[^a-z0-9]+", "_", str(col_label).lower())466 cell = r.get(key_guess, "")467 # Normalize text, emphasize keywords, then linkify URLs for clickable links468 cell_html = emphasize_keywords(_normalize_text_content(str(cell)))469 cell_with_links = linkify_urls(cell_html)470 row.append(cell_with_links)471 norm_rows.append(row)472 else:473 norm_rows.append([linkify_urls(emphasize_keywords(_normalize_text_content(str(cell)))) for cell in r])474 475 return RenderBlock(476 block_type="table",477 css_class="hb-table",478 data={479 "columns": [str(c) for c in cols],480 "rows": norm_rows,481 "variant": "standard",482 },483 )484 485 486def _normalize_table_v2(json_data: dict) -> RenderBlock:487 """Normalise table_v2 (comparison table with header groups)."""488 base_cols = json_data.get("base_columns", [])489 groups = json_data.get("header_groups", [])490 rows = json_data.get("rows", [])491 if not isinstance(base_cols, list):492 base_cols = []493 if not isinstance(groups, list):494 groups = []495 if not isinstance(rows, list):496 rows = []497 498 all_cols: list[dict] = []499 for c in base_cols:500 if isinstance(c, dict):501 all_cols.append({"key": str(c.get("key", "")), "label": str(c.get("label", ""))})502 for g in groups:503 if not isinstance(g, dict):504 continue505 g_cols = g.get("columns", [])506 if not isinstance(g_cols, list):507 g_cols = []508 for c in g_cols:509 if isinstance(c, dict):510 all_cols.append({"key": str(c.get("key", "")), "label": str(c.get("label", ""))})511 512 norm_rows = []513 for r in rows:514 if not isinstance(r, dict):515 continue516 row = {}517 for c in all_cols:518 k = c.get("key", "")519 val = r.get(k, "")520 if isinstance(val, dict):521 val = val.get("text", "")522 row[k] = emphasize_keywords(_normalize_text_content(str(val)))523 norm_rows.append(row)524 525 return RenderBlock(526 block_type="table",527 css_class="hb-table hb-table-comparison",528 data={529 "base_columns": [{"key": c.get("key", ""), "label": c.get("label", "")} for c in base_cols if isinstance(c, dict)],530 "header_groups": [531 {532 "label": str(g.get("label", "")),533 "columns": [{"key": str(c.get("key", "")), "label": str(c.get("label", ""))}534 for c in (g.get("columns", []) if isinstance(g.get("columns"), list) else [])535 if isinstance(c, dict)],536 }537 for g in groups if isinstance(g, dict)538 ],539 "all_columns": all_cols,540 "rows": norm_rows,541 "variant": "comparison",542 },543 )544 545 546# ───────────────────────────────────────────────────────────────547# Breakdown section post-processor548# ───────────────────────────────────────────────────────────────549 550def _postprocess_breakdown(551 blocks: list[RenderBlock],552 raw_blocks: list,553) -> list[RenderBlock]:554 """Rewrite the breakdown section to match the reference layout.555 556 - "Relocation Cost" becomes a banner heading with page-break-before557 - The relocation table gets a merged right cell (rowspan) with the558 cost-coverage note moved inside it559 - "ISP FINANCING" becomes an inline note with mixed bold/italic560 - "NB: CREDIT FACILITY" is styled green561 - Dollar amounts in parentheticals keep their original $ format562 """563 from markupsafe import Markup564 565 # Find raw blocks for the relocation cost table (pre-normalised, $ intact)566 raw_reloc_table = None567 raw_note_after_table = None568 found_reloc = False569 for i, rb in enumerate(raw_blocks):570 if not isinstance(rb, dict):571 continue572 if rb.get("type") == "subheading" and "relocation" in str(rb.get("text", "")).lower():573 found_reloc = True574 continue575 if found_reloc and rb.get("type") == "table_v1" and raw_reloc_table is None:576 raw_reloc_table = rb577 continue578 if found_reloc and raw_reloc_table and rb.get("type") == "paragraph" and raw_note_after_table is None:579 raw_note_after_table = rb580 break581 582 result: list[RenderBlock] = []583 i = 0584 while i < len(blocks):585 blk = blocks[i]586 587 # ── Detect "Relocation Cost" heading ──588 if (blk.block_type == "heading_2"589 and "relocation" in blk.data.get("text", "").lower()):590 591 # Banner heading with page break592 result.append(RenderBlock(593 block_type="heading_2",594 css_class="hb-heading-2 hb-banner-heading page-break",595 data={"text": blk.data["text"]},596 ))597 i += 1598 599 # Replace the next table with spanning variant that has merged cell600 if i < len(blocks) and blocks[i].block_type == "table" and raw_reloc_table:601 raw_rows = raw_reloc_table.get("rows", [])602 # Build the note text for the merged right cell603 note_text = ""604 if raw_note_after_table:605 note_text = str(raw_note_after_table.get("text", ""))606 607 spanning_rows = _build_relocation_spanning_rows(raw_rows, note_text)608 result.append(RenderBlock(609 block_type="table",610 css_class="hb-table hb-relocation-table",611 data={"rows": spanning_rows, "variant": "spanning"},612 ))613 i += 1 # skip the original table614 615 # Skip the paragraph that was moved into the merged cell616 if (i < len(blocks)617 and blocks[i].block_type == "paragraph"618 and note_text):619 i += 1620 continue621 622 # ── "ISP FINANCING" heading → inline note with mixed formatting ──623 if (blk.block_type == "heading_2"624 and "isp financing" in blk.data.get("text", "").lower()):625 # Next block should be the interest rate paragraph626 rate_text = ""627 if i + 1 < len(blocks) and blocks[i + 1].block_type == "paragraph":628 rate_text = blocks[i + 1].data.get("text", "")629 result.append(RenderBlock(630 block_type="note",631 css_class="hb-note hb-isp-financing",632 data={633 "parts": [634 {"text": "ISP FINANCING", "style": "bold"},635 {"text": " (" + _extract_rate_italic(rate_text) + "): " if rate_text else "", "style": "italic"},636 {"text": _extract_rate_amount(rate_text), "style": "bold"},637 ],638 "inline": True,639 },640 ))641 i += 1 # skip the heading642 if rate_text:643 i += 1 # skip the paragraph644 continue645 646 # ── "NB: CREDIT FACILITY" note → green styling ──647 if (blk.block_type == "note"648 and "credit facility" in blk.data.get("text", "").lower()):649 result.append(RenderBlock(650 block_type="note",651 css_class="hb-note hb-credit-note",652 data=blk.data,653 ))654 i += 1655 continue656 657 result.append(blk)658 i += 1659 660 return result661 662 663def _build_relocation_spanning_rows(664 raw_rows: list, note_text: str,665) -> list[list[dict]]:666 """Build spanning rows for the relocation cost table.667 668 Row 0: normal 2-column (consultation fees | Covered in the contribution)669 Rows 1-7: left cell per row, right cell merged (rowspan) with italic note670 Rows 8+: left cell only, empty right671 """672 from markupsafe import Markup673 674 if not raw_rows:675 return []676 677 rows: list[list[dict]] = []678 679 # Row 0 — has "Covered in the contribution"680 first = raw_rows[0] if raw_rows else ["", ""]681 rows.append([682 {"text": Markup(emphasize_keywords(str(first[0] if len(first) > 0 else ""))), "colspan": 1, "rowspan": 1},683 {"text": Markup("<em>" + h(str(first[1] if len(first) > 1 else "")) + "</em>"), "colspan": 1, "rowspan": 1},684 ])685 686 # Rows 1-7: items with dollar amounts that get the merged right cell687 # These are the visa/fee/rent/ticket rows (have parenthetical dollar amounts)688 merged_start = 1689 merged_end = min(8, len(raw_rows)) # Visa Integrity through Air ticket690 691 for idx in range(merged_start, len(raw_rows)):692 cell_text = str(raw_rows[idx][0] if len(raw_rows[idx]) > 0 else "")693 left = {"text": Markup(emphasize_keywords(cell_text)), "colspan": 1, "rowspan": 1}694 695 if idx == merged_start and note_text:696 # First merged row gets the rowspan cell697 span_count = merged_end - merged_start698 note_html = note_text.replace("\n\n", "<br/><br/>")699 right = {700 "text": Markup('<em class="hb-merged-note">' + h(note_html).replace("<br/><br/>", "<br/><br/>") + "</em>"),701 "colspan": 1,702 "rowspan": span_count,703 }704 rows.append([left, right])705 elif idx < merged_end:706 # Subsequent merged rows — no right cell (covered by rowspan)707 rows.append([left])708 else:709 # Remaining rows — empty right cell710 rows.append([711 left,712 {"text": "", "colspan": 1, "rowspan": 1},713 ])714 715 return rows716 717 718def _extract_rate_italic(text: str) -> str:719 """Extract the italic portion: 'Interest rate of 12% – 15% Market Rate PA'."""720 # Text is like: "Interest rate of 12% – 15% Market Rate: UP TO USD 10,000"721 m = re.match(r"(Interest rate.*?(?:Market Rate|PA))", text, re.IGNORECASE)722 if m:723 return m.group(1).rstrip(": ")724 # Fallback: everything before the colon725 if ":" in text:726 return text.split(":")[0].strip()727 return text728 729 730def _extract_rate_amount(text: str) -> str:731 """Extract the amount portion: 'UP TO USD 10,000'."""732 m = re.search(r"(UP TO.*)", text, re.IGNORECASE)733 if m:734 return m.group(1).strip()735 if ":" in text:736 return text.split(":", 1)[1].strip()737 return ""738 739 740# ───────────────────────────────────────────────────────────────741# Tier 2 (cosigner) section post-processor742# ───────────────────────────────────────────────────────────────743 744def _postprocess_tier2(blocks: list[RenderBlock]) -> list[RenderBlock]:745 """Style the Tier 2 section to match the reference layout.746 747 - Second consecutive bullet_list (sub-bullets under Sources of Funds)748 gets checkmark styling instead of arrows.749 """750 result: list[RenderBlock] = []751 prev_was_bullet = False752 for blk in blocks:753 if blk.block_type == "bullet_list":754 if prev_was_bullet:755 # This is the sub-bullet list → use checkmark class756 result.append(RenderBlock(757 block_type="bullet_list",758 css_class="hb-bullet-list hb-sub-bullets",759 data=blk.data,760 ))761 else:762 result.append(blk)763 prev_was_bullet = True764 else:765 prev_was_bullet = False766 result.append(blk)767 return result768 769 770def _normalize_doc_v1(blocks: list, *, skip_title: str = "") -> list[RenderBlock]:771 """Normalise doc_v1 blocks into typed RenderBlocks.772 773 Args:774 skip_title: When set, any leading heading/subheading block whose text775 matches this title (case-insensitive) is dropped to avoid776 duplicating the section heading already emitted by the caller.777 """778 from markupsafe import Markup779 _skip_norm = skip_title.strip().lower() if skip_title else ""780 result: list[RenderBlock] = []781 for b in blocks:782 if not isinstance(b, dict):783 continue784 btype = str(b.get("type", ""))785 786 # Skip heading/subheading blocks that duplicate the section title787 if _skip_norm and btype in ("heading", "subheading"):788 block_text = str(b.get("text", "")).strip().lower()789 if block_text == _skip_norm:790 continue791 792 if btype == "paragraph":793 t = _normalize_text_content(str(b.get("text", "")))794 if t.strip():795 result.append(RenderBlock(796 block_type="paragraph",797 css_class="hb-paragraph",798 data={799 "text": t,800 "html": Markup(emphasize_keywords(t)),801 },802 ))803 804 elif btype == "subheading":805 t = _normalize_text_content(str(b.get("text", "")))806 if t.strip():807 result.append(RenderBlock(808 block_type="heading_2",809 css_class="hb-heading-2",810 data={"text": t},811 ))812 813 elif btype == "bullets":814 items = b.get("items", [])815 if not isinstance(items, list):816 items = []817 normalized = [_normalize_text_content(str(it).strip()) for it in items if str(it).strip()]818 html_items = [Markup(emphasize_keywords(it)) for it in normalized]819 if normalized:820 result.append(RenderBlock(821 block_type="bullet_list",822 css_class="hb-bullet-list",823 data={"entries": html_items, "html_entries": True},824 ))825 826 elif btype == "numbered_list":827 items = b.get("items", [])828 if not isinstance(items, list):829 items = []830 normalized = [_normalize_text_content(str(it).strip()) for it in items if str(it).strip()]831 html_items = [Markup(emphasize_keywords(it)) for it in normalized]832 if normalized:833 result.append(RenderBlock(834 block_type="bullet_list",835 css_class="hb-bullet-list hb-numbered-list",836 data={"entries": html_items, "ordered": True, "html_entries": True},837 ))838 839 elif btype == "note":840 t = _normalize_text_content(str(b.get("text", "")))841 if t.strip():842 result.append(RenderBlock(843 block_type="note",844 css_class="hb-note",845 data={"text": t},846 ))847 848 elif btype == "note_inline":849 parts = b.get("parts", [])850 if not isinstance(parts, list):851 parts = []852 normalized_parts = []853 for p in parts:854 if not isinstance(p, dict):855 continue856 t = _normalize_text_content(str(p.get("text", "")))857 if t:858 normalized_parts.append({859 "text": t,860 "style": str(p.get("style", "")),861 })862 if normalized_parts:863 result.append(RenderBlock(864 block_type="note",865 css_class="hb-note",866 data={"parts": normalized_parts, "inline": True},867 ))868 869 elif btype == "table_v1":870 t_cols = b.get("columns", [])871 t_rows = b.get("rows", [])872 if not isinstance(t_cols, list):873 t_cols = []874 if not isinstance(t_rows, list):875 t_rows = []876 norm_rows = []877 for r in t_rows:878 if not isinstance(r, list):879 continue880 norm_rows.append([emphasize_keywords(_normalize_text_content(str(cell))) for cell in r])881 result.append(RenderBlock(882 block_type="table",883 css_class="hb-table",884 data={"columns": [str(c) for c in t_cols], "rows": norm_rows, "variant": "standard"},885 ))886 887 elif btype == "table":888 # Generic table (columns may be objects or strings, rows may be dicts or lists)889 t_cols = b.get("columns", [])890 t_rows = b.get("rows", [])891 if not isinstance(t_cols, list):892 t_cols = []893 if not isinstance(t_rows, list):894 t_rows = []895 col_labels = []896 col_keys = []897 for c in t_cols:898 if isinstance(c, dict):899 col_labels.append(str(c.get("label", c.get("key", ""))))900 col_keys.append(str(c.get("key", "")))901 else:902 col_labels.append(str(c))903 col_keys.append(re.sub(r"[^a-z0-9]+", "_", str(c).lower()))904 norm_rows = []905 for r in t_rows:906 if isinstance(r, dict):907 norm_rows.append([emphasize_keywords(_normalize_text_content(str(r.get(k, "")))) for k in col_keys])908 elif isinstance(r, list):909 norm_rows.append([emphasize_keywords(_normalize_text_content(str(cell))) for cell in r])910 result.append(RenderBlock(911 block_type="table",912 css_class="hb-table",913 data={"columns": col_labels, "rows": norm_rows, "variant": "standard"},914 ))915 916 elif btype in ("table_v3", "table_v4"):917 t_rows = b.get("rows", [])918 if not isinstance(t_rows, list):919 t_rows = []920 norm_rows = []921 for r in t_rows:922 if not isinstance(r, list):923 continue924 norm_row = []925 for cell in r:926 if isinstance(cell, dict):927 norm_row.append({928 "text": emphasize_keywords(_normalize_text_content(str(cell.get("text", "")))),929 "colspan": int(cell.get("colspan", 1)) if str(cell.get("colspan", "")).isdigit() else 1,930 "rowspan": int(cell.get("rowspan", 1)) if str(cell.get("rowspan", "")).isdigit() else 1,931 })932 else:933 norm_row.append({934 "text": emphasize_keywords(_normalize_text_content(str(cell))),935 "colspan": 1,936 "rowspan": 1,937 })938 norm_rows.append(norm_row)939 result.append(RenderBlock(940 block_type="table",941 css_class="hb-table",942 data={"rows": norm_rows, "variant": "spanning"},943 ))944 945 return result946