FaizanMirZa77/FormatX
0
1"""2document_extractor.py3─────────────────────4Extracts structured content blocks from a raw DOCX or PDF.5 6DOCX: reads Word XML directly — style names give high-confidence block types.7 Page numbers are tracked via explicit page break markers in the XML.8PDF: uses PyMuPDF (fitz) — page number comes directly from the page iterator.9 10Page number is passed to Gemini so it can apply cover-page rules accurately11regardless of how many blocks appear on page 1.12"""13 14import io15import re16import logging17import statistics18from collections import defaultdict19import fitz # PyMuPDF20from docx import Document21from docx.enum.text import WD_ALIGN_PARAGRAPH22from docx.oxml.ns import qn23from docx.table import Table as DocxTable24from docx.text.paragraph import Paragraph as DocxParagraph25from models import RawBlock, BlockType, TableBlock, TableCell, RunSpan26 27log = logging.getLogger("formatx.extractor")28 29 30_RTL_FONTS = [31 "naskh", "nastaleeq", "arabic", "urdu", "farsi", "persian",32 "amiri", "scheherazade", "lateef", "jameel", "nafees",33 "alvi", "mehr", "fajer", "tahoma",34]35 36_RTL_UNICODE_RANGES = range(0x0600, 0x06FF + 1)37 38# TOC dot-leader: text + (3+ dots/spaces OR a single tab) + page number at end39_TOC_RE = re.compile(r"^.{2,80}(?:[.\s]{3,}|\t)\s*[ivxlcdmIVXLCDM\d]{1,6}\s*$")40# TOF entry prefix41_TOF_RE = re.compile(r"^(figure|fig\.?|table|chart|appendix)\s+[\dA-Z]", re.IGNORECASE)42# Word built-in TOC style names43_TOC_STYLE_RE = re.compile(r"^toc\s*\d*$")44 45 46def _is_rtl_text(text: str) -> bool:47 if not text:48 return False49 alpha = [c for c in text if c.isalpha()]50 if not alpha:51 return False52 return sum(1 for c in alpha if ord(c) in _RTL_UNICODE_RANGES) / len(alpha) > 0.453 54 55def _is_rtl_font(font_name: str) -> bool:56 if not font_name:57 return False58 return any(r in font_name.lower() for r in _RTL_FONTS)59 60 61def _alignment_from_docx(para) -> str | None:62 """63 Returns the EXPLICIT alignment set directly on this paragraph, or None if64 no explicit alignment is set (meaning it should inherit from the style).65 66 We intentionally do NOT fall back to the style's alignment here — the67 assembler needs to know whether the raw document had an explicit alignment68 so it can decide whether to preserve it or let the template style win.69 """70 al = para.alignment # None means "not explicitly set on this paragraph"71 if al is None:72 return None73 return {74 WD_ALIGN_PARAGRAPH.LEFT: "left",75 WD_ALIGN_PARAGRAPH.RIGHT: "right",76 WD_ALIGN_PARAGRAPH.CENTER: "center",77 WD_ALIGN_PARAGRAPH.JUSTIFY: "justify",78 }.get(al, None)79 80 81def _para_rtl(para) -> bool:82 try:83 ppr = para._element.find(qn("w:pPr"))84 if ppr is not None:85 bidi = ppr.find(qn("w:bidi"))86 if bidi is not None:87 return bidi.get(qn("w:val"), "1") != "0"88 except Exception:89 pass90 return False91 92 93def _has_page_break_before(element) -> bool:94 """95 Returns True if this paragraph has w:pageBreakBefore set.96 This means THIS paragraph is the first on a new page.97 Only meaningful on <w:p> elements — skip tables.98 """99 try:100 WNS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"101 # w:pageBreakBefore is a paragraph property — only exists on <w:p>102 tag = element.tag.split("}")[-1] if "}" in element.tag else element.tag103 if tag != "p":104 return False105 ppr = element.find(f"{{{WNS}}}pPr")106 if ppr is not None:107 pbr = ppr.find(f"{{{WNS}}}pageBreakBefore")108 if pbr is not None and pbr.get(f"{{{WNS}}}val", "true") not in ("false", "0"):109 return True110 except Exception:111 pass112 return False113 114 115def _has_page_break_at_end(element) -> bool:116 """117 Returns True if this paragraph ends with an explicit page break118 (w:br type="page") OR contains a section break (w:sectPr) that119 causes a new page (nextPage, evenPage, oddPage types).120 121 This means the NEXT element starts a new page.122 Only checks <w:p> elements — skips tables to avoid false positives123 from page breaks inside table cells.124 125 NOTE: w:lastRenderedPageBreak is intentionally NOT checked here.126 It is a rendering cache hint inserted by Word and can appear mid-paragraph,127 causing false page-number increments. Only structural w:br type="page" is reliable.128 """129 try:130 WNS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"131 tag = element.tag.split("}")[-1] if "}" in element.tag else element.tag132 if tag != "p":133 return False134 135 # Run-level explicit page break: <w:br w:type="page"/>136 for br in element.iter(f"{{{WNS}}}br"):137 if br.get(f"{{{WNS}}}type") == "page":138 return True139 140 # Section break inside paragraph properties: <w:pPr><w:sectPr><w:type w:val="..."/>141 # nextPage, evenPage, oddPage all force a new page.142 # "continuous" does NOT create a page break — skip it.143 ppr = element.find(f"{{{WNS}}}pPr")144 if ppr is not None:145 sect_pr = ppr.find(f"{{{WNS}}}sectPr")146 if sect_pr is not None:147 sect_type = sect_pr.find(f"{{{WNS}}}type")148 if sect_type is not None:149 val = sect_type.get(f"{{{WNS}}}val", "nextPage")150 if val in ("nextPage", "evenPage", "oddPage"):151 return True152 else:153 # No explicit type element means default = nextPage154 return True155 156 except Exception:157 pass158 return False159 160 161def _infer_block_type_docx(para, is_bold: bool, font_size: float | None) -> tuple[BlockType, str]:162 style = para.style.name.lower() if para.style else ""163 164 if "heading 1" in style or style == "title":165 return BlockType.HEADING1, "high"166 if "heading 2" in style or style == "subtitle":167 return BlockType.HEADING2, "high"168 if "heading 3" in style:169 return BlockType.HEADING3, "high"170 if "heading 4" in style or "heading 5" in style or "heading 6" in style:171 return BlockType.HEADING4, "high"172 if "caption" in style:173 return BlockType.CAPTION, "high"174 if style in ("list paragraph", "list bullet", "list number",175 "list bullet 2", "list bullet 3", "list number 2", "list number 3") \176 or _has_numbering(para):177 return BlockType.LIST_ITEM, "high"178 if "header" in style:179 return BlockType.HEADER, "high"180 if "footer" in style:181 return BlockType.FOOTER, "high"182 183 text = para.text.strip()184 if is_bold and font_size and font_size >= 16 and len(text) < 120:185 return BlockType.HEADING1, "low"186 if is_bold and font_size and font_size >= 14 and len(text) < 120:187 return BlockType.HEADING2, "low"188 if is_bold and font_size and font_size >= 12 and len(text) < 80:189 return BlockType.HEADING3, "low"190 # ALL-CAPS heuristic: only apply when font is clearly larger than body text191 # (font_size >= 13) to avoid misclassifying cover page labels like "SUBMITTED BY"192 if (text.isupper() and len(text.split()) <= 8 and len(text) > 3193 and any(c.isalpha() for c in text)194 and font_size and font_size >= 13):195 return BlockType.HEADING2, "low"196 197 # Bold short-text heuristic for sub-headings styled as Normal+bold:198 # Authors often write section labels (e.g. "Phase 1: Discovery") as bold199 # Normal text instead of using a Heading style. These are short (≤ 10 words),200 # bold, and don't start with a list marker. Classify as HEADING3 at low201 # confidence so the AI can promote/demote as needed.202 if (is_bold and len(text.split()) <= 10 and len(text) > 2203 and not _is_text_pattern_list(text)204 and any(c.isalpha() for c in text)):205 return BlockType.HEADING3, "low"206 207 # Text-pattern list detection — catches manually authored lists that weren't208 # created with Word's numbering system (no numPr in XML).209 # Same patterns as PDF extraction — applied at "low" confidence so the AI210 # can override if the context suggests otherwise (e.g. a dash in a sentence).211 if _is_text_pattern_list(text):212 return BlockType.LIST_ITEM, "low"213 214 if style in ("normal", "default paragraph font", ""):215 return BlockType.PARAGRAPH, "low"216 217 return BlockType.PARAGRAPH, "high"218 219 220_TEXT_LIST_RE = re.compile(221 r"^\s*(?:"222 r"[\u2022\u2023\u2024\u2025\u2043\u25AA\u25AB\u25CF\u25CB\u25E6\u00B7]\s" # Unicode bullets223 r"|\u2013\s|\u2014\s" # en-dash / em-dash bullets224 r"|[-\*\+]\s" # ASCII dash/asterisk/plus225 r"|\d{1,2}[\.\)]\s" # "1. " or "1) "226 r"|[a-zA-Z][\.\)]\s" # "a. " or "a) "227 r"|\(\d{1,2}\)\s" # "(1) "228 r"|\([a-zA-Z]\)\s" # "(a) "229 r"|[ivxlIVXL]{1,4}[\.\)]\s" # "i. " "iv) " Roman numerals230 r")"231)232 233 234def _is_text_pattern_list(text: str) -> bool:235 """236 Returns True if the text STARTS with a recognisable list marker.237 Only short-to-medium paragraphs qualify — long sentences starting with238 a dash (e.g. "— However, the results show...") are not list items.239 """240 if not text:241 return False242 # Long paragraphs are almost certainly body text, not list items243 if len(text.split()) > 60:244 return False245 return bool(_TEXT_LIST_RE.match(text))246 247 248# (kept for potential future use — not currently referenced)249_TEXT_LIST_PRIMARY_MARKERS = re.compile(250 r"^\s*(?:"251 r"[\u2022\u2043\u25AA\u25CF\u25CB\u00B7\*\+]\s" # filled bullets, *, +252 r"|\d{1,2}[\.\)]\s" # numbered: "1. " "1) "253 r"|[a-zA-Z][\.\)]\s" # lettered: "a. " "a) "254 r"|\(\d{1,2}\)\s" # "(1) "255 r"|\([a-zA-Z]\)\s" # "(a) "256 r"|[ivxlIVXL]{1,4}[\.\)]\s" # roman: "i. " "iv) "257 r")"258)259 260 261def _infer_text_list_level(text: str, list_indent_stack: list[int]) -> tuple[int, list[int]]:262 """263 Infer the nesting level of a text-pattern list item using leading whitespace,264 and maintain an indent stack so levels stay consistent across the whole list.265 266 Algorithm:267 - Count leading spaces in the raw text (before the marker).268 - Compare against the indent stack (list of indent values seen so far,269 ordered from outermost to innermost).270 - If this indent is less than the current innermost → pop back up until we271 find a matching level (or hit the bottom).272 - If this indent is greater than the current innermost → it is a new,273 deeper level — push it.274 - If it equals the current innermost → same level, no change.275 276 Returns (level, updated_stack).277 278 The indent_stack is per-list-group — callers should reset it when a279 non-list paragraph appears between list items.280 """281 leading = len(text) - len(text.lstrip(" \t"))282 283 if not list_indent_stack:284 # First item in this list group — always level 0285 return 0, [leading]286 287 # Search stack from innermost outward for a matching indent288 for depth in range(len(list_indent_stack) - 1, -1, -1):289 if leading == list_indent_stack[depth]:290 # Same indent as an existing level → reuse that level291 # Pop any levels deeper than this292 return depth, list_indent_stack[:depth + 1]293 if leading > list_indent_stack[depth]:294 # Deeper than this level → new child level295 new_stack = list_indent_stack[:depth + 1] + [leading]296 return depth + 1, new_stack297 298 # Less indented than everything on the stack — new outermost level299 return 0, [leading]300 301 302def _has_numbering(para) -> bool:303 try:304 return para._element.find(305 ".//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}numPr"306 ) is not None307 except Exception:308 return False309 310 311def _extract_inline_runs(para) -> list[RunSpan] | None:312 """313 Extract run-level formatting spans from a DOCX paragraph.314 315 Returns a list of RunSpan objects when the paragraph has meaningful316 inline variation (mixed bold/italic/font across runs).317 Returns None when all runs are uniform — the assembler will use a318 single run in that case, which is faster and produces cleaner XML.319 320 Consecutive runs with identical formatting are merged to keep the321 span list compact. Empty runs are skipped.322 """323 try:324 raw_runs = [r for r in para.runs if r.text]325 if not raw_runs:326 return None327 328 spans: list[RunSpan] = []329 for r in raw_runs:330 bold = r.bold # True / False / None (inherit)331 italic = r.italic332 font_name = r.font.name if r.font.name else None333 font_size = round(r.font.size.pt, 1) if r.font.size else None334 335 # Merge with previous span if formatting is identical336 if spans:337 prev = spans[-1]338 if (prev.bold == bold and prev.italic == italic339 and prev.font_name == font_name340 and prev.font_size == font_size):341 spans[-1] = RunSpan(342 text = prev.text + r.text,343 bold = bold,344 italic = italic,345 font_name = font_name,346 font_size = font_size,347 )348 continue349 350 spans.append(RunSpan(351 text = r.text,352 bold = bold,353 italic = italic,354 font_name = font_name,355 font_size = font_size,356 ))357 358 # If there's only one span with no explicit formatting, no point359 # carrying the list — the assembler handles single-run blocks fine.360 if len(spans) <= 1:361 if not spans:362 return None363 s = spans[0]364 # Only skip if the single span has no explicit formatting at all365 if s.bold is None and s.italic is None and s.font_name is None and s.font_size is None:366 return None367 # Has explicit formatting — keep it so preserve_original mode uses it368 return spans369 370 # Check if all spans have the same formatting — if so, skip371 first = spans[0]372 all_same = all(373 s.bold == first.bold and s.italic == first.italic374 and s.font_name == first.font_name and s.font_size == first.font_size375 for s in spans376 )377 if all_same:378 return None379 380 return spans381 382 except Exception:383 return None384 385 386def _is_toc_entry(text: str, style_name: str) -> bool:387 if _TOC_STYLE_RE.match(style_name.lower()):388 return True389 if _TOC_RE.match(text.strip()):390 return True391 return False392 393 394def _is_tof_entry(text: str, style_name: str) -> bool:395 sl = style_name.lower()396 # High-confidence: explicit TOF style name397 if "table of figures" in sl or "list of figures" in sl:398 return True399 # Only classify as TOF entry if it ALSO has a dot-leader (page number at end)400 # This prevents body sentences like "Figure 2 shows..." from being misclassified.401 if _TOF_RE.match(text.strip()) and _TOC_RE.match(text.strip()):402 return True403 return False404 405 406def _get_figure_dimensions(para) -> tuple[float | None, float | None]:407 try:408 el = para._element409 DRAWING_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"410 WP_NS = "http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing"411 drawing = el.find(f".//{{{DRAWING_NS}}}drawing")412 if drawing is None:413 return None, None414 extent = drawing.find(f".//{{{WP_NS}}}extent")415 if extent is not None:416 cx, cy = extent.get("cx"), extent.get("cy")417 if cx and cy:418 return round(int(cx) / 360000, 1), round(int(cy) / 360000, 1)419 except Exception:420 pass421 return None, None422 423 424def _extract_image_from_para(para, doc: Document) -> dict | None:425 """426 Extract the actual image bytes and metadata from a figure paragraph.427 Returns a dict with:428 - image_bytes: raw image data429 - content_type: e.g. "image/png"430 - width_emu, height_emu: dimensions in EMU (for re-insertion)431 Returns None if extraction fails.432 """433 try:434 DRAWING_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"435 DRAWINGML_NS = "http://schemas.openxmlformats.org/drawingml/2006/main"436 REL_NS = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"437 WP_NS = "http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing"438 439 el = para._element440 drawing = el.find(f".//{{{DRAWING_NS}}}drawing")441 if drawing is None:442 return None443 444 # Get image relationship ID from blip445 blip = drawing.find(f".//{{{DRAWINGML_NS}}}blip")446 if blip is None:447 return None448 449 r_embed = blip.get(f"{{{REL_NS}}}embed")450 if not r_embed:451 return None452 453 # Get dimensions from extent454 width_emu = height_emu = None455 extent = drawing.find(f".//{{{WP_NS}}}extent")456 if extent is not None:457 cx, cy = extent.get("cx"), extent.get("cy")458 if cx: width_emu = int(cx)459 if cy: height_emu = int(cy)460 461 # Resolve relationship to get image part462 part = para.part463 image_part = part.related_parts.get(r_embed)464 if image_part is None:465 return None466 467 return {468 "image_bytes": image_part.blob,469 "content_type": image_part.content_type,470 "width_emu": width_emu,471 "height_emu": height_emu,472 }473 except Exception as e:474 log.warning(f"[EXTRACTOR] Image extraction failed: {e}")475 return None476 477 478def _check_for_figure(para) -> bool:479 try:480 el = para._element481 DRAWING_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"482 DRAWINGML_NS = "http://schemas.openxmlformats.org/drawingml/2006/main"483 PIC_NS = "http://schemas.openxmlformats.org/drawingml/2006/picture"484 CHART_NS = "http://schemas.openxmlformats.org/drawingml/2006/chart"485 486 drawing = el.find(f".//{{{DRAWING_NS}}}drawing")487 if drawing is not None:488 has_blip = drawing.find(f".//{{{DRAWINGML_NS}}}blip") is not None489 has_pic = drawing.find(f".//{{{PIC_NS}}}pic") is not None490 has_chart = drawing.find(f".//{{{CHART_NS}}}chart") is not None491 gd = drawing.find(f".//{{{DRAWINGML_NS}}}graphicData")492 has_gd = gd is not None and gd.get("uri", "") != "" and len(list(gd)) > 0493 if has_blip or has_pic or has_chart or has_gd:494 return True495 return False496 497 pict = el.find(".//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}pict")498 if pict is not None:499 if pict.find(".//{urn:schemas-microsoft-com:vml}imagedata") is not None:500 return True501 502 if el.find(".//{urn:schemas-microsoft-com:office:office}OLEObject") is not None:503 return True504 except Exception:505 pass506 return False507 508 509def _extract_table_block(table) -> TableBlock:510 # Detect header row: bold runs OR cell shading in row 0511 has_header = False512 try:513 for cell in table.rows[0].cells:514 # Check bold runs515 for para in cell.paragraphs:516 for run in para.runs:517 if run.bold:518 has_header = True519 break520 if has_header:521 break522 if has_header:523 break524 # Check cell shading (w:shd fill attribute)525 try:526 tc_pr = cell._tc.find(qn("w:tcPr"))527 if tc_pr is not None:528 shd = tc_pr.find(qn("w:shd"))529 if shd is not None:530 fill = shd.get(qn("w:fill"), "")531 # Any non-white, non-auto fill = shaded header532 if fill and fill.upper() not in ("FFFFFF", "AUTO", ""):533 has_header = True534 except Exception:535 pass536 if has_header:537 break # shading found — no need to check remaining cells538 except Exception:539 pass540 541 # Build rows — mark row-0 cells as headers only if has_header is True542 # Skip rows that have no cells (malformed table rows with no <w:tc> children)543 rows = []544 for i, row in enumerate(table.rows):545 cells = []546 for cell in row.cells:547 cells.append(TableCell(548 text = cell.text.strip(),549 is_header = (i == 0 and has_header),550 ))551 if cells: # only include rows that have at least one cell552 rows.append(cells)553 554 try:555 tbl_pr = table._tbl.find(qn("w:tblPr"))556 borders = tbl_pr.find(qn("w:tblBorders")) if tbl_pr is not None else None557 border_val = "grid" if borders is not None else "none"558 except Exception:559 border_val = "unknown"560 561 return TableBlock(562 rows = rows,563 has_header_row = has_header,564 col_count = len(table.columns) if rows else 0,565 original_style = {"border": border_val},566 )567 568 569def extract_raw_content(file_bytes: bytes, filename: str) -> list[RawBlock]:570 ext = filename.rsplit(".", 1)[-1].lower()571 log.info(f"[EXTRACTOR] Starting: file={filename!r}, size={len(file_bytes)} bytes, type={ext}")572 if ext == "docx":573 return _extract_from_docx(file_bytes)574 elif ext == "pdf":575 return _extract_from_pdf(file_bytes)576 else:577 raise ValueError("Unsupported file type: only DOCX and PDF are accepted")578 579 580# ── DOCX extraction ───────────────────────────────────────────────────────────581 582def _merge_tab_tables(blocks: list[RawBlock]) -> list[RawBlock]:583 """584 Detect consecutive PARAGRAPH blocks whose text is tab-delimited with585 a consistent column count (≥2 columns, ≥2 rows) and merge them into586 a single TABLE block.587 588 This converts "fake tables" authored as tab-separated text into real589 structured TableBlock objects that the assembler can render with the590 template's table styling.591 592 Rules:593 - A paragraph is a candidate if it contains ≥1 tab AND splitting on tab594 gives ≥2 non-empty cells.595 - Consecutive candidates on the same page with the same column count form596 a table group.597 - Groups with ≥2 rows become TABLE blocks; single-row groups are left as598 paragraphs (could be a header row orphan — not a table).599 - The first row is treated as a header if its cells are short (≤4 words600 each) and ALL-CAPS or bold-looking (heuristic: first row in a data table601 is usually column labels).602 """603 if not blocks:604 return blocks605 606 result: list[RawBlock] = []607 i = 0608 609 while i < len(blocks):610 b = blocks[i]611 612 # Only inspect PARAGRAPH blocks with tab characters613 if (b.block_type not in (BlockType.PARAGRAPH, BlockType.HEADING1,614 BlockType.HEADING2, BlockType.HEADING3,615 BlockType.HEADING4)616 or "\t" not in b.text):617 result.append(b)618 i += 1619 continue620 621 # Parse this block as a candidate table row622 cells = [c.strip() for c in b.text.split("\t")]623 if len(cells) < 2 or not any(cells):624 result.append(b)625 i += 1626 continue627 628 col_count = len(cells)629 630 # Collect consecutive tab-delimited blocks with the same col count631 group: list[tuple[RawBlock, list[str]]] = [(b, cells)]632 j = i + 1633 while j < len(blocks):634 nb = blocks[j]635 if (nb.block_type not in (BlockType.PARAGRAPH, BlockType.HEADING1,636 BlockType.HEADING2, BlockType.HEADING3,637 BlockType.HEADING4)638 or "\t" not in nb.text):639 break640 nb_cells = [c.strip() for c in nb.text.split("\t")]641 if len(nb_cells) != col_count or not any(nb_cells):642 break643 group.append((nb, nb_cells))644 j += 1645 646 if len(group) < 2:647 # Single row — not a table648 result.append(b)649 i += 1650 continue651 652 # Build TableBlock from the group653 # Heuristic: first row is header if all cells are ≤4 words654 first_cells = group[0][1]655 is_header_row = all(len(c.split()) <= 4 for c in first_cells if c)656 657 rows: list[list[TableCell]] = []658 for row_idx, (_, row_cells) in enumerate(group):659 is_hdr = (row_idx == 0 and is_header_row)660 rows.append([TableCell(text=c, is_header=is_hdr) for c in row_cells])661 662 table_block = TableBlock(663 rows = rows,664 has_header_row = is_header_row,665 col_count = col_count,666 original_style = {"border": "none", "source": "tab_delimited"},667 )668 669 merged = RawBlock(670 id = b.id,671 block_type = BlockType.TABLE,672 heuristic_confidence = "low",673 text = "",674 page_number = b.page_number,675 page_break_before = b.page_break_before,676 table = table_block,677 original_style = table_block.original_style,678 )679 result.append(merged)680 log.info(681 f"[EXTRACTOR] Merged {len(group)} tab-delimited paragraphs → "682 f"TABLE block {b.id} ({col_count}×{len(group)})"683 )684 i = j # skip past all merged blocks685 686 return result687 688 689def _parse_space_columns(text: str) -> list[str] | None:690 """691 Split a paragraph into columns based on 3+ consecutive spaces used as692 a column separator (space-padded table formatting).693 694 Returns a list of ≥2 non-empty column values, or None if the text695 doesn't look like a space-padded table row.696 697 Strategy: split on runs of 3+ spaces, strip each cell, drop empty ones.698 Require at least 2 non-empty cells for a valid row.699 """700 parts = re.split(r" {3,}", text)701 cells = [p.strip() for p in parts if p.strip()]702 if len(cells) >= 2:703 return cells704 return None705 706 707def _merge_space_tables(blocks: list[RawBlock]) -> list[RawBlock]:708 """709 Detect consecutive PARAGRAPH/HEADING blocks whose text uses 3+ consecutive710 spaces as column separators (space-padded table formatting) and merge them711 into a single TABLE block.712 713 This is the space-column equivalent of _merge_tab_tables. Documents where714 authors aligned columns with spaces instead of tabs produce these patterns.715 716 Rules mirror _merge_tab_tables:717 - Require ≥2 non-empty columns per row.718 - Require ≥2 consecutive rows with the same column count.719 - First row is a header if all its cells are ≤ 4 words.720 """721 if not blocks:722 return blocks723 724 result: list[RawBlock] = []725 i = 0726 727 while i < len(blocks):728 b = blocks[i]729 730 # Only inspect text blocks — skip tables and figures731 if b.block_type in (BlockType.TABLE, BlockType.FIGURE,732 BlockType.TOC_ENTRY, BlockType.TOF_ENTRY):733 result.append(b)734 i += 1735 continue736 737 cells = _parse_space_columns(b.text)738 if cells is None:739 result.append(b)740 i += 1741 continue742 743 col_count = len(cells)744 group: list[tuple[RawBlock, list[str]]] = [(b, cells)]745 j = i + 1746 747 while j < len(blocks):748 nb = blocks[j]749 if nb.block_type in (BlockType.TABLE, BlockType.FIGURE,750 BlockType.TOC_ENTRY, BlockType.TOF_ENTRY):751 break752 nb_cells = _parse_space_columns(nb.text)753 if nb_cells is None or len(nb_cells) != col_count:754 break755 group.append((nb, nb_cells))756 j += 1757 758 if len(group) < 2:759 result.append(b)760 i += 1761 continue762 763 first_cells = group[0][1]764 is_header_row = all(len(c.split()) <= 4 for c in first_cells if c)765 766 rows: list[list[TableCell]] = []767 for row_idx, (_, row_cells) in enumerate(group):768 is_hdr = (row_idx == 0 and is_header_row)769 rows.append([TableCell(text=c, is_header=is_hdr) for c in row_cells])770 771 table_block = TableBlock(772 rows = rows,773 has_header_row = is_header_row,774 col_count = col_count,775 original_style = {"border": "none", "source": "space_delimited"},776 )777 778 merged = RawBlock(779 id = b.id,780 block_type = BlockType.TABLE,781 heuristic_confidence = "low",782 text = "",783 page_number = b.page_number,784 page_break_before = b.page_break_before,785 table = table_block,786 original_style = table_block.original_style,787 )788 result.append(merged)789 log.info(790 f"[EXTRACTOR] Merged {len(group)} space-padded paragraphs → "791 f"TABLE block {b.id} ({col_count}×{len(group)})"792 )793 i = j794 795 return result796 797 798def _extract_from_docx(file_bytes: bytes) -> list[RawBlock]:799 doc = Document(io.BytesIO(file_bytes))800 blocks = []801 idx = 0802 page_num = 1 # 1-based page counter803 804 body = doc.element.body805 children = list(body)806 log.info(f"[EXTRACTOR] DOCX body has {len(children)} top-level elements")807 808 pending_page_break = False # set when previous element ended with a page break809 _list_indent_stack: list[int] = [] # tracks indent levels for text-pattern list nesting810 811 # Native Word TOC/TOF field boundary tracking812 # TOC field: w:instrText contains "TOC" but NOT the \c switch813 # TOF field: w:instrText contains "TOC" AND the \c switch (e.g. \c "Figure")814 in_toc_field = False # True while inside a w:fldChar TOC field815 toc_field_instr_seen = False # True once we've seen "TOC" without \c816 in_tof_field = False # True while inside a w:fldChar TOF field817 tof_field_instr_seen = False # True once we've seen "TOC" with \c818 819 for child in children:820 # ── Scan for native TOC/TOF field boundary markers ──────────821 # Walk all descendant w:fldChar and w:instrText elements in this822 # body child (paragraph or table cell) to update field-tracking state.823 for el in child.iter():824 el_local = el.tag.split("}")[-1] if "}" in el.tag else el.tag825 if el_local == "fldChar":826 fld_type = el.get(qn("w:fldCharType"), "")827 if fld_type == "begin":828 # Reset all four flags — wait for instrText to identify field type829 in_toc_field = False830 toc_field_instr_seen = False831 in_tof_field = False832 tof_field_instr_seen = False833 elif fld_type == "end":834 if tof_field_instr_seen:835 in_tof_field = False836 tof_field_instr_seen = False837 elif toc_field_instr_seen:838 in_toc_field = False839 toc_field_instr_seen = False840 elif el_local == "instrText":841 if el.text and "TOC" in el.text:842 instr = el.text843 # \c switch distinguishes TOF (Table of Figures) from TOC844 if "\\c" in instr or r"\c" in instr:845 in_tof_field = True846 tof_field_instr_seen = True847 else:848 in_toc_field = True849 toc_field_instr_seen = True850 tag = child.tag.split("}")[-1] if "}" in child.tag else child.tag851 852 has_pbefore = _has_page_break_before(child)853 has_pbend = _has_page_break_at_end(child)854 855 # w:pageBreakBefore: THIS element starts a new page.856 # If pending_page_break is already True (from the previous element's857 # end-break), the page was already incremented — don't double-count.858 if has_pbefore and not pending_page_break:859 page_num += 1860 log.debug(f"[EXTRACTOR] pageBreakBefore detected → now on page {page_num}")861 862 # This block starts a new page if either:863 # - the previous element ended with a page break (pending_page_break), OR864 # - this element has w:pageBreakBefore set865 page_break_detected = pending_page_break or has_pbefore866 pending_page_break = False867 868 # w:br type="page": NEXT element starts a new page.869 # Only increment page_num if this element did NOT already trigger a870 # begin-break increment above — avoids double-counting a paragraph871 # that both starts on a new page (w:pageBreakBefore) AND ends with872 # an explicit page break (w:br type="page"), e.g. a solo chapter-title page.873 if has_pbend:874 if not has_pbefore:875 page_num += 1876 pending_page_break = True877 log.debug(f"[EXTRACTOR] page break at end detected → next element on page {page_num}")878 879 if tag == "tbl":880 block_emitted = False881 try:882 table = DocxTable(child, doc)883 table_block = _extract_table_block(table)884 if table_block.rows:885 log.info(f"[EXTRACTOR] Block {idx}: TABLE p{page_num} ({table_block.col_count}×{len(table_block.rows)})")886 blocks.append(RawBlock(887 id = idx,888 block_type = BlockType.TABLE,889 text = "",890 page_number = page_num,891 page_break_before = page_break_detected,892 table = table_block,893 original_style = table_block.original_style,894 ))895 idx += 1896 block_emitted = True897 except Exception as e:898 log.warning(f"[EXTRACTOR] Table failed: {e}")899 # Do NOT clear page_break_detected here — the break flag belongs900 # to this position in the document and must carry to the next block.901 # If no block was emitted (empty table or exception), carry the break forward902 if not block_emitted and page_break_detected:903 pending_page_break = True904 log.debug("[EXTRACTOR] Table emitted no block — carrying page_break_before forward")905 906 elif tag == "p":907 try:908 para = DocxParagraph(child, doc)909 except Exception as e:910 log.warning(f"[EXTRACTOR] Paragraph failed: {e}")911 continue912 913 if _check_for_figure(para):914 caption = para.text.strip()915 fig_w, fig_h = _get_figure_dimensions(para)916 image_data = _extract_image_from_para(para, doc)917 if image_data:918 fig_w = round(image_data["width_emu"] / 360000, 1) if image_data["width_emu"] else fig_w919 fig_h = round(image_data["height_emu"] / 360000, 1) if image_data["height_emu"] else fig_h920 log.info(f"[EXTRACTOR] Block {idx}: FIGURE p{page_num} ({fig_w}×{fig_h}cm, image={'yes' if image_data else 'no'})")921 blocks.append(RawBlock(922 id = idx,923 block_type = BlockType.FIGURE,924 text = caption,925 page_number = page_num,926 page_break_before = page_break_detected,927 figure_width = fig_w,928 figure_height = fig_h,929 original_style = {930 "type": "figure",931 "image_bytes": image_data["image_bytes"] if image_data else None,932 "content_type": image_data["content_type"] if image_data else None,933 "width_emu": image_data["width_emu"] if image_data else None,934 "height_emu": image_data["height_emu"] if image_data else None,935 },936 ))937 idx += 1938 continue939 940 text = para.text.strip()941 if not text:942 continue943 944 style_name = para.style.name if para.style else ""945 946 if in_tof_field or _is_tof_entry(text, style_name):947 log.info(f"[EXTRACTOR] Block {idx}: TOF_ENTRY p{page_num}")948 blocks.append(RawBlock(949 id = idx,950 block_type = BlockType.TOF_ENTRY,951 heuristic_confidence = "high",952 text = text,953 page_number = page_num,954 page_break_before = page_break_detected,955 original_style = {"style": style_name},956 ))957 idx += 1958 continue959 960 elif in_toc_field or _is_toc_entry(text, style_name):961 log.info(f"[EXTRACTOR] Block {idx}: TOC_ENTRY p{page_num}")962 blocks.append(RawBlock(963 id = idx,964 block_type = BlockType.TOC_ENTRY,965 heuristic_confidence = "high",966 text = text,967 page_number = page_num,968 page_break_before = page_break_detected,969 original_style = {"style": style_name},970 ))971 idx += 1972 continue973 974 runs = [r for r in para.runs if r.text.strip()]975 bold_flags = [r.bold for r in runs]976 italic_flags = [r.italic for r in runs]977 sizes = [r.font.size.pt for r in runs if r.font.size]978 names = [r.font.name for r in runs if r.font.name]979 980 is_bold = any(b for b in bold_flags if b)981 is_italic = any(i for i in italic_flags if i)982 font_size = statistics.median(sizes) if sizes else None983 font_name = names[0] if names else None984 985 rtl = _para_rtl(para) or _is_rtl_text(text)986 if font_name:987 rtl = rtl or _is_rtl_font(font_name)988 989 alignment = _alignment_from_docx(para)990 block_type, confidence = _infer_block_type_docx(para, is_bold, font_size)991 992 list_level = 0993 try:994 ilvl = para._element.find(995 ".//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}ilvl"996 )997 if ilvl is not None:998 # Real Word numPr — ilvl is authoritative999 list_level = int(ilvl.get(qn("w:val"), 0))1000 elif block_type == BlockType.LIST_ITEM and confidence == "low":1001 # Text-pattern list — no real ilvl.1002 # Infer nesting from leading whitespace, maintaining an indent1003 # stack that resets whenever a non-list paragraph breaks the run.1004 # Use the RAW (unstripped) paragraph text so leading spaces are intact.1005 list_level, _list_indent_stack = _infer_text_list_level(1006 para.text, _list_indent_stack1007 )1008 except Exception:1009 pass1010 1011 # Reset the indent stack when we exit a list run (non-list block)1012 if block_type != BlockType.LIST_ITEM:1013 _list_indent_stack = []1014 1015 # Extract inline run spans for mixed-formatting paragraphs.1016 # Returns None when all runs are uniform (assembler uses single run).1017 inline_runs = _extract_inline_runs(para)1018 1019 log.info(1020 f"[EXTRACTOR] Block {idx}: {block_type.value.upper()} ({confidence}) p{page_num} | "1021 f"text={repr(text[:50])} | bold={is_bold} | size={font_size} | "1022 f"runs={len(inline_runs) if inline_runs else 1}"1023 )1024 1025 blocks.append(RawBlock(1026 id = idx,1027 block_type = block_type,1028 heuristic_confidence = confidence,1029 text = text,1030 page_number = page_num,1031 page_break_before = page_break_detected,1032 is_bold = is_bold,1033 is_italic = is_italic,1034 font_size = font_size,1035 font_name = font_name,1036 alignment = alignment,1037 rtl = rtl,1038 is_list = block_type == BlockType.LIST_ITEM,1039 list_level = list_level,1040 inline_runs = inline_runs,1041 original_style = {1042 "font_name": font_name,1043 "font_size": font_size,1044 "bold": is_bold,1045 "italic": is_italic,1046 "alignment": alignment,1047 "rtl": rtl,1048 "style": style_name, # used by assembler to detect list type1049 # NEW: explicit flag for Caption-family style origin1050 # Only set for paragraphs classified as CAPTION via style name1051 "original_style_name": style_name if block_type == BlockType.CAPTION and confidence == "high" else None,1052 },1053 ))1054 idx += 11055 1056 # Merge consecutive tab-delimited paragraphs into TABLE blocks1057 blocks = _merge_tab_tables(blocks)1058 # Merge consecutive space-padded paragraphs into TABLE blocks1059 blocks = _merge_space_tables(blocks)1060 1061 # Stamp position context1062 total = len(blocks)1063 for i, b in enumerate(blocks):1064 blocks[i] = b.model_copy(update={"block_index": i, "total_blocks": total})1065 1066 log.info(f"[EXTRACTOR] DOCX complete: {total} blocks across {page_num} pages")1067 return blocks1068 1069 1070# ── PDF extraction ────────────────────────────────────────────────────────────1071 1072def _pdf_is_list(text: str) -> bool:1073 patterns = [1074 r"^\s*[\u2022\u2023\u2024\u2025\u2043\u25AA\u25AB\u25CF\u25CB\u25E6\u2013\u00B7\-\*\u2014]\s",1075 r"^\s*\d+[\.\)]\s",1076 r"^\s*[a-zA-Z][\.\)]\s",1077 r"^\s*\(\d+\)\s", # (1) style1078 r"^\s*\([a-zA-Z]\)\s", # (a) style1079 ]1080 return any(re.match(p, text) for p in patterns)1081 1082 1083def _is_likely_noise(text: str) -> bool:1084 s = text.strip()1085 if not s:1086 return True1087 # Standalone page numbers (possibly with Roman numerals)1088 if re.match(r"^[ivxlcdmIVXLCDM\d]+$", s) and len(s) <= 6:1089 return True1090 # Very short non-alpha tokens1091 if len(s) <= 2 and not s.isalpha():1092 return True1093 # "Page N" or "Page N of M" or "- N -"1094 if re.match(r"^page\s+\d+(\s+of\s+\d+)?$", s.lower()):1095 return True1096 if re.match(r"^[-–—]\s*\d+\s*[-–—]$", s):1097 return True1098 # Pure punctuation / separator lines1099 if re.match(r"^[\s\-_=\|\.]{3,}$", s):1100 return True1101 # Single character (not a letter)1102 if len(s) == 1 and not s.isalpha():1103 return True1104 return False1105 1106 1107# Running header/footer deduplication across pages1108# Texts that appear on 3+ pages with identical content are likely headers/footers1109def _filter_running_headers(raw_pdf_blocks: list[dict]) -> list[dict]:1110 """1111 Remove blocks whose text appears verbatim on 3 or more different pages.1112 These are almost certainly running headers or footers injected as text blocks.1113 Only applies to short texts (≤ 12 words) to avoid removing repeated section titles.1114 """1115 text_pages: dict[str, set] = defaultdict(set)1116 for b in raw_pdf_blocks:1117 if b.get("type") != "text":1118 continue1119 t = b.get("text", "").strip()1120 if t and len(t.split()) <= 12:1121 text_pages[t].add(b.get("page", 0))1122 1123 # Texts appearing on 3+ pages are running headers/footers1124 noise_texts = {t for t, pages in text_pages.items() if len(pages) >= 3}1125 if noise_texts:1126 log.info(f"[EXTRACTOR] Filtering {len(noise_texts)} running header/footer text(s)")1127 1128 return [1129 b for b in raw_pdf_blocks1130 if not (b.get("type") == "text" and b.get("text", "").strip() in noise_texts)1131 ]1132 1133 1134def _infer_block_type_pdf(text: str, font_size: float, is_bold: bool, median_size: float) -> BlockType:1135 if _pdf_is_list(text):1136 return BlockType.LIST_ITEM1137 lower = text.lower().strip()1138 if re.match(r"^(figure|fig\.?|table|chart|graph)\s*\d*[\.\:]", lower):1139 return BlockType.CAPTION1140 ratio = font_size / median_size if median_size else 1.01141 word_count = len(text.split())1142 1143 # Adaptive thresholds: if the document has a narrow font size range,1144 # lower the ratio requirements so headings aren't missed.1145 # A ratio of 1.1 is enough to distinguish headings when body=12pt, heading=13pt.1146 h1_ratio = max(1.4, min(1.6, 1.0 + (median_size * 0.04)))1147 h2_ratio = max(1.2, min(1.35, 1.0 + (median_size * 0.025)))1148 h3_ratio = max(1.08, min(1.2, 1.0 + (median_size * 0.015)))1149 1150 if ratio >= h1_ratio and word_count <= 15:1151 return BlockType.HEADING11152 if ratio >= h2_ratio and word_count <= 15:1153 return BlockType.HEADING21154 if ratio >= h3_ratio and is_bold and word_count <= 20:1155 return BlockType.HEADING31156 # Bold short-text heuristic: only apply from page 2+ to avoid tagging1157 # cover page labels ("SUBMITTED BY", "SUPERVISED BY", etc.) as headings.1158 # page_number is not available here — caller guards this via Stage 0,1159 # but we tighten the ratio requirement to reduce false positives on page 1.1160 if is_bold and word_count <= 10 and text and text[0].isupper() and ratio >= 1.05:1161 return BlockType.HEADING31162 return BlockType.PARAGRAPH1163 1164 1165def _should_merge_pdf_blocks(prev: dict, curr: dict) -> bool:1166 """1167 Heuristic: two adjacent PDF text blocks are likely the same paragraph if:1168 - Same page1169 - Same font size (within 0.5pt)1170 - Same bold/italic flags1171 - Vertical gap is small (≤ 2.5x the font size — covers 1.5x line spacing)1172 - Neither is a heading1173 - Previous block doesn't end with sentence-ending punctuation AND1174 the current block starts with a lowercase letter (strong continuation signal)1175 - Blocks are horizontally overlapping (same column) — prevents merging1176 blocks from different columns in multi-column layouts.1177 1178 The sentence-end check is intentionally lenient: we only block merging when1179 the previous line ends with punctuation AND the next line starts uppercase,1180 which strongly suggests a new sentence/paragraph rather than a wrapped line.1181 """1182 if prev.get("page") != curr.get("page"):1183 return False1184 if abs(prev.get("font_size", 0) - curr.get("font_size", 0)) > 0.5:1185 return False1186 if prev.get("is_bold") != curr.get("is_bold"):1187 return False1188 # Don't merge headings1189 if prev.get("is_heading") or curr.get("is_heading"):1190 return False1191 # Check vertical gap — allow up to 2.5x font size to handle 1.5x line spacing1192 prev_bottom = prev.get("bbox_bottom", 0)1193 curr_top = curr.get("bbox_top", 0)1194 font_size = prev.get("font_size", 12)1195 gap = curr_top - prev_bottom1196 if gap < 0 or gap > font_size * 2.5:1197 return False1198 # Multi-column guard: if the blocks have bounding box info and their1199 # horizontal extents don't overlap, they are in different columns.1200 prev_x0 = prev.get("bbox_x0")