sfd-anonymous/sec-parser
SEC Filings Dataset Parser This repository contains the core parser used to convert SEC EDGAR filings into layout-faithful Markdown-style text for downstream dataset construction and evaluation. Contents sec_parser/sec_parser.py: main parser implementation sec_parser/special_chars.py: special-character normalization tables sec_parser/hardcodes.py: filing cleanup hardcodes sec_parser/config.py: parser configuration pdf_table_fastpath.py, table_ocr_backends.py… See the full description on the dataset page: https://huggingface.co/datasets/sfd-anonymous/sec-parser.
079
1#!/usr/bin/env python32from __future__ import annotations3 4import argparse5import json6from pathlib import Path7 8import table_ocr_backends9 10 11def parse_args() -> argparse.Namespace:12 parser = argparse.ArgumentParser(13 description=(14 "Run OCR on a PDF page, then overlay born-digital PDF cell bboxes "15 "and recovered bold/italic/underline formatting onto the returned table HTML."16 )17 )18 parser.add_argument("--pdf", required=True, help="Path to the source PDF.")19 parser.add_argument("--page", required=True, type=int, help="1-based PDF page number.")20 parser.add_argument(21 "--model-id",22 default=None,23 help="Optional OCR model id override. Defaults to the configured/default PDF page OCR model.",24 )25 parser.add_argument(26 "--input-html",27 default=None,28 help="Optional path to an existing OCR HTML fragment. When provided, the script skips OCR and only applies the PDF-native overlay.",29 )30 parser.add_argument(31 "--style-overlay-mode",32 default="auto",33 choices=["none", "attrs_only", "formatting_only", "auto", "aggressive"],34 help=(35 "Formatting overlay behavior. "36 "`formatting_only` preserves OCR text and only injects semantic bold/italic/underline tags. "37 "`auto` safely swaps in native styled cell HTML when the text match is strong. "38 "`aggressive` prefers native styled cell HTML whenever a cell matches."39 ),40 )41 parser.add_argument("--page-render-zoom", type=float, default=None, help="Optional PDF render zoom before OCR.")42 parser.add_argument("--output-html", default=None, help="Optional path to write the final annotated HTML.")43 parser.add_argument(44 "--output-raw-html",45 default=None,46 help=(47 "Optional path to write the original OCR HTML before PDF-native overlay. "48 "If omitted and --output-html is set, defaults to a sibling '*.raw.html' file."49 ),50 )51 parser.add_argument("--output-json", default=None, help="Optional path to write the full JSON payload.")52 return parser.parse_args()53 54 55def main() -> None:56 args = parse_args()57 if args.input_html:58 input_html_path = Path(args.input_html).resolve()59 input_html = input_html_path.read_text(encoding="utf-8")60 payload = table_ocr_backends.overlay_pdf_page_html_with_native_cells(61 input_html,62 pdf_path=args.pdf,63 page_number=max(1, int(args.page)),64 effective_model_id=args.model_id or "existing-html+pdf-overlay",65 style_overlay_mode=args.style_overlay_mode,66 )67 else:68 payload = table_ocr_backends.transcribe_pdf_page_to_payload(69 args.pdf,70 page_number=max(1, int(args.page)),71 model_id=args.model_id,72 page_render_zoom=args.page_render_zoom,73 overlay_pdf_cells=True,74 style_overlay_mode=args.style_overlay_mode,75 )76 77 output_html = str(payload.get("html") or "")78 raw_html = str(payload.get("raw_html") or output_html)79 output_html_path = Path(args.output_html).resolve() if args.output_html else None80 output_raw_html_path = None81 if args.output_raw_html:82 output_raw_html_path = Path(args.output_raw_html).resolve()83 elif output_html_path is not None:84 output_raw_html_path = output_html_path.with_name(f"{output_html_path.stem}.raw.html")85 86 if args.output_html:87 assert output_html_path is not None88 output_html_path.parent.mkdir(parents=True, exist_ok=True)89 output_html_path.write_text(output_html, encoding="utf-8")90 91 if output_raw_html_path is not None:92 output_raw_html_path.parent.mkdir(parents=True, exist_ok=True)93 output_raw_html_path.write_text(raw_html, encoding="utf-8")94 95 if args.output_json:96 output_json_path = Path(args.output_json).resolve()97 output_json_path.parent.mkdir(parents=True, exist_ok=True)98 output_json_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")99 100 if not args.output_html and not args.output_json:101 print(output_html)102 return103 104 summary = {105 "pdf": str(Path(args.pdf).resolve()),106 "page": int(args.page),107 "model_id": payload.get("effective_model_id"),108 "overlay_applied": bool(payload.get("overlay_applied")),109 "overlay_changed_html": output_html != raw_html,110 "style_overlay_mode": payload.get("style_overlay_mode"),111 "timings_ms": payload.get("timings_ms"),112 "output_html": str(output_html_path) if output_html_path is not None else None,113 "output_raw_html": str(output_raw_html_path) if output_raw_html_path is not None else None,114 "output_json": str(Path(args.output_json).resolve()) if args.output_json else None,115 }116 print(json.dumps(summary, indent=2, sort_keys=True))117 118 119if __name__ == "__main__":120 main()121 