CoolFace
Datasetpublic

sfd-anonymous/sec-parser

SEC Filings Dataset Parser This repository contains the core parser used to convert SEC EDGAR filings into layout-faithful Markdown-style text for downstream dataset construction and evaluation. Contents sec_parser/sec_parser.py: main parser implementation sec_parser/special_chars.py: special-character normalization tables sec_parser/hardcodes.py: filing cleanup hardcodes sec_parser/config.py: parser configuration pdf_table_fastpath.py, table_ocr_backends.py… See the full description on the dataset page: https://huggingface.co/datasets/sfd-anonymous/sec-parser.

sourceHugging Faceapache-2.0updated 5mo agoView on Hugging Face
0likes79downloads
mistral_pdf_ocr_overlay.py121 linesDownload Raw Back to root
1#!/usr/bin/env python32from __future__ import annotations3 4import argparse5import json6from pathlib import Path7 8import table_ocr_backends9 10 11def parse_args() -> argparse.Namespace:12    parser = argparse.ArgumentParser(13        description=(14            "Run OCR on a PDF page, then overlay born-digital PDF cell bboxes "15            "and recovered bold/italic/underline formatting onto the returned table HTML."16        )17    )18    parser.add_argument("--pdf", required=True, help="Path to the source PDF.")19    parser.add_argument("--page", required=True, type=int, help="1-based PDF page number.")20    parser.add_argument(21        "--model-id",22        default=None,23        help="Optional OCR model id override. Defaults to the configured/default PDF page OCR model.",24    )25    parser.add_argument(26        "--input-html",27        default=None,28        help="Optional path to an existing OCR HTML fragment. When provided, the script skips OCR and only applies the PDF-native overlay.",29    )30    parser.add_argument(31        "--style-overlay-mode",32        default="auto",33        choices=["none", "attrs_only", "formatting_only", "auto", "aggressive"],34        help=(35            "Formatting overlay behavior. "36            "`formatting_only` preserves OCR text and only injects semantic bold/italic/underline tags. "37            "`auto` safely swaps in native styled cell HTML when the text match is strong. "38            "`aggressive` prefers native styled cell HTML whenever a cell matches."39        ),40    )41    parser.add_argument("--page-render-zoom", type=float, default=None, help="Optional PDF render zoom before OCR.")42    parser.add_argument("--output-html", default=None, help="Optional path to write the final annotated HTML.")43    parser.add_argument(44        "--output-raw-html",45        default=None,46        help=(47            "Optional path to write the original OCR HTML before PDF-native overlay. "48            "If omitted and --output-html is set, defaults to a sibling '*.raw.html' file."49        ),50    )51    parser.add_argument("--output-json", default=None, help="Optional path to write the full JSON payload.")52    return parser.parse_args()53 54 55def main() -> None:56    args = parse_args()57    if args.input_html:58        input_html_path = Path(args.input_html).resolve()59        input_html = input_html_path.read_text(encoding="utf-8")60        payload = table_ocr_backends.overlay_pdf_page_html_with_native_cells(61            input_html,62            pdf_path=args.pdf,63            page_number=max(1, int(args.page)),64            effective_model_id=args.model_id or "existing-html+pdf-overlay",65            style_overlay_mode=args.style_overlay_mode,66        )67    else:68        payload = table_ocr_backends.transcribe_pdf_page_to_payload(69            args.pdf,70            page_number=max(1, int(args.page)),71            model_id=args.model_id,72            page_render_zoom=args.page_render_zoom,73            overlay_pdf_cells=True,74            style_overlay_mode=args.style_overlay_mode,75        )76 77    output_html = str(payload.get("html") or "")78    raw_html = str(payload.get("raw_html") or output_html)79    output_html_path = Path(args.output_html).resolve() if args.output_html else None80    output_raw_html_path = None81    if args.output_raw_html:82        output_raw_html_path = Path(args.output_raw_html).resolve()83    elif output_html_path is not None:84        output_raw_html_path = output_html_path.with_name(f"{output_html_path.stem}.raw.html")85 86    if args.output_html:87        assert output_html_path is not None88        output_html_path.parent.mkdir(parents=True, exist_ok=True)89        output_html_path.write_text(output_html, encoding="utf-8")90 91    if output_raw_html_path is not None:92        output_raw_html_path.parent.mkdir(parents=True, exist_ok=True)93        output_raw_html_path.write_text(raw_html, encoding="utf-8")94 95    if args.output_json:96        output_json_path = Path(args.output_json).resolve()97        output_json_path.parent.mkdir(parents=True, exist_ok=True)98        output_json_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")99 100    if not args.output_html and not args.output_json:101        print(output_html)102        return103 104    summary = {105        "pdf": str(Path(args.pdf).resolve()),106        "page": int(args.page),107        "model_id": payload.get("effective_model_id"),108        "overlay_applied": bool(payload.get("overlay_applied")),109        "overlay_changed_html": output_html != raw_html,110        "style_overlay_mode": payload.get("style_overlay_mode"),111        "timings_ms": payload.get("timings_ms"),112        "output_html": str(output_html_path) if output_html_path is not None else None,113        "output_raw_html": str(output_raw_html_path) if output_raw_html_path is not None else None,114        "output_json": str(Path(args.output_json).resolve()) if args.output_json else None,115    }116    print(json.dumps(summary, indent=2, sort_keys=True))117 118 119if __name__ == "__main__":120    main()121