dougvk/Unlimited-OCR-RDNA4

1from unlimited_ocr_rdna4.postprocess import EOS_STOP, clean_model_output, repetition_warning2 3 4def test_clean_model_output_preserves_unicode_and_content() -> None:5    raw = (6        "<|det|>header [1, 2, 3, 4]<|/det|>Überblick\n"7        "<|det|>table [1, 2, 3, 4]<|/det|><table><tr><td>€48.50</td></tr></table>\n"8        "[Non-Text]\n"9        "\\coloneqq"10        f"{EOS_STOP}"11    )12    assert clean_model_output(raw) == "Überblick\n<table><tr><td>€48.50</td></tr></table>\n\\coloneqq"13 14 15def test_clean_model_output_removes_combined_reference_tag() -> None:16    raw = "<|ref|>title<|/ref|><|det|>[[1, 2, 3, 4]]<|/det|>Actual title"17    assert clean_model_output(raw) == "titleActual title"18 19 20def test_clean_model_output_preserves_ordinary_words_and_tex() -> None:21    raw = "NonTextile material\nA \\eqqcolon B\nembedded NonText label"22    assert clean_model_output(raw) == raw23 24 25def test_clean_model_output_preserves_malformed_unclosed_tags() -> None:26    raw = "before <|det|>coordinate and recognized content"27    assert clean_model_output(raw) == raw28 29 30def test_repetition_warning_detects_three_lines() -> None:31    text = "start\nrepeating output\nrepeating output\nrepeating output"32    assert "repeated line" in (repetition_warning(text) or "")33 34 35def test_repetition_warning_accepts_normal_text() -> None:36    assert repetition_warning("first\nsecond\nthird") is None37