stevafernandes/Fine-Tuning-HunyuanOCR

1"""Shared constants and helpers for the MuseumSCAT HunyuanOCR fine-tuning pipeline."""2 3import json4import re5 6# Fixed instruction used for training, prediction, and the deployed app.7# It must stay byte-identical across all three, since the model is fine-tuned on it.8PROMPT = (9    "Read the specimen labels in this image. Transcribe the collection date and the "10    "collection locality exactly as written on the labels, using the Danish alphabet "11    '(replace historical umlauts). Answer with JSON only, in the form '12    '{"verbatimDate": "...", "verbatimLocality": "..."}. '13    'Use "MISSING" for a field that is not present on any label.'14)15 16MISSING = "MISSING"17 18# Longest image side, applied when preparing training data and again at19# inference (predict.py and the Space app). Must match the resolution the20# adapter was trained at.21MAX_SIDE = 153622 23 24def target_json(date: str, locality: str) -> str:25    """Serialize a training target. Key order is fixed: date first."""26    return json.dumps(27        {"verbatimDate": date, "verbatimLocality": locality},28        ensure_ascii=False,29    )30 31 32def parse_prediction(text: str) -> dict:33    """Parse a model response into {'verbatimDate': str, 'verbatimLocality': str}.34 35    Scans for the first parseable JSON object anywhere in the text, so exact36    JSON, JSON with surrounding prose, and repeated objects all work. Falls37    back to MISSING for fields that cannot be recovered.38    """39    text = text.strip()40    decoder = json.JSONDecoder()41    for match in re.finditer(r"\{", text):42        try:43            obj, _ = decoder.raw_decode(text, match.start())44        except ValueError:45            continue46        if isinstance(obj, dict):47            date = obj.get("verbatimDate")48            locality = obj.get("verbatimLocality")49            return {50                "verbatimDate": str(date) if date is not None else MISSING,51                "verbatimLocality": str(locality) if locality is not None else MISSING,52            }53    return {"verbatimDate": MISSING, "verbatimLocality": MISSING}54 55 56def normalized_edit_distance(pred: str, truth: str) -> float:57    """Levenshtein distance divided by the length of the longer string (0..1)."""58    if pred == truth:59        return 0.060    if not pred or not truth:61        return 1.062    prev = list(range(len(truth) + 1))63    for i, pc in enumerate(pred, start=1):64        cur = [i]65        for j, tc in enumerate(truth, start=1):66            cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (pc != tc)))67        prev = cur68    return prev[-1] / max(len(pred), len(truth))69