stevafernandes/Fine-Tuning-HunyuanOCR

1"""Score predictions against ground truth: normalized edit distance and AURC.2 3AURC (area under the risk-coverage curve) follows Geifman & El-Yaniv (2017):4sort predictions by confidence descending, then average the running mean risk5(normalized edit distance) over all coverage levels. Lower is better. The6competition score is the mean of the date AURC and the locality AURC.7 8Usage:9  python evaluate.py --predictions val_predictions.csv --truth-csv ../train.csv10"""11 12import argparse13import csv14 15from common import normalized_edit_distance16 17 18def aurc(risks: list[float], confidences: list[float]) -> float:19    order = sorted(range(len(risks)), key=lambda i: -confidences[i])20    # Average risk within equal-confidence groups so the result does not21    # depend on input order among ties.22    smoothed: list[float] = []23    i = 024    while i < len(order):25        j = i26        while j < len(order) and confidences[order[j]] == confidences[order[i]]:27            j += 128        group = [risks[idx] for idx in order[i:j]]29        smoothed.extend([sum(group) / len(group)] * len(group))30        i = j31    total = 0.032    running = 0.033    for k, risk in enumerate(smoothed, start=1):34        running += risk35        total += running / k36    return total / len(smoothed)37 38 39def main() -> None:40    parser = argparse.ArgumentParser(description=__doc__)41    parser.add_argument("--predictions", required=True)42    parser.add_argument("--truth-csv", required=True)43    args = parser.parse_args()44 45    with open(args.truth_csv, encoding="utf-8") as f:46        truth = {row["image_file"]: row for row in csv.DictReader(f)}47    with open(args.predictions, encoding="utf-8") as f:48        preds = list(csv.DictReader(f))49 50    preds = [p for p in preds if p["image_file"] in truth]51    if not preds:52        raise SystemExit("no overlapping image_file between predictions and truth")53 54    report = {}55    for field in ("verbatimDate", "verbatimLocality"):56        risks, confidences, exact = [], [], 057        for p in preds:58            t = truth[p["image_file"]][field].strip()59            risk = normalized_edit_distance(p[field], t)60            risks.append(risk)61            confidences.append(float(p[f"{field}_confidence"]))62            exact += risk == 0.063        report[field] = {64            "n": len(risks),65            "exact_match": round(exact / len(risks), 4),66            "mean_ned": round(sum(risks) / len(risks), 4),67            "aurc": round(aurc(risks, confidences), 4),68        }69 70    for field, metrics in report.items():71        print(72            f"{field}: n={metrics['n']} exact={metrics['exact_match']} "73            f"mean_ned={metrics['mean_ned']} aurc={metrics['aurc']}"74        )75    score = (report["verbatimDate"]["aurc"] + report["verbatimLocality"]["aurc"]) / 276    print(f"competition score (mean AURC, lower is better): {round(score, 4)}")77 78 79if __name__ == "__main__":80    main()81