aops02/math-annotation-demo

1#!/usr/bin/env python32"""Validate generated Potato data files."""3 4from __future__ import annotations5 6import argparse7import csv8import json9import sys10from pathlib import Path11 12 13PROJECT_DIR = Path(__file__).resolve().parents[1]14DEFAULT_DATA_DIR = PROJECT_DIR / "my-annotation-task" / "data"15 16DIMENSIONS = [17    "Content Correctness",18    "Learner-State Assessment",19    "Issue Localization",20    "Disclosure Appropriateness",21    "Providing Guidance",22    "Coherence",23    "Actionability",24    "Clarity",25    "Conciseness",26    "Humanness",27]28 29VALID_LABELS = {"Yes", "To some extent", "No"}30REQUIRED_COLUMNS = [31    "id",32    "split",33    "dialog_context",34    "correct_solution",35    "tutor_response",36    "text2show_html",37    *DIMENSIONS,38]39 40 41def read_csv(path: Path) -> list[dict[str, str]]:42    with path.open(newline="", encoding="utf-8") as handle:43        return list(csv.DictReader(handle))44 45 46def validate_csv(path: Path, expected_split: str | None = None) -> int:47    rows = read_csv(path)48    if not rows:49        raise ValueError(f"{path.name}: no rows")50    missing = [column for column in REQUIRED_COLUMNS if column not in rows[0]]51    if missing:52        raise ValueError(f"{path.name}: missing columns {missing}")53 54    ids = set()55    for row_number, row in enumerate(rows, start=2):56        item_id = row["id"]57        if not item_id:58            raise ValueError(f"{path.name}:{row_number}: empty id")59        if item_id in ids:60            raise ValueError(f"{path.name}:{row_number}: duplicate id {item_id}")61        ids.add(item_id)62        if expected_split and row["split"] != expected_split:63            raise ValueError(f"{path.name}:{row_number}: expected split {expected_split}, got {row['split']}")64        for column in REQUIRED_COLUMNS:65            if not row[column].strip():66                raise ValueError(f"{path.name}:{row_number}: empty {column}")67        for dimension in DIMENSIONS:68            if row[dimension] not in VALID_LABELS:69                raise ValueError(f"{path.name}:{row_number}: invalid {dimension}: {row[dimension]!r}")70    return len(rows)71 72 73def validate_json(path: Path, key_name: str) -> int:74    payload = json.loads(path.read_text(encoding="utf-8"))75    if not isinstance(payload, list) or not payload:76        raise ValueError(f"{path.name}: expected a non-empty list")77    ids = set()78    for index, item in enumerate(payload, start=1):79        item_id = item.get("id")80        if not item_id:81            raise ValueError(f"{path.name}:{index}: empty id")82        if item_id in ids:83            raise ValueError(f"{path.name}:{index}: duplicate id {item_id}")84        ids.add(item_id)85        labels = item.get(key_name)86        if not isinstance(labels, dict):87            raise ValueError(f"{path.name}:{index}: missing {key_name}")88        for dimension in DIMENSIONS:89            if labels.get(dimension) not in VALID_LABELS:90                raise ValueError(f"{path.name}:{index}: invalid {dimension}: {labels.get(dimension)!r}")91    return len(payload)92 93 94def main() -> int:95    parser = argparse.ArgumentParser(description=__doc__)96    parser.add_argument("--data-dir", type=Path, default=DEFAULT_DATA_DIR)97    args = parser.parse_args()98 99    checks = {100        "train_csv": validate_csv(args.data_dir / "math_annotator_training_set_with_id_text2show.csv", "train"),101        "test_csv": validate_csv(args.data_dir / "math_annotator_testing_set_with_id_text2show.csv", "test"),102        "combined_csv": validate_csv(args.data_dir / "math_annotator_demo_all_with_id_text2show.csv"),103        "training_questions": validate_json(args.data_dir / "training_questions.json", "correct_answers"),104        "gold_standards": validate_json(args.data_dir / "gold_standards.json", "gold_label"),105    }106    print(json.dumps(checks, indent=2))107    return 0108 109 110if __name__ == "__main__":111    try:112        raise SystemExit(main())113    except Exception as exc:114        print(f"Validation failed: {exc}", file=sys.stderr)115        raise SystemExit(1)116