miugod/CMB

1import json2import os3from typing import Dict, Any, List4 5OPTION_KEYS = ["A", "B", "C", "D", "E", "F"]6 7 8def normalize_option(option: Dict[str, Any]) -> Dict[str, str]:9    new_option = {}10    option = option or {}11    for k in OPTION_KEYS:12        new_option[k] = str(option.get(k, "")).strip()13    return new_option14 15 16def normalize_item(item: Dict[str, Any], idx: int) -> Dict[str, Any]:17    return {18        "id": item.get("id", idx),19        "exam_type": item.get("exam_type", ""),20        "exam_class": item.get("exam_class", ""),21        "exam_subject": item.get("exam_subject", ""),22        "question": item.get("question", ""),23        "answer": item.get("answer", ""),24        "explanation": item.get("explanation", ""),25        "question_type": item.get("question_type", ""),26        "option": normalize_option(item.get("option")),27    }28 29 30def clean_json(input_path: str) -> None:31    with open(input_path, "r", encoding="utf-8") as f:32        data: List[Dict[str, Any]] = json.load(f)33 34    cleaned = []35    for idx, item in enumerate(data, start=1):36        cleaned.append(normalize_item(item, idx))37 38    base, ext = os.path.splitext(input_path)39    output_path = f"{base}-clean{ext}"40 41    with open(output_path, "w", encoding="utf-8") as f:42        json.dump(cleaned, f, ensure_ascii=False, indent=2)43 44    print(f"[OK] {output_path}  ({len(cleaned)} samples)")45 46 47if __name__ == "__main__":48    input_files = [49        "CMB-Exam/CMB-train/CMB-train-merge.json",50        "CMB-Exam/CMB-val/CMB-val-merge.json",51        "CMB-Exam/CMB-test/CMB-test-choice-question-merge.json",52    ]53 54    for path in input_files:55        clean_json(path)56