MLRS/MELABench

1import json2import os3import re4from datetime import datetime, timezone5from pathlib import Path6 7from src.display.formatting import styled_error, styled_message, styled_warning8from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO, PROMPT_VERSIONS, PREDICTIONS_REPO9from src.submission.check_validity import already_submitted_models, is_model_on_hub, get_model_properties10 11REQUESTED_MODELS = None12 13 14def read_configuration(file_paths):15    configuration_file_paths = list(filter(lambda file_path: file_path.name.endswith(".json"), file_paths or []))16    if len(configuration_file_paths) != 1:17        return None, None, None, None, None, styled_error(f"Expected exactly one configuration file but found {len(configuration_file_paths)}!")18 19    configuration_file_path = file_paths.pop(file_paths.index(configuration_file_paths[0]))20 21    try:22        with open(configuration_file_path.name, "r", encoding="utf-8") as f:23            data = json.load(f)24    except Exception:25        return None, None, None, None, None, styled_error("Failed to read configuration file!")26 27    try:28        model_name = data["model_name"]29        model_args = {30            **dict({tuple(arg.split("=")) for arg in data["config"].get("model_args", "").split(",") if len(arg) > 0}),31            "revision": data["config"]["model_revision"],32            "trust_remote_code": True,33            "cache_dir": None34        }35        base_model = model_args.pop("pretrained")36        model_on_hub, error, _ = is_model_on_hub(model_name=base_model, model_args=model_args, token=TOKEN, test_tokenizer=True)37        if not model_on_hub:38            return None, None, model_name, None, None, styled_error(f"Model {model_name} {error}")39 40        limit = data["config"]["limit"]41        if limit is not None:42            return None, None, model_name, None, None, styled_error(f"Only full results are accepted but found a specified limit of {limit}!")43 44        prediction_files = {}45        versions = {}46        n_shots = {}47        for task_name, _ in data["configs"].items():48            sample_files = list(filter(lambda file_path: re.search(rf"samples_{task_name}_.*\.jsonl", file_path.name), file_paths))49            if len(sample_files) == 0:50                return None, None, model_name, None, None, styled_error(f"No prediction file found for configured task {task_name}!")51 52            prediction_files[task_name] = str(file_paths.pop(file_paths.index(sample_files[0])))53 54            versions[task_name] = data["versions"][task_name]55            n_shots[task_name] = data["n-shot"][task_name]56        if len(prediction_files) == 0:57            return None, None, model_name, None, None, styled_error("No tasks found in configuration!")58 59        versions = set(versions.values())60        if len(versions) != 1:61            return None, None, model_name, None, None, styled_error(f"All tasks should have the same version but found {versions}!")62        version = list(versions)[0]63        if version not in PROMPT_VERSIONS:64            return None, None, model_name, None, None, styled_error(f"Unknown version {version}, should be one of {PROMPT_VERSIONS}!")65 66        n_shots = set(n_shots.values())67        if len(n_shots) != 1:68            return None, None, model_name, version, None, styled_error(f"All tasks should have the same number of shots but found {n_shots}!")69        n_shot = list(n_shots)[0]70    except KeyError:71        return None, None, model_name, None, None, styled_error("Wrong configuration file format!")72 73    if len(file_paths) > 0:74        ignored_files = [Path(file_path).name for file_path in file_paths]75        return data, prediction_files, model_name, version, n_shot, styled_warning(f"The following files will be ignored: {ignored_files}")76    return data, prediction_files, model_name, version, n_shot, styled_message("Files parsed successfully, verify that read metadata is correct before submitting")77 78 79def add_new_eval(80    model_training: str,81    maltese_training: str,82    language_count: int,83    configuration: dict,84    prediction_files: dict[str, str],85):86    global REQUESTED_MODELS87    if not REQUESTED_MODELS:88        REQUESTED_MODELS = already_submitted_models(EVAL_REQUESTS_PATH)89 90    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S.%f")91 92    if configuration is None or configuration == {} or prediction_files is None or prediction_files == {}:93        return styled_error("No files selected for upload, please upload an output folder (or wait for the files to finish uploading).")94 95    if model_training is None or model_training == "":96        return styled_error("Please select the model's overall training.")97 98    if maltese_training is None or maltese_training == "":99        return styled_error("Please select the model's Maltese training.")100 101    if language_count is None or language_count < 1:102        language_count = None103 104    model_name, revision, precision, seed, prompt_version, n_shot = get_model_properties(configuration)105    model_id = configuration["model_name"]106 107    # Seems good, creating the eval108    print("Adding new eval")109 110    # Check for duplicate submission111    if f"{model_name}_{revision}_{precision}_{seed}_{prompt_version}_{n_shot}" in REQUESTED_MODELS:112        return styled_warning("This model has been already submitted.")113 114    request = {115        "model": model_id,116        "model_args": dict({tuple(arg.split("=")) for arg in configuration["config"].get("model_args", "").split(",") if len(arg) > 0}),117        "revision": revision,118        "precision": precision,119        "seed": seed,120        "n_shot": n_shot,121        "prompt_version": prompt_version,122        "tasks": list(configuration["configs"].keys()),123        "model_training": model_training,124        "maltese_training": maltese_training,125        "language_count": language_count,126        "submitted_time": current_time,127        "status": "PENDING",128    }129 130    for task_name, file_path in prediction_files.items():131        print(f"Uploading {model_id} {task_name} prediction file")132        API.upload_file(133            path_or_fileobj=file_path,134            path_in_repo=f"{n_shot}-shot_{prompt_version}/{model_name}_{revision}_{precision}/{seed}-seed/samples_{task_name}_{current_time}.jsonl",135            repo_id=PREDICTIONS_REPO,136            repo_type="dataset",137            commit_message=f"Add {configuration['model_name']} {task_name} {n_shot}-shot outputs",138        )139 140    print(f"Creating {model_id} configruation file")141    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{model_name}"142    os.makedirs(OUT_DIR, exist_ok=True)143    out_path = f"{OUT_DIR}/requests_{model_name}_{revision}_{precision}_{n_shot}shot_{prompt_version}_{seed}seed_{current_time}.json"144 145    with open(out_path, "w") as f:146        f.write(json.dumps({"leaderboard": request, "configuration": configuration}, ensure_ascii=False, indent=2))147 148    print(f"Uploading {model_id} configuration file")149    API.upload_file(150        path_or_fileobj=out_path,151        path_in_repo=out_path.split("eval-queue/")[1],152        repo_id=QUEUE_REPO,153        repo_type="dataset",154        commit_message=f"Add {configuration['model_name']} {n_shot}-shot to eval queue",155    )156 157    # Remove the local file158    os.remove(out_path)159 160    return styled_message(161        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."162    )163