autoevaluate/model-evaluator

1import inspect2import uuid3from typing import Dict, List, Union4 5import jsonlines6import requests7import streamlit as st8from evaluate import load9from huggingface_hub import HfApi, ModelFilter, Repository, dataset_info, list_metrics10from tqdm import tqdm11 12AUTOTRAIN_TASK_TO_HUB_TASK = {13    "binary_classification": "text-classification",14    "multi_class_classification": "text-classification",15    "natural_language_inference": "text-classification",16    "entity_extraction": "token-classification",17    "extractive_question_answering": "question-answering",18    "translation": "translation",19    "summarization": "summarization",20    "image_binary_classification": "image-classification",21    "image_multi_class_classification": "image-classification",22    "text_zero_shot_classification": "text-generation",23}24 25 26HUB_TASK_TO_AUTOTRAIN_TASK = {v: k for k, v in AUTOTRAIN_TASK_TO_HUB_TASK.items()}27LOGS_REPO = "evaluation-job-logs"28 29 30def get_auth_headers(token: str, prefix: str = "Bearer"):31    return {"Authorization": f"{prefix} {token}"}32 33 34def http_post(path: str, token: str, payload=None, domain: str = None, params=None) -> requests.Response:35    """HTTP POST request to the AutoNLP API, raises UnreachableAPIError if the API cannot be reached"""36    try:37        response = requests.post(38            url=domain + path,39            json=payload,40            headers=get_auth_headers(token=token),41            allow_redirects=True,42            params=params,43        )44    except requests.exceptions.ConnectionError:45        print("❌ Failed to reach AutoNLP API, check your internet connection")46    response.raise_for_status()47    return response48 49 50def http_get(path: str, domain: str, token: str = None, params: dict = None) -> requests.Response:51    """HTTP POST request to `path`, raises UnreachableAPIError if the API cannot be reached"""52    try:53        response = requests.get(54            url=domain + path,55            headers=get_auth_headers(token=token),56            allow_redirects=True,57            params=params,58        )59    except requests.exceptions.ConnectionError:60        print(f"❌ Failed to reach {path}, check your internet connection")61    response.raise_for_status()62    return response63 64 65def get_metadata(dataset_name: str, token: str) -> Union[Dict, None]:66    data = dataset_info(dataset_name, token=token)67    if data.cardData is not None and "train-eval-index" in data.cardData.keys():68        return data.cardData["train-eval-index"]69    else:70        return None71 72 73def get_compatible_models(task: str, dataset_ids: List[str]) -> List[str]:74    """75    Returns all model IDs that are compatible with the given task and dataset names.76 77    Args:78        task (`str`): The task to search for.79        dataset_names (`List[str]`): A list of dataset names to search for.80 81    Returns:82        A list of model IDs, sorted alphabetically.83    """84    compatible_models = []85    # Allow any summarization model to be used for summarization tasks86    # and allow any text-generation model to be used for text_zero_shot_classification87    if task in ("summarization", "text_zero_shot_classification"):88        model_filter = ModelFilter(89            task=AUTOTRAIN_TASK_TO_HUB_TASK[task],90            library=["transformers", "pytorch"],91        )92        compatible_models.extend(HfApi().list_models(filter=model_filter))93    # Include models trained on SQuAD datasets, since these can be evaluated on94    # other SQuAD-like datasets95    if task == "extractive_question_answering":96        dataset_ids.extend(["squad", "squad_v2"])97 98    # TODO: relax filter on PyTorch models if TensorFlow supported in AutoTrain99    for dataset_id in dataset_ids:100        model_filter = ModelFilter(101            task=AUTOTRAIN_TASK_TO_HUB_TASK[task],102            trained_dataset=dataset_id,103            library=["transformers", "pytorch"],104        )105        compatible_models.extend(HfApi().list_models(filter=model_filter))106    return sorted(set([model.modelId for model in compatible_models]))107 108 109def get_key(col_mapping, val):110    for key, value in col_mapping.items():111        if val == value:112            return key113 114    return "key doesn't exist"115 116 117def format_col_mapping(col_mapping: dict) -> dict:118    for k, v in col_mapping["answers"].items():119        col_mapping[f"answers.{k}"] = f"answers.{v}"120    del col_mapping["answers"]121    return col_mapping122 123 124def commit_evaluation_log(evaluation_log, hf_access_token=None):125    logs_repo_url = f"https://huggingface.co/datasets/autoevaluate/{LOGS_REPO}"126    logs_repo = Repository(127        local_dir=LOGS_REPO,128        clone_from=logs_repo_url,129        repo_type="dataset",130        use_auth_token=hf_access_token,131    )132    logs_repo.git_pull()133    with jsonlines.open(f"{LOGS_REPO}/logs.jsonl") as r:134        lines = []135        for obj in r:136            lines.append(obj)137 138    lines.append(evaluation_log)139    with jsonlines.open(f"{LOGS_REPO}/logs.jsonl", mode="w") as writer:140        for job in lines:141            writer.write(job)142    logs_repo.push_to_hub(143        commit_message=f"Evaluation submitted with project name {evaluation_log['payload']['proj_name']}"144    )145    print("INFO -- Pushed evaluation logs to the Hub")146 147 148@st.experimental_memo149def get_supported_metrics():150    """Helper function to get all metrics compatible with evaluation service.151 152    Requires all metric dependencies installed in the same environment, so wait until153    https://github.com/huggingface/evaluate/issues/138 is resolved before using this.154    """155    metrics = [metric.id for metric in list_metrics()]156    supported_metrics = []157    for metric in tqdm(metrics):158        # TODO: this currently requires all metric dependencies to be installed159        # in the same environment. Refactor to avoid needing to actually load160        # the metric.161        try:162            print(f"INFO -- Attempting to load metric: {metric}")163            metric_func = load(metric)164        except Exception as e:165            print(e)166            print("WARNING -- Skipping the following metric, which cannot load:", metric)167            continue168 169        argspec = inspect.getfullargspec(metric_func.compute)170        if "references" in argspec.kwonlyargs and "predictions" in argspec.kwonlyargs:171            # We require that "references" and "predictions" are arguments172            # to the metric function. We also require that the other arguments173            # besides "references" and "predictions" have defaults and so do not174            # need to be specified explicitly.175            defaults = True176            for key, value in argspec.kwonlydefaults.items():177                if key not in ("references", "predictions"):178                    if value is None:179                        defaults = False180                        break181 182            if defaults:183                supported_metrics.append(metric)184    return supported_metrics185 186 187def get_dataset_card_url(dataset_id: str) -> str:188    """Gets the URL to edit the dataset card for the given dataset ID."""189    if "/" in dataset_id:190        return f"https://huggingface.co/datasets/{dataset_id}/edit/main/README.md"191    else:192        return f"https://github.com/huggingface/datasets/edit/master/datasets/{dataset_id}/README.md"193 194 195def create_autotrain_project_name(dataset_id: str, dataset_config: str) -> str:196    """Creates an AutoTrain project name for the given dataset ID."""197    # Project names cannot have "/", so we need to format community datasets accordingly198    dataset_id_formatted = dataset_id.replace("/", "__")199    dataset_config_formatted = dataset_config.replace("--", "__")200    # Project names need to be unique, so we append a random string to guarantee this while adhering to naming rules201    basename = f"eval-{dataset_id_formatted}-{dataset_config_formatted}"202    basename = basename[:60] if len(basename) > 60 else basename  # Hub naming limitation203    return f"{basename}-{str(uuid.uuid4())[:6]}"204 205 206def get_config_metadata(config: str, metadata: List[Dict] = None) -> Union[Dict, None]:207    """Gets the dataset card metadata for the given config."""208    if metadata is None:209        return None210    config_metadata = [m for m in metadata if m["config"] == config]211    if len(config_metadata) >= 1:212        return config_metadata[0]213    else:214        return None215