HadrienByr/model-evaluator

1import copy2from dataclasses import dataclass3 4import streamlit as st5from huggingface_hub import DatasetFilter, HfApi6from huggingface_hub.hf_api import DatasetInfo7 8 9@dataclass(frozen=True, eq=True)10class EvaluationInfo:11    task: str12    model: str13    dataset_name: str14    dataset_config: str15    dataset_split: str16    metrics: set17 18 19def create_evaluation_info(dataset_info: DatasetInfo) -> int:20    if dataset_info.cardData is not None:21        metadata = dataset_info.cardData["eval_info"]22        metadata.pop("col_mapping", None)23        # TODO(lewtun): populate dataset cards with metric info24        if "metrics" not in metadata:25            metadata["metrics"] = frozenset()26        else:27            metadata["metrics"] = frozenset(metadata["metrics"])28        return EvaluationInfo(**metadata)29 30 31def get_evaluation_infos():32    filt = DatasetFilter(author="autoevaluate")33    evaluation_datasets = HfApi().list_datasets(filter=filt, full=True)34    return [create_evaluation_info(dset) for dset in evaluation_datasets]35 36 37def filter_evaluated_models(models, task, dataset_name, dataset_config, dataset_split, metrics):38    evaluation_infos = get_evaluation_infos()39    models_to_filter = copy.copy(models)40 41    for model in models_to_filter:42        evaluation_info = EvaluationInfo(43            task=task,44            model=model,45            dataset_name=dataset_name,46            dataset_config=dataset_config,47            dataset_split=dataset_split,48            metrics=frozenset(metrics),49        )50        if evaluation_info in evaluation_infos:51            st.info(52                f"Model [`{model}`](https://huggingface.co/{model}) has already been evaluated on this configuration. \53                    This model will be excluded from the evaluation job..."54            )55            models.remove(model)56 57    return models58