autoevaluate/model-evaluator

1import copy2from dataclasses import dataclass3 4import streamlit as st5from huggingface_hub import DatasetFilter, HfApi6from huggingface_hub.hf_api import DatasetInfo7 8 9@dataclass(frozen=True, eq=True)10class EvaluationInfo:11    task: str12    model: str13    dataset_name: str14    dataset_config: str15    dataset_split: str16    metrics: set17 18 19def create_evaluation_info(dataset_info: DatasetInfo) -> int:20    if dataset_info.cardData is not None:21        metadata = dataset_info.cardData["eval_info"]22        metadata.pop("col_mapping", None)23        # TODO(lewtun): populate dataset cards with metric info24        if "metrics" not in metadata:25            metadata["metrics"] = frozenset()26        else:27            metadata["metrics"] = frozenset(metadata["metrics"])28        return EvaluationInfo(**metadata)29 30 31def get_evaluation_infos():32    evaluation_datasets = []33    filt = DatasetFilter(author="autoevaluate")34    autoevaluate_datasets = HfApi().list_datasets(filter=filt, full=True)35    for dset in autoevaluate_datasets:36        try:37            evaluation_datasets.append(create_evaluation_info(dset))38        except Exception as e:39            print(f"Error processing dataset {dset}: {e}")40    return evaluation_datasets41 42 43def filter_evaluated_models(models, task, dataset_name, dataset_config, dataset_split, metrics):44    evaluation_infos = get_evaluation_infos()45    models_to_filter = copy.copy(models)46 47    for model in models_to_filter:48        evaluation_info = EvaluationInfo(49            task=task,50            model=model,51            dataset_name=dataset_name,52            dataset_config=dataset_config,53            dataset_split=dataset_split,54            metrics=frozenset(metrics),55        )56        if evaluation_info in evaluation_infos:57            st.info(58                f"Model [`{model}`](https://huggingface.co/{model}) has already been evaluated on this configuration. \59                    This model will be excluded from the evaluation job..."60            )61            models.remove(model)62 63    return models64