autoevaluate/model-evaluator
174
1import os2import time3from pathlib import Path4 5import pandas as pd6import streamlit as st7import yaml8from datasets import get_dataset_config_names9from dotenv import load_dotenv10from huggingface_hub import list_datasets11 12from evaluation import filter_evaluated_models13from utils import (14 AUTOTRAIN_TASK_TO_HUB_TASK,15 commit_evaluation_log,16 create_autotrain_project_name,17 format_col_mapping,18 get_compatible_models,19 get_config_metadata,20 get_dataset_card_url,21 get_key,22 get_metadata,23 http_get,24 http_post,25)26 27if Path(".env").is_file():28 load_dotenv(".env")29 30HF_TOKEN = os.getenv("HF_TOKEN")31AUTOTRAIN_USERNAME = os.getenv("AUTOTRAIN_USERNAME")32AUTOTRAIN_BACKEND_API = os.getenv("AUTOTRAIN_BACKEND_API")33DATASETS_PREVIEW_API = os.getenv("DATASETS_PREVIEW_API")34 35# Put image tasks on top36TASK_TO_ID = {37 "image_binary_classification": 17,38 "image_multi_class_classification": 18,39 "binary_classification": 1,40 "multi_class_classification": 2,41 "natural_language_inference": 22,42 "entity_extraction": 4,43 "extractive_question_answering": 5,44 "translation": 6,45 "summarization": 8,46 "text_zero_shot_classification": 23,47}48 49TASK_TO_DEFAULT_METRICS = {50 "binary_classification": ["f1", "precision", "recall", "auc", "accuracy"],51 "multi_class_classification": [52 "f1",53 "precision",54 "recall",55 "accuracy",56 ],57 "natural_language_inference": ["f1", "precision", "recall", "auc", "accuracy"],58 "entity_extraction": ["precision", "recall", "f1", "accuracy"],59 "extractive_question_answering": ["f1", "exact_match"],60 "translation": ["sacrebleu"],61 "summarization": ["rouge1", "rouge2", "rougeL", "rougeLsum"],62 "image_binary_classification": ["f1", "precision", "recall", "auc", "accuracy"],63 "image_multi_class_classification": [64 "f1",65 "precision",66 "recall",67 "accuracy",68 ],69 "text_zero_shot_classification": ["accuracy", "loss"],70}71 72AUTOTRAIN_TASK_TO_LANG = {73 "translation": "en2de",74 "image_binary_classification": "unk",75 "image_multi_class_classification": "unk",76}77 78AUTOTRAIN_MACHINE = {"text_zero_shot_classification": "r5.16x"}79 80 81SUPPORTED_TASKS = list(TASK_TO_ID.keys())82 83# Extracted from utils.get_supported_metrics84# Hardcoded for now due to speed / caching constraints85SUPPORTED_METRICS = [86 "accuracy",87 "bertscore",88 "bleu",89 "cer",90 "chrf",91 "code_eval",92 "comet",93 "competition_math",94 "coval",95 "cuad",96 "exact_match",97 "f1",98 "frugalscore",99 "google_bleu",100 "mae",101 "mahalanobis",102 "matthews_correlation",103 "mean_iou",104 "meteor",105 "mse",106 "pearsonr",107 "perplexity",108 "precision",109 "recall",110 "roc_auc",111 "rouge",112 "sacrebleu",113 "sari",114 "seqeval",115 "spearmanr",116 "squad",117 "squad_v2",118 "ter",119 "trec_eval",120 "wer",121 "wiki_split",122 "xnli",123 "angelina-wang/directional_bias_amplification",124 "jordyvl/ece",125 "lvwerra/ai4code",126 "lvwerra/amex",127]128 129 130#######131# APP #132#######133st.title("Evaluation on the Hub")134st.warning(135 "**⚠️ This project has been archived. If you want to evaluate LLMs, checkout [this collection](https://huggingface.co/collections/clefourrier/llm-leaderboards-and-benchmarks-✨-64f99d2e11e92ca5568a7cce) of leaderboards.**"136)137st.markdown(138 """139 Welcome to Hugging Face's automatic model evaluator 👋!140 141 This application allows you to evaluate 🤗 Transformers142 [models](https://huggingface.co/models?library=transformers&sort=downloads)143 across a wide variety of [datasets](https://huggingface.co/datasets) on the144 Hub. Please select the dataset and configuration below. The results of your145 evaluation will be displayed on the [public146 leaderboards](https://huggingface.co/spaces/autoevaluate/leaderboards). For147 more details, check out out our [blog148 post](https://huggingface.co/blog/eval-on-the-hub).149 """150)151 152# all_datasets = [d.id for d in list_datasets()]153# query_params = st.experimental_get_query_params()154# if "first_query_params" not in st.session_state:155# st.session_state.first_query_params = query_params156# first_query_params = st.session_state.first_query_params157# default_dataset = all_datasets[0]158# if "dataset" in first_query_params:159# if len(first_query_params["dataset"]) > 0 and first_query_params["dataset"][0] in all_datasets:160# default_dataset = first_query_params["dataset"][0]161 162# selected_dataset = st.selectbox(163# "Select a dataset",164# all_datasets,165# index=all_datasets.index(default_dataset),166# help="""Datasets with metadata can be evaluated with 1-click. Configure an evaluation job to add \167# new metadata to a dataset card.""",168# )169# st.experimental_set_query_params(**{"dataset": [selected_dataset]})170 171# # Check if selected dataset can be streamed172# is_valid_dataset = http_get(173# path="/is-valid",174# domain=DATASETS_PREVIEW_API,175# params={"dataset": selected_dataset},176# ).json()177# if is_valid_dataset["viewer"] is False and is_valid_dataset["preview"] is False:178# st.error(179# """The dataset you selected is not currently supported. Open a \180# [discussion](https://huggingface.co/spaces/autoevaluate/model-evaluator/discussions) for support."""181# )182 183# metadata = get_metadata(selected_dataset, token=HF_TOKEN)184# print(f"INFO -- Dataset metadata: {metadata}")185# if metadata is None:186# st.warning("No evaluation metadata found. Please configure the evaluation job below.")187 188# with st.expander("Advanced configuration"):189# # Select task190# selected_task = st.selectbox(191# "Select a task",192# SUPPORTED_TASKS,193# index=SUPPORTED_TASKS.index(metadata[0]["task_id"]) if metadata is not None else 0,194# help="""Don't see your favourite task here? Open a \195# [discussion](https://huggingface.co/spaces/autoevaluate/model-evaluator/discussions) to request it!""",196# )197# # Select config198# configs = get_dataset_config_names(selected_dataset)199# selected_config = st.selectbox(200# "Select a config",201# configs,202# help="""Some datasets contain several sub-datasets, known as _configurations_. \203# Select one to evaluate your models on. \204# See the [docs](https://huggingface.co/docs/datasets/master/en/load_hub#configurations) for more details.205# """,206# )207# # Some datasets have multiple metadata (one per config), so we grab the one associated with the selected config208# config_metadata = get_config_metadata(selected_config, metadata)209# print(f"INFO -- Config metadata: {config_metadata}")210 211# # Select splits212# splits_resp = http_get(213# path="/splits",214# domain=DATASETS_PREVIEW_API,215# params={"dataset": selected_dataset},216# )217# if splits_resp.status_code == 200:218# split_names = []219# all_splits = splits_resp.json()220# for split in all_splits["splits"]:221# if split["config"] == selected_config:222# split_names.append(split["split"])223 224# if config_metadata is not None:225# eval_split = config_metadata["splits"].get("eval_split", None)226# else:227# eval_split = None228# selected_split = st.selectbox(229# "Select a split",230# split_names,231# index=split_names.index(eval_split) if eval_split is not None else 0,232# help="Be wary when evaluating models on the `train` split.",233# )234 235# # Select columns236# rows_resp = http_get(237# path="/first-rows",238# domain=DATASETS_PREVIEW_API,239# params={240# "dataset": selected_dataset,241# "config": selected_config,242# "split": selected_split,243# },244# ).json()245# col_names = list(pd.json_normalize(rows_resp["rows"][0]["row"]).columns)246 247# st.markdown("**Map your dataset columns**")248# st.markdown(249# """The model evaluator uses a standardised set of column names for the input examples and labels. \250# Please define the mapping between your dataset columns (right) and the standardised column names (left)."""251# )252# col1, col2 = st.columns(2)253 254# # TODO: find a better way to layout these items255# # TODO: need graceful way of handling dataset <--> task mismatch for datasets with metadata256# col_mapping = {}257# if selected_task in ["binary_classification", "multi_class_classification"]:258# with col1:259# st.markdown("`text` column")260# st.text("")261# st.text("")262# st.text("")263# st.text("")264# st.markdown("`target` column")265# with col2:266# text_col = st.selectbox(267# "This column should contain the text to be classified",268# col_names,269# index=col_names.index(get_key(config_metadata["col_mapping"], "text"))270# if config_metadata is not None271# else 0,272# )273# target_col = st.selectbox(274# "This column should contain the labels associated with the text",275# col_names,276# index=col_names.index(get_key(config_metadata["col_mapping"], "target"))277# if config_metadata is not None278# else 0,279# )280# col_mapping[text_col] = "text"281# col_mapping[target_col] = "target"282 283# elif selected_task == "text_zero_shot_classification":284# with col1:285# st.markdown("`text` column")286# st.text("")287# st.text("")288# st.text("")289# st.text("")290# st.markdown("`classes` column")291# st.text("")292# st.text("")293# st.text("")294# st.text("")295# st.markdown("`target` column")296# with col2:297# text_col = st.selectbox(298# "This column should contain the text to be classified",299# col_names,300# index=col_names.index(get_key(config_metadata["col_mapping"], "text"))301# if config_metadata is not None302# else 0,303# )304# classes_col = st.selectbox(305# "This column should contain the classes associated with the text",306# col_names,307# index=col_names.index(get_key(config_metadata["col_mapping"], "classes"))308# if config_metadata is not None309# else 0,310# )311# target_col = st.selectbox(312# "This column should contain the index of the correct class",313# col_names,314# index=col_names.index(get_key(config_metadata["col_mapping"], "target"))315# if config_metadata is not None316# else 0,317# )318# col_mapping[text_col] = "text"319# col_mapping[classes_col] = "classes"320# col_mapping[target_col] = "target"321 322# if selected_task in ["natural_language_inference"]:323# config_metadata = get_config_metadata(selected_config, metadata)324# with col1:325# st.markdown("`text1` column")326# st.text("")327# st.text("")328# st.text("")329# st.text("")330# st.text("")331# st.markdown("`text2` column")332# st.text("")333# st.text("")334# st.text("")335# st.text("")336# st.text("")337# st.markdown("`target` column")338# with col2:339# text1_col = st.selectbox(340# "This column should contain the first text passage to be classified",341# col_names,342# index=col_names.index(get_key(config_metadata["col_mapping"], "text1"))343# if config_metadata is not None344# else 0,345# )346# text2_col = st.selectbox(347# "This column should contain the second text passage to be classified",348# col_names,349# index=col_names.index(get_key(config_metadata["col_mapping"], "text2"))350# if config_metadata is not None351# else 0,352# )353# target_col = st.selectbox(354# "This column should contain the labels associated with the text",355# col_names,356# index=col_names.index(get_key(config_metadata["col_mapping"], "target"))357# if config_metadata is not None358# else 0,359# )360# col_mapping[text1_col] = "text1"361# col_mapping[text2_col] = "text2"362# col_mapping[target_col] = "target"363 364# elif selected_task == "entity_extraction":365# with col1:366# st.markdown("`tokens` column")367# st.text("")368# st.text("")369# st.text("")370# st.text("")371# st.markdown("`tags` column")372# with col2:373# tokens_col = st.selectbox(374# "This column should contain the array of tokens to be classified",375# col_names,376# index=col_names.index(get_key(config_metadata["col_mapping"], "tokens"))377# if config_metadata is not None378# else 0,379# )380# tags_col = st.selectbox(381# "This column should contain the labels associated with each part of the text",382# col_names,383# index=col_names.index(get_key(config_metadata["col_mapping"], "tags"))384# if config_metadata is not None385# else 0,386# )387# col_mapping[tokens_col] = "tokens"388# col_mapping[tags_col] = "tags"389 390# elif selected_task == "translation":391# with col1:392# st.markdown("`source` column")393# st.text("")394# st.text("")395# st.text("")396# st.text("")397# st.markdown("`target` column")398# with col2:399# text_col = st.selectbox(400# "This column should contain the text to be translated",401# col_names,402# index=col_names.index(get_key(config_metadata["col_mapping"], "source"))403# if config_metadata is not None404# else 0,405# )406# target_col = st.selectbox(407# "This column should contain the target translation",408# col_names,409# index=col_names.index(get_key(config_metadata["col_mapping"], "target"))410# if config_metadata is not None411# else 0,412# )413# col_mapping[text_col] = "source"414# col_mapping[target_col] = "target"415 416# elif selected_task == "summarization":417# with col1:418# st.markdown("`text` column")419# st.text("")420# st.text("")421# st.text("")422# st.text("")423# st.markdown("`target` column")424# with col2:425# text_col = st.selectbox(426# "This column should contain the text to be summarized",427# col_names,428# index=col_names.index(get_key(config_metadata["col_mapping"], "text"))429# if config_metadata is not None430# else 0,431# )432# target_col = st.selectbox(433# "This column should contain the target summary",434# col_names,435# index=col_names.index(get_key(config_metadata["col_mapping"], "target"))436# if config_metadata is not None437# else 0,438# )439# col_mapping[text_col] = "text"440# col_mapping[target_col] = "target"441 442# elif selected_task == "extractive_question_answering":443# if config_metadata is not None:444# col_mapping = config_metadata["col_mapping"]445# # Hub YAML parser converts periods to hyphens, so we remap them here446# col_mapping = format_col_mapping(col_mapping)447# with col1:448# st.markdown("`context` column")449# st.text("")450# st.text("")451# st.text("")452# st.text("")453# st.markdown("`question` column")454# st.text("")455# st.text("")456# st.text("")457# st.text("")458# st.markdown("`answers.text` column")459# st.text("")460# st.text("")461# st.text("")462# st.text("")463# st.markdown("`answers.answer_start` column")464# with col2:465# context_col = st.selectbox(466# "This column should contain the question's context",467# col_names,468# index=col_names.index(get_key(col_mapping, "context")) if config_metadata is not None else 0,469# )470# question_col = st.selectbox(471# "This column should contain the question to be answered, given the context",472# col_names,473# index=col_names.index(get_key(col_mapping, "question")) if config_metadata is not None else 0,474# )475# answers_text_col = st.selectbox(476# "This column should contain example answers to the question, extracted from the context",477# col_names,478# index=col_names.index(get_key(col_mapping, "answers.text")) if config_metadata is not None else 0,479# )480# answers_start_col = st.selectbox(481# "This column should contain the indices in the context of the first character of each `answers.text`",482# col_names,483# index=col_names.index(get_key(col_mapping, "answers.answer_start"))484# if config_metadata is not None485# else 0,486# )487# col_mapping[context_col] = "context"488# col_mapping[question_col] = "question"489# col_mapping[answers_text_col] = "answers.text"490# col_mapping[answers_start_col] = "answers.answer_start"491# elif selected_task in ["image_binary_classification", "image_multi_class_classification"]:492# with col1:493# st.markdown("`image` column")494# st.text("")495# st.text("")496# st.text("")497# st.text("")498# st.markdown("`target` column")499# with col2:500# image_col = st.selectbox(501# "This column should contain the images to be classified",502# col_names,503# index=col_names.index(get_key(config_metadata["col_mapping"], "image"))504# if config_metadata is not None505# else 0,506# )507# target_col = st.selectbox(508# "This column should contain the labels associated with the images",509# col_names,510# index=col_names.index(get_key(config_metadata["col_mapping"], "target"))511# if config_metadata is not None512# else 0,513# )514# col_mapping[image_col] = "image"515# col_mapping[target_col] = "target"516 517# # Select metrics518# st.markdown("**Select metrics**")519# st.markdown("The following metrics will be computed")520# html_string = " ".join(521# [522# '<div style="padding-right:5px;padding-left:5px;padding-top:5px;padding-bottom:5px;float:left">'523# + '<div style="background-color:#D3D3D3;border-radius:5px;display:inline-block;padding-right:5px;'524# + 'padding-left:5px;color:white">'525# + metric526# + "</div></div>"527# for metric in TASK_TO_DEFAULT_METRICS[selected_task]528# ]529# )530# st.markdown(html_string, unsafe_allow_html=True)531# selected_metrics = st.multiselect(532# "(Optional) Select additional metrics",533# sorted(list(set(SUPPORTED_METRICS) - set(TASK_TO_DEFAULT_METRICS[selected_task]))),534# help="""User-selected metrics will be computed with their default arguments. \535# For example, `f1` will report results for binary labels. \536# Check out the [available metrics](https://huggingface.co/metrics) for more details.""",537# )538 539# with st.form(key="form"):540# compatible_models = get_compatible_models(selected_task, [selected_dataset])541# selected_models = st.multiselect(542# "Select the models you wish to evaluate",543# compatible_models,544# help="""Don't see your favourite model in this list? Add the dataset and task it was trained on to the \545# [model card metadata.](https://huggingface.co/docs/hub/models-cards#model-card-metadata)""",546# )547# print("INFO -- Selected models before filter:", selected_models)548 549# hf_username = st.text_input("Enter your 🤗 Hub username to be notified when the evaluation is finished")550 551# submit_button = st.form_submit_button("Evaluate models 🚀")552 553# if submit_button:554# if len(hf_username) == 0:555# st.warning("No 🤗 Hub username provided! Please enter your username and try again.")556# elif len(selected_models) == 0:557# st.warning("⚠️ No models were selected for evaluation! Please select at least one model and try again.")558# elif len(selected_models) > 10:559# st.warning("Only 10 models can be evaluated at once. Please select fewer models and try again.")560# else:561# # Filter out previously evaluated models562# selected_models = filter_evaluated_models(563# selected_models,564# selected_task,565# selected_dataset,566# selected_config,567# selected_split,568# selected_metrics,569# )570# print("INFO -- Selected models after filter:", selected_models)571# if len(selected_models) > 0:572# project_payload = {573# "username": AUTOTRAIN_USERNAME,574# "proj_name": create_autotrain_project_name(selected_dataset, selected_config),575# "task": TASK_TO_ID[selected_task],576# "config": {577# "language": AUTOTRAIN_TASK_TO_LANG[selected_task]578# if selected_task in AUTOTRAIN_TASK_TO_LANG579# else "en",580# "max_models": 5,581# "instance": {582# "provider": "sagemaker" if selected_task in AUTOTRAIN_MACHINE.keys() else "ovh",583# "instance_type": AUTOTRAIN_MACHINE[selected_task]584# if selected_task in AUTOTRAIN_MACHINE.keys()585# else "p3",586# "max_runtime_seconds": 172800,587# "num_instances": 1,588# "disk_size_gb": 200,589# },590# "evaluation": {591# "metrics": selected_metrics,592# "models": selected_models,593# "hf_username": hf_username,594# },595# },596# }597# print(f"INFO -- Payload: {project_payload}")598# project_json_resp = http_post(599# path="/projects/create",600# payload=project_payload,601# token=HF_TOKEN,602# domain=AUTOTRAIN_BACKEND_API,603# ).json()604# print(f"INFO -- Project creation response: {project_json_resp}")605 606# if project_json_resp["created"]:607# data_payload = {608# "split": 4, # use "auto" split choice in AutoTrain609# "col_mapping": col_mapping,610# "load_config": {"max_size_bytes": 0, "shuffle": False},611# "dataset_id": selected_dataset,612# "dataset_config": selected_config,613# "dataset_split": selected_split,614# }615# data_json_resp = http_post(616# path=f"/projects/{project_json_resp['id']}/data/dataset",617# payload=data_payload,618# token=HF_TOKEN,619# domain=AUTOTRAIN_BACKEND_API,620# ).json()621# print(f"INFO -- Dataset creation response: {data_json_resp}")622# if data_json_resp["download_status"] == 1:623# train_json_resp = http_post(624# path=f"/projects/{project_json_resp['id']}/data/start_processing",625# token=HF_TOKEN,626# domain=AUTOTRAIN_BACKEND_API,627# ).json()628# # For local development we process and approve projects on-the-fly629# if "localhost" in AUTOTRAIN_BACKEND_API:630# with st.spinner("⏳ Waiting for data processing to complete ..."):631# is_data_processing_success = False632# while is_data_processing_success is not True:633# project_status = http_get(634# path=f"/projects/{project_json_resp['id']}",635# token=HF_TOKEN,636# domain=AUTOTRAIN_BACKEND_API,637# ).json()638# if project_status["status"] == 3:639# is_data_processing_success = True640# time.sleep(10)641 642# # Approve training job643# train_job_resp = http_post(644# path=f"/projects/{project_json_resp['id']}/start_training",645# token=HF_TOKEN,646# domain=AUTOTRAIN_BACKEND_API,647# ).json()648# st.success("✅ Data processing and project approval complete - go forth and evaluate!")649# else:650# # Prod/staging submissions are evaluated in a cron job via run_evaluation_jobs.py651# print(f"INFO -- AutoTrain job response: {train_json_resp}")652# if train_json_resp["success"]:653# train_eval_index = {654# "train-eval-index": [655# {656# "config": selected_config,657# "task": AUTOTRAIN_TASK_TO_HUB_TASK[selected_task],658# "task_id": selected_task,659# "splits": {"eval_split": selected_split},660# "col_mapping": col_mapping,661# }662# ]663# }664# selected_metadata = yaml.dump(train_eval_index, sort_keys=False)665# dataset_card_url = get_dataset_card_url(selected_dataset)666# st.success("✅ Successfully submitted evaluation job!")667# st.markdown(668# f"""669# Evaluation can take up to 1 hour to complete, so grab a ☕️ or 🍵 while you wait:670 671# * 🔔 A [Hub pull request](https://huggingface.co/docs/hub/repositories-pull-requests-discussions) with the evaluation results will be opened for each model you selected. Check your email for notifications.672# * 📊 Click [here](https://hf.co/spaces/autoevaluate/leaderboards?dataset={selected_dataset}) to view the results from your submission once the Hub pull request is merged.673# * 🥱 Tired of configuring evaluations? Add the following metadata to the [dataset card]({dataset_card_url}) to enable 1-click evaluations:674# """ # noqa675# )676# st.markdown(677# f"""678# ```yaml679# {selected_metadata}680# """681# )682# print("INFO -- Pushing evaluation job logs to the Hub")683# evaluation_log = {}684# evaluation_log["project_id"] = project_json_resp["id"]685# evaluation_log["autotrain_env"] = (686# "staging" if "staging" in AUTOTRAIN_BACKEND_API else "prod"687# )688# evaluation_log["payload"] = project_payload689# evaluation_log["project_creation_response"] = project_json_resp690# evaluation_log["dataset_creation_response"] = data_json_resp691# evaluation_log["autotrain_job_response"] = train_json_resp692# commit_evaluation_log(evaluation_log, hf_access_token=HF_TOKEN)693# else:694# st.error("🙈 Oh no, there was an error submitting your evaluation job!")695# else:696# st.warning("⚠️ No models left to evaluate! Please select other models and try again.")697