AMR-KELEG/MLADI
0
1import os2import subprocess3import streamlit as st4 5import datasets6from tqdm import tqdm7from transformers import AutoModelForSequenceClassification, AutoTokenizer8 9from constants import DIALECTS_WITH_LABELS10from inspect import getmembers, isfunction11import eval_utils12import utils13import numpy as np14import pandas as pd15from sklearn.metrics import accuracy_score, f1_score, recall_score, precision_score16 17from huggingface_hub import HfApi18 19api = HfApi()20 21st.set_page_config(layout="wide")22st.title("MLADI Leaderboard")23st.write(24 "The Multi-label Arabic Dialect Identification (MLADI) leaderboard serves as a public interface for benchmarking ADI "25 "models using an 'extended version' of the NADI 2024 test set, "26 "the first multi-label country-level ADI dataset.\n\n"27 "🔜 More information can be found through the following paper: [Revisiting Common Assumptions about Arabic Dialects in NLP](https://arxiv.org/abs/2505.21816)\n\n"28)29 30SHARED_TASK_TEAMS = {31 "Elyadata": "https://aclanthology.org/2024.arabicnlp-1.85/",32 "NLP_DI": "https://aclanthology.org/2024.arabicnlp-1.82/",33 "dzNlp": "https://aclanthology.org/2024.arabicnlp-1.84/",34}35tab1, tab2 = st.tabs(["Leaderboard", "Submit a Model"])36with tab1:37 # Load the labels38 dataset_name = os.environ["DATASET_NAME"]39 dataset = datasets.load_dataset(dataset_name)["test"]40 labels = {dialect: dataset[dialect] for dialect in DIALECTS_WITH_LABELS}41 42 print("Loaded the labels, no. of samples:", len(dataset))43 44 # Load the models' predictions45 try:46 model_predictions_rows = datasets.load_dataset(47 os.environ["PREDICTIONS_DATASET_NAME"]48 )["train"]49 50 except Exception as e:51 st.info(f"Error in loading the results!")52 print(e)53 model_predictions_rows = []54 55 if model_predictions_rows:56 # TODO: Store these metrics in a separate dataset!57 evaluation_metrics = []58 for row in model_predictions_rows:59 # Evaluate the models60 accuracy_scores = {}61 f1_scores = {}62 recall_scores = {}63 precision_scores = {}64 predictions = row["predictions"]65 66 if not row["status"] == "completed":67 continue68 69 for dialect in DIALECTS_WITH_LABELS:70 y_true = labels[dialect]71 y_pred = [dialect in prediction for prediction in predictions]72 accuracy = accuracy_score(y_true, y_pred)73 f1 = f1_score(y_true, y_pred)74 recall = recall_score(y_true, y_pred)75 precision = precision_score(y_true, y_pred)76 77 accuracy_scores[dialect] = accuracy78 f1_scores[dialect] = f179 recall_scores[dialect] = recall80 precision_scores[dialect] = precision81 82 macro_avg_accuracy = np.mean(list(accuracy_scores.values()))83 macro_avg_f1 = np.mean(list(f1_scores.values()))84 macro_avg_recall = np.mean(list(recall_scores.values()))85 macro_avg_precision = np.mean(list(precision_scores.values()))86 87 evaluation_metrics.append(88 {89 "Model Name": row["model_name"],90 "Accuracy": macro_avg_accuracy,91 "Recall": macro_avg_recall,92 "Precision": macro_avg_precision,93 "F1 score": macro_avg_f1,94 "Inference Method": row["inference_function"],95 "URL": f"https://huggingface.co/{row['model_name']}"96 if ("shared task team" not in row["model_name"])97 else SHARED_TASK_TEAMS[row["model_name"].split(" (")[0]],98 "Commit ID": row["commit_id"][:5]99 if ("shared task team" not in row["model_name"])100 else "N/A",101 }102 )103 104 if evaluation_metrics:105 results_df = pd.DataFrame(evaluation_metrics).sort_values(106 "F1 score", ascending=False107 )108 results_df["Rank"] = range(1, len(results_df) + 1)109 110 results_df = results_df[111 [112 "Rank",113 "Model Name",114 "F1 score",115 "Precision",116 "Recall",117 "Accuracy",118 "Inference Method",119 "URL",120 "Commit ID",121 ]122 ]123 st.data_editor(124 results_df,125 column_config={126 "URL": st.column_config.LinkColumn("URL", required=False),127 },128 hide_index=True,129 )130 st.write("Note: The metrics are macro-averaged across all 11 dialects.")131 132 with st.expander("Click for more information."):133 inference_functions_names = [134 func_name for func_name, _ in getmembers(eval_utils, isfunction)135 ]136 # Show the docstring of the inference functions137 inference_functions_docstring = [138 getattr(eval_utils, func).__doc__ for func in inference_functions_names139 ]140 141 inference_functions_df = pd.DataFrame(142 {143 "Method": inference_functions_names,144 "Description": inference_functions_docstring,145 }146 )147 st.markdown("## Inference Methods' Descriptions", unsafe_allow_html=True)148 st.markdown(149 inference_functions_df.to_markdown(index=False), unsafe_allow_html=True150 )151 152 with open("leaderboard_info.md", "r") as f:153 MARKDOWN_TEXT = f.read()154 st.markdown(MARKDOWN_TEXT)155 st.markdown("For any inquiries, please do not hesistate to contact me: https://amr-keleg.github.io/")156 157 with st.expander("Cite this leaderboard!"):158 st.write(159 """160 Please cite the following paper in which we introduced the NADI 2024 evaluation sets:161 ```162 @inproceedings{keleg-etal-2025-revisiting,163 title = "Revisiting Common Assumptions about Arabic Dialects in NLP",164 author = "Keleg, Amr and165 Goldwater, Sharon and166 Magdy, Walid",167 booktitle = "Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics",168 month = july,169 year = "2025",170 address = "Vienna, Austria",171 publisher = "Association for Computational Linguistics",172 }173 174 @inproceedings{abdul-mageed-etal-2024-nadi,175 title = "{NADI} 2024: The Fifth Nuanced {A}rabic Dialect Identification Shared Task",176 author = "Abdul-Mageed, Muhammad and177 Keleg, Amr and178 Elmadany, AbdelRahim and179 Zhang, Chiyu and180 Hamed, Injy and181 Magdy, Walid and182 Bouamor, Houda and183 Habash, Nizar",184 editor = "Habash, Nizar and185 Bouamor, Houda and186 Eskander, Ramy and187 Tomeh, Nadi and188 Abu Farha, Ibrahim and189 Abdelali, Ahmed and190 Touileb, Samia and191 Hamed, Injy and192 Onaizan, Yaser and193 Alhafni, Bashar and194 Antoun, Wissam and195 Khalifa, Salam and196 Haddad, Hatem and197 Zitouni, Imed and198 AlKhamissi, Badr and199 Almatham, Rawan and200 Mrini, Khalil",201 booktitle = "Proceedings of The Second Arabic Natural Language Processing Conference",202 month = aug,203 year = "2024",204 address = "Bangkok, Thailand",205 publisher = "Association for Computational Linguistics",206 url = "https://aclanthology.org/2024.arabicnlp-1.79",207 doi = "10.18653/v1/2024.arabicnlp-1.79",208 pages = "709--728",209 }210 ```211 """212 )213 214 # Evaluate the models queued215 if model_predictions_rows:216 models_to_be_evaluated = []217 models_in_progress = []218 219 for row in model_predictions_rows:220 if row["status"] == "queued":221 models_to_be_evaluated.append(row)222 elif row["status"] == "in_progress":223 models_in_progress.append(row)224 225 for model in models_in_progress:226 # Check if the evaluation is staled for more than a day!227 timestamp = model["last_updated_timestamp"]228 if utils.current_seconds_time() - timestamp > 86400:229 utils.update_model_queue(230 repo_id=os.environ["PREDICTIONS_DATASET_NAME"],231 model_name=model["model_name"],232 commit_id=model["commit_id"],233 inference_function=model["inference_function"],234 status="queued",235 )236 print(f"Model {model['model_name']} is staled for more than a day.")237 models_to_be_evaluated.append(model)238 models_in_progress.remove(model)239 240 if models_in_progress == []:241 for row in models_to_be_evaluated:242 # Evaluate the model243 subprocess.Popen(244 [245 "python",246 "background_inference.py",247 row["model_name"],248 row["commit_id"],249 row["inference_function"],250 ]251 )252 print(f"Started the evaluation of {row['model_name']}.")253 254with tab2:255 model_name = st.text_input("Enter a model's name on HF")256 model_revision = st.text_input(257 "Enter a model's revision on HF (commit id, or branch name)",258 placeholder="main",259 value="main",260 )261 inference_functions_names = [262 func_name for func_name, _ in getmembers(eval_utils, isfunction)263 ]264 inference_function = st.selectbox(265 "Inference Method",266 inference_functions_names,267 )268 269 # TODO: Allow modifying the adhoc threshold values of the different inference methods270 271 # Show the docstring of the inference functions272 inference_functions_docstring = [273 getattr(eval_utils, func).__doc__ for func in inference_functions_names274 ]275 276 inference_functions_df = pd.DataFrame(277 {278 "Method": inference_functions_names,279 "Description": inference_functions_docstring,280 }281 )282 with st.expander("Check the inference methods' short descriptions"):283 st.markdown(284 inference_functions_df.to_markdown(index=False), unsafe_allow_html=True285 )286 st.write(287 "Note: We are happy to discuss adding new custom inference methods for your models."288 )289 290 if model_name and model_revision and inference_function:291 # Get the model's commit id292 commit_id = api.list_repo_commits(model_name, revision=model_revision)[293 0294 ].commit_id295 296 model_predictions_rows = datasets.load_dataset(297 os.environ["PREDICTIONS_DATASET_NAME"]298 )["train"]299 300 # Check if the model is already in the leaderboard301 model_exists = any(302 [303 row["model_name"] == model_name304 and row["commit_id"] == commit_id305 and row["inference_function"] == inference_function306 for row in model_predictions_rows307 ]308 )309 310 if not model_exists:311 # Add the model to the evaluation queue312 utils.update_model_queue(313 repo_id=os.environ["PREDICTIONS_DATASET_NAME"],314 model_name=model_name,315 commit_id=commit_id,316 inference_function=inference_function,317 status="queued",318 )319 st.info(320 f"The evaluation of the model {model_name} is queued for processing."321 )322 323 else:324 st.info(325 f"The model {model_name} has already submitted to the leaderboard before."326 )327