MLRS/MELABench

1import re2 3from dataclasses import dataclass4from enum import Enum5 6@dataclass7class TaskDetails:8    name: str9    display_name: str = ""10    symbol: str = "" # emoji11 12 13class TaskType(Enum):14    NLU = TaskDetails("nlu", "NLU", "🧠")15    NLG = TaskDetails("nlg", "NLG", "✍️")16 17 18@dataclass19class Task:20    benchmark: str21    metric: str22    col_name: str23    url: str24    task_type: TaskType25    is_primary_metric: bool = True26    zero_shot_only: bool = False27 28 29# Select your tasks here30# ---------------------------------------------------31class Tasks(Enum):32    # task_key in the json file, metric_key in the json file, name to display in the leaderboard33    task0 = Task("sentiment_mlt", "f1", "Sentiment Analysis (F1)", "https://github.com/jerbarnes/typology_of_crosslingual/tree/master/data/sentiment/mt", TaskType.NLU)34    task1 = Task("sib200_mlt", "f1", "SIB200 (F1)", "https://huggingface.co/datasets/Davlan/sib200/viewer/mlt_Latn", TaskType.NLU)35    task2 = Task("taxi1500_mlt", "f1", "Taxi1500 (F1)", "https://github.com/cisnlp/Taxi1500", TaskType.NLU)36    task3 = Task("maltese_news_categories", "loglikelihood", "Maltese News Categories (F1)", "https://huggingface.co/datasets/MLRS/maltese_news_categories", TaskType.NLU)37    task4 = Task("multieurlex_mlt", "loglikelihood", "MultiEURLEX (F1)", "https://huggingface.co/datasets/nlpaueb/multi_eurlex", TaskType.NLU)38    task5 = Task("belebele_mlt", "acc", "Belebele (Accuracy)", "https://huggingface.co/datasets/facebook/belebele/viewer/mlt_Latn", TaskType.NLU, zero_shot_only=True)39    task6 = Task("opus100_eng-mlt", "bleu", "OPUS-100 EN→MT (BLEU)", "https://huggingface.co/datasets/MLRS/OPUS-MT-EN-Fixed", TaskType.NLG, False)40    task7 = Task("opus100_eng-mlt", "chrf", "OPUS-100 EN→MT (ChrF)", "https://huggingface.co/datasets/MLRS/OPUS-MT-EN-Fixed", TaskType.NLG)41    task8 = Task("flores200_eng-mlt", "bleu", "Flores-200 EN→MT (BLEU)", "https://huggingface.co/datasets/Muennighoff/flores200", TaskType.NLG, False)42    task9 = Task("flores200_eng-mlt", "chrf", "Flores-200 EN→MT (ChrF)", "https://huggingface.co/datasets/Muennighoff/flores200", TaskType.NLG)43    task10 = Task("webnlg_mlt", "chrf", "WebNLG (ChrF)", "https://synalp.gitlabpages.inria.fr/webnlg-challenge/challenge_2023/", TaskType.NLG)44    task11 = Task("webnlg_mlt", "rouge", "WebNLG (Rouge-L)", "https://synalp.gitlabpages.inria.fr/webnlg-challenge/challenge_2023/", TaskType.NLG, False)45    task12 = Task("eurlexsum_mlt", "chrf", "EUR-Lex-Sum (ChrF)", "https://huggingface.co/datasets/dennlinger/eur-lex-sum", TaskType.NLG, False)46    task13 = Task("eurlexsum_mlt", "rouge", "EUR-Lex-Sum (Rouge-L)", "https://huggingface.co/datasets/dennlinger/eur-lex-sum", TaskType.NLG)47    task14 = Task("maltese_news_headlines", "chrf", "Maltese News Headlines (ChrF)", "https://huggingface.co/datasets/MLRS/maltese_news_headlines", TaskType.NLG, False)48    task15 = Task("maltese_news_headlines", "rouge", "Maltese News Headlines (Rouge-L)", "https://huggingface.co/datasets/MLRS/maltese_news_headlines", TaskType.NLG)49 50NUM_FEWSHOT = 0 # Change with your few shot51# ---------------------------------------------------52 53 54 55# Your leaderboard name56TITLE = """57<h1 align="center" id="space-title">58<img src="https://raw.githubusercontent.com/MLRS/MELABench/refs/heads/main/logo.jpg" alt="MELABench logo" width="200px">59Leaderboard60</h1>61"""62 63# What does your leaderboard evaluate?64INTRODUCTION_TEXT = """65<p align="center">A Maltese Evaluation Language Benchmark 🇲🇹</p>66"""67 68# Which evaluations are you running? how can people reproduce what you have?69tasks = {task_type.value.display_name: {} for task_type in TaskType}70for task in Tasks:71    tasks[task.value.task_type.value.display_name][re.sub(r" \(.*\)$", "", task.value.col_name)] = task.value.url72LLM_BENCHMARKS_TEXT = f"""73MELABench evaluates language model capabilities on Maltese.74Currently, the following tasks are supported:75""" + \76"\n".join([77    f"- {task_type}:\n" + "\n".join(f"  - [{task}]({url})" for task, url in sub_tasks.items()) + "\n"78    for task_type, sub_tasks in tasks.items()79]) + \80"""81The leaderboard is developed and maintained by people managing [MLRS](https://mlrs.research.um.edu.mt/).82We plan to expand our initial work with more tasks, if you would like to contribute your data, please reach out!83If you would like to include results for models/setups we did not include, we also accept submissions.84 85This work was introduced in [MELABenchv1: Benchmarking Large Language Models against Smaller Fine-Tuned Models for Low-Resource Maltese NLP](https://arxiv.org/abs/2506.04385).86"""87 88EVALUATION_QUEUE_TEXT = """89To include new results on this benchmark, follow the instructions on our [GitHub Repository](https://github.com/MLRS/MELABench/tree/main/prompting).90You can then upload the output files which should include the configuration/results file and all the prediction files.91In addition, we ask for additional metadata about model training. 92"""93 94CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"95CITATION_BUTTON_TEXT = r"""96@inproceedings{micallef-borg-2025-melabenchv1,97    title = "{MELAB}enchv1: Benchmarking Large Language Models against Smaller Fine-Tuned Models for Low-Resource {M}altese {NLP}",98    author = "Micallef, Kurt  and99      Borg, Claudia",100    editor = "Che, Wanxiang  and101      Nabende, Joyce  and102      Shutova, Ekaterina  and103      Pilehvar, Mohammad Taher",104    booktitle = "Findings of the Association for Computational Linguistics: ACL 2025",105    month = jul,106    year = "2025",107    address = "Vienna, Austria",108    publisher = "Association for Computational Linguistics",109    url = "https://aclanthology.org/2025.findings-acl.1053/",110    doi = "10.18653/v1/2025.findings-acl.1053",111    pages = "20505--20527",112    ISBN = "979-8-89176-256-5",113}114"""115