inception42/Leaderboards
49
1import os2import json3import math4import numpy as np5import pandas as pd6import gradio as gr7from huggingface_hub import HfApi, hf_hub_download8 9 10OWNER = "inceptionai"11 12ARAGEN_REQUESTS_REPO_ID = f"{OWNER}/aragen-requests-dataset"13HINDIGEN_REQUESTS_REPO_ID = f"{OWNER}/hindigen-requests-dataset"14IFEVAL_REQUESTS_REPO_ID = f"{OWNER}/arabicifeval-requests-dataset"15 16 17HEADER = """18<center>19<br></br>20<h1>Multilingual Leaderboards 🌍</h1>21<h2>Generative Evaluation for Global South</h2>22<br></br>23</center>24"""25 26ABOUT_SECTION = """27## About28 29In our `12-24` release, we introduced the **AraGen Benchmark**, along with the **3C3H** evaluation measure (aka the 3C3H Score). You can find more details about AraGen and 3C3H [here](https://huggingface.co/blog/leaderboard-3c3h-aragen). The first versions of the benchmark, **AraGen-12-24** and **AraGen-03-25 (v2)**, are publicly available in the [`inceptionai/AraGen`](https://huggingface.co/datasets/inceptionai/AraGen) dataset. The current AraGen leaderboard in this Space is powered by **AraGen-v3**.30 31Building on that foundation, we extend our evaluation beyond Arabic, introducing **HindiGen**, a generative benchmark for Hindi that will follow the same release philosophy as AraGen. The current **HindiGen-v1** powers the HindiGen leaderboards here; a future **HindiGen-v2** release will be publicly shared along with the v1 dataset.32 33In this release, we present three main leaderboards:34 35**AraGen-v3:**36 37- The AraGen Benchmark is designed to evaluate and compare the performance of Chat/Instruct Arabic Large Language Models on a suite of generative tasks that are culturally relevant to the Arab region, history, politics, cuisine, and more. By leveraging **3C3H** as an evaluation metric—which assesses a model's output across six dimensions: Correctness, Completeness, Conciseness, Helpfulness, Honesty, and Harmlessness—the leaderboard offers a comprehensive and holistic evaluation of a model’s chat capabilities and its ability to generate human-like and ethically responsible content.38 39**HindiGen-v1:**40 41- The HindiGen Benchmark evaluates Chat/Instruct LLMs on Hindi generative tasks such as question answering, grammar, and safety. It follows the same 3C3H evaluation methodology and bootstrapped confidence intervals, enabling statistically grounded comparisons between models on culturally and linguistically rich Hindi content.42 43**Instruction Following (IFEval – Arabic & English):**44 45- We have established a robust leaderboard that benchmarks models on Arabic and English instruction following, offering an open and comparative performance landscape for the research community. Concurrently, we released the first publicly available Arabic [dataset](https://huggingface.co/datasets/inceptionai/Arabic_IFEval) aimed at evaluating LLMs' ability to follow instructions. The Arabic IFEval samples are meticulously curated to capture the language’s unique nuances—such as diacritization and distinctive phonetic features—often overlooked in generic datasets. Our dedicated linguistic team generated original samples and adapted selections from the IFEval English dataset, ensuring that the material resonates with Arabic cultural contexts and meets the highest standards of authenticity and quality.46 47### Why Focus on Chat Models?48 49Our evaluations are conducted in a generative mode, meaning that we expect models to produce complete, context-rich responses rather than simply predicting the next token as base models do. This approach not only yields results that are more explainable and nuanced compared to logit-based measurements, but it also captures elements like creativity, coherence, and ethical considerations—providing deeper insights into overall model performance.50 51### Contact52 53For inquiries or assistance, please join the conversation on our [Discussions Tab](https://huggingface.co/spaces/inceptionai/Leaderboards/discussions) or reach out via [email](mailto:ali.filali@inceptionai.ai).54"""55 56BOTTOM_LOGO = """<img src="https://huggingface.co/spaces/inceptionai/Arabic-Leaderboards/resolve/main/assets/pictures/03-25/arabic-leaderboards-colab-march-preview-free-3.png" style="width:50%;display:block;margin-left:auto;margin-right:auto;border-radius:15px;">"""57 58CITATION_BUTTON_TEXT = """59@misc{leaderboards,60 author = {El Filali, Ali and Albarri, Sarah and Kamboj, Samta and Sengupta, Neha and Nakov, Preslav and Abouelseoud, Arwa},61 title = {Multilingual Leaderboards: Generative Evaluation for Global South},62 year = {2025},63 publisher = {Inception},64 howpublished = "url{https://huggingface.co/spaces/inceptionai/Leaderboards}"65}66"""67 68CITATION_BUTTON_LABEL = """69Copy the following snippet to cite the results from all Arabic Leaderboards in this Space.70"""71 72 73def extract_score_value(entry):74 """75 Helper to extract (value, lower, upper) from both old v2 format (float)76 and new v3/v1 formats (dict with "value"/"lower"/"upper").77 All values are returned in [0, 1] space; caller can convert to percentages.78 79 We use the "value" field as the point estimate.80 """81 if entry is None:82 return (math.nan, math.nan, math.nan)83 84 # Old format: just a float85 if isinstance(entry, (int, float)):86 v = float(entry)87 return (v, math.nan, math.nan)88 89 # New format: dict with "value", "lower", "upper"90 if isinstance(entry, dict):91 v = float(entry.get("value", math.nan))92 lower = entry.get("lower", math.nan)93 upper = entry.get("upper", math.nan)94 lower = float(lower) if isinstance(lower, (int, float)) else math.nan95 upper = float(upper) if isinstance(upper, (int, float)) else math.nan96 return (v, lower, upper)97 98 return (math.nan, math.nan, math.nan)99 100 101def compute_leaderboard_3c3h(df_3c3h_base: pd.DataFrame) -> pd.DataFrame:102 """103 Build the 3C3H leaderboard with:104 - Rank (by 3C3H Score)105 - Rank Spread (based on 3C3H Score CI)106 - 95% CI (±) for 3C3H Score (only)107 - Model Size Filter108 109 All scores are in percentage space.110 """111 df = df_3c3h_base.copy()112 113 # Model size filter helper114 max_model_size_value = 1000115 df["Model Size Filter"] = df["Model Size"].replace(np.inf, max_model_size_value)116 117 # Sort & rank by 3C3H Score (point estimate)118 if "3C3H Score" in df.columns:119 df = df.sort_values(by="3C3H Score", ascending=False)120 df = df.reset_index(drop=True)121 df.insert(0, "Rank", range(1, len(df) + 1))122 123 # Rank Spread based on 3C3H Score CI124 main_col = "3C3H Score"125 lower_col = "3C3H Score Lower"126 upper_col = "3C3H Score Upper"127 128 # Effective lower/upper: if not present, fall back to point estimate129 if lower_col in df.columns:130 lower_eff = df[lower_col].copy()131 else:132 lower_eff = df[main_col].copy()133 134 if upper_col in df.columns:135 upper_eff = df[upper_col].copy()136 else:137 upper_eff = df[main_col].copy()138 139 # order of base scenario: all models at their point estimates (value-based)140 sort_desc = df.sort_values(by=main_col, ascending=False)141 score_order = sort_desc[main_col].values # descending142 143 def rank_position(x, order):144 """145 Given a value x and a descending array 'order',146 return the rank index where x would land147 if all others stayed as in 'order'.148 149 Rank = 1 + number of scores strictly greater than x.150 """151 if np.isnan(x):152 return math.nan153 154 # Ignore NaNs in the score order155 valid = order[~np.isnan(order)]156 if valid.size == 0:157 return math.nan158 159 # 'valid' is descending; count how many scores are strictly greater than x160 num_greater = np.sum(valid > x)161 rank = num_greater + 1162 163 # Clamp rank to [1, len(valid)] for numerical safety164 if rank < 1:165 rank = 1166 elif rank > len(valid):167 rank = len(valid)168 169 return int(rank)170 171 best_ranks = []172 worst_ranks = []173 for low, high in zip(lower_eff.values, upper_eff.values):174 best = rank_position(high, score_order) # optimistic: use upper bound175 worst = rank_position(low, score_order) # pessimistic: use lower bound176 best_ranks.append(best)177 worst_ranks.append(worst)178 179 spread = []180 for b, w in zip(best_ranks, worst_ranks):181 if math.isnan(b) or math.isnan(w):182 spread.append("-")183 else:184 spread.append(f"{int(b)} <--> {int(w)}")185 df.insert(1, "Rank Spread", spread)186 187 # 95% CI (±) for 3C3H Score only (in percentage space)188 if lower_col in df.columns and upper_col in df.columns:189 ci = (df[upper_col] - df[lower_col]) / 2.0190 df["95% CI (±)"] = ci.round(4)191 else:192 df["95% CI (±)"] = np.nan193 194 # Round score columns195 score_columns_3c3h = [196 "3C3H Score",197 "Correctness",198 "Completeness",199 "Conciseness",200 "Helpfulness",201 "Honesty",202 "Harmlessness",203 ]204 for col in score_columns_3c3h:205 if col in df.columns:206 df[col] = df[col].round(4)207 208 df["95% CI (±)"] = df["95% CI (±)"].round(4)209 210 return df211 212 213def load_results(benchmark="aragen"):214 """215 Loads results for the given benchmark.216 217 benchmark:218 - "aragen" -> uses aragen_v3_results.json (or v2 fallback)219 - "hindigen" -> uses hindigen_v1_results.json220 221 Supports:222 - old v2 format (simple floats)223 - new v3/v1 format (dict with value/lower/upper)224 225 Returns:226 df_3c3h : 3C3H leaderboard dataframe (with Rank, Rank Spread, 95% CI (±))227 df_tasks : tasks leaderboard dataframe228 task_columns: list of task score columns229 """230 current_dir = os.path.dirname(os.path.abspath(__file__))231 232 if benchmark == "hindigen":233 results_file = os.path.join(current_dir, "assets", "results", "hindigen_v1_results.json")234 else:235 v3_file = os.path.join(current_dir, "assets", "results", "aragen_v3_results.json")236 v2_file = os.path.join(current_dir, "assets", "results", "aragen_v2_results.json")237 if os.path.exists(v3_file):238 results_file = v3_file239 else:240 results_file = v2_file241 242 with open(results_file, "r", encoding="utf-8") as f:243 data = json.load(f)244 245 # Filter out entries that only contain "_last_sync_timestamp"246 filtered_data = []247 for entry in data:248 if len(entry.keys()) == 1 and "_last_sync_timestamp" in entry:249 continue250 filtered_data.append(entry)251 data = filtered_data252 253 data_3c3h = []254 data_tasks = []255 256 for model_data in data:257 meta = model_data.get("Meta", {})258 model_name = meta.get("Model Name", "UNK")259 revision = meta.get("Revision", "UNK")260 precision = meta.get("Precision", "UNK")261 license_ = meta.get("License", "UNK")262 params = meta.get("Params", "UNK")263 264 # Parse model size265 try:266 model_size_numeric = float(params)267 except Exception:268 model_size_numeric = np.inf269 270 # Find the key that holds the scores (e.g. "claude-3-7-sonnet-20250219 Scores", "claude-3.5-sonnet Scores")271 scores_key = None272 for k in model_data.keys():273 if k.endswith("Scores"):274 scores_key = k275 break276 277 scores_data = model_data.get(scores_key, {}) if scores_key else {}278 scores_3c3h = scores_data.get("3C3H Scores", {})279 scores_tasks = scores_data.get("Tasks Scores", {})280 281 # --- 3C3H entry ---282 entry3 = {283 "Model Name": model_name,284 "Revision": revision,285 "License": license_,286 "Precision": precision,287 "Model Size": model_size_numeric,288 }289 290 for metric_name, metric_entry in scores_3c3h.items():291 v, lower, upper = extract_score_value(metric_entry)292 # Point estimate (percentage)293 entry3[metric_name] = v * 100 if not math.isnan(v) else np.nan294 295 # Only keep lower/upper for 3C3H Score (for CI & Rank Spread)296 if metric_name == "3C3H Score":297 entry3["3C3H Score Lower"] = (298 lower * 100 if not math.isnan(lower) else np.nan299 )300 entry3["3C3H Score Upper"] = (301 upper * 100 if not math.isnan(upper) else np.nan302 )303 304 data_3c3h.append(entry3)305 306 # --- Tasks entry ---307 entryt = {308 "Model Name": model_name,309 "Revision": revision,310 "License": license_,311 "Precision": precision,312 "Model Size": model_size_numeric,313 }314 315 for task_name, task_entry in scores_tasks.items():316 v, _, _ = extract_score_value(task_entry)317 entryt[task_name] = v * 100 if not math.isnan(v) else np.nan318 319 data_tasks.append(entryt)320 321 df_3c3h_base = pd.DataFrame(data_3c3h)322 df_tasks_base = pd.DataFrame(data_tasks)323 324 # Build 3C3H leaderboard (rank, rank spread, CI, size filter)325 df_3c3h = compute_leaderboard_3c3h(df_3c3h_base)326 327 # Build tasks leaderboard (no weighted average, no rank spread, no CI)328 if df_tasks_base.empty:329 df_tasks = df_tasks_base.copy()330 task_columns = []331 else:332 meta_cols_tasks = [333 "Model Name",334 "Revision",335 "License",336 "Precision",337 "Model Size",338 ]339 task_columns = [340 col341 for col in df_tasks_base.columns342 if col not in meta_cols_tasks343 ]344 345 df_tasks = df_tasks_base.copy()346 347 # Round task scores348 if task_columns:349 df_tasks[task_columns] = df_tasks[task_columns].round(4)350 351 # Model size filter352 max_model_size_value = 1000353 df_tasks["Model Size Filter"] = df_tasks["Model Size"].replace(354 np.inf, max_model_size_value355 )356 357 # Sort & rank: based on the first task (typically Question Answering (QA))358 if task_columns:359 first_task = task_columns[0]360 df_tasks = df_tasks.sort_values(by=first_task, ascending=False)361 else:362 df_tasks = df_tasks.sort_values(by="Model Name", ascending=True)363 364 df_tasks = df_tasks.reset_index(drop=True)365 df_tasks.insert(0, "Rank", range(1, len(df_tasks) + 1))366 367 return df_3c3h, df_tasks, task_columns368 369 370def load_if_data():371 """372 Loads the instruction-following data from ifeval_results.jsonl 373 and returns a dataframe with relevant columns, 374 converting decimal values to percentage format.375 """376 current_dir = os.path.dirname(os.path.abspath(__file__))377 results_file = os.path.join(current_dir, "assets", "results", "ifeval_results.jsonl")378 379 data = []380 with open(results_file, "r", encoding="utf-8") as f:381 for line in f:382 line = line.strip()383 if not line:384 continue385 data.append(json.loads(line))386 387 df = pd.DataFrame(data)388 389 # Convert numeric columns390 numeric_cols = ["En Prompt-lvl", "En Instruction-lvl", "Ar Prompt-lvl", "Ar Instruction-lvl"]391 for col in numeric_cols:392 df[col] = pd.to_numeric(df[col], errors="coerce")393 394 # Compute average accuracy for En and Ar395 df["Average Accuracy (En)"] = (df["En Prompt-lvl"] + df["En Instruction-lvl"]) / 2396 df["Average Accuracy (Ar)"] = (df["Ar Prompt-lvl"] + df["Ar Instruction-lvl"]) / 2397 398 # Convert them to percentage format (e.g., 0.871 -> 87.1)399 for col in numeric_cols:400 df[col] = (df[col] * 100).round(1)401 df["Average Accuracy (En)"] = (df["Average Accuracy (En)"] * 100).round(1)402 df["Average Accuracy (Ar)"] = (df["Average Accuracy (Ar)"] * 100).round(1)403 404 # Handle size as numeric405 def parse_size(x):406 try:407 return float(x)408 except:409 return np.inf410 411 df["Model Size"] = df["Size (B)"].apply(parse_size)412 413 # Add a filter column for size414 max_model_size_value = 1000415 df["Model Size Filter"] = df["Model Size"].replace(np.inf, max_model_size_value)416 417 # Sort by "Average Accuracy (Ar)" as an example418 df = df.sort_values(by="Average Accuracy (Ar)", ascending=False)419 df = df.reset_index(drop=True)420 df.insert(0, "Rank", range(1, len(df) + 1))421 422 return df423 424 425def submit_model(model_name, revision, precision, params, license, modality, leaderboards_selected):426 """427 Submits a model to one or more leaderboards:428 - AraGen -> inceptionai/aragen-requests-dataset429 - HindiGen -> inceptionai/hindigen-requests-dataset430 - IFEval -> inceptionai/arabicifeval-requests-dataset431 432 User must choose at least one leaderboard.433 """434 if not leaderboards_selected:435 return "**Error:** You must choose at least one leaderboard (AraGen, HindiGen, and/or IFEval)."436 437 # Normalize precision438 if precision == "Missing":439 precision_norm = None440 else:441 precision_norm = precision.strip().lower() if precision else None442 443 repo_map = {444 "AraGen": ARAGEN_REQUESTS_REPO_ID,445 "HindiGen": HINDIGEN_REQUESTS_REPO_ID,446 "IFEval": IFEVAL_REQUESTS_REPO_ID,447 }448 449 # Map leaderboards that use the 3C3H JSON result files (for dedup vs results)450 results_benchmark_map = {451 "AraGen": "aragen",452 "HindiGen": "hindigen",453 }454 455 api = HfApi()456 457 # Validate model exists on HuggingFace Hub once458 try:459 _ = api.model_info(model_name)460 except Exception:461 return f"**Error: Could not find model '{model_name}' on HuggingFace Hub. Please ensure the model name is correct and the model is public.**"462 463 org_model = model_name.split("/")464 if len(org_model) != 2:465 return "**Please enter the full model name including the organization or username, e.g., 'inceptionai/jais-family-30b-8k'**"466 org, model_id = org_model467 468 hf_api_token = os.environ.get("HF_API_TOKEN", None)469 470 # Dedup & upload per leaderboard471 success_targets = []472 skipped_targets = []473 errors = []474 475 for leaderboard in leaderboards_selected:476 repo_id = repo_map.get(leaderboard)477 if repo_id is None:478 errors.append(f"- Unknown leaderboard: {leaderboard}")479 continue480 481 # Deduplicate against existing results (only for AraGen/HindiGen)482 already_evaluated = False483 if leaderboard in results_benchmark_map:484 df_3c3h_lb, _, _ = load_results(results_benchmark_map[leaderboard])485 if not df_3c3h_lb.empty:486 existing_models_results = df_3c3h_lb[["Model Name", "Revision", "Precision"]]487 model_exists_in_results = (488 (existing_models_results["Model Name"] == model_name)489 & (existing_models_results["Revision"] == revision)490 & (existing_models_results["Precision"] == (precision_norm if precision_norm is not None else existing_models_results["Precision"]))491 ).any()492 if model_exists_in_results:493 skipped_targets.append(494 f"- **{leaderboard}**: Model already appears in the leaderboard results."495 )496 already_evaluated = True497 498 # Deduplicate against pending/finished requests in this repo499 def load_req(status_folder):500 return load_requests(repo_id, status_folder)501 502 df_pending = load_req("pending")503 df_finished = load_req("finished")504 505 if not already_evaluated:506 if not df_pending.empty:507 existing_models_pending = df_pending[["model_name", "revision", "precision"]]508 model_exists_in_pending = (509 (existing_models_pending["model_name"] == model_name)510 & (existing_models_pending["revision"] == revision)511 & (existing_models_pending["precision"] == precision_norm)512 ).any()513 if model_exists_in_pending:514 skipped_targets.append(515 f"- **{leaderboard}**: Model is already in pending evaluations."516 )517 already_evaluated = True518 519 if not already_evaluated:520 if not df_finished.empty:521 existing_models_finished = df_finished[["model_name", "revision", "precision"]]522 model_exists_in_finished = (523 (existing_models_finished["model_name"] == model_name)524 & (existing_models_finished["revision"] == revision)525 & (existing_models_finished["precision"] == precision_norm)526 ).any()527 if model_exists_in_finished:528 skipped_targets.append(529 f"- **{leaderboard}**: Model has already been evaluated (finished)."530 )531 already_evaluated = True532 533 if already_evaluated:534 continue535 536 # Prepare submission JSON537 status = "PENDING"538 submission = {539 "model_name": model_name,540 "license": license,541 "revision": revision,542 "precision": precision_norm,543 "params": params,544 "status": status,545 "modality": modality,546 "leaderboard": leaderboard,547 }548 submission_json = json.dumps(submission, indent=2)549 550 precision_str = precision_norm if precision_norm else "Missing"551 file_path_in_repo = f"pending/{org}/{model_id}_eval_request_{revision}_{precision_str}.json"552 553 try:554 api.upload_file(555 path_or_fileobj=submission_json.encode("utf-8"),556 path_in_repo=file_path_in_repo,557 repo_id=repo_id,558 repo_type="dataset",559 token=hf_api_token,560 )561 success_targets.append(leaderboard)562 except Exception as e:563 errors.append(f"- **{leaderboard}**: Error while submitting – {str(e)}")564 565 # Build user-facing message566 messages = []567 if success_targets:568 messages.append(569 f"✅ Model **'{model_name}'** has been submitted for evaluation to: "570 + ", ".join(f"**{lb}**" for lb in success_targets)571 + "."572 )573 if skipped_targets:574 messages.append("⚠️ Skipped submissions:\n" + "\n".join(skipped_targets))575 if errors:576 messages.append("❌ Errors:\n" + "\n".join(errors))577 578 if not messages:579 return "**No submissions were made.** Please check if the model is already pending or evaluated."580 581 return "\n\n".join(messages)582 583 584def load_requests(repo_id, status_folder):585 """586 Loads request JSON files from a given dataset repo and status folder:587 status_folder in {"pending", "finished", "failed"}588 """589 api = HfApi()590 requests_data = []591 592 hf_api_token = os.environ.get("HF_API_TOKEN", None)593 594 try:595 files_info = api.list_repo_files(596 repo_id=repo_id,597 repo_type="dataset",598 token=hf_api_token,599 )600 except Exception as e:601 print(f"Error accessing dataset repository {repo_id}: {e}")602 return pd.DataFrame()603 604 files_in_folder = [605 f for f in files_info if f.startswith(f"{status_folder}/") and f.endswith(".json")606 ]607 608 for file_path in files_in_folder:609 try:610 local_file_path = hf_hub_download(611 repo_id=repo_id,612 filename=file_path,613 repo_type="dataset",614 token=hf_api_token,615 )616 with open(local_file_path, "r") as f:617 request = json.load(f)618 requests_data.append(request)619 except Exception as e:620 print(f"Error loading file {file_path}: {e}")621 continue622 623 df = pd.DataFrame(requests_data)624 return df625 626 627# ---------- FILTER HELPERS (AraGen) ----------628 629def filter_df_3c3h(630 search_query,631 selected_cols,632 precision_filters,633 license_filters,634 min_size,635 max_size,636):637 # AraGen 3C3H638 df_3c3h, _, _ = load_results("aragen")639 df_ = df_3c3h.copy()640 641 # Sanity check on size range642 if min_size > max_size:643 min_size, max_size = max_size, min_size644 645 # Text search646 if search_query:647 df_ = df_[df_["Model Name"].str.contains(search_query, case=False, na=False)]648 649 # Precision filtering650 if precision_filters:651 include_missing = "Missing" in precision_filters652 selected_precisions = [p for p in precision_filters if p != "Missing"]653 if include_missing:654 df_ = df_[655 (df_["Precision"].isin(selected_precisions))656 | (df_["Precision"] == "UNK")657 | (df_["Precision"].isna())658 ]659 else:660 df_ = df_[df_["Precision"].isin(selected_precisions)]661 662 # License filtering663 if license_filters:664 include_missing = "Missing" in license_filters665 selected_licenses = [l for l in license_filters if l != "Missing"]666 if include_missing:667 df_ = df_[668 (df_["License"].isin(selected_licenses))669 | (df_["License"] == "UNK")670 | (df_["License"].isna())671 ]672 else:673 df_ = df_[df_["License"].isin(selected_licenses)]674 675 # Model size filter676 df_ = df_[677 (df_["Model Size Filter"] >= min_size) & (df_["Model Size Filter"] <= max_size)678 ]679 680 # Keep global Rank / Rank Spread; just reset the index681 df_ = df_.reset_index(drop=True)682 683 # Column ordering684 fixed_column_order = [685 "Rank",686 "Rank Spread",687 "Model Name",688 "3C3H Score",689 "95% CI (±)",690 "Correctness",691 "Completeness",692 "Conciseness",693 "Helpfulness",694 "Honesty",695 "Harmlessness",696 "Revision",697 "License",698 "Precision",699 "Model Size",700 ]701 702 selected_cols = [703 col704 for col in fixed_column_order705 if col in selected_cols and col in df_.columns706 ]707 708 return df_[selected_cols]709 710 711def filter_df_tasks(712 search_query,713 selected_cols,714 precision_filters,715 license_filters,716 min_size,717 max_size,718 task_columns,719):720 # AraGen tasks721 _, df_tasks, _ = load_results("aragen")722 df_ = df_tasks.copy()723 724 if min_size > max_size:725 min_size, max_size = max_size, min_size726 727 if search_query:728 df_ = df_[df_["Model Name"].str.contains(search_query, case=False, na=False)]729 730 if precision_filters:731 include_missing = "Missing" in precision_filters732 selected_precisions = [p for p in precision_filters if p != "Missing"]733 if include_missing:734 df_ = df_[735 (df_["Precision"].isin(selected_precisions))736 | (df_["Precision"] == "UNK")737 | (df_["Precision"].isna())738 ]739 else:740 df_ = df_[df_["Precision"].isin(selected_precisions)]741 742 if license_filters:743 include_missing = "Missing" in license_filters744 selected_licenses = [l for l in license_filters if l != "Missing"]745 if include_missing:746 df_ = df_[747 (df_["License"].isin(selected_licenses))748 | (df_["License"] == "UNK")749 | (df_["License"].isna())750 ]751 else:752 df_ = df_[df_["License"].isin(selected_licenses)]753 754 df_ = df_[755 (df_["Model Size Filter"] >= min_size) & (df_["Model Size Filter"] <= max_size)756 ]757 758 # Re-rank within filtered subset using first task as sort key759 if "Rank" in df_.columns:760 df_ = df_.drop(columns=["Rank"])761 762 if task_columns:763 first_task = task_columns[0]764 if first_task in df_.columns:765 df_ = df_.sort_values(by=first_task, ascending=False)766 else:767 df_ = df_.sort_values(by="Model Name", ascending=True)768 else:769 df_ = df_.sort_values(by="Model Name", ascending=True)770 771 df_ = df_.reset_index(drop=True)772 df_.insert(0, "Rank", range(1, len(df_) + 1))773 774 fixed_column_order = [775 "Rank",776 "Model Name",777 "Question Answering (QA)",778 "Orthographic and Grammatical Analysis",779 "Safety",780 "Reasoning",781 "Revision",782 "License",783 "Precision",784 "Model Size",785 ]786 787 selected_cols = [788 col for col in fixed_column_order if col in selected_cols and col in df_.columns789 ]790 return df_[selected_cols]791 792 793# ---------- FILTER HELPERS (HindiGen) ----------794 795def filter_df_3c3h_hindigen(796 search_query,797 selected_cols,798 precision_filters,799 license_filters,800 min_size,801 max_size,802):803 df_3c3h_hi, _, _ = load_results("hindigen")804 df_ = df_3c3h_hi.copy()805 806 if min_size > max_size:807 min_size, max_size = max_size, min_size808 809 if search_query:810 df_ = df_[df_["Model Name"].str.contains(search_query, case=False, na=False)]811 812 if precision_filters:813 include_missing = "Missing" in precision_filters814 selected_precisions = [p for p in precision_filters if p != "Missing"]815 if include_missing:816 df_ = df_[817 (df_["Precision"].isin(selected_precisions))818 | (df_["Precision"] == "UNK")819 | (df_["Precision"].isna())820 ]821 else:822 df_ = df_[df_["Precision"].isin(selected_precisions)]823 824 if license_filters:825 include_missing = "Missing" in license_filters826 selected_licenses = [l for l in license_filters if l != "Missing"]827 if include_missing:828 df_ = df_[829 (df_["License"].isin(selected_licenses))830 | (df_["License"] == "UNK")831 | (df_["License"].isna())832 ]833 else:834 df_ = df_[df_["License"].isin(selected_licenses)]835 836 df_ = df_[837 (df_["Model Size Filter"] >= min_size) & (df_["Model Size Filter"] <= max_size)838 ]839 840 df_ = df_.reset_index(drop=True)841 842 fixed_column_order = [843 "Rank",844 "Rank Spread",845 "Model Name",846 "3C3H Score",847 "95% CI (±)",848 "Correctness",849 "Completeness",850 "Conciseness",851 "Helpfulness",852 "Honesty",853 "Harmlessness",854 "Revision",855 "License",856 "Precision",857 "Model Size",858 ]859 860 selected_cols = [861 col862 for col in fixed_column_order863 if col in selected_cols and col in df_.columns864 ]865 866 return df_[selected_cols]867 868 869def filter_df_tasks_hindigen(870 search_query,871 selected_cols,872 precision_filters,873 license_filters,874 min_size,875 max_size,876 task_columns,877):878 _, df_tasks_hi, _ = load_results("hindigen")879 df_ = df_tasks_hi.copy()880 881 if min_size > max_size:882 min_size, max_size = max_size, min_size883 884 if search_query:885 df_ = df_[df_["Model Name"].str.contains(search_query, case=False, na=False)]886 887 if precision_filters:888 include_missing = "Missing" in precision_filters889 selected_precisions = [p for p in precision_filters if p != "Missing"]890 if include_missing:891 df_ = df_[892 (df_["Precision"].isin(selected_precisions))893 | (df_["Precision"] == "UNK")894 | (df_["Precision"].isna())895 ]896 else:897 df_ = df_[df_["Precision"].isin(selected_precisions)]898 899 if license_filters:900 include_missing = "Missing" in license_filters901 selected_licenses = [l for l in license_filters if l != "Missing"]902 if include_missing:903 df_ = df_[904 (df_["License"].isin(selected_licenses))905 | (df_["License"] == "UNK")906 | (df_["License"].isna())907 ]908 else:909 df_ = df_[df_["License"].isin(selected_licenses)]910 911 df_ = df_[912 (df_["Model Size Filter"] >= min_size) & (df_["Model Size Filter"] <= max_size)913 ]914 915 if "Rank" in df_.columns:916 df_ = df_.drop(columns=["Rank"])917 918 if task_columns:919 first_task = task_columns[0]920 if first_task in df_.columns:921 df_ = df_.sort_values(by=first_task, ascending=False)922 else:923 df_ = df_.sort_values(by="Model Name", ascending=True)924 else:925 df_ = df_.sort_values(by="Model Name", ascending=True)926 927 df_ = df_.reset_index(drop=True)928 df_.insert(0, "Rank", range(1, len(df_) + 1))929 930 fixed_column_order = [931 "Rank",932 "Model Name",933 "Question Answering (QA)",934 "Grammar",935 "Safety",936 "Revision",937 "License",938 "Precision",939 "Model Size",940 ]941 942 selected_cols = [943 col for col in fixed_column_order if col in selected_cols and col in df_.columns944 ]945 return df_[selected_cols]946 947 948def filter_if_df(search_query, selected_cols, family_filters, min_size, max_size):949 """950 Filters the instruction-following dataframe based on various criteria.951 We have removed 'Filter by Type' and 'Filter by Creator'.952 """953 df_ = load_if_data().copy()954 if min_size > max_size:955 min_size, max_size = max_size, min_size956 957 # Search by model name958 if search_query:959 df_ = df_[df_["Model Name"].str.contains(search_query, case=False, na=False)]960 961 # Filter by Family only (Creator and Type filters removed)962 if family_filters:963 df_ = df_[df_["Family"].isin(family_filters)]964 965 # Filter by Model Size966 df_ = df_[967 (df_["Model Size Filter"] >= min_size) & (df_["Model Size Filter"] <= max_size)968 ]969 970 # Re-rank within the filtered subset971 if "Rank" in df_.columns:972 df_ = df_.drop(columns=["Rank"])973 df_ = df_.reset_index(drop=True)974 df_.insert(0, "Rank", range(1, len(df_) + 1))975 976 fixed_column_order = [977 "Rank",978 "Model Name",979 "Average Accuracy (Ar)",980 "Ar Prompt-lvl",981 "Ar Instruction-lvl",982 "Average Accuracy (En)",983 "En Prompt-lvl",984 "En Instruction-lvl",985 "Type",986 "Creator",987 "Family",988 "Size (B)",989 "Base Model",990 "Context Window",991 "Lang.",992 ]993 994 selected_cols = [995 col for col in fixed_column_order if col in selected_cols and col in df_.columns996 ]997 return df_[selected_cols]998 999 1000def main():1001 # Load AraGen, HindiGen, and IFEval data1002 df_3c3h_ar, df_tasks_ar, task_columns_ar = load_results("aragen")1003 df_3c3h_hi, df_tasks_hi, task_columns_hi = load_results("hindigen")1004 df_if = load_if_data() # Instruction Following DF1005 1006 # ---------- AraGen options ----------1007 precision_options_3c3h = sorted(df_3c3h_ar["Precision"].dropna().unique().tolist())1008 precision_options_3c3h = [p for p in precision_options_3c3h if p != "UNK"]1009 precision_options_3c3h.append("Missing")1010 1011 license_options_3c3h = sorted(df_3c3h_ar["License"].dropna().unique().tolist())1012 license_options_3c3h = [l for l in license_options_3c3h if l != "UNK"]1013 license_options_3c3h.append("Missing")1014 1015 precision_options_tasks = sorted(df_tasks_ar["Precision"].dropna().unique().tolist())1016 precision_options_tasks = [p for p in precision_options_tasks if p != "UNK"]1017 precision_options_tasks.append("Missing")1018 1019 license_options_tasks = sorted(df_tasks_ar["License"].dropna().unique().tolist())1020 license_options_tasks = [l for l in license_options_tasks if l != "UNK"]1021 license_options_tasks.append("Missing")1022 1023 min_model_size_3c3h = int(df_3c3h_ar["Model Size Filter"].min())1024 max_model_size_3c3h = int(df_3c3h_ar["Model Size Filter"].max())1025 1026 min_model_size_tasks = int(df_tasks_ar["Model Size Filter"].min())1027 max_model_size_tasks = int(df_tasks_ar["Model Size Filter"].max())1028 1029 column_choices_3c3h = [1030 col1031 for col in df_3c3h_ar.columns.tolist()1032 if col1033 not in [1034 "Model Size Filter",1035 "3C3H Score Lower",1036 "3C3H Score Upper",1037 ]1038 ]1039 1040 column_choices_tasks = [1041 col1042 for col in df_tasks_ar.columns.tolist()1043 if col != "Model Size Filter"1044 ]1045 1046 # ---------- HindiGen options ----------1047 precision_options_3c3h_hi = sorted(df_3c3h_hi["Precision"].dropna().unique().tolist())1048 precision_options_3c3h_hi = [p for p in precision_options_3c3h_hi if p != "UNK"]1049 precision_options_3c3h_hi.append("Missing")1050 1051 license_options_3c3h_hi = sorted(df_3c3h_hi["License"].dropna().unique().tolist())1052 license_options_3c3h_hi = [l for l in license_options_3c3h_hi if l != "UNK"]1053 license_options_3c3h_hi.append("Missing")1054 1055 precision_options_tasks_hi = sorted(df_tasks_hi["Precision"].dropna().unique().tolist())1056 precision_options_tasks_hi = [p for p in precision_options_tasks_hi if p != "UNK"]1057 precision_options_tasks_hi.append("Missing")1058 1059 license_options_tasks_hi = sorted(df_tasks_hi["License"].dropna().unique().tolist())1060 license_options_tasks_hi = [l for l in license_options_tasks_hi if l != "UNK"]1061 license_options_tasks_hi.append("Missing")1062 1063 min_model_size_3c3h_hi = int(df_3c3h_hi["Model Size Filter"].min())1064 max_model_size_3c3h_hi = int(df_3c3h_hi["Model Size Filter"].max())1065 1066 min_model_size_tasks_hi = int(df_tasks_hi["Model Size Filter"].min())1067 max_model_size_tasks_hi = int(df_tasks_hi["Model Size Filter"].max())1068 1069 column_choices_3c3h_hi = [1070 col1071 for col in df_3c3h_hi.columns.tolist()1072 if col1073 not in [1074 "Model Size Filter",1075 "3C3H Score Lower",1076 "3C3H Score Upper",1077 ]1078 ]1079 1080 column_choices_tasks_hi = [1081 col1082 for col in df_tasks_hi.columns.tolist()1083 if col != "Model Size Filter"1084 ]1085 1086 # ---------- IFEval options ----------1087 family_options_if = sorted(df_if["Family"].dropna().unique().tolist())1088 min_model_size_if = int(df_if["Model Size Filter"].min())1089 max_model_size_if = int(df_if["Model Size Filter"].max())1090 1091 all_if_columns = [1092 "Rank",1093 "Model Name",1094 "Average Accuracy (Ar)",1095 "Ar Prompt-lvl",1096 "Ar Instruction-lvl",1097 "Average Accuracy (En)",1098 "En Prompt-lvl",1099 "En Instruction-lvl",1100 "Type",1101 "Creator",1102 "Family",1103 "Size (B)",1104 "Base Model",1105 "Context Window",1106 "Lang.",1107 ]1108 default_if_columns = [1109 "Rank",1110 "Model Name",1111 "Average Accuracy (Ar)",1112 "Ar Prompt-lvl",1113 "Ar Instruction-lvl",1114 "Average Accuracy (En)",1115 ]1116 1117 with gr.Blocks() as demo:1118 gr.HTML(HEADER)1119 1120 with gr.Tabs():1121 #1122 # AL Leaderboards Tab (AraGen + IFEval)1123 #1124 with gr.Tab("AL Leaderboards 🏅"):1125 with gr.Tabs():1126 # -------------------------1127 # Sub-Tab: AraGen Leaderboards1128 # -------------------------1129 with gr.Tab("🐪 AraGen Leaderboards (v3)"):1130 with gr.Tabs():1131 # 3C3H Scores1132 with gr.Tab("3C3H Scores"):1133 with gr.Accordion("⚙️ Filters", open=False):1134 with gr.Row():1135 search_box_3c3h = gr.Textbox(1136 placeholder="Search for models...", 1137 label="Search", 1138 interactive=True,1139 )1140 with gr.Row():1141 column_selector_3c3h = gr.CheckboxGroup(1142 choices=column_choices_3c3h,1143 value=[1144 "Rank",1145 "Rank Spread",1146 "Model Name",1147 "3C3H Score",1148 "95% CI (±)",1149 "Correctness",1150 "Completeness",1151 "Conciseness",1152 "Helpfulness",1153 "Honesty",1154 "Harmlessness",1155 ],1156 label="Select columns to display",1157 )1158 with gr.Row():1159 license_filter_3c3h = gr.CheckboxGroup(1160 choices=license_options_3c3h,1161 value=license_options_3c3h.copy(),1162 label="Filter by License",1163 )1164 precision_filter_3c3h = gr.CheckboxGroup(1165 choices=precision_options_3c3h,1166 value=precision_options_3c3h.copy(),1167 label="Filter by Precision",1168 )1169 with gr.Row():1170 model_size_min_filter_3c3h = gr.Slider(1171 minimum=min_model_size_3c3h,1172 maximum=max_model_size_3c3h,1173 value=min_model_size_3c3h,1174 step=1,1175 label="Minimum Model Size",1176 interactive=True,1177 )1178 model_size_max_filter_3c3h = gr.Slider(1179 minimum=min_model_size_3c3h,1180 maximum=max_model_size_3c3h,1181 value=max_model_size_3c3h,1182 step=1,1183 label="Maximum Model Size",1184 interactive=True,1185 )1186 leaderboard_3c3h = gr.Dataframe(1187 df_3c3h_ar[1188 [1189 "Rank",1190 "Rank Spread",1191 "Model Name",1192 "3C3H Score",1193 "95% CI (±)",1194 "Correctness",1195 "Completeness",1196 "Conciseness",1197 "Helpfulness",1198 "Honesty",1199 "Harmlessness",1200 ]