Vikhrmodels/Russian_ASR_Leaderboard
16
1import json2import pandas as pd3from statistics import mean4from huggingface_hub import HfApi, create_repo5from datasets import load_dataset, Dataset6from datasets.data_files import EmptyDatasetError7import re8 9from constants import (10 REPO_ID,11 HF_TOKEN,12 DATASETS,13 SHORT_DATASET_NAMES,14 DATASET_DESCRIPTIONS,15)16 17api = HfApi(token=HF_TOKEN)18 19 20OPEN_LICENSE_KEYWORDS = {21 "mit", "apache", "apache-2", "apache-2.0",22 "bsd", "bsd-2", "bsd-3", "bsd-2-clause", "bsd-3-clause",23 "isc", "mpl", "mpl-2.0",24 "lgpl", "lgpl-2.1", "lgpl-3.0",25 "gpl", "gpl-2.0", "gpl-3.0", "agpl", "agpl-3.0",26 "epl", "epl-2.0", "cddl", "cddl-1.0", "cddl-1.1",27 "bsl", "bsl-1.0", "boost", "zlib", "unlicense", "artistic-2.0",28 "cc0", "cc0-1.0",29 "cc-by", "cc-by-3.0", "cc-by-4.0",30 "cc-by-sa", "cc-by-sa-3.0", "cc-by-sa-4.0",31 "openrail", "openrail-m", "bigscience openrail", "bigscience openrail-m",32 "open-source", "opensource", "open source"33}34 35RESTRICTIVE_LICENSE_KEYWORDS = {36 "cc-by-nc", "cc-by-nc-sa", "cc-nc", "nc-sa", "nc-nd",37 "cc-by-nd", "cc-nd", "no-derivatives", "no derivatives",38 "non-commercial", "noncommercial", "research-only", "research only",39 "llama", "llama-2", "community license",40 "proprietary", "closed", "unknown", "custom"41}42 43def is_open_license(license_str: str) -> bool:44 s = (str(license_str) if license_str is not None else "").strip().lower()45 if not s:46 return False47 if any(pat in s for pat in RESTRICTIVE_LICENSE_KEYWORDS):48 return False49 return any(pat in s for pat in OPEN_LICENSE_KEYWORDS)50 51 52def init_repo():53 try:54 api.repo_info(REPO_ID, repo_type="dataset")55 except:56 create_repo(REPO_ID, repo_type="dataset", private=True, token=HF_TOKEN)57 58 59def load_data():60 columns = (61 ["model_name", "link", "license", "overall_wer", "overall_cer"]62 + [f"wer_{ds}" for ds in DATASETS]63 + [f"cer_{ds}" for ds in DATASETS]64 )65 try:66 dataset = load_dataset(REPO_ID, token=HF_TOKEN)67 df = dataset["train"].to_pandas()68 except EmptyDatasetError:69 df = pd.DataFrame(columns=columns)70 71 if not df.empty:72 df = df.sort_values("overall_wer").reset_index(drop=True)73 df.insert(0, "rank", df.index + 1)74 for col in (75 ["overall_wer", "overall_cer"]76 + [f"wer_{ds}" for ds in DATASETS]77 + [f"cer_{ds}" for ds in DATASETS]78 ):79 df[col] = (df[col] * 100).round(2)80 81 best_values = {ds: df[f"wer_{ds}"].min() for ds in DATASETS}82 for short_ds, ds in zip(SHORT_DATASET_NAMES, DATASETS):83 df[short_ds] = df.apply(84 lambda row: f'<span title="CER: {row[f"cer_{ds}"]:.2f}%" '85 f'class="metric-cell{" best-metric" if row[f"wer_{ds}"] == best_values[ds] else ""}">'86 f"{row[f'wer_{ds}']:.2f}%</span>",87 axis=1,88 )89 df = df.drop(columns=[f"wer_{ds}", f"cer_{ds}"])90 91 df["model_name"] = df.apply(92 lambda row: f'<a href="{row["link"]}" target="_blank">{row["model_name"]}</a>',93 axis=1,94 )95 df = df.drop(columns=["link"])96 97 df["license"] = df["license"].apply(lambda x: "Открытая" if is_open_license(x) else "Закрытая")98 99 df["rank"] = df["rank"].apply(100 lambda r: "🥇" if r == 1 else "🥈" if r == 2 else "🥉" if r == 3 else str(r)101 )102 103 df.rename(104 columns={105 "overall_wer": "Средний WER ⬇️",106 "overall_cer": "Средний CER ⬇️",107 "license": "Тип модели",108 "model_name": "Модель",109 "rank": "Ранг",110 },111 inplace=True,112 )113 114 table_html = df.to_html(115 escape=False, index=False, classes="display cell-border compact stripe"116 )117 return f'<div class="leaderboard-wrapper"><div class="leaderboard-table">{table_html}</div></div>'118 else:119 return (120 '<div class="leaderboard-wrapper"><div class="leaderboard-table"><table><thead><tr><th>Ранг</th><th>Модель</th><th>Тип модели</th><th>Средний WER ⬇️</th><th>Средний CER ⬇️</th>'121 + "".join(f"<th>{short}</th>" for short in SHORT_DATASET_NAMES)122 + "</tr></thead><tbody></tbody></table></div></div>"123 )124 125 126def process_submit(json_str):127 columns = (128 ["model_name", "link", "license", "overall_wer", "overall_cer"]129 + [f"wer_{ds}" for ds in DATASETS]130 + [f"cer_{ds}" for ds in DATASETS]131 )132 try:133 data = json.loads(json_str)134 required_keys = ["model_name", "link", "license", "metrics"]135 if not all(key in data for key in required_keys):136 raise ValueError(137 "Неверная структура JSON. Требуемые поля: model_name, link, license, metrics"138 )139 metrics = data["metrics"]140 if set(metrics.keys()) != set(DATASETS):141 raise ValueError(142 f"Метрики должны быть для всех датасетов: {', '.join(DATASETS)}"143 )144 wers, cers = [], []145 row = {146 "model_name": data["model_name"],147 "link": data["link"],148 "license": data["license"],149 }150 for ds in DATASETS:151 if "wer" not in metrics[ds] or "cer" not in metrics[ds]:152 raise ValueError(f"Для {ds} требуются wer и cer")153 row[f"wer_{ds}"] = metrics[ds]["wer"]154 row[f"cer_{ds}"] = metrics[ds]["cer"]155 wers.append(metrics[ds]["wer"])156 cers.append(metrics[ds]["cer"])157 row["overall_wer"] = mean(wers)158 row["overall_cer"] = mean(cers)159 160 try:161 dataset = load_dataset(REPO_ID, token=HF_TOKEN)162 df = dataset["train"].to_pandas()163 except EmptyDatasetError:164 df = pd.DataFrame(columns=columns)165 166 new_df = pd.concat([df, pd.DataFrame([row])], ignore_index=True)167 new_dataset = Dataset.from_pandas(new_df)168 new_dataset.push_to_hub(REPO_ID, token=HF_TOKEN)169 170 updated_html = load_data()171 return updated_html, "Успешно добавлено!", ""172 except Exception as e:173 return None, f"Ошибка: {str(e)}", json_str174 175 176def get_datasets_description():177 html = '<div class="datasets-container">'178 for short_ds, info in DATASET_DESCRIPTIONS.items():179 html += f"""180 <div class="dataset-card">181 <h3>{short_ds} <span class="full-name">{info["full_name"]}</span></h3>182 <p>{info["description"]}</p>183 <p class="records">📊 {info["num_rows"]} записей</p>184 </div>185 """186 html += "</div>"187 return html188 189 190def _strip_punct(text: str) -> str:191 return re.sub(r"[^\w\s]+", "", text, flags=re.UNICODE)192 193 194def normalize_text(s: str) -> str:195 return _strip_punct(s.lower()).strip()196 197 198def _edit_distance(a, b):199 n, m = len(a), len(b)200 dp = [[0] * (m + 1) for _ in range(n + 1)]201 for i in range(n + 1):202 dp[i][0] = i203 for j in range(m + 1):204 dp[0][j] = j205 for i in range(1, n + 1):206 ai = a[i - 1]207 for j in range(1, m + 1):208 cost = 0 if ai == b[j - 1] else 1209 dp[i][j] = min(dp[i - 1][j] + 1, dp[i][j - 1] + 1, dp[i - 1][j - 1] + cost)210 return dp[n][m]211 212 213def compute_wer_cer(ref: str, hyp: str, normalize: bool = True):214 if normalize:215 ref_norm, hyp_norm = normalize_text(ref), normalize_text(hyp)216 else:217 ref_norm, hyp_norm = ref, hyp218 ref_words, hyp_words = ref_norm.split(), hyp_norm.split()219 Nw = max(1, len(ref_words))220 wer = _edit_distance(ref_words, hyp_words) / Nw221 ref_chars, hyp_chars = list(ref_norm), list(hyp_norm)222 Nc = max(1, len(ref_chars))223 cer = _edit_distance(ref_chars, hyp_chars) / Nc224 return round(wer * 100, 2), round(cer * 100, 2)225 226 227def get_metrics_html():228 return """229<div class="metrics-grid">230 <div class="metric-card">231 <h3>WER — Word Error Rate</h3>232 <div class="formula">WER = ( <span>S</span> + <span>D</span> + <span>I</span> ) / <span>N</span></div>233 <div class="chips">234 <div class="chip"><b>S</b><small>замены</small></div>235 <div class="chip"><b>D</b><small>удаления</small></div>236 <div class="chip"><b>I</b><small>вставки</small></div>237 <div class="chip"><b>N</b><small>слов в референсе</small></div>238 </div>239 </div>240 <div class="metric-card">241 <h3>CER — Character Error Rate</h3>242 <div class="formula">CER = ( <span>S</span> + <span>D</span> + <span>I</span> ) / <span>N</span></div>243 <div class="chips">244 <div class="chip"><b>S, D, I</b><small>операции редактирования</small></div>245 <div class="chip"><b>N</b><small>символов в референсе</small></div>246 </div>247 </div>248 <div class="metric-card">249 <h3>Нормализация</h3>250 <p class="metric-text">Перед расчётом приводим текст к нижнему регистру и удаляем пунктуацию.</p>251 </div>252 <div class="metric-card">253 <h3>Сравнение</h3>254 <p class="metric-text">Сортировка по среднему WER по всем датасетам. Метрики отображаются в процентах.</p>255 </div>256</div>257"""258 259 260def get_submit_html():261 return """262<div class="submit-grid">263 <div class="form-card">264 <h3>Общая информация</h3>265 <ul>266 <li><b>Название модели</b> — коротко и понятно.</li>267 <li><b>Ссылка</b> — HuggingFace, GitHub или сайт.</li>268 <li><b>Лицензия</b> — MIT, Apache-2.0, GPL или Closed.</li>269 </ul>270 </div>271 <div class="form-card">272 <h3>Метрики</h3>273 <p>Укажите WER и CER для всех датасетов в формате JSON. Значения — от 0 до 1.</p>274 <pre class="code-block json">{275 <span class="key">"Russian_LibriSpeech"</span>: { <span class="key">"wer"</span>: <span class="number">0.1234</span>, <span class="key">"cer"</span>: <span class="number">0.0567</span> },276 <span class="key">"Common_Voice_Corpus_22.0"</span>: { <span class="key">"wer"</span>: <span class="number">0.2345</span>, <span class="key">"cer"</span>: <span class="number">0.0789</span> },277 <span class="key">"Tone_Webinars"</span>: { <span class="key">"wer"</span>: <span class="number">0.3456</span>, <span class="key">"cer"</span>: <span class="number">0.0987</span> },278 <span class="key">"Tone_Books"</span>: { <span class="key">"wer"</span>: <span class="number">0.4567</span>, <span class="key">"cer"</span>: <span class="number">0.1098</span> },279 <span class="key">"Tone_Speak"</span>: { <span class="key">"wer"</span>: <span class="number">0.5678</span>, <span class="key">"cer"</span>: <span class="number">0.1209</span> },280 <span class="key">"Sova_RuDevices"</span>: { <span class="key">"wer"</span>: <span class="number">0.6789</span>, <span class="key">"cer"</span>: <span class="number">0.1310</span> }281}</pre>282 </div>283</div>284"""285