Mnttnt21/tokenizer-bench
0
1<?php2 3declare(strict_types=1);4 5require __DIR__ . '/../vendor/autoload.php';6 7use Bench\BatchTokenizer;8use Bench\EngineRegistry;9use Bench\Translator;10 11$registry = new EngineRegistry();12$translator = new Translator();13$languages = $registry->languages();14$defaults = require __DIR__ . '/../fixtures/defaults.php';15 16// --- input ---------------------------------------------------------------17$lang = $_GET['lang'] ?? 'zh-Hans';18if (!isset($languages[$lang])) {19 $lang = 'zh-Hans';20}21 22$mode = $_GET['mode'] ?? 'perline'; // 'perline' | 'whole'23if ($mode !== 'whole') {24 $mode = 'perline';25}26 27$doTranslate = ($_GET['tr'] ?? '1') === '1';28 29$hasText = isset($_GET['text']);30$text = $hasText ? (string) $_GET['text'] : ($defaults[$lang] ?? '');31 32$engines = $registry->forLang($lang);33 34// Keyword lines (used by per-line mode and for translating each keyword).35$lines = array_values(array_filter(36 array_map('trim', preg_split('/\R/u', $text) ?: []),37 static fn (string $l): bool => $l !== '',38));39 40// --- tokenize ------------------------------------------------------------41$tokenSet = []; // unique tokens (for translation)42$whole = []; // [ ['engine'=>, 'tokens'=>, 'error'=>] ] (whole mode)43$table = []; // line => [engineName => ['tokens'=>] | ['error'=>]] (perline mode)44 45if ($text !== '') {46 if ($mode === 'whole') {47 foreach ($engines as $e) {48 try {49 $tok = $e->tokenize($text, $lang);50 $whole[] = ['engine' => $e, 'tokens' => $tok, 'error' => null];51 foreach ($tok as $t) {52 $tokenSet[$t] = true;53 }54 } catch (\Throwable $ex) {55 $whole[] = ['engine' => $e, 'tokens' => [], 'error' => $ex->getMessage()];56 }57 }58 } else {59 // One engine at a time: batch-capable engines (heavy Python sidecars)60 // tokenize the whole keyword list in a single subprocess.61 foreach ($engines as $e) {62 if ($e instanceof BatchTokenizer) {63 try {64 $batch = $e->tokenizeBatch($lines, $lang);65 foreach ($lines as $i => $line) {66 $tok = $batch[$i] ?? [];67 $table[$line][$e->name()] = ['tokens' => $tok];68 foreach ($tok as $t) {69 $tokenSet[$t] = true;70 }71 }72 } catch (\Throwable $ex) {73 foreach ($lines as $line) {74 $table[$line][$e->name()] = ['error' => $ex->getMessage()];75 }76 }77 continue;78 }79 80 foreach ($lines as $line) {81 try {82 $tok = $e->tokenize($line, $lang);83 $table[$line][$e->name()] = ['tokens' => $tok];84 foreach ($tok as $t) {85 $tokenSet[$t] = true;86 }87 } catch (\Throwable $ex) {88 $table[$line][$e->name()] = ['error' => $ex->getMessage()];89 }90 }91 }92 }93}94 95// --- translation (budgeted: cached items are free, fresh calls are capped) --96$NET_BUDGET = 140;97$capped = false;98$glosses = []; // token => en|null99$kwGloss = []; // keyword line => en|null100$fullEn = null;101 102$translate = static function (array $items) use ($translator, $lang, &$NET_BUDGET, &$capped): array {103 $out = [];104 foreach ($items as $it) {105 if (array_key_exists($it, $out)) {106 continue;107 }108 if ($translator->isCached($it, $lang)) {109 $out[$it] = $translator->toEnglish($it, $lang);110 continue;111 }112 if ($NET_BUDGET <= 0) {113 $out[$it] = null;114 $capped = true;115 continue;116 }117 $NET_BUDGET--;118 $out[$it] = $translator->toEnglish($it, $lang);119 }120 return $out;121};122 123if ($doTranslate && $text !== '') {124 if ($mode === 'whole') {125 $fullEn = $translator->toEnglish($text, $lang);126 $glosses = $translate(array_keys($tokenSet));127 } else {128 // Keywords first (most useful), then tokens with the remaining budget.129 $kwGloss = $translate($lines);130 $glosses = $translate(array_keys($tokenSet));131 }132}133 134function h(string $s): string135{136 return htmlspecialchars($s, ENT_QUOTES, 'UTF-8');137}138 139/** Whether an engine takes part in "highlight differences": n-gram and Jieba140 * search intentionally differ from word segmenters, so they're excluded. */141function isDiffable(string $name): bool142{143 return stripos($name, 'search') === false && stripos($name, 'n-gram') === false;144}145?>146<!doctype html>147<html lang="en">148<head>149<meta charset="utf-8">150<meta name="viewport" content="width=device-width, initial-scale=1">151<title>Tokenizer bench — Asolytics meta editor</title>152<style>153 :root { --bg:#0f1115; --panel:#181b22; --line:#2a2f3a; --txt:#e6e9ef; --mut:#9aa3b2;154 --chip:#243042; --chipln:#33415c; --accent:#4f8cff; }155 * { box-sizing: border-box; }156 body { margin:0; background:var(--bg); color:var(--txt);157 font:14px/1.5 ui-sans-serif,-apple-system,"Segoe UI",Roboto,"Noto Sans CJK",sans-serif; }158 .wrap { max-width:1280px; margin:0 auto; padding:24px; }159 h1 { font-size:20px; margin:0 0 4px; }160 .sub { color:var(--mut); margin:0 0 20px; }161 form { background:var(--panel); border:1px solid var(--line); border-radius:10px; padding:16px; }162 textarea { width:100%; min-height:300px; resize:vertical; background:#0c0e13; color:var(--txt);163 border:1px solid var(--line); border-radius:8px; padding:10px; font-size:18px; line-height:1.4; }164 .row { display:flex; gap:12px; align-items:center; margin-top:12px; flex-wrap:wrap; }165 select, button { background:#0c0e13; color:var(--txt); border:1px solid var(--line);166 border-radius:8px; padding:8px 12px; font-size:14px; }167 button.go { background:var(--accent); border-color:var(--accent); color:#fff; cursor:pointer; font-weight:600; }168 .seg { display:inline-flex; border:1px solid var(--line); border-radius:8px; overflow:hidden; }169 .seg label { padding:8px 12px; cursor:pointer; color:var(--mut); background:#0c0e13; font-size:13px; }170 .seg input { display:none; }171 .seg input:checked + span { color:#fff; }172 .seg label:has(input:checked) { background:var(--accent); }173 .seg label span { pointer-events:none; }174 .muted { color:var(--mut); font-size:13px; }175 .label { color:var(--mut); font-size:12px; text-transform:uppercase; letter-spacing:.04em; margin:18px 0 6px; }176 .translation { background:#101a2b; border:1px solid #21355c; border-radius:8px;177 padding:8px 12px; margin:4px 0 6px; font-size:15px; }178 .notice { background:#2a230f; border:1px solid #5a4a1c; border-radius:8px; padding:8px 12px;179 margin:8px 0; font-size:13px; color:#e8d9a8; }180 181 /* whole mode cards */182 .grid { display:grid; grid-template-columns:repeat(auto-fit,minmax(240px,1fr)); gap:14px; margin-top:10px; }183 .col { background:var(--panel); border:1px solid var(--line); border-radius:10px; padding:14px; }184 .col h3 { margin:0 0 2px; font-size:15px; }185 .col .note { color:var(--mut); font-size:12px; min-height:32px; margin:0 0 8px; }186 .count { color:var(--mut); font-size:12px; margin-bottom:8px; }187 .count b { color:var(--txt); }188 .chips { display:flex; flex-wrap:wrap; gap:6px; }189 .tok { display:inline-flex; flex-direction:column; align-items:center; gap:1px;190 background:var(--chip); border:1px solid var(--chipln); border-radius:6px; padding:4px 8px; }191 .tok-src { font-size:18px; line-height:1.1; }192 .tok-en { font-size:11px; color:var(--mut); line-height:1.1; max-width:130px; text-align:center; }193 .err { color:#ff8c8c; font-size:13px; }194 195 /* perline table */196 table { border-collapse:collapse; width:100%; margin-top:10px; }197 th, td { border:1px solid var(--line); padding:8px 10px; vertical-align:top; text-align:left; }198 th { background:var(--panel); font-size:13px; position:sticky; top:0; }199 td.kw { white-space:nowrap; }200 .kw-src { font-size:17px; }201 .kw-en { color:var(--mut); font-size:12px; }202 .chip { display:inline-flex; flex-direction:column; align-items:center; gap:1px;203 background:var(--chip); border:1px solid var(--chipln); border-radius:6px; padding:3px 7px; }204 .chip-src { font-size:16px; line-height:1.15; }205 .chip-en { font-size:10px; color:var(--mut); line-height:1.1; max-width:120px; text-align:center; }206 .cell { display:flex; align-items:flex-start; gap:8px; }207 .cell .chips { flex:1; }208 .cellcount { color:var(--mut); font-size:11px; white-space:nowrap; }209 210 /* highlight-differences toggle + row highlight */211 .diffbar { display:flex; align-items:center; gap:12px; margin:2px 0 0; }212 .tgl { display:inline-flex; align-items:center; gap:7px; cursor:pointer; user-select:none;213 background:#0c0e13; border:1px solid var(--line); border-radius:8px; padding:7px 12px;214 font-size:13px; color:var(--mut); }215 .tgl input { accent-color:var(--accent); }216 .tgl.on { color:#fff; border-color:var(--accent); }217 tr.differ td { background:rgba(255,196,84,.07); }218 tr.differ td.kw { box-shadow:inset 3px 0 0 #d8a23a; }219 220 /* loading overlay (shown while the request renders/translates) */221 #overlay { position:fixed; inset:0; background:rgba(15,17,21,.82); z-index:50;222 display:none; align-items:center; justify-content:center; flex-direction:column; gap:14px; }223 #overlay.on { display:flex; }224 #overlay .bar { width:280px; height:8px; background:#0c0e13; border:1px solid var(--line);225 border-radius:6px; overflow:hidden; }226 #overlay .barfill { height:100%; width:0; background:var(--accent); transition:width .25s ease; }227 #overlay .ov-text { color:var(--mut); font-size:13px; }228</style>229</head>230<body>231<div id="overlay">232 <div class="bar"><div class="barfill"></div></div>233 <div class="ov-text">Tokenizing & translating…</div>234</div>235<div class="wrap">236 <h1>Tokenizer bench</h1>237 <p class="sub">CJK / Thai / Korean word segmentation for the Asolytics meta editor — paste a keyword list, see how each splits.</p>238 239 <form method="get" id="f">240 <textarea name="text" id="ta" placeholder="One keyword per line…"><?= h($text) ?></textarea>241 <div class="row">242 <select name="lang" id="lang">243 <?php foreach ($languages as $code => $name): ?>244 <option value="<?= h($code) ?>" <?= $code === $lang ? 'selected' : '' ?>><?= h($name) ?></option>245 <?php endforeach; ?>246 </select>247 248 <div class="seg" id="mode">249 <label><input type="radio" name="mode" value="perline" <?= $mode === 'perline' ? 'checked' : '' ?>><span>Per keyword</span></label>250 <label><input type="radio" name="mode" value="whole" <?= $mode === 'whole' ? 'checked' : '' ?>><span>Whole text</span></label>251 </div>252 253 <input type="hidden" name="tr" id="trh" value="<?= $doTranslate ? '1' : '0' ?>">254 <label class="muted"><input type="checkbox" id="tr" <?= $doTranslate ? 'checked' : '' ?>> 🇬🇧 translate</label>255 256 <button type="submit" class="go">Tokenize</button>257 <span class="muted"><?= count($lines) ?> keywords · <?= mb_strlen($text) ?> chars</span>258 </div>259 </form>260 261 <?php if ($capped): ?>262 <div class="notice">⚠ Translation throttled to avoid rate limits — some glosses show <b>—</b>.263 Reload to fill more (results are cached, so each reload completes another batch).</div>264 <?php endif; ?>265 266 <?php /* ---------------- WHOLE MODE ---------------- */ ?>267 <?php if ($text !== '' && $mode === 'whole'): ?>268 <?php if ($fullEn !== null): ?>269 <div class="translation">🇬🇧 <?= h($fullEn) ?></div>270 <?php endif; ?>271 <div class="grid">272 <?php foreach ($whole as $r): ?>273 <?php $engine = $r['engine']; ?>274 <div class="col">275 <h3><?= h($engine->name()) ?></h3>276 <p class="note"><?= h($engine->note()) ?></p>277 <?php if ($r['error'] !== null): ?>278 <div class="err">⚠ <?= h($r['error']) ?></div>279 <?php else: ?>280 <div class="count"><b><?= count($r['tokens']) ?></b> tokens</div>281 <div class="chips">282 <?php foreach ($r['tokens'] as $tok): ?>283 <?php $g = $glosses[$tok] ?? null; ?>284 <span class="tok">285 <span class="tok-src"><?= h($tok) ?></span>286 <?php if ($doTranslate): ?><span class="tok-en"><?= $g !== null ? h($g) : '—' ?></span><?php endif; ?>287 </span>288 <?php endforeach; ?>289 </div>290 <?php endif; ?>291 </div>292 <?php endforeach; ?>293 </div>294 <?php endif; ?>295 296 <?php /* ---------------- PER-KEYWORD MODE ---------------- */ ?>297 <?php if ($text !== '' && $mode === 'perline'): ?>298 <div class="label">How each keyword splits into words</div>299 <div class="diffbar">300 <label class="tgl" id="difftgl"><input type="checkbox" id="diff"> ✨ Highlight differences</label>301 <span class="muted" id="diffcount"></span>302 </div>303 <table>304 <thead>305 <tr>306 <th>Keyword<?= $doTranslate ? ' / EN' : '' ?></th>307 <?php foreach ($engines as $e): ?>308 <th data-diff="<?= isDiffable($e->name()) ? '1' : '0' ?>"><?= h($e->name()) ?></th>309 <?php endforeach; ?>310 </tr>311 </thead>312 <tbody>313 <?php foreach ($lines as $line): ?>314 <tr>315 <td class="kw">316 <div class="kw-src"><?= h($line) ?></div>317 <?php if ($doTranslate): ?>318 <div class="kw-en"><?= isset($kwGloss[$line]) ? h((string) $kwGloss[$line]) : '—' ?></div>319 <?php endif; ?>320 </td>321 <?php foreach ($engines as $e): ?>322 <?php $cell = $table[$line][$e->name()] ?? null; ?>323 <td class="ecell" data-diff="<?= isDiffable($e->name()) ? '1' : '0' ?>">324 <?php if ($cell === null): ?>325 <span class="muted">—</span>326 <?php elseif (isset($cell['error'])): ?>327 <span class="err">⚠ <?= h($cell['error']) ?></span>328 <?php else: ?>329 <div class="cell">330 <span class="chips">331 <?php foreach ($cell['tokens'] as $tok): ?>332 <?php $g = $glosses[$tok] ?? null; ?>333 <span class="chip">334 <span class="chip-src"><?= h($tok) ?></span>335 <?php if ($doTranslate): ?><span class="chip-en"><?= $g !== null ? h($g) : '—' ?></span><?php endif; ?>336 </span>337 <?php endforeach; ?>338 </span>339 <span class="cellcount"><?= count($cell['tokens']) ?></span>340 </div>341 <?php endif; ?>342 </td>343 <?php endforeach; ?>344 </tr>345 <?php endforeach; ?>346 </tbody>347 </table>348 <?php endif; ?>349 350 <p class="muted" style="margin-top:24px; padding-top:14px; border-top:1px solid var(--line)">351 Engines per language: <b>zh</b> → Jieba (cut/search) + ICU · <b>ja</b> → MeCab + ICU ·352 <b>th</b> → ICU · <b>ko</b> → ICU + Space split (+ MeCab-ko if installed).353 The exact store algorithm is proprietary; Jieba <i>search</i> mode approximates its overlapping sub-grams.354 </p>355</div>356 357<script>358(function () {359 var DEFAULTS = <?= json_encode($defaults, JSON_UNESCAPED_UNICODE) ?>;360 var f = document.getElementById('f');361 var ta = document.getElementById('ta');362 var langSel = document.getElementById('lang');363 var K = {364 lang: 'tok:lang',365 mode: 'tok:mode',366 tr: 'tok:tr',367 diff: 'tok:diff',368 input: function (l) { return 'tok:input:' + l; }369 };370 var trBox = document.getElementById('tr');371 var trHidden = document.getElementById('trh');372 function curMode() { var c = f.querySelector('input[name=mode]:checked'); return c ? c.value : 'perline'; }373 function curTr() { return trBox.checked ? '1' : '0'; }374 function syncTr() { trHidden.value = curTr(); }375 376 // Loading overlay: the request is synchronous (server renders + translates in one377 // shot), so we show an animated bar that creeps toward ~92% until the new page loads.378 var ov = document.getElementById('overlay');379 var fill = ov.querySelector('.barfill');380 function showLoading() {381 ov.classList.add('on');382 var p = 8;383 fill.style.width = p + '%';384 var t = setInterval(function () {385 p += Math.max(0.5, (92 - p) * 0.08);386 if (p >= 92) { p = 92; clearInterval(t); }387 fill.style.width = p + '%';388 }, 200);389 }390 // Submit through here so the overlay always appears first.391 function go() { showLoading(); f.submit(); }392 f.addEventListener('submit', showLoading);393 394 var params = new URLSearchParams(location.search);395 396 // Fresh visit (no query): restore last session from localStorage and run once.397 if ([...params.keys()].length === 0) {398 var savedLang = localStorage.getItem(K.lang);399 if (savedLang && DEFAULTS[savedLang] !== undefined) {400 langSel.value = savedLang;401 var savedMode = localStorage.getItem(K.mode);402 if (savedMode) {403 var mi = f.querySelector('input[name=mode][value="' + savedMode + '"]');404 if (mi) mi.checked = true;405 }406 var savedTr = localStorage.getItem(K.tr);407 if (savedTr !== null) trBox.checked = (savedTr === '1');408 syncTr();409 410 var savedInput = localStorage.getItem(K.input(savedLang));411 ta.value = (savedInput !== null) ? savedInput : (DEFAULTS[savedLang] || '');412 go();413 return;414 }415 } else {416 // Came from a real submit/example: persist current state.417 localStorage.setItem(K.lang, langSel.value);418 localStorage.setItem(K.mode, curMode());419 localStorage.setItem(K.tr, curTr());420 localStorage.setItem(K.input(langSel.value), ta.value);421 }422 423 // Persist typing.424 ta.addEventListener('input', function () {425 localStorage.setItem(K.input(langSel.value), ta.value);426 });427 428 // Switching language swaps to that language's saved input (or its default) and runs.429 langSel.addEventListener('change', function () {430 var l = langSel.value;431 var saved = localStorage.getItem(K.input(l));432 ta.value = (saved !== null) ? saved : (DEFAULTS[l] || '');433 localStorage.setItem(K.lang, l);434 go();435 });436 437 // Toggling mode / translate re-runs immediately.438 f.querySelectorAll('input[name=mode]').forEach(function (el) {439 el.addEventListener('change', function () { localStorage.setItem(K.mode, curMode()); go(); });440 });441 trBox.addEventListener('change', function () {442 syncTr(); localStorage.setItem(K.tr, curTr()); go();443 });444 445 // --- Highlight differences (client-side, no resubmit) -----------------446 // A row is "different" when the real segmentation engines (Jieba search is447 // excluded — its overlapping sub-grams differ by design) don't all produce448 // the same token sequence for that keyword.449 var diffBox = document.getElementById('diff');450 if (diffBox) {451 var diffTgl = document.getElementById('difftgl');452 var diffCount = document.getElementById('diffcount');453 var tbl = f.parentNode.querySelector('table');454 455 function applyDiff() {456 if (!tbl) return;457 var on = diffBox.checked;458 diffTgl.classList.toggle('on', on);459 var rows = tbl.tBodies[0] ? tbl.tBodies[0].rows : [];460 var n = 0;461 for (var i = 0; i < rows.length; i++) {462 var tr = rows[i];463 tr.classList.remove('differ');464 if (!on) continue;465 var cells = tr.querySelectorAll('td.ecell[data-diff="1"]');466 var sigs = [];467 for (var j = 0; j < cells.length; j++) {468 var chips = cells[j].querySelectorAll('.chip-src');469 var toks = [];470 for (var k = 0; k < chips.length; k++) toks.push(chips[k].textContent);471 sigs.push(toks.join('␟'));472 }473 var allSame = sigs.every(function (s) { return s === sigs[0]; });474 if (sigs.length > 1 && !allSame) { tr.classList.add('differ'); n++; }475 }476 diffCount.textContent = on ? (n + ' of ' + rows.length + ' rows differ') : '';477 }478 479 diffBox.checked = (localStorage.getItem(K.diff) === '1');480 diffBox.addEventListener('change', function () {481 localStorage.setItem(K.diff, diffBox.checked ? '1' : '0');482 applyDiff();483 });484 applyDiff();485 }486})();487</script>488</body>489</html>490