Mnttnt21/tokenizer-bench

1<?php2 3declare(strict_types=1);4 5/**6 * Sample metadata per language, with the keywords we'd EXPECT a store-like7 * tokenizer to produce. Used to eyeball which engine is closest.8 *9 * Each entry: ['lang' => code, 'text' => input, 'expected' => [keywords], 'gloss' => note]10 */11return [12    [13        'lang'     => 'zh-Hans',14        'text'     => '密码',15        'expected' => ['密码'],16        'gloss'    => '"password". One word — NOT 密(secret)+码(code).',17    ],18    [19        'lang'     => 'zh-Hans',20        'text'     => '免费手机游戏',21        'expected' => ['免费', '手机', '游戏'],22        'gloss'    => '"free mobile games". 手机=phone (not 手 hand + 机 machine).',23    ],24    [25        'lang'     => 'zh-Hans',26        'text'     => '手机游戏下载',27        'expected' => ['手机', '游戏', '下载'],28        'gloss'    => '"mobile game download".',29    ],30    [31        'lang'     => 'zh-Hant',32        'text'     => '免費手機遊戲',33        'expected' => ['免費', '手機', '遊戲'],34        'gloss'    => 'Traditional form of "free mobile games".',35    ],36    [37        'lang'     => 'ja',38        'text'     => '無料ゲームアプリ',39        'expected' => ['無料', 'ゲーム', 'アプリ'],40        'gloss'    => '"free game app".',41    ],42    [43        'lang'     => 'ja',44        'text'     => '写真編集アプリ',45        'expected' => ['写真', '編集', 'アプリ'],46        'gloss'    => '"photo editing app".',47    ],48    [49        'lang'     => 'th',50        'text'     => 'เกมฟรี',51        'expected' => ['เกม', 'ฟรี'],52        'gloss'    => '"free game".',53    ],54    [55        'lang'     => 'th',56        'text'     => 'แอปแต่งรูป',57        'expected' => ['แอป', 'แต่ง', 'รูป'],58        'gloss'    => '"photo editing app".',59    ],60    [61        'lang'     => 'ko',62        'text'     => '무료 게임 앱',63        'expected' => ['무료', '게임', '앱'],64        'gloss'    => '"free game app" — already space-separated.',65    ],66    [67        'lang'     => 'ko',68        'text'     => '게임을 무료로',69        'expected' => ['게임', '무료'],70        'gloss'    => 'Particles 을/로 attached — does the engine strip them?',71    ],72];73