oddadmix/Arabic-Tokenizer-Comparison

1"""© KAND CA 2026 - Arabic tokenizer comparison Space.2 3A tokenizer is the cheapest thing to get wrong in an Arabic pipeline and the4hardest to see. Every model card quotes parameters and context length; almost5none tell you that the same Arabic paragraph costs 52% more context on one6tokenizer than another, which is a 52% tax on every prompt, every document and7every embedding you will ever run through it.8 9This puts the number on screen. Paste Arabic - MSA, dialect, or code-switched -10and see how many tokens each tokenizer actually spends on it.11 12The interesting comparison is not big-vocab vs small-vocab. Mistral and Emhotob13both have ~32K entries; Emhotob spends all of them on Arabic and Mistral spends14almost none, and the gap that opens up is the whole point.15"""16import html17import statistics18 19import gradio as gr20from transformers import AutoTokenizer21 22TOKENIZERS = [23    ("Emhotob 32K (Arabic-only)", "oddadmix/50M-2048-Emhotob",  "Arabic-only"),24    ("Gemma-4",                   "google/gemma-4-31B-it",      "multilingual"),25    ("Qwen3.6",                   "Qwen/Qwen3.6-27B",           "multilingual"),26    ("Qwen3.5",                   "Qwen/Qwen3.5-4B",            "multilingual"),27    ("Mistral-7B v0.3",           "mistralai/Mistral-7B-v0.3",  "western"),28    ("GPT-2",                     "openai-community/gpt2",      "western"),29]30 31EXAMPLES = {32    "فصحى — MSA news": (33        "أعلنت وزارة الاقتصاد أن معدل النمو المتوقع خلال العام المقبل سيبلغ نحو "34        "أربعة في المئة، مدفوعاً بارتفاع الصادرات غير النفطية وتحسن أداء قطاع "35        "السياحة، فيما أشار التقرير إلى أن الاستثمارات الأجنبية المباشرة سجلت "36        "زيادة ملحوظة مقارنة بالفترة نفسها من العام الماضي."),37    "مصري — Egyptian": (38        "يا جماعة أنا رايح السوق دلوقتي عشان أجيب شوية حاجات للبيت، لو محتاجين "39        "حاجة قولولي بسرعة قبل ما أنزل. الأسعار غليت أوي الفترة دي وبقى لازم "40        "الواحد يحسب حسابه كويس قبل ما يشتري أي حاجة."),41    "مغربي — Moroccan Darija": (42        "واش نتا فاهم شنو كايقع؟ الأسعار طلعات بزاف هاد الشهر وكلشي كايشتكي. "43        "غادي نمشي للسوق دابا باش نشوف شنو كاين، ولكن ما كنظنش غادي نلقى شي حاجة "44        "رخيصة. الله يسهل علينا."),45    "خليجي — Gulf": (46        "شلونك؟ أنا بروح الدوام بدري اليوم لأن عندي اجتماع مهم مع المدير. "47        "بعدين بمر على المحل عشان أشتري أغراض البيت، وإذا خلصت بدري بجيك."),48    "Code-switched": (49        "الـ deployment اتعمل امبارح على الـ production server بس فيه issue في "50        "الـ latency، محتاجين نعمل profiling للـ database queries عشان نشوف "51        "الـ bottleneck فين بالظبط."),52    "English (control)": (53        "The ministry announced that the expected growth rate for the coming year "54        "will reach approximately four percent, driven by rising non-oil exports "55        "and improved performance in the tourism sector."),56}57 58_CACHE = {}59 60 61def get(repo):62    if repo not in _CACHE:63        _CACHE[repo] = AutoTokenizer.from_pretrained(repo)64    return _CACHE[repo]65 66 67PALETTE = ["#dbeafe", "#fef3c7", "#dcfce7", "#fae8ff", "#ffe4e6", "#e0f2fe"]68 69 70def render_tokens(tk, ids):71    """Colour each token so the segmentation is visible, not just counted.72 73    Tokens are decoded ONE ID AT A TIME rather than read off74    `convert_ids_to_tokens`. For a byte-level BPE - which Emhotob, Gemma, Qwen75    and GPT-2 all are - that method returns the byte-mangled form, so Arabic76    comes back as 'أعÙĦÙĨت' instead of 'أعلنت'. Decoding per id gives the77    real characters, which is the whole point of showing the split.78    """79    out = []80    for i, tid in enumerate(ids):81        s = tk.decode([tid])82        lead = s.startswith(" ")83        s = html.escape(s.strip()) or "␣"84        out.append(85            f'<span style="background:{PALETTE[i % len(PALETTE)]};'86            f'padding:3px 5px;margin:2px;border-radius:4px;'87            f'display:inline-block;color:#111;font-size:15px;'88            f'border-left:{"3px solid #94a3b8" if lead else "0"}">{s}</span>')89    return ('<div dir="rtl" style="line-height:2.4;direction:rtl;'90            'text-align:right;padding:10px;background:#fafafa;'91            'border-radius:8px;border:1px solid #e5e7eb">'92            + "".join(out) + "</div>")93 94 95def compare(text, show_for):96    text = (text or "").strip()97    if not text:98        raise gr.Error("اكتب أو الصق نصاً عربياً أولاً. / Enter some text first.")99    n_words = len(text.split())100    n_chars = len(text)101 102    rows, counts = [], {}103    for name, repo, kind in TOKENIZERS:104        try:105            tk = get(repo)106            ids = tk(text, add_special_tokens=False)["input_ids"]107            counts[name] = len(ids)108            rows.append([name, kind, f"{tk.vocab_size:,}", len(ids),109                         round(len(ids) / max(n_words, 1), 3),110                         round(n_chars / max(len(ids), 1), 2)])111        except Exception as e:112            rows.append([name, kind, "—", None, None, None])113 114    base = counts.get("Emhotob 32K (Arabic-only)")115    for r in rows:116        r.append(round(r[3] / base, 2) if (base and r[3]) else None)117 118    ok = [r for r in rows if r[3]]119    ok.sort(key=lambda r: r[3])120    best, worst = ok[0], ok[-1]121    summary = (122        f"### {n_words} كلمة · {n_chars:,} حرف\n\n"123        f"**{best[0]}** is most efficient at **{best[3]:,} tokens** "124        f"({best[4]} tok/word). **{worst[0]}** needs **{worst[3]:,}** "125        f"— **{round(worst[3]/best[3], 2)}×** as many for the same text.\n\n"126        f"On a 128K context window that difference is "127        f"**{int(128000/best[4]) - int(128000/worst[4]):,} fewer words** of room.")128 129    tk = get(dict((n, r) for n, r, _ in TOKENIZERS)[show_for])130    ids = tk(text, add_special_tokens=False)["input_ids"]131    return rows, summary, render_tokens(tk, ids)132 133 134with gr.Blocks(title="Arabic Tokenizer Comparison") as demo:135    gr.Markdown(136        "# 🔤 Arabic Tokenizer Comparison\n"137        "### كم رمزاً يكلّفك النص العربي؟\n\n"138        "The same Arabic paragraph can cost **50% more context** on one tokenizer "139        "than another. That is a tax on every prompt, document and embedding you "140        "run — and no model card mentions it. Paste Arabic below and see.\n\n"141        "Note that **Mistral and Emhotob both have ~32K vocabularies**. The gap "142        "between them is not vocabulary *size*, it is what the vocabulary is "143        "*spent on*.")144 145    with gr.Row():146        with gr.Column(scale=3):147            text = gr.Textbox(label="النص / Text", lines=8, rtl=True,148                              text_align="right", value=EXAMPLES["فصحى — MSA news"])149            gr.Examples(examples=[[v] for v in EXAMPLES.values()], inputs=[text],150                        example_labels=list(EXAMPLES), label="أمثلة / Examples")151        with gr.Column(scale=2):152            summary = gr.Markdown()153            show_for = gr.Dropdown([n for n, _, _ in TOKENIZERS],154                                   value="Emhotob 32K (Arabic-only)",155                                   label="Show token split for")156    run = gr.Button("قارِن / Compare", variant="primary")157 158    table = gr.Dataframe(159        headers=["tokenizer", "kind", "vocab", "tokens", "tok/word", "chars/tok", "×Emhotob"],160        datatype=["str", "str", "str", "number", "number", "number", "number"],161        label="fewer tokens = better", wrap=True)162    viz = gr.HTML(label="token split — a grey edge marks a token that begins with a space")163 164    gr.Markdown(165        "---\n"166        "**Emhotob 32K** is the Arabic-only byte-level BPE behind "167        "[Nawah](https://huggingface.co/oddadmix/Nawah-50M-RAG-Support-2K) and "168        "[50M-2048-Emhotob](https://huggingface.co/oddadmix/50M-2048-Emhotob). "169        "Every one of its 32,000 entries is spent on Arabic, which is why it "170        "beats vocabularies 8× its size on Arabic text — and why it does *worse* "171        "on English, which it was never meant to handle.\n\n"172        "Measured on MSA it reaches ~1.39 tokens/word against Gemma-4's ~2.11. "173        "On dialect its lead narrows — dialectal orthography is where an "174        "MSA-trained vocabulary is weakest.\n\n"175        "© KAND CA 2026 — PROJECT NAWAH")176 177    run.click(compare, [text, show_for], [table, summary, viz])178    text.submit(compare, [text, show_for], [table, summary, viz])179    show_for.change(compare, [text, show_for], [table, summary, viz])180 181if __name__ == "__main__":182    demo.queue(max_size=24).launch(theme=gr.themes.Soft(primary_hue="teal"))183