dispatchAI/tokenizer-visualizer

1import gradio as gr2import json3import re4 5def tokenize_text(text: str, method: str = "wordpiece") -> str:6    """Tokenize text and return token count and details."""7    if not text:8        return json.dumps({"error": "No text provided"})9    10    if method == "whitespace":11        tokens = text.split()12    elif method == "character":13        tokens = list(text)14    elif method == "wordpiece":15        words = re.findall(r"\S+|\s+", text)16        tokens = []17        for w in words:18            if len(w) <= 4:19                tokens.append(w)20            else:21                for i in range(0, len(w), 4):22                    prefix = "##" if i > 0 else ""23                    tokens.append(prefix + w[i:i+4])24    elif method == "bpe":25        words = re.findall(r"\w+|\W", text)26        tokens = []27        for w in words:28            if w.isalpha() and len(w) <= 6:29                tokens.append(w)30            elif w.isalpha():31                mid = len(w) // 232                tokens.append(w[:mid])33                tokens.append("##" + w[mid:])34            else:35                tokens.append(w)36    37    return json.dumps({38        "method": method,39        "token_count": len(tokens),40        "char_count": len(text),41        "tokens": tokens[:50],42        "avg_token_length": round(sum(len(t) for t in tokens) / len(tokens), 1) if tokens else 0,43    }, indent=2)44 45def compare_methods(text: str) -> str:46    """Compare tokenization across methods."""47    methods = ["whitespace", "character", "wordpiece", "bpe"]48    results = {}49    for m in methods:50        if m == "whitespace":51            tokens = text.split()52        elif m == "character":53            tokens = list(text)54        elif m == "wordpiece":55            words = re.findall(r"\S+|\s+", text)56            tokens = []57            for w in words:58                if len(w) <= 4: tokens.append(w)59                else:60                    for i in range(0, len(w), 4):61                        tokens.append(("##" if i > 0 else "") + w[i:i+4])62        elif m == "bpe":63            words = re.findall(r"\w+|\W", text)64            tokens = []65            for w in words:66                if w.isalpha() and len(w) <= 6: tokens.append(w)67                elif w.isalpha():68                    mid = len(w) // 269                    tokens.append(w[:mid])70                    tokens.append("##" + w[mid:])71                else: tokens.append(w)72        results[m] = {"token_count": len(tokens), "tokens_per_char": round(len(tokens) / len(text), 3) if text else 0}73    74    return json.dumps({"char_count": len(text), "comparison": results}, indent=2)75 76with gr.Blocks(theme=gr.themes.Soft(primary_hue="blue"), title="dispatchAI Tokenizer Visualizer") as demo:77    gr.Markdown("# 🔤 dispatchAI Tokenizer Visualizer (MCP)")78    with gr.Tab("Visualize"):79        v_text = gr.Textbox(label="Text", value="The quick brown fox jumps over the lazy dog.", lines=3)80        v_method = gr.Dropdown(["whitespace", "character", "wordpiece", "bpe"], value="wordpiece", label="Method")81        v_btn = gr.Button("Tokenize", variant="primary")82        v_out = gr.Textbox(label="Tokens (JSON)", lines=15)83        v_btn.click(fn=tokenize_text, inputs=[v_text, v_method], outputs=v_out)84    with gr.Tab("Compare"):85        c_text = gr.Textbox(label="Text", value="The quick brown fox.", lines=3)86        c_btn = gr.Button("Compare Methods", variant="primary")87        c_out = gr.Textbox(label="Comparison (JSON)", lines=12)88        c_btn.click(fn=compare_methods, inputs=c_text, outputs=c_out)89    gr.Markdown("---\n🚀 [dispatchAI](https://huggingface.co/dispatchAI)")90 91demo.launch(mcp_server=True)92