ZurichNLP/subword-tokenization
1
1import gradio as gr2import utils3import os4 5# Create a custom theme6theme = gr.themes.Base(7 text_size="lg",8 radius_size="none",9 font=[gr.themes.GoogleFont('Source Sans 3'), 'ui-sans-serif', 'system-ui', 'sans-serif'],10)11 12# Load tokenizers only once during development13if gr.NO_RELOAD:14 print("Loading tokenizers...")15 all_tokenizers = utils.load_tokenizers()16 all_tokenizer_names = list(all_tokenizers.keys())17 print("Tokenizers loaded!")18 19def read_svg_file(name: str) -> str:20 """Read SVG file content."""21 icon_map = {22 "meta-llama/Llama-4-Scout-17B-16E-Instruct": "llama.svg",23 "deepseek-ai/DeepSeek-V3-0324": "deepseek.svg",24 "ZurichNLP/swissbert": "swissbert.svg",25 "mistralai/Mistral-Nemo-Instruct-2407": "mistral.svg",26 "google/gemma-3-27b-it": "gemma.svg",27 "gpt-4o": "chatgpt.svg"28 }29 icon_path = os.path.join("icons", icon_map.get(name, "chatgpt.svg"))30 try:31 with open(icon_path, 'r') as f:32 return f.read()33 except Exception as e:34 print(f"Error reading SVG file {icon_path}: {e}")35 return ""36 37def get_model_icon(name: str) -> str:38 """Get the HTML for the model icon."""39 # Skip icons for collapsed models40 if name in ["google/gemma-3-27b-it", "mistralai/Mistral-Nemo-Instruct-2407", "CohereLabs/aya-expanse-8b", "Qwen/Qwen3-4B-Base"]:41 return ""42 43 svg_content = read_svg_file(name)44 if svg_content:45 # Add viewBox and preserve aspect ratio to the SVG element46 svg_content = svg_content.replace('<svg', '<svg preserveAspectRatio="xMidYMid meet" style="height: 24px; width: 24px;"')47 # Wrap in a container that maintains aspect ratio48 return f'<div style="display: inline-block; vertical-align: middle; margin-right: 8px; height: 24px; width: 24px; overflow: hidden;">{svg_content}</div>'49 return ""50 51def process_text(text):52 """Process the input text and return visualizations for all tokenizers."""53 # Use the pre-loaded tokenizers54 visualizations = utils.visualize_tokens(text, all_tokenizers)55 return list(visualizations.values()) + [gr.update(visible=True)]56 57# Create the Gradio interface58with gr.Blocks(title="Tokens matter.", theme=theme, css="""59 .tokenizer-panel > div { background: var(--input-background-fill); }60 .no-padding { padding: 0 !important; }61 .form { border: 0 !important; }62 .html-container { line-height: 2em; !important; }63 .pending { opacity: 1; }64 65 @media (prefers-color-scheme: dark) {66 .gradio-container.gradio-container-5-29-0 .contain .html-container span.model-name { color: white !important; }67 .html-container span { color: black !important; }68 }69""") as demo:70 gr.Markdown("# Tokens matter.")71 72 with gr.Row():73 # Left column for inputs74 with gr.Column(scale=1):75 input_text = gr.Textbox(76 label="Input Text:",77 placeholder="Enter text to tokenize ...",78 value="Als Zürcher bini nöd so Fan vom FC Basel.",79 lines=3,80 elem_classes="no-padding",81 interactive=True,82 every=True, # This enables real-time updates83 )84 85 # Right column for outputs86 with gr.Column(scale=2):87 # Create output boxes for main tokenizers88 main_output_boxes = []89 more_output_boxes = []90 91 # Create 2x2 grid for main tokenizers92 with gr.Row():93 with gr.Column():94 for name in all_tokenizer_names[:2]:95 if name in ["google/gemma-3-27b-it", "mistralai/Mistral-Nemo-Instruct-2407", "CohereLabs/aya-expanse-8b", "Qwen/Qwen3-4B-Base"]:96 continue97 display_name = utils.MODEL_DISPLAY_NAMES.get(name, name)98 with gr.Group(elem_classes="tokenizer-panel"):99 gr.HTML(value=f'<div style="display: flex; align-items: center; margin-bottom: 8px;">{get_model_icon(name)}<span class="model-name" style="font-weight: bold;">{display_name}</span></div>')100 box = gr.HTML()101 main_output_boxes.append(box)102 with gr.Column():103 for name in all_tokenizer_names[2:4]:104 if name in ["google/gemma-3-27b-it", "mistralai/Mistral-Nemo-Instruct-2407", "CohereLabs/aya-expanse-8b", "Qwen/Qwen3-4B-Base"]:105 continue106 display_name = utils.MODEL_DISPLAY_NAMES.get(name, name)107 with gr.Group(elem_classes="tokenizer-panel"):108 gr.HTML(value=f'<div style="display: flex; align-items: center; margin-bottom: 8px;">{get_model_icon(name)}<span class="model-name" style="font-weight: bold;">{display_name}</span></div>')109 box = gr.HTML()110 main_output_boxes.append(box)111 112 # Display more tokenizers in accordion113 more_models = gr.Accordion("More Models", open=False, visible=False)114 with more_models:115 for name in all_tokenizer_names:116 if name in ["google/gemma-3-27b-it", "mistralai/Mistral-Nemo-Instruct-2407", "CohereLabs/aya-expanse-8b", "Qwen/Qwen3-4B-Base"]:117 display_name = utils.MODEL_DISPLAY_NAMES.get(name, name)118 with gr.Group(elem_classes="tokenizer-panel"):119 gr.HTML(value=f'<div style="display: flex; align-items: center; margin-bottom: 8px;">{get_model_icon(name)}<span class="model-name" style="font-weight: bold;">{display_name}</span></div>')120 box = gr.HTML()121 more_output_boxes.append(box)122 123 all_outputs = main_output_boxes + more_output_boxes + [more_models]124 125 # Use change event for real-time updates126 input_text.change(127 fn=process_text,128 inputs=[input_text],129 outputs=all_outputs,130 show_progress="hidden",131 )132 133 # Add examples134 gr.Examples(135 examples=[136 ["Als Zürcher bini nöd so Fan vom FC Basel."],137 ["Als Zürcher bin ich nicht sonderlich Fan des FC Basel."],138 ["En tant que Zurichois, je ne suis pas un grand fan du FC Bâle."],139 ["Come Zurighese, non sono un grande fan del FC Basilea."],140 ["Sco Turitgais na sun jau betg in grond fan da l'FC Basilea."],141 ["As a Zurich resident, I am not a big fan of FC Basel."],142 ],143 inputs=input_text144 )145 146if __name__ == "__main__":147 demo.launch()148