teragron/llama_tokenizer

1import gradio as gr2from transformers import LlamaTokenizer3import io4import json5import os6 7os.system("pip uninstall -y gradio")8os.system("pip install gradio==4.9.0")9 10print("grd version:", gr.__version__)11 12# Load the tokenizer from the specific folder13tokenizer = LlamaTokenizer.from_pretrained("llama_tokenizer")14 15def tokenize(input_text, file=None):16    if file:17        with open(file, encoding="utf-8") as f:18            full_text = "".join(f.readlines())19    else:20        full_text = input_text21 22    tokens = tokenizer.encode(full_text, add_special_tokens=False)23    num_tokens = len(tokens)24    return num_tokens25 26with gr.Blocks() as demo:27    gr.Markdown(28    """29    # Token Counter for LLAMA30    """)31    with gr.Row():32        text_input = gr.Textbox(placeholder="Enter prompt")33        file_input = gr.File(label="Upload File", type="filepath")34        with gr.Column():35            out = gr.Textbox(label="Number of tokens")36            run_btn = gr.Button("Run")37    run_btn.click(fn=tokenize, inputs=[text_input, file_input], outputs=out)38 39demo.launch()40