bigcode/tokenizer-testing

1import streamlit as st2from tokenizers.tools import EncodingVisualizer3from transformers import AutoTokenizer4 5st.set_page_config(page_title="BigCode Tokenizer", page_icon='👩‍💻', layout="wide")6 7models = [8    "bigcode/tokenizer-the-stack-march-sample-v3-no-prefix-spaces",9    "bigcode/tokenizer-the-stack-march-sample-v3",10    "bigcode/tokenizer-the-stack-march-sample-v2",11    "bigcode/tokenizer-the-stack-march-sample",12    "bigcode/santacoder",13    "bigcode/digit-bytelevel-bpe-jss-v1.1-49152",14    "facebook/incoder-6B",15    "Salesforce/codegen-16B-mono"16]17 18@st.cache()19def load_tokenizer():20    tokenizer = AutoTokenizer.from_pretrained(selected_model)21    return tokenizer22 23col1, col2 = st.columns([1, 2])24with col1:25    selected_model = st.selectbox("Select a tokenizer", models, key=1)26 27text = st.text_area(label="", placeholder="Text to tokenize")28button_clicked = st.button("Tokenize")29 30tokenizer = load_tokenizer()31visualizer = EncodingVisualizer(tokenizer=tokenizer._tokenizer, default_to_notebook=False)32 33if text or button_clicked:34    st.write(f"The input was split into {len(tokenizer.tokenize(text))} tokens.")35    st.components.v1.html(visualizer(text), height=1500)