bigscience-data/bigscience-tokenizer

1import streamlit as st2from tokenizers.tools import EncodingVisualizer3from transformers import AutoTokenizer4 5st.set_page_config(page_title="BigScience Tokenizer", page_icon='👩‍💻', layout="wide")6 7tokenizer = AutoTokenizer.from_pretrained('bigscience/tokenizer', use_auth_token=st.secrets["AUTH_TOKEN"])8visualizer = EncodingVisualizer(tokenizer=tokenizer._tokenizer, default_to_notebook=False)9 10text = st.text_area(label="", placeholder="Text to tokenize")11button_clicked = st.button("Tokenize")12if text or button_clicked:13    st.write(f"The input was split into {len(tokenizer.tokenize(text))} tokens.")14    st.components.v1.html(visualizer(text), height=1500)15       16