anveshplus/BPE-Tokenizer
1
1import streamlit as st2import encoder_parallel_telugu as encode_parallel3from consecutive_tokens import get_consecutive_tokens, search_consecutive_tokens4import tokenizer5 6def encode(text):7 if text == "":8 return "Enter text to encode..."9 encoded_tokens = [token.encode('utf-8') for token in text]10 consective_tokens = get_consecutive_tokens(encoded_tokens,window_size=4)11 # Reading vocabulary from file12 formatted_vocab = tokenizer.read_vocab_from_file()13 # Invert vocabulary14 inverted_vocab = {v: k for k, v in formatted_vocab.items()}15 # Expand vocabulary16 decoder_map = tokenizer.expand_vocab(inverted_vocab)17 # Invert back again after expansion18 re_inverted_vocab = {k: v for v, k in decoder_map.items()}19 20 # encoded_tokens = [re_inverted_vocab.get(token) for token in consective_tokens]21 encoded_tokens, printer_dict = search_consecutive_tokens(consective_tokens, re_inverted_vocab)22 print(encoded_tokens) 23 printer = [(b''.join(key).decode('utf-8'), value) for key, value in printer_dict.items()]24 return f"Encoded: {encoded_tokens} , Printer: {printer}"25 26def decode(text):27 # Placeholder for decoding logic28 toks_li = [token for token in text.split(',')]29 # Reading vocabulary from file30 formatted_vocab = tokenizer.read_vocab_from_file()31 # Invert vocabulary32 inverted_vocab = {v: k for k, v in formatted_vocab.items()}33 # Expand vocabulary34 decoder_map = tokenizer.expand_vocab(inverted_vocab)35 decoded_tokens = [decoder_map.get(int(token)) for token in toks_li]36 decoded_tokens = [item for token in decoded_tokens for item in token]37 tokens = [token.decode('utf-8') for token in decoded_tokens]38 decoded_tokens = b''.join(decoded_tokens)39 decoded_tokens = decoded_tokens.decode('utf-8')40 return f"->Decoded: {decoded_tokens} "41 42st.set_page_config(page_title="Telugu BPE Tokenizer", layout="centered", initial_sidebar_state="expanded")43st.markdown("<h1 style='color: #2ECC40; text-align: center;'>Telugu BPE Tokenizer</h1>", unsafe_allow_html=True)44 45# Add custom CSS for styling46st.markdown(47 """48 <style>49 .title {50 color: #FFFFFF;51 background-color: #2C3E50;52 font-family: "Arial", sans-serif;53 font-size: 2.5em;54 padding: 20px;55 text-align: center;56 }57 .subheader {58 color: #2980B9;59 font-size: 1.5em;60 }61 .text-area {62 background-color: #ECF0F1;63 border: 1px solid #BDC3C7;64 border-radius: 5px;65 }66 .orange-button {67 background-color: #FFA500; /* Bright orange color */68 color: white;69 border: none;70 border-radius: 5px;71 padding: 10px 20px;72 cursor: pointer;73 }74 </style>75 """, unsafe_allow_html=True76)77 78# Create two columns for encoder and decoder79col1, col2 = st.columns(2)80 81with col1:82 st.markdown("<div class='subheader' style='color: #FFA500;'>Encoder</div>", unsafe_allow_html=True)83 encoder_input = st.text_area("Input Text for Encoding", placeholder="Enter text to encode...", key="encoder_input", height=100)84 if st.button("Encode", key="encode_button"):85 encoder_output = encode(encoder_input)86 st.text_area("Encoded Output", value=encoder_output, height=100, disabled=True, key="encoder_output")87 88with col2:89 st.markdown("<div class='subheader' style='color: #FFA500;'>Decoder</div>", unsafe_allow_html=True)90 decoder_input = st.text_area("Input Text for Decoding", placeholder="51,32,63,94,15", key="decoder_input", height=100)91 if st.button("Decode", key="decode_button"):92 decoder_output = decode(decoder_input)93 st.text_area("Decoded Output", value=decoder_output, height=100, disabled=True, key="decoder_output")94 95st.markdown("<hr style='border: 1px solid #BDC3C7;'>", unsafe_allow_html=True) # Add a horizontal line above the section in grey96# Add sample texts at the end of the page97st.markdown("<div class='subheader'>Sample Texts</div>", unsafe_allow_html=True)98 99st.markdown("<br>", unsafe_allow_html=True)100 101st.markdown("<div style='margin-bottom: 10px;'> <span style='font-weight: bold;'>తెలుగు భాష ఒక ద్రావిడ భాష.</span></div>", unsafe_allow_html=True)102st.markdown("<div style='margin-bottom: 10px;'> <span style='font-weight: bold;'>మోదీ మార్కు రాజకీయం.</span></div>", unsafe_allow_html=True)103st.markdown("<div style='margin-bottom: 10px;'> <span style='font-weight: bold;'>రెండు విధాలా ఆలోచిస్తా.</span></div>", unsafe_allow_html=True)104 105if __name__ == "__main__":106 st.write("Streamlit app is running...")107 st.write("To view this page in your browser, run the command: `streamlit run app.py` and open the provided local URL.")108