towing/byt5-tokenizer-utils

1import gradio as gr2from transformers import ByT5Tokenizer3import json4 5ACTIONS = ["text2ids", "text2tokens", "ids2tokens", "tokens2ids and JSON requires double quotes", "ids2text"]6 7def translate(model, action, inputs):8    tokenizer = ByT5Tokenizer.from_pretrained(model)9    vocab_size = tokenizer.vocab_size10    len_tokenizer = len(tokenizer)11    input = inputs12    output = ''13    if action == ACTIONS[0]:14      input_ids = tokenizer(input)['input_ids']15      output = input_ids16 17    if action == ACTIONS[1]:18      input_ids = tokenizer(input)['input_ids']19      tokens = tokenizer.convert_ids_to_tokens(input_ids)20      output = tokens21    if action == ACTIONS[2]:22      list = json.loads(input)23      tokens = tokenizer.convert_ids_to_tokens(list)24      output = tokens25    if action == ACTIONS[3]:26      list = json.loads(input)27      tokens = tokenizer.convert_tokens_to_ids(list)28      output = tokens29    if action == ACTIONS[4]:30      list = json.loads(input)31      text = tokenizer.decode(list)32      output = text    33        34        35    return f'{output}\n\n\n\nother infos:\njson:{json.dumps(output)} \nvocab_size: {vocab_size}\nlen(tokenizer): {len_tokenizer}'36 37demo = gr.Interface(38    fn=translate,39    inputs=[40        gr.components.Textbox(label="MODEL NAME, eg: google/byt5-small", value="google/byt5-small"),41        gr.components.Dropdown(label="ACTIONS", choices=ACTIONS),42        gr.components.Textbox(label="INPUTS"),43    ],44    outputs=["text"],45    cache_examples=False,46    title="Test T5Tokenizer",47    description="▁Test, ▁To, ken, izer, s, ▁happily, !, </s>"48)49 50demo.launch(debug=True)