peterintoai/transformers_streaming

1from threading import Thread2 3import torch4import gradio as gr5from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, TextIteratorStreamer6 7model_id = "declare-lab/flan-alpaca-xl"8torch_device = "cuda" if torch.cuda.is_available() else "cpu"9print("Running on device:", torch_device)10print("CPU threads:", torch.get_num_threads())11 12 13model = AutoModelForSeq2SeqLM.from_pretrained(model_id, load_in_8bit=True, device_map="auto")14tokenizer = AutoTokenizer.from_pretrained(model_id)15 16 17def run_generation(user_text, top_p, temperature, top_k, max_new_tokens):18    # Get the model and tokenizer, and tokenize the user text.19    model_inputs = tokenizer([user_text], return_tensors="pt").to(torch_device)20 21    # Start generation on a separate thread, so that we don't block the UI. The text is pulled from the streamer22    # in the main thread. Adds timeout to the streamer to handle exceptions in the generation thread.23    streamer = TextIteratorStreamer(tokenizer, timeout=10., skip_prompt=True, skip_special_tokens=True)24    generate_kwargs = dict(25        model_inputs,26        streamer=streamer,27        max_new_tokens=max_new_tokens,28        do_sample=True,29        top_p=top_p,30        temperature=float(temperature),31        top_k=top_k32    )33    t = Thread(target=model.generate, kwargs=generate_kwargs)34    t.start()35 36    # Pull the generated text from the streamer, and update the model output.37    model_output = ""38    for new_text in streamer:39        model_output += new_text40        yield model_output41    return model_output42 43 44def reset_textbox():45    return gr.update(value='')46 47 48with gr.Blocks() as demo:49    duplicate_link = "https://huggingface.co/spaces/joaogante/transformers_streaming?duplicate=true"50    gr.Markdown(51        "# 🤗 Transformers 🔥Streaming🔥 on Gradio\n"52        "This demo showcases the use of the "53        "[streaming feature](https://huggingface.co/docs/transformers/main/en/generation_strategies#streaming) "54        "of 🤗 Transformers with Gradio to generate text in real-time. It uses "55        f"[{model_id}](https://huggingface.co/{model_id}), "56        "loaded in 8-bit quantized form.\n\n"57        f"Feel free to [duplicate this Space]({duplicate_link}) to try your own models or use this space as a "58        "template! 💛"59    )60 61    with gr.Row():62        with gr.Column(scale=4):63            user_text = gr.Textbox(64                placeholder="Write an email about an alpaca that likes flan",65                label="User input"66            )67            model_output = gr.Textbox(label="Model output", lines=10, interactive=False)68            button_submit = gr.Button(value="Submit")69 70        with gr.Column(scale=1):71            max_new_tokens = gr.Slider(72                minimum=1, maximum=1000, value=250, step=1, interactive=True, label="Max New Tokens",73            )74            top_p = gr.Slider(75                minimum=0.05, maximum=1.0, value=0.95, step=0.05, interactive=True, label="Top-p (nucleus sampling)",76            )77            top_k = gr.Slider(78                minimum=1, maximum=50, value=50, step=1, interactive=True, label="Top-k",79            )80            temperature = gr.Slider(81                minimum=0.1, maximum=5.0, value=0.8, step=0.1, interactive=True, label="Temperature",82            )83 84    user_text.submit(run_generation, [user_text, top_p, temperature, top_k, max_new_tokens], model_output)85    button_submit.click(run_generation, [user_text, top_p, temperature, top_k, max_new_tokens], model_output)86 87    demo.queue(max_size=32).launch(enable_queue=True)88