OpenSourceRonin/VPTQ-demo

1import spaces2import gradio as gr3from huggingface_hub import InferenceClient4 5 6"""7For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference8"""9from vptq.app_utils import get_chat_loop_generator10 11model_list=["VPTQ-community/Meta-Llama-3.1-8B-Instruct-v12-k65536-4096-woft", 12            "VPTQ-community/Meta-Llama-3.1-70B-Instruct-v8-k32768-0-woft", 13            "VPTQ-community/Qwen2.5-7B-Instruct-v8-k256-256-woft",14            "VPTQ-community/Qwen2.5-14B-Instruct-v8-k256-256-woft",15            "VPTQ-community/Qwen2.5-32B-Instruct-v16-k65536-65536-woft",16            "VPTQ-community/Qwen2.5-72B-Instruct-v8-k65536-0-woft",17            ]18 19current_model_g = model_list[0]20chat_completion = get_chat_loop_generator(current_model_g)21 22@spaces.GPU23def update_title_and_chatmodel(model):24    model = str(model)25    global chat_completion26    global current_model_g27    if model != current_model_g:28        current_model_g = model29        chat_completion = get_chat_loop_generator(current_model_g)30    return model31    32 33@spaces.GPU34def respond(35    message,36    history: list[tuple[str, str]],37    system_message,38    max_tokens,39    temperature,40    top_p,41):42    messages = [{"role": "system", "content": system_message}]43 44    for val in history:45        if val[0]:46            messages.append({"role": "user", "content": val[0]})47        if val[1]:48            messages.append({"role": "assistant", "content": val[1]})49 50    messages.append({"role": "user", "content": message})51 52    response = ""53 54    for message in chat_completion(55            messages,56            max_tokens=max_tokens,57            stream=True,58            temperature=temperature,59            top_p=top_p,60    ):61        token = message62 63        response += token64        yield response65 66 67 68css = """69h1 {70  text-align: center;71  display: block;72}73"""74"""75For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface76"""77chatbot = gr.Chatbot(label="Gradio ChatInterface")78with gr.Blocks() as demo:79    with gr.Column(scale=1):80        title_output = gr.Markdown("Please select a model to run")81        chat_demo = gr.ChatInterface(82            respond,83            #chatbot=chatbot,84            additional_inputs_accordion=gr.Accordion(85                label="⚙️ Parameters", open=False, render=False86            ),87            fill_height=False,88            additional_inputs=[89                gr.Textbox(value="You are a friendly Chatbot.", label="System message"),90                gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),91                gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),92                gr.Slider(93                    minimum=0.1,94                    maximum=1.0,95                    value=0.95,96                    step=0.05,97                    label="Top-p (nucleus sampling)",98                ),99            ],100        )101        model_select = gr.Dropdown(102                    choices=model_list, 103                    label="Models", 104                    value=model_list[0],105        )106 107        model_select.change(update_title_and_chatmodel, inputs=[model_select], outputs=title_output)108 109 110if __name__ == "__main__":111    demo.launch()