muj0987/testonly

1from huggingface_hub import InferenceClient2import gradio as gr3 4client = InferenceClient(5    "mistralai/Mistral-7B-Instruct-v0.1"6)7 8 9def format_prompt(message, history):10  prompt = "<s>"11  for user_prompt, bot_response in history:12    prompt += f"[INST] {user_prompt} [/INST]"13    prompt += f" {bot_response}</s> "14  prompt += f"[INST] {message} [/INST]"15  return prompt16 17def generate(18    prompt, history, temperature=0.9, max_new_tokens=256, top_p=0.95, repetition_penalty=1.0,19):20    temperature = float(temperature)21    if temperature < 1e-2:22        temperature = 1e-223    top_p = float(top_p)24 25    generate_kwargs = dict(26        temperature=temperature,27        max_new_tokens=max_new_tokens,28        top_p=top_p,29        repetition_penalty=repetition_penalty,30        do_sample=True,31        seed=42,32    )33 34    formatted_prompt = format_prompt(prompt, history)35 36    stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)37    output = ""38 39    for response in stream:40        output += response.token.text41        yield output42    return output43 44 45additional_inputs=[46    gr.Slider(47        label="Temperature",48        value=0.9,49        minimum=0.0,50        maximum=1.0,51        step=0.05,52        interactive=True,53        info="Higher values produce more diverse outputs",54    ),55    gr.Slider(56        label="Max new tokens",57        value=256,58        minimum=0,59        maximum=1048,60        step=64,61        interactive=True,62        info="The maximum numbers of new tokens",63    ),64    gr.Slider(65        label="Top-p (nucleus sampling)",66        value=0.90,67        minimum=0.0,68        maximum=1,69        step=0.05,70        interactive=True,71        info="Higher values sample more low-probability tokens",72    ),73    gr.Slider(74        label="Repetition penalty",75        value=1.2,76        minimum=1.0,77        maximum=2.0,78        step=0.05,79        interactive=True,80        info="Penalize repeated tokens",81    )82]83 84 85gr.ChatInterface(86    fn=generate,87    chatbot=gr.Chatbot(show_label=False, show_share_button=False, show_copy_button=True, likeable=True, layout="panel"),88    additional_inputs=additional_inputs,89    title="""Mistral 7B"""90).launch(show_api=False)