Mykes/test

1import gradio as gr2from llama_cpp import Llama3from huggingface_hub import hf_hub_download4 5# Download the model6model_name = "Mykes/med_tinyllama_gguf"7filename = "unsloth.Q4_K_M.gguf"8model_path = hf_hub_download(repo_id=model_name, filename=filename)9 10# Initialize the model11# model = Llama(model_path=model_path, n_ctx=2048, n_threads=4, n_batch=32, use_mmap=True, use_mlock=True, rope_freq_base=10000, rope_freq_scale=1.0)12model = Llama(model_path=model_path, n_ctx=256, n_threads=2, n_batch=8, use_mlock=True)13# def preload_model(model, preload_tokens=1024):14#     # Dummy call to load model into RAM by accessing parts of it15#     try:16#         dummy_input = " " * preload_tokens17#         _ = model(dummy_input, max_tokens=1)18#         print("Model preloaded into RAM.")19#     except Exception as e:20#         print(f"Error preloading model: {e}")21 22# # Preload the model into RAM23# preload_model(model)24def respond(25    message,26    history: list[tuple[str, str]],27    system_message,28    max_tokens,29    temperature,30    top_p,31):32    history = history[-3:]33    # Construct the prompt34    prompt = f"<s>{system_message}\n\n"35    for user_msg, assistant_msg in history:36        prompt += f"<|user|>{user_msg}<|end|></s> <|assistant|>{assistant_msg}<|end|></s>"37    prompt += f"<|user|>{message}<|end|></s> <|assistant|>"38 39    # Generate response40    response = ""41    for token in model(42        prompt,43        max_tokens=max_tokens,44        temperature=temperature,45        top_p=top_p,46        stream=True,47        stop=["<|end|>", "</s>"]48    ):49        response += token['choices'][0]['text']50        yield response.strip()51 52# Create the Gradio interface53demo = gr.ChatInterface(54    respond,55    undo_btn="Отменить",56    clear_btn="Очистить",57    additional_inputs=[58        # gr.Textbox(value="You are a friendly medical assistant.", label="System message"),59        gr.Textbox(value="", label="System message"),60        gr.Slider(minimum=128, maximum=4096, value=2048, step=1, label="Max new tokens"),61        gr.Slider(minimum=0.1, maximum=2.0, value=0.7, step=0.1, label="Temperature"),62        gr.Slider(63            minimum=0.1,64            maximum=1.0,65            value=0.9,66            step=0.05,67            label="Top-p (nucleus sampling)",68        ),69    ],70    title="Med TinyLlama Chat",71    description="Chat with the Med TinyLlama model for medical information.",72)73 74if __name__ == "__main__":75    demo.launch()