QuixiAI/chat

1import spaces2import json3import subprocess4from llama_cpp import Llama5from llama_cpp_agent import LlamaCppAgent6from llama_cpp_agent import MessagesFormatterType7from llama_cpp_agent.providers import LlamaCppPythonProvider8from llama_cpp_agent.chat_history import BasicChatHistory9from llama_cpp_agent.chat_history.messages import Roles10import gradio as gr11from huggingface_hub import hf_hub_download12from ui import css, PLACEHOLDER13 14llm = None15llm_model = None16# hf_hub_download(repo_id="bartowski/dolphin-2.9.1-yi-1.5-34b-GGUF", filename="dolphin-2.9.1-yi-1.5-34b-Q6_K.gguf",  local_dir = "./models")17# hf_hub_download(repo_id="crusoeai/dolphin-2.9.1-llama-3-70b-GGUF", filename="dolphin-2.9.1-llama-3-70b.Q3_K_M.gguf",  local_dir = "./models")18hf_hub_download(repo_id="bartowski/cognitivecomputations_Dolphin3.0-R1-Mistral-24B-GGUF", filename="cognitivecomputations_Dolphin3.0-R1-Mistral-24B-Q8_0.gguf",  local_dir = "./models")19# hf_hub_download(repo_id="mradermacher/Dolphin3.0-Mistral-24B-GGUF", filename="Dolphin3.0-Mistral-24B.Q8_0.gguf",  local_dir = "./models")20# hf_hub_download(repo_id="kroonen/dolphin-2.9.2-Phi-3-Medium-GGUF", filename="dolphin-2.9.2-Phi-3-Medium-Q6_K.gguf",  local_dir = "./models")21hf_hub_download(repo_id="cognitivecomputations/dolphin-2.9.2-qwen2-72b-gguf", filename="qwen2-Q3_K_M.gguf",  local_dir = "./models")22 23@spaces.GPU(duration=120)24def respond(25    message,26    history: list[tuple[str, str]],27    model,28    max_tokens,29    temperature,30    top_p,31    top_k,32    repeat_penalty,33):34    global llm35    global llm_model36 37    if llm is None or llm_model != model:38        llm = Llama(39            model_path=f"models/{model}",40            flash_attn=True,41            n_gpu_layers=81,42            n_batch=1024,43            n_ctx=8192,44        )45        llm_model=model46    provider = LlamaCppPythonProvider(llm)47 48    agent = LlamaCppAgent(49        provider,50        system_prompt="You are Dolphin, an AI assistant that helps humanity, trained to specialize in reasoning and first-principles analysis. When responding, always format your replies using <think>{reasoning}</think>{answer}. Use at least 6 reasoning steps and perform a root cause analysis before answering. However, if the answer is very easy and requires little thought, you may leave the <think></think> block empty. Your responses should be detailed, structured with rich Markdown formatting, and engaging with emojis. Be extensive in your explanations, just as the greatest scientific minds would be. Always reason through the problem first, unless it's trivial, in which case you may answer directly.",51        predefined_messages_formatter_type=MessagesFormatterType.CHATML,52        debug_output=True53    )54    55    settings = provider.get_provider_default_settings()56    settings.temperature = temperature57    settings.top_k = top_k58    settings.top_p = top_p59    settings.max_tokens = max_tokens60    settings.repeat_penalty = repeat_penalty61    settings.stream = True62 63    messages = BasicChatHistory()64 65    for msn in history:66        user = {67            'role': Roles.user,68            'content': msn[0]69        }70        assistant = {71            'role': Roles.assistant,72            'content': msn[1]73        }74        messages.add_message(user)75        messages.add_message(assistant)76    77    stream = agent.get_chat_response(message, llm_sampling_settings=settings, chat_history=messages, returns_streaming_generator=True, print_output=False)78    79    outputs = ""80    for output in stream:81        outputs += output82        yield outputs83 84demo = gr.ChatInterface(85    respond,86    additional_inputs=[87        gr.Dropdown([88            'cognitivecomputations_Dolphin3.0-R1-Mistral-24B-Q8_0.gguf',89            'qwen2-Q3_K_M.gguf'90        ], value="cognitivecomputations_Dolphin3.0-R1-Mistral-24B-Q8_0.gguf", label="Model"),91        gr.Slider(minimum=1, maximum=8192, value=8192, step=1, label="Max tokens"),92        gr.Slider(minimum=0.05, maximum=4.0, value=0.6, step=0.1, label="Temperature"),93        gr.Slider(94            minimum=0.1,95            maximum=1.0,96            value=0.95,97            step=0.05,98            label="Top-p",99        ),100        gr.Slider(101            minimum=0,102            maximum=100,103            value=40,104            step=1,105            label="Top-k",106        ),107        gr.Slider(108            minimum=0.0,109            maximum=2.0,110            value=1.1,111            step=0.1,112            label="Repetition penalty",113        ),114    ],115    theme=gr.themes.Soft(primary_hue="indigo", secondary_hue="blue", neutral_hue="gray",font=[gr.themes.GoogleFont("Exo"), "ui-sans-serif", "system-ui", "sans-serif"]).set(116        body_background_fill_dark="#0f172a",117        block_background_fill_dark="#0f172a",118        block_border_width="1px",119        block_title_background_fill_dark="#070d1b",120        input_background_fill_dark="#0c1425",121        button_secondary_background_fill_dark="#070d1b",122        border_color_accent_dark="#21293b",123        border_color_primary_dark="#21293b",124        background_fill_secondary_dark="#0f172a",125        color_accent_soft_dark="transparent"126    ),127    css=css,128    retry_btn="Retry",129    undo_btn="Undo",130    clear_btn="Clear",131    submit_btn="Send",132    description="Cognitive Computation: Chat Dolphin 🐬",133    chatbot=gr.Chatbot(134        scale=1,135        placeholder=PLACEHOLDER,136        show_copy_button=True137    )138)139 140if __name__ == "__main__":141    demo.launch()142