nazib61/llama-cpp-python-cpu-gradio

1import os2import copy3import gradio as gr4from llama_cpp import Llama5from huggingface_hub import hf_hub_download6 7# Fix for Python 3.13: audioop was removed from the standard library.8# This try/except block handles the missing dependency if audioop-lts is installed.9try:10    import audioop11except ImportError:12    try:13        import audioop_lts as audioop14    except ImportError:15        print("Warning: audioop not found. If Gradio fails to load, install 'audioop-lts'.")16 17# 1. Download the model correctly18# Repo: unsloth/NVIDIA-Nemotron-3-Nano-4B-GGUF19# File: NVIDIA-Nemotron-3-Nano-4B-Q4_K_M.gguf20model_path = hf_hub_download(21    repo_id=os.environ.get("REPO_ID", "unsloth/NVIDIA-Nemotron-3-Nano-4B-GGUF"),22    filename=os.environ.get("MODEL_FILE", "NVIDIA-Nemotron-3-Nano-4B-Q4_K_M.gguf"),23)24 25# 2. Initialize the Llama model26llm = Llama(27    model_path=model_path,28    n_ctx=2048,29    n_gpu_layers=-1, # -1 uses all available GPU layers, change to 0 for CPU only30) 31 32def generate_text(33    message,34    history: list[tuple[str, str]],35    system_message,36    max_tokens,37    temperature,38    top_p,39):40    temp = ""41    # Standard ChatML / Llama format logic42    input_prompt = f"[INST] <<SYS>>\n{system_message}\n<</SYS>>\n\n "43    for interaction in history:44        input_prompt += f"{interaction[0]} [/INST] {interaction[1]} </s><s> [INST] "45    46    input_prompt += f"{message} [/INST] "47 48    output = llm(49        input_prompt,50        temperature=temperature,51        top_p=top_p,52        top_k=40,53        repeat_penalty=1.1,54        max_tokens=max_tokens,55        stop=[56            "[/INST]",57            "</s>",58            "<|endoftext|>",59            "USER:",60            "ASSISTANT:",61        ],62        stream=True,63    )64    65    for out in output:66        stream = copy.deepcopy(out)67        temp += stream["choices"][0]["text"]68        yield temp69 70# 3. Define the Gradio Interface71demo = gr.ChatInterface(72    generate_text,73    title="NVIDIA Nemotron-3 Nano (Llama-cpp)",74    description="Running NVIDIA Nemotron-3-Nano-4B via llama-cpp-python",75    examples=[76        ['How to setup a human base on Mars? Give short answer.'],77        ['Explain theory of relativity to me like I’m 8 years old.'],78        ['What is 9,000 * 9,000?']79    ],80    cache_examples=False,81    additional_inputs=[82        gr.Textbox(value="You are a helpful and friendly AI assistant.", label="System message"),83        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),84        gr.Slider(minimum=0.1, maximum=2.0, value=0.7, step=0.1, label="Temperature"),85        gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p"),86    ],87)88 89if __name__ == "__main__":90    demo.launch(server_name="0.0.0.0", server_port=7860)