Hamid248/ornith-runner

1import gradio as gr2import spaces3import llama_cpp4from huggingface_hub import hf_hub_download5 6 7MODEL_REPO = "Qwen/Qwen2.5-0.5B-Instruct-GGUF"8MODEL_FILE = "qwen2.5-0.5b-instruct-q4_k_m.gguf"9 10model = None11 12 13@spaces.GPU14def generate(prompt):15    global model16 17    if model is None:18        model_path = hf_hub_download(19            repo_id=MODEL_REPO,20            filename=MODEL_FILE,21        )22 23        model = llama_cpp.Llama(24            model_path=model_path,25            n_ctx=2048,26            n_gpu_layers=-1,27            verbose=True,28        )29 30    result = model(31        prompt,32        max_tokens=80,33        temperature=0.2,34    )35 36    return result["choices"][0]["text"]37 38 39demo = gr.Interface(40    fn=generate,41    inputs=gr.Textbox(42        label="Prompt",43        value="Reply with exactly: GPU TEST OK",44    ),45    outputs=gr.Textbox(label="Response"),46    title="Ornith Runner - Real GPU Inference Test",47)48 49demo.launch()