ngxtm/self-hosted-llm

1import os2 3import gradio as gr4import spaces5from openai import OpenAI6 7 8# Hugging Face's ZeroGPU runtime requires at least one top-level @spaces.GPU9# function at startup. Inference itself runs on the school's vLLM server, so10# this compatibility function is intentionally never called.11@spaces.GPU12def _zerogpu_startup_check():13    return None14 15 16client = OpenAI(17    base_url=os.environ["BASE_URL"].rstrip("/"),18    api_key=os.environ["API_KEY"],19)20MODEL = os.getenv("MODEL", "gemma-3-4b-it")21 22 23def _text_from_content(content):24    if isinstance(content, str):25        return content26    if isinstance(content, list):27        return "".join(28            item.get("text", "")29            for item in content30            if isinstance(item, dict) and item.get("type") == "text"31        )32    return ""33 34 35def chat(message, history):36    messages = []37    for item in history:38        role = item.get("role")39        content = _text_from_content(item.get("content"))40        if role in ("user", "assistant") and content:41            messages.append({"role": role, "content": content})42 43    messages.append({"role": "user", "content": message})44 45    stream = client.chat.completions.create(46        model=MODEL,47        messages=messages,48        stream=True,49        max_tokens=1024,50        temperature=0.7,51    )52 53    reply = ""54    for chunk in stream:55        if chunk.choices and chunk.choices[0].delta.content:56            reply += chunk.choices[0].delta.content57            yield reply58 59 60demo = gr.ChatInterface(61    chat,62    title="Gemma 3 trên RTX 3090 của trường",63    description="Model chạy trên vLLM của server trường.",64)65 66demo.launch()67