build-small-hackathon/QED

1import modal2import subprocess3 4app = modal.App("llama-server")5 6MINUTES = 607GPU_CONFIG = "A100-40GB"8cache_dir = "/root/.cache/llama.cpp"9 10#REPO_ID = "google/gemma-4-31B-it-qat-q4_0-gguf"11#MODEL_FILE = "gemma-4-31B_q4_0-it.gguf"12#MMPROJ_FILE = "gemma-4-31B-it-mmproj.gguf"13 14 15REPO_ID = "unsloth/Qwen3.6-27B-GGUF"16MODEL_FILE = "Qwen3.6-27B-Q4_K_M.gguf"17 18 19cuda_tag = "12.4.0-devel-ubuntu22.04"20 21model_cache = modal.Volume.from_name("llamacpp-cache", create_if_missing=True)22 23image = (24    modal.Image.from_registry(f"nvidia/cuda:{cuda_tag}", add_python="3.11")25    .apt_install(26        "git",27        "build-essential",28        "cmake",29        "curl",30        "libcurl4-openssl-dev",31        "libssl-dev",32    )33    .run_commands("git clone https://github.com/ggerganov/llama.cpp && cd llama.cpp && git pull origin master")34    .run_commands(35        "cmake llama.cpp -B llama.cpp/build "36        "-DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=ON -DLLAMA_CURL=ON -DLLAMA_OPENSSL=ON"37    )38    .run_commands(39        "cmake --build llama.cpp/build --config Release -j "40        "--target llama-server "41    )42    .run_commands("cp llama.cpp/build/bin/llama-server /usr/local/bin/")43    .entrypoint([])44)45 46download_image = (47    modal.Image.debian_slim(python_version="3.11")48    .pip_install("huggingface_hub[hf_transfer]==0.26.2")49    .env({"HF_HUB_ENABLE_HF_TRANSFER": "1"})50)51 52 53@app.function(54    image=download_image,55    volumes={cache_dir: model_cache},56    timeout=20 * MINUTES,57)58def download_model():59    from huggingface_hub import hf_hub_download60 61    for filename in [MODEL_FILE]: #, MMPROJ_FILE 62        hf_hub_download(63            repo_id=REPO_ID,64            filename=filename,65            local_dir=cache_dir,66        )67 68    model_cache.commit()69 70 71@app.function(72    image=image,73    gpu=GPU_CONFIG,74    volumes={cache_dir: model_cache},75    timeout=60 * MINUTES,76    max_containers=1,77    min_containers=1, #← for judging so that judges can start with a warm container78)79 80@modal.web_server(port=8080, startup_timeout=5 * MINUTES)81def serve():82     import shutil83     import os84     import urllib.request85     import json86     import time87 88     local_model = f"/tmp/{MODEL_FILE}"89     #local_mmproj = f"/tmp/{MMPROJ_FILE}"90 91     if not os.path.exists(local_model):92        print("Copying model to local storage...", flush=True)93        shutil.copy2(f"{cache_dir}/{MODEL_FILE}", local_model)94 95        #shutil.copy2(f"{cache_dir}/{MMPROJ_FILE}", local_mmproj)96        print("Copy complete.", flush=True)97 98     subprocess.Popen([99        "llama-server",100        "-m",local_model ,101       # "--mmproj", local_mmproj, 102        "--host", "0.0.0.0",103        "--port", "8080",104        "--ctx-size", "4096",105        "-ngl", "999",106        "--flash-attn","on", 107        "-np","1",108        "-b", "2048",109        "-ub", "512",110        "--cache-type-k", "q8_0",111        "--cache-type-v", "q8_0",112        "-t", "8",113        #"--no-mmap",  ← loads weight into ram/not needed because of tmp 114        #"--no-warmup",  skip empty run 115    ])116 117        # wait for server ready118     for _ in range(60):119            try:120                urllib.request.urlopen("http://localhost:8080/health")121                break122            except Exception:123                time.sleep(5)124 125        # fire a real request to compile actual CUDA graphs126     payload = json.dumps({127            "model": "any",128            "messages": [{"role": "user", "content": "hi"}],129            "max_tokens": 10,130            "chat_template_kwargs": {"enable_thinking": False}131        }).encode()132     req = urllib.request.Request(133            "http://localhost:8080/v1/chat/completions",134            data=payload,135            headers={"Content-Type": "application/json"}136        )137     urllib.request.urlopen(req)138     print("Warmup complete.", flush=True)139 140 141 142 143@app.local_entrypoint()144def main():145    download_model.remote()146