patdev/k3-a40-bootstrap

1#!/usr/bin/env bash2# Flash-Next sous SGLang officiel (image lmsysorg/sglang:qwen38flashnext).3#4# POURQUOI CETTE VOIE. vLLM refuse le KV fp8 sur ce modele :5#   NotImplementedError: Qwen3.8-Flash-Next QSA requires a BF16 main KV cache6# -> 26 078 octets/jeton irreductibles, pool 604 705, donc 500K en SOLO.7# SGLang sert le MEME checkpoint RadixArk en fp8 (pool mesure 824 384 chez8# Pennyroyal). On ne prend PAS Pennyroyal : ses releases n'ont aucune roue9# binaire et il exige sgl-kernel 0.4.6.post1 (index public : 0.3.21) + CUDA 13.310# + GCC 15.3.1 -> compilation longue et chaine incoherente garantie.11# L'image officielle donne le meme moteur sans rien compiler.12#13# PRUDENCE : montee par etages, un levier a la fois.14#   VL_SPEC=off / VL_HICACHE=off par defaut ; on valide d'abord fp8 + 524288.15# En cas d'echec de flag, on publie `--help` dans le journal au lieu de deviner.16set -u17DEPOT=patdev/k3-a40-bootstrap18MODELE="${VL_MODEL:-RadixArk/Qwen3.8-Flash-Next-NVFP4}"19CTX="${VL_CTX:-524288}"20PORT_SGL=3000021PORT_PONT="${VL_PROXY_PORT:-8080}"22echo "[SGL] $(date -u +%H:%M:%S) modele=$MODELE ctx=$CTX"23mkdir -p /travail && cd /travail24 25cat > /travail/publier.py <<'PYFIN'26import os, sys27from huggingface_hub import HfApi28HfApi().upload_file(path_or_fileobj="/travail/journal.txt", path_in_repo=sys.argv[1],29                    repo_id="patdev/k3-a40-bootstrap", commit_message="journal sglang",30                    token=os.environ["HF_TOKEN"])31PYFIN32 33if [ -n "${HF_TOKEN:-}" ]; then34  CLE_J="etat/${RUNPOD_POD_ID:-sgl}.log"35  echo "[SGL] journal : https://huggingface.co/$DEPOT/resolve/main/$CLE_J"36  (37    while true; do38      {39        echo "=== sglang-flashnext | $(date -u +%H:%M:%S) UTC ==="40        nvidia-smi --query-gpu=memory.used,memory.total --format=csv,noheader 2>/dev/null41        free -g 2>/dev/null | head -242        echo "--- aide (si flag refuse) ---"; tail -c 4000 /travail/aide.txt 2>/dev/null43        echo "--- sglang.log (fin) ---"; tail -c 40000 /travail/sglang.log 2>/dev/null44        echo "--- pont.log (fin) ---";   tail -c 3000 /travail/pont.log 2>/dev/null45      } > /travail/journal.txt 2>/dev/null46      python3 /travail/publier.py "$CLE_J" >/dev/null 2>&147      if curl -sf --max-time 4 "http://127.0.0.1:$PORT_PONT/v1/models" >/dev/null 2>&1; then48        sleep 30049      else50        sleep 3051      fi52    done53  ) &54fi55 56# YaRN facteur 2 : 262144 x 2 = 524288 exactement. Syntaxe SGLang =57# --json-model-override-args, structure text_config.rope_parameters (la meme58# que vLLM ; a la racine elle serait ignoree en silence).59OVERRIDE=""60if [ "$CTX" -gt 262144 ]; then61  export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=162  OVERRIDE='{"text_config":{"rope_parameters":{"mrope_interleaved":true,"mrope_section":[11,11,10],"rope_type":"yarn","rope_theta":10000000,"partial_rotary_factor":0.25,"factor":'"${VL_YARN_FACTOR:-2.0}"',"original_max_position_embeddings":262144}}}'63  echo "[SGL] YaRN facteur ${VL_YARN_FACTOR:-2.0} -> $CTX"64fi65 66SPEC=()67if [ "${VL_SPEC:-off}" = "mtp" ]; then68  SPEC=( --speculative-algorithm NEXTN --speculative-num-steps "${VL_MTP_N:-3}"69         --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 )70  echo "[SGL] speculation NEXTN ${VL_MTP_N:-3}"71fi72 73HICACHE=()74if [ "${VL_HICACHE:-off}" != "off" ]; then75  # L2 en RAM hote uniquement : backend kernel, PAS de NIXL (io_uring est76  # bloque par seccomp chez RunPod, mesure du 04/09).77  HICACHE=( --enable-hierarchical-cache --hicache-size "${VL_HICACHE:-32}"78            --hicache-write-policy write_through --hicache-io-backend kernel )79  echo "[SGL] HiCache L2 RAM ${VL_HICACHE} Go (sans NIXL)"80fi81 82echo "[SGL] lancement sur 127.0.0.1:$PORT_SGL"83python3 -m sglang.launch_server \84  --model-path "$MODELE" \85  --served-model-name flashnext \86  --host 127.0.0.1 --port "$PORT_SGL" \87  --trust-remote-code \88  --tp 1 \89  --context-length "$CTX" \90  ${OVERRIDE:+--json-model-override-args "$OVERRIDE"} \91  --kv-cache-dtype "${VL_KV:-fp8_e4m3}" \92  --mem-fraction-static "${VL_MEMFRAC:-0.90}" \93  --page-size "${VL_PAGE:-64}" \94  --max-running-requests "${VL_SEQS:-4}" \95  --chunked-prefill-size "${VL_CHUNK:-4096}" \96  --max-mamba-cache-size "${VL_MAMBA:-24}" \97  --moe-runner-backend "${VL_MOE:-flashinfer_cutlass}" \98  ${VL_PLE_FLAG:---ple-offload-embedding} \99  "${SPEC[@]}" "${HICACHE[@]}" \100  > /travail/sglang.log 2>&1 &101 102python3 -c "import fastapi, httpx, uvicorn" 2>/dev/null || pip install -q fastapi uvicorn httpx103 104for i in $(seq 1 240); do105  sleep 20106  curl -sf --max-time 5 "http://127.0.0.1:$PORT_SGL/v1/models" >/dev/null 2>&1 && break107  if grep -aqE "unrecognized arguments|invalid choice|error: argument" /travail/sglang.log; then108    echo "[SGL] FLAG REFUSE — publication de l'aide"109    grep -aE "unrecognized arguments|invalid choice|error: argument" /travail/sglang.log | head -3110    python3 -m sglang.launch_server --help > /travail/aide.txt 2>&1111    sleep 3600; exit 1112  fi113done114if ! curl -sf --max-time 5 "http://127.0.0.1:$PORT_SGL/v1/models" >/dev/null 2>&1; then115  echo "[SGL] SGLang jamais pret"; tail -40 /travail/sglang.log; sleep 3600; exit 1116fi117echo "[SGL] SGLang pret a $(date -u +%H:%M:%S)"118grep -aE "KV cache|max_total_num_tokens|Capture cuda graph" /travail/sglang.log | tail -4119 120for i in 1 2 3 4 5; do121  curl -sL "https://huggingface.co/$DEPOT/resolve/main/anthropic_proxy.py" -o /travail/anthropic_proxy.py122  [ -s /travail/anthropic_proxy.py ] && break123  sleep 4124done125echo "[SGL] pont Anthropic sur 0.0.0.0:$PORT_PONT"126cd /travail127VL_UPSTREAM="http://127.0.0.1:$PORT_SGL" VL_SERVED_NAME=flashnext \128VL_REAL_MODEL="$MODELE" VL_MAX_OUTPUT=32768 VL_TIMEOUT=240 \129setsid nohup python3 -m uvicorn anthropic_proxy:app --host 0.0.0.0 --port "$PORT_PONT" \130  --log-level warning > /travail/pont.log 2>&1 < /dev/null &131for i in $(seq 1 20); do132  sleep 3133  curl -sf --max-time 4 "http://127.0.0.1:$PORT_PONT/v1/models" >/dev/null 2>&1 \134    && { echo "[SGL] PRET a $(date -u +%H:%M:%S)"; break; }135done136sleep infinity137