numind/NuExtract3

1#!/usr/bin/env bash2set -euo pipefail3 4MODEL_NAME="${MODEL_NAME:-NM-dev/NuExtract3.4_4B-RL-400}"5VLLM_PORT="${VLLM_PORT:-8001}"6GRADIO_PORT="${GRADIO_SERVER_PORT:-7860}"7API_KEY="${OPENAI_API_KEY:-EMPTY}"8 9echo "Starting vLLM with model: ${MODEL_NAME}"10 11vllm serve "${MODEL_NAME}" \12  --served-model-name "${MODEL_NAME}" \13  --host 127.0.0.1 \14  --port "${VLLM_PORT}" \15  --trust-remote-code \16  --dtype auto \17  --max-model-len "${MAX_MODEL_LEN:-8192}" \18  --gpu-memory-utilization "${GPU_MEMORY_UTILIZATION:-0.90}" \19  --api-key "${API_KEY}" &20 21VLLM_PID=$!22 23echo "Waiting for vLLM to become ready..."24until curl -sf \25  -H "Authorization: Bearer ${API_KEY}" \26  "http://127.0.0.1:${VLLM_PORT}/v1/models" >/dev/null; do27 28  if ! kill -0 "${VLLM_PID}" 2>/dev/null; then29    echo "vLLM exited before becoming ready."30    exit 131  fi32 33  sleep 234done35 36echo "vLLM is ready. Starting Gradio..."37 38python3 /home/user/app/app.py \39  --model-name "${MODEL_NAME}" \40  --api-base "http://127.0.0.1:${VLLM_PORT}/v1" \41  --api-key "${API_KEY}" \42  --server-name "0.0.0.0" \43  --server-port "${GRADIO_PORT}"