Vishva007/NuExtract3-W4A16-AutoRound-GPTQ

vllm serve Vishva007/NuExtract3-W4A16-AutoRound-GPTQ \
  --trust-remote-code \
  --limit-mm-per-prompt '{"image": 10, "video": 0}' \
  --chat-template-content-format openai \
  --generation-config vllm \
  --max-model-len 16384 \
  --dtype bfloat16 \
  --gpu-memory-utilization 0.90

import base64
import json
from openai import OpenAI

client = OpenAI(api_key="EMPTY", base_url="http://localhost:8000/v1")

def encode_image(image_path):
    with open(image_path, "rb") as f:
        return base64.b64encode(f.read()).decode("utf-8")

data_url = f"data:image/png;base64,{encode_image('document.png')}"

# Structured JSON Extraction
template = {
    "invoice_number": "verbatim-string",
    "date": "date-time",
    "total": "number",
    "line_items": [{"description": "verbatim-string", "amount": "number"}]
}

response = client.chat.completions.create(
    model="Vishva007/NuExtract3-W4A16-AutoRound-GPTQ",
    temperature=0.2,
    messages=[
        {"role": "user", "content": [{"type": "image_url", "image_url": {"url": data_url}}]}
    ],
    extra_body={
        "chat_template_kwargs": {
            "enable_thinking": False
        }
    }
)

print(response.choices[0].message.content)