NajahUniv/AraUni-Granite-311M-Intent-Classifier

npm install @huggingface/transformers onnxruntime-web

import { AutoTokenizer } from "@huggingface/transformers";
import * as ort from "onnxruntime-web/webgpu";

ort.env.wasm.wasmPaths = "/ort/"; // Serve matching ORT Web helper files here.
const executionProviders = ["webgpu"];

const modelId = "NajahUniv/AraUni-Granite-311M-Intent-Classifier";
const revision = "v0.4.1";
const artifactRoot =
  `https://huggingface.co/${modelId}/resolve/${revision}/webgpu/`;
const config = await fetch(`${artifactRoot}webgpu_config.json`).then((response) => {
  if (!response.ok) throw new Error(`Could not load browser config: ${response.status}`);
  return response.json();
});
const tokenizer = await AutoTokenizer.from_pretrained(modelId, { revision });
const session = await ort.InferenceSession.create(
  `${artifactRoot}${config.model_file}`,
  { executionProviders, graphOptimizationLevel: "all" },
);

const text = "ما هي شروط القبول في الجامعة؟";
const encoded = await tokenizer(text, {
  padding: "max_length",
  truncation: true,
  max_length: config.max_length,
});
const feeds = Object.fromEntries(
  config.input_names.map((name) => [
    name,
    new ort.Tensor("int64", encoded[name].data, encoded[name].dims),
  ]),
);
const output = await session.run(feeds);
const logits = output[config.output_name].data;
const predictions = config.labels.map((label, index) => {
  const score = 1 / (1 + Math.exp(-Number(logits[index])));
  const threshold = Number(config.thresholds[label]);
  return { label, score, threshold, selected: score >= threshold };
});
console.table(predictions);

import * as ort from "onnxruntime-web/wasm";

ort.env.wasm.wasmPaths = "/ort/";
ort.env.wasm.numThreads = 1; // Works without cross-origin isolation.
const executionProviders = ["wasm"];

import torch
from transformers import AutoModelForSequenceClassification, AutoTokenizer

model_id = "NajahUniv/AraUni-Granite-311M-Intent-Classifier"
model_revision = "v0.4.1"
tokenizer = AutoTokenizer.from_pretrained(
    model_id,
    revision=model_revision,
    trust_remote_code=True,
)
model = AutoModelForSequenceClassification.from_pretrained(
    model_id,
    revision=model_revision,
    trust_remote_code=True,
).eval()

text = "ما هي شروط القبول في الجامعة؟"
inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=model.config.max_length)
with torch.inference_mode():
    probabilities = torch.sigmoid(model(**inputs).logits[0])

result = {}
for index, probability in enumerate(probabilities.tolist()):
    label = model.config.id2label[index]
    threshold = model.config.thresholds[label]
    result[label] = {"probability": probability, "selected": probability >= threshold}

selected_labels = [label for label, value in result.items() if value["selected"]]
print(selected_labels)

python examples/basic_inference.py \
  --model-id NajahUniv/AraUni-Granite-311M-Intent-Classifier \
  --revision v0.4.1 \
  --backend auto \
  --precision auto \
  --text "ما هي شروط التسجيل؟"

"""Run multi-label inference with a published PyTorch or CPU INT8 ONNX model."""

from __future__ import annotations

import argparse
import json
from pathlib import Path

import numpy as np
import torch
from huggingface_hub import hf_hub_download
from transformers import AutoConfig, AutoModelForSequenceClassification, AutoTokenizer

DEFAULT_MODEL_ID = "NajahUniv/AraUni-Granite-311M-Intent-Classifier"


def choose_device(requested: str) -> str:
    if requested != "auto":
        return requested
    if torch.cuda.is_available():
        return "cuda"
    if torch.backends.mps.is_available():
        return "mps"
    return "cpu"


def resolve_runtime(backend: str, precision: str, device: str) -> tuple[str, str]:
    resolved_backend = "pytorch" if backend == "auto" else backend
    resolved_precision = precision
    if precision == "auto":
        if resolved_backend == "onnx":
            resolved_precision = "int8"
        elif device == "cuda":
            resolved_precision = "bf16" if torch.cuda.is_bf16_supported() else "fp16"
        else:
            resolved_precision = "fp32"
    if resolved_backend == "onnx" and (device != "cpu" or resolved_precision != "int8"):
        raise ValueError("the published ONNX artifact supports CPU INT8 only")
    if resolved_backend == "pytorch" and resolved_precision == "int8":
        raise ValueError("precision=int8 requires backend=onnx")
    if resolved_backend == "pytorch" and resolved_precision in {"bf16", "fp16"} and device != "cuda":
        raise ValueError("the example enables bf16/fp16 only on CUDA")
    return resolved_backend, resolved_precision


def hub_or_local_file(model_id: str, filename: str, revision: str | None) -> str:
    local = Path(model_id) / filename
    if local.is_file():
        return str(local)
    return hf_hub_download(model_id, filename, revision=revision)


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--model-id", default=DEFAULT_MODEL_ID)
    parser.add_argument("--revision", help="Pin a release tag or reviewed commit SHA")
    parser.add_argument("--text", required=True)
    parser.add_argument("--device", default="auto", choices=("auto", "cpu", "mps", "cuda"))
    parser.add_argument("--backend", default="auto", choices=("auto", "pytorch", "onnx"))
    parser.add_argument(
        "--precision",
        default="auto",
        choices=("auto", "fp32", "bf16", "fp16", "int8"),
    )
    parser.add_argument("--top-k", type=int, default=5)
    args = parser.parse_args()

    device = "cpu" if args.backend == "onnx" and args.device == "auto" else choose_device(args.device)
    backend, precision = resolve_runtime(args.backend, args.precision, device)
    load_kwargs = {"revision": args.revision} if args.revision else {}
    tokenizer = AutoTokenizer.from_pretrained(
        args.model_id,
        trust_remote_code=True,
        **load_kwargs,
    )
    config = AutoConfig.from_pretrained(
        args.model_id,
        trust_remote_code=True,
        **load_kwargs,
    )
    thresholds = config.thresholds
    if backend == "onnx":
        import onnxruntime as ort

        onnx_config_path = hub_or_local_file(
            args.model_id,
            "onnx/onnx_config.json",
            args.revision,
        )
        onnx_config = json.loads(Path(onnx_config_path).read_text(encoding="utf-8"))
        thresholds = onnx_config["thresholds"]
        model_path = hub_or_local_file(
            args.model_id,
            "onnx/model_int8.onnx",
            args.revision,
        )
        session = ort.InferenceSession(model_path, providers=["CPUExecutionProvider"])
        encoded = tokenizer(
            args.text,
            return_tensors="np",
            truncation=True,
            max_length=config.max_length,
        )
        logits = session.run(
            ["logits"],
            {
                "input_ids": encoded["input_ids"].astype(np.int64),
                "attention_mask": encoded["attention_mask"].astype(np.int64),
            },
        )[0][0]
        probabilities = 1.0 / (1.0 + np.exp(-logits))
    else:
        dtype = {
            "fp32": torch.float32,
            "bf16": torch.bfloat16,
            "fp16": torch.float16,
        }[precision]
        model = AutoModelForSequenceClassification.from_pretrained(
            args.model_id,
            trust_remote_code=True,
            torch_dtype=dtype,
            **load_kwargs,
        ).to(device).eval()
        encoded = tokenizer(
            args.text,
            return_tensors="pt",
            truncation=True,
            max_length=model.config.max_length,
        ).to(device)
        with torch.inference_mode():
            probabilities = torch.sigmoid(model(**encoded).logits[0]).cpu().float().numpy()

    labels = [config.id2label[index] for index in range(config.num_labels)]
    ranked = sorted(
        (
            {
                "label": label,
                "probability": float(probabilities[index]),
                "threshold": float(thresholds[label]),
                "selected": float(probabilities[index]) >= float(thresholds[label]),
            }
            for index, label in enumerate(labels)
        ),
        key=lambda item: item["probability"],
        reverse=True,
    )
    print(
        json.dumps(
            {
                "text": args.text,
                "backend": backend,
                "precision": precision,
                "device": device,
                "selected_labels": [item["label"] for item in ranked if item["selected"]],
                "scores": ranked[: max(1, args.top_k)],
            },
            ensure_ascii=False,
            indent=2,
        )
    )


if __name__ == "__main__":
    main()

pip install -r requirements.txt
MODEL_ID=NajahUniv/AraUni-Granite-311M-Intent-Classifier MODEL_REVISION=v0.4.1 MODEL_BACKEND=auto \
  MODEL_PRECISION=auto MODEL_API_KEY=change-me \
  uvicorn examples.fastapi_app:app --host 0.0.0.0 --port 8000

curl http://localhost:8000/classify \
  -H 'Content-Type: application/json' \
  -H 'Authorization: Bearer change-me' \
  -d '{"texts":["كيف يمكنني دفع الرسوم؟"],"top_k":5}'

"""FastAPI service with automatic PyTorch or CPU INT8 ONNX inference."""

from __future__ import annotations

import hmac
import json
import os
import threading
from contextlib import asynccontextmanager
from pathlib import Path
from typing import Annotated

import numpy as np
import torch
from fastapi import Depends, FastAPI, HTTPException
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
from huggingface_hub import hf_hub_download
from pydantic import BaseModel, Field
from transformers import AutoConfig, AutoModelForSequenceClassification, AutoTokenizer

DEFAULT_MODEL_ID = "NajahUniv/AraUni-Granite-311M-Intent-Classifier"
MODEL_ID = os.getenv("MODEL_ID", DEFAULT_MODEL_ID)
MODEL_REVISION = os.getenv("MODEL_REVISION")
MODEL_DEVICE = os.getenv("MODEL_DEVICE", "auto")
MODEL_BACKEND = os.getenv("MODEL_BACKEND", "auto")
MODEL_PRECISION = os.getenv("MODEL_PRECISION", "auto")
MODEL_API_KEY = os.getenv("MODEL_API_KEY")
MAX_BATCH_SIZE = int(os.getenv("MAX_BATCH_SIZE", "64"))


def choose_device() -> str:
    if MODEL_BACKEND == "onnx" and MODEL_DEVICE == "auto":
        return "cpu"
    if MODEL_DEVICE != "auto":
        return MODEL_DEVICE
    if torch.cuda.is_available():
        return "cuda"
    if torch.backends.mps.is_available():
        return "mps"
    return "cpu"


def resolve_runtime(device: str) -> tuple[str, str]:
    if MODEL_BACKEND not in {"auto", "pytorch", "onnx"}:
        raise ValueError("MODEL_BACKEND must be auto, pytorch, or onnx")
    if MODEL_PRECISION not in {"auto", "fp32", "bf16", "fp16", "int8"}:
        raise ValueError("MODEL_PRECISION must be auto, fp32, bf16, fp16, or int8")
    backend = "pytorch" if MODEL_BACKEND == "auto" else MODEL_BACKEND
    precision = MODEL_PRECISION
    if precision == "auto":
        if backend == "onnx":
            precision = "int8"
        elif device == "cuda":
            precision = "bf16" if torch.cuda.is_bf16_supported() else "fp16"
        else:
            precision = "fp32"
    if backend == "onnx" and (device != "cpu" or precision != "int8"):
        raise ValueError("the published ONNX artifact supports CPU INT8 only")
    if backend == "pytorch" and precision == "int8":
        raise ValueError("MODEL_PRECISION=int8 requires MODEL_BACKEND=onnx")
    if backend == "pytorch" and precision in {"bf16", "fp16"} and device != "cuda":
        raise ValueError("this example enables bf16/fp16 only on CUDA")
    return backend, precision


def hub_or_local_file(filename: str) -> str:
    local = Path(MODEL_ID) / filename
    if local.is_file():
        return str(local)
    return hf_hub_download(MODEL_ID, filename, revision=MODEL_REVISION)


class ClassifyRequest(BaseModel):
    texts: list[str] = Field(min_length=1)
    top_k: int = Field(default=5, ge=1)
    threshold: float | None = Field(default=None, gt=0, lt=1)


class ModelRuntime:
    def __init__(self) -> None:
        load_kwargs = {"revision": MODEL_REVISION} if MODEL_REVISION else {}
        self.tokenizer = AutoTokenizer.from_pretrained(
            MODEL_ID,
            trust_remote_code=True,
            **load_kwargs,
        )
        self.config = AutoConfig.from_pretrained(
            MODEL_ID,
            trust_remote_code=True,
            **load_kwargs,
        )
        self.device = choose_device()
        self.backend, self.precision = resolve_runtime(self.device)
        self.lock = threading.Lock()
        self.thresholds = self.config.thresholds
        self.model = None
        self.session = None
        if self.backend == "onnx":
            import onnxruntime as ort

            onnx_config = json.loads(
                Path(hub_or_local_file("onnx/onnx_config.json")).read_text(encoding="utf-8")
            )
            self.thresholds = onnx_config["thresholds"]
            self.session = ort.InferenceSession(
                hub_or_local_file("onnx/model_int8.onnx"),
                providers=["CPUExecutionProvider"],
            )
        else:
            dtype = {
                "fp32": torch.float32,
                "bf16": torch.bfloat16,
                "fp16": torch.float16,
            }[self.precision]
            self.model = AutoModelForSequenceClassification.from_pretrained(
                MODEL_ID,
                trust_remote_code=True,
                torch_dtype=dtype,
                **load_kwargs,
            ).to(self.device).eval()

    def probabilities(self, texts: list[str]) -> np.ndarray:
        if self.backend == "onnx":
            encoded = self.tokenizer(
                texts,
                return_tensors="np",
                padding=True,
                truncation=True,
                max_length=self.config.max_length,
            )
            with self.lock:
                logits = self.session.run(
                    ["logits"],
                    {
                        "input_ids": encoded["input_ids"].astype(np.int64),
                        "attention_mask": encoded["attention_mask"].astype(np.int64),
                    },
                )[0]
            return 1.0 / (1.0 + np.exp(-logits))
        encoded = self.tokenizer(
            texts,
            return_tensors="pt",
            padding=True,
            truncation=True,
            max_length=self.config.max_length,
        ).to(self.device)
        with self.lock, torch.inference_mode():
            return torch.sigmoid(self.model(**encoded).logits).cpu().float().numpy()

    def classify(self, request: ClassifyRequest) -> list[dict[str, object]]:
        if len(request.texts) > MAX_BATCH_SIZE:
            raise HTTPException(413, f"at most {MAX_BATCH_SIZE} texts are allowed per request")
        probabilities = self.probabilities(request.texts)
        labels = [self.config.id2label[index] for index in range(self.config.num_labels)]
        results = []
        for text, row in zip(request.texts, probabilities, strict=True):
            scores = []
            for index, label in enumerate(labels):
                threshold = (
                    request.threshold
                    if request.threshold is not None
                    else float(self.thresholds[label])
                )
                scores.append(
                    {
                        "label": label,
                        "probability": float(row[index]),
                        "threshold": threshold,
                        "selected": float(row[index]) >= threshold,
                    }
                )
            scores.sort(key=lambda item: item["probability"], reverse=True)
            results.append(
                {
                    "text": text,
                    "selected_labels": [item["label"] for item in scores if item["selected"]],
                    "scores": scores[: min(request.top_k, len(scores))],
                }
            )
        return results


runtime: ModelRuntime | None = None


@asynccontextmanager
async def lifespan(_: FastAPI):
    global runtime
    runtime = ModelRuntime()
    yield
    runtime = None


app = FastAPI(title="AraUni Multi-label Intent Classifier", lifespan=lifespan)
bearer_scheme = HTTPBearer(
    auto_error=False,
    scheme_name="BearerAuth",
    description="Enter the MODEL_API_KEY value. Swagger adds the 'Bearer' prefix.",
)


def authorize(
    credentials: Annotated[
        HTTPAuthorizationCredentials | None,
        Depends(bearer_scheme),
    ],
) -> None:
    if MODEL_API_KEY is None:
        return
    if (
        credentials is None
        or credentials.scheme.lower() != "bearer"
        or not hmac.compare_digest(credentials.credentials, MODEL_API_KEY)
    ):
        raise HTTPException(
            401,
            "invalid bearer token",
            headers={"WWW-Authenticate": "Bearer"},
        )


@app.get("/health")
def health() -> dict[str, object]:
    return {
        "status": "ok",
        "model_id": MODEL_ID,
        "device": runtime.device if runtime else None,
        "backend": runtime.backend if runtime else None,
        "precision": runtime.precision if runtime else None,
    }


@app.get("/labels", dependencies=[Depends(authorize)])
def labels() -> dict[int, str]:
    if runtime is None:
        raise HTTPException(503, "model is not ready")
    return dict(runtime.config.id2label)


@app.post("/classify", dependencies=[Depends(authorize)])
def classify(request: ClassifyRequest) -> list[dict[str, object]]:
    if runtime is None:
        raise HTTPException(503, "model is not ready")
    return runtime.classify(request)