vigneshlabs/cohere-transcribe-03-2026-int8-onnx

pip install onnxruntime numpy soundfile transformers

pip install onnxruntime-gpu  # replaces onnxruntime

pip install onnxruntime  # CoreML provider is included by default on macOS

from huggingface_hub import snapshot_download

snapshot_download(
    repo_id="vigneshlabs/cohere-transcribe-03-2026-int8-onnx",
    local_dir="./cohere-int8",
)

git lfs install
git clone https://huggingface.co/vigneshlabs/cohere-transcribe-03-2026-int8-onnx

cohere-encoder.int8.onnx          101 MB   Conformer encoder graph (INT8)
cohere-encoder.int8.onnx.data     2.4 GB   Encoder weights (external data)
cohere-decoder.int8.onnx          569 KB   Transformer decoder graph (INT8)
cohere-decoder.int8.onnx.data     146 MB   Decoder weights (external data)
tokens.txt                              203 KB   Vocabulary (16,384 tokens)

import numpy as np
import onnxruntime as ort
import soundfile as sf
from transformers import AutoProcessor

# ── Config ──────────────────────────────────────────────────────────
MODEL_DIR        = "./cohere-int8"
DECODER_START_ID = 13764
EOS_ID           = 3
EN_LANG_ID       = 62    # see Language Tokens section below
MAX_NEW_TOKENS   = 256

# ── Tokenizer ──────────────────────────────────────────────────────
id_to_token = {}
with open(f"{MODEL_DIR}/tokens.txt") as f:
    for line in f:
        parts = line.rstrip("\n").split(" ")
        id_to_token[int(parts[-1])] = " ".join(parts[:-1])

def detokenize(ids):
    pieces = [id_to_token.get(i, "") for i in ids
              if not (id_to_token.get(i, "").startswith("<|")
                      and id_to_token.get(i, "").endswith("|>"))]
    return "".join(pieces).replace("▁", " ").strip()

# ── Load processor (for mel spectrogram extraction) ────────────────
processor = AutoProcessor.from_pretrained(
    "CohereLabs/cohere-transcribe-03-2026", trust_remote_code=True
)

# ── Load ONNX sessions on CPU ─────────────────────────────────────
opts = ort.SessionOptions()
opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL

encoder = ort.InferenceSession(
    f"{MODEL_DIR}/cohere-encoder.int8.onnx", opts,
    providers=["CPUExecutionProvider"]
)
decoder = ort.InferenceSession(
    f"{MODEL_DIR}/cohere-decoder.int8.onnx", opts,
    providers=["CPUExecutionProvider"]
)

# ── Load & preprocess audio ───────────────────────────────────────
audio, sr = sf.read("audio.wav")
if audio.ndim > 1:
    audio = audio.mean(axis=1)  # stereo → mono
mel = processor(
    audio.astype(np.float32), sampling_rate=sr, return_tensors="np"
)["input_features"].astype(np.float32)

# ── Encode ────────────────────────────────────────────────────────
cross_k, cross_v = encoder.run(
    ["n_layer_cross_k", "n_layer_cross_v"], {"mel": mel}
)

# ── Decode (greedy) ──────────────────────────────────────────────
n_layers = cross_k.shape[0]  # 8
self_k = np.zeros((n_layers, 1, 8, 1024, 128), dtype=np.float32)
self_v = np.zeros((n_layers, 1, 8, 1024, 128), dtype=np.float32)
tokens = np.array([[DECODER_START_ID, EN_LANG_ID]], dtype=np.int64)
offset = np.array(0, dtype=np.int64)
generated = []

for _ in range(MAX_NEW_TOKENS):
    logits, self_k, self_v = decoder.run(
        ["logits", "out_n_layer_self_k_cache", "out_n_layer_self_v_cache"],
        {
            "tokens": tokens,
            "in_n_layer_self_k_cache": self_k,
            "in_n_layer_self_v_cache": self_v,
            "n_layer_cross_k": cross_k,
            "n_layer_cross_v": cross_v,
            "offset": offset,
        },
    )
    next_token = int(np.argmax(logits[0, -1, :]))
    if next_token == EOS_ID:
        break
    generated.append(next_token)
    offset = np.array(int(offset) + tokens.shape[1], dtype=np.int64)
    tokens = np.array([[next_token]], dtype=np.int64)

print(detokenize(generated))

import onnxruntime as ort

# Use CoreML provider — accelerates via Apple Neural Engine / GPU
encoder = ort.InferenceSession(
    "cohere-encoder.int8.onnx",
    providers=["CoreMLExecutionProvider", "CPUExecutionProvider"]
)
decoder = ort.InferenceSession(
    "cohere-decoder.int8.onnx",
    providers=["CoreMLExecutionProvider", "CPUExecutionProvider"]
)

# Rest of the inference code is identical to the CPU example above.
# CoreML will automatically accelerate supported ops on the Neural Engine.
# Unsupported ops fall back to CPU transparently.

pip install onnxruntime-gpu

import onnxruntime as ort

# Use CUDA provider
encoder = ort.InferenceSession(
    "cohere-encoder.int8.onnx",
    providers=["CUDAExecutionProvider", "CPUExecutionProvider"]
)
decoder = ort.InferenceSession(
    "cohere-decoder.int8.onnx",
    providers=["CUDAExecutionProvider", "CPUExecutionProvider"]
)

# Rest of the inference code is identical.
# Tensors are transferred to GPU automatically by ONNX Runtime.

pip install onnxruntime-gpu  # TensorRT must also be installed system-wide

import onnxruntime as ort

# TensorRT will compile an optimized engine on first run (cached afterward)
encoder = ort.InferenceSession(
    "cohere-encoder.int8.onnx",
    providers=["TensorrtExecutionProvider", "CUDAExecutionProvider", "CPUExecutionProvider"]
)
decoder = ort.InferenceSession(
    "cohere-decoder.int8.onnx",
    providers=["TensorrtExecutionProvider", "CUDAExecutionProvider", "CPUExecutionProvider"]
)

# Rest of the inference code is identical.
# First inference is slow (TensorRT engine compilation), subsequent runs are fast.

pip install onnxruntime-directml

import onnxruntime as ort

encoder = ort.InferenceSession(
    "cohere-encoder.int8.onnx",
    providers=["DmlExecutionProvider", "CPUExecutionProvider"]
)
decoder = ort.InferenceSession(
    "cohere-decoder.int8.onnx",
    providers=["DmlExecutionProvider", "CPUExecutionProvider"]
)

# Rest of the inference code is identical.

LibriSpeech test-clean (read speech, studio quality)
  FP32  █  0.99%
  INT8  █  0.99%   Δ  0.00%

VoxPopuli (parliament, formal speech)
  FP32  ██████  12.16%
  INT8  ██████  12.16%   Δ  0.00%

AMI IHM (meeting, conversational)
  FP32  ████████  16.24%
  INT8  ████████  15.38%   Δ -0.85%

People's Speech (diverse, crowd-sourced)
  FP32  ███████████  21.97%
  INT8  ███████████  22.42%   Δ +0.45%

Earnings22 (financial, spontaneous)
  FP32  █████████████  26.38%
  INT8  ██████████████  27.87%   Δ +1.49%

AVERAGE
  FP32  ████████  15.55%
  INT8  ████████  15.77%   Δ +0.22%

Original (safetensors)  ██████████████████████████████████████  3.80 GB  (baseline)
FP32 ONNX               ████████████████████████████████████████████████████████████████████████████ 7.79 GB  (+105%)
INT8 ONNX               ██████████████████████████  2.69 GB  (-29%)
INT4 ONNX               ███████████████████  1.95 GB  (-49%)

# Example: transcribe French audio
FR_LANG_ID = 69
tokens = np.array([[DECODER_START_ID, FR_LANG_ID]], dtype=np.int64)

┌─────────────────────────────────────────────────────┐
│  Encoder                                             │
│  Input:  mel (batch, 128, T)                         │
│  Output: n_layer_cross_k (8, batch, T_enc, 1024)     │
│          n_layer_cross_v (8, batch, T_enc, 1024)     │
└──────────────────────┬──────────────────────────────┘
                       │ cross-attention K/V
┌──────────────────────▼──────────────────────────────┐
│  Decoder (autoregressive)                            │
│  Input:  tokens (batch, n_tokens)                    │
│          self_k/v caches (8, batch, 8, 1024, 128)    │
│          cross_k/v from encoder                      │
│          offset (scalar)                             │
│  Output: logits (batch, n_tokens, 16384)             │
│          updated self_k/v caches                     │
└─────────────────────────────────────────────────────┘

from onnxruntime.quantization import quantize_dynamic, QuantType

quantize_dynamic(
    model_input="cohere-encoder.onnx",
    model_output="cohere-encoder.int8.onnx",
    weight_type=QuantType.QInt8,
    nodes_to_exclude=conv_and_batchnorm_nodes,  # 149 nodes protected
    use_external_data_format=True,
)

@misc{cohere-transcribe-03-2026,
  author    = {CohereLabs},
  title     = {cohere-transcribe-03-2026},
  year      = {2026},
  url       = {https://huggingface.co/CohereLabs/cohere-transcribe-03-2026}
}