jinaai/jina-embeddings-v5-omni-small

# Retrieve the configuration of the preconfigured omni-small inference endpoint
GET /_inference/embedding/.jina-embeddings-v5-omni-small

# Generate an embedding for a single piece of text using the predefined endpoint
POST _inference/embedding/.jina-embeddings-v5-omni-small
{
  "input": [
    "This is a test"
  ]
}

# Fuse a text description and an image into a single embedding via a multimodal content block
POST _inference/embedding/.jina-embeddings-v5-omni-small
{
  "input": [
    {
      "content": [
        { "type": "text",  "value": "A small blue square" },
        { "type": "image", "format": "base64", "value": "<BASE64_IMAGE_DATA>" }
      ]
    }
  ]
}

# Create a custom endpoint that truncates omni-small embeddings to 32 dimensions
PUT _inference/embedding/jina-omni-small-32d
{
  "service": "elastic",
  "service_settings": {
    "model_id": "jina-embeddings-v5-omni-small",
    "dimensions": 32
  }
}

# core
pip install transformers torch pillow numpy

# Optional — install only the extras for the modalities you actually use:
pip install librosa soundfile      # audio decoding
pip install torchcodec                # video decoding for proc(videos=path) (transformers default backend)
pip install av                        # video decoding for SentenceTransformer model.encode("clip.mp4")
pip install pdf2image pypdfium2    # PDF rendering
pip install cairosvg pillow    # SVG rendering
pip install "vllm==0.20.1"        # high-throughput serving (validated)
pip install sentence-transformers  # one-call multimodal API

from PIL import Image
import librosa, torch
from transformers import AutoModel, AutoProcessor, WhisperFeatureExtractor

repo = "jinaai/jina-embeddings-v5-omni-small"
model = AutoModel.from_pretrained(repo, trust_remote_code=True, default_task="retrieval").eval()
proc  = AutoProcessor.from_pretrained(repo, trust_remote_code=True)

# model.embed(**inputs) returns L2-normalized last-token embeddings.
t_vec = model.embed(**proc(text="Query: Which planet is known as the Red Planet?", return_tensors="pt").to(model.device))
i_vec = model.embed(**proc(images=Image.open("photo.jpg"), text="<|vision_start|><|image_pad|><|vision_end|>", return_tensors="pt").to(model.device))
v_vec = model.embed(**proc(videos="clip.mp4", text="<|vision_start|><|video_pad|><|vision_end|>", return_tensors="pt").to(model.device))
# Needs torchcodec or torchvision. On Windows or envs without either, use model.encode("clip.mp4") from the sentence-transformers section below (av-only, no codec libs required).

# Audio has no string placeholder — build the prompt from config. Size the
# audio run from the REAL Whisper frame mask: the feature extractor pads every
# clip out to the 30s window, so counting padded frames emits 750 audio tokens
# for a 2s clip and a 30s clip alike, which is not how the model was trained.
# (The sentence-transformers path below does all of this for you.)
audio, _ = librosa.load("speech.wav", sr=16000)
feats = WhisperFeatureExtractor(feature_size=128)(
    audio, sampling_rate=16000, return_tensors="pt",
    padding="max_length", return_attention_mask=True,
)
feat, feat_mask = feats["input_features"], feats["attention_mask"]
real = int(feat_mask.sum())                    # unpadded mel frames
n    = ((real - 1) // 2 + 1 - 2) // 2 + 1      # audio encoder output length
cfg, tok = model.config, proc.tokenizer
audio_run = (tok.convert_ids_to_tokens(cfg.audio_start_token_id)
             + tok.convert_ids_to_tokens(cfg.audio_token_id) * n
             + tok.convert_ids_to_tokens(cfg.audio_end_token_id))
prompt = tok.apply_chat_template([{"role": "user", "content": audio_run}],
                                 tokenize=False, add_generation_prompt=False)
ids = torch.tensor([tok(prompt)["input_ids"]])  # keep the tokenizer's specials
a_vec = model.embed(
    input_ids=ids.to(model.device),
    attention_mask=torch.ones_like(ids).to(model.device),
    input_features=feat.to(model.device, dtype=next(model.parameters()).dtype),
    feature_attention_mask=feat_mask.to(model.device),
)

from transformers import AutoModel

AutoModel.from_pretrained("jinaai/jina-embeddings-v5-omni-small", trust_remote_code=True, modality="omni")    # all (default)
AutoModel.from_pretrained("jinaai/jina-embeddings-v5-omni-small", trust_remote_code=True, modality="vision")  # vision + text
AutoModel.from_pretrained("jinaai/jina-embeddings-v5-omni-small", trust_remote_code=True, modality="audio")   # audio + text
AutoModel.from_pretrained("jinaai/jina-embeddings-v5-omni-small", trust_remote_code=True, modality="text")    # text only

SentenceTransformer("jinaai/jina-embeddings-v5-omni-small", trust_remote_code=True, model_kwargs={"modality": "vision"})

from sentence_transformers import SentenceTransformer

# Base repo holds all 4 task adapters — pick one at load time.
model = SentenceTransformer(
    "jinaai/jina-embeddings-v5-omni-small",
    trust_remote_code=True,
    model_kwargs={"default_task": "retrieval"},
)

# URLs, local paths (with or without extension), PIL.Image, np.ndarray,
# torch.Tensor, bytes, and BytesIO are all accepted directly.
q_vec = model.encode_query("Which planet is known as the Red Planet?")
d_vec = model.encode_document("Mars is often referred to as the Red Planet due to its reddish appearance.")

# The Query:/Document: distinction applies to EVERY modality, not just text —
# pass the image / video / audio (URL, path, or object) straight to
# encode_query() / encode_document() to encode it as that retrieval side:
img_as_document = model.encode_document("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/car.jpg")
img_as_query    = model.encode_query("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/car.jpg")
i_vec = model.encode("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/car.jpg")
v_vec = model.encode("https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4")  # needs `pip install av`
a_vec = model.encode("https://huggingface.co/datasets/Narsil/asr_dummy/resolve/main/mlk.flac")  # needs `pip install librosa soundfile`

# Fused multimodal — a tuple becomes ONE embedding in a single forward pass:
emb = model.encode(("Winter boots, waterproof leather upper",
                    "https://.../boot.jpg",
                    "https://.../boot.mp4"))

import numpy as np
# (T, H, W, 3) uint8 — e.g. from decord, imageio, or an rgb frame buffer
frames = np.zeros((16, 224, 224, 3), dtype=np.uint8)
v_vec = model.encode(frames)

from vllm import LLM
llm = LLM(
    model="jinaai/jina-embeddings-v5-omni-small",
    runner="pooling",
    trust_remote_code=True,
    hf_overrides={"task": "retrieval"},  # or: classification / clustering / text-matching
)
# Retrieval: prepend "Query: " for queries, "Document: " for documents.
# Non-retrieval (classification / clustering / text-matching): prepend "Document: " to every text input.
outs = llm.embed([{"prompt": "Query: Which planet is known as the Red Planet?"}])

vllm serve jinaai/jina-embeddings-v5-omni-small \
  --trust-remote-code \
  --hf-overrides '{"task": "retrieval"}'

# transformers
vec = model.embed(truncate_dim=256, **proc(text="hello", return_tensors="pt"))
# or
vec = model.encode(["hello"], task="retrieval", truncate_dim=256)

# sentence-transformers
vec = model.encode("hello", truncate_dim=256)

# vLLM — ask the pooler for a smaller embedding
from vllm import PoolingParams
outs = llm.embed(prompts, pooling_params=PoolingParams(dimensions=256))
# or truncate + renormalize the full-dim output yourself:
import numpy as np
full = np.asarray(outs[0].outputs.embedding)
vec = full[:256] / np.linalg.norm(full[:256])

# sentence-transformers — any modality
t_vecs = model.encode(["query 1", "query 2"])
i_vecs = model.encode([Image.open("a.jpg"), Image.open("b.jpg")])
v_vecs = model.encode(["clip1.mp4", "clip2.mp4"])
a_vecs = model.encode(["speech1.wav", "speech2.wav"])

# raw transformers — text (native padded batch)
inputs = proc(text=["query 1", "query 2"], padding=True, truncation=True, return_tensors="pt").to(model.device)
vecs = model.embed(**inputs)  # (2, dim)

# vLLM — list of request dicts, any modality
outs = llm.embed([
    {"prompt": "query 1"},
    {"prompt": "query 2"},
])

@article{honicke2026jina,
  title={jina-embeddings-v5-omni: Geometry-preserving Embeddings via Locked Aligned Towers},
  author={H{\"o}nicke, Florian and G{\"u}nther, Michael and Koukounas, Andreas and Akram, Kalim and Martens, Scott and Sturua, Saba and Xiao, Han},
  journal={arXiv preprint arXiv:2605.08384},
  year={2026}
}