OsaurusAI/Nemotron-3-Nano-Omni-30B-A3B-MXFP4

pip install jang_tools mlx mlx_lm pillow soundfile scipy librosa imageio[ffmpeg]

import mlx.core as mx
from jang_tools.nemotron_omni.model import NemotronHOmni

chat = NemotronHOmni("OsaurusAI/Nemotron-3-Nano-Omni-30B-A3B-MXFP4", dtype=mx.float32)

# Text only
print(chat.turn("Capital of France?"))                                # "Paris."

# Image input — RADIO ViT runs natively in MLX on Metal (1.4s)
print(chat.turn("What's in this image?", images=["cat.jpg"]))

# Audio input — parakeet Conformer encoder native MLX (2.1s for 20s clip)
print(chat.turn("Transcribe what was said.", audio="speech.wav"))

# Video input — frame extraction + RADIO video_embedder + EVS pruning native MLX (3.6s for 8 frames)
print(chat.turn("Describe what happens.", video="clip.mp4",
                video_target_frames=8, video_apply_evs=True))

# Mixed modality
print(chat.turn(
    "Compare the image with the spoken description.",
    images=["scene.jpg"], audio="description.wav",
))

# Multi-turn — KV + Mamba state persists across turns
print(chat.turn("And what about the previous image?"))      # references prior turn
chat.reset()                                                  # new conversation

# Reasoning ON (default for Reasoning SKU): emits <think>...</think> + answer
chat.turn("Solve: 17 + 28 = ?", enable_thinking=True)

# Reasoning OFF: faster, more direct
chat.turn("Solve: 17 + 28 = ?", enable_thinking=False)

from mlx_lm import load, generate
model, tokenizer = load("OsaurusAI/Nemotron-3-Nano-Omni-30B-A3B-MXFP4")
prompt = tokenizer.apply_chat_template(
    [{"role": "user", "content": "Capital of France?"}],
    tokenize=False, add_generation_prompt=True,
)
print(generate(model, tokenizer, prompt=prompt, max_tokens=20))   # "Paris."

hybrid_override_pattern (52 chars):
"MEMEM*EMEMEM*EMEMEM*EMEMEM*EMEMEM*EMEMEMEM*EMEMEMEME"

   23 × M = Mamba-2 SSM             (state-space, O(1) cache per token)
   23 × E = MoE (128 routed × 6)     (ReLU² activation, no gate_proj)
    6 × * = Attention                 (GQA 32q / 2kv heads, NO RoPE, head_dim=128)

Multimodal towers (fp16, native MLX):
   vision_model        →  RADIO ViT (NVIDIA C-RADIOv2-H)
   mlp1                →  LayerNorm + Linear + GELU + Linear → llm_hidden
   sound_encoder       →  ParakeetEncoder (24 Conformer layers)
   sound_projection    →  RMSNorm + Linear + SquaredReLU + Linear → llm_hidden

Cache (multi-turn):
   M layers  →  MambaCache (size=2: conv state + ssm state)  O(1)/token
   * layers  →  KVCacheSimple                                  O(L)/token
   E layers  →  stateless

Native context: 262 144 tokens (no RoPE extrapolation needed)

import MLXVLM

let frames = try await vlmExtractFramesUniform(url: videoURL, targetFrames: 32)
let pixels = vlmStackFramesIntoChannels(frames, imageSize: 512, temporalPatchDim: 2)
// → MLXArray (n_groups, T*3=6, 512, 512) for RADIO video_embedder