Modelpublic

openbmb/MiniCPM-RobotManip

sourceHugging Faceapache-2.0updated 2mo agoView on Hugging Face

188likes262downloads

Model Card

<h1 align="center"> MiniCPM-RobotManip </h1>

A Smarter and Faster On-Device AI Brain for Robots

<img src="githublogo.png" alt="github" width="15" height="15" style="margin-right: 4px;"> <a href="https://github.com/OpenBMB/MiniCPM-Robot" target="blank"> Github</a>  | <img src="discordlogo.png" alt="Discord" width="15" height="15" style="margin-right: 4px;"> <a href="https://huggingface.co/openbmb/MiniCPM-RobotManip/resolve/main/discord.jpeg" target="blank"> Discord</a>  |

MiniCPM-RobotManip is a 1.5B vision-language-action model for embodied manipulation with the following highlights: <ul> <li>Generalist Manipulation: A unified 1.5B generalist policy that uses one set of weights across all downstream tasks and outperforms larger models such as π₀.₅ and Qwen-VLA across representative evaluations.</li> <li>Streaming Context: Historical observations are continuously incorporated into the model context through streaming inference, reducing per-step compute from 125 TFLOPs to 3.3 TFLOPs while retaining 60 frames of history and supporting up to one minute of visual memory. This moves VLA beyond reactive action generation from single-frame observations toward continuous decision-making grounded in long-horizon visual context.</li> <li>Efficient Inference: Inherits MiniCPM-V 4.6's visual token compression, reducing each frame from 256 to 64 visual tokens for 4× compression. With H100, BF16, and single-frame input, model-forward latency per decision step is 120 ms, compared with 234 ms for π0.5. The measurement excludes task autoregressive decoding.</li> </ul>

Benchmark Results

Inference Example

Please Ensure transformers==5.7.0

<pre><code class="language-python">from _future_ import annotations

import argparse import json from pathlib import Path from typing import Sequence

import numpy as np import torch from PIL import Image from transformers import AutoModel, AutoProcessor

STATEDIM = 80 IMAGESIZE = (448, 448)

class MiniCPMVLAInference: """Processor and model wrapper for single-sample VLA inference."""

def _init( self, checkpointpath: str | Path = "openbmb/MiniCPM-RobotManip", device: str | torch.device | None = None, ): if device is None: device = "cuda" if torch.cuda.isavailable() else "cpu" self.device = torch.device(device) checkpoint = str(checkpointpath) self.processor = AutoProcessor.frompretrained(checkpoint, trustremotecode=True) self.model = AutoModel.frompretrained(checkpoint, trustremotecode=True) self.model.to(self.device).eval()

@staticmethod def loadimages(images: Sequence[str | Path | Image.Image | np.ndarray]) -> list[np.ndarray]: if not images: raise ValueError("At least one image is required") loaded = [] for image in images: if isinstance(image, (str, Path)): with Image.open(image) as pilimage: array = np.asarray(pilimage.convert("RGB")) elif isinstance(image, Image.Image): array = np.asarray(image.convert("RGB")) elif isinstance(image, np.ndarray): array = image else: raise TypeError(f"Unsupported image type: {type(image)!r}") if array.ndim != 3 or array.shape[-1] != 3: raise ValueError(f"Expected an HxWx3 image, got shape {array.shape}") # Match the training pipeline's ResizeImage(size=(448, 448)), # including PIL's default resize interpolation. resized = Image.fromarray(array).resize(IMAGE_SIZE) loaded.append(np.asarray(resized).copy()) return loaded

def preprocess(self, images: Sequence, text: str) -> dict[str, torch.Tensor]: """Apply the same MiniCPM-V chat template and processor as training.""" content = [ {"type": "image", "image": image} for image in self.loadimages(images) ] content.append({"type": "text", "text": text}) messages = [{"role": "user", "content": content}] inputs = self.processor.applychattemplate( messages, tokenize=True, addgenerationprompt=True, returndict=True, returntensors="pt", processor_kwargs={"padding": False}, ) return { key: value.to(self.device) for key, value in inputs.items() if isinstance(value, torch.Tensor) }

def preparestate(self, state: torch.Tensor | np.ndarray | Sequence[float]) -> torch.Tensor: state = torch.astensor(state, dtype=torch.float32, device=self.device) if state.ndim == 1: state = state.unsqueeze(0).unsqueeze(0) elif state.ndim == 2: state = state.unsqueeze(1) if state.shape != (1, 1, STATEDIM): raise ValueError(f"state must have shape (80,), (1, 80), or (1, 1, 80); got {tuple(state.shape)}") return state

@torch.inferencemode() def predict( self, images: Sequence[str | Path | Image.Image | np.ndarray], text: str, state: torch.Tensor | np.ndarray | Sequence[float] | None = None, embodimentid: int = 0, seed: int | None = None, ) -> torch.Tensor: """Return one action chunk with shape `(30, 80)` on CPU.""" if not 0 <= embodimentid < self.model.actionhead.maxnumembodiments: raise ValueError( f"embodimentid must be in [0, {self.model.actionhead.maxnumembodiments - 1}]" ) if state is None: state = torch.zeros(STATEDIM) statetensor = self.preparestate(state) embodiment = torch.tensor([embodimentid], dtype=torch.long, device=self.device) if seed is not None: torch.manualseed(seed) if self.device.type == "cuda": torch.cuda.manualseedall(seed)

vlminputs = self.preprocess(images, text) actions = self.model.predictaction( state=statetensor, embodimentid=embodiment, **vlm_inputs, ) return actions[0].float().cpu()

def parseargs() -> argparse.Namespace: parser = argparse.ArgumentParser(description=doc) parser.addargument("--image", action="append", required=True, help="Input image; repeat for multiple views") parser.addargument("--text", required=True, help="Robot instruction/prompt") parser.addargument("--device", default=None, help="Default: cuda if available, otherwise cpu") stategroup = parser.addmutuallyexclusivegroup() stategroup.addargument("--state-file", help="A .npy file containing 80 state values") stategroup.addargument("--state", nargs=STATEDIM, type=float, metavar="VALUE") parser.addargument("--embodiment-id", type=int, default=0) parser.addargument("--seed", type=int, default=None) parser.addargument("--output", help="Optional output .npy path; otherwise print JSON") return parser.parse_args()

if _name == "main": args = parseargs() if args.statefile: state = np.load(args.statefile) elif args.state is not None: state = args.state else: state = np.zeros(STATE_DIM, dtype=np.float32)

inferrunner = MiniCPMVLAInference( checkpointpath="openbmb/MiniCPM-RobotManip", device=args.device, ) action = inferrunner.predict( images=args.image, text=args.text, state=state, embodimentid=args.embodimentid, seed=args.seed, ) if args.output: outputpath = Path(args.output) outputpath.parent.mkdir(parents=True, existok=True) np.save(outputpath, action.numpy()) print(f"Saved action {tuple(action.shape)} to {outputpath}") else: print(json.dumps(action.tolist()))

</code></pre>

Acknowledgement

This project builds on and references <a href="https://github.com/starVLA/starVLA">starVLA</a> and <a href="https://github.com/huggingface/lerobot">LeRobot</a>. We thank the authors for their open-source contributions.

License

Model weights and code are open-sourced under the Apache-2.0 license.