InternRobotics/RoboInter-Data

RoboInter-Data/
│
├── Annotation_with_action_lerobotv21/    # [Main] LeRobot v2.1 format (actions + annotations + videos)
│   ├── lerobot_droid_anno/               #   DROID: 152,986 episodes
│   └── lerobot_rh20t_anno/              #   RH20T:  82,894 episodes
│
├── Annotation_pure/                      # Annotation-only LMDB (no actions/videos)
│   └── annotations/                      #   35 GB, all 235,920 episodes
│
├── Annotation_raw/                       # Original unprocessed annotations
│   ├── droid_annotation.pkl              #   Raw DROID annotations (~20 GB)
│   ├── rh20t_annotation.pkl              #   Raw RH20T annotations (~11 GB)
│   └── segmentation_npz.zip.*            #   Segmentation masks (~50 GB, split archives)
│
├── Annotation_demo_app/                  # Small demo subset for online visualization
│   ├── demo_data/                        #   LMDB annotations for 20 sampled videos
│   └── videos/                           #   20 MP4 videos
│
├── Annotation_demo_larger/               # Larger demo subset for local visualization
│   ├── demo_annotations/                 #   LMDB annotations for 120 videos
│   └── videos/                           #   120 MP4 videos
│
├── All_Keys_of_Primary.json              # Episode names where all annotations are Primary quality
├── RoboInter_Data_Qsheet.json            # Per-episode quality ratings for each annotation type
├── RoboInter_Data_Qsheet_value_stats.json# Distribution statistics of quality ratings
├── RoboInter_Data_RawPath_Qmapping.json  # Mapping: original data source path -> episode splits & quality
├── range_nop.json                        # Non-idle frame ranges for all 230k episodes
├── range_nop_droid_all.json              # Non-idle frame ranges (DROID only)
├── range_nop_rh20t_all.json              # Non-idle frame ranges (RH20T only)
├── val_video.json                        # Validation set: 7,246 episode names
└── VideoID_2_SegmentationNPZ.json        # Episode video ID -> segmentation NPZ file path mapping

cd Annotation_with_action_lerobotv21

for dataset in lerobot_droid_anno lerobot_rh20t_anno; do
  for subdir in data videos; do
    cd ${dataset}/${subdir}
    for f in *.tar; do tar xf "$f" && rm "$f"; done
    cd ../..
  done
done

lerobot_droid_anno/  (or lerobot_rh20t_anno/)
├── meta/
│   ├── info.json              # Dataset metadata (fps=10, features, etc.)
│   ├── episodes.jsonl         # Episode information
│   └── tasks.jsonl            # Task/instruction mapping
├── data/
│   └── chunk-{NNN}/           # Parquet files (1,000 episodes per chunk)
│       └── episode_{NNNNNN}.parquet
└── videos/
    └── chunk-{NNN}/
        ├── observation.images.primary/
        │   └── episode_{NNNNNN}.mp4
        └── observation.images.wrist/
            └── episode_{NNNNNN}.mp4

from lerobot_dataloader import create_dataloader

# Single dataset
dataloader = create_dataloader(
    "path/to/Annotation_with_action_lerobotv21/lerobot_droid_anno",
    batch_size=32,
    action_horizon=16,
)

for batch in dataloader:
    images = batch["observation.images.primary"]   # (B, H, W, 3)
    actions = batch["action"]                      # (B, 16, 7)
    trace = batch["annotation.trace"]              # JSON strings
    skill = batch["annotation.primitive_skill"]    # List[str]
    break

# Multiple datasets (DROID + RH20T)
dataloader = create_dataloader(
    [
        "path/to/lerobot_droid_anno",
        "path/to/lerobot_rh20t_anno",
    ],
    batch_size=32,
    action_horizon=16,
)

from lerobot_dataloader import create_dataloader, QAnnotationFilter

dataloader = create_dataloader(
    "path/to/lerobot_droid_anno",
    batch_size=32,
    range_nop_path="path/to/range_nop.json",       # Remove idle frames
    q_filters=[
        QAnnotationFilter("Q_annotation.trace", ["Primary"]),
        QAnnotationFilter("Q_annotation.gripper_box", ["Primary", "Secondary"]),
    ],
)

{
    0: {  # frame_id
        "time_clip": [[0, 132], [132, 197], [198, 224]],   # subtask segments
        "instruction_add": "pick up the red cup",           # language instruction
        "substask": "reach for the cup",                    # current subtask
        "primitive_skill": "reach",                         # skill label
        "segmentation": None,                               # (stored separately in Annotation_raw)
        "object_box": [[45, 30], [120, 95]],                # manipulated object bbox
        "placement_proposal": [[150, 80], [220, 140]],      # target placement bbox
        "trace": [[x, y], ...],                             # next 10 gripper waypoints
        "gripper_box": [[60, 50], [100, 80]],               # gripper bbox
        "contact_frame": 101,                               # contact event frame (−1 if past contact)
        "state_affordance": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6],# 6D EEF state at contact
        "affordance_box": [[62, 48], [98, 82]],             # gripper bbox at contact frame
        "contact_points": [[75, 65], [85, 65]],             # contact pixel coordinates
        ...
    },
    1: { ... },
    ...
}

import lmdb
import pickle

lmdb_path = "Annotation_pure/annotations"
env = lmdb.open(lmdb_path, readonly=True, lock=False, readahead=False)

with env.begin() as txn:
    # List all episode keys
    cursor = txn.cursor()
    for key, value in cursor:
        episode_name = key.decode("utf-8")
        episode_data = pickle.loads(value)

        # Access frame 0
        frame_0 = episode_data[0]
        print(f"{episode_name}: {frame_0['instruction_add']}")
        print(f"  object_box: {frame_0['object_box']}")
        print(f"  trace: {frame_0['trace'][:3]}...")  # first 3 waypoints
        break

env.close()

cd RoboInter/RoboInterData/lmdb_tool

# Basic info
python read_lmdb.py --lmdb_path Annotation_pure/annotations --action info

# View a specific episode
python read_lmdb.py --lmdb_path Annotation_pure/annotations --action item --key "3072_exterior_image_1_left"

# Field coverage statistics
python read_lmdb.py --lmdb_path Annotation_pure/annotations --action stats --key "3072_exterior_image_1_left"

# Multi-episode summary
python read_lmdb.py --lmdb_path Annotation_pure/annotations --action summary --limit 100

cd /RoboInter-Data/Annotation_raw
cat segmentation_npz.zip.* > segmentation_npz.zip
unzip segmentation_npz.zip

import pickle

with open("Annotation_raw/droid_annotation.pkl", "rb") as f:
    droid_data = pickle.load(f)  # Warning: ~20 GB, takes several minutes

# droid_data[episode_key] contains raw intermediate representation data
# including: all_language, all_gripper_box, all_grounding_box, all_contact_point, all_traj, etc.

git clone https://github.com/InternRobotics/RoboInter.git
cd RoboInter/RoboInterData-Demo

# Option A: Use the small demo subset (for Spaces)
ln -s /path/to/Annotation_demo_app/demo_data ./demo_data
ln -s /path/to/Annotation_demo_app/videos ./videos

# Option B: Use the larger demo subset (for local)
ln -s /path/to/Annotation_demo_larger/demo_annotations ./demo_data
ln -s /path/to/Annotation_demo_larger/videos ./videos

pip install -r requirements.txt
python app.py
# Open http://localhost:7860

import json

with open("range_nop.json") as f:
    range_nop = json.load(f)

# Example: "3072_exterior_image_1_left": [12, 217, 206]
# Means: valid action frames are 12~217, total 206 valid frames
# (frames 0~11 and 218+ are idle/stationary)

@article{li2026robointer,
  title={RoboInter: A Holistic Intermediate Representation Suite Towards Robotic Manipulation},
  author={Li, Hao and Wang, Ziqin and Ding, Zi-han and Yang, Shuai and Chen, Yilun and Tian, Yang and Hu, Xiaolin and Wang, Tai and Lin, Dahua and Zhao, Feng and others},
  journal={arXiv preprint arXiv:2602.09973},
  year={2026}
}