cagataydev/protomotions-gtp-unitree-g1

Kimodo diffusion (100 steps, 11 it/s)         → 8.0s
  → 120 frames @ 30Hz kinematic ref
  → SLERP retime → 199 frames @ 50Hz
  → ProtoMotions GTP ONNX (this model) @ 50Hz  → 1.2s
  → G1 physics @ 1kHz (MuJoCo)                 → 3.2× realtime
  → 100 rendered frames @ 25fps                → ~2s

0-5   left_hip_[pitch,roll,yaw]_joint, left_knee, left_ankle_[pitch,roll]
6-11  right_hip_[pitch,roll,yaw]_joint, right_knee, right_ankle_[pitch,roll]
12-14 waist_[yaw,roll,pitch]_joint
15-21 left_shoulder_[pitch,roll,yaw], left_elbow, left_wrist_[roll,pitch,yaw]
22-28 right_shoulder_[pitch,roll,yaw], right_elbow, right_wrist_[roll,pitch,yaw]

0  pelvis
1  head
2-7  left leg: [hip_pitch, hip_roll, hip_yaw, knee, ankle_pitch, ankle_roll]_link
8-13 right leg (mirror)
14   waist_yaw_link
15   waist_roll_link
16   torso_link       ← anchor
17-23 left arm: [shoulder_pitch, shoulder_roll, shoulder_yaw, elbow, wrist_roll, wrist_pitch, wrist_yaw]_link
24   left_rubber_hand
25-31 right arm (mirror)
32   right_rubber_hand

import numpy as np, onnxruntime as ort, yaml

sess = ort.InferenceSession("unified_pipeline.onnx", providers=["CPUExecutionProvider"])
cfg  = yaml.safe_load(open("unified_pipeline.yaml"))

# Build observation from your robot state + kinematic reference motion
obs = {
    "current_anchor_rot":           np.zeros((1, 4),      dtype=np.float32),   # torso quat
    "current_dof_pos":              np.zeros((1, 29),     dtype=np.float32),   # rad
    "current_dof_vel":              np.zeros((1, 29),     dtype=np.float32),   # rad/s
    "current_root_local_ang_vel":   np.zeros((1, 3),      dtype=np.float32),
    "historical_processed_actions": np.zeros((1, 1, 29),  dtype=np.float32),
    "mimic_future_anchor_rot":      np.zeros((1, 4, 4),   dtype=np.float32),   # 4 lookahead steps
    "mimic_future_dof_pos":         np.zeros((1, 4, 29),  dtype=np.float32),
    "mimic_future_dof_vel":         np.zeros((1, 4, 29),  dtype=np.float32),
}

actions, joint_targets, stiff, damp = sess.run(
    ["actions", "joint_pos_targets", "stiffness_targets", "damping_targets"],
    obs,
)
# → feed joint_targets/stiff/damp to G1 PD controller at 50 Hz

# 1) Generate kinematic reference from text (nvidia/Kimodo-G1-RP-v1)
from diffusers import DiffusionPipeline  # or Kimodo's native API
motion = kimodo.generate("a person walking forward with confident strides")
# motion: (120, 3+29) root_pos + joint_pos at 30 Hz

# 2) Retime to 50 Hz (SLERP for rotations, linear for positions)
motion_50hz = slerp_retime(motion, from_fps=30, to_fps=50)   # → (199, 32)

# 3) Roll out through GTP tracker in MuJoCo (see full script link below)
for step in range(len(motion_50hz)):
    obs = build_obs(mj_data, motion_50hz, step, lookahead=[1,2,4,8])
    _, joint_targets, kp, kd = sess.run(None, obs)
    apply_pd(mj_data, joint_targets, kp, kd)
    for _ in range(20):  # 20× physics substeps → 1 kHz
        mujoco.mj_step(mj_model, mj_data)

pip install "strands-robots[kimodo,protomotions,sim-mujoco]"
export STRANDS_TRUST_REMOTE_CODE=1     # NVIDIA Open Model License gate
export MUJOCO_GL=egl                   # headless render on Jetson/Thor

from strands_robots import Robot, create_policy

sim = Robot("g1", mesh=False)
sim.add_camera("cinematic", position=[3, 0, 1.2], target=[0, 0, 0.9])
sim.start_recording(repo_id="my/kimodo-g1", root="./data", fps=25)

kimodo = create_policy(
    "kimodo",
    diffusion_steps=100,
    guidance_scale=7.5,
    device="cuda",
)
tracker = create_policy(
    "protomotions",
    onnx_path="cagataydev/protomotions-gtp-unitree-g1/unified_pipeline.onnx",
    yaml_path="cagataydev/protomotions-gtp-unitree-g1/unified_pipeline.yaml",
)

sim.run_policy(
    robot_name="g1",
    policy=kimodo,
    tracker=tracker,                       # feeds Kimodo's qpos into GTP → PD targets
    instruction="a person walking forward with confident strides",
    n_steps=200,
    control_frequency=50,                  # ProtoMotions native rate
    n_episodes=10,
    reset_between=True,
)
sim.stop_recording()

from strands_robots.policies.kimodo.hardware import kimodo_action_to_lerobot_g1

# Kimodo emits 29-DoF whole-body qpos targets in KIMODO_G1_JOINTS ordering.
# The bridge is a pure key rename to lerobot_unitree's driver vocabulary:
lerobot_action = kimodo_action_to_lerobot_g1(kimodo_action)

@article{beyondmimic,
  title={BeyondMimic: From Motion Tracking to Versatile Humanoid Control via Guided Diffusion},
  year={2024},
  eprint={2408.07295},
  archivePrefix={arXiv},
}
@misc{protomotions,
  author={NVIDIA GEAR},
  title={ProtoMotions},
  howpublished={\url{https://github.com/NVlabs/ProtoMotions}},
}