OpenWAM/OpenWAM-Alpha-Sim-RoboDojo

1model:2  video_backbone:3    encoder:4      name: wan22_vae5      model_path: /path/to/Wan2.2-TI2V-5B6    name: wan22_ti2v_5b7    model_path: /path/to/Wan2.2-TI2V-5B8    from_scratch: false9    shift_video: 5.010    components:11    - attr: dit12      model_class: openwam.model.video_backbone.wan.models.dit.WanModel13      extra_kwargs:14        has_image_input: false15        patch_size:16        - 117        - 218        - 219        in_dim: 4820        dim: 307221        ffn_dim: 1433622        freq_dim: 25623        text_dim: 409624        out_dim: 4825        num_heads: 2426        num_layers: 3027        eps: 1.0e-0628        seperated_timestep: true29        require_clip_embedding: false30        require_vae_embedding: false31        fuse_vae_embedding_in_latents: true32    - attr: vae33      model_class: openwam.model.video_backbone.wan.models.vae.WanVideoVAE3834      extra_kwargs: {}35    - attr: text_encoder36      model_class: openwam.model.video_backbone.wan.models.text_encoder.WanTextEncoder37      extra_kwargs: {}38    tokenizer:39      class: openwam.model.video_backbone.wan.models.text_encoder.HuggingfaceTokenizer40      attr: tokenizer41      subdir: tokenizer/google/umt5-xxl42      path_kwarg: name43      kwargs:44        seq_len: 51245        clean: whitespace46  action_backbone:47    dim: 102448    ffn_dim: 409649    shift_action: 5.050  freeze:51  - video_backbone.text_encoder52  - video_backbone.reason153  - video_backbone.vae54  - video_backbone.image_encoder55  - video_backbone.video_encoder56  architecture:57    framework: dual_system58    variant: joint_self_attn59    action_dim: 8060    use_proprioception: true61    state_dim: 8062    bridge_layers: null63    bridge_interval: 164    mot_checkpoint_mixed_attn: true65    attention_mask_mode: mutual66    video_attention_mask_mode: first_frame_causal67    detach_bridge: false68    idm_video_cond_noise_prob: 0.569dataloader:70  type: robodojo71  dataset_dir: /path/to/robodojo_data72  embodiment: arx_x573  variant: sim74  action_mode: eef75  unify_action: true76  unify_action_map:77  - 0-978  - 34-4379  unify_state_map: null80  num_frames: 3381  video_stride: 482  window_stride: 183  split: train84  height: 38485  width: 32086  multiview: true87  camera_layout:88  - cam_head89  - cam_left_wrist90  - cam_right_wrist91  target_camera: cam_head92  normalize_mode: min-max93  color_jitter:94    enabled: true95    brightness: 0.296    contrast: 0.297    saturation: 0.298    hue: 0.099  seed: 42100training:101  debug: false102  learning_rate: 0.0001103  adam_betas:104  - 0.9105  - 0.95106  weight_decay: 0.01107  max_grad_norm: 1.0108  num_epochs: null109  max_steps: 60000110  batch_size: 4111  gradient_accumulation_steps: 1112  lr_scheduler: cosine113  warmup_ratio: 0.05114  lr_min_ratio: 0.01115  action_lr: null116  video_lr: null117  mixed_precision: bf16118  zero_stage: 2119  use_gradient_checkpointing: false120  use_gradient_checkpointing_offload: false121  initialize_model_on_cpu: false122  offload_optimizer_device: none123  lambda_video: 1.0124  lambda_action: 1.0125  max_timestep_boundary: 1.0126  min_timestep_boundary: 0.0127  output_path: outputs/openwam_checkpoints128  save_steps: 2000129  save_full_states_for_resume: false130  keep_last_k_ckpts: 10131  finetune_ckpt_path: /path/to/OpenWAM-Pretrain-Foundation-Model132  resume_ckpt_path: null133  dataset_num_workers: 8134project:135  name: openwam136  output_dir: outputs/${now:%Y-%m-%d_%H-%M-%S}137  seed: 42138  wandb:139    project: openwam140    run_name: null141    entity: null142