nvidia/SO_ARM_Starter_Gr00tN17
1127
1{2 "processor_class": "Gr00tN1d7Processor",3 "processor_kwargs": {4 "modality_configs": {5 "real_g1_relative_eef_relative_joints": {6 "video": {7 "delta_indices": [8 -20,9 010 ],11 "modality_keys": [12 "ego_view"13 ],14 "sin_cos_embedding_keys": null,15 "mean_std_embedding_keys": null,16 "action_configs": null17 },18 "state": {19 "delta_indices": [20 021 ],22 "modality_keys": [23 "left_wrist_eef_9d",24 "right_wrist_eef_9d",25 "left_hand",26 "right_hand",27 "left_arm",28 "right_arm",29 "waist"30 ],31 "sin_cos_embedding_keys": null,32 "mean_std_embedding_keys": null,33 "action_configs": null34 },35 "action": {36 "delta_indices": [37 0,38 1,39 2,40 3,41 4,42 5,43 6,44 7,45 8,46 9,47 10,48 11,49 12,50 13,51 14,52 15,53 16,54 17,55 18,56 19,57 20,58 21,59 22,60 23,61 24,62 25,63 26,64 27,65 28,66 29,67 30,68 31,69 32,70 33,71 34,72 35,73 36,74 37,75 38,76 3977 ],78 "modality_keys": [79 "left_wrist_eef_9d",80 "right_wrist_eef_9d",81 "left_hand",82 "right_hand",83 "left_arm",84 "right_arm",85 "waist",86 "base_height_command",87 "navigate_command"88 ],89 "sin_cos_embedding_keys": null,90 "mean_std_embedding_keys": null,91 "action_configs": [92 {93 "rep": "RELATIVE",94 "type": "EEF",95 "format": "XYZ_ROT6D",96 "state_key": "left_wrist_eef_9d"97 },98 {99 "rep": "RELATIVE",100 "type": "EEF",101 "format": "XYZ_ROT6D",102 "state_key": "right_wrist_eef_9d"103 },104 {105 "rep": "ABSOLUTE",106 "type": "NON_EEF",107 "format": "DEFAULT",108 "state_key": "left_hand"109 },110 {111 "rep": "ABSOLUTE",112 "type": "NON_EEF",113 "format": "DEFAULT",114 "state_key": "right_hand"115 },116 {117 "rep": "RELATIVE",118 "type": "NON_EEF",119 "format": "DEFAULT",120 "state_key": "left_arm"121 },122 {123 "rep": "RELATIVE",124 "type": "NON_EEF",125 "format": "DEFAULT",126 "state_key": "right_arm"127 },128 {129 "rep": "ABSOLUTE",130 "type": "NON_EEF",131 "format": "DEFAULT",132 "state_key": "waist"133 },134 {135 "rep": "ABSOLUTE",136 "type": "NON_EEF",137 "format": "DEFAULT",138 "state_key": "base_height_command"139 },140 {141 "rep": "ABSOLUTE",142 "type": "NON_EEF",143 "format": "DEFAULT",144 "state_key": "navigate_command"145 }146 ]147 },148 "language": {149 "delta_indices": [150 0151 ],152 "modality_keys": [153 "annotation.human.task_description"154 ],155 "sin_cos_embedding_keys": null,156 "mean_std_embedding_keys": null,157 "action_configs": null158 }159 },160 "real_r1_pro_sharpa_relative_eef_mecka": {161 "video": {162 "delta_indices": [163 -30,164 0165 ],166 "modality_keys": [167 "ego_view_cropratio_res320x240_freq30"168 ],169 "sin_cos_embedding_keys": null,170 "mean_std_embedding_keys": null,171 "action_configs": null172 },173 "state": {174 "delta_indices": [175 0176 ],177 "modality_keys": [178 "left_wrist_eef",179 "right_wrist_eef",180 "left_hand_joints",181 "right_hand_joints"182 ],183 "sin_cos_embedding_keys": null,184 "mean_std_embedding_keys": null,185 "action_configs": null186 },187 "action": {188 "delta_indices": [189 0,190 1,191 2,192 3,193 4,194 5,195 6,196 7,197 8,198 9,199 10,200 11,201 12,202 13,203 14,204 15,205 16,206 17,207 18,208 19,209 20,210 21,211 22,212 23,213 24,214 25,215 26,216 27,217 28,218 29,219 30,220 31,221 32,222 33,223 34,224 35,225 36,226 37,227 38,228 39229 ],230 "modality_keys": [231 "left_wrist_eef",232 "right_wrist_eef",233 "left_hand_joints",234 "right_hand_joints"235 ],236 "sin_cos_embedding_keys": null,237 "mean_std_embedding_keys": null,238 "action_configs": [239 {240 "rep": "RELATIVE",241 "type": "EEF",242 "format": "XYZ_ROT6D",243 "state_key": "left_wrist_eef"244 },245 {246 "rep": "RELATIVE",247 "type": "EEF",248 "format": "XYZ_ROT6D",249 "state_key": "right_wrist_eef"250 },251 {252 "rep": "ABSOLUTE",253 "type": "NON_EEF",254 "format": "DEFAULT",255 "state_key": "left_hand_joints"256 },257 {258 "rep": "ABSOLUTE",259 "type": "NON_EEF",260 "format": "DEFAULT",261 "state_key": "right_hand_joints"262 }263 ]264 },265 "language": {266 "delta_indices": [267 0268 ],269 "modality_keys": [270 "annotation.human.coarse_action"271 ],272 "sin_cos_embedding_keys": null,273 "mean_std_embedding_keys": null,274 "action_configs": null275 }276 },277 "real_r1_pro_sharpa_relative_eef_human": {278 "video": {279 "delta_indices": [280 -20,281 0282 ],283 "modality_keys": [284 "ego_view_res320x240_freq20",285 "left_wrist_view_res320x240_freq20",286 "right_wrist_view_res320x240_freq20"287 ],288 "sin_cos_embedding_keys": null,289 "mean_std_embedding_keys": null,290 "action_configs": null291 },292 "state": {293 "delta_indices": [294 0295 ],296 "modality_keys": [297 "left_wrist_eef",298 "right_wrist_eef",299 "left_hand_joints",300 "right_hand_joints"301 ],302 "sin_cos_embedding_keys": null,303 "mean_std_embedding_keys": null,304 "action_configs": null305 },306 "action": {307 "delta_indices": [308 0,309 1,310 2,311 3,312 4,313 5,314 6,315 7,316 8,317 9,318 10,319 11,320 12,321 13,322 14,323 15,324 16,325 17,326 18,327 19,328 20,329 21,330 22,331 23,332 24,333 25,334 26,335 27,336 28,337 29,338 30,339 31,340 32,341 33,342 34,343 35,344 36,345 37,346 38,347 39348 ],349 "modality_keys": [350 "left_wrist_eef",351 "right_wrist_eef",352 "left_hand_joints",353 "right_hand_joints"354 ],355 "sin_cos_embedding_keys": null,356 "mean_std_embedding_keys": null,357 "action_configs": [358 {359 "rep": "RELATIVE",360 "type": "EEF",361 "format": "XYZ_ROT6D",362 "state_key": "left_wrist_eef"363 },364 {365 "rep": "RELATIVE",366 "type": "EEF",367 "format": "XYZ_ROT6D",368 "state_key": "right_wrist_eef"369 },370 {371 "rep": "ABSOLUTE",372 "type": "NON_EEF",373 "format": "DEFAULT",374 "state_key": "left_hand_joints"375 },376 {377 "rep": "ABSOLUTE",378 "type": "NON_EEF",379 "format": "DEFAULT",380 "state_key": "right_hand_joints"381 }382 ]383 },384 "language": {385 "delta_indices": [386 0387 ],388 "modality_keys": [389 "annotation.human.coarse_action"390 ],391 "sin_cos_embedding_keys": null,392 "mean_std_embedding_keys": null,393 "action_configs": null394 }395 },396 "real_r1_pro_sharpa_relative_eef": {397 "video": {398 "delta_indices": [399 -20,400 0401 ],402 "modality_keys": [403 "ego_view_res320x240_freq20",404 "left_wrist_view_res320x240_freq20",405 "right_wrist_view_res320x240_freq20"406 ],407 "sin_cos_embedding_keys": null,408 "mean_std_embedding_keys": null,409 "action_configs": null410 },411 "state": {412 "delta_indices": [413 0414 ],415 "modality_keys": [416 "left_wrist_eef",417 "right_wrist_eef",418 "left_hand_joints",419 "right_hand_joints"420 ],421 "sin_cos_embedding_keys": null,422 "mean_std_embedding_keys": null,423 "action_configs": null424 },425 "action": {426 "delta_indices": [427 0,428 1,429 2,430 3,431 4,432 5,433 6,434 7,435 8,436 9,437 10,438 11,439 12,440 13,441 14,442 15,443 16,444 17,445 18,446 19,447 20,448 21,449 22,450 23,451 24,452 25,453 26,454 27,455 28,456 29,457 30,458 31,459 32,460 33,461 34,462 35,463 36,464 37,465 38,466 39467 ],468 "modality_keys": [469 "left_wrist_eef",470 "right_wrist_eef",471 "left_hand_joints",472 "right_hand_joints"473 ],474 "sin_cos_embedding_keys": null,475 "mean_std_embedding_keys": null,476 "action_configs": [477 {478 "rep": "RELATIVE",479 "type": "EEF",480 "format": "XYZ_ROT6D",481 "state_key": "left_wrist_eef"482 },483 {484 "rep": "RELATIVE",485 "type": "EEF",486 "format": "XYZ_ROT6D",487 "state_key": "right_wrist_eef"488 },489 {490 "rep": "ABSOLUTE",491 "type": "NON_EEF",492 "format": "DEFAULT",493 "state_key": "left_hand_joints"494 },495 {496 "rep": "ABSOLUTE",497 "type": "NON_EEF",498 "format": "DEFAULT",499 "state_key": "right_hand_joints"500 }501 ]502 },503 "language": {504 "delta_indices": [505 0506 ],507 "modality_keys": [508 "annotation.human.coarse_action"509 ],510 "sin_cos_embedding_keys": null,511 "mean_std_embedding_keys": null,512 "action_configs": null513 }514 },515 "xdof_relative_eef_relative_joint": {516 "video": {517 "delta_indices": [518 -30,519 0520 ],521 "modality_keys": [522 "top_camera-images-rgb_320_240",523 "left_camera-images-rgb_320_240",524 "right_camera-images-rgb_320_240"525 ],526 "sin_cos_embedding_keys": null,527 "mean_std_embedding_keys": null,528 "action_configs": null529 },530 "state": {531 "delta_indices": [532 0533 ],534 "modality_keys": [535 "left_wrist_eef",536 "right_wrist_eef",537 "left_gripper_pos",538 "right_gripper_pos",539 "left_joint_pos",540 "right_joint_pos"541 ],542 "sin_cos_embedding_keys": null,543 "mean_std_embedding_keys": null,544 "action_configs": null545 },546 "action": {547 "delta_indices": [548 0,549 1,550 2,551 3,552 4,553 5,554 6,555 7,556 8,557 9,558 10,559 11,560 12,561 13,562 14,563 15,564 16,565 17,566 18,567 19,568 20,569 21,570 22,571 23,572 24,573 25,574 26,575 27,576 28,577 29,578 30,579 31,580 32,581 33,582 34,583 35,584 36,585 37,586 38,587 39588 ],589 "modality_keys": [590 "left_wrist_eef",591 "right_wrist_eef",592 "left_gripper_pos",593 "right_gripper_pos",594 "left_joint_pos",595 "right_joint_pos"596 ],597 "sin_cos_embedding_keys": null,598 "mean_std_embedding_keys": null,599 "action_configs": [600 {601 "rep": "RELATIVE",602 "type": "EEF",603 "format": "XYZ_ROT6D",604 "state_key": "left_wrist_eef"605 },606 {607 "rep": "RELATIVE",608 "type": "EEF",609 "format": "XYZ_ROT6D",610 "state_key": "right_wrist_eef"611 },612 {613 "rep": "ABSOLUTE",614 "type": "NON_EEF",615 "format": "DEFAULT",616 "state_key": "left_gripper_pos"617 },618 {619 "rep": "ABSOLUTE",620 "type": "NON_EEF",621 "format": "DEFAULT",622 "state_key": "right_gripper_pos"623 },624 {625 "rep": "RELATIVE",626 "type": "NON_EEF",627 "format": "DEFAULT",628 "state_key": "left_joint_pos"629 },630 {631 "rep": "RELATIVE",632 "type": "NON_EEF",633 "format": "DEFAULT",634 "state_key": "right_joint_pos"635 }636 ]637 },638 "language": {639 "delta_indices": [640 0641 ],642 "modality_keys": [643 "annotation.task"644 ],645 "sin_cos_embedding_keys": null,646 "mean_std_embedding_keys": null,647 "action_configs": null648 }649 },650 "real_r1_pro_sharpa_relative_eef_maxinsights": {651 "video": {652 "delta_indices": [653 -30,654 0655 ],656 "modality_keys": [657 "ego_view_cropratio_res320x240_freq30"658 ],659 "sin_cos_embedding_keys": null,660 "mean_std_embedding_keys": null,661 "action_configs": null662 },663 "state": {664 "delta_indices": [665 0666 ],667 "modality_keys": [668 "left_wrist_eef",669 "right_wrist_eef",670 "left_hand_joints",671 "right_hand_joints"672 ],673 "sin_cos_embedding_keys": null,674 "mean_std_embedding_keys": null,675 "action_configs": null676 },677 "action": {678 "delta_indices": [679 0,680 1,681 2,682 3,683 4,684 5,685 6,686 7,687 8,688 9,689 10,690 11,691 12,692 13,693 14,694 15,695 16,696 17,697 18,698 19,699 20,700 21,701 22,702 23,703 24,704 25,705 26,706 27,707 28,708 29,709 30,710 31,711 32,712 33,713 34,714 35,715 36,716 37,717 38,718 39719 ],720 "modality_keys": [721 "left_wrist_eef",722 "right_wrist_eef",723 "left_hand_joints",724 "right_hand_joints"725 ],726 "sin_cos_embedding_keys": null,727 "mean_std_embedding_keys": null,728 "action_configs": [729 {730 "rep": "RELATIVE",731 "type": "EEF",732 "format": "XYZ_ROT6D",733 "state_key": "left_wrist_eef"734 },735 {736 "rep": "RELATIVE",737 "type": "EEF",738 "format": "XYZ_ROT6D",739 "state_key": "right_wrist_eef"740 },741 {742 "rep": "ABSOLUTE",743 "type": "NON_EEF",744 "format": "DEFAULT",745 "state_key": "left_hand_joints"746 },747 {748 "rep": "ABSOLUTE",749 "type": "NON_EEF",750 "format": "DEFAULT",751 "state_key": "right_hand_joints"752 }753 ]754 },755 "language": {756 "delta_indices": [757 0758 ],759 "modality_keys": [760 "annotation.human.coarse_action"761 ],762 "sin_cos_embedding_keys": null,763 "mean_std_embedding_keys": null,764 "action_configs": null765 }766 },767 "xdof_relative_eef_relative_joint_subtask": {768 "video": {769 "delta_indices": [770 -30,771 0772 ],773 "modality_keys": [774 "top_camera-images-rgb_320_240",775 "left_camera-images-rgb_320_240",776 "right_camera-images-rgb_320_240"777 ],778 "sin_cos_embedding_keys": null,779 "mean_std_embedding_keys": null,780 "action_configs": null781 },782 "state": {783 "delta_indices": [784 0785 ],786 "modality_keys": [787 "left_wrist_eef",788 "right_wrist_eef",789 "left_gripper_pos",790 "right_gripper_pos",791 "left_joint_pos",792 "right_joint_pos"793 ],794 "sin_cos_embedding_keys": null,795 "mean_std_embedding_keys": null,796 "action_configs": null797 },798 "action": {799 "delta_indices": [800 0,801 1,802 2,803 3,804 4,805 5,806 6,807 7,808 8,809 9,810 10,811 11,812 12,813 13,814 14,815 15,816 16,817 17,818 18,819 19,820 20,821 21,822 22,823 23,824 24,825 25,826 26,827 27,828 28,829 29,830 30,831 31,832 32,833 33,834 34,835 35,836 36,837 37,838 38,839 39840 ],841 "modality_keys": [842 "left_wrist_eef",843 "right_wrist_eef",844 "left_gripper_pos",845 "right_gripper_pos",846 "left_joint_pos",847 "right_joint_pos"848 ],849 "sin_cos_embedding_keys": null,850 "mean_std_embedding_keys": null,851 "action_configs": [852 {853 "rep": "RELATIVE",854 "type": "EEF",855 "format": "XYZ_ROT6D",856 "state_key": "left_wrist_eef"857 },858 {859 "rep": "RELATIVE",860 "type": "EEF",861 "format": "XYZ_ROT6D",862 "state_key": "right_wrist_eef"863 },864 {865 "rep": "ABSOLUTE",866 "type": "NON_EEF",867 "format": "DEFAULT",868 "state_key": "left_gripper_pos"869 },870 {871 "rep": "ABSOLUTE",872 "type": "NON_EEF",873 "format": "DEFAULT",874 "state_key": "right_gripper_pos"875 },876 {877 "rep": "RELATIVE",878 "type": "NON_EEF",879 "format": "DEFAULT",880 "state_key": "left_joint_pos"881 },882 {883 "rep": "RELATIVE",884 "type": "NON_EEF",885 "format": "DEFAULT",886 "state_key": "right_joint_pos"887 }888 ]889 },890 "language": {891 "delta_indices": [892 0893 ],894 "modality_keys": [895 "annotation.sub_task"896 ],897 "sin_cos_embedding_keys": null,898 "mean_std_embedding_keys": null,899 "action_configs": null900 }901 },902 "oxe_droid_relative_eef_relative_joint": {903 "video": {904 "delta_indices": [905 -15,906 0907 ],908 "modality_keys": [909 "exterior_image_1_left",910 "wrist_image_left"911 ],912 "sin_cos_embedding_keys": null,913 "mean_std_embedding_keys": null,914 "action_configs": null915 },916 "state": {917 "delta_indices": [918 0919 ],920 "modality_keys": [921 "eef_9d",922 "gripper_position",923 "joint_position"924 ],925 "sin_cos_embedding_keys": null,926 "mean_std_embedding_keys": null,927 "action_configs": null928 },929 "action": {930 "delta_indices": [931 0,932 1,933 2,934 3,935 4,936 5,937 6,938 7,939 8,940 9,941 10,942 11,943 12,944 13,945 14,946 15,947 16,948 17,949 18,950 19,951 20,952 21,953 22,954 23,955 24,956 25,957 26,958 27,959 28,960 29,961 30,962 31,963 32,964 33,965 34,966 35,967 36,968 37,969 38,970 39971 ],972 "modality_keys": [973 "eef_9d",974 "gripper_position",975 "joint_position"976 ],977 "sin_cos_embedding_keys": null,978 "mean_std_embedding_keys": null,979 "action_configs": [980 {981 "rep": "RELATIVE",982 "type": "EEF",983 "format": "XYZ_ROT6D",984 "state_key": "eef_9d"985 },986 {987 "rep": "ABSOLUTE",988 "type": "NON_EEF",989 "format": "DEFAULT",990 "state_key": "gripper_position"991 },992 {993 "rep": "RELATIVE",994 "type": "NON_EEF",995 "format": "DEFAULT",996 "state_key": "joint_position"997 }998 ]999 },1000 "language": {1001 "delta_indices": [1002 01003 ],1004 "modality_keys": [1005 "annotation.language.language_instruction"1006 ],1007 "sin_cos_embedding_keys": null,1008 "mean_std_embedding_keys": null,1009 "action_configs": null1010 }1011 },1012 "new_embodiment": {1013 "video": {1014 "delta_indices": [1015 01016 ],1017 "modality_keys": [1018 "room",1019 "wrist"1020 ],1021 "sin_cos_embedding_keys": null,1022 "mean_std_embedding_keys": null,1023 "action_configs": null1024 },1025 "state": {1026 "delta_indices": [1027 01028 ],1029 "modality_keys": [1030 "single_arm",1031 "gripper"1032 ],1033 "sin_cos_embedding_keys": null,1034 "mean_std_embedding_keys": null,1035 "action_configs": null1036 },1037 "action": {1038 "delta_indices": [1039 0,1040 1,1041 2,1042 3,1043 4,1044 5,1045 6,1046 7,1047 8,1048 9,1049 10,1050 11,1051 12,1052 13,1053 14,1054 151055 ],1056 "modality_keys": [1057 "single_arm",1058 "gripper"1059 ],1060 "sin_cos_embedding_keys": null,1061 "mean_std_embedding_keys": null,1062 "action_configs": [1063 {1064 "rep": "ABSOLUTE",1065 "type": "NON_EEF",1066 "format": "DEFAULT",1067 "state_key": null1068 },1069 {1070 "rep": "ABSOLUTE",1071 "type": "NON_EEF",1072 "format": "DEFAULT",1073 "state_key": null1074 }1075 ]1076 },1077 "language": {1078 "delta_indices": [1079 01080 ],1081 "modality_keys": [1082 "annotation.human.task_description"1083 ],1084 "sin_cos_embedding_keys": null,1085 "mean_std_embedding_keys": null,1086 "action_configs": null1087 }1088 }1089 },1090 "image_crop_size": [1091 230,1092 2301093 ],1094 "image_target_size": [1095 256,1096 2561097 ],1098 "use_albumentations": true,1099 "random_rotation_angle": 0,1100 "color_jitter_params": {1101 "brightness": 0.3,1102 "contrast": 0.4,1103 "saturation": 0.5,1104 "hue": 0.081105 },1106 "shortest_image_edge": 256,1107 "crop_fraction": 0.95,1108 "letter_box_transform": false,1109 "model_name": "nvidia/Cosmos-Reason2-2B",1110 "model_type": "qwen",1111 "formalize_language": true,1112 "max_state_dim": 132,1113 "max_action_dim": 132,1114 "max_action_horizon": 40,1115 "use_percentiles": true,1116 "use_mean_std": false,1117 "clip_outliers": true,1118 "apply_sincos_state_encoding": false,1119 "use_relative_action": true,1120 "exclude_state": false,1121 "state_dropout_prob": 0.21122 }1123}