CMSManhattan/GemmaRoboticsTokenizer

1{2  "audio_token": "<|audio|>",3  "backend": "tokenizers",4  "boa_token": "<|audio>",5  "boi_token": "<|image>",6  "bos_token": "<bos>",7  "eoa_token": "<audio|>",8  "eoc_token": "<channel|>",9  "eoi_token": "<image|>",10  "eos_token": "<eos>",11  "eot_token": "<turn|>",12  "escape_token": "<|\"|>",13  "etc_token": "<tool_call|>",14  "etd_token": "<tool|>",15  "etr_token": "<tool_response|>",16  "extra_special_tokens": [17    "<|image|>",18    "<|video|>",19    "<|sound|>",20    "<|voice|>",21    "<|listening|>",22    "<|vision|>",23    "<|mood_happy|>",24    "<|mood_sad|>",25    "<|mood_angry|>",26    "<|mood_neutral|>",27    "<fim_prefix>",28    "<fim_middle>",29    "<fim_suffix>",30    "<|action_start|>",31    "<|action_end|>",32    "<|trajectory_start|>",33    "<|trajectory_end|>",34    "<|joint_start|>",35    "<|joint_end|>",36    "<|sensor_start|>",37    "<|sensor_end|>",38    "<|command_start|>",39    "<|command_end|>",40    "<|state_start|>",41    "<|state_end|>",42    "<|pose|>",43    "<|velocity|>",44    "<|force|>",45    "<|torque|>",46    "<|gripper|>",47    "<|navigation|>",48    "<|obstacle|>",49    "<|task_start|>",50    "<|task_end|>",51    "<|plan_start|>",52    "<|plan_end|>",53    "<|behavior_start|>",54    "<|behavior_end|>",55    "<|skill_start|>",56    "<|skill_end|>",57    "<|motor|>",58    "<|servo|>",59    "<|imu|>",60    "<|lidar|>",61    "<|camera|>",62    "<|depth|>",63    "<|waypoint|>",64    "<|path|>",65    "<|collision|>",66    "<|grasp|>",67    "<|release|>",68    "<|homing|>",69    "<|emergency_stop|>",70    "<|calibration|>",71    "<|manipulation|>",72    "<|locomotion|>",73    "<|feedback|>",74    "<|control_loop|>",75    "__SCIENCE__",76    "__CODING__",77    "__STOCK_EXCHANGE__",78    "__MEDICINE__",79    "__GOVERNMENT__",80    "__NEWS__",81    "__GENERAL__",82    "__MATERIAL_SCIENCE__",83    "__ELECTRONICS__",84    "__MICROELECTRONICS__",85    "__ENGINEERING__",86    "__ROBOTICS__",87    "__ENERGY__",88    "__AUTOMOTIVE__",89    "__AVIATION__",90    "__MATH__",91    "__PYTHON__",92    "__C__",93    "__CPP__",94    "__C_SHARP__",95    "__JAVA__",96    "__JAVASCRIPT__",97    "__TYPESCRIPT__",98    "__RUST__",99    "__GO__",100    "__RUBY__",101    "__PHP__",102    "__SWIFT__",103    "__KOTLIN__",104    "__BASH__",105    "__SQL__",106    "__ASSEMBLY__",107    "__PHILOSOPHY__",108    "__LITERATURE__",109    "__SOCIOLOGY__",110    "__PSYCHOLOGY__",111    "__POLITICAL_SCIENCE__",112    "__CULTURAL_STUDIES__",113    "__ETHNOGRAPHY__",114    "__HUMAN_RIGHTS__",115    "__COMPLIANCE__",116    "__MILITARY__",117    "__BANKING__",118    "__OIL_INDUSTRY__",119    "__LIGHT_INDUSTRY__",120    "__NATURE__",121    "__OCEAN__",122    "__SPORT__",123    "__CULINARY__",124    "__TRAVEL__",125    "__HOBBY__"126  ],127  "image_token": "<|image|>",128  "is_local": false,129  "local_files_only": false,130  "mask_token": "<mask>",131  "model_max_length": 1000000000000000019884624838656,132  "model_specific_special_tokens": {133    "audio_token": "<|audio|>",134    "boa_token": "<|audio>",135    "boi_token": "<|image>",136    "eoa_token": "<audio|>",137    "eoc_token": "<channel|>",138    "eoi_token": "<image|>",139    "eot_token": "<turn|>",140    "escape_token": "<|\"|>",141    "etc_token": "<tool_call|>",142    "etd_token": "<tool|>",143    "etr_token": "<tool_response|>",144    "image_token": "<|image|>",145    "soc_token": "<|channel>",146    "sot_token": "<|turn>",147    "stc_token": "<|tool_call>",148    "std_token": "<|tool>",149    "str_token": "<|tool_response>",150    "think_token": "<|think|>"151  },152  "pad_token": "<pad>",153  "padding_side": "left",154  "processor_class": "Gemma4Processor",155  "soc_token": "<|channel>",156  "sot_token": "<|turn>",157  "stc_token": "<|tool_call>",158  "std_token": "<|tool>",159  "str_token": "<|tool_response>",160  "think_token": "<|think|>",161  "tokenizer_class": "GemmaTokenizer",162  "unk_token": "<unk>"163}164