OpenFormosa/PangolinTokenizer

1{2  "unk_token": "<unk>",3  "bos_token": "<s>",4  "eos_token": "</s>",5  "pad_token": "<pad>",6  "additional_special_tokens": [7    "<|bos|>",8    "<|eos|>",9    "<|unk|>",10    "<|pad|>",11    "<|system|>",12    "<|user_channel|>",13    "<|assistant_channel|>",14    "<|task:speech_to_text|>",15    "<|task:text_to_speech|>",16    "<|input_audio_start|>",17    "<|input_audio_end|>",18    "<|audio_ref_start|>",19    "<|audio_ref_end|>",20    "<|audio_start|>",21    "<|audio_end|>",22    "<|speech_start|>",23    "<|speech_end|>",24    "<|transcript_start|>",25    "<|transcript_end|>",26    "<|segment_start|>",27    "<|segment_end|>",28    "<|speaker|>",29    "<|start_time|>",30    "<|end_time|>",31    "<|duration|>",32    "<|content|>",33    "<|non_speech_event|>",34    "<|retrieval_result_start|>",35    "<|retrieval_result_end|>",36    "<|ocr_start|>",37    "<|ocr_end|>",38    "<|image_start|>",39    "<|image_end|>",40    "<|video_start|>",41    "<|video_end|>",42    "<|user|>",43    "<|assistant|>",44    "<|tool_call|>",45    "<|tool_response|>",46    "<|endoftext|>",47    "<think>",48    "</think>",49    "<|no_think|>",50    "<|think|>",51    "<|think_max|>",52    "<|task:text_to_text|>",53    "<|task:speech_to_speech|>",54    "<|task:text_speech_to_text|>",55    "<|task:text_speech_to_speech|>",56    "<|task:full_duplex_speech|>",57    "<|task:agent|>",58    "<|task:tool_use|>",59    "<|task:rag|>",60    "<|task:code_execution|>",61    "<|task:document_qa|>",62    "<|task:data_analysis|>",63    "<|task:workflow|>",64    "<|reasoning_mode:none|>",65    "<|reasoning_mode:short|>",66    "<|reasoning_mode:deep|>",67    "<|reasoning_mode:verify|>",68    "<|private_reasoning_start|>",69    "<|private_reasoning_end|>",70    "<|reasoning_summary_start|>",71    "<|reasoning_summary_end|>",72    "<|plan_start|>",73    "<|plan_end|>",74    "<|step_start|>",75    "<|step_end|>",76    "<|action_start|>",77    "<|action_end|>",78    "<|observation_start|>",79    "<|observation_end|>",80    "<|reflection_start|>",81    "<|reflection_end|>",82    "<|verification_start|>",83    "<|verification_end|>",84    "<|tool_schema_start|>",85    "<|tool_schema_end|>",86    "<|tool_call_start|>",87    "<|tool_call_end|>",88    "<|tool_result_start|>",89    "<|tool_result_end|>",90    "<|tool_error_start|>",91    "<|tool_error_end|>",92    "<|retrieval_query_start|>",93    "<|retrieval_query_end|>",94    "<|citation_start|>",95    "<|citation_end|>",96    "<|memory_read_start|>",97    "<|memory_read_end|>",98    "<|memory_write_start|>",99    "<|memory_write_end|>",100    "<|final_answer_start|>",101    "<|final_answer_end|>",102    "<|json_start|>",103    "<|json_end|>",104    "<|code_start|>",105    "<|code_end|>",106    "<|markdown_start|>",107    "<|markdown_end|>",108    "<|duplex_start|>",109    "<|duplex_end|>",110    "<|system_channel|>",111    "<|listen|>",112    "<|speak|>",113    "<|listen_speak|>",114    "<|output_audio_start|>",115    "<|output_audio_end|>",116    "<|text_start|>",117    "<|text_end|>",118    "<|overlap|>",119    "<|barge_in|>",120    "<|interruption|>",121    "<|interruption_repair|>",122    "<|backchannel|>",123    "<|turn_yield|>",124    "<|hold|>",125    "<|silence|>",126    "<|non_speech|>",127    "<|voice_reference_start|>",128    "<|voice_reference_end|>",129    "<|voice_reference|>",130    "<|voice_switch|>",131    "<|speaker_style|>",132    "<|prosody_control|>",133    "<|zh_tw|>",134    "<|zh_hant|>",135    "<|taigi|>",136    "<|hakka|>",137    "<|bopomofo|>",138    "<|mixed_en|>",139    "<|en|>",140    "<|ja|>",141    "<|ko|>",142    "<|vi|>",143    "<|id|>",144    "<|th|>",145    "<|asr|>",146    "<|tts|>",147    "<|speaker_0|>",148    "<|speaker_1|>",149    "<|speaker_2|>",150    "<|speaker_3|>",151    "<|timestamp|>",152    "<|noise|>",153    "<|laugh|>",154    "<|breath|>",155    "<|pause|>",156    "<|prosody|>",157    "<|pron|>",158    "</|pron|>",159    "<|image|>",160    "<|ocr|>",161    "<|bbox|>",162    "<|line|>",163    "<|table|>",164    "<|row|>",165    "<|col|>",166    "<|cell|>",167    "<|reading_order|>",168    "<|source|>",169    "<|cite|>",170    "<|evidence|>",171    "<|quote|>"172  ]173}174