OpenFormosa/PangolinTokenizer

1{2  "ok": true,3  "tokenizer": "voidful/PangolinTokenizer",4  "base_vocab_size": 114688,5  "effective_vocab_size": 114822,6  "required_special_token_count": 157,7  "required_special_tokens_missing": [],8  "required_special_tokens_single_id": true,9  "standard_special_tokens": {10    "bos_token": "<s>",11    "eos_token": "</s>",12    "unk_token": "<unk>",13    "pad_token": "<pad>"14  },15  "banned_token_matches": {16    "audio_numeric_tokens": [],17    "dense_timestamp_tokens": []18  },19  "smoke_texts": {20    "traditional_chinese": "台灣本土語言模型需要保留繁體中文、注音ㄅㄆㄇ與台語漢字。",21    "duplex": "<|task:full_duplex_speech|><|audio_ref_start|>audio://utterance_000001<|audio_ref_end|>",22    "tool_call": "<|tool_call_start|>{\"name\":\"search_documents\",\"arguments\":{\"query\":\"健保年金\"}}<|tool_call_end|>"23  }24}25