m-a-p/CT-LLM-Base

1{2  "auto_map": {3    "AutoTokenizer": ["tokenization_baichuan.BaichuanTokenizer", null]4  },5  "add_bos_token": false,6  "add_eos_token": false,7  "use_fast": false,8  "clean_up_tokenization_spaces": false,9  "eos_token": {10    "__type": "AddedToken",11    "content": "</s>",12    "lstrip": false,13    "normalized": true,14    "rstrip": false,15    "single_word": true16  },17  "model_max_length": 4096,18  "sp_model_kwargs": {},19  "tokenizer_class": "BaichuanTokenizer",20  "pad_token": {21    "__type": "AddedToken",22    "content": "<unk>",23    "lstrip": false,24    "normalized": true,25    "rstrip": false,26    "single_word": true27  },28  "unk_token": {29    "__type": "AddedToken",30    "content": "<unk>",31    "lstrip": false,32    "normalized": true,33    "rstrip": false,34    "single_word": true35  },36  "added_tokens_decoder": {37    "125696": {38      "content": "<|EOD|>",39      "lstrip": false,40      "normalized": false,41      "rstrip": false,42      "single_word": false,43      "special": true44    },45    "125697": {46      "content": "<|MASK|>",47      "lstrip": false,48      "normalized": false,49      "rstrip": false,50      "single_word": false,51      "special": true52    },53    "125698": {54      "content": "<|PAD|>",55      "lstrip": false,56      "normalized": false,57      "rstrip": false,58      "single_word": false,59      "special": true60    }61  },62  "additional_special_tokens": [63    "<|EOD|>",64    "<|MASK|>",65    "<|PAD|>"66  ]67}68