Go4miii/DISC-FinLLM

1{2  "add_bos_token": false,3  "add_eos_token": false,4  "auto_map": {5    "AutoTokenizer": [6      "tokenization_baichuan.BaichuanTokenizer",7      null8    ]9  },10  "bos_token": {11    "__type": "AddedToken",12    "content": "<s>",13    "lstrip": false,14    "normalized": true,15    "rstrip": false,16    "single_word": true17  },18  "clean_up_tokenization_spaces": false,19  "eos_token": {20    "__type": "AddedToken",21    "content": "</s>",22    "lstrip": false,23    "normalized": true,24    "rstrip": false,25    "single_word": true26  },27  "model_max_length": 4096,28  "pad_token": {29    "__type": "AddedToken",30    "content": "<unk>",31    "lstrip": false,32    "normalized": true,33    "rstrip": false,34    "single_word": true35  },36  "padding_side": "right",37  "sp_model_kwargs": {},38  "split_special_tokens": false,39  "tokenizer_class": "BaichuanTokenizer",40  "unk_token": {41    "__type": "AddedToken",42    "content": "<unk>",43    "lstrip": false,44    "normalized": true,45    "rstrip": false,46    "single_word": true47  }48}49