inference-optimization/DSV4-tiny-empty

1{2  "add_bos_token": false,3  "add_eos_token": false,4  "bos_token": {5    "__type": "AddedToken",6    "content": "<｜begin▁of▁sentence｜>",7    "lstrip": false,8    "normalized": true,9    "rstrip": false,10    "single_word": false11  },12  "clean_up_tokenization_spaces": false,13  "eos_token": {14    "__type": "AddedToken",15    "content": "<｜end▁of▁sentence｜>",16    "lstrip": false,17    "normalized": true,18    "rstrip": false,19    "single_word": false20  },21  "legacy": true,22  "model_max_length": 1048576,23  "pad_token": {24    "__type": "AddedToken",25    "content": "<｜end▁of▁sentence｜>",26    "lstrip": false,27    "normalized": true,28    "rstrip": false,29    "single_word": false30  },31  "sp_model_kwargs": {},32  "unk_token": null,33  "tokenizer_class": "PreTrainedTokenizerFast"34}