CoolFace
Modelpublic

openbmb/MiniCPM3-4B-GPTQ-Int4

sourceHugging Faceapache-2.0updated 2y agoView on Hugging Face
12likes163downloads
tokenizer_config.json120 linesDownload Raw Back to root
1{2  "auto_map": {3    "AutoTokenizer": ["tokenization_minicpm.MiniCPMTokenizer", null]4  },5  "add_bos_token": true,6  "add_eos_token": false,7  "added_tokens_decoder": {8    "0": {9      "content": "<unk>",10      "lstrip": false,11      "normalized": false,12      "rstrip": false,13      "single_word": false,14      "special": true15    },16    "1": {17      "content": "<s>",18      "lstrip": false,19      "normalized": false,20      "rstrip": false,21      "single_word": false,22      "special": true23    },24    "2": {25      "content": "</s>",26      "lstrip": false,27      "normalized": false,28      "rstrip": false,29      "single_word": false,30      "special": true31    },32    "73440": {33      "content": "<|im_end|>",34      "lstrip": false,35      "normalized": false,36      "rstrip": false,37      "single_word": false,38      "special": true39    },40    "73441": {41      "content": "<|im_start|>",42      "lstrip": false,43      "normalized": false,44      "rstrip": false,45      "single_word": false,46      "special": true47    },48    "73442": {49      "content": "<|tool_call|>",50      "lstrip": false,51      "normalized": false,52      "rstrip": false,53      "single_word": false,54      "special": true55    },56    "73443": {57      "content": "<|execute_start|>",58      "lstrip": false,59      "normalized": false,60      "rstrip": false,61      "single_word": false,62      "special": true63    },64    "73444": {65      "content": "<|execute_end|>",66      "lstrip": false,67      "normalized": false,68      "rstrip": false,69      "single_word": false,70      "special": true71    },72    "73445": {73      "content": "<|fim_prefix|>",74      "lstrip": false,75      "normalized": false,76      "rstrip": false,77      "single_word": false,78      "special": true79    },80    "73446": {81      "content": "<|fim_middle|>",82      "lstrip": false,83      "normalized": false,84      "rstrip": false,85      "single_word": false,86      "special": true87    },88    "73447": {89      "content": "<|fim_suffix|>",90      "lstrip": false,91      "normalized": false,92      "rstrip": false,93      "single_word": false,94      "special": true95    }96  },97  "additional_special_tokens": [98    "<|im_end|>",99    "<|im_start|>",100    "<|tool_call|>",101    "<|execute_start|>",102    "<|execute_end|>",103    "<|fim_prefix|>",104    "<|fim_middle|>",105    "<|fim_suffix|>"106  ],107  "bos_token": "<s>",108  "clean_up_tokenization_spaces": false,109  "eos_token": "<|im_end|>",110  "legacy": true,111  "model_max_length": 1000000000000000019884624838656,112  "pad_token": null,113  "sp_model_kwargs": {},114  "spaces_between_special_tokens": false,115  "tokenizer_class": "MiniCPMTokenizer",116  "unk_token": "<unk>",117  "use_default_system_prompt": false,118  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}"119}120