samiyousef/Command-R-4-bit

1{2    "add_bos_token": true,3    "add_eos_token": false,4    "added_tokens_decoder": {5      "0": {6        "content": "<PAD>",7        "lstrip": false,8        "normalized": false,9        "rstrip": false,10        "single_word": false,11        "special": true12      },13      "1": {14        "content": "<UNK>",15        "lstrip": false,16        "normalized": false,17        "rstrip": false,18        "single_word": false,19        "special": true20      },21      "2": {22        "content": "<CLS>",23        "lstrip": false,24        "normalized": false,25        "rstrip": false,26        "single_word": false,27        "special": true28      },29      "3": {30        "content": "<SEP>",31        "lstrip": false,32        "normalized": false,33        "rstrip": false,34        "single_word": false,35        "special": true36      },37      "4": {38        "content": "<MASK_TOKEN>",39        "lstrip": false,40        "normalized": false,41        "rstrip": false,42        "single_word": false,43        "special": true44      },45      "5": {46        "content": "<BOS_TOKEN>",47        "lstrip": false,48        "normalized": false,49        "rstrip": false,50        "single_word": false,51        "special": true52      },53      "6": {54        "content": "<EOS_TOKEN>",55        "lstrip": false,56        "normalized": false,57        "rstrip": false,58        "single_word": false,59        "special": true60      },61      "7": {62        "content": "<EOP_TOKEN>",63        "lstrip": false,64        "normalized": false,65        "rstrip": false,66        "single_word": false,67        "special": true68      },69      "255000": {70        "content": "<|START_OF_TURN_TOKEN|>",71        "lstrip": false,72        "normalized": false,73        "rstrip": false,74        "single_word": false,75        "special": false76      },77      "255001": {78        "content": "<|END_OF_TURN_TOKEN|>",79        "lstrip": false,80        "normalized": false,81        "rstrip": false,82        "single_word": false,83        "special": false84      },85      "255002": {86        "content": "<|YES_TOKEN|>",87        "lstrip": false,88        "normalized": false,89        "rstrip": false,90        "single_word": false,91        "special": false92      },93      "255003": {94        "content": "<|NO_TOKEN|>",95        "lstrip": false,96        "normalized": false,97        "rstrip": false,98        "single_word": false,99        "special": false100      },101      "255004": {102        "content": "<|GOOD_TOKEN|>",103        "lstrip": false,104        "normalized": false,105        "rstrip": false,106        "single_word": false,107        "special": false108      },109      "255005": {110        "content": "<|BAD_TOKEN|>",111        "lstrip": false,112        "normalized": false,113        "rstrip": false,114        "single_word": false,115        "special": false116      },117      "255006": {118        "content": "<|USER_TOKEN|>",119        "lstrip": false,120        "normalized": false,121        "rstrip": false,122        "single_word": false,123        "special": false124      },125      "255007": {126        "content": "<|CHATBOT_TOKEN|>",127        "lstrip": false,128        "normalized": false,129        "rstrip": false,130        "single_word": false,131        "special": false132      },133      "255008": {134        "content": "<|SYSTEM_TOKEN|>",135        "lstrip": false,136        "normalized": false,137        "rstrip": false,138        "single_word": false,139        "special": false140      },141      "255009": {142        "content": "<|USER_0_TOKEN|>",143        "lstrip": false,144        "normalized": false,145        "rstrip": false,146        "single_word": false,147        "special": false148      },149      "255010": {150        "content": "<|USER_1_TOKEN|>",151        "lstrip": false,152        "normalized": false,153        "rstrip": false,154        "single_word": false,155        "special": false156      },157      "255011": {158        "content": "<|USER_2_TOKEN|>",159        "lstrip": false,160        "normalized": false,161        "rstrip": false,162        "single_word": false,163        "special": false164      },165      "255012": {166        "content": "<|USER_3_TOKEN|>",167        "lstrip": false,168        "normalized": false,169        "rstrip": false,170        "single_word": false,171        "special": false172      },173      "255013": {174        "content": "<|USER_4_TOKEN|>",175        "lstrip": false,176        "normalized": false,177        "rstrip": false,178        "single_word": false,179        "special": false180      },181      "255014": {182        "content": "<|USER_5_TOKEN|>",183        "lstrip": false,184        "normalized": false,185        "rstrip": false,186        "single_word": false,187        "special": false188      },189      "255015": {190        "content": "<|USER_6_TOKEN|>",191        "lstrip": false,192        "normalized": false,193        "rstrip": false,194        "single_word": false,195        "special": false196      },197      "255016": {198        "content": "<|USER_7_TOKEN|>",199        "lstrip": false,200        "normalized": false,201        "rstrip": false,202        "single_word": false,203        "special": false204      },205      "255017": {206        "content": "<|USER_8_TOKEN|>",207        "lstrip": false,208        "normalized": false,209        "rstrip": false,210        "single_word": false,211        "special": false212      },213      "255018": {214        "content": "<|USER_9_TOKEN|>",215        "lstrip": false,216        "normalized": false,217        "rstrip": false,218        "single_word": false,219        "special": false220      },221      "255019": {222        "content": "<|EXTRA_0_TOKEN|>",223        "lstrip": false,224        "normalized": false,225        "rstrip": false,226        "single_word": false,227        "special": false228      },229      "255020": {230        "content": "<|EXTRA_1_TOKEN|>",231        "lstrip": false,232        "normalized": false,233        "rstrip": false,234        "single_word": false,235        "special": false236      },237      "255021": {238        "content": "<|EXTRA_2_TOKEN|>",239        "lstrip": false,240        "normalized": false,241        "rstrip": false,242        "single_word": false,243        "special": false244      },245      "255022": {246        "content": "<|EXTRA_3_TOKEN|>",247        "lstrip": false,248        "normalized": false,249        "rstrip": false,250        "single_word": false,251        "special": false252      },253      "255023": {254        "content": "<|EXTRA_4_TOKEN|>",255        "lstrip": false,256        "normalized": false,257        "rstrip": false,258        "single_word": false,259        "special": false260      },261      "255024": {262        "content": "<|EXTRA_5_TOKEN|>",263        "lstrip": false,264        "normalized": false,265        "rstrip": false,266        "single_word": false,267        "special": false268      },269      "255025": {270        "content": "<|EXTRA_6_TOKEN|>",271        "lstrip": false,272        "normalized": false,273        "rstrip": false,274        "single_word": false,275        "special": false276      },277      "255026": {278        "content": "<|EXTRA_7_TOKEN|>",279        "lstrip": false,280        "normalized": false,281        "rstrip": false,282        "single_word": false,283        "special": false284      },285      "255027": {286        "content": "<|EXTRA_8_TOKEN|>",287        "lstrip": false,288        "normalized": false,289        "rstrip": false,290        "single_word": false,291        "special": false292      },293      "255028": {294        "content": "<|EXTRA_9_TOKEN|>",295        "lstrip": false,296        "normalized": false,297        "rstrip": false,298        "single_word": false,299        "special": false300      }301    },302    "auto_map": {303      "AutoTokenizer": [304        null,305        "tokenization_cohere_fast.CohereTokenizerFast"306      ]307    },308    "bos_token": "<BOS_TOKEN>",309    "clean_up_tokenization_spaces": false,310    "eos_token": "<|END_OF_TURN_TOKEN|>",311    "legacy": true,312    "model_max_length": 1000000000000000019884624838656,313    "pad_token": "<PAD>",314    "sp_model_kwargs": {},315    "spaces_between_special_tokens": false,316    "tokenizer_class": "CohereTokenizer",317    "unk_token": null,318    "use_default_system_prompt": false319  }