karths/WizardCodeLLM_commit_maintain

1{2  "added_tokens_decoder": {3    "0": {4      "content": "<unk>",5      "lstrip": false,6      "normalized": true,7      "rstrip": false,8      "single_word": false,9      "special": true10    },11    "1": {12      "content": "<s>",13      "lstrip": false,14      "normalized": true,15      "rstrip": false,16      "single_word": false,17      "special": true18    },19    "2": {20      "content": "</s>",21      "lstrip": false,22      "normalized": false,23      "rstrip": false,24      "single_word": false,25      "special": true26    },27    "32000": {28      "content": "[PAD]",29      "lstrip": false,30      "normalized": false,31      "rstrip": false,32      "single_word": false,33      "special": true34    }35  },36  "additional_special_tokens": [37    "<unk>",38    "<s>",39    "</s>",40    "[PAD]"41  ],42  "bos_token": "</s>",43  "clean_up_tokenization_spaces": false,44  "eos_token": "</s>",45  "legacy": null,46  "model_max_length": 1000000000000000019884624838656,47  "pad_token": "</s>",48  "padding_side": "right",49  "sp_model_kwargs": {},50  "tokenizer_class": "LlamaTokenizer",51  "unk_token": "</s>",52  "use_default_system_prompt": false53}54