Modelpublic

ajibawa-2023/Python-Code-33B

sourceHugging Facecc-by-nc-nd-4.0updated 3y agoView on Hugging Face

9likes181downloads

tokenizer_config.json35 linesDownload Raw Back to root

1{2  "add_bos_token": true,3  "add_eos_token": false,4  "bos_token": {5    "__type": "AddedToken",6    "content": "<s>",7    "lstrip": false,8    "normalized": true,9    "rstrip": false,10    "single_word": false11  },12  "clean_up_tokenization_spaces": false,13  "eos_token": {14    "__type": "AddedToken",15    "content": "</s>",16    "lstrip": false,17    "normalized": true,18    "rstrip": false,19    "single_word": false20  },21  "model_max_length": 2048,22  "pad_token": null,23  "padding_side": "right",24  "sp_model_kwargs": {},25  "tokenizer_class": "LlamaTokenizer",26  "unk_token": {27    "__type": "AddedToken",28    "content": "<unk>",29    "lstrip": false,30    "normalized": true,31    "rstrip": false,32    "single_word": false33  }34}35