Modelpublic

YOZ1/RadAI4

sourceHugging Faceupdated 3y agoView on Hugging Face

0likes4downloads

tokenizer_config.json32 linesDownload Raw Back to root

1{2  "bos_token": {3    "__type": "AddedToken",4    "content": "<s>",5    "lstrip": false,6    "normalized": true,7    "rstrip": false,8    "single_word": false9  },10  "clean_up_tokenization_spaces": false,11  "eos_token": {12    "__type": "AddedToken",13    "content": "</s>",14    "lstrip": false,15    "normalized": true,16    "rstrip": false,17    "single_word": false18  },19  "model_max_length": 1000000000000000019884624838656,20  "pad_token": null,21  "sp_model_kwargs": {},22  "tokenizer_class": "LlamaTokenizer",23  "unk_token": {24    "__type": "AddedToken",25    "content": "<unk>",26    "lstrip": false,27    "normalized": true,28    "rstrip": false,29    "single_word": false30  }31}32