Modelpublic

bhandariKaran/postsathi

sourceHugging Faceupdated 4mo agoView on Hugging Face

0likes5downloads

tokenizer_config.json15 linesDownload Raw Back to root

1{
2  "add_prefix_space": false,
3  "backend": "tokenizers",
4  "bos_token": "<|endoftext|>",
5  "clean_up_tokenization_spaces": true,
6  "eos_token": "<|endoftext|>",
7  "is_local": false,
8  "local_files_only": true,
9  "model_max_length": 2048,
10  "pad_token": "<|endoftext|>",
11  "return_token_type_ids": false,
12  "tokenizer_class": "CodeGenTokenizer",
13  "unk_token": "<|endoftext|>"
14}
15