Modelpublic

lamahugface/sqlcoder-stackexchange

sourceHugging Faceupdated 5mo agoView on Hugging Face

0likes5downloads

tokenizer_config.json18 linesDownload Raw Back to root

1{
2  "backend": "tokenizers",
3  "bos_token": "<s>",
4  "clean_up_tokenization_spaces": false,
5  "eos_token": "</s>",
6  "extra_special_tokens": [],
7  "is_local": false,
8  "legacy": true,
9  "local_files_only": false,
10  "model_max_length": 1000000000000000019884624838656,
11  "pad_token": "</s>",
12  "sp_model_kwargs": {},
13  "spaces_between_special_tokens": false,
14  "tokenizer_class": "TokenizersBackend",
15  "unk_token": "<unk>",
16  "use_default_system_prompt": true
17}
18