Notacape/EdgeDecoder-JAX

1{2  "add_prefix_space": false,3  "backend": "tokenizers",4  "bos_token": "<|endoftext|>",5  "clean_up_tokenization_spaces": false,6  "eos_token": "<|im_end|>",7  "errors": "replace",8  "extra_special_tokens": [9    "<|endoftext|>",10    "<|im_start|>",11    "<|im_end|>",12    "<repo_name>",13    "<reponame>",14    "<file_sep>",15    "<filename>",16    "<gh_stars>",17    "<issue_start>",18    "<issue_comment>",19    "<issue_closed>",20    "<jupyter_start>",21    "<jupyter_text>",22    "<jupyter_code>",23    "<jupyter_output>",24    "<jupyter_script>",25    "<empty_output>"26  ],27  "is_local": false,28  "local_files_only": false,29  "model_max_length": 1024,30  "pad_token": "<|endoftext|>",31  "tokenizer_class": "GPT2Tokenizer",32  "unk_token": "<|endoftext|>",33  "vocab_size": 49152,34  "add_bos_token": false,35  "add_eos_token": false36}