bigcode/tokenizer-the-stack-march-sample-v2

1{2  "add_prefix_space": false,3  "additional_special_tokens": [4    "<|endoftext|>",5    "<fim-prefix>",6    "<fim-middle>",7    "<fim-suffix>",8    "<fim-pad>",9    "<filename>",10    "<gh-stars>",11    "<issue_start>",12    "<issue_comment>",13    "<issue_closed>",14    "<jupyter_start>",15    "<jupyter_text>",16    "<jupyter_code>",17    "<jupyter_output>",18    "<empty_output>",19    "<commit_before>",20    "<commit_msg>",21    "<commit_after>"22  ],23  "bos_token": "<|endoftext|>",24  "eos_token": "<|endoftext|>",25  "model_max_length": 1000000000000000019884624838656,26  "tokenizer_class": "GPT2Tokenizer",27  "unk_token": "<|endoftext|>",28  "vocab_size": 4915229}30