bigcode/tokenizer-the-stack-march-sample-v2

1{2  "additional_special_tokens": [3    "<|endoftext|>",4    "<fim-prefix>",5    "<fim-middle>",6    "<fim-suffix>",7    "<fim-pad>",8    "<filename>",9    "<gh-stars>",10    "<issue_start>",11    "<issue_comment>",12    "<issue_closed>",13    "<jupyter_start>",14    "<jupyter_text>",15    "<jupyter_code>",16    "<jupyter_output>",17    "<empty_output>",18    "<commit_before>",19    "<commit_msg>",20    "<commit_after>"21  ],22  "bos_token": "<|endoftext|>",23  "eos_token": "<|endoftext|>",24  "unk_token": "<|endoftext|>"25}26