dipikakhullar/olmo-code-python3-chunk-aa

1{2  "add_prefix_space": false,3  "added_tokens_decoder": {4    "100256": {5      "content": "<|extra_id_0|>",6      "lstrip": false,7      "normalized": false,8      "rstrip": false,9      "single_word": false,10      "special": false11    },12    "100257": {13      "content": "<|endoftext|>",14      "lstrip": false,15      "normalized": false,16      "rstrip": false,17      "single_word": false,18      "special": true19    },20    "100258": {21      "content": "<|fim_prefix|>",22      "lstrip": false,23      "normalized": false,24      "rstrip": false,25      "single_word": false,26      "special": true27    },28    "100259": {29      "content": "<|fim_middle|>",30      "lstrip": false,31      "normalized": false,32      "rstrip": false,33      "single_word": false,34      "special": true35    },36    "100260": {37      "content": "<|fim_suffix|>",38      "lstrip": false,39      "normalized": false,40      "rstrip": false,41      "single_word": false,42      "special": true43    },44    "100261": {45      "content": "|||PHONE_NUMBER|||",46      "lstrip": false,47      "normalized": false,48      "rstrip": false,49      "single_word": false,50      "special": false51    },52    "100262": {53      "content": "|||EMAIL_ADDRESS|||",54      "lstrip": false,55      "normalized": false,56      "rstrip": false,57      "single_word": false,58      "special": false59    },60    "100263": {61      "content": "|||IP_ADDRESS|||",62      "lstrip": false,63      "normalized": false,64      "rstrip": false,65      "single_word": false,66      "special": false67    },68    "100264": {69      "content": "<|im_start|>",70      "lstrip": false,71      "normalized": false,72      "rstrip": false,73      "single_word": false,74      "special": true75    },76    "100265": {77      "content": "<|im_end|>",78      "lstrip": false,79      "normalized": false,80      "rstrip": false,81      "single_word": false,82      "special": true83    },84    "100266": {85      "content": "<|extra_id_1|>",86      "lstrip": false,87      "normalized": false,88      "rstrip": false,89      "single_word": false,90      "special": false91    },92    "100267": {93      "content": "<|extra_id_2|>",94      "lstrip": false,95      "normalized": false,96      "rstrip": false,97      "single_word": false,98      "special": false99    },100    "100268": {101      "content": "<|extra_id_3|>",102      "lstrip": false,103      "normalized": false,104      "rstrip": false,105      "single_word": false,106      "special": false107    },108    "100269": {109      "content": "<|extra_id_4|>",110      "lstrip": false,111      "normalized": false,112      "rstrip": false,113      "single_word": false,114      "special": false115    },116    "100270": {117      "content": "<|extra_id_5|>",118      "lstrip": false,119      "normalized": false,120      "rstrip": false,121      "single_word": false,122      "special": false123    },124    "100271": {125      "content": "<|extra_id_6|>",126      "lstrip": false,127      "normalized": false,128      "rstrip": false,129      "single_word": false,130      "special": false131    },132    "100272": {133      "content": "<|extra_id_7|>",134      "lstrip": false,135      "normalized": false,136      "rstrip": false,137      "single_word": false,138      "special": false139    },140    "100273": {141      "content": "<|extra_id_8|>",142      "lstrip": false,143      "normalized": false,144      "rstrip": false,145      "single_word": false,146      "special": false147    },148    "100274": {149      "content": "<|extra_id_9|>",150      "lstrip": false,151      "normalized": false,152      "rstrip": false,153      "single_word": false,154      "special": false155    },156    "100275": {157      "content": "<|extra_id_10|>",158      "lstrip": false,159      "normalized": false,160      "rstrip": false,161      "single_word": false,162      "special": false163    },164    "100276": {165      "content": "<|endofprompt|>",166      "lstrip": false,167      "normalized": false,168      "rstrip": false,169      "single_word": false,170      "special": true171    },172    "100277": {173      "content": "<|pad|>",174      "lstrip": false,175      "normalized": false,176      "rstrip": false,177      "single_word": false,178      "special": true179    }180  },181  "bos_token": "<|endoftext|>",182  "clean_up_tokenization_spaces": false,183  "eos_token": "<|endoftext|>",184  "extra_special_tokens": {},185  "model_max_length": 1000000000000000019884624838656,186  "pad_token": "<|pad|>",187  "tokenizer_class": "GPT2Tokenizer",188  "unk_token": "<|endoftext|>"189}190