emplitude/rubycon2

1{2  "add_prefix_space": false,3  "added_tokens_decoder": {4    "100256": {5      "content": "<|reg_extra|>",6      "lstrip": false,7      "normalized": false,8      "rstrip": false,9      "single_word": false,10      "special": true11    },12    "100257": {13      "content": "<|endoftext|>",14      "lstrip": false,15      "normalized": false,16      "rstrip": false,17      "single_word": false,18      "special": true19    },20    "100258": {21      "content": "<|fim_prefix|>",22      "lstrip": false,23      "normalized": false,24      "rstrip": false,25      "single_word": false,26      "special": true27    },28    "100259": {29      "content": "<|fim_middle|>",30      "lstrip": false,31      "normalized": false,32      "rstrip": false,33      "single_word": false,34      "special": true35    },36    "100260": {37      "content": "<|fim_suffix|>",38      "lstrip": false,39      "normalized": false,40      "rstrip": false,41      "single_word": false,42      "special": true43    },44    "100261": {45      "content": "<|fim_pad|>",46      "lstrip": false,47      "normalized": false,48      "rstrip": false,49      "single_word": false,50      "special": true51    },52    "100262": {53      "content": "<gh_stars>",54      "lstrip": false,55      "normalized": false,56      "rstrip": false,57      "single_word": false,58      "special": true59    },60    "100263": {61      "content": "<filename>",62      "lstrip": false,63      "normalized": false,64      "rstrip": false,65      "single_word": false,66      "special": true67    },68    "100264": {69      "content": "<issue_start>",70      "lstrip": false,71      "normalized": false,72      "rstrip": false,73      "single_word": false,74      "special": true75    },76    "100265": {77      "content": "<issue_comment>",78      "lstrip": false,79      "normalized": false,80      "rstrip": false,81      "single_word": false,82      "special": true83    },84    "100266": {85      "content": "<issue_closed>",86      "lstrip": false,87      "normalized": false,88      "rstrip": false,89      "single_word": false,90      "special": true91    },92    "100267": {93      "content": "<jupyter_start>",94      "lstrip": false,95      "normalized": false,96      "rstrip": false,97      "single_word": false,98      "special": true99    },100    "100268": {101      "content": "<jupyter_text>",102      "lstrip": false,103      "normalized": false,104      "rstrip": false,105      "single_word": false,106      "special": true107    },108    "100269": {109      "content": "<jupyter_code>",110      "lstrip": false,111      "normalized": false,112      "rstrip": false,113      "single_word": false,114      "special": true115    },116    "100270": {117      "content": "<jupyter_output>",118      "lstrip": false,119      "normalized": false,120      "rstrip": false,121      "single_word": false,122      "special": true123    },124    "100271": {125      "content": "<empty_output>",126      "lstrip": false,127      "normalized": false,128      "rstrip": false,129      "single_word": false,130      "special": true131    },132    "100272": {133      "content": "<commit_before>",134      "lstrip": false,135      "normalized": false,136      "rstrip": false,137      "single_word": false,138      "special": true139    },140    "100273": {141      "content": "<commit_msg>",142      "lstrip": false,143      "normalized": false,144      "rstrip": false,145      "single_word": false,146      "special": true147    },148    "100274": {149      "content": "<commit_after>",150      "lstrip": false,151      "normalized": false,152      "rstrip": false,153      "single_word": false,154      "special": true155    },156    "100275": {157      "content": "<reponame>",158      "lstrip": false,159      "normalized": false,160      "rstrip": false,161      "single_word": false,162      "special": true163    },164    "100276": {165      "content": "<|endofprompt|>",166      "lstrip": false,167      "normalized": false,168      "rstrip": false,169      "single_word": false,170      "special": true171    },172    "100277": {173      "content": "<|im_start|>",174      "lstrip": false,175      "normalized": false,176      "rstrip": false,177      "single_word": false,178      "special": true179    },180    "100278": {181      "content": "<|im_end|>",182      "lstrip": false,183      "normalized": false,184      "rstrip": false,185      "single_word": false,186      "special": true187    },188    "100279": {189      "content": "<|pause|>",190      "lstrip": false,191      "normalized": false,192      "rstrip": false,193      "single_word": false,194      "special": true195    },196    "100280": {197      "content": "<|reg0|>",198      "lstrip": false,199      "normalized": false,200      "rstrip": false,201      "single_word": false,202      "special": true203    },204    "100281": {205      "content": "<|reg1|>",206      "lstrip": false,207      "normalized": false,208      "rstrip": false,209      "single_word": false,210      "special": true211    },212    "100282": {213      "content": "<|reg2|>",214      "lstrip": false,215      "normalized": false,216      "rstrip": false,217      "single_word": false,218      "special": true219    },220    "100283": {221      "content": "<|reg3|>",222      "lstrip": false,223      "normalized": false,224      "rstrip": false,225      "single_word": false,226      "special": true227    },228    "100284": {229      "content": "<|reg4|>",230      "lstrip": false,231      "normalized": false,232      "rstrip": false,233      "single_word": false,234      "special": true235    },236    "100285": {237      "content": "<|reg5|>",238      "lstrip": false,239      "normalized": false,240      "rstrip": false,241      "single_word": false,242      "special": true243    },244    "100286": {245      "content": "<|reg6|>",246      "lstrip": false,247      "normalized": false,248      "rstrip": false,249      "single_word": false,250      "special": true251    },252    "100287": {253      "content": "<|reg7|>",254      "lstrip": false,255      "normalized": false,256      "rstrip": false,257      "single_word": false,258      "special": true259    },260    "100288": {261      "content": "<|extra0|>",262      "lstrip": false,263      "normalized": false,264      "rstrip": false,265      "single_word": false,266      "special": true267    }268  },269  "additional_special_tokens": [270    "<|reg_extra|>",271    "<|endoftext|>",272    "<|fim_prefix|>",273    "<|fim_middle|>",274    "<|fim_suffix|>",275    "<|fim_pad|>",276    "<gh_stars>",277    "<filename>",278    "<issue_start>",279    "<issue_comment>",280    "<issue_closed>",281    "<jupyter_start>",282    "<jupyter_text>",283    "<jupyter_code>",284    "<jupyter_output>",285    "<empty_output>",286    "<commit_before>",287    "<commit_msg>",288    "<commit_after>",289    "<reponame>",290    "<|endofprompt|>",291    "<|im_start|>",292    "<|im_end|>",293    "<|pause|>",294    "<|reg0|>",295    "<|reg1|>",296    "<|reg2|>",297    "<|reg3|>",298    "<|reg4|>",299    "<|reg5|>",300    "<|reg6|>",301    "<|reg7|>",302    "<|extra0|>"303  ],304  "bos_token": "<|im_start|>",305  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",306  "clean_up_tokenization_spaces": true,307  "eos_token": "<|im_end|>",308  "model_max_length": 2048,309  "pad_token": "<|endoftext|>",310  "padding_side": "left",311  "tokenizer_class": "GPT2Tokenizer",312  "unk_token": "<|endoftext|>"313}314