impossibleexchange/dipft

1{2  "add_bos_token": true,3  "add_prefix_space": false,4  "added_tokens_decoder": {5    "100256": {6      "content": "<|reg_extra|>",7      "lstrip": false,8      "normalized": false,9      "rstrip": false,10      "single_word": false,11      "special": true12    },13    "100257": {14      "content": "<|endoftext|>",15      "lstrip": false,16      "normalized": false,17      "rstrip": false,18      "single_word": false,19      "special": true20    },21    "100258": {22      "content": "<|fim_prefix|>",23      "lstrip": false,24      "normalized": false,25      "rstrip": false,26      "single_word": false,27      "special": true28    },29    "100259": {30      "content": "<|fim_middle|>",31      "lstrip": false,32      "normalized": false,33      "rstrip": false,34      "single_word": false,35      "special": true36    },37    "100260": {38      "content": "<|fim_suffix|>",39      "lstrip": false,40      "normalized": false,41      "rstrip": false,42      "single_word": false,43      "special": true44    },45    "100261": {46      "content": "<|fim_pad|>",47      "lstrip": false,48      "normalized": false,49      "rstrip": false,50      "single_word": false,51      "special": true52    },53    "100262": {54      "content": "<gh_stars>",55      "lstrip": false,56      "normalized": false,57      "rstrip": false,58      "single_word": false,59      "special": true60    },61    "100263": {62      "content": "<filename>",63      "lstrip": false,64      "normalized": false,65      "rstrip": false,66      "single_word": false,67      "special": true68    },69    "100264": {70      "content": "<issue_start>",71      "lstrip": false,72      "normalized": false,73      "rstrip": false,74      "single_word": false,75      "special": true76    },77    "100265": {78      "content": "<issue_comment>",79      "lstrip": false,80      "normalized": false,81      "rstrip": false,82      "single_word": false,83      "special": true84    },85    "100266": {86      "content": "<issue_closed>",87      "lstrip": false,88      "normalized": false,89      "rstrip": false,90      "single_word": false,91      "special": true92    },93    "100267": {94      "content": "<jupyter_start>",95      "lstrip": false,96      "normalized": false,97      "rstrip": false,98      "single_word": false,99      "special": true100    },101    "100268": {102      "content": "<jupyter_text>",103      "lstrip": false,104      "normalized": false,105      "rstrip": false,106      "single_word": false,107      "special": true108    },109    "100269": {110      "content": "<jupyter_code>",111      "lstrip": false,112      "normalized": false,113      "rstrip": false,114      "single_word": false,115      "special": true116    },117    "100270": {118      "content": "<jupyter_output>",119      "lstrip": false,120      "normalized": false,121      "rstrip": false,122      "single_word": false,123      "special": true124    },125    "100271": {126      "content": "<empty_output>",127      "lstrip": false,128      "normalized": false,129      "rstrip": false,130      "single_word": false,131      "special": true132    },133    "100272": {134      "content": "<commit_before>",135      "lstrip": false,136      "normalized": false,137      "rstrip": false,138      "single_word": false,139      "special": true140    },141    "100273": {142      "content": "<commit_msg>",143      "lstrip": false,144      "normalized": false,145      "rstrip": false,146      "single_word": false,147      "special": true148    },149    "100274": {150      "content": "<commit_after>",151      "lstrip": false,152      "normalized": false,153      "rstrip": false,154      "single_word": false,155      "special": true156    },157    "100275": {158      "content": "<reponame>",159      "lstrip": false,160      "normalized": false,161      "rstrip": false,162      "single_word": false,163      "special": true164    },165    "100276": {166      "content": "<|endofprompt|>",167      "lstrip": false,168      "normalized": false,169      "rstrip": false,170      "single_word": false,171      "special": true172    },173    "100277": {174      "content": "<|im_start|>",175      "lstrip": false,176      "normalized": false,177      "rstrip": false,178      "single_word": false,179      "special": true180    },181    "100278": {182      "content": "<|im_end|>",183      "lstrip": false,184      "normalized": false,185      "rstrip": false,186      "single_word": false,187      "special": true188    },189    "100279": {190      "content": "<|pause|>",191      "lstrip": false,192      "normalized": false,193      "rstrip": false,194      "single_word": false,195      "special": true196    },197    "100280": {198      "content": "<|reg0|>",199      "lstrip": false,200      "normalized": false,201      "rstrip": false,202      "single_word": false,203      "special": true204    },205    "100281": {206      "content": "<|reg1|>",207      "lstrip": false,208      "normalized": false,209      "rstrip": false,210      "single_word": false,211      "special": true212    },213    "100282": {214      "content": "<|reg2|>",215      "lstrip": false,216      "normalized": false,217      "rstrip": false,218      "single_word": false,219      "special": true220    },221    "100283": {222      "content": "<|reg3|>",223      "lstrip": false,224      "normalized": false,225      "rstrip": false,226      "single_word": false,227      "special": true228    },229    "100284": {230      "content": "<|reg4|>",231      "lstrip": false,232      "normalized": false,233      "rstrip": false,234      "single_word": false,235      "special": true236    },237    "100285": {238      "content": "<|reg5|>",239      "lstrip": false,240      "normalized": false,241      "rstrip": false,242      "single_word": false,243      "special": true244    },245    "100286": {246      "content": "<|reg6|>",247      "lstrip": false,248      "normalized": false,249      "rstrip": false,250      "single_word": false,251      "special": true252    },253    "100287": {254      "content": "<|reg7|>",255      "lstrip": false,256      "normalized": false,257      "rstrip": false,258      "single_word": false,259      "special": true260    },261    "100288": {262      "content": "<|extra0|>",263      "lstrip": false,264      "normalized": false,265      "rstrip": false,266      "single_word": false,267      "special": true268    }269  },270  "additional_special_tokens": [271    "<|reg_extra|>",272    "<|endoftext|>",273    "<|fim_prefix|>",274    "<|fim_middle|>",275    "<|fim_suffix|>",276    "<|fim_pad|>",277    "<gh_stars>",278    "<filename>",279    "<issue_start>",280    "<issue_comment>",281    "<issue_closed>",282    "<jupyter_start>",283    "<jupyter_text>",284    "<jupyter_code>",285    "<jupyter_output>",286    "<empty_output>",287    "<commit_before>",288    "<commit_msg>",289    "<commit_after>",290    "<reponame>",291    "<|endofprompt|>",292    "<|im_start|>",293    "<|im_end|>",294    "<|pause|>",295    "<|reg0|>",296    "<|reg1|>",297    "<|reg2|>",298    "<|reg3|>",299    "<|reg4|>",300    "<|reg5|>",301    "<|reg6|>",302    "<|reg7|>",303    "<|extra0|>"304  ],305  "bos_token": "<|endoftext|>",306  "clean_up_tokenization_spaces": true,307  "eos_token": "<|endoftext|>",308  "max_length": 3896,309  "model_max_length": 1000000000000000019884624838656,310  "pad_to_multiple_of": null,311  "pad_token": "<|endoftext|>",312  "pad_token_type_id": 0,313  "padding_side": "left",314  "stride": 0,315  "tokenizer_class": "GPT2Tokenizer",316  "truncation_side": "right",317  "truncation_strategy": "longest_first",318  "unk_token": "<|endoftext|>"319}320