CCBD-Interns/Mechinterp-revised-tokenizer

1{2  "version": "1.0",3  "truncation": null,4  "padding": null,5  "added_tokens": [6    {7      "id": 0,8      "content": "<pad>",9      "single_word": false,10      "lstrip": false,11      "rstrip": false,12      "normalized": false,13      "special": true14    },15    {16      "id": 1,17      "content": "<bos>",18      "single_word": false,19      "lstrip": false,20      "rstrip": false,21      "normalized": false,22      "special": true23    },24    {25      "id": 2,26      "content": "<eos>",27      "single_word": false,28      "lstrip": false,29      "rstrip": false,30      "normalized": false,31      "special": true32    },33    {34      "id": 3,35      "content": "<unk>",36      "single_word": false,37      "lstrip": false,38      "rstrip": false,39      "normalized": false,40      "special": true41    }42  ],43  "normalizer": null,44  "pre_tokenizer": {45    "type": "Split",46    "pattern": {47      "Regex": "[\\s\\S]"48    },49    "behavior": "Isolated",50    "invert": false51  },52  "post_processor": {53    "type": "TemplateProcessing",54    "single": [55      {56        "Sequence": {57          "id": "A",58          "type_id": 059        }60      }61    ],62    "pair": [63      {64        "Sequence": {65          "id": "A",66          "type_id": 067        }68      },69      {70        "Sequence": {71          "id": "B",72          "type_id": 173        }74      }75    ],76    "special_tokens": {}77  },78  "decoder": {79    "type": "Fuse"80  },81  "model": {82    "type": "WordLevel",83    "vocab": {84      "<pad>": 0,85      "<bos>": 1,86      "<eos>": 2,87      "<unk>": 3,88      "\n": 4,89      "+": 5,90      "0": 6,91      "1": 7,92      "2": 8,93      "3": 9,94      "4": 10,95      "5": 11,96      "6": 12,97      "7": 13,98      "8": 14,99      "9": 15,100      "=": 16,101      "०": 17,102      "१": 18,103      "२": 19,104      "३": 20,105      "४": 21,106      "५": 22,107      "६": 23,108      "७": 24,109      "८": 25,110      "९": 26,111      "一": 27,112      "七": 28,113      "三": 29,114      "九": 30,115      "二": 31,116      "五": 32,117      "八": 33,118      "六": 34,119      "十": 35,120      "千": 36,121      "四": 37,122      "百": 38,123      "零": 39124    },125    "unk_token": "<unk>"126  }127}