CrossBaseArithmetic/crossbase-tokenizer

1{2  "version": "1.0",3  "truncation": null,4  "padding": null,5  "added_tokens": [6    {7      "id": 0,8      "content": "<pad>",9      "single_word": false,10      "lstrip": false,11      "rstrip": false,12      "normalized": false,13      "special": true14    },15    {16      "id": 1,17      "content": "<bos>",18      "single_word": false,19      "lstrip": false,20      "rstrip": false,21      "normalized": false,22      "special": true23    },24    {25      "id": 2,26      "content": "<eos>",27      "single_word": false,28      "lstrip": false,29      "rstrip": false,30      "normalized": false,31      "special": true32    },33    {34      "id": 3,35      "content": "<unk>",36      "single_word": false,37      "lstrip": false,38      "rstrip": false,39      "normalized": false,40      "special": true41    }42  ],43  "normalizer": null,44  "pre_tokenizer": {45    "type": "Split",46    "pattern": {47      "Regex": "[\\s\\S]"48    },49    "behavior": "Isolated",50    "invert": false51  },52  "post_processor": {53    "type": "TemplateProcessing",54    "single": [55      {56        "Sequence": {57          "id": "A",58          "type_id": 059        }60      }61    ],62    "pair": [63      {64        "Sequence": {65          "id": "A",66          "type_id": 067        }68      },69      {70        "Sequence": {71          "id": "B",72          "type_id": 173        }74      }75    ],76    "special_tokens": {}77  },78  "decoder": {79    "type": "Fuse"80  },81  "model": {82    "type": "WordLevel",83    "vocab": {84      "<pad>": 0,85      "<bos>": 1,86      "<eos>": 2,87      "<unk>": 3,88      "\n": 4,89      "+": 5,90      "0": 6,91      "1": 7,92      "2": 8,93      "3": 9,94      "4": 10,95      "5": 11,96      "6": 12,97      "7": 13,98      "8": 14,99      "9": 15,100      "=": 16,101      "A": 17,102      "B": 18,103      "C": 19,104      "D": 20,105      "E": 21,106      "F": 22107    },108    "unk_token": "<unk>"109  }110}