web3se/SmartBERT-v3

1{2  "add_prefix_space": false,3  "bos_token": {4    "__type": "AddedToken",5    "content": "<s>",6    "lstrip": false,7    "normalized": true,8    "rstrip": false,9    "single_word": false10  },11  "clean_up_tokenization_spaces": true,12  "cls_token": {13    "__type": "AddedToken",14    "content": "<s>",15    "lstrip": false,16    "normalized": true,17    "rstrip": false,18    "single_word": false19  },20  "eos_token": {21    "__type": "AddedToken",22    "content": "</s>",23    "lstrip": false,24    "normalized": true,25    "rstrip": false,26    "single_word": false27  },28  "errors": "replace",29  "mask_token": {30    "__type": "AddedToken",31    "content": "<mask>",32    "lstrip": true,33    "normalized": true,34    "rstrip": false,35    "single_word": false36  },37  "model_max_length": 512,38  "pad_token": {39    "__type": "AddedToken",40    "content": "<pad>",41    "lstrip": false,42    "normalized": true,43    "rstrip": false,44    "single_word": false45  },46  "sep_token": {47    "__type": "AddedToken",48    "content": "</s>",49    "lstrip": false,50    "normalized": true,51    "rstrip": false,52    "single_word": false53  },54  "tokenizer_class": "RobertaTokenizer",55  "unk_token": {56    "__type": "AddedToken",57    "content": "<unk>",58    "lstrip": false,59    "normalized": true,60    "rstrip": false,61    "single_word": false62  }63}64