VictorEigen/docstring_codet5_20232731_1927

1{2  "add_prefix_space": false,3  "bos_token": {4    "__type": "AddedToken",5    "content": "<s>",6    "lstrip": false,7    "normalized": true,8    "rstrip": false,9    "single_word": false10  },11  "clean_up_tokenization_spaces": true,12  "cls_token": {13    "__type": "AddedToken",14    "content": "<s>",15    "lstrip": false,16    "normalized": true,17    "rstrip": false,18    "single_word": false19  },20  "eos_token": {21    "__type": "AddedToken",22    "content": "</s>",23    "lstrip": false,24    "normalized": true,25    "rstrip": false,26    "single_word": false27  },28  "errors": "replace",29  "mask_token": {30    "__type": "AddedToken",31    "content": "<mask>",32    "lstrip": true,33    "normalized": true,34    "rstrip": false,35    "single_word": false36  },37  "model_max_length": 512,38  "pad_token": {39    "__type": "AddedToken",40    "content": "<pad>",41    "lstrip": false,42    "normalized": true,43    "rstrip": false,44    "single_word": false45  },46  "return_tensors": "tf",47  "sep_token": {48    "__type": "AddedToken",49    "content": "</s>",50    "lstrip": false,51    "normalized": true,52    "rstrip": false,53    "single_word": false54  },55  "tokenizer_class": "RobertaTokenizer",56  "trim_offsets": true,57  "unk_token": {58    "__type": "AddedToken",59    "content": "<unk>",60    "lstrip": false,61    "normalized": true,62    "rstrip": false,63    "single_word": false64  }65}66