fhamborg/newsframes-aff-bin

1{2  "bos_token": {3    "__type": "AddedToken",4    "content": "<s>",5    "lstrip": false,6    "normalized": true,7    "rstrip": false,8    "single_word": false9  },10  "clean_up_tokenization_spaces": true,11  "cls_token": {12    "__type": "AddedToken",13    "content": "<s>",14    "lstrip": false,15    "normalized": true,16    "rstrip": false,17    "single_word": false18  },19  "do_basic_tokenize": true,20  "do_lower_case": true,21  "eos_token": {22    "__type": "AddedToken",23    "content": "</s>",24    "lstrip": false,25    "normalized": true,26    "rstrip": false,27    "single_word": false28  },29  "mask_token": {30    "__type": "AddedToken",31    "content": "<mask>",32    "lstrip": true,33    "normalized": true,34    "rstrip": false,35    "single_word": false36  },37  "max_length": 512,38  "model_max_length": 512,39  "never_split": null,40  "pad_to_multiple_of": null,41  "pad_token": {42    "__type": "AddedToken",43    "content": "<pad>",44    "lstrip": false,45    "normalized": true,46    "rstrip": false,47    "single_word": false48  },49  "pad_token_type_id": 0,50  "padding_side": "right",51  "sep_token": {52    "__type": "AddedToken",53    "content": "</s>",54    "lstrip": false,55    "normalized": true,56    "rstrip": false,57    "single_word": false58  },59  "stride": 0,60  "strip_accents": null,61  "tokenize_chinese_chars": true,62  "tokenizer_class": "MPNetTokenizer",63  "truncation_side": "right",64  "truncation_strategy": "longest_first",65  "unk_token": {66    "__type": "AddedToken",67    "content": "[UNK]",68    "lstrip": false,69    "normalized": true,70    "rstrip": false,71    "single_word": false72  }73}74