aframson/RDPDLM

1{2  "add_bos_token": false,3  "add_eos_token": false,4  "auto_map": {5    "AutoTokenizer": [6      "tokenizeConfig.OBTokenzier",7      null8    ]9  },10  "bos_token": {11    "__type": "AddedToken",12    "content": "<s>",13    "lstrip": false,14    "normalized": true,15    "rstrip": false,16    "single_word": true17  },18  "clean_up_tokenization_spaces": false,19  "eos_token": {20    "__type": "AddedToken",21    "content": "</s>",22    "lstrip": false,23    "normalized": true,24    "rstrip": false,25    "single_word": true26  },27  "model_max_length": 4096,28  "pad_token": {29    "__type": "AddedToken",30    "content": "<unk>",31    "lstrip": false,32    "normalized": true,33    "rstrip": false,34    "single_word": true35  },36  "sp_model_kwargs": {},37  "tokenizer_class": "OBTokenzier",38  "unk_token": {39    "__type": "AddedToken",40    "content": "<unk>",41    "lstrip": false,42    "normalized": true,43    "rstrip": false,44    "single_word": true45  }46}