Modelpublic

planhanasan/test-trainer

sourceHugging Faceapache-2.0updated 4y agoView on Hugging Face

0likes38downloads

tokenizer_config.json25 linesDownload Raw Back to root

1{2  "additional_special_tokens": [3    "<s>NOTUSED",4    "</s>NOTUSED"5  ],6  "bos_token": "<s>",7  "cls_token": "<s>",8  "eos_token": "</s>",9  "mask_token": {10    "__type": "AddedToken",11    "content": "<mask>",12    "lstrip": true,13    "normalized": true,14    "rstrip": false,15    "single_word": false16  },17  "model_max_length": 512,18  "name_or_path": "camembert-base",19  "pad_token": "<pad>",20  "sep_token": "</s>",21  "special_tokens_map_file": null,22  "tokenizer_class": "CamembertTokenizer",23  "unk_token": "<unk>"24}25