Modelpublic

RowRed/ComplexityRouter

sourceHugging Faceapache-2.0updated 2mo agoView on Hugging Face

0likes21downloads

tokenizer_config.json25 linesDownload Raw Back to root

1{2  "add_prefix_space": true,3  "backend": "tokenizers",4  "bos_token": "[CLS]",5  "cls_token": "[CLS]",6  "do_lower_case": false,7  "eos_token": "[SEP]",8  "extra_special_tokens": [9    "[PAD]",10    "[CLS]",11    "[SEP]"12  ],13  "is_local": false,14  "local_files_only": false,15  "mask_token": "[MASK]",16  "model_max_length": 1000000000000000019884624838656,17  "pad_token": "[PAD]",18  "sep_token": "[SEP]",19  "split_by_punct": false,20  "tokenizer_class": "DebertaV2Tokenizer",21  "unk_id": 3,22  "unk_token": "[UNK]",23  "vocab_type": "spm"24}25