CoolFace
Modelpublic

nz/RITA_l

sourceHugging Faceupdated 4y agoView on Hugging Face
0likes33downloads
tokenizer.json328 linesDownload Raw Back to root
1{2  "version": "1.0",3  "truncation": null,4  "padding": null,5  "added_tokens": [6    {7      "id": 1,8      "content": "<PAD>",9      "single_word": false,10      "lstrip": false,11      "rstrip": false,12      "normalized": false,13      "special": true14    },15    {16      "id": 2,17      "content": "<EOS>",18      "single_word": false,19      "lstrip": false,20      "rstrip": false,21      "normalized": false,22      "special": true23    },24    {25      "id": 3,26      "content": "L",27      "single_word": false,28      "lstrip": false,29      "rstrip": false,30      "normalized": true,31      "special": false32    },33    {34      "id": 4,35      "content": "A",36      "single_word": false,37      "lstrip": false,38      "rstrip": false,39      "normalized": true,40      "special": false41    },42    {43      "id": 5,44      "content": "G",45      "single_word": false,46      "lstrip": false,47      "rstrip": false,48      "normalized": true,49      "special": false50    },51    {52      "id": 6,53      "content": "V",54      "single_word": false,55      "lstrip": false,56      "rstrip": false,57      "normalized": true,58      "special": false59    },60    {61      "id": 7,62      "content": "E",63      "single_word": false,64      "lstrip": false,65      "rstrip": false,66      "normalized": true,67      "special": false68    },69    {70      "id": 8,71      "content": "S",72      "single_word": false,73      "lstrip": false,74      "rstrip": false,75      "normalized": true,76      "special": false77    },78    {79      "id": 9,80      "content": "I",81      "single_word": false,82      "lstrip": false,83      "rstrip": false,84      "normalized": true,85      "special": false86    },87    {88      "id": 10,89      "content": "K",90      "single_word": false,91      "lstrip": false,92      "rstrip": false,93      "normalized": true,94      "special": false95    },96    {97      "id": 11,98      "content": "R",99      "single_word": false,100      "lstrip": false,101      "rstrip": false,102      "normalized": true,103      "special": false104    },105    {106      "id": 12,107      "content": "D",108      "single_word": false,109      "lstrip": false,110      "rstrip": false,111      "normalized": true,112      "special": false113    },114    {115      "id": 13,116      "content": "T",117      "single_word": false,118      "lstrip": false,119      "rstrip": false,120      "normalized": true,121      "special": false122    },123    {124      "id": 14,125      "content": "P",126      "single_word": false,127      "lstrip": false,128      "rstrip": false,129      "normalized": true,130      "special": false131    },132    {133      "id": 15,134      "content": "N",135      "single_word": false,136      "lstrip": false,137      "rstrip": false,138      "normalized": true,139      "special": false140    },141    {142      "id": 16,143      "content": "Q",144      "single_word": false,145      "lstrip": false,146      "rstrip": false,147      "normalized": true,148      "special": false149    },150    {151      "id": 17,152      "content": "F",153      "single_word": false,154      "lstrip": false,155      "rstrip": false,156      "normalized": true,157      "special": false158    },159    {160      "id": 18,161      "content": "Y",162      "single_word": false,163      "lstrip": false,164      "rstrip": false,165      "normalized": true,166      "special": false167    },168    {169      "id": 19,170      "content": "M",171      "single_word": false,172      "lstrip": false,173      "rstrip": false,174      "normalized": true,175      "special": false176    },177    {178      "id": 20,179      "content": "H",180      "single_word": false,181      "lstrip": false,182      "rstrip": false,183      "normalized": true,184      "special": false185    },186    {187      "id": 21,188      "content": "C",189      "single_word": false,190      "lstrip": false,191      "rstrip": false,192      "normalized": true,193      "special": false194    },195    {196      "id": 22,197      "content": "W",198      "single_word": false,199      "lstrip": false,200      "rstrip": false,201      "normalized": true,202      "special": false203    },204    {205      "id": 23,206      "content": "B",207      "single_word": false,208      "lstrip": false,209      "rstrip": false,210      "normalized": true,211      "special": false212    },213    {214      "id": 24,215      "content": "O",216      "single_word": false,217      "lstrip": false,218      "rstrip": false,219      "normalized": true,220      "special": false221    },222    {223      "id": 25,224      "content": "U",225      "single_word": false,226      "lstrip": false,227      "rstrip": false,228      "normalized": true,229      "special": false230    },231    {232      "id": 26,233      "content": "Z",234      "single_word": false,235      "lstrip": false,236      "rstrip": false,237      "normalized": true,238      "special": false239    }240  ],241  "normalizer": {242    "type": "Sequence",243    "normalizers": [244      {245        "type": "Replace",246        "pattern": {247          "String": "X"248        },249        "content": "L"250      },251      {252        "type": "Replace",253        "pattern": {254          "String": "B"255        },256        "content": "D"257      },258      {259        "type": "Replace",260        "pattern": {261          "String": "Z"262        },263        "content": "E"264      },265      {266        "type": "Replace",267        "pattern": {268          "String": "J"269        },270        "content": "L"271      }272    ]273  },274  "pre_tokenizer": null,275  "post_processor": {276    "type": "TemplateProcessing",277    "single": [278      {279        "Sequence": {280          "id": "A",281          "type_id": 0282        }283      },284      {285        "SpecialToken": {286          "id": "<EOS>",287          "type_id": 0288        }289      }290    ],291    "pair": [292      {293        "Sequence": {294          "id": "A",295          "type_id": 0296        }297      },298      {299        "Sequence": {300          "id": "B",301          "type_id": 1302        }303      }304    ],305    "special_tokens": {306      "<EOS>": {307        "id": "<EOS>",308        "ids": [309          2310        ],311        "tokens": [312          "<EOS>"313        ]314      }315    }316  },317  "decoder": null,318  "model": {319    "type": "Unigram",320    "unk_id": 0,321    "vocab": [322      [323        "<unk>",324        0.0325      ]326    ]327  }328}