pythonstudentiam/tinyllm

1{2  "version": "1.0",3  "truncation": null,4  "padding": null,5  "added_tokens": [6    {7      "id": 0,8      "content": "<unk>",9      "single_word": false,10      "lstrip": false,11      "rstrip": false,12      "normalized": false,13      "special": true14    },15    {16      "id": 1,17      "content": "<s>",18      "single_word": false,19      "lstrip": false,20      "rstrip": false,21      "normalized": false,22      "special": true23    },24    {25      "id": 2,26      "content": "<|im_end|>",27      "single_word": false,28      "lstrip": false,29      "rstrip": false,30      "normalized": false,31      "special": true32    },33    {34      "id": 3,35      "content": "<pad>",36      "single_word": false,37      "lstrip": false,38      "rstrip": false,39      "normalized": false,40      "special": true41    }42  ],43  "normalizer": null,44  "pre_tokenizer": {45    "type": "Metaspace",46    "replacement": "▁",47    "prepend_scheme": "first",48    "split": false49  },50  "post_processor": {51    "type": "TemplateProcessing",52    "single": [53      {54        "SpecialToken": {55          "id": "<s>",56          "type_id": 057        }58      },59      {60        "Sequence": {61          "id": "A",62          "type_id": 063        }64      }65    ],66    "pair": [67      {68        "SpecialToken": {69          "id": "<s>",70          "type_id": 071        }72      },73      {74        "Sequence": {75          "id": "A",76          "type_id": 077        }78      },79      {80        "SpecialToken": {81          "id": "<s>",82          "type_id": 183        }84      },85      {86        "Sequence": {87          "id": "B",88          "type_id": 189        }90      }91    ],92    "special_tokens": {93      "<s>": {94        "id": "<s>",95        "ids": [96          197        ],98        "tokens": [99          "<s>"100        ]101      }102    }103  },104  "decoder": {105    "type": "Sequence",106    "decoders": [107      {108        "type": "Replace",109        "pattern": {110          "String": "▁"111        },112        "content": " "113      },114      {115        "type": "ByteFallback"116      },117      {118        "type": "Fuse"119      },120      {121        "type": "Strip",122        "content": " ",123        "start": 1,124        "stop": 0125      }126    ]127  },128  "model": {129    "type": "BPE",130    "dropout": null,131    "unk_token": null,132    "continuing_subword_prefix": null,133    "end_of_word_suffix": null,134    "fuse_unk": true,135    "byte_fallback": true,136    "ignore_merges": false,137    "vocab": {138      "<unk>": 0,139      "<s>": 1,140      "<|im_end|>": 2141    },142    "merges": []143  }144}