jree423/diffsketcher

1{2  "version": "1.0",3  "truncation": null,4  "padding": null,5  "added_tokens": [6    {7      "id": 49406,8      "content": "<|startoftext|>",9      "single_word": false,10      "lstrip": false,11      "rstrip": false,12      "normalized": true,13      "special": true14    },15    {16      "id": 49407,17      "content": "<|endoftext|>",18      "single_word": false,19      "lstrip": false,20      "rstrip": false,21      "normalized": true,22      "special": true23    }24  ],25  "normalizer": {26    "type": "Lowercase"27  },28  "pre_tokenizer": {29    "type": "ByteLevel",30    "add_prefix_space": false,31    "trim_offsets": true,32    "use_regex": true33  },34  "post_processor": {35    "type": "ByteLevel",36    "add_prefix_space": false,37    "trim_offsets": true38  },39  "decoder": {40    "type": "ByteLevel",41    "add_prefix_space": false,42    "trim_offsets": true,43    "use_regex": true44  },45  "model": {46    "type": "BPE",47    "dropout": null,48    "unk_token": null,49    "continuing_subword_prefix": null,50    "end_of_word_suffix": null,51    "fuse_unk": false,52    "vocab": {},53    "merges": []54  }55}