hardiksinh/master-python-coder-llm

1{2  "add_prefix_space": false,3  "backend": "tokenizers",4  "bos_token": null,5  "clean_up_tokenization_spaces": false,6  "eos_token": "<|im_end|>",7  "errors": "replace",8  "extra_special_tokens": [9    "<|im_start|>",10    "<|im_end|>",11    "<|object_ref_start|>",12    "<|object_ref_end|>",13    "<|box_start|>",14    "<|box_end|>",15    "<|quad_start|>",16    "<|quad_end|>",17    "<|vision_start|>",18    "<|vision_end|>",19    "<|vision_pad|>",20    "<|image_pad|>",21    "<|video_pad|>"22  ],23  "is_local": false,24  "model_max_length": 32768,25  "pad_token": "<|PAD_TOKEN|>",26  "padding_side": "left",27  "split_special_tokens": false,28  "tokenizer_class": "Qwen2Tokenizer",29  "unk_token": null,30  "added_tokens_decoder": {31    "151643": {32      "content": "<|endoftext|>",33      "single_word": false,34      "lstrip": false,35      "rstrip": false,36      "normalized": false,37      "special": true38    },39    "151644": {40      "content": "<|im_start|>",41      "single_word": false,42      "lstrip": false,43      "rstrip": false,44      "normalized": false,45      "special": true46    },47    "151645": {48      "content": "<|im_end|>",49      "single_word": false,50      "lstrip": false,51      "rstrip": false,52      "normalized": false,53      "special": true54    },55    "151646": {56      "content": "<|object_ref_start|>",57      "single_word": false,58      "lstrip": false,59      "rstrip": false,60      "normalized": false,61      "special": true62    },63    "151647": {64      "content": "<|object_ref_end|>",65      "single_word": false,66      "lstrip": false,67      "rstrip": false,68      "normalized": false,69      "special": true70    },71    "151648": {72      "content": "<|box_start|>",73      "single_word": false,74      "lstrip": false,75      "rstrip": false,76      "normalized": false,77      "special": true78    },79    "151649": {80      "content": "<|box_end|>",81      "single_word": false,82      "lstrip": false,83      "rstrip": false,84      "normalized": false,85      "special": true86    },87    "151650": {88      "content": "<|quad_start|>",89      "single_word": false,90      "lstrip": false,91      "rstrip": false,92      "normalized": false,93      "special": true94    },95    "151651": {96      "content": "<|quad_end|>",97      "single_word": false,98      "lstrip": false,99      "rstrip": false,100      "normalized": false,101      "special": true102    },103    "151652": {104      "content": "<|vision_start|>",105      "single_word": false,106      "lstrip": false,107      "rstrip": false,108      "normalized": false,109      "special": true110    },111    "151653": {112      "content": "<|vision_end|>",113      "single_word": false,114      "lstrip": false,115      "rstrip": false,116      "normalized": false,117      "special": true118    },119    "151654": {120      "content": "<|vision_pad|>",121      "single_word": false,122      "lstrip": false,123      "rstrip": false,124      "normalized": false,125      "special": true126    },127    "151655": {128      "content": "<|image_pad|>",129      "single_word": false,130      "lstrip": false,131      "rstrip": false,132      "normalized": false,133      "special": true134    },135    "151656": {136      "content": "<|video_pad|>",137      "single_word": false,138      "lstrip": false,139      "rstrip": false,140      "normalized": false,141      "special": true142    },143    "151657": {144      "content": "<tool_call>",145      "single_word": false,146      "lstrip": false,147      "rstrip": false,148      "normalized": false,149      "special": false150    },151    "151658": {152      "content": "</tool_call>",153      "single_word": false,154      "lstrip": false,155      "rstrip": false,156      "normalized": false,157      "special": false158    },159    "151659": {160      "content": "<|fim_prefix|>",161      "single_word": false,162      "lstrip": false,163      "rstrip": false,164      "normalized": false,165      "special": false166    },167    "151660": {168      "content": "<|fim_middle|>",169      "single_word": false,170      "lstrip": false,171      "rstrip": false,172      "normalized": false,173      "special": false174    },175    "151661": {176      "content": "<|fim_suffix|>",177      "single_word": false,178      "lstrip": false,179      "rstrip": false,180      "normalized": false,181      "special": false182    },183    "151662": {184      "content": "<|fim_pad|>",185      "single_word": false,186      "lstrip": false,187      "rstrip": false,188      "normalized": false,189      "special": false190    },191    "151663": {192      "content": "<|repo_name|>",193      "single_word": false,194      "lstrip": false,195      "rstrip": false,196      "normalized": false,197      "special": false198    },199    "151664": {200      "content": "<|file_sep|>",201      "single_word": false,202      "lstrip": false,203      "rstrip": false,204      "normalized": false,205      "special": false206    },207    "151665": {208      "content": "<|PAD_TOKEN|>",209      "single_word": false,210      "lstrip": false,211      "rstrip": false,212      "normalized": false,213      "special": true214    }215  }216}217