sanjaymalladi/DataSense-Spec-Code

1{2  "audio_token": "<|audio|>",3  "backend": "tokenizers",4  "boa_token": "<|audio>",5  "boi_token": "<|image>",6  "bos_token": "<bos>",7  "eoa_token": "<audio|>",8  "eoc_token": "<channel|>",9  "eoi_token": "<image|>",10  "eos_token": "<eos>",11  "eot_token": "<turn|>",12  "escape_token": "<|\"|>",13  "etc_token": "<tool_call|>",14  "etd_token": "<tool|>",15  "etr_token": "<tool_response|>",16  "extra_special_tokens": [17    "<|video|>"18  ],19  "image_token": "<|image|>",20  "is_local": false,21  "mask_token": "<mask>",22  "model_max_length": 131072,23  "model_specific_special_tokens": {24    "audio_token": "<|audio|>",25    "boa_token": "<|audio>",26    "boi_token": "<|image>",27    "eoa_token": "<audio|>",28    "eoc_token": "<channel|>",29    "eoi_token": "<image|>",30    "eot_token": "<turn|>",31    "escape_token": "<|\"|>",32    "etc_token": "<tool_call|>",33    "etd_token": "<tool|>",34    "etr_token": "<tool_response|>",35    "image_token": "<|image|>",36    "soc_token": "<|channel>",37    "sot_token": "<|turn>",38    "stc_token": "<|tool_call>",39    "std_token": "<|tool>",40    "str_token": "<|tool_response>",41    "think_token": "<|think|>"42  },43  "pad_token": "<pad>",44  "padding_side": "right",45  "processor_class": "Gemma4Processor",46  "response_schema": {47    "properties": {48      "content": {49        "type": "string"50      },51      "role": {52        "const": "assistant"53      },54      "thinking": {55        "type": "string"56      },57      "tool_calls": {58        "items": {59          "properties": {60            "function": {61              "properties": {62                "arguments": {63                  "additionalProperties": {},64                  "type": "object",65                  "x-parser": "gemma4-tool-call"66                },67                "name": {68                  "type": "string"69                }70              },71              "type": "object",72              "x-regex": "call\\:(?P<name>\\w+)(?P<arguments>\\{.*\\})"73            },74            "type": {75              "const": "function"76            }77          },78          "type": "object"79        },80        "type": "array",81        "x-regex-iterator": "<\\|tool_call>(.*?)<tool_call\\|>"82      }83    },84    "type": "object",85    "x-regex": "(\\<\\|channel\\>thought\\n(?P<thinking>.*?)\\<channel\\|\\>)?(?P<tool_calls>\\<\\|tool_call\\>.*\\<tool_call\\|\\>)?(?P<content>(?:(?!\\<turn\\|\\>)(?!\\<\\|tool_response\\>).)+)?(?:\\<turn\\|\\>|\\<\\|tool_response\\>)?"86  },87  "soc_token": "<|channel>",88  "sot_token": "<|turn>",89  "stc_token": "<|tool_call>",90  "std_token": "<|tool>",91  "str_token": "<|tool_response>",92  "think_token": "<|think|>",93  "tokenizer_class": "GemmaTokenizer",94  "unk_token": "<unk>",95  "added_tokens_decoder": {96    "0": {97      "content": "<pad>",98      "single_word": false,99      "lstrip": false,100      "rstrip": false,101      "normalized": false,102      "special": true103    },104    "1": {105      "content": "<eos>",106      "single_word": false,107      "lstrip": false,108      "rstrip": false,109      "normalized": false,110      "special": true111    },112    "2": {113      "content": "<bos>",114      "single_word": false,115      "lstrip": false,116      "rstrip": false,117      "normalized": false,118      "special": true119    },120    "3": {121      "content": "<unk>",122      "single_word": false,123      "lstrip": false,124      "rstrip": false,125      "normalized": false,126      "special": true127    },128    "4": {129      "content": "<mask>",130      "single_word": false,131      "lstrip": false,132      "rstrip": false,133      "normalized": false,134      "special": true135    },136    "46": {137      "content": "<|tool>",138      "single_word": false,139      "lstrip": false,140      "rstrip": false,141      "normalized": false,142      "special": true143    },144    "47": {145      "content": "<tool|>",146      "single_word": false,147      "lstrip": false,148      "rstrip": false,149      "normalized": false,150      "special": true151    },152    "48": {153      "content": "<|tool_call>",154      "single_word": false,155      "lstrip": false,156      "rstrip": false,157      "normalized": false,158      "special": true159    },160    "49": {161      "content": "<tool_call|>",162      "single_word": false,163      "lstrip": false,164      "rstrip": false,165      "normalized": false,166      "special": true167    },168    "50": {169      "content": "<|tool_response>",170      "single_word": false,171      "lstrip": false,172      "rstrip": false,173      "normalized": false,174      "special": true175    },176    "51": {177      "content": "<tool_response|>",178      "single_word": false,179      "lstrip": false,180      "rstrip": false,181      "normalized": false,182      "special": true183    },184    "52": {185      "content": "<|\"|>",186      "single_word": false,187      "lstrip": false,188      "rstrip": false,189      "normalized": false,190      "special": true191    },192    "98": {193      "content": "<|think|>",194      "single_word": false,195      "lstrip": false,196      "rstrip": false,197      "normalized": false,198      "special": true199    },200    "100": {201      "content": "<|channel>",202      "single_word": false,203      "lstrip": false,204      "rstrip": false,205      "normalized": false,206      "special": true207    },208    "101": {209      "content": "<channel|>",210      "single_word": false,211      "lstrip": false,212      "rstrip": false,213      "normalized": false,214      "special": true215    },216    "105": {217      "content": "<|turn>",218      "single_word": false,219      "lstrip": false,220      "rstrip": false,221      "normalized": false,222      "special": true223    },224    "106": {225      "content": "<turn|>",226      "single_word": false,227      "lstrip": false,228      "rstrip": false,229      "normalized": false,230      "special": true231    },232    "255999": {233      "content": "<|image>",234      "single_word": false,235      "lstrip": false,236      "rstrip": false,237      "normalized": false,238      "special": true239    },240    "256000": {241      "content": "<|audio>",242      "single_word": false,243      "lstrip": false,244      "rstrip": false,245      "normalized": false,246      "special": true247    },248    "258880": {249      "content": "<|image|>",250      "single_word": false,251      "lstrip": false,252      "rstrip": false,253      "normalized": false,254      "special": true255    },256    "258881": {257      "content": "<|audio|>",258      "single_word": false,259      "lstrip": false,260      "rstrip": false,261      "normalized": false,262      "special": true263    },264    "258882": {265      "content": "<image|>",266      "single_word": false,267      "lstrip": false,268      "rstrip": false,269      "normalized": false,270      "special": true271    },272    "258883": {273      "content": "<audio|>",274      "single_word": false,275      "lstrip": false,276      "rstrip": false,277      "normalized": false,278      "special": true279    },280    "258884": {281      "content": "<|video|>",282      "single_word": false,283      "lstrip": false,284      "rstrip": false,285      "normalized": false,286      "special": true287    }288  }289}290