Felipe97/llama-cpp-compiled
01.1k
1from __future__ import annotations2 3import json4 5from typing import Any, Callable, Iterable, TYPE_CHECKING6 7if TYPE_CHECKING:8 from torch import Tensor9 10from .base import MmprojModel, ModelBase, gguf, logger11 12from .qwen import Qwen3Model, Qwen3MoeModel13from .qwenvl import Qwen25AudioModel14 15 16@ModelBase.register("Qwen3VLForConditionalGeneration", "Qwen3VLMoeForConditionalGeneration", "Qwen3_5ForConditionalGeneration", "Qwen3_5MoeForConditionalGeneration")17@ModelBase.example("Qwen/Qwen3-VL-4B-Instruct", "Qwen/Qwen3-VL-30B-A3B-Instruct", "Qwen/Qwen3.5-9B", "Qwen/Qwen3.5-35B-A3B")18class Qwen3VLVisionModel(MmprojModel):19 def __init__(self, *args, **kwargs):20 super().__init__(*args, **kwargs)21 if self.hparams_vision is None:22 logger.info("No vision config found, skipping vision tensor processing")23 return24 25 # Compute image_size if not present26 if "image_size" not in self.hparams_vision:27 # For Qwen3VL/Qwen3VLMoe, compute from num_position_embeddings28 num_pos = self.hparams_vision.get("num_position_embeddings", 2304)29 patch_size = self.hparams_vision.get("patch_size", 16)30 # num_position_embeddings = (image_size / patch_size) ** 231 # So image_size = sqrt(num_position_embeddings) * patch_size32 image_size = int(num_pos**0.5 * patch_size)33 self.hparams_vision["image_size"] = image_size34 35 # Rename config values for compatibility36 self.hparams_vision["num_attention_heads"] = self.hparams_vision.get("num_heads")37 self.hparams_vision["num_hidden_layers"] = self.hparams_vision.get("depth")38 39 self.is_deepstack_layers = [False] * int(self.hparams_vision["num_hidden_layers"] or 0)40 for idx in self.hparams_vision.get("deepstack_visual_indexes", []):41 self.is_deepstack_layers[idx] = True42 43 def set_gguf_parameters(self):44 super().set_gguf_parameters()45 # in case mixed modalities, the arch will be handled by subclass46 if not self.has_audio_encoder:47 self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.QWEN3VL)48 self.gguf_writer.add_vision_use_gelu(True)49 50 if self.hparams_vision is not None:51 merge_size = self.hparams_vision.get("spatial_merge_size")52 if merge_size is not None:53 self.gguf_writer.add_vision_spatial_merge_size(int(merge_size))54 55 # Use text config's rms_norm_eps for vision attention layernorm eps56 rms_norm_eps = self.global_config.get("text_config", {}).get("rms_norm_eps", 1e-6)57 self.gguf_writer.add_vision_attention_layernorm_eps(rms_norm_eps)58 59 if self.is_deepstack_layers:60 self.gguf_writer.add_vision_is_deepstack_layers(self.is_deepstack_layers)61 62 @classmethod63 def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:64 name, gen = item65 66 # Skip text model tensors67 if name.startswith("lm_head."):68 return None69 70 # Skip MTP tensors71 if name.startswith("mtp."):72 return None73 74 if name.startswith("model.visual."):75 name = name.replace("model.visual.", "visual.", 1)76 77 if not name.startswith("visual."):78 return None79 80 return super().filter_tensors((name, gen))81 82 def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:83 assert self.hparams_vision is not None84 85 if name.startswith("visual.deepstack_merger_list."):86 prefix, rest = name.split(".", maxsplit=3)[2:]87 # prefix is the layer index, convert to absolute clip layer index!88 idx = self.hparams_vision.get("deepstack_visual_indexes", [])[int(prefix)]89 target = rest90 91 tensor_type: gguf.MODEL_TENSOR92 if target.startswith("norm."):93 tensor_type = gguf.MODEL_TENSOR.V_DS_NORM94 suffix = target.split(".", 1)[1]95 elif target.startswith("linear_fc1."):96 tensor_type = gguf.MODEL_TENSOR.V_DS_FC197 suffix = target.split(".", 1)[1]98 elif target.startswith("linear_fc2."):99 tensor_type = gguf.MODEL_TENSOR.V_DS_FC2100 suffix = target.split(".", 1)[1]101 else:102 raise ValueError(f"Unexpected deepstack tensor: {name}")103 104 new_name = self.format_tensor_name(tensor_type, idx, suffix=f".{suffix}")105 yield from super().modify_tensors(data_torch, new_name, bid)106 return107 108 if name.startswith("visual.merger."):109 suffix = name.split(".", 2)[2]110 if suffix.startswith("linear_fc"):111 fc_idx_str, tail = suffix.split(".", 1)112 fc_num = int(fc_idx_str.replace("linear_fc", ""))113 # Qwen3VL has linear_fc1 and linear_fc2114 # Map to indices 0 and 2 (matching Qwen2VL which uses indices 0 and 2)115 if fc_num == 1:116 fc_idx = 0117 elif fc_num == 2:118 fc_idx = 2119 else:120 raise ValueError(f"unexpected fc index {fc_num} in {name}")121 new_name = self.format_tensor_name(gguf.MODEL_TENSOR.V_MMPROJ, fc_idx, suffix=f".{tail}")122 elif suffix.startswith("norm."):123 new_name = self.format_tensor_name(gguf.MODEL_TENSOR.V_POST_NORM, suffix=f".{suffix.split('.', 1)[1]}")124 else:125 raise ValueError(f"Unexpected merger tensor: {name}")126 yield (new_name, data_torch)127 return128 129 if name == "visual.patch_embed.proj.weight":130 # split Conv3D into Conv2Ds along temporal dimension131 c1, c2, kt, _, _ = data_torch.shape132 del c1, c2133 if kt != 2:134 raise ValueError("Current implementation only supports temporal_patch_size of 2")135 yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight", data_torch[:, :, 0, ...])136 yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight.1", data_torch[:, :, 1, ...])137 return138 139 if name == "visual.patch_embed.proj.bias":140 # Include the bias - it's used by the C++ code141 yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".bias", data_torch)142 return143 144 yield from MmprojModel.modify_tensors(self, data_torch, name, bid)145 146 147@ModelBase.register("Qwen3OmniMoeForConditionalGeneration")148@ModelBase.example("Qwen/Qwen3-Omni-30B-A3B-Instruct")149class Qwen3OmniMmprojModel(Qwen3VLVisionModel, Qwen25AudioModel):150 has_audio_encoder = True151 has_vision_encoder = True152 153 def get_vision_config(self) -> dict[str, Any] | None:154 if self.has_vision_encoder:155 return self.global_config["thinker_config"].get("vision_config")156 else:157 return None158 159 def get_audio_config(self) -> dict[str, Any] | None:160 if self.has_audio_encoder:161 return self.global_config["thinker_config"].get("audio_config")162 else:163 return None164 165 def set_gguf_parameters(self):166 if self.has_vision_encoder:167 Qwen3VLVisionModel.set_gguf_parameters(self)168 self.gguf_writer.add_clip_vision_projector_type(gguf.VisionProjectorType.QWEN3VL)169 if self.has_audio_encoder:170 Qwen25AudioModel.set_gguf_parameters(self)171 self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.QWEN3A)172 173 @classmethod174 def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:175 name, gen = item176 177 # Skip text model tensors178 if name.startswith("lm_head."):179 return None180 181 # Skip MTP tensors182 if name.startswith("mtp."):183 return None184 185 if name.startswith("model.visual."):186 name = name.replace("model.visual.", "visual.", 1)187 188 if name.startswith("thinker.audio_tower."):189 name = name.replace("thinker.audio_tower.", "audio_tower.", 1)190 191 if "visual." not in name and "audio_tower." not in name:192 return None193 194 return MmprojModel.filter_tensors((name, gen))195 196 def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:197 if "visual." in name:198 if not self.has_vision_encoder:199 raise ValueError(f"Model does not have vision encoder, but found tensor {name}")200 # need to transform vision tensor naming, so that modify_tensors() logic can be used correctly201 name = name.replace("thinker.visual.", "model.visual.")202 if ".merger_list." in name:203 name = name.replace(".merger_list.", ".deepstack_merger_list.")204 name = name.replace(".ln_q", ".norm")205 name = name.replace(".mlp.0", ".linear_fc1")206 name = name.replace(".mlp.2", ".linear_fc2")207 elif ".merger." in name:208 name = name.replace(".ln_q", ".norm")209 name = name.replace(".mlp.0", ".linear_fc1")210 name = name.replace(".mlp.2", ".linear_fc2")211 yield from Qwen3VLVisionModel.modify_tensors(self, data_torch, name, bid)212 elif "audio_tower." in name:213 if not self.has_audio_encoder:214 raise ValueError(f"Model does not have audio encoder, but found tensor {name}")215 if "conv2d" in name and name.endswith(".bias"):216 # transform conv2d bias [n_embd] --> [1, 1, n_embd]217 data_torch = data_torch.unsqueeze(-1).unsqueeze(-1)218 yield from Qwen25AudioModel.modify_tensors(self, data_torch, name, bid)219 220 221@ModelBase.register("Qwen3ASRForConditionalGeneration")222@ModelBase.example("Qwen/Qwen3-ASR-0.6B-hf")223class Qwen3ASRMmprojModel(Qwen3OmniMmprojModel):224 has_audio_encoder = True225 has_vision_encoder = False226 227 228@ModelBase.register("Glm4vForConditionalGeneration", "Glm4vMoeForConditionalGeneration", "GlmOcrForConditionalGeneration")229@ModelBase.example("zai-org/GLM-4.1V-9B-Thinking", "zai-org/GLM-4.5V")230class Glm4VVisionModel(Qwen3VLVisionModel):231 def set_gguf_parameters(self):232 MmprojModel.set_gguf_parameters(self) # skip Qwen3VLVisionModel parameters233 assert self.hparams_vision is not None234 self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.GLM4V)235 236 hidden_act = str(self.hparams_vision.get("hidden_act", "")).lower()237 if hidden_act == "gelu":238 self.gguf_writer.add_vision_use_gelu(True)239 elif hidden_act == "silu":240 self.gguf_writer.add_vision_use_silu(True)241 242 rms_norm_eps = self.hparams_vision.get("rms_norm_eps", 1e-5)243 self.gguf_writer.add_vision_attention_layernorm_eps(rms_norm_eps)244 245 def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:246 if name.startswith("visual.merger."):247 yield from ModelBase.modify_tensors(self, data_torch, name, bid)248 return249 yield from super().modify_tensors(data_torch, name, bid)250 251 252@ModelBase.register("Qwen3VLForConditionalGeneration")253@ModelBase.example("Qwen/Qwen3-VL-4B-Instruct")254class Qwen3VLTextModel(Qwen3Model):255 model_arch = gguf.MODEL_ARCH.QWEN3VL256 257 def set_gguf_parameters(self):258 super().set_gguf_parameters()259 if "thinker_config" in self.hparams:260 vision_config = self.hparams["thinker_config"].get("vision_config", {})261 else:262 vision_config = self.hparams.get("vision_config", {})263 deepstack_layer_num = len(vision_config.get("deepstack_visual_indexes", []))264 self.gguf_writer.add_num_deepstack_layers(deepstack_layer_num)265 266 @classmethod267 def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:268 name, gen = item269 270 name = name.replace("thinker.", "")271 272 return super().filter_tensors((name, gen))273 274 275@ModelBase.register("Qwen3VLMoeForConditionalGeneration")276@ModelBase.example("Qwen/Qwen3-VL-30B-A3B-Instruct")277class Qwen3VLMoeTextModel(Qwen3MoeModel):278 model_arch = gguf.MODEL_ARCH.QWEN3VLMOE279 280 def set_gguf_parameters(self):281 super().set_gguf_parameters()282 vision_config = self.hparams.get("vision_config", {})283 deepstack_layer_num = len(vision_config.get("deepstack_visual_indexes", []))284 self.gguf_writer.add_num_deepstack_layers(deepstack_layer_num)285 286 @classmethod287 def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:288 name, gen = item289 290 name = name.replace("thinker.", "")291 292 return super().filter_tensors((name, gen))293 294 def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:295 # Qwen3VL has transposed packed tensors, so we treat it differently from general Qwen2MoE packed tensors296 if name.endswith("mlp.experts.down_proj") or name.endswith("mlp.experts.down_proj.weight"):297 mapped = f"{name}.weight" if not name.endswith(".weight") else name298 permuted = data_torch.permute(0, 2, 1).contiguous()299 yield from ModelBase.modify_tensors(self, permuted, mapped, bid)300 return301 302 if name.endswith("mlp.experts.gate_up_proj") or name.endswith("mlp.experts.gate_up_proj.weight"):303 if data_torch.ndim < 3 or data_torch.shape[-1] % 2 != 0:304 raise ValueError(f"Unexpected gate_up_proj shape for {name}: {tuple(data_torch.shape)}")305 split_dim = data_torch.shape[-1] // 2306 gate = data_torch[..., :split_dim].contiguous()307 up = data_torch[..., split_dim:].contiguous()308 # Input gate/up: (n_expert=128, n_embd=2048, n_ff_exp=768)309 # Want GGML ne: {n_embd, n_ff_exp, n_expert} = {2048, 768, 128}310 # Need PyTorch: (128, 768, 2048) [reversed of GGML]311 # So: permute(0, 2, 1): (128, 2048, 768) -> (128, 768, 2048)312 base_name = name.removesuffix(".weight")313 base = base_name.rsplit('.', 1)[0]314 mapped_gate = f"{base}.gate_proj.weight"315 mapped_up = f"{base}.up_proj.weight"316 perm_gate = gate.permute(0, 2, 1).contiguous()317 perm_up = up.permute(0, 2, 1).contiguous()318 yield from ModelBase.modify_tensors(self, perm_gate, mapped_gate, bid)319 yield from ModelBase.modify_tensors(self, perm_up, mapped_up, bid)320 return321 322 yield from super().modify_tensors(data_torch, name, bid)323 324 325@ModelBase.register("Qwen3OmniMoeForConditionalGeneration")326@ModelBase.example("Qwen/Qwen3-Omni-30B-A3B-Instruct")327class Qwen3OmniMoeTextModel(Qwen3VLMoeTextModel):328 model_arch = gguf.MODEL_ARCH.QWEN3VLMOE329 330 def set_vocab(self):331 super().set_vocab()332 # correct BOS/EOS tokens333 with open(self.dir_model / "tokenizer_config.json", "r", encoding="utf-8") as f:334 tokenizer_config = json.load(f)335 added_tokens = tokenizer_config.get("added_tokens_decoder", {})336 for token_id, data in added_tokens.items():337 if data.get("content") == "<|im_end|>":338 self.gguf_writer.add_bos_token_id(int(token_id))339 self.gguf_writer.add_eos_token_id(int(token_id))340 break341 342 def set_gguf_parameters(self):343 super().set_gguf_parameters()344 self.gguf_writer.add_num_deepstack_layers(0)345 346 347@ModelBase.register("Qwen3ASRForConditionalGeneration")348@ModelBase.example("Qwen/Qwen3-ASR-0.6B-hf")349class Qwen3ASRTextModel(Qwen3VLTextModel):350 model_arch = gguf.MODEL_ARCH.QWEN3VL351 352 def set_gguf_parameters(self):353 super().set_gguf_parameters()354 self.gguf_writer.add_num_deepstack_layers(0)355 356 def set_vocab(self):357 super().set_vocab()358 # fix chat template, use correct chatml format359 self.gguf_writer.add_chat_template("{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}")360 # correct BOS/EOS tokens361 with open(self.dir_model / "tokenizer_config.json", "r", encoding="utf-8") as f:362 tokenizer_config = json.load(f)363 added_tokens = tokenizer_config.get("added_tokens_decoder", {})364 for token_id, data in added_tokens.items():365 if data.get("content") == "<|im_end|>":366 self.gguf_writer.add_bos_token_id(int(token_id))367 self.gguf_writer.add_eos_token_id(int(token_id))368 break369