Felipe97/llama-cpp-compiled
01.1k
1from __future__ import annotations2 3import json4import re5 6from typing import Callable, Iterable, TYPE_CHECKING, Sequence7 8import torch9 10if TYPE_CHECKING:11 from torch import Tensor12 13from .base import MmprojModel, ModelBase, TextModel, gguf, logger14 15 16@ModelBase.register("GemmaForCausalLM")17# [TAG_HF_EXAMPLE_GATED] google/gemma-2b is gated18@ModelBase.example("trl-internal-testing/tiny-GemmaForCausalLM")19class GemmaModel(TextModel):20 model_arch = gguf.MODEL_ARCH.GEMMA21 22 def set_vocab(self):23 self._set_vocab_sentencepiece()24 25 # TODO: these special tokens should be exported only for the CodeGemma family26 special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=False,27 special_token_types = ['prefix', 'suffix', 'middle', 'fsep', 'eot'])28 special_vocab._set_special_token("prefix", 67)29 special_vocab._set_special_token("suffix", 69)30 special_vocab._set_special_token("middle", 68)31 special_vocab._set_special_token("fsep", 70)32 special_vocab._set_special_token("eot", 107)33 special_vocab.chat_template = None # do not add it twice34 special_vocab.add_to_gguf(self.gguf_writer)35 36 self.gguf_writer.add_add_space_prefix(False)37 38 def set_gguf_parameters(self):39 hparams = self.hparams40 41 self.gguf_writer.add_context_length(hparams["max_position_embeddings"])42 self.gguf_writer.add_embedding_length(hparams["hidden_size"])43 self.gguf_writer.add_block_count(self.block_count)44 self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"])45 self.gguf_writer.add_head_count(hparams["num_attention_heads"])46 self.gguf_writer.add_head_count_kv(self.hparams["num_key_value_heads"] if "num_key_value_heads" in hparams else hparams["num_attention_heads"])47 self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])48 self.gguf_writer.add_key_length(hparams["head_dim"])49 self.gguf_writer.add_value_length(hparams["head_dim"])50 self.gguf_writer.add_file_type(self.ftype)51 52 @classmethod53 def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:54 name, gen = item55 56 # lm_head is not used in llama.cpp, while autoawq will include this tensor in model57 # To prevent errors, skip loading lm_head.weight.58 if name == "lm_head.weight":59 logger.debug(f"Skipping get tensor {name!r} in safetensors so that convert can end normally.")60 return None61 62 return super().filter_tensors(item)63 64 def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:65 # ref: https://github.com/huggingface/transformers/blob/fc37f38915372c15992b540dfcbbe00a916d4fc6/src/transformers/models/gemma/modeling_gemma.py#L8966 if name.endswith("norm.weight"):67 data_torch = data_torch + 168 69 yield from super().modify_tensors(data_torch, name, bid)70 71 72@ModelBase.register("Gemma2ForCausalLM")73# [TAG_HF_EXAMPLE_GATED] google/gemma-2-9b-it is gated74@ModelBase.example("trl-internal-testing/tiny-Gemma2ForCausalLM")75class Gemma2Model(TextModel):76 model_arch = gguf.MODEL_ARCH.GEMMA277 78 def set_vocab(self):79 self._set_vocab_sentencepiece()80 81 self.gguf_writer.add_add_space_prefix(False)82 83 def set_gguf_parameters(self):84 hparams = self.hparams85 86 self.gguf_writer.add_context_length(hparams["max_position_embeddings"])87 self.gguf_writer.add_embedding_length(hparams["hidden_size"])88 self.gguf_writer.add_block_count(self.block_count)89 self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"])90 self.gguf_writer.add_head_count(hparams["num_attention_heads"])91 self.gguf_writer.add_head_count_kv(self.hparams["num_key_value_heads"] if "num_key_value_heads" in hparams else hparams["num_attention_heads"])92 self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])93 self.gguf_writer.add_key_length(hparams["head_dim"])94 self.gguf_writer.add_value_length(hparams["head_dim"])95 self.gguf_writer.add_file_type(self.ftype)96 self.gguf_writer.add_attn_logit_softcapping(97 self.hparams["attn_logit_softcapping"]98 )99 self.gguf_writer.add_final_logit_softcapping(100 self.hparams["final_logit_softcapping"]101 )102 self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])103 104 @classmethod105 def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:106 name, gen = item107 108 # lm_head is not used in llama.cpp, while autoawq will include this tensor in model109 # To prevent errors, skip loading lm_head.weight.110 if name == "lm_head.weight":111 logger.debug(f"Skipping get tensor {name!r} in safetensors so that convert can end normally.")112 return None113 114 return super().filter_tensors(item)115 116 def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:117 # ref: https://github.com/huggingface/transformers/blob/fc37f38915372c15992b540dfcbbe00a916d4fc6/src/transformers/models/gemma/modeling_gemma.py#L89118 if name.endswith("norm.weight"):119 data_torch = data_torch + 1120 121 yield from super().modify_tensors(data_torch, name, bid)122 123 124@ModelBase.register("Gemma3ForCausalLM", "Gemma3ForConditionalGeneration")125# [TAG_HF_EXAMPLE_GATED] google/gemma-3-4b-it is gated126@ModelBase.example("trl-internal-testing/tiny-Gemma3ForConditionalGeneration", "hf-tiny-v2/tiny-random-Gemma3ForCausalLM")127class Gemma3Model(TextModel):128 model_arch = gguf.MODEL_ARCH.GEMMA3129 130 def norm_shift(self, name: str) -> float:131 return 1.0 if name.endswith("norm.weight") else 0.0 # Gemma3RMSNorm adds 1.0 to the norm value132 133 def set_vocab(self):134 if (self.dir_model / "tokenizer.model").is_file():135 self._set_vocab_sentencepiece()136 self.gguf_writer.add_add_space_prefix(False)137 else:138 self._set_vocab_gpt2()139 140 def set_gguf_parameters(self):141 super().set_gguf_parameters()142 hparams = self.hparams143 144 # some default values are not specified in the hparams145 self.gguf_writer.add_context_length(hparams.get("max_position_embeddings", 131072))146 self.gguf_writer.add_head_count(hparams.get("num_attention_heads", 8))147 self.gguf_writer.add_layer_norm_rms_eps(self.hparams.get("rms_norm_eps", 1e-6))148 self.gguf_writer.add_key_length(hparams.get("head_dim", 256))149 self.gguf_writer.add_value_length(hparams.get("head_dim", 256))150 self.gguf_writer.add_rope_freq_base(self.rope_parameters.get("full_attention", self.rope_parameters).get("rope_theta", 1_000_000.0)) # for global layers151 # attn_logit_softcapping is removed in Gemma3152 assert hparams.get("attn_logit_softcapping") is None153 if (final_logit_softcap := hparams.get("final_logit_softcapping")):154 self.gguf_writer.add_final_logit_softcapping(final_logit_softcap)155 if hparams.get("sliding_window_pattern") != 1:156 self.gguf_writer.add_sliding_window(hparams["sliding_window"])157 self.gguf_writer.add_head_count_kv(hparams.get("num_key_value_heads", 4))158 159 def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:160 # remove OOV (out-of-vocabulary) rows in token_embd161 if "embed_tokens.weight" in name:162 n_vocab_real = -1163 if (self.dir_model / "tokenizer.model").is_file():164 tokens = self._create_vocab_sentencepiece()[0]165 n_vocab_real = len(tokens)166 else:167 with open(self.dir_model / "tokenizer.json", "r", encoding="utf-8") as f:168 tokenizer_json = json.load(f)169 n_vocab_real = len(tokenizer_json["model"]["vocab"]) + len(tokenizer_json["added_tokens"])170 data_torch = data_torch[:n_vocab_real]171 172 # ref code in Gemma3RMSNorm173 # output = output * (1.0 + self.weight.float())174 # note: this is not the case on gemma3n175 f_shift = self.norm_shift(name)176 if f_shift != 0.0:177 data_torch = data_torch + f_shift178 179 yield from super().modify_tensors(data_torch, name, bid)180 181 182@ModelBase.register("Gemma3TextModel")183# [TAG_HF_EXAMPLE_GATED] google/embeddinggemma-300m is gated184@ModelBase.example("hf-tiny-v2/tiny-random-Gemma3TextModel")185class EmbeddingGemma(Gemma3Model):186 model_arch = gguf.MODEL_ARCH.GEMMA_EMBEDDING187 module_paths = []188 dense_features_dims = {}189 190 def __init__(self, *args, **kwargs):191 super().__init__(*args, **kwargs)192 if self.sentence_transformers_dense_modules:193 # read modules.json to determine if model has Dense layers194 modules_file = self.dir_model / "modules.json"195 if modules_file.is_file():196 with open(modules_file, encoding="utf-8") as modules_json_file:197 mods = json.load(modules_json_file)198 for mod in mods:199 if mod["type"].endswith("Dense"):200 mod_path = mod["path"]201 # check if model.safetensors file for Dense layer exists202 model_tensors_file = self.dir_model / mod_path / "model.safetensors"203 if model_tensors_file.is_file():204 self.module_paths.append(mod_path)205 # read config.json of the Dense layer to get in/out features206 mod_conf_file = self.dir_model / mod_path / "config.json"207 if mod_conf_file.is_file():208 with open(mod_conf_file, encoding="utf-8") as mod_conf_json_file:209 mod_conf = json.load(mod_conf_json_file)210 # hparams dense_2_feat_out and dense_3_feat_in are required when loading model's dense weights211 prefix = self._get_dense_prefix(mod_path)212 if mod_conf["in_features"] is not None and mod_conf["out_features"] is not None:213 self.dense_features_dims[prefix] = (mod_conf["in_features"], mod_conf["out_features"])214 215 def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:216 from safetensors.torch import load_file217 module_paths = list(self.module_paths)218 for i, module_path in enumerate(module_paths):219 tensors_file = self.dir_model / module_path / "model.safetensors"220 local_tensors = load_file(tensors_file)221 tensor_name = self._get_dense_prefix(module_path)222 for name, local_tensor in local_tensors.items():223 if not name.endswith(".weight"):224 continue225 orig_name = name.replace("linear", tensor_name)226 name = self.map_tensor_name(orig_name)227 yield name, local_tensor.clone()228 229 @staticmethod230 def _get_dense_prefix(module_path) -> str:231 """Get the tensor name prefix for the Dense layer from module path."""232 tensor_name = "dense_2" if module_path == "2_Dense" else "dense_3"233 return tensor_name234 235 def set_gguf_parameters(self):236 super().set_gguf_parameters()237 238 # Override the sliding window size as it gets adjusted by the Gemma3TextConfig239 # constructor. We want to use the value from the original model's config.json.240 # ref: https://github.com/huggingface/transformers/pull/40700241 with open(self.dir_model / "config.json", "r", encoding="utf-8") as f:242 config = json.load(f)243 orig_sliding_window = config.get("sliding_window")244 if orig_sliding_window is None:245 raise ValueError("sliding_window not found in model config - this is required for the model")246 247 logger.info(f"Using original sliding_window from config: {orig_sliding_window} "248 f"instead of {self.hparams['sliding_window']}")249 self.gguf_writer.add_sliding_window(orig_sliding_window)250 if self.sentence_transformers_dense_modules:251 for dense, dims in self.dense_features_dims.items():252 logger.info(f"Setting dense layer {dense} in/out features to {dims}")253 self.gguf_writer.add_dense_features_dims(dense, dims[0], dims[1])254 255 self._try_set_pooling_type()256 257 258@ModelBase.register("Gemma3ForConditionalGeneration")259# [TAG_HF_EXAMPLE_GATED] google/gemma-3-4b-it is gated260@ModelBase.example("trl-internal-testing/tiny-Gemma3ForConditionalGeneration")261class Gemma3VisionModel(MmprojModel):262 def set_gguf_parameters(self):263 super().set_gguf_parameters()264 hparams = self.hparams265 self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.GEMMA3)266 # default values below are taken from HF transformers code267 self.gguf_writer.add_vision_attention_layernorm_eps(hparams.get("layer_norm_eps", 1e-6))268 self.gguf_writer.add_vision_use_gelu(True)269 # calculate proj_scale_factor (used by tinygemma3 test model)270 image_seq_length = self.preprocessor_config.get("image_seq_length", 256)271 n_per_side = int(image_seq_length ** 0.5)272 image_size = self.hparams["image_size"]273 patch_size = self.hparams["patch_size"]274 proj_scale_factor = (image_size // patch_size) // n_per_side275 if proj_scale_factor > 0 and proj_scale_factor != 4:276 # we only need to write this if it's not the default value277 # in this case, we are converting a test model278 self.gguf_writer.add_vision_projector_scale_factor(proj_scale_factor)279 280 def tensor_force_quant(self, name, new_name, bid, n_dims):281 # related to https://github.com/ggml-org/llama.cpp/issues/13025282 if "input_projection" in name:283 return gguf.GGMLQuantizationType.F16284 if ".embeddings." in name:285 return gguf.GGMLQuantizationType.F32286 return super().tensor_force_quant(name, new_name, bid, n_dims)287 288 @classmethod289 def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:290 name, gen = item291 292 if "vision_model.head." in name:293 # skip redundant tensors for tinygemma3294 return None295 296 if not name.startswith(("multi_modal_projector.", "vision_tower.", "multimodal_projector.", "vision_model.")):297 return None298 299 name = name.replace("_weight", ".weight")300 301 return super().filter_tensors((name, gen))302 303 def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:304 # correct norm value ; only this "soft_emb_norm" need to be corrected as it's part of Gemma projector305 # the other norm values are part of SigLIP model, and they are already correct306 # ref code: Gemma3RMSNorm307 if "soft_emb_norm.weight" in name:308 logger.info(f"Correcting norm value for '{name}'")309 data_torch = data_torch + 1310 311 yield from super().modify_tensors(data_torch, name, bid)312 313 314class ConformerAudioModel(MmprojModel):315 _batch_norm_tensors: list[dict[str, Tensor]] | None = None316 317 @staticmethod318 def is_audio_tensor(name: str):319 return any(p in name for p in ["audio", "codebook", "conformer", "depth_embedding", "depthformer", "depth_linear"])320 321 def tensor_force_quant(self, name, new_name, bid, n_dims):322 if ConformerAudioModel.is_audio_tensor(name):323 if ".conv" in name or "_conv" in name and ".weight" in name:324 return gguf.GGMLQuantizationType.F32325 return super().tensor_force_quant(name, new_name, bid, n_dims)326 327 def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:328 # fold running_mean, running_var and eps into weight and bias for batch_norm329 if "batch_norm" in name:330 if self._batch_norm_tensors is None:331 self._batch_norm_tensors = [{} for _ in range(self.block_count)]332 assert bid is not None333 self._batch_norm_tensors[bid][name] = data_torch334 335 if len(self._batch_norm_tensors[bid]) < 5:336 return337 338 weight = self._batch_norm_tensors[bid][f"conformer.layers.{bid}.conv.batch_norm.weight"]339 bias = self._batch_norm_tensors[bid][f"conformer.layers.{bid}.conv.batch_norm.bias"]340 running_mean = self._batch_norm_tensors[bid][f"conformer.layers.{bid}.conv.batch_norm.running_mean"]341 running_var = self._batch_norm_tensors[bid][f"conformer.layers.{bid}.conv.batch_norm.running_var"]342 eps = 1e-5 # default value343 344 a = weight / torch.sqrt(running_var + eps)345 b = bias - running_mean * a346 yield from super().modify_tensors(a, f"conformer.layers.{bid}.conv.batch_norm.weight", bid)347 yield from super().modify_tensors(b, f"conformer.layers.{bid}.conv.batch_norm.bias", bid)348 return349 350 # reshape conv weights351 if name.startswith("conformer.pre_encode.conv.") and name.endswith(".bias"):352 data_torch = data_torch[:, None, None]353 if "conv.depthwise_conv" in name and name.endswith(".weight"):354 assert data_torch.shape[1] == 1355 data_torch = data_torch.reshape(data_torch.shape[0], data_torch.shape[2])356 if "conv.pointwise_conv" in name and name.endswith(".weight"):357 assert data_torch.shape[2] == 1358 data_torch = data_torch.reshape(data_torch.shape[0], data_torch.shape[1])359 360 mapped_name = self.map_tensor_name(name, (".weight", ".bias", ".input_max", ".input_min", ".output_max", ".output_min"))361 yield (mapped_name, data_torch)362 363 364@ModelBase.register("Gemma3nForConditionalGeneration")365# [TAG_HF_EXAMPLE_GATED] google/gemma-3n-E2B-it is gated366@ModelBase.example("hf-tiny-v2/tiny-random-Gemma3nForConditionalGeneration")367class Gemma3nVisionAudioModel(ConformerAudioModel):368 has_audio_encoder = True369 has_vision_encoder = True370 371 # Double indexed mapping for MobileNetV5 blocks (not supported by tensor_mapping.py)372 # This is the only known model having this, so we prefer implementing it outside of tensor_mapping.py373 block_tensor_mapping = {374 "model.vision_tower.timm_model.blocks.{bid}.{sid}.conv_exp.weight": "v.blk.{bid}.{sid}.conv_exp.weight",375 "model.vision_tower.timm_model.blocks.{bid}.{sid}.bn1.weight": "v.blk.{bid}.{sid}.bn1.weight",376 "model.vision_tower.timm_model.blocks.{bid}.{sid}.conv_pwl.weight": "v.blk.{bid}.{sid}.conv_pwl.weight",377 "model.vision_tower.timm_model.blocks.{bid}.{sid}.bn2.weight": "v.blk.{bid}.{sid}.bn2.weight",378 "model.vision_tower.timm_model.blocks.{bid}.{sid}.dw_start.conv.weight": "v.blk.{bid}.{sid}.dw_start.conv.weight",379 "model.vision_tower.timm_model.blocks.{bid}.{sid}.dw_start.bn.weight": "v.blk.{bid}.{sid}.dw_start.bn.weight",380 "model.vision_tower.timm_model.blocks.{bid}.{sid}.dw_mid.conv.weight": "v.blk.{bid}.{sid}.dw_mid.conv.weight",381 "model.vision_tower.timm_model.blocks.{bid}.{sid}.dw_mid.bn.weight": "v.blk.{bid}.{sid}.dw_mid.bn.weight",382 "model.vision_tower.timm_model.blocks.{bid}.{sid}.pw_exp.conv.weight": "v.blk.{bid}.{sid}.pw_exp.conv.weight",383 "model.vision_tower.timm_model.blocks.{bid}.{sid}.pw_exp.bn.weight": "v.blk.{bid}.{sid}.pw_exp.bn.weight",384 "model.vision_tower.timm_model.blocks.{bid}.{sid}.pw_proj.conv.weight": "v.blk.{bid}.{sid}.pw_proj.conv.weight",385 "model.vision_tower.timm_model.blocks.{bid}.{sid}.pw_proj.bn.weight": "v.blk.{bid}.{sid}.pw_proj.bn.weight",386 "model.vision_tower.timm_model.blocks.{bid}.{sid}.layer_scale.gamma": "v.blk.{bid}.{sid}.layer_scale.gamma",387 "model.vision_tower.timm_model.blocks.{bid}.{sid}.attn.query.proj.weight": "v.blk.{bid}.{sid}.attn.query.proj.weight",388 "model.vision_tower.timm_model.blocks.{bid}.{sid}.attn.key.proj.weight": "v.blk.{bid}.{sid}.attn.key.proj.weight",389 "model.vision_tower.timm_model.blocks.{bid}.{sid}.attn.value.proj.weight": "v.blk.{bid}.{sid}.attn.value.proj.weight",390 "model.vision_tower.timm_model.blocks.{bid}.{sid}.attn.output.proj.weight": "v.blk.{bid}.{sid}.attn.output.proj.weight",391 "model.vision_tower.timm_model.blocks.{bid}.{sid}.attn.key.down_conv.weight": "v.blk.{bid}.{sid}.attn.key.down_conv.weight",392 "model.vision_tower.timm_model.blocks.{bid}.{sid}.attn.key.norm.weight": "v.blk.{bid}.{sid}.attn.key.norm.weight",393 "model.vision_tower.timm_model.blocks.{bid}.{sid}.attn.value.down_conv.weight": "v.blk.{bid}.{sid}.attn.value.down_conv.weight",394 "model.vision_tower.timm_model.blocks.{bid}.{sid}.attn.value.norm.weight": "v.blk.{bid}.{sid}.attn.value.norm.weight",395 "model.vision_tower.timm_model.blocks.{bid}.{sid}.norm.weight": "v.blk.{bid}.{sid}.norm.weight",396 }397 398 def __init__(self, *args, **kwargs):399 # Parent init will call find_hparam which now returns 0 for empty keys400 super().__init__(*args, **kwargs)401 assert self.hparams_vision is not None402 self.hparams_vision["n_layers"] = 128 # fake value for audio encoder, vision encoder doesn't use it403 self.hparams_vision["intermediate_size"] = self.hparams_vision.get("intermediate_size", 2048) * 4404 self.hparams_vision["num_attention_heads"] = self.hparams_vision.get("num_attention_heads", 8)405 406 # MobileNetV5 does not use image_mean/std407 self.preprocessor_config["image_mean"] = [0.0 ,0.0 , 0.0]408 self.preprocessor_config["image_std"] = [1.0 ,1.0 ,1.0]409 self.hparams_vision["image_size"] = self.preprocessor_config.get(410 "size", {"height": 768, "width": 768}411 )["height"]412 413 # Image sequence length (256 tokens = 16x16 for Gemma3n)414 image_seq_length = self.preprocessor_config.get("image_seq_length", 256)415 image_size = self.hparams_vision["image_size"]416 self.hparams_vision["patch_size"] = image_size // image_seq_length417 418 # remap audio hparams419 assert self.hparams_audio is not None420 self.hparams_audio["n_layers"] = self.hparams_audio["conf_num_hidden_layers"]421 self.hparams_audio["num_attention_heads"] = self.hparams_audio["conf_num_attention_heads"]422 self.hparams_audio["feat_in"] = self.hparams_audio["input_feat_size"]423 self.hparams_audio["intermediate_size"] = self.hparams_audio.get("intermediate_size", 6144)424 425 def set_gguf_parameters(self):426 super().set_gguf_parameters()427 428 # vision params429 self.gguf_writer.add_clip_vision_projector_type(gguf.VisionProjectorType.GEMMA3NV)430 self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams.get("layer_norm_eps", 1e-6))431 432 # audio params433 assert self.hparams_audio is not None434 self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.GEMMA3NA)435 self.gguf_writer.add_audio_num_mel_bins(self.hparams_audio["feat_in"])436 self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)437 438 def tensor_force_quant(self, name, new_name, bid, n_dims):439 # Force quantization settings for specific tensor types440 if "input_projection" in name or "input_proj" in name:441 return gguf.GGMLQuantizationType.F16442 if ".embeddings." in name or "stem" in name:443 return gguf.GGMLQuantizationType.F32444 return super().tensor_force_quant(name, new_name, bid, n_dims)445 446 def custom_map(self, name: str) -> str:447 """Parses names like model.vision_tower.timm_model.blocks.1.2.suffix and applies template mapping."""448 parts = name.split(".")449 # MobileNet blocks have at least 7 parts: model, vision_tower, timm_model, blocks, bid, sid, and suffix450 if len(parts) >= 7:451 bid, sid = parts[4], parts[5]452 suffix = ".".join(parts[6:])453 template = f"model.vision_tower.timm_model.blocks.{{bid}}.{{sid}}.{suffix}"454 if template in self.block_tensor_mapping:455 return self.block_tensor_mapping[template].format(bid=bid, sid=sid)456 457 raise ValueError(f"Unknown name: {name}")458 459 def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:460 if (ConformerAudioModel.is_audio_tensor(name)):461 name = name.replace("model.audio_tower.conformer.", "conformer.layers.")462 yield from super().modify_tensors(data_torch, name, bid)463 464 # Gemma3n uses465 # - model.embed_vision.* for projection layers466 # - model.vision_tower.* for vision encoder467 # Skip non-vision tensors468 if not (name.startswith("model.embed_vision.") or name.startswith("model.vision_tower.")):469 return470 471 if name.startswith("model.vision_tower.timm_model.blocks."):472 # Double-indexed block tensors through custom logic473 yield (self.custom_map(name), data_torch)474 return475 else:476 # Route non-repeating (conv_stem, msfa, embedding, etc.) and un-catched through tensor_mapping.py477 new_name = self.map_tensor_name(name)478 479 if new_name.endswith("conv_stem.conv.bias") or new_name.endswith("layer_scale.gamma"):480 data_torch = data_torch.unsqueeze(0).unsqueeze(-1).unsqueeze(-1) # [1, C, 1, 1]481 482 yield from ModelBase.modify_tensors(self, data_torch, new_name, bid)483 484 485@ModelBase.register("Gemma3nForCausalLM", "Gemma3nForConditionalGeneration")486# [TAG_HF_EXAMPLE_GATED] google/gemma-3n-E2B-it is gated487@ModelBase.example("hf-tiny-v2/tiny-random-Gemma3nForConditionalGeneration")488class Gemma3NModel(Gemma3Model):489 model_arch = gguf.MODEL_ARCH.GEMMA3N490 491 _altup_proj: list[Tensor] = []492 _altup_unembd: list[Tensor] = []493 494 def __init__(self, *args, **kwargs):495 super().__init__(*args, **kwargs)496 assert self.hparams["altup_num_inputs"] == 4, "Current conversion only supports 4 altup inputs"497 self._altup_proj = [498 torch.Tensor(), # to be replaced499 torch.Tensor(), # to be replaced500 torch.Tensor(), # to be replaced501 ]502 self._altup_unembd = [503 torch.Tensor(), # to be replaced504 torch.Tensor(), # to be replaced505 torch.Tensor(), # to be replaced506 ]507 508 def norm_shift(self, name: str) -> float:509 del name510 return 0.0 # same value with Gemma3p5RMSNorm scale_shift on python code511 512 def set_vocab(self):513 # For Gemma3n multimodal models, we need the FULL vocab_size (262400)514 # which includes special tokens from 262144-262399 for vision/audio.515 # The vocab_size_per_layer_input (262144) is only the embedding size per layer.516 # Temporarily override the hparams lookup order to prioritize vocab_size.517 518 # Store original vocab_size_per_layer_input if it exists519 vocab_size_per_layer_input = self.hparams.get("vocab_size_per_layer_input")520 521 # Temporarily remove vocab_size_per_layer_input to force using vocab_size522 if vocab_size_per_layer_input is not None:523 del self.hparams["vocab_size_per_layer_input"]524 525 # Call parent set_vocab which will now use vocab_size (262400)526 super().set_vocab()527 528 # Restore vocab_size_per_layer_input for later use529 if vocab_size_per_layer_input is not None:530 self.hparams["vocab_size_per_layer_input"] = vocab_size_per_layer_input531 532 def set_gguf_parameters(self):533 super().set_gguf_parameters()534 self.gguf_writer.add_altup_active_idx(self.hparams["altup_active_idx"])535 self.gguf_writer.add_altup_num_inputs(self.hparams["altup_num_inputs"])536 self.gguf_writer.add_embedding_length_per_layer_input(self.hparams["hidden_size_per_layer_input"])537 self.gguf_writer.add_shared_kv_layers(self.hparams["num_kv_shared_layers"])538 539 activation_sparsity_scale = []540 for s in self.hparams["activation_sparsity_pattern"]:541 normal_dist = torch.distributions.normal.Normal(0, 1)542 std_multiplier = normal_dist.icdf(torch.tensor(s, dtype=torch.float32))543 activation_sparsity_scale.append(std_multiplier.item())544 self.gguf_writer.add_activation_sparsity_scale(activation_sparsity_scale)545 546 sliding_window_pattern = []547 for t in self.hparams["layer_types"]:548 sliding_window_pattern.append(t == "sliding_attention")549 self.gguf_writer.add_sliding_window_pattern(sliding_window_pattern)550 551 def _stack_matrices(self, matrices: list[Tensor]) -> Tensor | None:552 has_all = all(m.numel() > 0 for m in matrices)553 if not has_all:554 return None555 else:556 return torch.stack(matrices, dim=0)557 558 @classmethod559 def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:560 name, gen = item561 562 if name.endswith("_scale"):563 name = name + ".weight"564 565 return super().filter_tensors((name, gen))566 567 def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:568 # TODO: implement self.prediction_coefs.weight.clamp_(...)569 570 # Pad token embeddings for vision/audio special tokens (262144-262399)571 if "embed_tokens.weight" in name or "embed_tokens_per_layer" in name:572 # Move to CPU to avoid meta device issues during padding573 data_torch = data_torch.to(device="cpu")574 575 vocab_size = self.hparams.get("vocab_size", 262400)576 current_size = data_torch.shape[0] # First dimension is vocab_size577 578 if current_size < vocab_size:579 # Pad with zeros for vision/audio tokens (they get embeddings from vision tower)580 padding_size = vocab_size - current_size581 tensor_type = "per-layer embeddings" if "per_layer" in name else "token embeddings"582 logger.info(f"Padding {tensor_type} shape {list(data_torch.shape)} from {current_size} to {vocab_size} (adding {padding_size} vision/audio token slots)")583 584 # Create padding with zeros (vision tokens won't use these embeddings)585 padding = torch.zeros((padding_size, data_torch.shape[1]), dtype=data_torch.dtype, device=data_torch.device)586 data_torch = torch.cat([data_torch, padding], dim=0)587 588 # Continue with normal processing589 yield from ModelBase.modify_tensors(self, data_torch, name, bid)590 return591 592 if "altup_unembed_projections" in name:593 data_torch = data_torch.to(device="cpu")594 # altup_unembed matrices are [hidden_size, hidden_size], NOT vocab-based595 # They should NOT be padded596 if ".0." in name:597 self._altup_unembd[0] = data_torch598 elif ".1." in name:599 self._altup_unembd[1] = data_torch600 elif ".2." in name:601 self._altup_unembd[2] = data_torch602 else:603 raise ValueError(f"Unknown name: {name}")604 out = self._stack_matrices(self._altup_unembd)605 if out is not None:606 yield from ModelBase.modify_tensors(self, out, "model.altup_unembed_projections.weight", bid)607 return608 else:609 return610 611 if "altup_projections" in name:612 data_torch = data_torch.to(device="cpu")613 if ".0." in name:614 self._altup_proj[0] = data_torch615 elif ".1." in name:616 self._altup_proj[1] = data_torch617 elif ".2." in name:618 self._altup_proj[2] = data_torch619 else:620 raise ValueError(f"Unknown name: {name}")621 out = self._stack_matrices(self._altup_proj)622 if out is not None:623 yield from ModelBase.modify_tensors(self, out, "model.altup_projections.weight", bid)624 return625 else:626 return627 628 yield from super().modify_tensors(data_torch, name, bid)629 630 631@ModelBase.register("Gemma4ForConditionalGeneration", "Gemma4ForCausalLM")632@ModelBase.example("google/gemma-4-31B-it", "google/gemma-4-26B-A4B-it", "google/gemma-4-E2B-it")633class Gemma4Model(Gemma3Model):634 model_arch = gguf.MODEL_ARCH.GEMMA4635 636 def norm_shift(self, name: str) -> float:637 del name # unused638 return 0.0639 640 def set_vocab(self):641 vocab = gguf.LlamaHfVocab(self.dir_model)642 tokens = []643 scores = []644 toktypes = []645 visible_tokens = {"<|channel>", "<channel|>", "<|tool_call>", "<tool_call|>", "<|tool_response>", "<tool_response|>", "<|\"|>"}646 647 for text, score, toktype in vocab.all_tokens():648 tokens.append(text)649 scores.append(score)650 text_str = text.decode()651 if text_str in visible_tokens:652 # always render these tokens, so that the chat parser can read them653 toktypes.append(gguf.TokenType.USER_DEFINED)654 logger.info(f"Token '{text_str}' is set to USER_DEFINED")655 else:656 toktypes.append(toktype)657 658 assert len(tokens) == vocab.vocab_size659 660 self.gguf_writer.add_tokenizer_model("gemma4")661 self.gguf_writer.add_token_list(tokens)662 self.gguf_writer.add_token_scores(scores)663 self.gguf_writer.add_token_types(toktypes)664 665 special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)666 special_vocab.add_to_gguf(self.gguf_writer)667 self.gguf_writer.add_add_space_prefix(False)668 self.gguf_writer.add_add_bos_token(True)669 670 def set_gguf_parameters(self):671 super().set_gguf_parameters()672 673 num_kv_shared_layers = self.hparams["num_kv_shared_layers"]674 self.gguf_writer.add_shared_kv_layers(num_kv_shared_layers)675 676 # per-layer embedding is optional677 n_pl_embd = self.hparams.get("hidden_size_per_layer_input") or 0678 self.gguf_writer.add_embedding_length_per_layer_input(n_pl_embd)679 680 swa_layers = [t == "sliding_attention" for t in self.hparams["layer_types"]]681 self.gguf_writer.add_sliding_window_pattern(swa_layers)682 683 per_layer_config = self.hparams.get("per_layer_config")684 layer_types = self.hparams.get("layer_types", [])685 if (head_dim_full := self.hparams.get("global_head_dim")) is None and per_layer_config is not None:686 for layer_idx, layer_config in per_layer_config.items():687 layer_idx = int(layer_idx)688 if layer_idx < len(layer_types):689 if layer_types[layer_idx] == "full_attention" and "head_dim" in layer_config:690 head_dim_full = layer_config["head_dim"]691 break692 693 assert head_dim_full is not None694 695 head_dim_swa = self.hparams["head_dim"]696 # correct the head dim for global/swa layers697 self.gguf_writer.add_key_length(head_dim_full)698 self.gguf_writer.add_value_length(head_dim_full)699 self.gguf_writer.add_key_length_swa(head_dim_swa)700 self.gguf_writer.add_value_length_swa(head_dim_swa)701 702 expert_intermediate_size = self.find_hparam(["expert_intermediate_size", "moe_intermediate_size"])703 if expert_intermediate_size is not None:704 self.gguf_writer.add_expert_feed_forward_length(expert_intermediate_size)705 706 # if use_double_wide_mlp is set, we need to adjust the value for kv shared layers707 use_double_wide_mlp = self.hparams.get("use_double_wide_mlp", False)708 first_kv_shared_layer_idx = self.block_count - num_kv_shared_layers709 if use_double_wide_mlp:710 n_ff = self.hparams["intermediate_size"]711 n_ff_arr = [n_ff if il < first_kv_shared_layer_idx else n_ff * 2 for il in range(self.block_count)]712 self.gguf_writer.add_feed_forward_length(n_ff_arr)713 714 if (num_key_value_heads_full := self.hparams.get("num_global_key_value_heads")) is None and per_layer_config is not None:715 for layer_idx, layer_config in per_layer_config.items():716 layer_idx = int(layer_idx)717 if layer_idx < len(layer_types):718 if layer_types[layer_idx] == "full_attention" and "num_key_value_heads" in layer_config:719 num_key_value_heads_full = layer_config["num_key_value_heads"]720 break721 722 num_key_value_heads_swa = self.hparams.get("num_key_value_heads")723 if num_key_value_heads_full is not None and num_key_value_heads_swa is not None:724 value_arr = [num_key_value_heads_swa if is_swa else num_key_value_heads_full for is_swa in swa_layers]725 self.gguf_writer.add_head_count_kv(value_arr)726 727 # handle n_rot differently for global vs swa layers728 partial_rotary_factor_swa = self.rope_parameters.get("partial_rotary_factor", 1.0)729 n_rot_full = int(head_dim_full) # "proportional" is used, see generate_extra_tensors730 n_rot_swa = int(head_dim_swa * partial_rotary_factor_swa)731 self.gguf_writer.add_rope_dimension_count(n_rot_full)732 self.gguf_writer.add_rope_dimension_count_swa(n_rot_swa)733 734 def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:735 # full layer uses "proportional" rope with partial_rotary_factor=0.25736 # the expected ordering is cc000000ss000000 (c = cos, s = sin, 0 = unrotated),737 # but ggml neox only supports ccss000000000000, and we cannot rearrange the head because that will break use_alternative_attention738 # solution is to set specific freq_factors for the unrotated dims739 740 # IMPORTANT: this ROPE_FREQS tensor is ONLY used by the full_attention layers741 rope_params_full = self.hparams["rope_parameters"]["full_attention"]742 assert rope_params_full["rope_type"] == "proportional"743 744 per_layer_config = self.hparams.get("per_layer_config")745 if (head_dim_full := self.hparams.get("global_head_dim")) is None and per_layer_config is not None:746 layer_types = self.hparams.get("layer_types", [])747 for layer_idx, layer_config in per_layer_config.items():748 layer_idx = int(layer_idx)749 if layer_idx < len(layer_types):750 if layer_types[layer_idx] == "full_attention" and "head_dim" in layer_config:751 head_dim_full = layer_config["head_dim"]752 break753 754 assert head_dim_full is not None755 756 partial_rotary_factor_full = rope_params_full["partial_rotary_factor"]757 n_rot_full = int(head_dim_full * partial_rotary_factor_full / 2)758 n_unrot_full = int(head_dim_full / 2) - n_rot_full759 values = [1.0] * n_rot_full + [1e30] * n_unrot_full760 rope_freqs_full = torch.tensor(values, dtype=torch.float32)761 yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), rope_freqs_full)762 763 def _generate_nvfp4_tensors(self):764 # Gemma-4 stores a per-layer router.per_expert_scale ([n_expert]) that scales765 # each expert's contribution. It's mathematically equivalent to a per-expert766 # scalar on the down_proj output, which is exactly where ffn_down_exps_s is767 # applied at inference. Fold it into each expert's NVFP4 weight_scale_2 so the768 # existing NVFP4 path produces the right scales.769 n_experts = self.find_hparam(["num_local_experts", "num_experts"], optional=True) or 0770 for name in [n for n in self.model_tensors if n.endswith(".router.per_expert_scale")]:771 bid_match = re.search(r"\.layers\.(\d+)\.", name)772 if bid_match is None:773 continue774 bid = bid_match.group(1)775 prefix = name[: name.index(f".layers.{bid}.") + len(f".layers.{bid}.")]776 w2_targets = [f"{prefix}experts.{e}.down_proj.weight_scale_2" for e in range(n_experts)]777 present = [w2 in self.model_tensors for w2 in w2_targets]778 if not any(present):779 continue780 assert all(present), f"layer {bid}: partial NVFP4 quantization across experts"781 r = self.model_tensors.pop(name)782 for e, w2 in enumerate(w2_targets):783 s = self.model_tensors[w2]784 self.model_tensors[w2] = lambda s=s, r=r, i=e: s() * r()[i]785 super()._generate_nvfp4_tensors()786 787 @classmethod788 def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:789 name, gen = item790 791 if name.endswith("per_dim_scale") or name.endswith("layer_scalar"):792 name = name + ".weight"793 if ".experts." in name and not name.endswith((".weight", ".weight_scale", ".weight_scale_2", ".input_scale")):794 name += ".weight"795 796 return super().filter_tensors((name, gen))797 798 def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:799 if name.endswith("router.scale"):800 name = self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_INP, bid, ".scale")801 yield (name, data_torch)802 return803 if ".per_expert_scale" in name:804 # convert per-expert scale to FFN down scale805 name = self.format_tensor_name(gguf.MODEL_TENSOR.FFN_DOWN_EXP, bid, ".scale")806 yield (name, data_torch)807 return808 809 yield from super().modify_tensors(data_torch, name, bid)810 811 812@ModelBase.register("Gemma4UnifiedForConditionalGeneration")813@ModelBase.example("hf-tiny-v2/tiny-random-Gemma4UnifiedForConditionalGeneration")814class Gemma4UnifiedModel(Gemma4Model):815 model_arch = gguf.MODEL_ARCH.GEMMA4816 817 def _get_suppress_tokens(self) -> Sequence[int] | None:818 gen_cfg_path = self.dir_model / "generation_config.json"819 if gen_cfg_path.is_file():820 with open(gen_cfg_path, encoding="utf-8") as f:821 gen_cfg = json.load(f)822 return gen_cfg.get("suppress_tokens")823 return None824 825 def set_gguf_parameters(self):826 super().set_gguf_parameters()827 828 suppress_tokens = self._get_suppress_tokens()829 if suppress_tokens is not None:830 self.gguf_writer.add_suppress_tokens(suppress_tokens)831 832 833@ModelBase.register("Gemma4AssistantForCausalLM", "Gemma4UnifiedAssistantForCausalLM")834@ModelBase.example("google/gemma-4-31B-it-assistant", "google/gemma-4-26B-A4B-it-assistant", "google/gemma-4-E2B-it-assistant")835class Gemma4AssistantModel(Gemma4Model):836 model_arch = gguf.MODEL_ARCH.GEMMA4_ASSISTANT837 838 @classmethod839 def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:840 name, gen = item841 842 if "masked_embedding" in name:843 logger.debug(f"Skipping get tensor {name!r} in safetensors so that convert can end normally.")844 return None845 846 return super().filter_tensors(item)847 848 def set_gguf_parameters(self):849 super().set_gguf_parameters()850 self.gguf_writer.add_embedding_length_out(self.hparams["backbone_hidden_size"])851 self.gguf_writer.add_nextn_predict_layers(self.block_count)852 853 854@ModelBase.register("Gemma4ForConditionalGeneration")855@ModelBase.example("google/gemma-4-31B-it", "google/gemma-4-26B-A4B-it", "google/gemma-4-E2B-it")856class Gemma4VisionAudioModel(MmprojModel):857 has_audio_encoder = True858 has_vision_encoder = True859 860 def __init__(self, *args, **kwargs):861 super().__init__(*args, **kwargs)862 assert self.hparams_vision is not None863 self.hparams_vision["image_size"] = 224 # unused, but set to avoid error864 865 # remap audio hparams866 if self.hparams_audio:867 self.hparams_audio["feat_in"] = self.hparams_audio.get("input_feat_size", 128)868 if "hidden_size" in self.hparams_audio:869 self.hparams_audio["intermediate_size"] = self.hparams_audio["hidden_size"] * 4870 else:871 self.has_audio_encoder = False872 873 def set_gguf_parameters(self):874 super().set_gguf_parameters()875 876 # vision params877 assert self.hparams_vision is not None878 self.gguf_writer.add_clip_vision_projector_type(gguf.VisionProjectorType.GEMMA4V)879 self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams_vision.get("layer_norm_eps", 1e-6))880 881 # audio params882 if self.has_audio_encoder:883 assert self.hparams_audio is not None884 self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.GEMMA4A)885 self.gguf_writer.add_audio_num_mel_bins(self.hparams_audio["feat_in"])886 self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams_audio.get("layer_norm_eps", 1e-6))887 888 def is_audio_tensor(self, name: str) -> bool:889 return "audio_tower" in name or "embed_audio" in name890 891 def tensor_force_quant(self, name, new_name, bid, n_dims):892 if self.is_audio_tensor(name):893 if ".conv" in name or "_conv" in name and ".weight" in name:894 return gguf.GGMLQuantizationType.F32895 if "position_embedding_table" in name:896 return gguf.GGMLQuantizationType.F32897 return super().tensor_force_quant(name, new_name, bid, n_dims)898 899 def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:900 del bid # unused901 902 if len(data_torch.shape) == 0:903 # convert scalar tensors (input/output_mix/max) to 1D tensors904 data_torch = data_torch.unsqueeze(0)905 906 if self.is_audio_tensor(name):907 assert self.hparams_audio is not None908 name = name.replace("model.audio_tower.", "conformer.")909 name = name.replace(".linear.", ".")910 if name.endswith("per_dim_key_scale") or name.endswith("per_dim_scale"):911 name = name + ".weight"912 data_torch = torch.nn.functional.softplus(data_torch)913 if "lconv1d.depthwise_conv1d" in name and name.endswith(".weight"):914 assert data_torch.shape[1] == 1915 data_torch = data_torch.reshape(data_torch.shape[0], data_torch.shape[2])916 mapped_name = self.map_tensor_name(name, (".weight", ".bias", ".input_max", ".input_min", ".output_max", ".output_min"))917 yield (mapped_name, data_torch)918 919 else:920 name = name.replace("model.vision_tower.encoder.", "vision_model.model.")921 name = name.replace(".linear.weight", ".weight")922 if name.endswith("layer_scalar") or name.endswith("position_embedding_table"):923 name = name + ".weight"924 if name.endswith("patch_embedder.input_proj.weight"):925 n_embd, ksize_sq_c = data_torch.shape926 patch_size = int((ksize_sq_c // 3) ** 0.5)927 data_torch = data_torch.reshape(n_embd, patch_size, patch_size, 3)928 data_torch = data_torch.permute(0, 3, 1, 2).contiguous()929 mapped_name = self.map_tensor_name(name, (".weight", ".bias", ".input_max", ".input_min", ".output_max", ".output_min"))930 yield (mapped_name, data_torch)931 932 933@ModelBase.register("Gemma4UnifiedForConditionalGeneration")934@ModelBase.example("hf-tiny-v2/tiny-random-Gemma4UnifiedForConditionalGeneration")935class Gemma4UnifiedVisionAudioModel(Gemma4VisionAudioModel):936 has_audio_encoder = True937 has_vision_encoder = True938 939 def __init__(self, *args, **kwargs):940 super().__init__(*args, **kwargs)941 assert self.hparams_vision is not None942 assert self.hparams_audio is not None943 text_embd_dim = self.hparams_vision["mm_embed_dim"]944 self.hparams_vision["hidden_size"] = text_embd_dim945 self.hparams_audio["hidden_size"] = self.hparams_audio["audio_embed_dim"]946 # this is a transformer-less vision tower, the params below are redundant but set to avoid error947 self.hparams_vision["intermediate_size"] = 0948 self.hparams_vision["num_layers"] = 0949 self.hparams_vision["num_attention_heads"] = 0950 self.hparams_audio["intermediate_size"] = 0951 self.hparams_audio["num_layers"] = 0952 self.hparams_audio["num_attention_heads"] = 0953 954 def set_gguf_parameters(self):955 super().set_gguf_parameters()956 self.gguf_writer.add_clip_vision_projector_type(gguf.VisionProjectorType.GEMMA4UV)957 self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.GEMMA4UA)958 959 def modify_tensors(self, data_torch, name, bid):960 if name.endswith("pos_embedding"):961 name += ".weight"962 data_torch = data_torch.permute(1, 0, 2)963 elif ".pos_norm." in name:964 # rename to patch_ln3 to reuse the tensor name scheme965 name = name.replace(".pos_norm.", ".patch_ln3.")966 elif "patch_dense.weight" in name:967 # ggml im2col outputs in RR..GG..BB.. (CHW) order, but weight expects RGBRGB.. (HWC).968 # Permute columns so column i aligns with CHW input position i.969 assert self.hparams_vision is not None970 if "model_patch_size" in self.hparams_vision:971 p = self.hparams_vision["model_patch_size"]972 else:973 p = self.hparams_vision["patch_size"] * self.hparams_vision["pooling_kernel_size"]974 i = torch.arange(p * p * 3)975 ch = i // (p * p)976 row = (i % (p * p)) // p977 col = i % p978 # perm[i] = HWC column index for CHW position i979 perm = row * p * 3 + col * 3 + ch980 data_torch = data_torch[:, perm]981 elif "patch_ln1.weight" in name or "patch_ln1.bias" in name:982 # same permutation for patch_ln1 as patch_dense to align with CHW input order983 assert self.hparams_vision is not None984 if "model_patch_size" in self.hparams_vision:985 p = self.hparams_vision["model_patch_size"]986 else:987 p = self.hparams_vision["patch_size"] * self.hparams_vision["pooling_kernel_size"]988 i = torch.arange(p * p * 3)989 ch = i // (p * p)990 row = (i % (p * p)) // p991 col = i % p992 # perm[i] = HWC index for CHW position i993 perm = row * p * 3 + col * 3 + ch994 data_torch = data_torch[perm]995 return super().modify_tensors(data_torch, name, bid)996