Felipe97/llama-cpp-compiled

1from __future__ import annotations2 3import json4import re5 6from typing import Any, Callable, Iterable, TYPE_CHECKING7 8import torch9 10if TYPE_CHECKING:11    from torch import Tensor12 13from .base import MmprojModel, ModelBase, TextModel, gguf14 15 16@ModelBase.register("MiMoV2FlashForCausalLM", "MiMoV2ForCausalLM")17@ModelBase.example("XiaomiMiMo/MiMo-V2.5")18class MimoV2Model(TextModel):19    model_arch = gguf.MODEL_ARCH.MIMO220 21    # MiMo V2-Flash, V2.5 and V2.5-Pro all ship 3 trained MTP layers under model.mtp.layers.{0,1,2}.22    # The HF config does not expose the count, so it's hardcoded to match the count found in the safetensors.23    _n_nextn = 324 25    def __init__(self, *args, **kwargs):26        super().__init__(*args, **kwargs)27 28        self.block_count = self.hparams["num_hidden_layers"] + self._n_nextn29        self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)30 31    @staticmethod32    def _tp_aware_qkv_dequant(weight: Tensor, scale_inv: Tensor,33                              n_q: int, n_kv: int, hd: int, vhd: int,34                              bs: int = 128) -> Tensor:35        # MiMo-V2.5 (TP=4) and V2.5-Pro (TP=8) ship qkv_proj sharded across TP36        # ranks; per rank, rows are stacked as [Q_per | K_per | V_per].37        # weight_scale_inv has ceil(rows_per_rank/bs) block-rows per rank (last38        # may extend past rows_per_rank with phantom rows not in the weight).39        # Naive repeat_interleave aligns rank 0 only and mis-applies scales to40        # later ranks once rows_per_rank isn't a multiple of bs.41        # Re-group the per-rank [Q_per|K_per|V_per] rows into a single fused42        # [Q | K | V] tensor matching the un-sharded original layout.43        q_size = n_q * hd44        k_size = n_kv * hd45        v_size = n_kv * vhd46        total_rows = q_size + k_size + v_size47        if weight.shape[0] != total_rows:48            raise ValueError(f"qkv_proj weight rows {weight.shape[0]} != q+k+v {total_rows}")49 50        # detect TP from scale_inv block count, descending order so larger matches first51        tp = None52        for cand in (8, 4):53            if total_rows % cand != 0:54                continue55            rpr = total_rows // cand56            bpr = (rpr + bs - 1) // bs57            if scale_inv.shape[0] == cand * bpr:58                tp = cand59                break60        if tp is None:61            raise ValueError(62                f"qkv_proj: cannot detect TP - scale_inv rows {scale_inv.shape[0]}, "63                f"q+k+v {total_rows}")64 65        q_per = q_size // tp66        k_per = k_size // tp67        v_per = v_size // tp68        rows_per_rank = q_per + k_per + v_per69        blocks_per_rank = (rows_per_rank + bs - 1) // bs70 71        scale_inv = scale_inv.float()72        # per-row scale-row index: rank * blocks_per_rank + (rr_in_rank // bs)73        row_idx = torch.arange(total_rows)74        rr = row_idx % rows_per_rank75        rank = row_idx // rows_per_rank76        scale_row_idx = rank * blocks_per_rank + (rr // bs)77        # gather: (total_rows, n_col_blocks)78        scale_per_row_block = scale_inv[scale_row_idx]79        # expand col-blocks -> cols: each block-col covers `bs` weight cols80        scale_full = scale_per_row_block.repeat_interleave(bs, dim=1)81        # crop to weight col count (in case last col-block isn't full)82        scale_full = scale_full[:, : weight.shape[1]]83        dequant = weight.float() * scale_full84 85        if tp == 1:86            return dequant87 88        # Re-group per-rank [Q_per|K_per|V_per] rows into unified [Q | K | V]89        qs, ks, vs = [], [], []90        for r in range(tp):91            base = r * rows_per_rank92            qs.append(dequant[base : base + q_per])93            ks.append(dequant[base + q_per : base + q_per + k_per])94            vs.append(dequant[base + q_per + k_per : base + rows_per_rank])95        return torch.cat(qs + ks + vs, dim=0)96 97    def dequant_model(self):98        # Capture raw FP8 (weight, scale_inv) lambdas for qkv_proj BEFORE super99        # rewrites them with the existing dequant. Replace super's lambda after100        # it runs so scale_inv removal still happens via the standard path.101        qkv_overrides: dict[str, tuple[Callable, Callable, int]] = {}102        qc = self.hparams.get("quantization_config")103        if isinstance(qc, dict) and qc.get("quant_method") == "fp8":104            pat = re.compile(r"^model\.layers\.(\d+)\.self_attn\.qkv_proj\.weight_scale_inv$")105            for name in list(self.model_tensors.keys()):106                m = pat.match(name)107                if not m:108                    continue109                weight_name = name.removesuffix("_scale_inv")110                if weight_name not in self.model_tensors:111                    continue112                qkv_overrides[weight_name] = (113                    self.model_tensors[weight_name],114                    self.model_tensors[name],115                    int(m.group(1)),116                )117 118        super().dequant_model()119 120        if not qkv_overrides:121            return122 123        n_q = self.hparams["num_attention_heads"]124        hd = self.hparams["head_dim"]125        vhd = self.hparams["v_head_dim"]126        hybrid = self.hparams["hybrid_layer_pattern"]127        n_layer_text = self.hparams["num_hidden_layers"]128        for weight_name, (w_fn, s_fn, bid) in qkv_overrides.items():129            # MTP layers (bid >= n_layer_text) use SWA-style attention dims130            is_swa = True if bid >= n_layer_text else hybrid[bid] == 1131            n_kv = self.hparams["swa_num_key_value_heads" if is_swa else "num_key_value_heads"]132            self.model_tensors[weight_name] = (133                lambda w_fn=w_fn, s_fn=s_fn, n_q=n_q, n_kv=n_kv, hd=hd, vhd=vhd:134                    MimoV2Model._tp_aware_qkv_dequant(w_fn(), s_fn(), n_q, n_kv, hd, vhd)135            )136 137    def set_gguf_parameters(self):138        super().set_gguf_parameters()139 140        assert self.hparams["swa_head_dim"] == self.hparams["head_dim"]141        assert self.hparams["swa_num_attention_heads"] == self.hparams["num_attention_heads"]142        assert self.hparams["swa_v_head_dim"] == self.hparams["v_head_dim"]143        assert self.hparams["topk_method"] == "noaux_tc"144 145        n_head_kv = self.hparams["num_key_value_heads"]146        n_head_kv_swa = self.hparams["swa_num_key_value_heads"]147        # Extend the per-layer pattern with SWA entries for the MTP blocks so the148        # runtime arrays (sized to extended block_count) are fully populated.149        hybrid = list(self.hparams["hybrid_layer_pattern"]) + [1] * self._n_nextn150        n_head_kv_arr = [n_head_kv_swa if use_swa == 1 else n_head_kv for use_swa in hybrid]151        self.gguf_writer.add_head_count_kv(n_head_kv_arr)152 153        self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])154        self.gguf_writer.add_sliding_window_pattern(hybrid)155        self.gguf_writer.add_value_length(self.hparams["v_head_dim"])156        self.gguf_writer.add_expert_count(self.hparams["n_routed_experts"])157        self.gguf_writer.add_expert_feed_forward_length(self.hparams["moe_intermediate_size"])158 159        rope_dim = int(self.hparams["head_dim"] * self.rope_parameters["partial_rotary_factor"])160        self.gguf_writer.add_rope_dimension_count(rope_dim)161 162        self.gguf_writer.add_layer_norm_rms_eps(self.hparams.get("layernorm_epsilon", 1e-5))163 164        v_scale = self.hparams.get("attention_value_scale")165        if v_scale is not None:166            self.gguf_writer.add_attn_value_scale(float(v_scale))167 168        self.gguf_writer.add_nextn_predict_layers(self._n_nextn)169 170    _experts: list[dict[str, Tensor]] | None = None171 172    @classmethod173    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:174        name, gen = item175 176        if "attention_sink" in name and not name.endswith(".weight"):177            name += ".weight"178 179        return super().filter_tensors((name, gen))180 181    def modify_tensors(self, data_torch, name, bid):182        # Remap MTP/NextN tensors to additional layer slots so the standard tensor map handles them.183        # HF: model.mtp.layers.{i}.foo  ->  model.layers.{n_layer_text + i}.foo184        m = re.match(r"^model\.mtp\.layers\.(\d+)\.(.*)$", name)185        if m is not None:186            mtp_idx = int(m.group(1))187            assert mtp_idx < self._n_nextn, f"MTP layer index {mtp_idx} >= _n_nextn ({self._n_nextn})"188            rest = m.group(2)189            n_layer_text = self.hparams["num_hidden_layers"]190            new_bid = n_layer_text + mtp_idx191            name = f"model.layers.{new_bid}.{rest}"192            bid = new_bid193 194        # process the experts separately195        if name.find("mlp.experts") != -1:196            n_experts = self.hparams["n_routed_experts"]197            assert bid is not None198 199            if self._experts is None:200                self._experts = [{} for _ in range(self.block_count)]201 202            self._experts[bid][name] = data_torch203 204            if len(self._experts[bid]) >= n_experts * 3:205                # merge the experts into a single 3d tensor206                for w_name in ["gate_proj", "up_proj", "down_proj"]:207                    datas: list[Tensor] = []208 209                    for xid in range(n_experts):210                        ename_to_retrieve = f"model.layers.{bid}.mlp.experts.{xid}.{w_name}.weight"211                        datas.append(self._experts[bid][ename_to_retrieve])212                        del self._experts[bid][ename_to_retrieve]213 214                    data_torch = torch.stack(datas, dim=0)215                    merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"216 217                    yield from super().modify_tensors(data_torch, merged_name, bid)218                return219            else:220                return221        yield from super().modify_tensors(data_torch, name, bid)222 223    def prepare_tensors(self):224        super().prepare_tensors()225 226        if self._experts is not None:227            # flatten `list[dict[str, Tensor]]` into `list[str]`228            experts = [k for d in self._experts for k in d.keys()]229            if len(experts) > 0:230                raise ValueError(f"Unprocessed experts: {experts}")231 232 233@ModelBase.register("MiMoV2ForCausalLM")234@ModelBase.example("XiaomiMiMo/MiMo-V2.5")235class MiMoV2VisionAudioModel(MmprojModel):236    has_audio_encoder = True237 238    _audio_tok_hparams: dict[str, Any] | None = None239    _rvq_codebook_sizes: list[int] | None = None240    _code_embd: dict[int, Tensor] | None = None241 242    def __init__(self, *args, **kwargs):243        super().__init__(*args, **kwargs)244        assert self.hparams_vision is not None245        hp = self.hparams_vision246 247        hp["image_size"] = hp.get("image_size", 560)248        hp["num_attention_heads"] = hp.get("num_heads", 32)249        hp["num_hidden_layers"] = hp.get("depth", 28)250 251        self.n_q_heads = int(hp["num_heads"])252        self.num_kv_heads = int(hp.get("num_key_value_heads", 8))253        self.head_dim = int(hp.get("qk_channels", 64))254        self.spatial_merge_size = int(hp["spatial_merge_size"])255        # MiMoV2 vision RMSNorm: HF uses getattr(config, "rms_norm_eps", 1e-6) and the256        # field is absent from MiMo-V2.5's vision_config257        self.rms_norm_eps = float(hp.get("rms_norm_eps", 1e-6))258 259        # fullatt_block_indexes are also reflected in vit_window_attn_types as -1260        self.fullatt_block_indexes = list(hp.get("fullatt_block_indexes") or [])261        self.vit_window_attn_types = list(hp.get("vit_window_attn_types") or [])262        self.visual_token_window_size = int(hp.get("visual_token_window_size", -1))263        self.use_sink = bool(hp.get("use_sink", False))264 265    def get_audio_config(self) -> dict[str, Any] | None:266        if self._audio_tok_hparams is None:267            path = self.dir_model / "audio_tokenizer" / "config.json"268            with open(path, "r", encoding="utf-8") as f:269                cfg = json.load(f)270            # aliases so MmprojModel.find_aparam() / n_block_keys can resolve them271            cfg["hidden_size"] = cfg["d_model"]272            cfg["intermediate_size"] = cfg["encoder_ffn_dim"]273            cfg["num_attention_heads"] = cfg["encoder_attention_heads"]274            self._audio_tok_hparams = cfg275        return self._audio_tok_hparams276 277    def set_gguf_parameters(self):278        super().set_gguf_parameters()279 280        self.gguf_writer.add_clip_vision_projector_type(gguf.VisionProjectorType.MIMOVL)281        self.gguf_writer.add_vision_use_silu(True)282        self.gguf_writer.add_vision_head_count_kv(self.num_kv_heads)283        self.gguf_writer.add_vision_spatial_merge_size(self.spatial_merge_size)284        self.gguf_writer.add_uint32(gguf.Keys.ClipVision.WINDOW_SIZE, self.visual_token_window_size)285        self.gguf_writer.add_vision_wa_pattern_mode(self.vit_window_attn_types)286        self.gguf_writer.add_vision_attention_layernorm_eps(self.rms_norm_eps)287        self.gguf_writer.add_vision_min_pixels(int(self.preprocessor_config["min_pixels"]))288        self.gguf_writer.add_vision_max_pixels(int(self.preprocessor_config["max_pixels"]))289 290        assert self.hparams_audio is not None291        self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.MIMO_AUDIO)292        self.gguf_writer.add_audio_num_mel_bins(self.hparams_audio["n_mels"])293        self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams_audio.get("layer_norm_eps", 1e-5))294 295        assert self._rvq_codebook_sizes is not None296        self.gguf_writer.add_audio_rvq_num_quantizers(len(self._rvq_codebook_sizes))297        self.gguf_writer.add_audio_rvq_codebook_size(self._rvq_codebook_sizes)298 299        n_layer = self.hparams_audio["encoder_layers"]300        swa_per_block = self.hparams_audio.get("swa_per_block", 1)301        if self.hparams_audio.get("hybrid_attention") and swa_per_block > 1:302            wa_pattern = [0 if i % swa_per_block < swa_per_block - 1 else -1 for i in range(n_layer)]303        else:304            wa_pattern = [-1] * n_layer305        self.gguf_writer.add_audio_wa_pattern_mode(wa_pattern)306        self.gguf_writer.add_audio_window_size(int(self.hparams_audio["encoder_attn_window_size"][0]))307 308        audio_cfg = self.global_config["audio_config"]309        self.gguf_writer.add_audio_local_block_count(int(audio_cfg["input_local_layers"]))310        self.gguf_writer.add_audio_local_group_size(int(audio_cfg["group_size"]))311 312    def tensor_force_quant(self, name, new_name, bid, n_dims):313        # for audio encoder: keep codebook in F32314        if new_name in (315            gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.A_ENC_RVQ_CODEBOOK] + ".weight",316            gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.A_MM_CODE_EMBD] + ".weight",317        ):318            return gguf.GGMLQuantizationType.F32319        if ("encoder.conv" in name or "encoder.down_sample_layer" in name) and name.endswith(".weight"):320            return gguf.GGMLQuantizationType.F32321        return super().tensor_force_quant(name, new_name, bid, n_dims)322 323    @classmethod324    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:325        name, _ = item326        if name.startswith("visual.") or name.startswith("speech_embeddings.") or name.startswith("audio_encoder."):327            return super().filter_tensors(item)328        return None329 330    def modify_tensors(self, data_torch, name, bid):331        # Conv3D patch embed: split along the temporal axis (kt=2) into two Conv2D332        # weights that the existing qwen2vl-style two-Conv2D path consumes.333        if name == "visual.patch_embed.proj.weight":334            _, _, kt, _, _ = data_torch.shape335            if kt != 2:336                raise ValueError(f"unexpected temporal_patch_size: {kt}")337            embd_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH]338            yield (embd_name + ".weight",   data_torch[:, :, 0, ...])339            yield (embd_name + ".weight.1", data_torch[:, :, 1, ...])340            return341 342        if m := re.match(r"^speech_embeddings\.(\d+)\.weight$", name):343            if self._code_embd is None:344                self._code_embd = {}345            self._code_embd[int(m.group(1))] = data_torch346 347            n_channels = int(self.global_config["audio_config"]["audio_channels"])348            if len(self._code_embd) < n_channels:349                return350            merged = torch.stack([self._code_embd.pop(i) for i in range(n_channels)], dim=0)351            yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_MM_CODE_EMBD), merged)352            return353 354        if "conv1.bias" in name or "conv2.bias" in name:355            # transpose conv1/conv2 bias so it broadcasts against [n_frames, C_out, 1]356            data_torch = data_torch.unsqueeze(-1)357 358        if name == "audio_encoder.projection.mlp.0.weight":359            yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_MMPROJ, 1), data_torch)360            return361        if name == "audio_encoder.projection.mlp.2.weight":362            yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_MMPROJ, 2), data_torch)363            return364 365        yield from super().modify_tensors(data_torch, name, bid)366 367    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:368        # note: audio encoder is in its own subdir "audio_tokenizer"369        from safetensors.torch import load_file370 371        tok_dir = self.dir_model / "audio_tokenizer"372        state_dict = load_file(tok_dir / "model.safetensors")373 374        codebook_re = re.compile(r"^encoder\.quantizer\.vq\.layers\.(\d+)\._codebook\.embed$")375        codebooks: dict[int, Tensor] = {}376 377        # EMA/training-only RVQ buffers - not needed for inference (nearest-codebook378        # lookup only reads "_codebook.embed")379        skip_suffixes = (380            "_codebook.cluster_size",381            "_codebook.embed_avg",382            "_codebook.inited",383        )384        for name, tensor in state_dict.items():385            if name.endswith(skip_suffixes):386                continue387            if m := codebook_re.match(name):388                codebooks[int(m.group(1))] = tensor389                continue390            yield name, tensor391 392        # gather codebooks and merge into 3D tensor, similar to MoE MLP tensors393        n_q = len(codebooks)394        ordered = [codebooks[i] for i in range(n_q)]395        self._rvq_codebook_sizes = [int(cb.shape[0]) for cb in ordered]396        max_bins = max(self._rvq_codebook_sizes)397        dim = ordered[0].shape[1]398        merged = ordered[0].new_zeros(n_q, max_bins, dim)399        for i, cb in enumerate(ordered):400            merged[i, : cb.shape[0], :] = cb401 402        yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_ENC_RVQ_CODEBOOK), merged)403