Felipe97/llama-cpp-compiled

1from __future__ import annotations2 3from typing import Any, Callable, Iterable, TYPE_CHECKING4 5if TYPE_CHECKING:6    from torch import Tensor7 8from .base import MmprojModel, ModelBase, TextModel, gguf9 10 11@ModelBase.register("UltravoxModel")12@ModelBase.example("fixie-ai/ultravox-v0_5-llama-3_2-1b")13class UltravoxModel(TextModel):14    model_arch = gguf.MODEL_ARCH.LLAMA # dummy15 16    def __init__(self, *args, **kwargs):17        super().__init__(*args, **kwargs)18        raise NotImplementedError("Ultravox does not have text decoder. Instead, it uses Llama or other models for text. If you want to get the audio encoder, please use --mmproj argument")19 20 21@ModelBase.register("GlmasrModel")22@ModelBase.example("zai-org/GLM-ASR-Nano-2512")23class GlmASRWhisperEncoderModel(MmprojModel):24    has_vision_encoder = False25    has_audio_encoder = True26 27    def __init__(self, *args, **kwargs):28        super().__init__(*args, **kwargs)29        if "hidden_size" not in self.hparams and "intermediate_size" not in self.hparams:30            self.hparams["hidden_size"] = self.hparams["d_model"]31            self.hparams["intermediate_size"] = self.hparams["encoder_ffn_dim"]32            self.hparams["num_attention_heads"] = self.hparams["encoder_attention_heads"]33 34    def set_gguf_parameters(self):35        super().set_gguf_parameters()36        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.GLMA)37        self.gguf_writer.add_audio_num_mel_bins(self.hparams["num_mel_bins"])38        self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams.get("layer_norm_eps", 1e-5))39        self.gguf_writer.add_audio_stack_factor(self.global_config["merge_factor"])40 41    def tensor_force_quant(self, name, new_name, bid, n_dims):42        if ".conv" in name and ".weight" in name:43            return gguf.GGMLQuantizationType.F1644        return super().tensor_force_quant(name, new_name, bid, n_dims)45 46    @classmethod47    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:48        name, gen = item49 50        if name.startswith(("model.", "lm_head.")):51            # skip language model tensors52            return None53 54        if name.startswith("audio_encoder.whisper."):55            name = name.replace("audio_encoder.whisper.","audio_tower.")56        if "audio_encoder.layer_norm." in name or "audio_encoder.proj." in name:57            name = name.replace("audio_encoder.", "audio_encoder.adapting.")58        if name.startswith("audio_encoder.adapting."):59            name = name.replace("audio_encoder.adapting.","audio.multi_modal_projector.")60            if ".layer_norm." in name:61                name = name.replace(".layer_norm.", ".ln_pre.")62            if ".0." in name:63                name = name.replace(".0.", ".linear_1.")64            if ".2." in name:65                name = name.replace(".2.", ".linear_2.")66 67        return super().filter_tensors((name, gen))68 69    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:70        if name.startswith("audio_encoder.audio_bos_eos_token."):71            yield from super().modify_tensors(data_torch[0], "model.vision.boi", bid)72            yield from super().modify_tensors(data_torch[1], "model.vision.eoi", bid)73            return74 75        if name.startswith("audio_encoder.adapting."):76            if ".proj." in name:77                return78 79        if "conv1.bias" in name or "conv2.bias" in name:80            # transpose conv1 and conv2 bias81            data_torch = data_torch.unsqueeze(-1)82 83        yield from super().modify_tensors(data_torch, name, bid)84 85 86@ModelBase.register("Qwen2AudioForConditionalGeneration")87@ModelBase.example("Qwen/Qwen2-Audio-7B-Instruct")88class WhisperEncoderModel(MmprojModel):89    has_vision_encoder = False # no vision encoder90    has_audio_encoder = True91 92    def __init__(self, *args, **kwargs):93        super().__init__(*args, **kwargs)94        if "hidden_size" not in self.hparams and "intermediate_size" not in self.hparams:95            self.hparams["hidden_size"] = self.hparams["d_model"]96            self.hparams["intermediate_size"] = self.hparams["encoder_ffn_dim"]97            self.hparams["num_attention_heads"] = self.hparams["encoder_attention_heads"]98 99    def set_gguf_parameters(self):100        super().set_gguf_parameters()101        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.QWEN2A)102        self.gguf_writer.add_audio_num_mel_bins(self.hparams["num_mel_bins"])103        self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams.get("layer_norm_eps", 1e-5))104 105    def tensor_force_quant(self, name, new_name, bid, n_dims):106        if ".conv" in name and ".weight" in name:107            return gguf.GGMLQuantizationType.F16108        return super().tensor_force_quant(name, new_name, bid, n_dims)109 110    @classmethod111    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:112        name, gen = item113 114        # prevent clash naming with vision tensors115        if name.startswith("multi_modal_projector"):116            name = "audio." + name117 118        return super().filter_tensors((name, gen))119 120    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:121        if "conv1.bias" in name or "conv2.bias" in name:122            # transpose conv1 and conv2 bias123            data_torch = data_torch.unsqueeze(-1)124 125        yield from super().modify_tensors(data_torch, name, bid)126 127 128@ModelBase.register("UltravoxModel")129@ModelBase.example("fixie-ai/ultravox-v0_5-llama-3_2-1b")130class UltravoxWhisperEncoderModel(WhisperEncoderModel):131    has_vision_encoder = False # no vision encoder132    has_audio_encoder = True133 134    def set_gguf_parameters(self):135        super().set_gguf_parameters()136        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.ULTRAVOX)137        self.gguf_writer.add_audio_stack_factor(self.global_config["stack_factor"])138 139 140@ModelBase.register("MERaLiON2ForConditionalGeneration")141@ModelBase.example("MERaLiON/MERaLiON-2-3B")142class MERaLiONWhisperEncoderModel(WhisperEncoderModel):143    has_vision_encoder = False144    has_audio_encoder = True145 146    def get_audio_config(self) -> dict[str, Any] | None:147        return self.global_config.get("speech_config")148 149    def set_gguf_parameters(self):150        super().set_gguf_parameters()151        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MERALION)152        self.gguf_writer.add_audio_stack_factor(self.global_config.get("speech_mlp_scale_factor", 15))153 154    @classmethod155    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:156        name, gen = item157 158        if name.startswith("text_decoder."):159            return None160 161        if name.startswith("speech_encoder."):162            name = name.replace("speech_encoder.", "audio_tower.")163 164        return super().filter_tensors((name, gen))165 166    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:167        suffix = "." + name.rsplit(".", 1)[-1]168 169        if name.startswith("ln_speech."):170            yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_MM_NORM_PRE, suffix=suffix), data_torch)171            return172 173        if name.startswith("speech_audio_adapter."):174            if ".mlp_adapter.0." in name:175                yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_MMPROJ, 0, suffix=suffix), data_torch)176            elif ".gate_proj." in name:177                yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_MMPROJ, 1, suffix=suffix), data_torch)178            elif ".pool_proj." in name:179                yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_MMPROJ, 2, suffix=suffix), data_torch)180            elif ".out_proj." in name:181                yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_MMPROJ, 3, suffix=suffix), data_torch)182            return183 184        yield from super().modify_tensors(data_torch, name, bid)185 186 187@ModelBase.register("VoxtralForConditionalGeneration")188@ModelBase.example("mistralai/Voxtral-Mini-3B-2507")189class VoxtralWhisperEncoderModel(WhisperEncoderModel):190    has_vision_encoder = False # no vision encoder191    has_audio_encoder = True192 193    def set_gguf_parameters(self):194        super().set_gguf_parameters()195        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.VOXTRAL)196        self.gguf_writer.add_audio_stack_factor(4) # == intermediate_size // hidden_size197 198 199@ModelBase.register("AudioFlamingo3ForConditionalGeneration")200@ModelBase.example("nvidia/audio-flamingo-3-hf")201class AudioFlamingo3WhisperEncoderModel(WhisperEncoderModel):202    def set_gguf_parameters(self):203        super().set_gguf_parameters()204        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MUSIC_FLAMINGO)205 206    def tensor_force_quant(self, name, new_name, bid, n_dims):207        if ".conv" in name and ".weight" in name:208            # Was trained in BF16, being safe, avoiding quantizing to FP16209            return gguf.GGMLQuantizationType.F32210        return super().tensor_force_quant(name, new_name, bid, n_dims)211