Aluode/PerceptionLabPortable

1# Copyright 2024 The ggml.ai team and The HuggingFace Inc. team. and pygguf author (github.com/99991)2# https://github.com/99991/pygguf3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15 16import re17from typing import NamedTuple, Optional18 19import numpy as np20from tqdm.auto import tqdm21 22from .integrations import (23    GGUF_CONFIG_MAPPING,24    GGUF_TOKENIZER_MAPPING,25    _gguf_parse_value,26)27from .utils import is_torch_available28from .utils.import_utils import is_gguf_available29from .utils.logging import get_logger30 31 32if is_torch_available():33    import torch34 35logger = get_logger(__name__)36 37 38GGUF_TO_TRANSFORMERS_MAPPING = {39    "ignore": {40        "GGUF": {41            "version": "version",42            "tensor_count": "tensor_count",43            "kv_count": "kv_count",44        },45        "general": {"file_type": "file_type", "quantization_version": "quantization_version"},46    },47    "config": GGUF_CONFIG_MAPPING,48    "tokenizer": {"tokenizer": GGUF_TOKENIZER_MAPPING["tokenizer"]},49    "tokenizer_config": {"tokenizer": GGUF_TOKENIZER_MAPPING["tokenizer_config"]},50}51 52GGUF_SUPPORTED_ARCHITECTURES = list(GGUF_TO_TRANSFORMERS_MAPPING["config"].keys())53 54 55class GGUFTensor(NamedTuple):56    weights: np.ndarray57    name: str58    metadata: dict59 60 61class TensorProcessor:62    def __init__(self, config=None):63        self.config = config or {}64 65    def process(self, weights, name, **kwargs):66        return GGUFTensor(weights, name, {})67 68 69class LlamaTensorProcessor(TensorProcessor):70    def __init__(self, config=None):71        super().__init__(config=config)72 73    def process(self, weights, name, **kwargs):74        if ".attn_k." in name or ".attn_q." in name:75            num_heads = self.config.get("num_attention_heads")76            num_kv_heads = self.config.get("num_key_value_heads")77 78            if None in (num_heads, num_kv_heads):79                return GGUFTensor(weights, name, {})80            if ".attn_q." in name:81                weights = self._reverse_permute_weights(weights, num_heads, num_heads)82            elif ".attn_k." in name:83                weights = self._reverse_permute_weights(weights, num_heads, num_kv_heads)84        return GGUFTensor(weights, name, {})85 86    def _reverse_permute_weights(87        self, weights: np.ndarray, n_head: int, num_kv_heads: Optional[int] = None88    ) -> np.ndarray:89        # Original permutation implementation90        # https://github.com/ggerganov/llama.cpp/blob/a38b884c6c4b0c256583acfaaabdf556c62fabea/convert_hf_to_gguf.py#L1402-L140891        if num_kv_heads is not None and n_head != num_kv_heads:92            n_head = num_kv_heads93 94        dim = weights.shape[0] // n_head // 295        w = weights.reshape(n_head, dim, 2, *weights.shape[1:])96        return w.swapaxes(2, 1).reshape(weights.shape)97 98 99class Qwen2MoeTensorProcessor(TensorProcessor):100    def __init__(self, config=None):101        super().__init__(config=config)102 103    def process(self, weights, name, **kwargs):104        if "_exp" in name:105            tensor_key_mapping = kwargs.get("tensor_key_mapping")106            parsed_parameters = kwargs.get("parsed_parameters")107            if tensor_key_mapping:108                self._split_moe_expert_tensor(weights, parsed_parameters, name, tensor_key_mapping)109                return GGUFTensor(weights, None, {})110        if "ffn_gate_inp_shexp" in name:111            # for compatibility tensor shared_expert_gate must be (1, 2048) dim,112            # quantized one is (2048)113            weights = np.expand_dims(weights, axis=0)114        return GGUFTensor(weights, name, {})115 116    def _split_moe_expert_tensor(117        self, weights: np.ndarray, parsed_parameters: dict[str, dict], name: str, tensor_key_mapping: dict118    ):119        # Original merge implementation120        # https://github.com/ggerganov/llama.cpp/blob/master/convert_hf_to_gguf.py#L1994-L2022121        name = tensor_key_mapping[name]122        w_counter = self.config.get("num_experts", 60)123        for i in range(0, w_counter):124            temp_name = name.replace("mlp.experts.", f"mlp.experts.{i}.")125            exp_weight = weights[i]126            parsed_parameters["tensors"][temp_name] = torch.from_numpy(np.copy(exp_weight))127 128 129class BloomTensorProcessor(TensorProcessor):130    def __init__(self, config=None):131        super().__init__(config=config)132 133    def process(self, weights, name, **kwargs):134        if "attn_qkv" in name:135            num_heads = self.config["n_head"]136            n_embed = self.config["hidden_size"]137            if "weight" in name:138                weights = self._reverse_reshape_weights(weights, num_heads, n_embed)139            else:140                weights = self._reverse_reshape_bias(weights, num_heads, n_embed)141        return GGUFTensor(weights, name, {})142 143    def _reverse_reshape_weights(self, weights: np.ndarray, n_head: int, n_embed: int):144        # Original reshape implementation145        # https://github.com/ggerganov/llama.cpp/blob/master/convert_hf_to_gguf.py#L972-L985146        q, k, v = np.array_split(weights, 3, axis=0)147 148        q = q.reshape(n_head, n_embed // n_head, n_embed)149        k = k.reshape(n_head, n_embed // n_head, n_embed)150        v = v.reshape(n_head, n_embed // n_head, n_embed)151        qkv_weights = np.stack([q, k, v], axis=1)152 153        return qkv_weights.reshape(n_head * 3 * (n_embed // n_head), n_embed)154 155    def _reverse_reshape_bias(self, weights: np.ndarray, n_head: int, n_embed: int):156        # Original reshape implementation157        # https://github.com/ggerganov/llama.cpp/blob/master/convert_hf_to_gguf.py#L986-L998158        q_bias, k_bias, v_bias = np.array_split(weights, 3)159 160        q_bias = q_bias.reshape(n_head, n_embed // n_head)161        k_bias = k_bias.reshape(n_head, n_embed // n_head)162        v_bias = v_bias.reshape(n_head, n_embed // n_head)163 164        qkv_bias = np.stack([q_bias, k_bias, v_bias], axis=1).flatten()165        return qkv_bias166 167 168class T5TensorProcessor(TensorProcessor):169    def __init__(self, config=None):170        super().__init__(config=config)171 172    def process(self, weights, name, **kwargs):173        bid = None174        for chunk in name.split("."):175            if chunk.isdigit():176                bid = int(chunk)177                break178        return GGUFTensor(weights, name, {"bid": bid})179 180 181class GPT2TensorProcessor(TensorProcessor):182    def __init__(self, config=None):183        super().__init__(config=config)184 185    def process(self, weights, name, **kwargs):186        # Original transpose implementation187        # https://github.com/ggerganov/llama.cpp/blob/a38b884c6c4b0c256583acfaaabdf556c62fabea/convert_hf_to_gguf.py#L2060-L2061188        if (189            "attn_qkv.weight" in name190            or "ffn_down.weight" in name191            or "ffn_up.weight" in name192            or "attn_output.weight" in name193        ):194            weights = weights.T195 196        # Handle special case for output.weight197        if name == "output.weight":198            # output.weight has conflicts with attn_output.weight in name checking199            # Store the tensor directly and signal to skip further processing200            name = "lm_head.weight"201            parsed_parameters = kwargs.get("parsed_parameters", {})202            parsed_parameters["tensors"][name] = torch.from_numpy(np.copy(weights))203            name = None  # Signal to skip further processing204        return GGUFTensor(weights, name, {})205 206 207class MambaTensorProcessor(TensorProcessor):208    def __init__(self, config=None):209        super().__init__(config=config)210 211    def process(self, weights, name, **kwargs):212        if "ssm_conv1d.weight" in name:213            # for compatibility tensor ssm_conv1d must be (5120, 1, 4]) dim,214            # quantized one is (5120, 4)215            weights = np.expand_dims(weights, axis=1)216        if "ssm_a" in name:217            # Original exponential implementation218            # https://github.com/ggerganov/llama.cpp/blob/master/convert_hf_to_gguf.py#L2975-L2977219            weights = np.log(-weights)220        return GGUFTensor(weights, name, {})221 222 223class NemotronTensorProcessor(TensorProcessor):224    def __init__(self, config=None):225        super().__init__(config=config)226 227    # ref : https://github.com/ggerganov/llama.cpp/blob/master/convert_hf_to_gguf.py#L4666228    def process(self, weights, name, **kwargs):229        if "norm.weight" in name:230            weights = weights - 1231        return GGUFTensor(weights, name, {})232 233 234class Gemma2TensorProcessor(TensorProcessor):235    def __init__(self, config=None):236        super().__init__(config=config)237 238    # ref: https://github.com/ggerganov/llama.cpp/blob/d79d8f39b4da6deca4aea8bf130c6034c482b320/convert_hf_to_gguf.py#L3191239    # ref: https://github.com/huggingface/transformers/blob/fc37f38915372c15992b540dfcbbe00a916d4fc6/src/transformers/models/gemma/modeling_gemma.py#L89240    def process(self, weights, name, **kwargs):241        if "norm.weight" in name:242            weights = weights - 1243        return GGUFTensor(weights, name, {})244 245 246class Lfm2TensorProcessor(TensorProcessor):247    def __init__(self, config=None):248        super().__init__(config=config)249 250    def process(self, weights, name, **kwargs):251        if "shortconv.conv.weight" in name:252            ## GGUF shape is [hidden_dim, L_cache], HF expects [hidden_dim, 1, L_cache]253            weights = np.expand_dims(weights, axis=1)  ## equivalent to unsqueeze(1)254        return GGUFTensor(weights, name, {})255 256 257TENSOR_PROCESSORS = {258    "llama": LlamaTensorProcessor,259    "qwen2moe": Qwen2MoeTensorProcessor,260    "qwen3moe": Qwen2MoeTensorProcessor,261    "bloom": BloomTensorProcessor,262    "t5": T5TensorProcessor,263    "t5encoder": T5TensorProcessor,264    "gpt2": GPT2TensorProcessor,265    "mamba": MambaTensorProcessor,266    "nemotron": NemotronTensorProcessor,267    "gemma2": Gemma2TensorProcessor,268    "gemma3": Gemma2TensorProcessor,269    "lfm2": Lfm2TensorProcessor,270}271 272 273def read_field(reader, field):274    if field not in reader.fields:275        return []276    value = reader.fields[field]277    return [_gguf_parse_value(value.parts[_data_index], value.types) for _data_index in value.data]278 279 280# modified from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/model_executor/model_loader/loader.py#L1115-L1147281def get_gguf_hf_weights_map(282    hf_model,283    model_type: Optional[str] = None,284    num_layers: Optional[int] = None,285    qual_name: str = "",286):287    """288    GGUF uses this naming convention for their tensors from HF checkpoint:289    `blk.N.BB.weight` and `blk.N.BB.bias`290    where N signifies the block number of a layer, and BB signifies the291    attention/mlp layer components.292    See "Standardized tensor names" in293    https://github.com/ggerganov/ggml/blob/master/docs/gguf.md for details.294    """295    if is_gguf_available() and is_torch_available():296        from gguf import MODEL_ARCH_NAMES, get_tensor_name_map297    else:298        logger.error(299            "Loading a GGUF checkpoint in PyTorch, requires both PyTorch and GGUF>=0.10.0 to be installed. Please see "300            "https://pytorch.org/ and https://github.com/ggerganov/llama.cpp/tree/master/gguf-py for installation instructions."301        )302        raise ImportError("Please install torch and gguf>=0.10.0 to load a GGUF checkpoint in PyTorch.")303 304    model_type = hf_model.config.model_type if model_type is None else model_type305    num_layers = hf_model.config.num_hidden_layers if num_layers is None else num_layers306    # hack: ggufs have a different name for cohere307    if model_type == "cohere":308        model_type = "command-r"309    elif model_type == "qwen2_moe":310        model_type = "qwen2moe"311    elif model_type == "qwen3_moe":312        model_type = "qwen3moe"313    elif model_type == "gemma3_text":314        model_type = "gemma3"315    elif model_type == "umt5":316        model_type = "t5"317    arch = None318    for key, value in MODEL_ARCH_NAMES.items():319        if value == model_type:320            arch = key321            break322    if arch is None:323        raise NotImplementedError(324            f"Unknown gguf model_type: {model_type} in gguf-py. "325            "This might because you're using an outdated version of gguf-py package, "326            "you can install `gguf` package from source refer to "327            "https://github.com/ggerganov/llama.cpp/tree/master/gguf-py#development"328        )329    name_map = get_tensor_name_map(arch, num_layers)330 331    # Use a dummy conversion to get the mapping, because332    # hf => gguf and gguf => hf mappings are reversed333    gguf_to_hf_name_map = {}334    state_dict = hf_model.state_dict()335    for hf_name in state_dict:336        # An exception for qwen2moe/qwen3moe model, where the expert layers are packed337        if model_type in ("qwen2moe", "qwen3moe") and "mlp.experts." in hf_name:338            hf_name = re.sub(r"mlp.experts.\d+.", "mlp.experts.", hf_name)339 340        name, suffix = hf_name, ""341        if hf_name.endswith(".weight") or hf_name.endswith(".bias"):342            name, suffix = hf_name.rsplit(".", 1)343            suffix = "." + suffix344 345        gguf_name = name_map.get_name(name)346        if gguf_name is None:347            continue348 349        gguf_to_hf_name_map[gguf_name + suffix] = qual_name + hf_name350 351    # Some model like Bloom converted from BloomModel instead of BloomForCausalLM352    # Therefore, we need to check submodule as well to get a correct mapping353    if named_children := hf_model.named_children():354        for name, child in named_children:355            sub_map = get_gguf_hf_weights_map(child, model_type, num_layers, qual_name=f"{qual_name}{name}.")356            # Ignore the keys that are already in the main map to avoid overwriting357            sub_map = {k: v for k, v in sub_map.items() if k not in gguf_to_hf_name_map}358            gguf_to_hf_name_map.update(sub_map)359 360    return gguf_to_hf_name_map361 362 363def load_gguf_checkpoint(gguf_checkpoint_path, return_tensors=False, model_to_load=None):364    """365    Load a GGUF file and return a dictionary of parsed parameters containing tensors, the parsed366    tokenizer and config attributes.367 368    Args:369        gguf_checkpoint_path (`str`):370            The path the to GGUF file to load371        return_tensors (`bool`, defaults to `False`):372            Whether to read the tensors from the file and return them. Not doing so is faster373            and only loads the metadata in memory.374    """375    if is_gguf_available() and is_torch_available():376        from gguf import GGUFReader, dequantize377    else:378        logger.error(379            "Loading a GGUF checkpoint in PyTorch, requires both PyTorch and GGUF>=0.10.0 to be installed. Please see "380            "https://pytorch.org/ and https://github.com/ggerganov/llama.cpp/tree/master/gguf-py for installation instructions."381        )382        raise ImportError("Please install torch and gguf>=0.10.0 to load a GGUF checkpoint in PyTorch.")383 384    reader = GGUFReader(gguf_checkpoint_path)385    fields = reader.fields386    reader_keys = list(fields.keys())387 388    parsed_parameters = {k: {} for k in GGUF_TO_TRANSFORMERS_MAPPING}389 390    architecture = read_field(reader, "general.architecture")[0]391    # NOTE: Some GGUF checkpoints may miss `general.name` field in metadata392    model_name = read_field(reader, "general.name")393 394    updated_architecture = None395    # in llama.cpp mistral models use the same architecture as llama. We need396    # to add this patch to ensure things work correctly on our side.397    if "llama" in architecture and "mistral" in model_name:398        updated_architecture = "mistral"399    # FIXME: Currently this implementation is only for flan-t5 architecture.400    # It needs to be developed for supporting legacy t5.401    elif "t5" in architecture or "t5encoder" in architecture:402        parsed_parameters["config"]["is_gated_act"] = True403        if model_name and "umt5" in model_name[0].lower():404            updated_architecture = "umt5"405            if "t5encoder" in architecture:406                parsed_parameters["config"]["architectures"] = ["UMT5EncoderModel"]407        else:408            if "t5encoder" in architecture:409                parsed_parameters["config"]["architectures"] = ["T5EncoderModel"]410            updated_architecture = "t5"411    else:412        updated_architecture = architecture413 414    if "qwen2moe" in architecture:415        updated_architecture = "qwen2_moe"416    elif "qwen3moe" in architecture:417        updated_architecture = "qwen3_moe"418 419    # For stablelm architecture, we need to set qkv_bias and use_parallel_residual from tensors420    # If `qkv_bias=True`, qkv_proj with bias will be present in the tensors421    # If `use_parallel_residual=False`, ffn_norm will be present in the tensors422    if "stablelm" in architecture:423        attn_bias_name = {"attn_q.bias", "attn_k.bias", "attn_v.bias"}424        ffn_norm_name = "ffn_norm"425        qkv_bias = any(bias_name in tensor.name for tensor in reader.tensors for bias_name in attn_bias_name)426        use_parallel_residual = any(ffn_norm_name in tensor.name for tensor in reader.tensors)427        parsed_parameters["config"]["use_qkv_bias"] = qkv_bias428        parsed_parameters["config"]["use_parallel_residual"] = not use_parallel_residual429 430    if architecture not in GGUF_SUPPORTED_ARCHITECTURES and updated_architecture not in GGUF_SUPPORTED_ARCHITECTURES:431        raise ValueError(f"GGUF model with architecture {architecture} is not supported yet.")432 433    # Handle tie_word_embeddings, if lm_head.weight is not present in tensors,434    # tie_word_embeddings is true otherwise false435    exceptions = ["falcon", "bloom"]436    parsed_parameters["config"]["tie_word_embeddings"] = (437        all("output.weight" != tensor.name for tensor in reader.tensors) or architecture in exceptions438    )439 440    # List all key-value pairs in a columnized format441    for gguf_key, field in reader.fields.items():442        gguf_key = gguf_key.replace(architecture, updated_architecture)443        split = gguf_key.split(".")444        prefix = split[0]445        config_key = ".".join(split[1:])446 447        value = [_gguf_parse_value(field.parts[_data_index], field.types) for _data_index in field.data]448 449        if len(value) == 1:450            value = value[0]451 452        if isinstance(value, str) and architecture in value:453            value = value.replace(architecture, updated_architecture)454 455        for parameter, parameter_renames in GGUF_TO_TRANSFORMERS_MAPPING.items():456            if prefix in parameter_renames and config_key in parameter_renames[prefix]:457                renamed_config_key = parameter_renames[prefix][config_key]458                if renamed_config_key == -1:459                    continue460 461                if renamed_config_key is not None:462                    parsed_parameters[parameter][renamed_config_key] = value463 464                if gguf_key in reader_keys:465                    reader_keys.remove(gguf_key)466 467        if gguf_key in reader_keys:468            logger.info(f"Some keys were not parsed and added into account {gguf_key} | {value}")469 470    # Gemma3 GGUF checkpoint only contains weights of text backbone471    if parsed_parameters["config"]["model_type"] == "gemma3":472        parsed_parameters["config"]["model_type"] = "gemma3_text"473 474    if parsed_parameters["config"]["model_type"] == "lfm2":475        gguf_num_key_value_heads = parsed_parameters["config"]["num_key_value_heads"]476        # LFM2 GGUF checkpoint defines num_key_value_heads as a list of integers .e.g [0, 0, 8, 0, 0, 8, 0, 0, 8, 0, 8, 0, 8, 0, 8, 0] but we need to set it to the max value for HF477        parsed_parameters["config"]["num_key_value_heads"] = max(gguf_num_key_value_heads)478        ## we already read the correct intermediate_size from the GGUF checkpoint so we need to set block_auto_adjust_ff_dim to False479        parsed_parameters["config"]["block_auto_adjust_ff_dim"] = False480 481        ## llama.cpp defines the layers that are full-attention by looking at num_key_value_heads482        ## we need to set the full_attn_idxs to the layers that are full-attention483        parsed_parameters["config"]["full_attn_idxs"] = [484            i for i, num_kv_heads in enumerate(gguf_num_key_value_heads) if num_kv_heads > 0485        ]486 487    # retrieve config vocab_size from tokenizer488    # Please refer to https://github.com/huggingface/transformers/issues/32526 for more details489    if "vocab_size" not in parsed_parameters["config"]:490        tokenizer_parameters = parsed_parameters["tokenizer"]491        if "tokens" in tokenizer_parameters:492            parsed_parameters["config"]["vocab_size"] = len(tokenizer_parameters["tokens"])493        else:494            logger.warning(495                "Can't find a way to retrieve missing config vocab_size from tokenizer parameters. "496                "This will use default value from model config class and cause unexpected behavior."497            )498 499    if return_tensors:500        parsed_parameters["tensors"] = {}501 502        tensor_key_mapping = get_gguf_hf_weights_map(model_to_load)503        config = parsed_parameters.get("config", {})504 505        ProcessorClass = TENSOR_PROCESSORS.get(architecture, TensorProcessor)506        processor = ProcessorClass(config=config)507 508        for tensor in tqdm(reader.tensors, desc="Converting and de-quantizing GGUF tensors..."):509            name = tensor.name510            weights = dequantize(tensor.data, tensor.tensor_type)511 512            result = processor.process(513                weights=weights,514                name=name,515                tensor_key_mapping=tensor_key_mapping,516                parsed_parameters=parsed_parameters,517            )518 519            weights = result.weights520            name = result.name521 522            if name not in tensor_key_mapping:523                continue524 525            name = tensor_key_mapping[name]526 527            parsed_parameters["tensors"][name] = torch.from_numpy(np.copy(weights))528 529    if len(reader_keys) > 0:530        logger.info(f"Some keys of the GGUF file were not considered: {reader_keys}")531 532    return parsed_parameters533