Aluode/PerceptionLabPortable
0
1#!/usr/bin/env python2# coding=utf-83 4# Copyright 2023 The HuggingFace Inc. team. All rights reserved.5# Modifications Copyright (C) 2025, Advanced Micro Devices, Inc. All rights reserved.6#7# Licensed under the Apache License, Version 2.0 (the "License");8# you may not use this file except in compliance with the License.9# You may obtain a copy of the License at10#11# http://www.apache.org/licenses/LICENSE-2.012#13# Unless required by applicable law or agreed to in writing, software14# distributed under the License is distributed on an "AS IS" BASIS,15# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.16# See the License for the specific language governing permissions and17# limitations under the License.18import copy19import dataclasses20import importlib.metadata21import json22import os23from dataclasses import dataclass, is_dataclass24from enum import Enum25from inspect import Parameter, signature26from typing import Any, Optional, Union27 28from packaging import version29 30from ..utils import (31 is_auto_awq_available,32 is_compressed_tensors_available,33 is_gptqmodel_available,34 is_hqq_available,35 is_quark_available,36 is_torch_available,37 is_torchao_available,38 logging,39)40from .import_utils import is_auto_gptq_available41 42 43if is_torch_available():44 import torch45 46logger = logging.get_logger(__name__)47 48 49class QuantizationMethod(str, Enum):50 BITS_AND_BYTES = "bitsandbytes"51 GPTQ = "gptq"52 AWQ = "awq"53 AQLM = "aqlm"54 VPTQ = "vptq"55 QUANTO = "quanto"56 EETQ = "eetq"57 HIGGS = "higgs"58 HQQ = "hqq"59 COMPRESSED_TENSORS = "compressed-tensors"60 FBGEMM_FP8 = "fbgemm_fp8"61 TORCHAO = "torchao"62 BITNET = "bitnet"63 SPQR = "spqr"64 FP8 = "fp8"65 QUARK = "quark"66 FPQUANT = "fp_quant"67 AUTOROUND = "auto-round"68 MXFP4 = "mxfp4"69 70 71class AWQLinearVersion(str, Enum):72 GEMM = "gemm"73 GEMV = "gemv"74 EXLLAMA = "exllama"75 IPEX = "ipex"76 77 @staticmethod78 def from_str(version: str):79 version = version.lower()80 if version == "gemm":81 return AWQLinearVersion.GEMM82 elif version == "gemv":83 return AWQLinearVersion.GEMV84 elif version == "exllama":85 return AWQLinearVersion.EXLLAMA86 elif version == "ipex":87 return AWQLinearVersion.IPEX88 else:89 raise ValueError(f"Unknown AWQLinearVersion {version}")90 91 92class AwqBackendPackingMethod(str, Enum):93 AUTOAWQ = "autoawq"94 LLMAWQ = "llm-awq"95 96 97@dataclass98class QuantizationConfigMixin:99 """100 Mixin class for quantization config101 """102 103 quant_method: QuantizationMethod104 105 @classmethod106 def from_dict(cls, config_dict, return_unused_kwargs=False, **kwargs):107 """108 Instantiates a [`QuantizationConfigMixin`] from a Python dictionary of parameters.109 110 Args:111 config_dict (`dict[str, Any]`):112 Dictionary that will be used to instantiate the configuration object.113 return_unused_kwargs (`bool`,*optional*, defaults to `False`):114 Whether or not to return a list of unused keyword arguments. Used for `from_pretrained` method in115 `PreTrainedModel`.116 kwargs (`dict[str, Any]`):117 Additional parameters from which to initialize the configuration object.118 119 Returns:120 [`QuantizationConfigMixin`]: The configuration object instantiated from those parameters.121 """122 config = cls(**config_dict)123 124 to_remove = []125 for key, value in kwargs.items():126 if hasattr(config, key):127 setattr(config, key, value)128 to_remove.append(key)129 for key in to_remove:130 kwargs.pop(key, None)131 132 if return_unused_kwargs:133 return config, kwargs134 else:135 return config136 137 def to_json_file(self, json_file_path: Union[str, os.PathLike]):138 """139 Save this instance to a JSON file.140 141 Args:142 json_file_path (`str` or `os.PathLike`):143 Path to the JSON file in which this configuration instance's parameters will be saved.144 use_diff (`bool`, *optional*, defaults to `True`):145 If set to `True`, only the difference between the config instance and the default146 `QuantizationConfig()` is serialized to JSON file.147 """148 with open(json_file_path, "w", encoding="utf-8") as writer:149 config_dict = self.to_dict()150 json_string = json.dumps(config_dict, indent=2, sort_keys=True) + "\n"151 152 writer.write(json_string)153 154 def to_dict(self) -> dict[str, Any]:155 """156 Serializes this instance to a Python dictionary. Returns:157 `dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance.158 """159 return copy.deepcopy(self.__dict__)160 161 def __iter__(self):162 """allows `dict(obj)` for situations where obj may be a dict or QuantizationConfigMixin"""163 for attr, value in copy.deepcopy(self.__dict__).items():164 yield attr, value165 166 def __repr__(self):167 return f"{self.__class__.__name__} {self.to_json_string()}"168 169 def to_json_string(self, use_diff: bool = True) -> str:170 """171 Serializes this instance to a JSON string.172 173 Args:174 use_diff (`bool`, *optional*, defaults to `True`):175 If set to `True`, only the difference between the config instance and the default `PretrainedConfig()`176 is serialized to JSON string.177 178 Returns:179 `str`: String containing all the attributes that make up this configuration instance in JSON format.180 """181 if use_diff is True:182 config_dict = self.to_diff_dict()183 else:184 config_dict = self.to_dict()185 return json.dumps(config_dict, indent=2, sort_keys=True) + "\n"186 187 def update(self, **kwargs):188 """189 Updates attributes of this class instance with attributes from `kwargs` if they match existing attributes,190 returning all the unused kwargs.191 192 Args:193 kwargs (`dict[str, Any]`):194 Dictionary of attributes to tentatively update this class.195 196 Returns:197 `dict[str, Any]`: Dictionary containing all the key-value pairs that were not used to update the instance.198 """199 to_remove = []200 for key, value in kwargs.items():201 if hasattr(self, key):202 setattr(self, key, value)203 to_remove.append(key)204 205 # Remove all the attributes that were updated, without modifying the input dict206 unused_kwargs = {key: value for key, value in kwargs.items() if key not in to_remove}207 return unused_kwargs208 209 210@dataclass211class AutoRoundConfig(QuantizationConfigMixin):212 """This is a wrapper class about all possible attributes and features that you can play with a model that has been213 loaded AutoRound quantization.214 215 Args:216 bits (`int`, *optional*, defaults to 4):217 The number of bits to quantize to, supported numbers are (2, 3, 4, 8).218 group_size (`int`, *optional*, defaults to 128): Group-size value219 sym (`bool`, *optional*, defaults to `True`): Symmetric quantization or not220 backend (`str`, *optional*, defaults to `"auto"`): The kernel to use, e.g., ipex,marlin, exllamav2, triton, etc. Ref. https://github.com/intel/auto-round?tab=readme-ov-file#specify-backend221 """222 223 def __init__(224 self,225 bits: int = 4,226 group_size: int = 128,227 sym: bool = True,228 backend: str = "auto",229 **kwargs,230 ):231 self.bits = bits232 self.group_size = group_size233 self.sym = sym234 self.backend = backend235 self.packing_format = "auto_round:gptq"236 if kwargs is not None:237 for key, value in kwargs.items():238 setattr(self, key, value)239 self.quant_method = QuantizationMethod.AUTOROUND240 self.post_init()241 242 def post_init(self):243 r"""Safety checker that arguments are correct."""244 if self.bits not in [2, 3, 4, 8]:245 raise ValueError(f"Only support quantization to [2,3,4,8] bits but found {self.bits}")246 if self.group_size != -1 and self.group_size <= 0:247 raise ValueError("group_size must be greater than 0 or equal to -1")248 249 def get_loading_attributes(self):250 loading_attributes_dict = {"backend": self.backend}251 return loading_attributes_dict252 253 def to_dict(self):254 config_dict = super().to_dict()255 return config_dict256 257 @classmethod258 def from_dict(cls, config_dict, return_unused_kwargs=False, **kwargs):259 quant_method = config_dict["quant_method"]260 if "auto-round" not in quant_method and "gptq" not in quant_method and "awq" not in quant_method:261 raise NotImplementedError(262 "Failed to convert to auto_round format. Only `gptqv1`, `awq`, and `auto-round` formats are supported."263 )264 265 if "gptq" in quant_method and "meta" in config_dict:266 raise NotImplementedError("Failed to convert gptq format to auto_round format. Only supports `gptqv1`")267 268 if "awq" in quant_method and config_dict.get("version", "gemm") != "gemm":269 raise NotImplementedError(270 "Failed to convert awq format to auto_round format. Only supports awq format with gemm version"271 )272 273 if "auto-round" not in quant_method:274 config_dict["packing_format"] = f"auto_round:{quant_method}"275 276 return super().from_dict(config_dict, return_unused_kwargs=return_unused_kwargs, **kwargs)277 278 279@dataclass280class HqqConfig(QuantizationConfigMixin):281 """282 This is wrapper around hqq's BaseQuantizeConfig.283 284 Args:285 nbits (`int`, *optional*, defaults to 4):286 Number of bits. Supported values are (8, 4, 3, 2, 1).287 group_size (`int`, *optional*, defaults to 64):288 Group-size value. Supported values are any value that is divisible by weight.shape[axis]).289 view_as_float (`bool`, *optional*, defaults to `False`):290 View the quantized weight as float (used in distributed training) if set to `True`.291 axis (`Optional[int]`, *optional*):292 Axis along which grouping is performed. Supported values are 0 or 1.293 dynamic_config (dict, *optional*):294 Parameters for dynamic configuration. The key is the name tag of the layer and the value is a quantization config.295 If set, each layer specified by its id will use its dedicated quantization configuration.296 skip_modules (`list[str]`, *optional*, defaults to `['lm_head']`):297 List of `nn.Linear` layers to skip.298 kwargs (`dict[str, Any]`, *optional*):299 Additional parameters from which to initialize the configuration object.300 """301 302 def __init__(303 self,304 nbits: int = 4,305 group_size: int = 64,306 view_as_float: bool = False,307 axis: Optional[int] = None,308 dynamic_config: Optional[dict] = None,309 skip_modules: list[str] = ["lm_head"],310 **kwargs,311 ):312 if is_hqq_available():313 from hqq.core.quantize import BaseQuantizeConfig as HQQBaseQuantizeConfig314 else:315 raise ImportError(316 "A valid HQQ version (>=0.2.1) is not available. Please follow the instructions to install it: `https://github.com/mobiusml/hqq/`."317 )318 319 for deprecated_key in ["quant_zero", "quant_scale", "offload_meta"]:320 if deprecated_key in kwargs:321 logger.info(322 deprecated_key + " is deprecated. This parameter will be ignored in quantization settings."323 )324 325 if axis is None:326 axis = 1327 logger.info("Setting axis=1 as faster backends such as TorchAO or BitBlas are only compatible with it.")328 329 if axis not in [0, 1]:330 raise ValueError("Invalid axis value. Only 0 and 1 are allowed.")331 332 if dynamic_config is not None:333 self.quant_config = {}334 for key in dynamic_config:335 self.quant_config[key] = HQQBaseQuantizeConfig(**dynamic_config[key])336 else:337 self.quant_config = HQQBaseQuantizeConfig(338 **{339 "nbits": nbits,340 "group_size": group_size,341 "view_as_float": view_as_float,342 "axis": axis,343 }344 )345 346 self.quant_method = QuantizationMethod.HQQ347 self.skip_modules = skip_modules348 349 self.post_init()350 351 def post_init(self):352 r"""353 Safety checker that arguments are correct - also replaces some NoneType arguments with their default values.354 """355 pass356 357 @classmethod358 def from_dict(cls, config: dict[str, Any]):359 """360 Override from_dict, used in AutoQuantizationConfig.from_dict in quantizers/auto.py361 """362 instance = cls()363 instance.quant_config = config["quant_config"]364 instance.skip_modules = config["skip_modules"]365 return instance366 367 def to_dict(self) -> dict[str, Any]:368 """369 Serializes this instance to a Python dictionary. Returns:370 `dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance.371 """372 return {373 "quant_config": self.quant_config,374 "quant_method": self.quant_method,375 "skip_modules": self.skip_modules,376 }377 378 def __repr__(self):379 config_dict = self.to_dict()380 return f"{self.__class__.__name__} {json.dumps(config_dict, indent=2, sort_keys=True)}\n"381 382 def to_diff_dict(self) -> dict[str, Any]:383 """384 Removes all attributes from config which correspond to the default config attributes for better readability and385 serializes to a Python dictionary.386 Returns:387 `dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance,388 """389 config_dict = self.to_dict()390 391 # get the default config dict392 default_config_dict = HqqConfig().to_dict()393 394 serializable_config_dict = {}395 396 # only serialize values that differ from the default config397 for key, value in config_dict.items():398 if value != default_config_dict[key]:399 serializable_config_dict[key] = value400 401 return serializable_config_dict402 403 404@dataclass405class BitsAndBytesConfig(QuantizationConfigMixin):406 """407 This is a wrapper class about all possible attributes and features that you can play with a model that has been408 loaded using `bitsandbytes`.409 410 This replaces `load_in_8bit` or `load_in_4bit`therefore both options are mutually exclusive.411 412 Currently only supports `LLM.int8()`, `FP4`, and `NF4` quantization. If more methods are added to `bitsandbytes`,413 then more arguments will be added to this class.414 415 Args:416 load_in_8bit (`bool`, *optional*, defaults to `False`):417 This flag is used to enable 8-bit quantization with LLM.int8().418 load_in_4bit (`bool`, *optional*, defaults to `False`):419 This flag is used to enable 4-bit quantization by replacing the Linear layers with FP4/NF4 layers from420 `bitsandbytes`.421 llm_int8_threshold (`float`, *optional*, defaults to 6.0):422 This corresponds to the outlier threshold for outlier detection as described in `LLM.int8() : 8-bit Matrix423 Multiplication for Transformers at Scale` paper: https://huggingface.co/papers/2208.07339 Any hidden states value424 that is above this threshold will be considered an outlier and the operation on those values will be done425 in fp16. Values are usually normally distributed, that is, most values are in the range [-3.5, 3.5], but426 there are some exceptional systematic outliers that are very differently distributed for large models.427 These outliers are often in the interval [-60, -6] or [6, 60]. Int8 quantization works well for values of428 magnitude ~5, but beyond that, there is a significant performance penalty. A good default threshold is 6,429 but a lower threshold might be needed for more unstable models (small models, fine-tuning).430 llm_int8_skip_modules (`list[str]`, *optional*):431 An explicit list of the modules that we do not want to convert in 8-bit. This is useful for models such as432 Jukebox that has several heads in different places and not necessarily at the last position. For example433 for `CausalLM` models, the last `lm_head` is kept in its original `dtype`.434 llm_int8_enable_fp32_cpu_offload (`bool`, *optional*, defaults to `False`):435 This flag is used for advanced use cases and users that are aware of this feature. If you want to split436 your model in different parts and run some parts in int8 on GPU and some parts in fp32 on CPU, you can use437 this flag. This is useful for offloading large models such as `google/flan-t5-xxl`. Note that the int8438 operations will not be run on CPU.439 llm_int8_has_fp16_weight (`bool`, *optional*, defaults to `False`):440 This flag runs LLM.int8() with 16-bit main weights. This is useful for fine-tuning as the weights do not441 have to be converted back and forth for the backward pass.442 bnb_4bit_compute_dtype (`torch.dtype` or str, *optional*, defaults to `torch.float32`):443 This sets the computational type which might be different than the input type. For example, inputs might be444 fp32, but computation can be set to bf16 for speedups.445 bnb_4bit_quant_type (`str`, *optional*, defaults to `"fp4"`):446 This sets the quantization data type in the bnb.nn.Linear4Bit layers. Options are FP4 and NF4 data types447 which are specified by `fp4` or `nf4`.448 bnb_4bit_use_double_quant (`bool`, *optional*, defaults to `False`):449 This flag is used for nested quantization where the quantization constants from the first quantization are450 quantized again.451 bnb_4bit_quant_storage (`torch.dtype` or str, *optional*, defaults to `torch.uint8`):452 This sets the storage type to pack the quantized 4-bit params.453 kwargs (`dict[str, Any]`, *optional*):454 Additional parameters from which to initialize the configuration object.455 """456 457 def __init__(458 self,459 load_in_8bit=False,460 load_in_4bit=False,461 llm_int8_threshold=6.0,462 llm_int8_skip_modules=None,463 llm_int8_enable_fp32_cpu_offload=False,464 llm_int8_has_fp16_weight=False,465 bnb_4bit_compute_dtype=None,466 bnb_4bit_quant_type="fp4",467 bnb_4bit_use_double_quant=False,468 bnb_4bit_quant_storage=None,469 **kwargs,470 ):471 self.quant_method = QuantizationMethod.BITS_AND_BYTES472 473 if load_in_4bit and load_in_8bit:474 raise ValueError("load_in_4bit and load_in_8bit are both True, but only one can be used at the same time")475 476 self._load_in_8bit = load_in_8bit477 self._load_in_4bit = load_in_4bit478 self.llm_int8_threshold = llm_int8_threshold479 self.llm_int8_skip_modules = llm_int8_skip_modules480 self.llm_int8_enable_fp32_cpu_offload = llm_int8_enable_fp32_cpu_offload481 self.llm_int8_has_fp16_weight = llm_int8_has_fp16_weight482 self.bnb_4bit_quant_type = bnb_4bit_quant_type483 self.bnb_4bit_use_double_quant = bnb_4bit_use_double_quant484 485 if bnb_4bit_compute_dtype is None:486 self.bnb_4bit_compute_dtype = torch.float32487 elif isinstance(bnb_4bit_compute_dtype, str):488 self.bnb_4bit_compute_dtype = getattr(torch, bnb_4bit_compute_dtype)489 elif isinstance(bnb_4bit_compute_dtype, torch.dtype):490 self.bnb_4bit_compute_dtype = bnb_4bit_compute_dtype491 else:492 raise ValueError("bnb_4bit_compute_dtype must be a string or a torch.dtype")493 494 if bnb_4bit_quant_storage is None:495 self.bnb_4bit_quant_storage = torch.uint8496 elif isinstance(bnb_4bit_quant_storage, str):497 if bnb_4bit_quant_storage not in ["float16", "float32", "int8", "uint8", "float64", "bfloat16"]:498 raise ValueError(499 "`bnb_4bit_quant_storage` must be a valid string (one of 'float16', 'float32', 'int8', 'uint8', 'float64', 'bfloat16') "500 )501 self.bnb_4bit_quant_storage = getattr(torch, bnb_4bit_quant_storage)502 elif isinstance(bnb_4bit_quant_storage, torch.dtype):503 self.bnb_4bit_quant_storage = bnb_4bit_quant_storage504 else:505 raise ValueError("bnb_4bit_quant_storage must be a string or a torch.dtype")506 507 if kwargs:508 logger.info(f"Unused kwargs: {list(kwargs.keys())}. These kwargs are not used in {self.__class__}.")509 510 self.post_init()511 512 @property513 def load_in_4bit(self):514 return self._load_in_4bit515 516 @load_in_4bit.setter517 def load_in_4bit(self, value: bool):518 if not isinstance(value, bool):519 raise TypeError("load_in_4bit must be a boolean")520 521 if self.load_in_8bit and value:522 raise ValueError("load_in_4bit and load_in_8bit are both True, but only one can be used at the same time")523 self._load_in_4bit = value524 525 @property526 def load_in_8bit(self):527 return self._load_in_8bit528 529 @load_in_8bit.setter530 def load_in_8bit(self, value: bool):531 if not isinstance(value, bool):532 raise TypeError("load_in_8bit must be a boolean")533 534 if self.load_in_4bit and value:535 raise ValueError("load_in_4bit and load_in_8bit are both True, but only one can be used at the same time")536 self._load_in_8bit = value537 538 def post_init(self):539 r"""540 Safety checker that arguments are correct - also replaces some NoneType arguments with their default values.541 """542 if not isinstance(self.load_in_4bit, bool):543 raise TypeError("load_in_4bit must be a boolean")544 545 if not isinstance(self.load_in_8bit, bool):546 raise TypeError("load_in_8bit must be a boolean")547 548 if not isinstance(self.llm_int8_threshold, float):549 raise TypeError("llm_int8_threshold must be a float")550 551 if self.llm_int8_skip_modules is not None and not isinstance(self.llm_int8_skip_modules, list):552 raise TypeError("llm_int8_skip_modules must be a list of strings")553 if not isinstance(self.llm_int8_enable_fp32_cpu_offload, bool):554 raise TypeError("llm_int8_enable_fp32_cpu_offload must be a boolean")555 556 if not isinstance(self.llm_int8_has_fp16_weight, bool):557 raise TypeError("llm_int8_has_fp16_weight must be a boolean")558 559 if self.bnb_4bit_compute_dtype is not None and not isinstance(self.bnb_4bit_compute_dtype, torch.dtype):560 raise TypeError("bnb_4bit_compute_dtype must be torch.dtype")561 562 if not isinstance(self.bnb_4bit_quant_type, str):563 raise TypeError("bnb_4bit_quant_type must be a string")564 565 if not isinstance(self.bnb_4bit_use_double_quant, bool):566 raise TypeError("bnb_4bit_use_double_quant must be a boolean")567 568 if self.load_in_4bit and not version.parse(importlib.metadata.version("bitsandbytes")) >= version.parse(569 "0.39.0"570 ):571 raise ValueError(572 "4 bit quantization requires bitsandbytes>=0.39.0 - please upgrade your bitsandbytes version"573 )574 575 def is_quantizable(self):576 r"""577 Returns `True` if the model is quantizable, `False` otherwise.578 """579 return self.load_in_8bit or self.load_in_4bit580 581 def quantization_method(self):582 r"""583 This method returns the quantization method used for the model. If the model is not quantizable, it returns584 `None`.585 """586 if self.load_in_8bit:587 return "llm_int8"588 elif self.load_in_4bit and self.bnb_4bit_quant_type == "fp4":589 return "fp4"590 elif self.load_in_4bit and self.bnb_4bit_quant_type == "nf4":591 return "nf4"592 else:593 return None594 595 def to_dict(self) -> dict[str, Any]:596 """597 Serializes this instance to a Python dictionary. Returns:598 `dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance.599 """600 output = copy.deepcopy(self.__dict__)601 output["bnb_4bit_compute_dtype"] = str(output["bnb_4bit_compute_dtype"]).split(".")[1]602 output["bnb_4bit_quant_storage"] = str(output["bnb_4bit_quant_storage"]).split(".")[1]603 output["load_in_4bit"] = self.load_in_4bit604 output["load_in_8bit"] = self.load_in_8bit605 606 return output607 608 def __repr__(self):609 config_dict = self.to_dict()610 return f"{self.__class__.__name__} {json.dumps(config_dict, indent=2, sort_keys=True)}\n"611 612 def to_diff_dict(self) -> dict[str, Any]:613 """614 Removes all attributes from config which correspond to the default config attributes for better readability and615 serializes to a Python dictionary.616 617 Returns:618 `dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance,619 """620 config_dict = self.to_dict()621 622 # get the default config dict623 default_config_dict = BitsAndBytesConfig().to_dict()624 625 serializable_config_dict = {}626 627 # only serialize values that differ from the default config628 for key, value in config_dict.items():629 if value != default_config_dict[key]:630 serializable_config_dict[key] = value631 632 return serializable_config_dict633 634 635class ExllamaVersion(int, Enum):636 ONE = 1637 TWO = 2638 639 640@dataclass641class GPTQConfig(QuantizationConfigMixin):642 """643 This is a wrapper class about all possible attributes and features that you can play with a model that has been644 loaded using `optimum` api for gptq quantization relying on auto_gptq backend.645 646 Args:647 bits (`int`):648 The number of bits to quantize to, supported numbers are (2, 3, 4, 8).649 tokenizer (`str` or `PreTrainedTokenizerBase`, *optional*):650 The tokenizer used to process the dataset. You can pass either:651 - A custom tokenizer object.652 - A string, the *model id* of a predefined tokenizer hosted inside a model repo on huggingface.co.653 - A path to a *directory* containing vocabulary files required by the tokenizer, for instance saved654 using the [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`.655 dataset (`Union[list[str]]`, *optional*):656 The dataset used for quantization. You can provide your own dataset in a list of string or just use the657 original datasets used in GPTQ paper ['wikitext2','c4','c4-new']658 group_size (`int`, *optional*, defaults to 128):659 The group size to use for quantization. Recommended value is 128 and -1 uses per-column quantization.660 damp_percent (`float`, *optional*, defaults to 0.1):661 The percent of the average Hessian diagonal to use for dampening. Recommended value is 0.1.662 desc_act (`bool`, *optional*, defaults to `False`):663 Whether to quantize columns in order of decreasing activation size. Setting it to False can significantly664 speed up inference but the perplexity may become slightly worse. Also known as act-order.665 sym (`bool`, *optional*, defaults to `True`):666 Whether to use symmetric quantization.667 true_sequential (`bool`, *optional*, defaults to `True`):668 Whether to perform sequential quantization even within a single Transformer block. Instead of quantizing669 the entire block at once, we perform layer-wise quantization. As a result, each layer undergoes670 quantization using inputs that have passed through the previously quantized layers.671 checkpoint_format (`str`, *optional*, defaults to `"gptq"`):672 GPTQ weight format. `gptq`(v1) is supported by both gptqmodel and auto-gptq. `gptq_v2` is gptqmodel only.673 meta (`dict[str, any]`, *optional*):674 Properties, such as tooling:version, that do not directly contributes to quantization or quant inference are stored in meta.675 i.e. `meta.quantizer`: ["optimum:_version_", "gptqmodel:_version_"]676 backend (`str`, *optional*):677 Controls which gptq kernel to be used. Valid values for gptqmodel are `auto`, `auto_trainable` and more. For auto-gptq, only678 valid value is None and `auto_trainable`. Ref gptqmodel backends: https://github.com/ModelCloud/GPTQModel/blob/main/gptqmodel/utils/backend.py679 use_cuda_fp16 (`bool`, *optional*, defaults to `False`):680 Whether or not to use optimized cuda kernel for fp16 model. Need to have model in fp16. Auto-gptq only.681 model_seqlen (`int`, *optional*):682 The maximum sequence length that the model can take.683 block_name_to_quantize (`str`, *optional*):684 The transformers block name to quantize. If None, we will infer the block name using common patterns (e.g. model.layers)685 module_name_preceding_first_block (`list[str]`, *optional*):686 The layers that are preceding the first Transformer block.687 batch_size (`int`, *optional*, defaults to 1):688 The batch size used when processing the dataset689 pad_token_id (`int`, *optional*):690 The pad token id. Needed to prepare the dataset when `batch_size` > 1.691 use_exllama (`bool`, *optional*):692 Whether to use exllama backend. Defaults to `True` if unset. Only works with `bits` = 4.693 max_input_length (`int`, *optional*):694 The maximum input length. This is needed to initialize a buffer that depends on the maximum expected input695 length. It is specific to the exllama backend with act-order.696 exllama_config (`dict[str, Any]`, *optional*):697 The exllama config. You can specify the version of the exllama kernel through the `version` key. Defaults698 to `{"version": 1}` if unset.699 cache_block_outputs (`bool`, *optional*, defaults to `True`):700 Whether to cache block outputs to reuse as inputs for the succeeding block.701 modules_in_block_to_quantize (`list[list[str]]`, *optional*):702 List of list of module names to quantize in the specified block. This argument is useful to exclude certain linear modules from being quantized.703 The block to quantize can be specified by setting `block_name_to_quantize`. We will quantize each list sequentially. If not set, we will quantize all linear layers.704 Example: `modules_in_block_to_quantize =[["self_attn.k_proj", "self_attn.v_proj", "self_attn.q_proj"], ["self_attn.o_proj"]]`.705 In this example, we will first quantize the q,k,v layers simultaneously since they are independent.706 Then, we will quantize `self_attn.o_proj` layer with the q,k,v layers quantized. This way, we will get707 better results since it reflects the real input `self_attn.o_proj` will get when the model is quantized.708 """709 710 def __init__(711 self,712 bits: int,713 tokenizer: Any = None,714 dataset: Optional[Union[list[str], str]] = None,715 group_size: int = 128,716 damp_percent: float = 0.1,717 desc_act: bool = False,718 sym: bool = True,719 true_sequential: bool = True,720 checkpoint_format: str = "gptq",721 meta: Optional[dict[str, Any]] = None,722 backend: Optional[str] = None,723 use_cuda_fp16: bool = False,724 model_seqlen: Optional[int] = None,725 block_name_to_quantize: Optional[str] = None,726 module_name_preceding_first_block: Optional[list[str]] = None,727 batch_size: int = 1,728 pad_token_id: Optional[int] = None,729 use_exllama: Optional[bool] = None,730 max_input_length: Optional[int] = None,731 exllama_config: Optional[dict[str, Any]] = None,732 cache_block_outputs: bool = True,733 modules_in_block_to_quantize: Optional[list[list[str]]] = None,734 **kwargs,735 ):736 self.quant_method = QuantizationMethod.GPTQ737 self.bits = bits738 self.tokenizer = tokenizer739 self.dataset = dataset740 self.group_size = group_size741 self.damp_percent = damp_percent742 self.desc_act = desc_act743 self.sym = sym744 self.true_sequential = true_sequential745 self.checkpoint_format = checkpoint_format.lower()746 self.meta = meta747 self.backend = backend.lower() if isinstance(backend, str) else backend748 self.use_cuda_fp16 = use_cuda_fp16749 self.model_seqlen = model_seqlen750 self.block_name_to_quantize = block_name_to_quantize751 self.module_name_preceding_first_block = module_name_preceding_first_block752 self.batch_size = batch_size753 self.pad_token_id = pad_token_id754 self.use_exllama = use_exllama755 self.max_input_length = max_input_length756 self.exllama_config = exllama_config757 self.cache_block_outputs = cache_block_outputs758 self.modules_in_block_to_quantize = modules_in_block_to_quantize759 self.post_init()760 761 def get_loading_attributes(self):762 attributes_dict = copy.deepcopy(self.__dict__)763 loading_attributes = [764 "use_exllama",765 "exllama_config",766 "use_cuda_fp16",767 "max_input_length",768 "backend",769 ]770 loading_attributes_dict = {i: j for i, j in attributes_dict.items() if i in loading_attributes}771 return loading_attributes_dict772 773 def post_init(self):774 r"""775 Safety checker that arguments are correct776 """777 if self.bits not in [2, 3, 4, 8]:778 raise ValueError(f"Only support quantization to [2,3,4,8] bits but found {self.bits}")779 if self.group_size != -1 and self.group_size <= 0:780 raise ValueError("group_size must be greater than 0 or equal to -1")781 if not (0 < self.damp_percent < 1):782 raise ValueError("damp_percent must between 0 and 1.")783 if self.dataset is not None:784 if isinstance(self.dataset, str):785 if self.dataset in ["ptb", "ptb-new"]:786 raise ValueError(787 f"""{self.dataset} dataset was deprecated. You can only choose between788 ['wikitext2','c4','c4-new']"""789 )790 if self.dataset not in ["wikitext2", "c4", "c4-new"]:791 raise ValueError(792 f"""You have entered a string value for dataset. You can only choose between793 ['wikitext2','c4','c4-new'], but we found {self.dataset}"""794 )795 elif not isinstance(self.dataset, list):796 raise ValueError(797 f"""dataset needs to be either a list of string or a value in798 ['wikitext2','c4','c4-new'], but we found {self.dataset}"""799 )800 801 # make sure backend is back/forward compatible with both gptqmodel (full) and auto-gptq (partial)802 if is_gptqmodel_available():803 # convert auto-gptq control into gptqmodel backend804 if self.backend is None:805 self.backend = "auto_trainable" if self.use_exllama is not None and not self.use_exllama else "auto"806 else:807 # convert gptqmodel backend `auto_trainable` into auto-gptq control808 if self.backend == "auto_trainable":809 self.use_exllama = False810 811 # auto-gptq specific kernel control logic812 if self.use_exllama is None:813 # New default behaviour814 self.use_exllama = True815 816 if self.exllama_config is None:817 self.exllama_config = {"version": ExllamaVersion.ONE}818 else:819 if "version" not in self.exllama_config:820 raise ValueError("`exllama_config` needs to have a `version` key.")821 elif self.exllama_config["version"] not in [ExllamaVersion.ONE, ExllamaVersion.TWO]:822 exllama_version = self.exllama_config["version"]823 raise ValueError(824 f"Only supported versions are in [ExllamaVersion.ONE, ExllamaVersion.TWO] - not recognized version {exllama_version}"825 )826 827 if self.bits == 4 and self.use_exllama:828 if self.exllama_config["version"] == ExllamaVersion.ONE:829 logger.info(830 "You have activated exllama backend. Note that you can get better inference "831 "speed using exllamav2 kernel by setting `exllama_config`."832 )833 elif self.exllama_config["version"] == ExllamaVersion.TWO:834 if is_auto_gptq_available():835 optimum_version = version.parse(importlib.metadata.version("optimum"))836 autogptq_version = version.parse(importlib.metadata.version("auto_gptq"))837 if optimum_version <= version.parse("1.13.2") or autogptq_version <= version.parse("0.4.2"):838 raise ValueError(839 f"You need optimum > 1.13.2 and auto-gptq > 0.4.2 . Make sure to have that version installed - detected version : optimum {optimum_version} and autogptq {autogptq_version}"840 )841 if self.modules_in_block_to_quantize is not None:842 optimum_version = version.parse(importlib.metadata.version("optimum"))843 if optimum_version < version.parse("1.15.0"):844 raise ValueError(845 "You current version of `optimum` does not support `modules_in_block_to_quantize` quantization argument, please upgrade `optimum` package to a version superior than 1.15.0 ."846 )847 848 def to_dict(self) -> dict[str, Any]:849 config_dict = super().to_dict()850 config_dict.pop("disable_exllama", None)851 return config_dict852 853 def to_dict_optimum(self):854 """855 Get compatible dict for optimum gptq config856 """857 quant_dict = self.to_dict()858 # make it compatible with optimum config859 quant_dict["disable_exllama"] = not self.use_exllama860 return quant_dict861 862 @classmethod863 def from_dict_optimum(cls, config_dict):864 """865 Get compatible class with optimum gptq config dict866 """867 868 if "disable_exllama" in config_dict:869 config_dict["use_exllama"] = not config_dict["disable_exllama"]870 # switch to None to not trigger the warning871 config_dict.pop("disable_exllama")872 873 config = cls(**config_dict)874 return config875 876 877@dataclass878class AwqConfig(QuantizationConfigMixin):879 """880 This is a wrapper class about all possible attributes and features that you can play with a model that has been881 loaded using `auto-awq` library awq quantization relying on auto_awq backend.882 883 Args:884 bits (`int`, *optional*, defaults to 4):885 The number of bits to quantize to.886 group_size (`int`, *optional*, defaults to 128):887 The group size to use for quantization. Recommended value is 128 and -1 uses per-column quantization.888 zero_point (`bool`, *optional*, defaults to `True`):889 Whether to use zero point quantization.890 version (`AWQLinearVersion`, *optional*, defaults to `AWQLinearVersion.GEMM`):891 The version of the quantization algorithm to use. GEMM is better for big batch_size (e.g. >= 8) otherwise,892 GEMV is better (e.g. < 8 ). GEMM models are compatible with Exllama kernels.893 backend (`AwqBackendPackingMethod`, *optional*, defaults to `AwqBackendPackingMethod.AUTOAWQ`):894 The quantization backend. Some models might be quantized using `llm-awq` backend. This is useful for users895 that quantize their own models using `llm-awq` library.896 do_fuse (`bool`, *optional*, defaults to `False`):897 Whether to fuse attention and mlp layers together for faster inference898 fuse_max_seq_len (`int`, *optional*):899 The Maximum sequence length to generate when using fusing.900 modules_to_fuse (`dict`, *optional*, default to `None`):901 Overwrite the natively supported fusing scheme with the one specified by the users.902 modules_to_not_convert (`list`, *optional*, default to `None`):903 The list of modules to not quantize, useful for quantizing models that explicitly require to have904 some modules left in their original precision (e.g. Whisper encoder, Llava encoder, Mixtral gate layers).905 Note you cannot quantize directly with transformers, please refer to `AutoAWQ` documentation for quantizing HF models.906 exllama_config (`dict[str, Any]`, *optional*):907 You can specify the version of the exllama kernel through the `version` key, the maximum sequence908 length through the `max_input_len` key, and the maximum batch size through the `max_batch_size` key.909 Defaults to `{"version": 2, "max_input_len": 2048, "max_batch_size": 8}` if unset.910 """911 912 def __init__(913 self,914 bits: int = 4,915 group_size: int = 128,916 zero_point: bool = True,917 version: AWQLinearVersion = AWQLinearVersion.GEMM,918 backend: AwqBackendPackingMethod = AwqBackendPackingMethod.AUTOAWQ,919 do_fuse: Optional[bool] = None,920 fuse_max_seq_len: Optional[int] = None,921 modules_to_fuse: Optional[dict] = None,922 modules_to_not_convert: Optional[list] = None,923 exllama_config: Optional[dict[str, int]] = None,924 **kwargs,925 ):926 self.quant_method = QuantizationMethod.AWQ927 928 self.bits = bits929 self.group_size = group_size930 self.zero_point = zero_point931 self.version = version932 self.backend = backend933 self.fuse_max_seq_len = fuse_max_seq_len934 self.modules_to_not_convert = modules_to_not_convert935 self.exllama_config = exllama_config936 937 self.modules_to_fuse = modules_to_fuse938 if do_fuse is None:939 self.do_fuse = modules_to_fuse is not None and len(modules_to_fuse) > 0940 else:941 self.do_fuse = do_fuse942 self.fuse_max_seq_len = fuse_max_seq_len943 944 self.post_init()945 946 def post_init(self):947 r"""948 Safety checker that arguments are correct949 """950 if self.backend not in [AwqBackendPackingMethod.AUTOAWQ, AwqBackendPackingMethod.LLMAWQ]:951 raise ValueError(952 f"Only supported quantization backends in {AwqBackendPackingMethod.AUTOAWQ} and {AwqBackendPackingMethod.LLMAWQ} - not recognized backend {self.backend}"953 )954 955 self.version = AWQLinearVersion.from_str(self.version)956 if self.version not in [957 AWQLinearVersion.GEMM,958 AWQLinearVersion.GEMV,959 AWQLinearVersion.EXLLAMA,960 AWQLinearVersion.IPEX,961 ]:962 raise ValueError(963 f"Only supported versions are in [AWQLinearVersion.GEMM, AWQLinearVersion.GEMV, AWQLinearVersion.EXLLAMA, AWQLinearVersion.IPEX] - not recognized version {self.version}"964 )965 966 if self.backend == AwqBackendPackingMethod.LLMAWQ:967 # Only cuda device can run this function968 if not (torch.cuda.is_available() or torch.xpu.is_available()):969 raise ValueError("LLM-AWQ backend is only supported on CUDA and XPU")970 if torch.cuda.is_available():971 compute_capability = torch.cuda.get_device_capability()972 major, minor = compute_capability973 if major < 8:974 raise ValueError("LLM-AWQ backend is only supported on CUDA GPUs with compute capability >= 8.0")975 976 if self.do_fuse and self.fuse_max_seq_len is None:977 raise ValueError(978 "You cannot enable fused modules without specifying a `fuse_max_seq_len`, make sure to pass a valid `fuse_max_seq_len` for your usecase"979 )980 981 if self.do_fuse:982 awq_version_supports_fusing = False983 MIN_AWQ_VERSION = "0.1.7"984 if is_auto_awq_available():985 awq_version_supports_fusing = version.parse(importlib.metadata.version("autoawq")) >= version.parse(986 MIN_AWQ_VERSION987 )988 989 if not awq_version_supports_fusing:990 raise ValueError(991 f"You current version of `autoawq` does not support module fusing, please upgrade `autoawq` package to at least {MIN_AWQ_VERSION}."992 )993 994 if self.modules_to_not_convert is not None:995 awq_version_supports_non_conversion = False996 MIN_AWQ_VERSION = "0.1.8"997 if is_auto_awq_available():998 awq_version_supports_non_conversion = version.parse(999 importlib.metadata.version("autoawq")1000 ) >= version.parse(MIN_AWQ_VERSION)1001 1002 if not awq_version_supports_non_conversion:1003 raise ValueError(1004 f"You current version of `autoawq` does not support module quantization skipping, please upgrade `autoawq` package to at least {MIN_AWQ_VERSION}."1005 )1006 1007 if self.do_fuse and self.modules_to_fuse is not None:1008 required_keys = [1009 "hidden_size",1010 "num_attention_heads",1011 "num_key_value_heads",1012 "mlp",1013 "attention",1014 "layernorm",1015 "use_alibi",1016 ]1017 if not all(key in self.modules_to_fuse for key in required_keys):1018 raise ValueError(1019 f"Required fields are missing in the fusing mapping, required fields are {required_keys}"1020 )1021 1022 if self.version == AWQLinearVersion.EXLLAMA:1023 awq_version_supports_exllama = False1024 MIN_AWQ_VERSION = "0.2.0"1025 if is_auto_awq_available():1026 awq_version_supports_exllama = version.parse(importlib.metadata.version("autoawq")) >= version.parse(1027 MIN_AWQ_VERSION1028 )1029 1030 if not awq_version_supports_exllama:1031 raise ValueError(1032 f"You current version of `autoawq` does not support exllama backend, "1033 f"please upgrade `autoawq` package to at least {MIN_AWQ_VERSION}."1034 )1035 1036 if self.exllama_config is None:1037 self.exllama_config = {"version": ExllamaVersion.TWO, "max_input_len": 2048, "max_batch_size": 8}1038 else:1039 if "version" not in self.exllama_config:1040 raise ValueError("`exllama_config` needs to have a `version` key.")1041 elif self.exllama_config["version"] not in [ExllamaVersion.ONE, ExllamaVersion.TWO]:1042 exllama_version = self.exllama_config["version"]1043 raise ValueError(1044 f"Only supported versions are in [ExllamaVersion.ONE, ExllamaVersion.TWO] - not recognized version {exllama_version}"1045 )1046 1047 def get_loading_attributes(self):1048 attributes_dict = copy.deepcopy(self.__dict__)1049 loading_attributes = ["version", "do_fuse", "modules_to_fuse", "fuse_max_seq_len", "exllama_config"]1050 loading_attributes_dict = {i: j for i, j in attributes_dict.items() if i in loading_attributes}1051 return loading_attributes_dict1052 1053 1054@dataclass1055class AqlmConfig(QuantizationConfigMixin):1056 """1057 This is a wrapper class about `aqlm` parameters.1058 1059 Args:1060 in_group_size (`int`, *optional*, defaults to 8):1061 The group size along the input dimension.1062 out_group_size (`int`, *optional*, defaults to 1):1063 The group size along the output dimension. It's recommended to always use 1.1064 num_codebooks (`int`, *optional*, defaults to 1):1065 Number of codebooks for the Additive Quantization procedure.1066 nbits_per_codebook (`int`, *optional*, defaults to 16):1067 Number of bits encoding a single codebook vector. Codebooks size is 2**nbits_per_codebook.1068 linear_weights_not_to_quantize (`Optional[list[str]]`, *optional*):1069 List of full paths of `nn.Linear` weight parameters that shall not be quantized.1070 kwargs (`dict[str, Any]`, *optional*):1071 Additional parameters from which to initialize the configuration object.1072 """1073 1074 def __init__(1075 self,1076 in_group_size: int = 8,1077 out_group_size: int = 1,1078 num_codebooks: int = 1,1079 nbits_per_codebook: int = 16,1080 linear_weights_not_to_quantize: Optional[list[str]] = None,1081 **kwargs,1082 ):1083 self.quant_method = QuantizationMethod.AQLM1084 self.in_group_size = in_group_size1085 self.out_group_size = out_group_size1086 self.num_codebooks = num_codebooks1087 self.nbits_per_codebook = nbits_per_codebook1088 self.linear_weights_not_to_quantize = linear_weights_not_to_quantize1089 1090 self.post_init()1091 1092 def post_init(self):1093 r"""1094 Safety checker that arguments are correct - also replaces some NoneType arguments with their default values.1095 """1096 if not isinstance(self.in_group_size, int):1097 raise TypeError("in_group_size must be a float")1098 if not isinstance(self.out_group_size, int):1099 raise TypeError("out_group_size must be a float")1100 if not isinstance(self.num_codebooks, int):1101 raise TypeError("num_codebooks must be a float")1102 if not isinstance(self.nbits_per_codebook, int):1103 raise TypeError("nbits_per_codebook must be a float")1104 1105 if self.linear_weights_not_to_quantize is not None and not isinstance(1106 self.linear_weights_not_to_quantize, list1107 ):1108 raise ValueError("linear_weights_not_to_quantize must be a list of strings")1109 1110 if self.linear_weights_not_to_quantize is None:1111 self.linear_weights_not_to_quantize = []1112 1113 1114@dataclass1115class VptqLayerConfig(QuantizationConfigMixin):1116 """1117 This is used to explain vptq config params for each layer1118 Args:1119 enable_norm (`bool`, *optional*, defaults to `True`): to control if we have scale/bias for fp-weight1120 enable_perm (`bool`, *optional*, defaults to `True`): to perm input_channel or not1121 group_num (`int`, *optional*, defaults to `1`): how many single groups for vector-quantization1122 group_size (`int`, *optional*, defaults to `-1`): depends on out-features1123 indices_as_float (`bool`, *optional*, defaults to `False`): for Finetuning1124 is_indice_packed (`bool`, *optional*, defaults to `True`): should always be True1125 num_centroids (`list`, *optional*, defaults to `[-1, -1]`): centroid numbers of clusters1126 num_res_centroids (`list`, *optional*, defaults to `[-1, -1]`): ditto for residual1127 outlier_size (`int`, *optional*, defaults to `1`): outliers1128 vector_lens (`list`, *optional*, defaults to `[-1, -1]`): centroid vector length in quantization1129 """1130 1131 def __init__(1132 self,1133 enable_norm: bool = True,1134 enable_perm: bool = True,1135 group_num: int = 1,1136 group_size: int = -1,1137 in_features: int = -1,1138 indices_as_float: bool = False,1139 is_indice_packed: bool = True,1140 num_centroids: tuple = [-1, -1],1141 num_res_centroids: tuple = [-1, -1],1142 out_features: int = -1,1143 outlier_size: int = 0,1144 vector_lens: tuple = [-1, -1],1145 **kwargs,1146 ):1147 self.enable_norm = enable_norm1148 self.enable_perm = enable_perm1149 self.group_num = group_num1150 self.group_size = group_size1151 self.in_features = in_features1152 self.indices_as_float = indices_as_float1153 self.is_indice_packed = is_indice_packed1154 self.num_centroids = num_centroids1155 self.num_res_centroids = num_res_centroids1156 self.out_features = out_features1157 self.outlier_size = outlier_size1158 self.vector_lens = vector_lens1159 self.post_init()1160 1161 def post_init(self):1162 r"""1163 Safety checker that arguments are correct1164 """1165 if self.is_indice_packed is False:1166 raise ValueError("is_indice_packed should always be True")1167 1168 1169@dataclass1170class VptqConfig(QuantizationConfigMixin):1171 """1172 This is a wrapper class about `vptq` parameters.1173 1174 Args:1175 enable_proxy_error (`bool`, *optional*, defaults to `False`): calculate proxy error for each layer1176 config_for_layers (`Dict`, *optional*, defaults to `{}`): quantization params for each layer1177 shared_layer_config (`Dict`, *optional*, defaults to `{}`): shared quantization params among layers1178 modules_to_not_convert (`list`, *optional*, default to `None`):1179 The list of modules to not quantize, useful for quantizing models that explicitly require to have1180 some modules left in their original precision (e.g. Whisper encoder, Llava encoder, Mixtral gate layers).1181 kwargs (`dict[str, Any]`, *optional*):1182 Additional parameters from which to initialize the configuration object.1183 """1184 1185 def __init__(1186 self,1187 enable_proxy_error: bool = False,1188 config_for_layers: dict[str, Any] = {},1189 shared_layer_config: dict[str, Any] = {},1190 modules_to_not_convert: Optional[list] = None,1191 **kwargs,1192 ):1193 self.quant_method = QuantizationMethod.VPTQ1194 self.enable_proxy_error = enable_proxy_error1195 self.config_for_layers: dict[str, Any] = config_for_layers1196 self.shared_layer_config: dict[str, Any] = shared_layer_config1197 self.modules_to_not_convert = modules_to_not_convert1198 self.post_init()1199 1200 def post_init(self):