Aluode/PerceptionLabPortable
0
1# Copyright 2023 The HuggingFace Team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7# http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15import importlib16import inspect17import re18from typing import Any, Optional, Union19 20from packaging import version21 22from ..utils import (23 check_peft_version,24 find_adapter_config_file,25 is_accelerate_available,26 is_peft_available,27 is_torch_available,28 logging,29)30 31 32if is_torch_available():33 import torch34 35if is_accelerate_available():36 from accelerate import dispatch_model37 from accelerate.utils import get_balanced_memory, infer_auto_device_map38 39# Minimum PEFT version supported for the integration40MIN_PEFT_VERSION = "0.5.0"41 42 43logger = logging.get_logger(__name__)44 45 46# DO NOT MODIFY, KEPT FOR BC ONLY47VLMS = [48 "aria",49 "ayavision",50 "emu3",51 "fuyu",52 "gotocr2",53 "gemma3",54 "internvl",55 "llava", # all llava prefixed models fall under this check56 "mistral3",57 "mllama",58 "paligemma",59 "qwen2vl",60 "qwen2_5_vl",61 "videollava",62 "vipllava",63]64 65 66class PeftAdapterMixin:67 """68 A class containing all functions for loading and using adapters weights that are supported in PEFT library. For69 more details about adapters and injecting them on a transformer-based model, check out the documentation of PEFT70 library: https://huggingface.co/docs/peft/index71 72 Currently supported PEFT methods are all non-prompt learning methods (LoRA, IA³, etc.). Other PEFT models such as73 prompt tuning, prompt learning are out of scope as these adapters are not "injectable" into a torch module. For74 using these methods, please refer to the usage guide of PEFT library.75 76 With this mixin, if the correct PEFT version is installed, it is possible to:77 78 - Load an adapter stored on a local path or in a remote Hub repository, and inject it in the model79 - Attach new adapters in the model and train them with Trainer or by your own.80 - Attach multiple adapters and iteratively activate / deactivate them81 - Activate / deactivate all adapters from the model.82 - Get the `state_dict` of the active adapter.83 """84 85 _hf_peft_config_loaded = False86 87 def load_adapter(88 self,89 peft_model_id: Optional[str] = None,90 adapter_name: Optional[str] = None,91 revision: Optional[str] = None,92 token: Optional[str] = None,93 device_map: str = "auto",94 max_memory: Optional[str] = None,95 offload_folder: Optional[str] = None,96 offload_index: Optional[int] = None,97 peft_config: Optional[dict[str, Any]] = None,98 adapter_state_dict: Optional[dict[str, "torch.Tensor"]] = None,99 low_cpu_mem_usage: bool = False,100 is_trainable: bool = False,101 adapter_kwargs: Optional[dict[str, Any]] = None,102 ) -> None:103 """104 Load adapter weights from file or remote Hub folder. If you are not familiar with adapters and PEFT methods, we105 invite you to read more about them on PEFT official documentation: https://huggingface.co/docs/peft106 107 Requires PEFT to be installed as a backend to load the adapter weights.108 109 Args:110 peft_model_id (`str`, *optional*):111 The identifier of the model to look for on the Hub, or a local path to the saved adapter config file112 and adapter weights.113 adapter_name (`str`, *optional*):114 The adapter name to use. If not set, will use the name "default".115 revision (`str`, *optional*, defaults to `"main"`):116 The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a117 git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any118 identifier allowed by git.119 120 > [!TIP]121 > To test a pull request you made on the Hub, you can pass `revision="refs/pr/<pr_number>"`.122 123 token (`str`, `optional`):124 Whether to use authentication token to load the remote folder. Useful to load private repositories125 that are on HuggingFace Hub. You might need to call `hf auth login` and paste your tokens to126 cache it.127 device_map (`str` or `dict[str, Union[int, str, torch.device]]` or `int` or `torch.device`, *optional*):128 A map that specifies where each submodule should go. It doesn't need to be refined to each129 parameter/buffer name, once a given module name is inside, every submodule of it will be sent to the130 same device. If we only pass the device (*e.g.*, `"cpu"`, `"cuda:1"`, `"mps"`, or a GPU ordinal rank131 like `1`) on which the model will be allocated, the device map will map the entire model to this132 device. Passing `device_map = 0` means put the whole model on GPU 0.133 134 To have Accelerate compute the most optimized `device_map` automatically, set `device_map="auto"`. For135 more information about each option see [designing a device136 map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map).137 max_memory (`Dict`, *optional*):138 A dictionary device identifier to maximum memory. Will default to the maximum memory available for each139 GPU and the available CPU RAM if unset.140 offload_folder (`str` or `os.PathLike`, `optional`):141 If the `device_map` contains any value `"disk"`, the folder where we will offload weights.142 offload_index (`int`, `optional`):143 `offload_index` argument to be passed to `accelerate.dispatch_model` method.144 peft_config (`dict[str, Any]`, *optional*):145 The configuration of the adapter to add, supported adapters are all non-prompt learning configs (LoRA,146 IA³, etc). This argument is used in case users directly pass PEFT state dicts.147 adapter_state_dict (`dict[str, torch.Tensor]`, *optional*):148 The state dict of the adapter to load. This argument is used in case users directly pass PEFT state149 dicts.150 low_cpu_mem_usage (`bool`, *optional*, defaults to `False`):151 Reduce memory usage while loading the PEFT adapter. This should also speed up the loading process.152 Requires PEFT version 0.13.0 or higher.153 is_trainable (`bool`, *optional*, defaults to `False`):154 Whether the adapter should be trainable or not. If `False`, the adapter will be frozen and can only be155 used for inference.156 adapter_kwargs (`dict[str, Any]`, *optional*):157 Additional keyword arguments passed along to the `from_pretrained` method of the adapter config and158 `find_adapter_config_file` method.159 """160 check_peft_version(min_version=MIN_PEFT_VERSION)161 162 # peft only supports low_cpu_mem_usage starting from v0.13.0163 peft_load_kwargs = {}164 key_mapping = adapter_kwargs.pop("key_mapping", None) if adapter_kwargs is not None else None165 if key_mapping is None and any(allowed_name in self.__class__.__name__.lower() for allowed_name in VLMS):166 key_mapping = self._checkpoint_conversion_mapping167 if low_cpu_mem_usage:168 min_version_lcmu = "0.13.0"169 if version.parse(importlib.metadata.version("peft")) >= version.parse(min_version_lcmu):170 peft_load_kwargs["low_cpu_mem_usage"] = low_cpu_mem_usage171 else:172 raise ValueError(173 "The version of PEFT you are using does not support `low_cpu_mem_usage` yet, "174 f"please install PEFT >= {min_version_lcmu}."175 )176 177 adapter_name = adapter_name if adapter_name is not None else "default"178 if adapter_kwargs is None:179 adapter_kwargs = {}180 181 from peft import PeftConfig, inject_adapter_in_model, load_peft_weights182 from peft.utils import set_peft_model_state_dict183 184 if self._hf_peft_config_loaded and adapter_name in self.peft_config:185 raise ValueError(f"Adapter with name {adapter_name} already exists. Please use a different name.")186 187 if peft_model_id is None and (adapter_state_dict is None and peft_config is None):188 raise ValueError(189 "You should either pass a `peft_model_id` or a `peft_config` and `adapter_state_dict` to load an adapter."190 )191 192 if "device" not in adapter_kwargs:193 device = self.device if not hasattr(self, "hf_device_map") else list(self.hf_device_map.values())[0]194 else:195 device = adapter_kwargs.pop("device")196 197 # To avoid PEFT errors later on with safetensors.198 if isinstance(device, torch.device):199 device = str(device)200 201 # We keep `revision` in the signature for backward compatibility202 if revision is not None and "revision" not in adapter_kwargs:203 adapter_kwargs["revision"] = revision204 elif revision is not None and "revision" in adapter_kwargs and revision != adapter_kwargs["revision"]:205 logger.error(206 "You passed a `revision` argument both in `adapter_kwargs` and as a standalone argument. "207 "The one in `adapter_kwargs` will be used."208 )209 210 # Override token with adapter_kwargs' token211 if "token" in adapter_kwargs:212 token = adapter_kwargs.pop("token")213 214 if peft_config is None:215 adapter_config_file = find_adapter_config_file(216 peft_model_id,217 token=token,218 **adapter_kwargs,219 )220 221 if adapter_config_file is None:222 raise ValueError(223 f"adapter model file not found in {peft_model_id}. Make sure you are passing the correct path to the "224 "adapter model."225 )226 227 peft_config = PeftConfig.from_pretrained(228 peft_model_id,229 token=token,230 **adapter_kwargs,231 )232 peft_config.inference_mode = not is_trainable233 234 # Create and add fresh new adapters into the model.235 inject_adapter_in_model(peft_config, self, adapter_name, **peft_load_kwargs)236 237 if not self._hf_peft_config_loaded:238 self._hf_peft_config_loaded = True239 240 if peft_model_id is not None:241 adapter_state_dict = load_peft_weights(peft_model_id, token=token, device=device, **adapter_kwargs)242 243 # We need to pre-process the state dict to remove unneeded prefixes - for backward compatibility244 processed_adapter_state_dict = {}245 prefix = "base_model.model."246 for key, value in adapter_state_dict.items():247 if key.startswith(prefix):248 new_key = key[len(prefix) :]249 else:250 new_key = key251 252 if key_mapping:253 for pattern, replacement in key_mapping.items():254 new_key, n_replace = re.subn(pattern, replacement, new_key)255 # Early exit of the loop256 if n_replace > 0:257 break258 processed_adapter_state_dict[new_key] = value259 260 # Load state dict261 incompatible_keys = set_peft_model_state_dict(262 self, processed_adapter_state_dict, adapter_name, **peft_load_kwargs263 )264 265 if incompatible_keys is not None:266 err_msg = ""267 origin_name = peft_model_id if peft_model_id is not None else "state_dict"268 # Check for unexpected keys.269 if hasattr(incompatible_keys, "unexpected_keys") and len(incompatible_keys.unexpected_keys) > 0:270 err_msg = (271 f"Loading adapter weights from {origin_name} led to unexpected keys not found in the model: "272 f"{', '.join(incompatible_keys.unexpected_keys)}. "273 )274 275 # Check for missing keys.276 missing_keys = getattr(incompatible_keys, "missing_keys", None)277 if missing_keys:278 # Filter missing keys specific to the current adapter, as missing base model keys are expected.279 lora_missing_keys = [k for k in missing_keys if "lora_" in k and adapter_name in k]280 if lora_missing_keys:281 err_msg += (282 f"Loading adapter weights from {origin_name} led to missing keys in the model: "283 f"{', '.join(lora_missing_keys)}"284 )285 286 if err_msg:287 logger.warning(err_msg)288 289 if peft_config.inference_mode:290 self.eval()291 292 # Re-dispatch model and hooks in case the model is offloaded to CPU / Disk.293 if (294 (getattr(self, "hf_device_map", None) is not None)295 and (len(set(self.hf_device_map.values()).intersection({"cpu", "disk"})) > 0)296 and len(self.peft_config) == 1297 ):298 self._dispatch_accelerate_model(299 device_map=device_map,300 max_memory=max_memory,301 offload_folder=offload_folder,302 offload_index=offload_index,303 )304 305 def add_adapter(self, adapter_config, adapter_name: Optional[str] = None) -> None:306 r"""307 If you are not familiar with adapters and PEFT methods, we invite you to read more about them on the PEFT308 official documentation: https://huggingface.co/docs/peft309 310 Adds a fresh new adapter to the current model for training purpose. If no adapter name is passed, a default311 name is assigned to the adapter to follow the convention of PEFT library (in PEFT we use "default" as the312 default adapter name).313 314 Note that the newly added adapter is not automatically activated. To activate it, use `model.set_adapter`.315 316 Args:317 adapter_config (`~peft.PeftConfig`):318 The configuration of the adapter to add, supported adapters are non-prompt learning methods (LoRA,319 IA³, etc.).320 adapter_name (`str`, *optional*, defaults to `"default"`):321 The name of the adapter to add. If no name is passed, a default name is assigned to the adapter.322 """323 check_peft_version(min_version=MIN_PEFT_VERSION)324 325 from peft import PeftConfig, inject_adapter_in_model326 327 adapter_name = adapter_name or "default"328 329 if not self._hf_peft_config_loaded:330 self._hf_peft_config_loaded = True331 elif adapter_name in self.peft_config:332 raise ValueError(f"Adapter with name {adapter_name} already exists. Please use a different name.")333 334 if not isinstance(adapter_config, PeftConfig):335 raise TypeError(f"adapter_config should be an instance of PeftConfig. Got {type(adapter_config)} instead.")336 337 # Retrieve the name or path of the model, one could also use self.config._name_or_path338 # but to be consistent with what we do in PEFT: https://github.com/huggingface/peft/blob/6e783780ca9df3a623992cc4d1d665001232eae0/src/peft/mapping.py#L100339 adapter_config.base_model_name_or_path = self.__dict__.get("name_or_path", None)340 inject_adapter_in_model(adapter_config, self, adapter_name)341 342 self.set_adapter(adapter_name)343 344 def set_adapter(self, adapter_name: Union[list[str], str]) -> None:345 """346 If you are not familiar with adapters and PEFT methods, we invite you to read more about them on the PEFT347 official documentation: https://huggingface.co/docs/peft348 349 Sets a specific adapter by forcing the model to use a that adapter and disable the other adapters.350 351 Args:352 adapter_name (`Union[list[str], str]`):353 The name of the adapter to set. Can be also a list of strings to set multiple adapters.354 """355 check_peft_version(min_version=MIN_PEFT_VERSION)356 if not self._hf_peft_config_loaded:357 raise ValueError("No adapter loaded. Please load an adapter first.")358 elif isinstance(adapter_name, list):359 missing = set(adapter_name) - set(self.peft_config)360 if len(missing) > 0:361 raise ValueError(362 f"Following adapter(s) could not be found: {', '.join(missing)}. Make sure you are passing the correct adapter name(s)."363 f" current loaded adapters are: {list(self.peft_config.keys())}"364 )365 elif adapter_name not in self.peft_config:366 raise ValueError(367 f"Adapter with name {adapter_name} not found. Please pass the correct adapter name among {list(self.peft_config.keys())}"368 )369 370 from peft.tuners.tuners_utils import BaseTunerLayer371 from peft.utils import ModulesToSaveWrapper372 373 _adapters_has_been_set = False374 375 for _, module in self.named_modules():376 if isinstance(module, (BaseTunerLayer, ModulesToSaveWrapper)):377 # For backward compatibility with previous PEFT versions378 if hasattr(module, "set_adapter"):379 module.set_adapter(adapter_name)380 else:381 module.active_adapter = adapter_name382 _adapters_has_been_set = True383 384 if not _adapters_has_been_set:385 raise ValueError(386 "Did not succeeded in setting the adapter. Please make sure you are using a model that supports adapters."387 )388 389 def disable_adapters(self) -> None:390 r"""391 If you are not familiar with adapters and PEFT methods, we invite you to read more about them on the PEFT392 official documentation: https://huggingface.co/docs/peft393 394 Disable all adapters that are attached to the model. This leads to inferring with the base model only.395 """396 check_peft_version(min_version=MIN_PEFT_VERSION)397 398 if not self._hf_peft_config_loaded:399 raise ValueError("No adapter loaded. Please load an adapter first.")400 401 from peft.tuners.tuners_utils import BaseTunerLayer402 from peft.utils import ModulesToSaveWrapper403 404 for _, module in self.named_modules():405 if isinstance(module, (BaseTunerLayer, ModulesToSaveWrapper)):406 # The recent version of PEFT need to call `enable_adapters` instead407 if hasattr(module, "enable_adapters"):408 module.enable_adapters(enabled=False)409 else:410 module.disable_adapters = True411 412 def enable_adapters(self) -> None:413 """414 If you are not familiar with adapters and PEFT methods, we invite you to read more about them on the PEFT415 official documentation: https://huggingface.co/docs/peft416 417 Enable adapters that are attached to the model.418 """419 check_peft_version(min_version=MIN_PEFT_VERSION)420 421 if not self._hf_peft_config_loaded:422 raise ValueError("No adapter loaded. Please load an adapter first.")423 424 from peft.tuners.tuners_utils import BaseTunerLayer425 426 for _, module in self.named_modules():427 if isinstance(module, BaseTunerLayer):428 # The recent version of PEFT need to call `enable_adapters` instead429 if hasattr(module, "enable_adapters"):430 module.enable_adapters(enabled=True)431 else:432 module.disable_adapters = False433 434 def active_adapters(self) -> list[str]:435 """436 If you are not familiar with adapters and PEFT methods, we invite you to read more about them on the PEFT437 official documentation: https://huggingface.co/docs/peft438 439 Gets the current active adapters of the model. In case of multi-adapter inference (combining multiple adapters440 for inference) returns the list of all active adapters so that users can deal with them accordingly.441 442 For previous PEFT versions (that does not support multi-adapter inference), `module.active_adapter` will return443 a single string.444 """445 check_peft_version(min_version=MIN_PEFT_VERSION)446 447 if not is_peft_available():448 raise ImportError("PEFT is not available. Please install PEFT to use this function: `pip install peft`.")449 450 if not self._hf_peft_config_loaded:451 raise ValueError("No adapter loaded. Please load an adapter first.")452 453 from peft.tuners.tuners_utils import BaseTunerLayer454 455 for _, module in self.named_modules():456 if isinstance(module, BaseTunerLayer):457 active_adapters = module.active_adapter458 break459 460 # For previous PEFT versions461 if isinstance(active_adapters, str):462 active_adapters = [active_adapters]463 464 return active_adapters465 466 def get_adapter_state_dict(self, adapter_name: Optional[str] = None, state_dict: Optional[dict] = None) -> dict:467 """468 If you are not familiar with adapters and PEFT methods, we invite you to read more about them on the PEFT469 official documentation: https://huggingface.co/docs/peft470 471 Gets the adapter state dict that should only contain the weights tensors of the specified adapter_name adapter.472 If no adapter_name is passed, the active adapter is used.473 474 Args:475 adapter_name (`str`, *optional*):476 The name of the adapter to get the state dict from. If no name is passed, the active adapter is used.477 state_dict (nested dictionary of `torch.Tensor`, *optional*)478 The state dictionary of the model. Will default to `self.state_dict()`, but can be used if special479 precautions need to be taken when recovering the state dictionary of a model (like when using model480 parallelism).481 """482 check_peft_version(min_version=MIN_PEFT_VERSION)483 484 if not self._hf_peft_config_loaded:485 raise ValueError("No adapter loaded. Please load an adapter first.")486 487 from peft import get_peft_model_state_dict488 489 if adapter_name is None:490 adapter_name = self.active_adapters()[0]491 492 adapter_state_dict = get_peft_model_state_dict(self, state_dict=state_dict, adapter_name=adapter_name)493 return adapter_state_dict494 495 def _dispatch_accelerate_model(496 self,497 device_map: str,498 max_memory: Optional[int] = None,499 offload_folder: Optional[str] = None,500 offload_index: Optional[int] = None,501 ) -> None:502 """503 Optional re-dispatch the model and attach new hooks to the model in case the model has been loaded with504 accelerate (i.e. with `device_map=xxx`)505 506 Args:507 device_map (`str` or `dict[str, Union[int, str, torch.device]]` or `int` or `torch.device`, *optional*):508 A map that specifies where each submodule should go. It doesn't need to be refined to each509 parameter/buffer name, once a given module name is inside, every submodule of it will be sent to the510 same device. If we only pass the device (*e.g.*, `"cpu"`, `"cuda:1"`, `"mps"`, or a GPU ordinal rank511 like `1`) on which the model will be allocated, the device map will map the entire model to this512 device. Passing `device_map = 0` means put the whole model on GPU 0.513 514 To have Accelerate compute the most optimized `device_map` automatically, set `device_map="auto"`. For515 more information about each option see [designing a device516 map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map).517 max_memory (`Dict`, *optional*):518 A dictionary device identifier to maximum memory. Will default to the maximum memory available for each519 GPU and the available CPU RAM if unset.520 offload_folder (`str` or `os.PathLike`, *optional*):521 If the `device_map` contains any value `"disk"`, the folder where we will offload weights.522 offload_index (`int`, *optional*):523 The offload_index argument to be passed to `accelerate.dispatch_model` method.524 """525 dispatch_model_kwargs = {}526 # Safety checker for previous `accelerate` versions527 # `offload_index` was introduced in https://github.com/huggingface/accelerate/pull/873/528 if "offload_index" in inspect.signature(dispatch_model).parameters:529 dispatch_model_kwargs["offload_index"] = offload_index530 531 no_split_module_classes = self._no_split_modules532 533 if device_map != "sequential":534 max_memory = get_balanced_memory(535 self,536 max_memory=max_memory,537 no_split_module_classes=no_split_module_classes,538 low_zero=(device_map == "balanced_low_0"),539 )540 if isinstance(device_map, str):541 device_map = infer_auto_device_map(542 self, max_memory=max_memory, no_split_module_classes=no_split_module_classes543 )544 dispatch_model(545 self,546 device_map=device_map,547 offload_dir=offload_folder,548 **dispatch_model_kwargs,549 )550 551 def delete_adapter(self, adapter_names: Union[list[str], str]) -> None:552 """553 Delete a PEFT adapter from the underlying model.554 555 Args:556 adapter_names (`Union[list[str], str]`):557 The name(s) of the adapter(s) to delete.558 """559 560 check_peft_version(min_version=MIN_PEFT_VERSION)561 min_version_delete_adapter = "0.18.0"562 563 if not self._hf_peft_config_loaded:564 raise ValueError("No adapter loaded. Please load an adapter first.")565 566 # TODO: delete old version once support for PEFT < 0.18.0 is dropped567 def old_delete_adapter(model, adapter_name, prefix=None):568 from peft.tuners.tuners_utils import BaseTunerLayer569 from peft.utils import ModulesToSaveWrapper570 571 has_modules_to_save = False572 for module in model.modules():573 if isinstance(module, ModulesToSaveWrapper):574 has_modules_to_save |= True575 continue576 if isinstance(module, BaseTunerLayer):577 if hasattr(module, "delete_adapter"):578 module.delete_adapter(adapter_name)579 else:580 raise ValueError(581 "The version of PEFT you are using is not compatible, please use a version that is greater than 0.6.1"582 )583 584 if has_modules_to_save:585 logger.warning(586 "The deleted adapter contains modules_to_save, which could not be deleted. For this to work, PEFT version "587 f">= {min_version_delete_adapter} is required."588 )589 590 if version.parse(importlib.metadata.version("peft")) >= version.parse(min_version_delete_adapter):591 from peft.functional import delete_adapter592 else:593 delete_adapter = old_delete_adapter594 595 if isinstance(adapter_names, str):596 adapter_names = [adapter_names]597 598 # Check that all adapter names are present in the config599 missing_adapters = [name for name in adapter_names if name not in self.peft_config]600 if missing_adapters:601 raise ValueError(602 f"The following adapter(s) are not present and cannot be deleted: {', '.join(missing_adapters)}"603 )604 605 prefixes = [f"{self.peft_config[adapter_name].peft_type.value.lower()}_" for adapter_name in adapter_names]606 for adapter_name, prefix in zip(adapter_names, prefixes):607 delete_adapter(self, adapter_name=adapter_name, prefix=prefix)608 # For transformers integration - we need to pop the adapter from the config609 if getattr(self, "_hf_peft_config_loaded", False) and hasattr(self, "peft_config"):610 self.peft_config.pop(adapter_name, None)611 612 # In case all adapters are deleted, we need to delete the config613 # and make sure to set the flag to False614 if len(self.peft_config) == 0:615 del self.peft_config616 self._hf_peft_config_loaded = False617