Aluode/PerceptionLabPortable
0
1# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.2# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""Configuration base class and utilities."""16 17import copy18import json19import os20import warnings21from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union22 23from packaging import version24 25from . import __version__26from .dynamic_module_utils import custom_object_save27from .modeling_gguf_pytorch_utils import load_gguf_checkpoint28from .utils import (29 CONFIG_NAME,30 PushToHubMixin,31 cached_file,32 copy_func,33 download_url,34 extract_commit_hash,35 is_remote_url,36 is_torch_available,37 logging,38)39from .utils.generic import is_timm_config_dict40 41 42if TYPE_CHECKING:43 import torch44 45 46logger = logging.get_logger(__name__)47 48 49# type hinting: specifying the type of config class that inherits from PretrainedConfig50SpecificPretrainedConfigType = TypeVar("SpecificPretrainedConfigType", bound="PretrainedConfig")51 52 53class PretrainedConfig(PushToHubMixin):54 # no-format55 r"""56 Base class for all configuration classes. Handles a few parameters common to all models' configurations as well as57 methods for loading/downloading/saving configurations.58 59 <Tip>60 61 A configuration file can be loaded and saved to disk. Loading the configuration file and using this file to62 initialize a model does **not** load the model weights. It only affects the model's configuration.63 64 </Tip>65 66 Class attributes (overridden by derived classes):67 68 - **model_type** (`str`) -- An identifier for the model type, serialized into the JSON file, and used to recreate69 the correct object in [`~transformers.AutoConfig`].70 - **has_no_defaults_at_init** (`bool`) -- Whether the config class can be initialized without providing input arguments.71 Some configurations requires inputs to be defined at init and have no default values, usually these are composite configs,72 (but not necessarily) such as [`~transformers.EncoderDecoderConfig`] or [`~RagConfig`]. They have to be initialized from73 two or more configs of type [`~transformers.PretrainedConfig`].74 - **keys_to_ignore_at_inference** (`list[str]`) -- A list of keys to ignore by default when looking at dictionary75 outputs of the model during inference.76 - **attribute_map** (`dict[str, str]`) -- A dict that maps model specific attribute names to the standardized77 naming of attributes.78 - **base_model_tp_plan** (`dict[str, Any]`) -- A dict that maps sub-modules FQNs of a base model to a tensor79 parallel plan applied to the sub-module when `model.tensor_parallel` is called.80 - **base_model_pp_plan** (`dict[str, tuple[list[str]]]`) -- A dict that maps child-modules of a base model to a81 pipeline parallel plan that enables users to place the child-module on the appropriate device.82 83 Common attributes (present in all subclasses):84 85 - **vocab_size** (`int`) -- The number of tokens in the vocabulary, which is also the first dimension of the86 embeddings matrix (this attribute may be missing for models that don't have a text modality like ViT).87 - **hidden_size** (`int`) -- The hidden size of the model.88 - **num_attention_heads** (`int`) -- The number of attention heads used in the multi-head attention layers of the89 model.90 - **num_hidden_layers** (`int`) -- The number of blocks in the model.91 92 <Tip warning={true}>93 94 Setting parameters for sequence generation in the model config is deprecated. For backward compatibility, loading95 some of them will still be possible, but attempting to overwrite them will throw an exception -- you should set96 them in a [~transformers.GenerationConfig]. Check the documentation of [~transformers.GenerationConfig] for more97 information about the individual parameters.98 99 </Tip>100 101 Arg:102 name_or_path (`str`, *optional*, defaults to `""`):103 Store the string that was passed to [`PreTrainedModel.from_pretrained`] or104 [`TFPreTrainedModel.from_pretrained`] as `pretrained_model_name_or_path` if the configuration was created105 with such a method.106 output_hidden_states (`bool`, *optional*, defaults to `False`):107 Whether or not the model should return all hidden-states.108 output_attentions (`bool`, *optional*, defaults to `False`):109 Whether or not the model should returns all attentions.110 return_dict (`bool`, *optional*, defaults to `True`):111 Whether or not the model should return a [`~transformers.utils.ModelOutput`] instead of a plain tuple.112 is_encoder_decoder (`bool`, *optional*, defaults to `False`):113 Whether the model is used as an encoder/decoder or not.114 is_decoder (`bool`, *optional*, defaults to `False`):115 Whether to only use the decoder in an encoder-decoder architecture, otherwise it has no effect on116 decoder-only or encoder-only architectures.117 cross_attention_hidden_size (`bool`, *optional*):118 The hidden size of the cross-attention layer in case the model is used as a decoder in an encoder-decoder119 setting and the cross-attention hidden dimension differs from `self.config.hidden_size`.120 add_cross_attention (`bool`, *optional*, defaults to `False`):121 Whether cross-attention layers should be added to the model. Note, this option is only relevant for models122 that can be used as decoder models within the [`EncoderDecoderModel`] class, which consists of all models123 in `AUTO_MODELS_FOR_CAUSAL_LM`.124 tie_encoder_decoder (`bool`, *optional*, defaults to `False`):125 Whether all encoder weights should be tied to their equivalent decoder weights. This requires the encoder126 and decoder model to have the exact same parameter names.127 prune_heads (`dict[int, list[int]]`, *optional*, defaults to `{}`):128 Pruned heads of the model. The keys are the selected layer indices and the associated values, the list of129 heads to prune in said layer.130 131 For instance `{1: [0, 2], 2: [2, 3]}` will prune heads 0 and 2 on layer 1 and heads 2 and 3 on layer 2.132 chunk_size_feed_forward (`int`, *optional*, defaults to `0`):133 The chunk size of all feed forward layers in the residual attention blocks. A chunk size of `0` means that134 the feed forward layer is not chunked. A chunk size of n means that the feed forward layer processes `n` <135 sequence_length embeddings at a time. For more information on feed forward chunking, see [How does Feed136 Forward Chunking work?](../glossary.html#feed-forward-chunking).137 138 > Parameters for fine-tuning tasks139 140 architectures (`list[str]`, *optional*):141 Model architectures that can be used with the model pretrained weights.142 finetuning_task (`str`, *optional*):143 Name of the task used to fine-tune the model. This can be used when converting from an original (TensorFlow144 or PyTorch) checkpoint.145 id2label (`dict[int, str]`, *optional*):146 A map from index (for instance prediction index, or target index) to label.147 label2id (`dict[str, int]`, *optional*):148 A map from label to index for the model.149 num_labels (`int`, *optional*):150 Number of labels to use in the last layer added to the model, typically for a classification task.151 task_specific_params (`dict[str, Any]`, *optional*):152 Additional keyword arguments to store for the current task.153 problem_type (`str`, *optional*):154 Problem type for `XxxForSequenceClassification` models. Can be one of `"regression"`,155 `"single_label_classification"` or `"multi_label_classification"`.156 157 > Parameters linked to the tokenizer158 159 tokenizer_class (`str`, *optional*):160 The name of the associated tokenizer class to use (if none is set, will use the tokenizer associated to the161 model by default).162 prefix (`str`, *optional*):163 A specific prompt that should be added at the beginning of each text before calling the model.164 bos_token_id (`int`, *optional*):165 The id of the _beginning-of-stream_ token.166 pad_token_id (`int`, *optional*):167 The id of the _padding_ token.168 eos_token_id (`int`, *optional*):169 The id of the _end-of-stream_ token.170 decoder_start_token_id (`int`, *optional*):171 If an encoder-decoder model starts decoding with a different token than _bos_, the id of that token.172 sep_token_id (`int`, *optional*):173 The id of the _separation_ token.174 175 > PyTorch specific parameters176 177 torchscript (`bool`, *optional*, defaults to `False`):178 Whether or not the model should be used with Torchscript.179 tie_word_embeddings (`bool`, *optional*, defaults to `True`):180 Whether the model's input and output word embeddings should be tied. Note that this is only relevant if the181 model has a output word embedding layer.182 dtype (`str`, *optional*):183 The `dtype` of the weights. This attribute can be used to initialize the model to a non-default `dtype`184 (which is normally `float32`) and thus allow for optimal storage allocation. For example, if the saved185 model is `float16`, ideally we want to load it back using the minimal amount of memory needed to load186 `float16` weights.187 """188 189 model_type: str = ""190 base_config_key: str = ""191 sub_configs: dict[str, type["PretrainedConfig"]] = {}192 has_no_defaults_at_init: bool = False193 attribute_map: dict[str, str] = {}194 base_model_tp_plan: Optional[dict[str, Any]] = None195 base_model_pp_plan: Optional[dict[str, tuple[list[str]]]] = None196 base_model_ep_plan: Optional[dict[str, tuple[list[str]]]] = None197 _auto_class: Optional[str] = None198 199 def __setattr__(self, key, value):200 if key in super().__getattribute__("attribute_map"):201 key = super().__getattribute__("attribute_map")[key]202 super().__setattr__(key, value)203 204 def __getattribute__(self, key):205 if key != "attribute_map" and key in super().__getattribute__("attribute_map"):206 key = super().__getattribute__("attribute_map")[key]207 return super().__getattribute__(key)208 209 def __init__(210 self,211 *,212 # All models common arguments213 output_hidden_states: bool = False,214 output_attentions: bool = False,215 return_dict: bool = True,216 torchscript: bool = False,217 dtype: Optional[Union[str, "torch.dtype"]] = None,218 # Common arguments219 pruned_heads: Optional[dict[int, list[int]]] = None,220 tie_word_embeddings: bool = True,221 chunk_size_feed_forward: int = 0,222 is_encoder_decoder: bool = False,223 is_decoder: bool = False,224 cross_attention_hidden_size: Optional[int] = None,225 add_cross_attention: bool = False,226 tie_encoder_decoder: bool = False,227 # Fine-tuning task arguments228 architectures: Optional[list[str]] = None,229 finetuning_task: Optional[str] = None,230 id2label: Optional[dict[int, str]] = None,231 label2id: Optional[dict[str, int]] = None,232 num_labels: Optional[int] = None,233 task_specific_params: Optional[dict[str, Any]] = None,234 problem_type: Optional[str] = None,235 # Tokenizer kwargs236 tokenizer_class: Optional[str] = None,237 prefix: Optional[str] = None,238 bos_token_id: Optional[int] = None,239 pad_token_id: Optional[int] = None,240 eos_token_id: Optional[int] = None,241 sep_token_id: Optional[int] = None,242 decoder_start_token_id: Optional[int] = None,243 **kwargs,244 ):245 # Validation for some arguments246 if label2id is not None and not isinstance(label2id, dict):247 raise ValueError("Argument label2id should be a dictionary.")248 if id2label is not None and not isinstance(id2label, dict):249 raise ValueError("Argument id2label should be a dictionary.")250 if num_labels is not None and id2label is not None and len(id2label) != num_labels:251 logger.warning(252 f"You passed `num_labels={num_labels}` which is incompatible to "253 f"the `id2label` map of length `{len(id2label)}`."254 )255 if problem_type is not None and problem_type not in (256 "regression",257 "single_label_classification",258 "multi_label_classification",259 ):260 raise ValueError(261 f"The config parameter `problem_type` was not understood: received {problem_type} "262 "but only 'regression', 'single_label_classification' and 'multi_label_classification' are valid."263 )264 # BC for the `torch_dtype` argument instead of the simpler `dtype`265 # Do not warn, as it would otherwise always be triggered since most configs on the hub have `torch_dtype`266 if (torch_dtype := kwargs.pop("torch_dtype", None)) is not None:267 # If both are provided, keep `dtype`268 dtype = dtype if dtype is not None else torch_dtype269 if dtype is not None and isinstance(dtype, str) and is_torch_available():270 # we will start using self.dtype in v5, but to be consistent with271 # from_pretrained's dtype arg convert it to an actual torch.dtype object272 import torch273 274 dtype = getattr(torch, dtype)275 276 # Attributes common for all models277 self.return_dict = return_dict278 self.output_hidden_states = output_hidden_states279 self.torchscript = torchscript280 self.dtype = dtype281 self._output_attentions = output_attentions # has public property282 283 # Less common kwargs, only used by some models284 self.pruned_heads = pruned_heads if pruned_heads is not None else {}285 self.tie_word_embeddings = tie_word_embeddings286 self.chunk_size_feed_forward = chunk_size_feed_forward287 288 # Encoder-decoder models attributes289 self.is_encoder_decoder = is_encoder_decoder290 self.is_decoder = is_decoder # used in encoder-decoder models to differentiate encoder from decoder291 self.cross_attention_hidden_size = cross_attention_hidden_size292 self.add_cross_attention = add_cross_attention293 self.tie_encoder_decoder = tie_encoder_decoder294 295 # Fine-tuning task attributes296 self.architectures = architectures297 self.finetuning_task = finetuning_task298 self.id2label = id2label299 self.label2id = label2id300 self.task_specific_params = task_specific_params301 self.problem_type = problem_type302 303 if self.id2label is None:304 self._create_id_label_maps(num_labels if num_labels is not None else 2)305 else:306 # Keys are always strings in JSON so convert ids to int here.307 self.id2label = {int(key): value for key, value in self.id2label.items()}308 309 # Tokenizer attributes310 self.tokenizer_class = tokenizer_class311 self.prefix = prefix312 self.bos_token_id = bos_token_id313 self.pad_token_id = pad_token_id314 self.eos_token_id = eos_token_id315 self.sep_token_id = sep_token_id316 self.decoder_start_token_id = decoder_start_token_id317 318 # Retrocompatibility: Parameters for sequence generation. While we will keep the ability to load these319 # parameters, saving them will be deprecated. In a distant future, we won't need to load them.320 for parameter_name, default_value in self._get_global_generation_defaults().items():321 setattr(self, parameter_name, kwargs.pop(parameter_name, default_value))322 323 # Name or path to the pretrained checkpoint324 self._name_or_path = str(kwargs.pop("name_or_path", ""))325 self._commit_hash = kwargs.pop("_commit_hash", None)326 327 # Attention implementation to use, if relevant (it sets it recursively on sub-configs)328 self._attn_implementation = kwargs.pop("attn_implementation", None)329 330 # Drop the transformers version info331 self.transformers_version = kwargs.pop("transformers_version", None)332 333 # Deal with gradient checkpointing334 if kwargs.get("gradient_checkpointing", False):335 warnings.warn(336 "Passing `gradient_checkpointing` to a config initialization is deprecated and will be removed in v5 "337 "Transformers. Using `model.gradient_checkpointing_enable()` instead, or if you are using the "338 "`Trainer` API, pass `gradient_checkpointing=True` in your `TrainingArguments`."339 )340 341 # Additional attributes without default values342 for key, value in kwargs.items():343 try:344 setattr(self, key, value)345 except AttributeError as err:346 logger.error(f"Can't set {key} with value {value} for {self}")347 raise err348 349 # TODO: remove later, deprecated arguments for TF models350 self.tf_legacy_loss = kwargs.pop("tf_legacy_loss", False)351 self.use_bfloat16 = kwargs.pop("use_bfloat16", False)352 353 def _create_id_label_maps(self, num_labels: int):354 self.id2label = {i: f"LABEL_{i}" for i in range(num_labels)}355 self.label2id = dict(zip(self.id2label.values(), self.id2label.keys()))356 357 @property358 def name_or_path(self) -> Optional[str]:359 return getattr(self, "_name_or_path", None)360 361 @name_or_path.setter362 def name_or_path(self, value):363 self._name_or_path = str(value) # Make sure that name_or_path is a string (for JSON encoding)364 365 @property366 def output_attentions(self):367 """368 `bool`: Whether or not the model should returns all attentions.369 """370 return self._output_attentions371 372 @output_attentions.setter373 def output_attentions(self, value: bool):374 # If we set `output_attentions` explicitly before the attn implementation, dispatch eager375 if value and self._attn_implementation is None:376 self._attn_implementation = "eager"377 if value and self._attn_implementation != "eager":378 raise ValueError(379 "The `output_attentions` attribute is not supported when using the `attn_implementation` set to "380 f"{self._attn_implementation}. Please set it to 'eager' instead."381 )382 self._output_attentions = value383 384 @property385 def use_return_dict(self) -> bool:386 """387 `bool`: Whether or not return [`~utils.ModelOutput`] instead of tuples.388 """389 # If torchscript is set, force `return_dict=False` to avoid jit errors390 return self.return_dict and not self.torchscript391 392 @property393 def num_labels(self) -> int:394 """395 `int`: The number of labels for classification models.396 """397 return len(self.id2label)398 399 @num_labels.setter400 def num_labels(self, num_labels: int):401 # we do not store `num_labels` attribute in config, but instead402 # compute it based on the length of the `id2label` map403 if self.id2label is None or self.num_labels != num_labels:404 self._create_id_label_maps(num_labels)405 406 @property407 def _attn_implementation(self):408 return self._attn_implementation_internal409 410 @_attn_implementation.setter411 def _attn_implementation(self, value: Optional[Union[str, dict]]):412 """We set it recursively on the sub-configs as well"""413 # Set if for current config414 current_attn = getattr(self, "_attn_implementation", None)415 attn_implementation = value if not isinstance(value, dict) else value.get("", current_attn)416 self._attn_implementation_internal = attn_implementation417 418 # Set it recursively on the subconfigs419 for subconfig_key in self.sub_configs:420 subconfig = getattr(self, subconfig_key, None)421 if subconfig is not None:422 current_subconfig_attn = getattr(subconfig, "_attn_implementation", None)423 sub_implementation = (424 value if not isinstance(value, dict) else value.get(subconfig_key, current_subconfig_attn)425 )426 subconfig._attn_implementation = sub_implementation427 428 @property429 def torch_dtype(self):430 logger.warning_once("`torch_dtype` is deprecated! Use `dtype` instead!")431 return self.dtype432 433 @torch_dtype.setter434 def torch_dtype(self, value):435 logger.warning_once("`torch_dtype` is deprecated! Use `dtype` instead!")436 self.dtype = value437 438 def save_pretrained(self, save_directory: Union[str, os.PathLike], push_to_hub: bool = False, **kwargs):439 """440 Save a configuration object to the directory `save_directory`, so that it can be re-loaded using the441 [`~PretrainedConfig.from_pretrained`] class method.442 443 Args:444 save_directory (`str` or `os.PathLike`):445 Directory where the configuration JSON file will be saved (will be created if it does not exist).446 push_to_hub (`bool`, *optional*, defaults to `False`):447 Whether or not to push your model to the Hugging Face model hub after saving it. You can specify the448 repository you want to push to with `repo_id` (will default to the name of `save_directory` in your449 namespace).450 kwargs (`dict[str, Any]`, *optional*):451 Additional key word arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method.452 """453 self._set_token_in_kwargs(kwargs)454 455 if os.path.isfile(save_directory):456 raise AssertionError(f"Provided path ({save_directory}) should be a directory, not a file")457 458 non_default_generation_parameters = self._get_non_default_generation_parameters()459 if len(non_default_generation_parameters) > 0:460 # TODO (joao): this should be an exception if the user has modified the loaded config. See #33886461 warnings.warn(462 "Some non-default generation parameters are set in the model config. These should go into either a) "463 "`model.generation_config` (as opposed to `model.config`); OR b) a GenerationConfig file "464 "(https://huggingface.co/docs/transformers/generation_strategies#save-a-custom-decoding-strategy-with-your-model)."465 "This warning will become an exception in the future."466 f"\nNon-default generation parameters: {str(non_default_generation_parameters)}",467 UserWarning,468 )469 470 os.makedirs(save_directory, exist_ok=True)471 472 if push_to_hub:473 commit_message = kwargs.pop("commit_message", None)474 repo_id = kwargs.pop("repo_id", save_directory.split(os.path.sep)[-1])475 repo_id = self._create_repo(repo_id, **kwargs)476 files_timestamps = self._get_files_timestamps(save_directory)477 478 # This attribute is important to know on load, but should not be serialized on save.479 if "transformers_weights" in self:480 delattr(self, "transformers_weights")481 482 # If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be483 # loaded from the Hub.484 if self._auto_class is not None:485 custom_object_save(self, save_directory, config=self)486 487 # If we save using the predefined names, we can load using `from_pretrained`488 output_config_file = os.path.join(save_directory, CONFIG_NAME)489 490 self.to_json_file(output_config_file, use_diff=True)491 logger.info(f"Configuration saved in {output_config_file}")492 493 if push_to_hub:494 self._upload_modified_files(495 save_directory,496 repo_id,497 files_timestamps,498 commit_message=commit_message,499 token=kwargs.get("token"),500 )501 502 @staticmethod503 def _set_token_in_kwargs(kwargs, token=None):504 """Temporary method to deal with `token` and `use_auth_token`.505 506 This method is to avoid apply the same changes in all model config classes that overwrite `from_pretrained`.507 508 Need to clean up `use_auth_token` in a follow PR.509 """510 # Some model config classes like CLIP define their own `from_pretrained` without the new argument `token` yet.511 if token is None:512 token = kwargs.pop("token", None)513 use_auth_token = kwargs.pop("use_auth_token", None)514 515 if use_auth_token is not None:516 warnings.warn(517 "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.",518 FutureWarning,519 )520 if token is not None:521 raise ValueError(522 "`token` and `use_auth_token` are both specified. Please set only the argument `token`."523 )524 token = use_auth_token525 526 if token is not None:527 kwargs["token"] = token528 529 @classmethod530 def from_pretrained(531 cls: type[SpecificPretrainedConfigType],532 pretrained_model_name_or_path: Union[str, os.PathLike],533 cache_dir: Optional[Union[str, os.PathLike]] = None,534 force_download: bool = False,535 local_files_only: bool = False,536 token: Optional[Union[str, bool]] = None,537 revision: str = "main",538 **kwargs,539 ) -> SpecificPretrainedConfigType:540 r"""541 Instantiate a [`PretrainedConfig`] (or a derived class) from a pretrained model configuration.542 543 Args:544 pretrained_model_name_or_path (`str` or `os.PathLike`):545 This can be either:546 547 - a string, the *model id* of a pretrained model configuration hosted inside a model repo on548 huggingface.co.549 - a path to a *directory* containing a configuration file saved using the550 [`~PretrainedConfig.save_pretrained`] method, e.g., `./my_model_directory/`.551 - a path or url to a saved configuration JSON *file*, e.g., `./my_model_directory/configuration.json`.552 cache_dir (`str` or `os.PathLike`, *optional*):553 Path to a directory in which a downloaded pretrained model configuration should be cached if the554 standard cache should not be used.555 force_download (`bool`, *optional*, defaults to `False`):556 Whether or not to force to (re-)download the configuration files and override the cached versions if557 they exist.558 resume_download:559 Deprecated and ignored. All downloads are now resumed by default when possible.560 Will be removed in v5 of Transformers.561 proxies (`dict[str, str]`, *optional*):562 A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',563 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.564 token (`str` or `bool`, *optional*):565 The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use566 the token generated when running `hf auth login` (stored in `~/.huggingface`).567 revision (`str`, *optional*, defaults to `"main"`):568 The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a569 git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any570 identifier allowed by git.571 572 <Tip>573 574 To test a pull request you made on the Hub, you can pass `revision="refs/pr/<pr_number>"`.575 576 </Tip>577 578 return_unused_kwargs (`bool`, *optional*, defaults to `False`):579 If `False`, then this function returns just the final configuration object.580 581 If `True`, then this functions returns a `Tuple(config, unused_kwargs)` where *unused_kwargs* is a582 dictionary consisting of the key/value pairs whose keys are not configuration attributes: i.e., the583 part of `kwargs` which has not been used to update `config` and is otherwise ignored.584 subfolder (`str`, *optional*, defaults to `""`):585 In case the relevant files are located inside a subfolder of the model repo on huggingface.co, you can586 specify the folder name here.587 kwargs (`dict[str, Any]`, *optional*):588 The values in kwargs of any keys which are configuration attributes will be used to override the loaded589 values. Behavior concerning key/value pairs whose keys are *not* configuration attributes is controlled590 by the `return_unused_kwargs` keyword parameter.591 592 Returns:593 [`PretrainedConfig`]: The configuration object instantiated from this pretrained model.594 595 Examples:596 597 ```python598 # We can't instantiate directly the base class *PretrainedConfig* so let's show the examples on a599 # derived class: BertConfig600 config = BertConfig.from_pretrained(601 "google-bert/bert-base-uncased"602 ) # Download configuration from huggingface.co and cache.603 config = BertConfig.from_pretrained(604 "./test/saved_model/"605 ) # E.g. config (or model) was saved using *save_pretrained('./test/saved_model/')*606 config = BertConfig.from_pretrained("./test/saved_model/my_configuration.json")607 config = BertConfig.from_pretrained("google-bert/bert-base-uncased", output_attentions=True, foo=False)608 assert config.output_attentions == True609 config, unused_kwargs = BertConfig.from_pretrained(610 "google-bert/bert-base-uncased", output_attentions=True, foo=False, return_unused_kwargs=True611 )612 assert config.output_attentions == True613 assert unused_kwargs == {"foo": False}614 ```"""615 kwargs["cache_dir"] = cache_dir616 kwargs["force_download"] = force_download617 kwargs["local_files_only"] = local_files_only618 kwargs["revision"] = revision619 620 cls._set_token_in_kwargs(kwargs, token)621 622 config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)623 if cls.base_config_key and cls.base_config_key in config_dict:624 config_dict = config_dict[cls.base_config_key]625 626 if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:627 # sometimes the config has no `base_config_key` if the config is used in several composite models628 # e.g. LlamaConfig. In that case we try to see if there is match in `model_type` before raising a warning629 for v in config_dict.values():630 if isinstance(v, dict) and v.get("model_type") == cls.model_type:631 config_dict = v632 633 # raise warning only if we still can't see a match in `model_type`634 if config_dict["model_type"] != cls.model_type:635 logger.warning(636 f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "637 f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."638 )639 640 return cls.from_dict(config_dict, **kwargs)641 642 @classmethod643 def get_config_dict(644 cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs645 ) -> tuple[dict[str, Any], dict[str, Any]]:646 """647 From a `pretrained_model_name_or_path`, resolve to a dictionary of parameters, to be used for instantiating a648 [`PretrainedConfig`] using `from_dict`.649 650 Parameters:651 pretrained_model_name_or_path (`str` or `os.PathLike`):652 The identifier of the pre-trained checkpoint from which we want the dictionary of parameters.653 654 Returns:655 `tuple[Dict, Dict]`: The dictionary(ies) that will be used to instantiate the configuration object.656 657 """658 cls._set_token_in_kwargs(kwargs)659 660 original_kwargs = copy.deepcopy(kwargs)661 # Get config dict associated with the base config file662 config_dict, kwargs = cls._get_config_dict(pretrained_model_name_or_path, **kwargs)663 if config_dict is None:664 return {}, kwargs665 if "_commit_hash" in config_dict:666 original_kwargs["_commit_hash"] = config_dict["_commit_hash"]667 668 # That config file may point us toward another config file to use.669 if "configuration_files" in config_dict:670 configuration_file = get_configuration_file(config_dict["configuration_files"])671 config_dict, kwargs = cls._get_config_dict(672 pretrained_model_name_or_path, _configuration_file=configuration_file, **original_kwargs673 )674 675 return config_dict, kwargs676 677 @classmethod678 def _get_config_dict(679 cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs680 ) -> tuple[dict[str, Any], dict[str, Any]]:681 cache_dir = kwargs.pop("cache_dir", None)682 force_download = kwargs.pop("force_download", False)683 resume_download = kwargs.pop("resume_download", None)684 proxies = kwargs.pop("proxies", None)685 token = kwargs.pop("token", None)686 local_files_only = kwargs.pop("local_files_only", False)687 revision = kwargs.pop("revision", None)688 trust_remote_code = kwargs.pop("trust_remote_code", None)689 subfolder = kwargs.pop("subfolder", "")690 from_pipeline = kwargs.pop("_from_pipeline", None)691 from_auto_class = kwargs.pop("_from_auto", False)692 commit_hash = kwargs.pop("_commit_hash", None)693 694 gguf_file = kwargs.get("gguf_file")695 696 if trust_remote_code is True:697 logger.warning(698 "The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is"699 " ignored."700 )701 702 user_agent = {"file_type": "config", "from_auto_class": from_auto_class}703 if from_pipeline is not None:704 user_agent["using_pipeline"] = from_pipeline705 706 pretrained_model_name_or_path = str(pretrained_model_name_or_path)707 708 is_local = os.path.isdir(pretrained_model_name_or_path)709 if os.path.isfile(os.path.join(subfolder, pretrained_model_name_or_path)):710 # Special case when pretrained_model_name_or_path is a local file711 resolved_config_file = pretrained_model_name_or_path712 is_local = True713 elif is_remote_url(pretrained_model_name_or_path):714 configuration_file = pretrained_model_name_or_path if gguf_file is None else gguf_file715 resolved_config_file = download_url(pretrained_model_name_or_path)716 else:717 configuration_file = kwargs.pop("_configuration_file", CONFIG_NAME) if gguf_file is None else gguf_file718 719 try:720 # Load from local folder or from cache or download from model Hub and cache721 resolved_config_file = cached_file(722 pretrained_model_name_or_path,723 configuration_file,724 cache_dir=cache_dir,725 force_download=force_download,726 proxies=proxies,727 resume_download=resume_download,728 local_files_only=local_files_only,729 token=token,730 user_agent=user_agent,731 revision=revision,732 subfolder=subfolder,733 _commit_hash=commit_hash,734 )735 if resolved_config_file is None:736 return None, kwargs737 commit_hash = extract_commit_hash(resolved_config_file, commit_hash)738 except OSError:739 # Raise any environment error raise by `cached_file`. It will have a helpful error message adapted to740 # the original exception.741 raise742 except Exception:743 # For any other exception, we throw a generic error.744 raise OSError(745 f"Can't load the configuration of '{pretrained_model_name_or_path}'. If you were trying to load it"746 " from 'https://huggingface.co/models', make sure you don't have a local directory with the same"747 f" name. Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a directory"748 f" containing a {configuration_file} file"749 )750 751 try:752 if gguf_file:753 config_dict = load_gguf_checkpoint(resolved_config_file, return_tensors=False)["config"]754 else:755 # Load config dict756 config_dict = cls._dict_from_json_file(resolved_config_file)757 758 config_dict["_commit_hash"] = commit_hash759 except (json.JSONDecodeError, UnicodeDecodeError):760 raise OSError(f"It looks like the config file at '{resolved_config_file}' is not a valid JSON file.")761 762 if is_local:763 logger.info(f"loading configuration file {resolved_config_file}")764 else:765 logger.info(f"loading configuration file {configuration_file} from cache at {resolved_config_file}")766 767 # timm models are not saved with the model_type in the config file768 if "model_type" not in config_dict and is_timm_config_dict(config_dict):769 config_dict["model_type"] = "timm_wrapper"770 771 return config_dict, kwargs772 773 @classmethod774 def from_dict(775 cls: type[SpecificPretrainedConfigType], config_dict: dict[str, Any], **kwargs776 ) -> SpecificPretrainedConfigType:777 """778 Instantiates a [`PretrainedConfig`] from a Python dictionary of parameters.779 780 Args:781 config_dict (`dict[str, Any]`):782 Dictionary that will be used to instantiate the configuration object. Such a dictionary can be783 retrieved from a pretrained checkpoint by leveraging the [`~PretrainedConfig.get_config_dict`] method.784 kwargs (`dict[str, Any]`):785 Additional parameters from which to initialize the configuration object.786 787 Returns:788 [`PretrainedConfig`]: The configuration object instantiated from those parameters.789 """790 return_unused_kwargs = kwargs.pop("return_unused_kwargs", False)791 # Those arguments may be passed along for our internal telemetry.792 # We remove them so they don't appear in `return_unused_kwargs`.793 kwargs.pop("_from_auto", None)794 kwargs.pop("_from_pipeline", None)795 # The commit hash might have been updated in the `config_dict`, we don't want the kwargs to erase that update.796 if "_commit_hash" in kwargs and "_commit_hash" in config_dict:797 kwargs["_commit_hash"] = config_dict["_commit_hash"]798 799 # For BC on the old `torch_dtype`800 if (torch_dtype := kwargs.pop("torch_dtype", None)) is not None:801 logger.warning_once("`torch_dtype` is deprecated! Use `dtype` instead!")802 # If both are present, use `dtype`803 kwargs["dtype"] = kwargs.get("dtype", torch_dtype)804 805 # We remove it from kwargs so that it does not appear in `return_unused_kwargs`.806 config_dict["attn_implementation"] = kwargs.pop("attn_implementation", None)807 808 config = cls(**config_dict)809 810 if hasattr(config, "pruned_heads"):811 config.pruned_heads = {int(key): value for key, value in config.pruned_heads.items()}812 813 # Update config with kwargs if needed814 if "num_labels" in kwargs and "id2label" in kwargs:815 num_labels = kwargs["num_labels"]816 id2label = kwargs["id2label"] if kwargs["id2label"] is not None else []817 if len(id2label) != num_labels:818 raise ValueError(819 f"You passed along `num_labels={num_labels}` with an incompatible id to label map: "820 f"{kwargs['id2label']}. Since those arguments are inconsistent with each other, you should remove "821 "one of them."822 )823 to_remove = []824 for key, value in kwargs.items():825 if hasattr(config, key):826 current_attr = getattr(config, key)827 # To authorize passing a custom subconfig as kwarg in models that have nested configs.828 # We need to update only custom kwarg values instead and keep other attributes in subconfig.829 if isinstance(current_attr, PretrainedConfig) and isinstance(value, dict):830 current_attr_updated = current_attr.to_dict()831 current_attr_updated.update(value)832 value = current_attr.__class__(**current_attr_updated)833 setattr(config, key, value)834 if key != "dtype":835 to_remove.append(key)836 for key in to_remove:837 kwargs.pop(key, None)838 839 logger.info(f"Model config {config}")840 if return_unused_kwargs:841 return config, kwargs842 else:843 return config844 845 @classmethod846 def from_json_file(847 cls: type[SpecificPretrainedConfigType], json_file: Union[str, os.PathLike]848 ) -> SpecificPretrainedConfigType:849 """850 Instantiates a [`PretrainedConfig`] from the path to a JSON file of parameters.851 852 Args:853 json_file (`str` or `os.PathLike`):854 Path to the JSON file containing the parameters.855 856 Returns:857 [`PretrainedConfig`]: The configuration object instantiated from that JSON file.858 859 """860 config_dict = cls._dict_from_json_file(json_file)861 return cls(**config_dict)862 863 @classmethod864 def _dict_from_json_file(cls, json_file: Union[str, os.PathLike]):865 with open(json_file, encoding="utf-8") as reader:866 text = reader.read()867 return json.loads(text)868 869 def __eq__(self, other):870 return isinstance(other, PretrainedConfig) and (self.__dict__ == other.__dict__)871 872 def __repr__(self):873 return f"{self.__class__.__name__} {self.to_json_string()}"874 875 def __iter__(self):876 yield from self.__dict__877 878 def to_diff_dict(self) -> dict[str, Any]:879 """880 Removes all attributes from the configuration that correspond to the default config attributes for881 better readability, while always retaining the `config` attribute from the class. Serializes to a882 Python dictionary.883 884 Returns:885 dict[str, Any]: Dictionary of all the attributes that make up this configuration instance.886 """887 config_dict = self.to_dict()888 889 # Get the default config dict (from a fresh PreTrainedConfig instance)890 default_config_dict = PretrainedConfig().to_dict()891 892 # get class specific config dict893 class_config_dict = self.__class__().to_dict() if not self.has_no_defaults_at_init else {}894 895 serializable_config_dict = {}896 897 # Only serialize values that differ from the default config,898 # except always keep the 'config' attribute.899 for key, value in config_dict.items():900 if (901 isinstance(getattr(self, key, None), PretrainedConfig)902 and key in class_config_dict903 and isinstance(class_config_dict[key], dict)904 or key in self.sub_configs905 ):906 # For nested configs we need to clean the diff recursively907 diff = recursive_diff_dict(value, default_config_dict, config_obj=getattr(self, key, None))908 if "model_type" in value:909 # Needs to be set even if it's not in the diff910 diff["model_type"] = value["model_type"]911 912 serializable_config_dict[key] = diff913 elif (914 key not in default_config_dict915 or key == "transformers_version"916 or key == "vocab_file"917 or value != default_config_dict[key]918 or (key in default_config_dict and value != class_config_dict.get(key, value))919 ):920 serializable_config_dict[key] = value921 922 self._remove_keys_not_serialized(serializable_config_dict)923 924 # Key removed only in diff dict925 if "_name_or_path" in serializable_config_dict:926 del serializable_config_dict["_name_or_path"]927 928 if hasattr(self, "quantization_config"):929 serializable_config_dict["quantization_config"] = (930 self.quantization_config.to_dict()931 if not isinstance(self.quantization_config, dict)932 else self.quantization_config933 )934 self.dict_dtype_to_str(serializable_config_dict)935 936 return serializable_config_dict937 938 def to_dict(self) -> dict[str, Any]:939 """940 Serializes this instance to a Python dictionary.941 942 Returns:943 `dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance.944 """945 output = copy.deepcopy(self.__dict__)946 if hasattr(self.__class__, "model_type"):947 output["model_type"] = self.__class__.model_type948 949 # Transformers version when serializing the model950 output["transformers_version"] = __version__951 952 for key, value in output.items():953 # Deal with nested configs like CLIP954 if isinstance(value, PretrainedConfig):955 value = value.to_dict()956 del value["transformers_version"]957 958 output[key] = value959 960 self._remove_keys_not_serialized(output)961 962 if hasattr(self, "quantization_config"):963 output["quantization_config"] = (964 self.quantization_config.to_dict()965 if not isinstance(self.quantization_config, dict)966 else self.quantization_config967 )968 self.dict_dtype_to_str(output)969 970 return output971 972 def to_json_string(self, use_diff: bool = True) -> str:973 """974 Serializes this instance to a JSON string.975 976 Args:977 use_diff (`bool`, *optional*, defaults to `True`):978 If set to `True`, only the difference between the config instance and the default `PretrainedConfig()`979 is serialized to JSON string.980 981 Returns:982 `str`: String containing all the attributes that make up this configuration instance in JSON format.983 """984 if use_diff is True:985 config_dict = self.to_diff_dict()986 else:987 config_dict = self.to_dict()988 return json.dumps(config_dict, indent=2, sort_keys=True) + "\n"989 990 def to_json_file(self, json_file_path: Union[str, os.PathLike], use_diff: bool = True):991 """992 Save this instance to a JSON file.993 994 Args:995 json_file_path (`str` or `os.PathLike`):996 Path to the JSON file in which this configuration instance's parameters will be saved.997 use_diff (`bool`, *optional*, defaults to `True`):998 If set to `True`, only the difference between the config instance and the default `PretrainedConfig()`999 is serialized to JSON file.1000 """1001 with open(json_file_path, "w", encoding="utf-8") as writer:1002 writer.write(self.to_json_string(use_diff=use_diff))1003 1004 def update(self, config_dict: dict[str, Any]):1005 """1006 Updates attributes of this class with attributes from `config_dict`.1007 1008 Args:1009 config_dict (`dict[str, Any]`): Dictionary of attributes that should be updated for this class.1010 """1011 for key, value in config_dict.items():1012 setattr(self, key, value)1013 1014 def update_from_string(self, update_str: str):1015 """1016 Updates attributes of this class with attributes from `update_str`.1017 1018 The expected format is ints, floats and strings as is, and for booleans use `true` or `false`. For example:1019 "n_embd=10,resid_pdrop=0.2,scale_attn_weights=false,summary_type=cls_index"1020 1021 The keys to change have to already exist in the config object.1022 1023 Args:1024 update_str (`str`): String with attributes that should be updated for this class.1025 1026 """1027 1028 d = dict(x.split("=") for x in update_str.split(","))1029 for k, v in d.items():1030 if not hasattr(self, k):1031 raise ValueError(f"key {k} isn't in the original config dict")1032 1033 old_v = getattr(self, k)1034 if isinstance(old_v, bool):1035 if v.lower() in ["true", "1", "y", "yes"]:1036 v = True1037 elif v.lower() in ["false", "0", "n", "no"]:1038 v = False1039 else:1040 raise ValueError(f"can't derive true or false from {v} (key {k})")1041 elif isinstance(old_v, int):1042 v = int(v)1043 elif isinstance(old_v, float):1044 v = float(v)1045 elif not isinstance(old_v, str):1046 raise TypeError(1047 f"You can only update int, float, bool or string values in the config, got {v} for key {k}"1048 )1049 1050 setattr(self, k, v)1051 1052 def dict_dtype_to_str(self, d: dict[str, Any]) -> None:1053 """1054 Checks whether the passed dictionary and its nested dicts have a *dtype* key and if it's not None,1055 converts torch.dtype to a string of just the type. For example, `torch.float32` get converted into *"float32"*1056 string, which can then be stored in the json format.1057 """1058 if d.get("dtype") is not None:1059 if isinstance(d["dtype"], dict):1060 d["dtype"] = {k: str(v).split(".")[-1] for k, v in d["dtype"].items()}1061 # models like Emu3 can have "dtype" as token in config's vocabulary map,1062 # so we also exclude int type here to avoid error in this special case.1063 elif not isinstance(d["dtype"], (str, int)):1064 d["dtype"] = str(d["dtype"]).split(".")[1]1065 for value in d.values():1066 if isinstance(value, dict):1067 self.dict_dtype_to_str(value)1068 1069 def _remove_keys_not_serialized(self, d: dict[str, Any]) -> None:1070 """1071 Checks and removes if there are any keys in the dict that should not be serialized when saving the config.1072 Runs recursive check on the dict, to remove from all sub configs.1073 """1074 if hasattr(self, "quantization_config"):1075 # Pop the `_pre_quantization_dtype` as torch.dtypes are not serializable.1076 _ = d.pop("_pre_quantization_dtype", None)1077 1078 if "_auto_class" in d:1079 del d["_auto_class"]1080 if "_output_attentions" in d:1081 d["output_attentions"] = d.pop("_output_attentions")1082 if "_commit_hash" in d:1083 del d["_commit_hash"]1084 if "_attn_implementation_internal" in d:1085 del d["_attn_implementation_internal"]1086 # Do not serialize `base_model_tp_plan` for now1087 if "base_model_tp_plan" in d:1088 del d["base_model_tp_plan"]1089 # Do not serialize `base_model_pp_plan` for now1090 if "base_model_pp_plan" in d:1091 del d["base_model_pp_plan"]1092 for value in d.values():1093 if isinstance(value, dict):1094 self._remove_keys_not_serialized(value)1095 1096 @classmethod1097 def register_for_auto_class(cls, auto_class="AutoConfig"):1098 """1099 Register this class with a given auto class. This should only be used for custom configurations as the ones in1100 the library are already mapped with `AutoConfig`.1101 1102 1103 1104 Args:1105 auto_class (`str` or `type`, *optional*, defaults to `"AutoConfig"`):1106 The auto class to register this new configuration with.1107 """1108 if not isinstance(auto_class, str):1109 auto_class = auto_class.__name__1110 1111 import transformers.models.auto as auto_module1112 1113 if not hasattr(auto_module, auto_class):1114 raise ValueError(f"{auto_class} is not a valid auto class.")1115 1116 cls._auto_class = auto_class1117 1118 @staticmethod1119 def _get_global_generation_defaults() -> dict[str, Any]:1120 return {1121 "max_length": 20,1122 "min_length": 0,1123 "do_sample": False,1124 "early_stopping": False,1125 "num_beams": 1,1126 "temperature": 1.0,1127 "top_k": 50,1128 "top_p": 1.0,1129 "typical_p": 1.0,1130 "repetition_penalty": 1.0,1131 "length_penalty": 1.0,1132 "no_repeat_ngram_size": 0,1133 "encoder_no_repeat_ngram_size": 0,1134 "bad_words_ids": None,1135 "num_return_sequences": 1,1136 "output_scores": False,1137 "return_dict_in_generate": False,1138 "forced_bos_token_id": None,1139 "forced_eos_token_id": None,1140 "remove_invalid_values": False,1141 "exponential_decay_length_penalty": None,1142 "suppress_tokens": None,1143 "begin_suppress_tokens": None,1144 # Deprecated arguments (moved to the Hub). TODO joao, manuel: remove in v4.62.01145 "num_beam_groups": 1,1146 "diversity_penalty": 0.0,1147 }1148 1149 def _get_non_default_generation_parameters(self) -> dict[str, Any]:1150 """1151 Gets the non-default generation parameters on the PretrainedConfig instance1152 """1153 non_default_generation_parameters = {}1154 decoder_attribute_name = None1155 1156 # Composite models don't have a default config, use their decoder config as a fallback for default values1157 # If no known pattern is matched, then `default_config = None` -> check against the global generation defaults1158 try:1159 default_config = self.__class__()1160 except ValueError:1161 decoder_config = self.get_text_config(decoder=True)1162 if decoder_config is not self:1163 default_config = decoder_config.__class__()1164 else:1165 default_config = None1166 1167 # If it is a composite model, we want to check the subconfig that will be used for generation1168 self_decoder_config = self if decoder_attribute_name is None else getattr(self, decoder_attribute_name)1169 1170 for parameter_name, default_global_value in self._get_global_generation_defaults().items():1171 if hasattr(self_decoder_config, parameter_name):1172 is_default_in_config = is_default_generation_value = None1173 parameter_value = getattr(self_decoder_config, parameter_name)1174 # Three cases in which is okay for the model config to hold generation config parameters:1175 # 1. The parameter is set to `None`, effectively delegating its value to the generation config1176 if parameter_value is None:1177 continue1178 # 2. If we have a default config, then the instance should hold the same generation defaults1179 if default_config is not None:1180 is_default_in_config = parameter_value == getattr(default_config, parameter_name)1181 # 3. if we don't have a default config, then the instance should hold the global generation defaults1182 else:1183 is_default_generation_value = parameter_value == default_global_value1184 1185 is_non_default = (is_default_in_config is False) or (1186 is_default_in_config is None and is_default_generation_value is False1187 )1188 if is_non_default:1189 non_default_generation_parameters[parameter_name] = getattr(self_decoder_config, parameter_name)1190 1191 return non_default_generation_parameters1192 1193 def get_text_config(self, decoder=None, encoder=None) -> "PretrainedConfig":1194 """1195 Returns the text config related to the text input (encoder) or text output (decoder) of the model. The1196 `decoder` and `encoder` input arguments can be used to specify which end of the model we are interested in,1197 which is useful on models that have both text input and output modalities.1198 1199 There are three possible outcomes of using this method:1200 1. On most models, it returns the original config instance itself.