Aluode/PerceptionLabPortable
0
1# coding=utf-82# Copyright 2022 The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""Generation configuration class and utilities."""16 17import copy18import json19import os20import warnings21from abc import ABC, abstractmethod22from dataclasses import dataclass, is_dataclass23from typing import TYPE_CHECKING, Any, Callable, Optional, Union24 25from .. import __version__26from ..configuration_utils import PretrainedConfig27from ..utils import (28 GENERATION_CONFIG_NAME,29 ExplicitEnum,30 PushToHubMixin,31 cached_file,32 download_url,33 extract_commit_hash,34 is_remote_url,35 is_torch_available,36 logging,37)38 39 40if TYPE_CHECKING:41 from ..modeling_utils import PreTrainedModel42 43 44logger = logging.get_logger(__name__)45METADATA_FIELDS = ("_from_model_config", "_commit_hash", "_original_object_hash", "transformers_version")46STATIC_CACHE_IMPLEMENTATIONS = ("static", "offloaded_static")47DYNAMIC_CACHE_IMPLEMENTATIONS = ("dynamic", "dynamic_full", "offloaded", "quantized")48# All the following are redundant and deprecated, but kept for BC49DEPRECATED_STATIC_CACHE_IMPLEMENTATIONS = (50 "sliding_window",51 "hybrid",52 "hybrid_chunked",53 "offloaded_hybrid",54 "offloaded_hybrid_chunked",55)56ALL_STATIC_CACHE_IMPLEMENTATIONS = STATIC_CACHE_IMPLEMENTATIONS + DEPRECATED_STATIC_CACHE_IMPLEMENTATIONS57ALL_CACHE_IMPLEMENTATIONS = ALL_STATIC_CACHE_IMPLEMENTATIONS + DYNAMIC_CACHE_IMPLEMENTATIONS58 59 60if is_torch_available():61 from .logits_process import SynthIDTextWatermarkLogitsProcessor, WatermarkLogitsProcessor62 63 64class GenerationMode(ExplicitEnum):65 """66 Possible generation modes, downstream of the [`~generation.GenerationMixin.generate`] method.67 """68 69 # Non-beam methods70 CONTRASTIVE_SEARCH = "contrastive_search"71 GREEDY_SEARCH = "greedy_search"72 SAMPLE = "sample"73 ASSISTED_GENERATION = "assisted_generation"74 DOLA_GENERATION = "dola_generation"75 # Beam methods76 BEAM_SEARCH = "beam_search"77 BEAM_SAMPLE = "beam_sample"78 CONSTRAINED_BEAM_SEARCH = "constrained_beam_search"79 GROUP_BEAM_SEARCH = "group_beam_search"80 81 82class GenerationConfig(PushToHubMixin):83 # no-format84 """85 Class that holds a configuration for a generation task. A `generate` call supports the following generation methods86 for text-decoder, text-to-text, speech-to-text, and vision-to-text models:87 88 - *greedy decoding* if `num_beams=1` and `do_sample=False`89 - *multinomial sampling* if `num_beams=1` and `do_sample=True`90 - *beam-search decoding* if `num_beams>1` and `do_sample=False`91 - *beam-search multinomial sampling* if `num_beams>1` and `do_sample=True`92 - *assisted decoding* if `assistant_model` or `prompt_lookup_num_tokens` is passed to `.generate()`93 94 To learn more about decoding strategies refer to the [text generation strategies guide](../generation_strategies).95 96 <Tip>97 98 A large number of these flags control the logits or the stopping criteria of the generation. Make sure you check99 the [generate-related classes](https://huggingface.co/docs/transformers/internal/generation_utils) for a full100 description of the possible manipulations, as well as examples of their usage.101 102 </Tip>103 104 Arg:105 > Parameters that control the length of the output106 107 max_length (`int`, *optional*, defaults to 20):108 The maximum length the generated tokens can have. Corresponds to the length of the input prompt +109 `max_new_tokens`. Its effect is overridden by `max_new_tokens`, if also set.110 max_new_tokens (`int`, *optional*):111 The maximum numbers of tokens to generate, ignoring the number of tokens in the prompt.112 min_length (`int`, *optional*, defaults to 0):113 The minimum length of the sequence to be generated. Corresponds to the length of the input prompt +114 `min_new_tokens`. Its effect is overridden by `min_new_tokens`, if also set.115 min_new_tokens (`int`, *optional*):116 The minimum numbers of tokens to generate, ignoring the number of tokens in the prompt.117 early_stopping (`bool` or `str`, *optional*, defaults to `False`):118 Controls the stopping condition for beam-based methods, like beam-search. It accepts the following values:119 `True`, where the generation stops as soon as there are `num_beams` complete candidates; `False`, where an120 heuristic is applied and the generation stops when is it very unlikely to find better candidates;121 `"never"`, where the beam search procedure only stops when there cannot be better candidates (canonical122 beam search algorithm).123 max_time (`float`, *optional*):124 The maximum amount of time you allow the computation to run for in seconds. generation will still finish125 the current pass after allocated time has been passed.126 stop_strings (`str or list[str]`, *optional*):127 A string or a list of strings that should terminate generation if the model outputs them.128 129 > Parameters that control the generation strategy used130 131 do_sample (`bool`, *optional*, defaults to `False`):132 Whether or not to use sampling ; use greedy decoding otherwise.133 num_beams (`int`, *optional*, defaults to 1):134 Number of beams for beam search. 1 means no beam search.135 136 > Parameters that control the cache137 138 use_cache (`bool`, *optional*, defaults to `True`):139 Whether or not the model should use the past last key/values attentions (if applicable to the model) to140 speed up decoding.141 cache_implementation (`str`, *optional*, default to `None`):142 Name of the cache class that will be instantiated in `generate`, for faster decoding. Possible values are:143 144 - `"dynamic"`: [`DynamicCache`]145 - `"static"`: [`StaticCache`]146 - `"offloaded"`: [`DynamicCache(offloaded=True)`]147 - `"offloaded_static"`: [`StaticCache(offloaded=True)`]148 - `"quantized"`: [`QuantizedCache`]149 150 If none is specified, we will use the default cache for the model (which is often [`DynamicCache`]). See151 our [cache documentation](https://huggingface.co/docs/transformers/en/kv_cache) for further information.152 cache_config (`dict`, *optional*, default to `None`):153 Arguments used in the key-value cache class can be passed in `cache_config`.154 return_legacy_cache (`bool`, *optional*, default to `True`):155 Whether to return the legacy or new format of the cache when `DynamicCache` is used by default.156 157 > Parameters for manipulation of the model output logits158 159 temperature (`float`, *optional*, defaults to 1.0):160 The value used to module the next token probabilities. This value is set in a model's `generation_config.json` file. If it isn't set, the default value is 1.0161 top_k (`int`, *optional*, defaults to 50):162 The number of highest probability vocabulary tokens to keep for top-k-filtering. This value is set in a model's `generation_config.json` file. If it isn't set, the default value is 50.163 top_p (`float`, *optional*, defaults to 1.0):164 If set to float < 1, only the smallest set of most probable tokens with probabilities that add up to165 `top_p` or higher are kept for generation. This value is set in a model's `generation_config.json` file. If it isn't set, the default value is 1.0166 min_p (`float`, *optional*):167 Minimum token probability, which will be scaled by the probability of the most likely token. It must be a168 value between 0 and 1. Typical values are in the 0.01-0.2 range, comparably selective as setting `top_p` in169 the 0.99-0.8 range (use the opposite of normal `top_p` values).170 typical_p (`float`, *optional*, defaults to 1.0):171 Local typicality measures how similar the conditional probability of predicting a target token next is to172 the expected conditional probability of predicting a random token next, given the partial text already173 generated. If set to float < 1, the smallest set of the most locally typical tokens with probabilities that174 add up to `typical_p` or higher are kept for generation. See [this175 paper](https://huggingface.co/papers/2202.00666) for more details.176 epsilon_cutoff (`float`, *optional*, defaults to 0.0):177 If set to float strictly between 0 and 1, only tokens with a conditional probability greater than178 `epsilon_cutoff` will be sampled. In the paper, suggested values range from 3e-4 to 9e-4, depending on the179 size of the model. See [Truncation Sampling as Language Model180 Desmoothing](https://huggingface.co/papers/2210.15191) for more details.181 eta_cutoff (`float`, *optional*, defaults to 0.0):182 Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to float strictly between183 0 and 1, a token is only considered if it is greater than either `eta_cutoff` or `sqrt(eta_cutoff) *184 exp(-entropy(softmax(next_token_logits)))`. The latter term is intuitively the expected next token185 probability, scaled by `sqrt(eta_cutoff)`. In the paper, suggested values range from 3e-4 to 2e-3,186 depending on the size of the model. See [Truncation Sampling as Language Model187 Desmoothing](https://huggingface.co/papers/2210.15191) for more details.188 repetition_penalty (`float`, *optional*, defaults to 1.0):189 The parameter for repetition penalty. 1.0 means no penalty. See [this190 paper](https://huggingface.co/papers/1909.05858) for more details.191 encoder_repetition_penalty (`float`, *optional*, defaults to 1.0):192 The parameter for encoder_repetition_penalty. An exponential penalty on sequences that are not in the193 original input. 1.0 means no penalty.194 length_penalty (`float`, *optional*, defaults to 1.0):195 Exponential penalty to the length that is used with beam-based generation. It is applied as an exponent to196 the sequence length, which in turn is used to divide the score of the sequence. Since the score is the log197 likelihood of the sequence (i.e. negative), `length_penalty` > 0.0 promotes longer sequences, while198 `length_penalty` < 0.0 encourages shorter sequences.199 no_repeat_ngram_size (`int`, *optional*, defaults to 0):200 If set to int > 0, all ngrams of that size can only occur once.201 bad_words_ids (`list[list[int]]`, *optional*):202 List of list of token ids that are not allowed to be generated. Check203 [`~generation.NoBadWordsLogitsProcessor`] for further documentation and examples.204 renormalize_logits (`bool`, *optional*, defaults to `False`):205 Whether to renormalize the logits after applying all the logits processors (including the custom206 ones). It's highly recommended to set this flag to `True` as the search algorithms suppose the score logits207 are normalized but some logit processors break the normalization.208 forced_bos_token_id (`int`, *optional*, defaults to `model.config.forced_bos_token_id`):209 The id of the token to force as the first generated token after the `decoder_start_token_id`. Useful for210 multilingual models like [mBART](../model_doc/mbart) where the first generated token needs to be the target211 language token.212 forced_eos_token_id (`int` or list[int]`, *optional*, defaults to `model.config.forced_eos_token_id`):213 The id of the token to force as the last generated token when `max_length` is reached. Optionally, use a214 list to set multiple *end-of-sequence* tokens.215 remove_invalid_values (`bool`, *optional*, defaults to `model.config.remove_invalid_values`):216 Whether to remove possible *nan* and *inf* outputs of the model to prevent the generation method to crash.217 Note that using `remove_invalid_values` can slow down generation.218 exponential_decay_length_penalty (`tuple(int, float)`, *optional*):219 This Tuple adds an exponentially increasing length penalty, after a certain amount of tokens have been220 generated. The tuple shall consist of: `(start_index, decay_factor)` where `start_index` indicates where221 penalty starts and `decay_factor` represents the factor of exponential decay222 suppress_tokens (`list[int]`, *optional*):223 A list of tokens that will be suppressed at generation. The `SuppressTokens` logit processor will set their224 log probs to `-inf` so that they are not sampled.225 begin_suppress_tokens (`list[int]`, *optional*):226 A list of tokens that will be suppressed at the beginning of the generation. The `SuppressBeginTokens` logit227 processor will set their log probs to `-inf` so that they are not sampled.228 sequence_bias (`dict[tuple[int], float]`, *optional*)):229 Dictionary that maps a sequence of tokens to its bias term. Positive biases increase the odds of the230 sequence being selected, while negative biases do the opposite. Check231 [`~generation.SequenceBiasLogitsProcessor`] for further documentation and examples.232 token_healing (`bool`, *optional*, defaults to `False`):233 Heal tail tokens of prompts by replacing them with their appropriate extensions.234 This enhances the quality of completions for prompts affected by greedy tokenization bias.235 guidance_scale (`float`, *optional*):236 The guidance scale for classifier free guidance (CFG). CFG is enabled by setting `guidance_scale > 1`.237 Higher guidance scale encourages the model to generate samples that are more closely linked to the input238 prompt, usually at the expense of poorer quality.239 watermarking_config (`BaseWatermarkingConfig` or `dict`, *optional*):240 Arguments used to watermark the model outputs by adding a small bias to randomly selected set of "green"241 tokens. See the docs of [`SynthIDTextWatermarkingConfig`] and [`WatermarkingConfig`] for more242 details. If passed as `Dict`, it will be converted to a `WatermarkingConfig` internally.243 244 > Parameters that define the output variables of generate245 246 num_return_sequences (`int`, *optional*, defaults to 1):247 The number of independently computed returned sequences for each element in the batch.248 output_attentions (`bool`, *optional*, defaults to `False`):249 Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned250 tensors for more details.251 output_hidden_states (`bool`, *optional*, defaults to `False`):252 Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for253 more details.254 output_scores (`bool`, *optional*, defaults to `False`):255 Whether or not to return the prediction scores. See `scores` under returned tensors for more details.256 output_logits (`bool`, *optional*):257 Whether or not to return the unprocessed prediction logit scores. See `logits` under returned tensors for258 more details.259 return_dict_in_generate (`bool`, *optional*, defaults to `False`):260 Whether or not to return a [`~utils.ModelOutput`], as opposed to returning exclusively the generated261 sequence. This flag must be set to `True` to return the generation cache (when `use_cache` is `True`)262 or optional outputs (see flags starting with `output_`)263 264 > Special tokens that can be used at generation time265 266 pad_token_id (`int`, *optional*):267 The id of the *padding* token.268 bos_token_id (`int`, *optional*):269 The id of the *beginning-of-sequence* token.270 eos_token_id (`Union[int, list[int]]`, *optional*):271 The id of the *end-of-sequence* token. Optionally, use a list to set multiple *end-of-sequence* tokens.272 273 > Generation parameters exclusive to encoder-decoder models274 275 encoder_no_repeat_ngram_size (`int`, *optional*, defaults to 0):276 If set to int > 0, all ngrams of that size that occur in the `encoder_input_ids` cannot occur in the277 `decoder_input_ids`.278 decoder_start_token_id (`int` or `list[int]`, *optional*):279 If an encoder-decoder model starts decoding with a different token than *bos*, the id of that token or a list of length280 `batch_size`. Indicating a list enables different start ids for each element in the batch281 (e.g. multilingual models with different target languages in one batch)282 283 > Generation parameters exclusive to assistant generation284 is_assistant (`bool`, *optional*, defaults to `False`):285 Whether the model is an assistant (draft) model.286 num_assistant_tokens (`int`, *optional*, defaults to 20):287 Defines the number of _speculative tokens_ that shall be generated by the assistant model before being288 checked by the target model at each iteration. Higher values for `num_assistant_tokens` make the generation289 more _speculative_ : If the assistant model is performant larger speed-ups can be reached, if the assistant290 model requires lots of corrections, lower speed-ups are reached.291 num_assistant_tokens_schedule (`str`, *optional*, defaults to `"constant"`):292 Defines the schedule at which max assistant tokens shall be changed during inference.293 - `"heuristic"`: When all speculative tokens are correct, increase `num_assistant_tokens` by 2 else294 reduce by 1. `num_assistant_tokens` value is persistent over multiple generation calls with the same assistant model.295 - `"heuristic_transient"`: Same as `"heuristic"` but `num_assistant_tokens` is reset to its initial value after each generation call.296 - `"constant"`: `num_assistant_tokens` stays unchanged during generation297 assistant_confidence_threshold (`float`, *optional*, defaults to 0.4):298 The confidence threshold for the assistant model. If the assistant model's confidence in its prediction for the current token is lower299 than this threshold, the assistant model stops the current token generation iteration, even if the number of _speculative tokens_300 (defined by `num_assistant_tokens`) is not yet reached. The assistant's confidence threshold is adjusted throughout the speculative iterations to reduce the number of unnecessary draft and target forward passes, biased towards avoiding false negatives.301 `assistant_confidence_threshold` value is persistent over multiple generation calls with the same assistant model.302 It is an unsupervised version of the dynamic speculation lookahead303 from Dynamic Speculation Lookahead Accelerates Speculative Decoding of Large Language Models <https://huggingface.co/papers/2405.04304>.304 prompt_lookup_num_tokens (`int`, *optional*):305 The number of tokens to be output as candidate tokens.306 max_matching_ngram_size (`int`, *optional*):307 The maximum ngram size to be considered for matching in the prompt. Default to 2 if not provided.308 assistant_early_exit(`int`, *optional*):309 If set to a positive integer, early exit of the model will be used as an assistant. Can only be used with310 models that support early exit (i.e. models where logits from intermediate layers can be interpreted by the LM head).311 assistant_lookbehind(`int`, *optional*, defaults to 10):312 If set to a positive integer, the re-encodeing process will additionally consider the last `assistant_lookbehind` assistant tokens313 to correctly align tokens. Can only be used with different tokenizers in speculative decoding.314 See this [blog](https://huggingface.co/blog/universal_assisted_generation) for more details.315 target_lookbehind(`int`, *optional*, defaults to 10):316 If set to a positive integer, the re-encodeing process will additionally consider the last `target_lookbehind` target tokens317 to correctly align tokens. Can only be used with different tokenizers in speculative decoding.318 See this [blog](https://huggingface.co/blog/universal_assisted_generation) for more details.319 320 > Parameters related to performances and compilation321 322 compile_config (CompileConfig, *optional*):323 If using a compilable cache, this controls how `generate` will `compile` the forward pass for faster324 inference.325 disable_compile (`bool`, *optional*):326 Whether to disable the automatic compilation of the forward pass. Automatic compilation happens when327 specific criteria are met, including using a compilable cache. Please open an issue if you find the328 need to use this flag.329 """330 331 extra_output_flags = ("output_attentions", "output_hidden_states", "output_scores", "output_logits")332 333 def __init__(self, **kwargs):334 # Parameters that control the length of the output335 self.max_length = kwargs.pop("max_length", 20)336 self.max_new_tokens = kwargs.pop("max_new_tokens", None)337 self.min_length = kwargs.pop("min_length", 0)338 self.min_new_tokens = kwargs.pop("min_new_tokens", None)339 self.early_stopping = kwargs.pop("early_stopping", False)340 self.max_time = kwargs.pop("max_time", None)341 self.stop_strings = kwargs.pop("stop_strings", None)342 343 # Parameters that control the generation strategy used344 self.do_sample = kwargs.pop("do_sample", False)345 self.num_beams = kwargs.pop("num_beams", 1)346 347 # Parameters that control the cache348 self.use_cache = kwargs.pop("use_cache", True)349 self.cache_implementation = kwargs.pop("cache_implementation", None)350 self.cache_config = kwargs.pop("cache_config", None)351 352 self.return_legacy_cache = kwargs.pop("return_legacy_cache", None)353 self.prefill_chunk_size = kwargs.pop("prefill_chunk_size", None)354 355 # Parameters for manipulation of the model output logits356 self.temperature = kwargs.pop("temperature", 1.0)357 self.top_k = kwargs.pop("top_k", 50)358 self.top_p = kwargs.pop("top_p", 1.0)359 self.min_p = kwargs.pop("min_p", None)360 self.typical_p = kwargs.pop("typical_p", 1.0)361 self.epsilon_cutoff = kwargs.pop("epsilon_cutoff", 0.0)362 self.eta_cutoff = kwargs.pop("eta_cutoff", 0.0)363 self.repetition_penalty = kwargs.pop("repetition_penalty", 1.0)364 self.encoder_repetition_penalty = kwargs.pop("encoder_repetition_penalty", 1.0)365 self.length_penalty = kwargs.pop("length_penalty", 1.0)366 self.no_repeat_ngram_size = kwargs.pop("no_repeat_ngram_size", 0)367 self.bad_words_ids = kwargs.pop("bad_words_ids", None)368 self.renormalize_logits = kwargs.pop("renormalize_logits", False)369 self.forced_bos_token_id = kwargs.pop("forced_bos_token_id", None)370 self.forced_eos_token_id = kwargs.pop("forced_eos_token_id", None)371 self.remove_invalid_values = kwargs.pop("remove_invalid_values", False)372 self.exponential_decay_length_penalty = kwargs.pop("exponential_decay_length_penalty", None)373 self.suppress_tokens = kwargs.pop("suppress_tokens", None)374 self.begin_suppress_tokens = kwargs.pop("begin_suppress_tokens", None)375 self.sequence_bias = kwargs.pop("sequence_bias", None)376 self.token_healing = kwargs.pop("token_healing", False)377 self.guidance_scale = kwargs.pop("guidance_scale", None)378 379 watermarking_config = kwargs.pop("watermarking_config", None)380 if watermarking_config is None:381 self.watermarking_config = None382 elif isinstance(watermarking_config, BaseWatermarkingConfig):383 self.watermarking_config = watermarking_config384 else:385 self.watermarking_config = WatermarkingConfig.from_dict(watermarking_config)386 387 # Parameters that define the output variables of `generate`388 self.num_return_sequences = kwargs.pop("num_return_sequences", 1)389 self.output_attentions = kwargs.pop("output_attentions", False)390 self.output_hidden_states = kwargs.pop("output_hidden_states", False)391 self.output_scores = kwargs.pop("output_scores", False)392 self.output_logits = kwargs.pop("output_logits", None)393 self.return_dict_in_generate = kwargs.pop("return_dict_in_generate", False)394 395 # Special tokens that can be used at generation time396 self.pad_token_id = kwargs.pop("pad_token_id", None)397 self.bos_token_id = kwargs.pop("bos_token_id", None)398 self.eos_token_id = kwargs.pop("eos_token_id", None)399 400 # Generation parameters exclusive to encoder-decoder models401 self.encoder_no_repeat_ngram_size = kwargs.pop("encoder_no_repeat_ngram_size", 0)402 self.decoder_start_token_id = kwargs.pop("decoder_start_token_id", None)403 404 # Assistant generation405 self.is_assistant = False406 self.num_assistant_tokens = kwargs.pop("num_assistant_tokens", 20)407 self.num_assistant_tokens_schedule = kwargs.pop("num_assistant_tokens_schedule", "constant")408 self.assistant_confidence_threshold = kwargs.pop("assistant_confidence_threshold", 0.4)409 self.prompt_lookup_num_tokens = kwargs.pop("prompt_lookup_num_tokens", None)410 self.max_matching_ngram_size = kwargs.pop("max_matching_ngram_size", None)411 self.assistant_early_exit = kwargs.pop("assistant_early_exit", None)412 ## assistant generation for different tokenizers, the windows size for assistant/target model413 self.assistant_lookbehind = kwargs.pop("assistant_lookbehind", 10)414 self.target_lookbehind = kwargs.pop("target_lookbehind", 10)415 416 # Performance417 self.compile_config = kwargs.pop("compile_config", None)418 self.disable_compile = kwargs.pop("disable_compile", False)419 420 # Deprecated (moved to the Hub). TODO joao, manuel: remove in v4.62.0421 self.low_memory = kwargs.pop("low_memory", None)422 self.penalty_alpha = kwargs.pop("penalty_alpha", None)423 self.dola_layers = kwargs.pop("dola_layers", None)424 self.diversity_penalty = kwargs.pop("diversity_penalty", 0.0)425 self.num_beam_groups = kwargs.pop("num_beam_groups", 1)426 self.constraints = kwargs.pop("constraints", None)427 self.force_words_ids = kwargs.pop("force_words_ids", None)428 429 # The remaining attributes do not parametrize `.generate()`, but are informative and/or used by the hub430 # interface.431 self._from_model_config = kwargs.pop("_from_model_config", False)432 self._commit_hash = kwargs.pop("_commit_hash", None)433 self.transformers_version = kwargs.pop("transformers_version", __version__)434 435 # Additional attributes without default values436 if not self._from_model_config:437 # we don't want to copy values from the model config if we're initializing a `GenerationConfig` from a438 # model's default configuration file439 for key, value in kwargs.items():440 try:441 setattr(self, key, value)442 except AttributeError as err:443 logger.error(f"Can't set {key} with value {value} for {self}")444 raise err445 446 # Validate the values of the attributes447 self.validate()448 449 def __hash__(self):450 return hash(self.to_json_string(ignore_metadata=True))451 452 def __eq__(self, other):453 if not isinstance(other, GenerationConfig):454 return False455 456 self_without_metadata = self.to_json_string(use_diff=False, ignore_metadata=True)457 other_without_metadata = other.to_json_string(use_diff=False, ignore_metadata=True)458 return self_without_metadata == other_without_metadata459 460 def __repr__(self):461 return f"{self.__class__.__name__} {self.to_json_string(ignore_metadata=True)}"462 463 def get_generation_mode(self, assistant_model: Optional["PreTrainedModel"] = None) -> GenerationMode:464 """465 Returns the generation mode triggered by the [`GenerationConfig`] instance.466 467 Arg:468 assistant_model (`PreTrainedModel`, *optional*):469 The assistant model to be used for assisted generation. If set, the generation mode will be470 assisted generation.471 472 Returns:473 `GenerationMode`: The generation mode triggered by the instance.474 """475 # TODO joao: find out a way of not depending on external fields (e.g. `assistant_model`), then make this a476 # property and part of the `__repr__`477 if self.constraints is not None or self.force_words_ids is not None:478 generation_mode = GenerationMode.CONSTRAINED_BEAM_SEARCH479 elif self.num_beams == 1:480 if self.do_sample is False:481 if (482 self.top_k is not None483 and self.top_k > 1484 and self.penalty_alpha is not None485 and self.penalty_alpha > 0486 ):487 generation_mode = GenerationMode.CONTRASTIVE_SEARCH488 else:489 generation_mode = GenerationMode.GREEDY_SEARCH490 else:491 generation_mode = GenerationMode.SAMPLE492 else:493 if self.num_beam_groups > 1:494 generation_mode = GenerationMode.GROUP_BEAM_SEARCH495 elif self.do_sample is True:496 generation_mode = GenerationMode.BEAM_SAMPLE497 else:498 generation_mode = GenerationMode.BEAM_SEARCH499 500 # Assisted generation may extend some generation modes501 if (502 assistant_model is not None503 or self.prompt_lookup_num_tokens is not None504 or self.assistant_early_exit is not None505 ):506 if generation_mode in ("greedy_search", "sample"):507 generation_mode = GenerationMode.ASSISTED_GENERATION508 else:509 logger.warning(510 "You've set `assistant_model`, which triggers assisted generate. Currently, assisted generate "511 "is only supported with Greedy Search and Sample. However, the base decoding mode (based on "512 f"current flags) is {generation_mode} -- some of the set flags will be ignored."513 )514 515 # DoLa generation may extend some generation modes516 # TODO joao, manuel: remove this in v4.62.0517 if self.dola_layers is not None:518 if generation_mode in ("greedy_search", "sample"):519 generation_mode = GenerationMode.DOLA_GENERATION520 else:521 logger.warning(522 "You've set `dola_layers`, which triggers DoLa generate. Currently, DoLa generate "523 "is only supported with Greedy Search and Sample. However, the base decoding mode (based on "524 f"current flags) is {generation_mode} -- some of the set flags will be ignored."525 )526 return generation_mode527 528 def validate(self, strict=False):529 """530 Validates the values of the attributes of the [`GenerationConfig`] instance. Raises exceptions in the presence531 of parameterization that can be detected as incorrect from the configuration instance alone.532 533 Note that some parameters not validated here are best validated at generate runtime, as they may depend on534 other inputs and/or the model, such as parameters related to the generation length.535 536 Args:537 strict (bool): If True, raise an exception for any issues found. If False, only log issues.538 """539 minor_issues = {} # format: {attribute_name: issue_description}540 541 # 1. Validation of individual attributes542 # 1.1. Decoding attributes543 if self.early_stopping not in {True, False, "never"}:544 raise ValueError(f"`early_stopping` must be a boolean or 'never', but is {self.early_stopping}.")545 if self.max_new_tokens is not None and self.max_new_tokens <= 0:546 raise ValueError(f"`max_new_tokens` must be greater than 0, but is {self.max_new_tokens}.")547 if self.pad_token_id is not None and self.pad_token_id < 0:548 minor_issues["pad_token_id"] = (549 f"`pad_token_id` should be positive but got {self.pad_token_id}. This will cause errors when batch "550 "generating, if there is padding. Please set `pad_token_id` explicitly as "551 "`model.generation_config.pad_token_id=PAD_TOKEN_ID` to avoid errors in generation"552 )553 # 1.2. Cache attributes554 if self.cache_implementation is not None and self.cache_implementation not in ALL_CACHE_IMPLEMENTATIONS:555 raise ValueError(556 f"Invalid `cache_implementation` ({self.cache_implementation}). Choose one of: "557 f"{ALL_CACHE_IMPLEMENTATIONS}"558 )559 # 1.3. Performance attributes560 if self.compile_config is not None and not isinstance(self.compile_config, CompileConfig):561 raise ValueError(562 f"You provided `compile_config` as an instance of {type(self.compile_config)}, but it must be an "563 "instance of `CompileConfig`."564 )565 # 1.4. Watermarking attributes566 if self.watermarking_config is not None:567 self.watermarking_config.validate()568 569 # 2. Validation of attribute combinations570 # 2.1. detect sampling-only parameterization when not in sampling mode571 if self.do_sample is False:572 greedy_wrong_parameter_msg = (573 "`do_sample` is set to `False`. However, `{flag_name}` is set to `{flag_value}` -- this flag is only "574 "used in sample-based generation modes. You should set `do_sample=True` or unset `{flag_name}`."575 )576 if self.temperature is not None and self.temperature != 1.0:577 minor_issues["temperature"] = greedy_wrong_parameter_msg.format(578 flag_name="temperature", flag_value=self.temperature579 )580 if self.top_p is not None and self.top_p != 1.0:581 minor_issues["top_p"] = greedy_wrong_parameter_msg.format(flag_name="top_p", flag_value=self.top_p)582 if self.min_p is not None:583 minor_issues["min_p"] = greedy_wrong_parameter_msg.format(flag_name="min_p", flag_value=self.min_p)584 if self.typical_p is not None and self.typical_p != 1.0:585 minor_issues["typical_p"] = greedy_wrong_parameter_msg.format(586 flag_name="typical_p", flag_value=self.typical_p587 )588 if self.top_k is not None and self.top_k != 50:589 minor_issues["top_k"] = greedy_wrong_parameter_msg.format(flag_name="top_k", flag_value=self.top_k)590 if self.epsilon_cutoff is not None and self.epsilon_cutoff != 0.0:591 minor_issues["epsilon_cutoff"] = greedy_wrong_parameter_msg.format(592 flag_name="epsilon_cutoff", flag_value=self.epsilon_cutoff593 )594 if self.eta_cutoff is not None and self.eta_cutoff != 0.0:595 minor_issues["eta_cutoff"] = greedy_wrong_parameter_msg.format(596 flag_name="eta_cutoff", flag_value=self.eta_cutoff597 )598 599 # 2.2. detect beam-only parameterization when not in beam mode600 if self.num_beams == 1:601 single_beam_wrong_parameter_msg = (602 "`num_beams` is set to 1. However, `{flag_name}` is set to `{flag_value}` -- this flag is only used "603 "in beam-based generation modes. You should set `num_beams>1` or unset `{flag_name}`."604 )605 if self.early_stopping is not False:606 minor_issues["early_stopping"] = single_beam_wrong_parameter_msg.format(607 flag_name="early_stopping", flag_value=self.early_stopping608 )609 if self.length_penalty is not None and self.length_penalty != 1.0:610 minor_issues["length_penalty"] = single_beam_wrong_parameter_msg.format(611 flag_name="length_penalty", flag_value=self.length_penalty612 )613 614 # 2.4. check `num_return_sequences`615 if self.num_return_sequences != 1:616 if self.num_beams == 1:617 if self.do_sample is False:618 raise ValueError(619 "Greedy methods without beam search do not support `num_return_sequences` different than 1 "620 f"(got {self.num_return_sequences})."621 )622 elif self.num_return_sequences > self.num_beams:623 raise ValueError(624 f"`num_return_sequences` ({self.num_return_sequences}) has to be smaller or equal to `num_beams` "625 f"({self.num_beams})."626 )627 628 # 2.5. check cache-related arguments629 if self.use_cache is False:630 # In this case, all cache-related arguments should be unset. However, since `use_cache=False` is often used631 # passed to `generate` directly to hot-fix cache issues, let's raise a warning instead of an error632 # (otherwise a user might need to overwrite several parameters).633 no_cache_warning = (634 "You have set `use_cache` to `False`, but {cache_arg} is set to {cache_arg_value}. {cache_arg} will "635 "have no effect."636 )637 for arg_name in ("cache_implementation", "cache_config", "return_legacy_cache"):638 if getattr(self, arg_name) is not None:639 minor_issues[arg_name] = no_cache_warning.format(640 cache_arg=arg_name, cache_arg_value=getattr(self, arg_name)641 )642 643 # 2.6. other incorrect combinations644 if self.return_dict_in_generate is not True:645 for extra_output_flag in self.extra_output_flags:646 if getattr(self, extra_output_flag) is True:647 minor_issues[extra_output_flag] = (648 f"`return_dict_in_generate` is NOT set to `True`, but `{extra_output_flag}` is. When "649 f"`return_dict_in_generate` is not `True`, `{extra_output_flag}` is ignored."650 )651 652 # 3. Check common issue: passing `generate` arguments inside the generation config653 generate_arguments = (654 "logits_processor",655 "stopping_criteria",656 "prefix_allowed_tokens_fn",657 "synced_gpus",658 "assistant_model",659 "streamer",660 "negative_prompt_ids",661 "negative_prompt_attention_mask",662 "use_model_defaults",663 )664 for arg in generate_arguments:665 if hasattr(self, arg):666 raise ValueError(667 f"Argument `{arg}` is not a valid argument of `GenerationConfig`. It should be passed to "668 "`generate()` (or a pipeline) directly."669 )670 671 # Finally, handle caught minor issues. With default parameterization, we will throw a minimal warning.672 if len(minor_issues) > 0:673 # Full list of issues with potential fixes674 info_message = []675 for attribute_name, issue_description in minor_issues.items():676 info_message.append(f"- `{attribute_name}`: {issue_description}")677 info_message = "\n".join(info_message)678 info_message += (679 "\nIf you're using a pretrained model, note that some of these attributes may be set through the "680 "model's `generation_config.json` file."681 )682 683 if strict:684 raise ValueError("GenerationConfig is invalid: \n" + info_message)685 else:686 attributes_with_issues = list(minor_issues.keys())687 warning_message = (688 f"The following generation flags are not valid and may be ignored: {attributes_with_issues}."689 )690 if logging.get_verbosity() >= logging.WARNING:691 warning_message += " Set `TRANSFORMERS_VERBOSITY=info` for more details."692 logger.warning_once(warning_message)693 logger.info_once(info_message)694 695 def save_pretrained(696 self,697 save_directory: Union[str, os.PathLike],698 config_file_name: Optional[Union[str, os.PathLike]] = None,699 push_to_hub: bool = False,700 **kwargs,701 ):702 r"""703 Save a generation configuration object to the directory `save_directory`, so that it can be re-loaded using the704 [`~GenerationConfig.from_pretrained`] class method.705 706 Args:707 save_directory (`str` or `os.PathLike`):708 Directory where the configuration JSON file will be saved (will be created if it does not exist).709 config_file_name (`str` or `os.PathLike`, *optional*, defaults to `"generation_config.json"`):710 Name of the generation configuration JSON file to be saved in `save_directory`.711 push_to_hub (`bool`, *optional*, defaults to `False`):712 Whether or not to push your model to the Hugging Face model hub after saving it. You can specify the713 repository you want to push to with `repo_id` (will default to the name of `save_directory` in your714 namespace).715 kwargs (`dict[str, Any]`, *optional*):716 Additional key word arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method.717 """718 719 # At save time, validate the instance enforcing strictness -- if any warning/exception would be thrown, we720 # refuse to save the instance.721 # This strictness is enforced to prevent bad configurations from being saved and re-used.722 try:723 self.validate(strict=True)724 except ValueError as exc:725 raise ValueError(str(exc) + "\n\nFix these issues to save the configuration.")726 727 use_auth_token = kwargs.pop("use_auth_token", None)728 729 if use_auth_token is not None:730 warnings.warn(731 "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. "732 "Please use `token` instead.",733 FutureWarning,734 )735 if kwargs.get("token") is not None:736 raise ValueError(737 "`token` and `use_auth_token` are both specified. Please set only the argument `token`."738 )739 kwargs["token"] = use_auth_token740 741 config_file_name = config_file_name if config_file_name is not None else GENERATION_CONFIG_NAME742 743 if os.path.isfile(save_directory):744 raise AssertionError(f"Provided path ({save_directory}) should be a directory, not a file")745 746 os.makedirs(save_directory, exist_ok=True)747 748 if push_to_hub:749 commit_message = kwargs.pop("commit_message", None)750 repo_id = kwargs.pop("repo_id", save_directory.split(os.path.sep)[-1])751 repo_id = self._create_repo(repo_id, **kwargs)752 files_timestamps = self._get_files_timestamps(save_directory)753 754 output_config_file = os.path.join(save_directory, config_file_name)755 756 self.to_json_file(output_config_file, use_diff=True)757 logger.info(f"Configuration saved in {output_config_file}")758 759 if push_to_hub:760 self._upload_modified_files(761 save_directory,762 repo_id,763 files_timestamps,764 commit_message=commit_message,765 token=kwargs.get("token"),766 )767 768 @classmethod769 def from_pretrained(770 cls,771 pretrained_model_name: Union[str, os.PathLike],772 config_file_name: Optional[Union[str, os.PathLike]] = None,773 cache_dir: Optional[Union[str, os.PathLike]] = None,774 force_download: bool = False,775 local_files_only: bool = False,776 token: Optional[Union[str, bool]] = None,777 revision: str = "main",778 **kwargs,779 ) -> "GenerationConfig":780 r"""781 Instantiate a [`GenerationConfig`] from a generation configuration file.782 783 Args:784 pretrained_model_name (`str` or `os.PathLike`):785 This can be either:786 787 - a string, the *model id* of a pretrained model configuration hosted inside a model repo on788 huggingface.co.789 - a path to a *directory* containing a configuration file saved using the790 [`~GenerationConfig.save_pretrained`] method, e.g., `./my_model_directory/`.791 config_file_name (`str` or `os.PathLike`, *optional*, defaults to `"generation_config.json"`):792 Name of the generation configuration JSON file to be loaded from `pretrained_model_name`.793 cache_dir (`str` or `os.PathLike`, *optional*):794 Path to a directory in which a downloaded pretrained model configuration should be cached if the795 standard cache should not be used.796 force_download (`bool`, *optional*, defaults to `False`):797 Whether or not to force to (re-)download the configuration files and override the cached versions if798 they exist.799 resume_download:800 Deprecated and ignored. All downloads are now resumed by default when possible.801 Will be removed in v5 of Transformers.802 proxies (`dict[str, str]`, *optional*):803 A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',804 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.805 token (`str` or `bool`, *optional*):806 The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use807 the token generated when running `hf auth login` (stored in `~/.huggingface`).808 revision (`str`, *optional*, defaults to `"main"`):809 The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a810 git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any811 identifier allowed by git.812 813 <Tip>814 815 To test a pull request you made on the Hub, you can pass `revision="refs/pr/<pr_number>"`.816 817 </Tip>818 819 return_unused_kwargs (`bool`, *optional*, defaults to `False`):820 If `False`, then this function returns just the final configuration object.821 822 If `True`, then this functions returns a `Tuple(config, unused_kwargs)` where *unused_kwargs* is a823 dictionary consisting of the key/value pairs whose keys are not configuration attributes: i.e., the824 part of `kwargs` which has not been used to update `config` and is otherwise ignored.825 subfolder (`str`, *optional*, defaults to `""`):826 In case the relevant files are located inside a subfolder of the model repo on huggingface.co, you can827 specify the folder name here.828 kwargs (`dict[str, Any]`, *optional*):829 The values in kwargs of any keys which are configuration attributes will be used to override the loaded830 values. Behavior concerning key/value pairs whose keys are *not* configuration attributes is controlled831 by the `return_unused_kwargs` keyword parameter.832 833 Returns:834 [`GenerationConfig`]: The configuration object instantiated from this pretrained model.835 836 Examples:837 838 ```python839 >>> from transformers import GenerationConfig840 841 >>> # Download configuration from huggingface.co and cache.842 >>> generation_config = GenerationConfig.from_pretrained("openai-community/gpt2")843 844 >>> # E.g. config was saved using *save_pretrained('./test/saved_model/')*845 >>> generation_config.save_pretrained("./test/saved_model/")846 >>> generation_config = GenerationConfig.from_pretrained("./test/saved_model/")847 848 >>> # You can also specify configuration names to your generation configuration file849 >>> generation_config.save_pretrained("./test/saved_model/", config_file_name="my_configuration.json")850 >>> generation_config = GenerationConfig.from_pretrained("./test/saved_model/", "my_configuration.json")851 852 >>> # If you'd like to try a minor variation to an existing configuration, you can also pass generation853 >>> # arguments to `.from_pretrained()`. Be mindful that typos and unused arguments will be ignored854 >>> generation_config, unused_kwargs = GenerationConfig.from_pretrained(855 ... "openai-community/gpt2", top_k=1, foo=False, do_sample=True, return_unused_kwargs=True856 ... )857 >>> generation_config.top_k858 1859 860 >>> unused_kwargs861 {'foo': False}862 ```"""863 config_file_name = config_file_name if config_file_name is not None else GENERATION_CONFIG_NAME864 865 resume_download = kwargs.pop("resume_download", None)866 proxies = kwargs.pop("proxies", None)867 use_auth_token = kwargs.pop("use_auth_token", None)868 subfolder = kwargs.pop("subfolder", "")869 from_pipeline = kwargs.pop("_from_pipeline", None)870 from_auto_class = kwargs.pop("_from_auto", False)871 commit_hash = kwargs.pop("_commit_hash", None)872 873 if use_auth_token is not None:874 warnings.warn(875 "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.",876 FutureWarning,877 )878 if token is not None:879 raise ValueError(880 "`token` and `use_auth_token` are both specified. Please set only the argument `token`."881 )882 token = use_auth_token883 884 user_agent = {"file_type": "config", "from_auto_class": from_auto_class}885 if from_pipeline is not None:886 user_agent["using_pipeline"] = from_pipeline887 888 config_path = os.path.join(pretrained_model_name, config_file_name)889 config_path = str(config_path)890 891 is_local = os.path.exists(config_path)892 if os.path.isfile(os.path.join(subfolder, config_path)):893 # Special case when config_path is a local file894 resolved_config_file = config_path895 is_local = True896 elif is_remote_url(config_path):897 configuration_file = config_path898 resolved_config_file = download_url(config_path)899 else:900 configuration_file = config_file_name901 try:902 # Load from local folder or from cache or download from model Hub and cache903 resolved_config_file = cached_file(904 pretrained_model_name,905 configuration_file,906 cache_dir=cache_dir,907 force_download=force_download,908 proxies=proxies,909 resume_download=resume_download,910 local_files_only=local_files_only,911 token=token,912 user_agent=user_agent,913 revision=revision,914 subfolder=subfolder,915 _commit_hash=commit_hash,916 )917 commit_hash = extract_commit_hash(resolved_config_file, commit_hash)918 except OSError:919 # Raise any environment error raise by `cached_file`. It will have a helpful error message adapted to920 # the original exception.921 raise922 except Exception:923 # For any other exception, we throw a generic error.924 raise OSError(925 f"Can't load the configuration of '{pretrained_model_name}'. If you were trying to load it"926 " from 'https://huggingface.co/models', make sure you don't have a local directory with the same"927 f" name. Otherwise, make sure '{pretrained_model_name}' is the correct path to a directory"928 f" containing a {configuration_file} file"929 )930 931 try:932 # Load config dict933 config_dict = cls._dict_from_json_file(resolved_config_file)934 config_dict["_commit_hash"] = commit_hash935 except (json.JSONDecodeError, UnicodeDecodeError):936 raise OSError(f"It looks like the config file at '{resolved_config_file}' is not a valid JSON file.")937 938 if is_local:939 logger.info(f"loading configuration file {resolved_config_file}")940 else:941 logger.info(f"loading configuration file {configuration_file} from cache at {resolved_config_file}")942 943 if kwargs.get("return_unused_kwargs") is True:944 config, unused_kwargs = cls.from_dict(config_dict, **kwargs)945 config._original_object_hash = hash(config) # Hash to detect whether the instance was modified946 return config, unused_kwargs947 else:948 config = cls.from_dict(config_dict, **kwargs)949 config._original_object_hash = hash(config) # Hash to detect whether the instance was modified950 return config951 952 @classmethod953 def _dict_from_json_file(cls, json_file: Union[str, os.PathLike]):954 with open(json_file, "r", encoding="utf-8") as reader:955 text = reader.read()956 return json.loads(text)957 958 @classmethod959 def from_dict(cls, config_dict: dict[str, Any], **kwargs) -> "GenerationConfig":960 """961 Instantiates a [`GenerationConfig`] from a Python dictionary of parameters.962 963 Args:964 config_dict (`dict[str, Any]`):965 Dictionary that will be used to instantiate the configuration object.966 kwargs (`dict[str, Any]`):967 Additional parameters from which to initialize the configuration object.968 969 Returns:970 [`GenerationConfig`]: The configuration object instantiated from those parameters.971 """972 return_unused_kwargs = kwargs.pop("return_unused_kwargs", False)973 # Those arguments may be passed along for our internal telemetry.974 # We remove them so they don't appear in `return_unused_kwargs`.975 kwargs.pop("_from_auto", None)976 kwargs.pop("_from_pipeline", None)977 # The commit hash might have been updated in the `config_dict`, we don't want the kwargs to erase that update.978 if "_commit_hash" in kwargs and "_commit_hash" in config_dict:979 kwargs["_commit_hash"] = config_dict["_commit_hash"]980 981 # The line below allows model-specific config to be loaded as well through kwargs, with safety checks.982 # See https://github.com/huggingface/transformers/pull/21269983 config = cls(**{**config_dict, **kwargs})984 unused_kwargs = config.update(**kwargs)985 986 logger.info(f"Generate config {config}")987 if return_unused_kwargs:988 return config, unused_kwargs989 else:990 return config991 992 def dict_dtype_to_str(self, d: dict[str, Any]) -> None:993 """994 Checks whether the passed dictionary and its nested dicts have a *dtype* key and if it's not None,995 converts torch.dtype to a string of just the type. For example, `torch.float32` get converted into *"float32"*996 string, which can then be stored in the json format.997 """998 if d.get("dtype") is not None and not isinstance(d["dtype"], str):999 d["dtype"] = str(d["dtype"]).split(".")[1]1000 for value in d.values():1001 if isinstance(value, dict):1002 self.dict_dtype_to_str(value)1003 1004 def to_diff_dict(self) -> dict[str, Any]:1005 """1006 Removes all attributes from config which correspond to the default config attributes for better readability and1007 serializes to a Python dictionary.1008 1009 Returns:1010 `dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance,1011 """1012 config_dict = self.to_dict()1013 1014 # get the default config dict1015 default_config_dict = GenerationConfig().to_dict()1016 1017 serializable_config_dict = {}1018 1019 # only serialize values that differ from the default config1020 for key, value in config_dict.items():1021 if key not in default_config_dict or key == "transformers_version" or value != default_config_dict[key]:1022 serializable_config_dict[key] = value1023 1024 self.dict_dtype_to_str(serializable_config_dict)1025 return serializable_config_dict1026 1027 def to_dict(self) -> dict[str, Any]:1028 """1029 Serializes this instance to a Python dictionary.1030 1031 Returns:1032 `dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance.1033 """1034 output = copy.deepcopy(self.__dict__)1035 1036 # Fields to ignore at serialization time1037 if "_commit_hash" in output:1038 del output["_commit_hash"]1039 if "_original_object_hash" in output:1040 del output["_original_object_hash"]1041 if "compile_config" in output:1042 del output["compile_config"]1043 1044 # Transformers version when serializing this file1045 output["transformers_version"] = __version__1046 1047 self.dict_dtype_to_str(output)1048 return output1049 1050 def to_json_string(self, use_diff: bool = True, ignore_metadata: bool = False) -> str:1051 """1052 Serializes this instance to a JSON string.1053 1054 Args:1055 use_diff (`bool`, *optional*, defaults to `True`):1056 If set to `True`, only the difference between the config instance and the default `GenerationConfig()`1057 is serialized to JSON string.1058 ignore_metadata (`bool`, *optional*, defaults to `False`):1059 Whether to ignore the metadata fields present in the instance1060 1061 Returns:1062 `str`: String containing all the attributes that make up this configuration instance in JSON format.1063 """1064 if use_diff is True:1065 config_dict = self.to_diff_dict()1066 else:1067 config_dict = self.to_dict()1068 1069 if ignore_metadata:1070 for metadata_field in METADATA_FIELDS:1071 config_dict.pop(metadata_field, None)1072 1073 def convert_keys_to_string(obj):1074 if isinstance(obj, dict):1075 return {str(key): convert_keys_to_string(value) for key, value in obj.items()}1076 elif isinstance(obj, list):1077 return [convert_keys_to_string(item) for item in obj]1078 else:1079 return obj1080 1081 def convert_dataclass_to_dict(obj):1082 if isinstance(obj, dict):1083 return {key: convert_dataclass_to_dict(value) for key, value in obj.items()}1084 elif is_dataclass(obj):1085 return obj.to_dict()1086 else:1087 return obj1088 1089 config_dict = convert_keys_to_string(config_dict)1090 config_dict = convert_dataclass_to_dict(config_dict)1091 1092 return json.dumps(config_dict, indent=2, sort_keys=True) + "\n"1093 1094 def to_json_file(self, json_file_path: Union[str, os.PathLike], use_diff: bool = True):1095 """1096 Save this instance to a JSON file.1097 1098 Args:1099 json_file_path (`str` or `os.PathLike`):1100 Path to the JSON file in which this configuration instance's parameters will be saved.1101 use_diff (`bool`, *optional*, defaults to `True`):1102 If set to `True`, only the difference between the config instance and the default `GenerationConfig()`1103 is serialized to JSON file.1104 """1105 with open(json_file_path, "w", encoding="utf-8") as writer:1106 writer.write(self.to_json_string(use_diff=use_diff))1107 1108 @classmethod1109 def from_model_config(cls, model_config: PretrainedConfig) -> "GenerationConfig":1110 """1111 Instantiates a [`GenerationConfig`] from a [`PretrainedConfig`]. This function is useful to convert legacy1112 [`PretrainedConfig`] objects, which may contain generation parameters, into a stand-alone [`GenerationConfig`].1113 1114 Args:1115 model_config (`PretrainedConfig`):1116 The model config that will be used to instantiate the generation config.1117 1118 Returns:1119 [`GenerationConfig`]: The configuration object instantiated from those parameters.1120 """1121 config_dict = model_config.to_dict()1122 config_dict.pop("_from_model_config", None)1123 1124 # Removes all `None` from the model config dict -- this lets the generation config defaults to take hold1125 config_dict = {key: value for key, value in config_dict.items() if value is not None}1126 1127 generation_config = cls.from_dict(config_dict, return_unused_kwargs=False, _from_model_config=True)1128 1129 # Special case: some models have generation attributes set in the decoder. Use them if still unset in the1130 # generation config (which in turn is defined from the outer attributes of model config).1131 decoder_config = model_config.get_text_config(decoder=True)1132 if decoder_config is not model_config:1133 default_generation_config = GenerationConfig()1134 decoder_config_dict = decoder_config.to_dict()1135 for attr in generation_config.to_dict():1136 is_unset = getattr(generation_config, attr) == getattr(default_generation_config, attr)1137 if attr in decoder_config_dict and is_unset:1138 setattr(generation_config, attr, decoder_config_dict[attr])1139 1140 # If any `output_...` flag is set to `True`, we ensure `return_dict_in_generate` is set to `True`.1141 if generation_config.return_dict_in_generate is False:1142 if any(1143 getattr(generation_config, extra_output_flag, False)1144 for extra_output_flag in generation_config.extra_output_flags1145 ):1146 generation_config.return_dict_in_generate = True1147 1148 # Hash to detect whether the instance was modified1149 generation_config._original_object_hash = hash(generation_config)1150 return generation_config1151 1152 def update(self, **kwargs):1153 """1154 Updates attributes of this class instance with attributes from `kwargs` if they match existing attributes,1155 returning all the unused kwargs.1156 1157 Args:1158 kwargs (`dict[str, Any]`):1159 Dictionary of attributes to tentatively update this class.1160 1161 Returns:1162 `dict[str, Any]`: Dictionary containing all the key-value pairs that were not used to update the instance.1163 """1164 to_remove = []1165 for key, value in kwargs.items():1166 if hasattr(self, key):1167 setattr(self, key, value)1168 to_remove.append(key)1169 1170 # Confirm that the updated instance is still valid1171 self.validate()1172 1173 # Remove all the attributes that were updated, without modifying the input dict1174 unused_kwargs = {key: value for key, value in kwargs.items() if key not in to_remove}1175 return unused_kwargs1176 1177 1178@dataclass1179class BaseWatermarkingConfig(ABC):1180 """Generic watermarking config"""1181 1182 @classmethod1183 def from_dict(cls, config_dict, **kwargs):1184 """1185 Constructs a BaseWatermarkingConfig instance from a dictionary of parameters.1186 1187 Args:1188 config_dict (dict[str, Any]): Dictionary containing configuration parameters.1189 **kwargs: Additional keyword arguments to override dictionary values.1190 1191 Returns:1192 BaseWatermarkingConfig: Instance of BaseWatermarkingConfig constructed from the dictionary.1193 """1194 config = cls(**config_dict)1195 to_remove = []1196 for key, value in kwargs.items():1197 if hasattr(config, key):1198 setattr(config, key, value)1199 to_remove.append(key)1200 for key in to_remove: