Aluode/PerceptionLabPortable
0
1# coding=utf-82# Copyright 2024 The HuggingFace Inc. team and Google DeepMind.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15 16import inspect17import math18from collections.abc import Iterable19from typing import TYPE_CHECKING, Callable, Optional, Union20 21import numpy as np22import torch23 24from ..pytorch_utils import isin_mps_friendly25from ..utils import add_start_docstrings26from ..utils.logging import get_logger27 28 29# TODO (joao): We shouldn't need this, but there would be a circular import30if TYPE_CHECKING:31 from ..generation.configuration_utils import GenerationConfig32 33logger = get_logger(__name__)34 35 36LOGITS_PROCESSOR_INPUTS_DOCSTRING = r"""37 Args:38 input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):39 Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids)40 scores (`torch.FloatTensor` of shape `(batch_size, config.vocab_size)`):41 Prediction scores of a language modeling head. These can be logits for each vocabulary when not using beam42 search or log softmax for each vocabulary token when using beam search43 44 Return:45 `torch.FloatTensor` of shape `(batch_size, config.vocab_size)`: The processed prediction scores.46 47"""48 49 50class LogitsProcessor:51 """Abstract base class for all logit processors that can be applied during generation."""52 53 @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)54 def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:55 raise NotImplementedError(56 f"{self.__class__} is an abstract class. Only classes inheriting this class can be called."57 )58 59 60class LogitsProcessorList(list):61 """62 This class can be used to create a list of [`LogitsProcessor`] to subsequently process a `scores` input tensor.63 This class inherits from list and adds a specific *__call__* method to apply each [`LogitsProcessor`] to the64 inputs.65 """66 67 def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> torch.FloatTensor:68 r"""69 Args:70 input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):71 Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids)72 scores (`torch.FloatTensor` of shape `(batch_size, config.vocab_size)`):73 Prediction scores of a language modeling head. These can be logits for each vocabulary when not using74 beam search or log softmax for each vocabulary token when using beam search75 kwargs (`dict[str, Any]`, *optional*):76 Additional kwargs that are specific to a logits processor.77 78 Return:79 `torch.FloatTensor` of shape `(batch_size, config.vocab_size)`:80 The processed prediction scores.81 82 """83 for processor in self:84 function_args = inspect.signature(processor.__call__).parameters85 if len(function_args) > 2:86 if not all(arg in kwargs for arg in list(function_args.keys())[2:]):87 raise ValueError(88 f"Make sure that all the required parameters: {list(function_args.keys())} for "89 f"{processor.__class__} are passed to the logits processor."90 )91 scores = processor(input_ids, scores, **kwargs)92 else:93 scores = processor(input_ids, scores)94 95 return scores96 97 98class MinLengthLogitsProcessor(LogitsProcessor):99 r"""100 [`LogitsProcessor`] enforcing a min-length by setting EOS probability to 0. Note that, for decoder-only models101 like most LLMs, the length includes the prompt.102 103 Args:104 min_length (`int`):105 The minimum length below which the score of `eos_token_id` is set to `-float("Inf")`.106 eos_token_id (`Union[int, list[int], torch.Tensor]`):107 The id(s) of the *end-of-sequence* token.108 device (`str`, *optional*, defaults to `"cpu"`):109 The device to allocate the tensors.110 111 Examples:112 113 ```python114 >>> from transformers import AutoModelForCausalLM, AutoTokenizer115 116 >>> tokenizer = AutoTokenizer.from_pretrained("bigscience/bloomz-560m")117 >>> model = AutoModelForCausalLM.from_pretrained("bigscience/bloomz-560m")118 119 >>> inputs = tokenizer("A number:", return_tensors="pt")120 >>> gen_out = model.generate(**inputs)121 >>> print(tokenizer.batch_decode(gen_out, skip_special_tokens=True)[0])122 A number: one123 124 >>> # setting `min_length` to a value smaller than the uncontrolled output length has no impact125 >>> gen_out = model.generate(**inputs, min_length=3)126 >>> print(tokenizer.batch_decode(gen_out, skip_special_tokens=True)[0])127 A number: one128 129 >>> # setting a larger `min_length` will force the model to generate beyond its natural ending point, which is not130 >>> # necessarily incorrect131 >>> gen_out = model.generate(**inputs, min_length=10)132 >>> print(tokenizer.batch_decode(gen_out, skip_special_tokens=True)[0])133 A number: one thousand, nine hundred and ninety-four134 ```135 """136 137 def __init__(self, min_length: int, eos_token_id: Union[int, list[int], torch.Tensor], device: str = "cpu"):138 if not isinstance(min_length, int) or min_length < 0:139 raise ValueError(f"`min_length` has to be a non-negative integer, but is {min_length}")140 141 if not isinstance(eos_token_id, torch.Tensor):142 if isinstance(eos_token_id, int):143 eos_token_id = [eos_token_id]144 eos_token_id = torch.tensor(eos_token_id, device=device)145 146 self.min_length = min_length147 self.eos_token_id = eos_token_id148 149 @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)150 def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:151 vocab_tensor = torch.arange(scores.shape[-1], device=scores.device)152 eos_token_mask = isin_mps_friendly(vocab_tensor, self.eos_token_id)153 scores_processed = scores.clone()154 if input_ids.shape[-1] < self.min_length:155 scores_processed = torch.where(eos_token_mask, -math.inf, scores)156 return scores_processed157 158 159class MinNewTokensLengthLogitsProcessor(LogitsProcessor):160 r"""161 [`LogitsProcessor`] enforcing a min-length of new tokens by setting EOS (End-Of-Sequence) token probability to 0.162 Contrarily to [`MinLengthLogitsProcessor`], this processor ignores the prompt.163 164 Args:165 prompt_length_to_skip (`int`):166 The input tokens length. Not a valid argument when used with `generate` as it will automatically assign the167 input length.168 min_new_tokens (`int`):169 The minimum *new* tokens length below which the score of `eos_token_id` is set to `-float("Inf")`.170 eos_token_id (`Union[int, list[int], torch.Tensor]`):171 The id(s) of the *end-of-sequence* token.172 device (`str`, *optional*, defaults to `"cpu"`):173 The device to allocate the tensors.174 175 Examples:176 177 ```python178 >>> from transformers import AutoModelForCausalLM, AutoTokenizer179 180 >>> tokenizer = AutoTokenizer.from_pretrained("bigscience/bloomz-560m")181 >>> model = AutoModelForCausalLM.from_pretrained("bigscience/bloomz-560m")182 183 >>> inputs = tokenizer(["A number:"], return_tensors="pt")184 >>> gen_out = model.generate(**inputs)185 >>> print(tokenizer.batch_decode(gen_out, skip_special_tokens=True)[0])186 A number: one187 188 >>> # setting `min_new_tokens` will force the model to generate beyond its natural ending point, which is not189 >>> # necessarily incorrect190 >>> gen_out = model.generate(**inputs, min_new_tokens=2)191 >>> print(tokenizer.batch_decode(gen_out, skip_special_tokens=True)[0])192 A number: one thousand193 ```194 """195 196 def __init__(197 self,198 prompt_length_to_skip: int,199 min_new_tokens: int,200 eos_token_id: Union[int, list[int], torch.Tensor],201 device: str = "cpu",202 ):203 for arg_name, arg_value in [204 ("prompt_length_to_skip", prompt_length_to_skip),205 ("min_new_tokens", min_new_tokens),206 ]:207 if not isinstance(arg_value, int) or arg_value < 0:208 raise ValueError(f"`{arg_name}` has to be a positive integer, but is {arg_value}")209 210 if not isinstance(eos_token_id, torch.Tensor):211 if isinstance(eos_token_id, int):212 eos_token_id = [eos_token_id]213 eos_token_id = torch.tensor(eos_token_id, device=device)214 215 self.prompt_length_to_skip = prompt_length_to_skip216 self.min_new_tokens = min_new_tokens217 self.eos_token_id = eos_token_id218 219 @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)220 def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:221 new_tokens_length = input_ids.shape[-1] - self.prompt_length_to_skip222 scores_processed = scores.clone()223 vocab_tensor = torch.arange(scores.shape[-1], device=scores.device)224 eos_token_mask = isin_mps_friendly(vocab_tensor, self.eos_token_id)225 if new_tokens_length < self.min_new_tokens:226 scores_processed = torch.where(eos_token_mask, -math.inf, scores)227 228 return scores_processed229 230 231class TemperatureLogitsWarper(LogitsProcessor):232 r"""233 [`LogitsProcessor`] for temperature (exponential scaling output probability distribution), which effectively means234 that it can control the randomness of the predicted tokens. Often used together with [`TopPLogitsWarper`] and235 [`TopKLogitsWarper`].236 237 <Tip>238 239 Make sure that `do_sample=True` is included in the `generate` arguments otherwise the temperature value won't have240 any effect.241 242 </Tip>243 244 Args:245 temperature (`float`):246 Strictly positive float value used to modulate the logits distribution. A value smaller than `1` decreases247 randomness (and vice versa), with `0` being equivalent to shifting all probability mass to the most likely248 token.249 250 Examples:251 252 ```python253 >>> import torch254 >>> from transformers import AutoTokenizer, AutoModelForCausalLM, set_seed255 256 >>> set_seed(0) # for reproducibility257 258 >>> tokenizer = AutoTokenizer.from_pretrained("openai-community/gpt2")259 >>> model = AutoModelForCausalLM.from_pretrained("openai-community/gpt2")260 >>> model.config.pad_token_id = model.config.eos_token_id261 >>> inputs = tokenizer(["Hugging Face Company is"], return_tensors="pt")262 263 >>> # With temperature=1.0, the default, we consistently get random outputs due to random sampling.264 >>> generate_kwargs = {"max_new_tokens": 10, "do_sample": True, "temperature": 1.0, "num_return_sequences": 2}265 >>> outputs = model.generate(**inputs, **generate_kwargs)266 >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True))267 ['Hugging Face Company is one of these companies that is going to take a',268 "Hugging Face Company is a brand created by Brian A. O'Neil"]269 270 >>> # However, with temperature close to 0, it approximates greedy decoding strategies (invariant)271 >>> generate_kwargs["temperature"] = 0.0001272 >>> outputs = model.generate(**inputs, **generate_kwargs)273 >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True))274 ['Hugging Face Company is a company that has been around for over 20 years',275 'Hugging Face Company is a company that has been around for over 20 years']276 ```277 """278 279 def __init__(self, temperature: float):280 if not isinstance(temperature, float) or not (temperature > 0):281 except_msg = (282 f"`temperature` (={temperature}) has to be a strictly positive float, otherwise your next token "283 "scores will be invalid."284 )285 if isinstance(temperature, float) and temperature == 0.0:286 except_msg += " If you're looking for greedy decoding strategies, set `do_sample=False`."287 raise ValueError(except_msg)288 289 self.temperature = temperature290 291 @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)292 def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:293 scores_processed = scores / self.temperature294 return scores_processed295 296 297class RepetitionPenaltyLogitsProcessor(LogitsProcessor):298 r"""299 [`LogitsProcessor`] that prevents the repetition of previous tokens through a penalty. This penalty is applied at300 most once per token. Note that, for decoder-only models like most LLMs, the considered tokens include the prompt301 by default.302 303 In the original [paper](https://huggingface.co/papers/1909.05858), the authors suggest the use of a penalty of around304 1.2 to achieve a good balance between truthful generation and lack of repetition. To penalize and reduce305 repetition, use `penalty` values above 1.0, where a higher value penalizes more strongly. To reward and encourage306 repetition, use `penalty` values between 0.0 and 1.0, where a lower value rewards more strongly.307 308 Args:309 penalty (`float`):310 The parameter for repetition penalty. 1.0 means no penalty. Above 1.0 penalizes previously generated311 tokens. Between 0.0 and 1.0 rewards previously generated tokens.312 prompt_ignore_length (`int`, *optional*):313 The original input ids sequence length, which if provided, will not be used in the penalty calculation.314 315 Examples:316 317 ```py318 >>> from transformers import AutoTokenizer, AutoModelForCausalLM, RepetitionPenaltyLogitsProcessor319 320 >>> # Initializing the model and tokenizer for it321 >>> model = AutoModelForCausalLM.from_pretrained("distilbert/distilgpt2")322 >>> tokenizer = AutoTokenizer.from_pretrained("distilbert/distilgpt2")323 >>> inputs = tokenizer(["I'm not going to"], return_tensors="pt")324 325 >>> # This shows a normal generate without any specific parameters326 >>> summary_ids = model.generate(**inputs)327 >>> print(tokenizer.batch_decode(summary_ids, skip_special_tokens=True)[0])328 I'm not going to be able to do that. I'm going to be able to do that329 330 >>> # This generates a penalty for repeated tokens331 >>> penalized_ids = model.generate(**inputs, repetition_penalty=1.1)332 >>> print(tokenizer.batch_decode(penalized_ids, skip_special_tokens=True)[0])333 I'm not going to be able to do that. I'll just have to go out and play334 335 >>> # We can also exclude the input prompt by creating an instance of this class336 >>> # with a `prompt_ignore_length` and passing it as a custom logit processor337 >>> rep_pen_processor = RepetitionPenaltyLogitsProcessor(338 ... penalty=1.1,339 ... prompt_ignore_length=inputs["input_ids"].shape[-1]340 ... )341 >>> penalized_ids = model.generate(**inputs, logits_processor=[rep_pen_processor])342 >>> print(tokenizer.batch_decode(penalized_ids, skip_special_tokens=True)[0])343 I'm not going to be able to do that. I'm going to have to go through a lot of things, and344 ```345 """346 347 def __init__(self, penalty: float, prompt_ignore_length: Optional[int] = None):348 if not isinstance(penalty, float) or not (penalty > 0):349 raise ValueError(f"`penalty` has to be a strictly positive float, but is {penalty}")350 351 if prompt_ignore_length is not None and (352 not isinstance(prompt_ignore_length, int) or prompt_ignore_length < 0353 ):354 raise ValueError(f"`prompt_ignore_length` has to be a positive integer, but is {prompt_ignore_length}")355 356 self.penalty = penalty357 self.prompt_ignore_length = prompt_ignore_length358 self.logits_indices = None359 self.cu_seq_lens_q = None360 361 def set_continuous_batching_context(self, logits_indices: torch.Tensor, cu_seq_lens_q: torch.Tensor):362 self.logits_indices = logits_indices363 self.cu_seq_lens_q = cu_seq_lens_q364 365 @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)366 def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:367 if self.prompt_ignore_length:368 input_ids = input_ids[:, self.prompt_ignore_length :]369 370 if scores.dim() == 3:371 if self.logits_indices is not None and self.cu_seq_lens_q is not None:372 last_positions = self.logits_indices373 last_scores = scores[0, last_positions, :]374 375 # Prepare token mask376 token_mask = torch.zeros_like(last_scores, dtype=torch.bool)377 cu_seq_lens = self.cu_seq_lens_q378 lengths = cu_seq_lens[1:] - cu_seq_lens[:-1]379 seq_indices = torch.repeat_interleave(torch.arange(len(lengths), device=input_ids.device), lengths)380 token_mask[seq_indices, input_ids] = True381 382 # Apply penalty383 penalty_scores = torch.where(last_scores < 0, last_scores * self.penalty, last_scores / self.penalty)384 scores[0, last_positions, :] = torch.where(token_mask, penalty_scores, last_scores)385 else:386 batch_size, seq_len, vocab_size = scores.shape387 last_scores = scores[:, -1, :]388 token_mask = torch.zeros_like(last_scores, dtype=torch.bool)389 if input_ids.dim() == 1:390 unique_tokens = torch.unique(input_ids)391 token_mask.scatter_(1, unique_tokens.unsqueeze(0), True)392 else:393 token_mask.scatter_(1, input_ids, True)394 # if last_scores < 0 then repetition penalty has to be multiplied to reduce the token probabilities395 penalty_scores = torch.where(last_scores < 0, last_scores * self.penalty, last_scores / self.penalty)396 scores[:, -1, :] = torch.where(token_mask, penalty_scores, last_scores)397 return scores398 399 if input_ids.dim() == 1:400 input_ids = input_ids.unsqueeze(1)401 402 score = torch.gather(scores, 1, input_ids)403 # if score < 0 then repetition penalty has to be multiplied to reduce the token probabilities404 score = torch.where(score < 0, score * self.penalty, score / self.penalty)405 scores_processed = scores.scatter(1, input_ids, score)406 return scores_processed407 408 409class EncoderRepetitionPenaltyLogitsProcessor(LogitsProcessor):410 r"""411 [`LogitsProcessor`] that works similarly to [`RepetitionPenaltyLogitsProcessor`], but with an *inverse* penalty412 that is applied to the tokens present in the prompt. In other words, a penalty above 1.0 increases the odds of413 selecting tokens that were present in the prompt.414 415 It was designed to avoid hallucination in input-grounded tasks, like summarization. Although originally intended416 for encoder-decoder models, it can also be used with decoder-only models like LLMs.417 418 Args:419 penalty (`float`):420 The parameter for repetition penalty. 1.0 means no penalty. Above 1.0 rewards prompt tokens. Between 0.0421 and 1.0 penalizes prompt tokens.422 encoder_input_ids (`torch.LongTensor`):423 The encoder_input_ids that should be repeated within the decoder ids.424 425 Examples:426 427 ```python428 >>> from transformers import AutoModelForCausalLM, AutoTokenizer429 430 >>> tokenizer = AutoTokenizer.from_pretrained("bigscience/bloomz-560m")431 >>> model = AutoModelForCausalLM.from_pretrained("bigscience/bloomz-560m")432 433 >>> inputs = tokenizer(["Alice and Bob. The third member's name was"], return_tensors="pt")434 >>> gen_out = model.generate(**inputs)435 >>> print(tokenizer.batch_decode(gen_out, skip_special_tokens=True)[0])436 Alice and Bob. The third member's name was not mentioned.437 438 >>> # With the `encoder_repetition_penalty` argument we can trigger this logits processor in `generate`, which can439 >>> # promote the use of prompt tokens ("Bob" in this example)440 >>> gen_out = model.generate(**inputs, encoder_repetition_penalty=1.2)441 >>> print(tokenizer.batch_decode(gen_out, skip_special_tokens=True)[0])442 Alice and Bob. The third member's name was Bob. The third member's name was Bob.443 ```444 """445 446 def __init__(self, penalty: float, encoder_input_ids: torch.LongTensor):447 if not isinstance(penalty, float) or not (penalty > 0):448 raise ValueError(f"`penalty` has to be a strictly positive float, but is {penalty}")449 450 self.penalty = 1 / penalty451 self.encoder_input_ids = encoder_input_ids452 453 @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)454 def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:455 score = torch.gather(scores, 1, self.encoder_input_ids)456 457 # if score < 0 then hallucination penalty has to be multiplied to increase the token probabilities458 score = torch.where(score < 0, score * self.penalty, score / self.penalty)459 460 scores_processed = scores.scatter(1, self.encoder_input_ids, score)461 return scores_processed462 463 464class TopPLogitsWarper(LogitsProcessor):465 """466 [`LogitsProcessor`] that performs top-p, i.e. restricting to top tokens summing to prob_cut_off <= prob_cut_off.467 Often used together with [`TemperatureLogitsWarper`] and [`TopKLogitsWarper`].468 469 Args:470 top_p (`float`):471 If set to < 1, only the smallest set of most probable tokens with probabilities that add up to `top_p` or472 higher are kept for generation.473 filter_value (`float`, *optional*, defaults to -inf):474 All filtered values will be set to this float value.475 min_tokens_to_keep (`int`, *optional*, defaults to 1):476 Minimum number of tokens that cannot be filtered.477 478 Examples:479 480 ```python481 >>> from transformers import AutoTokenizer, AutoModelForCausalLM, set_seed482 483 >>> set_seed(1)484 >>> model = AutoModelForCausalLM.from_pretrained("distilbert/distilgpt2")485 >>> tokenizer = AutoTokenizer.from_pretrained("distilbert/distilgpt2")486 487 >>> inputs = tokenizer("A sequence: 1, 2", return_tensors="pt")488 489 >>> # With sampling, the output is unexpected -- sometimes too unexpected.490 >>> outputs = model.generate(**inputs, do_sample=True)491 >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0])492 A sequence: 1, 2, 3 | < 4 (left-hand pointer) ;493 <BLANKLINE>494 <BLANKLINE>495 496 >>> # With `top_p` sampling, the output gets restricted to high-probability tokens.497 >>> # Pro tip: In practice, LLMs use `top_p` in the 0.9-0.95 range.498 >>> outputs = model.generate(**inputs, do_sample=True, top_p=0.1)499 >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0])500 A sequence: 1, 2, 3, 4, 5, 6, 7, 8, 9501 ```502 """503 504 def __init__(self, top_p: float, filter_value: float = -float("Inf"), min_tokens_to_keep: int = 1):505 top_p = float(top_p)506 if top_p < 0 or top_p > 1.0:507 raise ValueError(f"`top_p` has to be a float > 0 and < 1, but is {top_p}")508 if not isinstance(min_tokens_to_keep, int) or (min_tokens_to_keep < 1):509 raise ValueError(f"`min_tokens_to_keep` has to be a positive integer, but is {min_tokens_to_keep}")510 511 self.top_p = top_p512 self.filter_value = filter_value513 self.min_tokens_to_keep = min_tokens_to_keep514 515 @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)516 def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:517 sorted_logits, sorted_indices = torch.sort(scores, descending=False)518 cumulative_probs = sorted_logits.softmax(dim=-1).cumsum(dim=-1)519 520 # Remove tokens with cumulative top_p above the threshold (token with 0 are kept)521 sorted_indices_to_remove = cumulative_probs <= (1 - self.top_p)522 # Keep at least min_tokens_to_keep523 sorted_indices_to_remove[..., -self.min_tokens_to_keep :] = 0524 525 # scatter sorted tensors to original indexing526 indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)527 scores_processed = scores.masked_fill(indices_to_remove, self.filter_value)528 return scores_processed529 530 531class TopKLogitsWarper(LogitsProcessor):532 r"""533 [`LogitsProcessor`] that performs top-k, i.e. restricting to the k highest probability elements. Often used534 together with [`TemperatureLogitsWarper`] and [`TopPLogitsWarper`].535 536 Args:537 top_k (`int`):538 The number of highest probability vocabulary tokens to keep for top-k-filtering.539 filter_value (`float`, *optional*, defaults to -inf):540 All filtered values will be set to this float value.541 min_tokens_to_keep (`int`, *optional*, defaults to 1):542 Minimum number of tokens that cannot be filtered.543 544 Examples:545 546 ```python547 >>> from transformers import AutoTokenizer, AutoModelForCausalLM, set_seed548 549 >>> set_seed(1)550 >>> model = AutoModelForCausalLM.from_pretrained("distilbert/distilgpt2")551 >>> tokenizer = AutoTokenizer.from_pretrained("distilbert/distilgpt2")552 553 >>> inputs = tokenizer("A sequence: A, B, C, D", return_tensors="pt")554 555 >>> # With sampling, the output is unexpected -- sometimes too unexpected.556 >>> outputs = model.generate(**inputs, do_sample=True)557 >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0])558 A sequence: A, B, C, D, E — S — O, P — R559 560 >>> # With `top_k` sampling, the output gets restricted the k most likely tokens.561 >>> # Pro tip: In practice, LLMs use `top_k` in the 5-50 range.562 >>> outputs = model.generate(**inputs, do_sample=True, top_k=2)563 >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0])564 A sequence: A, B, C, D, E, F, G, H, I565 ```566 """567 568 def __init__(self, top_k: int, filter_value: float = -float("Inf"), min_tokens_to_keep: int = 1):569 if not isinstance(top_k, int) or top_k <= 0:570 raise ValueError(f"`top_k` has to be a strictly positive integer, but is {top_k}")571 572 self.top_k = max(top_k, min_tokens_to_keep)573 self.filter_value = filter_value574 575 @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)576 def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:577 top_k = min(self.top_k, scores.size(-1)) # Safety check578 # Remove all tokens with a probability less than the last token of the top-k579 indices_to_remove = scores < torch.topk(scores, top_k)[0][..., -1, None]580 scores_processed = scores.masked_fill(indices_to_remove, self.filter_value)581 return scores_processed582 583 584class MinPLogitsWarper(LogitsProcessor):585 """586 [`LogitsProcessor`] that performs min-p, i.e. keeps all tokens that are above a minimum probability, scaled by the587 probability of the most likely token. As a result, the filter becomes more aggressive in the presence of588 high-probability tokens, which is a sign of a confident output that we shouldn't deviate from.589 590 Often used together with [`TemperatureLogitsWarper`]. Used as an alternative to [`TopPLogitsWarper`] and591 [`TopKLogitsWarper`].592 593 Created by @menhguin and @kalomaze (github handles). Code adapted from [this external PR](https://github.com/oobabooga/text-generation-webui/pull/4449/files)594 595 Args:596 min_p (`float`):597 Minimum token probability, which will be scaled by the probability of the most likely token. It must be a598 value between 0 and 1. Typical values are in the 0.01-0.2 range, comparably selective as setting `top_p` in599 the 0.99-0.8 range (use the opposite of normal `top_p` values).600 filter_value (`float`, *optional*, defaults to -inf):601 All filtered values will be set to this float value.602 min_tokens_to_keep (`int`, *optional*, defaults to 1):603 Minimum number of tokens that cannot be filtered.604 605 Examples:606 607 ```python608 >>> from transformers import AutoTokenizer, AutoModelForCausalLM, set_seed609 610 >>> set_seed(1)611 >>> model = AutoModelForCausalLM.from_pretrained("distilbert/distilgpt2")612 >>> tokenizer = AutoTokenizer.from_pretrained("distilbert/distilgpt2")613 614 >>> inputs = tokenizer("A sequence: 1, 2", return_tensors="pt")615 616 >>> # With sampling, the output is unexpected -- sometimes too unexpected.617 >>> outputs = model.generate(**inputs, do_sample=True)618 >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0])619 A sequence: 1, 2, 3 | < 4 (left-hand pointer) ;620 <BLANKLINE>621 <BLANKLINE>622 623 >>> # With `min_p` sampling, the output gets restricted to high-probability tokens.624 >>> # Pro tip: In practice, LLMs use `min_p` in the 0.01-0.2 range.625 >>> outputs = model.generate(**inputs, do_sample=True, min_p=0.1)626 >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0])627 A sequence: 1, 2, 3, 4, 5, 6, 7, 8, 9628 ```629 """630 631 def __init__(self, min_p: float, filter_value: float = -float("Inf"), min_tokens_to_keep: int = 1):632 if not (0 <= min_p <= 1.0):633 raise ValueError(f"`min_p` has to be a float in the [0, 1] interval, but is {min_p}")634 if not isinstance(min_tokens_to_keep, int) or (min_tokens_to_keep < 1):635 raise ValueError(f"`min_tokens_to_keep` has to be a positive integer, but is {min_tokens_to_keep}")636 637 self.min_p = min_p638 self.filter_value = filter_value639 self.min_tokens_to_keep = min_tokens_to_keep640 641 def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:642 # Convert logits to probabilities643 probs = torch.softmax(scores, dim=-1)644 # Get the probability of the top token for each sequence in the batch645 top_probs, _ = probs.max(dim=-1, keepdim=True)646 # Calculate the actual min_p threshold by scaling min_p with the top token's probability647 scaled_min_p = self.min_p * top_probs648 # Create a mask for tokens that have a probability less than the scaled min_p649 tokens_to_remove = probs < scaled_min_p650 651 sorted_indices = torch.argsort(scores, descending=True, dim=-1)652 sorted_indices_to_remove = torch.gather(tokens_to_remove, dim=-1, index=sorted_indices)653 # Keep at least min_tokens_to_keep654 sorted_indices_to_remove[..., : self.min_tokens_to_keep] = False655 656 indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)657 scores_processed = scores.masked_fill(indices_to_remove, self.filter_value)658 return scores_processed659 660 661class TypicalLogitsWarper(LogitsProcessor):662 r"""663 [`LogitsProcessor`] that performs typical decoding. Inspired on how humans use language, it prioritizes tokens664 whose log probability is close to the entropy of the token probability distribution. This means that the most665 likely tokens may be discarded in the process.666 667 See [Typical Decoding for Natural Language Generation](https://huggingface.co/papers/2202.00666) for more information.668 669 Args:670 mass (`float`, *optional*, defaults to 0.9):671 Value of typical_p between 0 and 1 inclusive, defaults to 0.9.672 filter_value (`float`, *optional*, defaults to -inf):673 All filtered values will be set to this float value.674 min_tokens_to_keep (`int`, *optional*, defaults to 1):675 Minimum number of tokens that cannot be filtered.676 677 Examples:678 679 ```python680 >>> from transformers import AutoTokenizer, AutoModelForCausalLM, set_seed681 682 >>> model = AutoModelForCausalLM.from_pretrained("bigscience/bloomz-560m")683 >>> tokenizer = AutoTokenizer.from_pretrained("bigscience/bloomz-560m")684 685 >>> inputs = tokenizer("1, 2, 3", return_tensors="pt")686 687 >>> # We can see that greedy decoding produces a sequence of numbers688 >>> outputs = model.generate(**inputs)689 >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0])690 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,691 692 >>> # For this particular seed, we can see that sampling produces nearly the same low-information (= low entropy)693 >>> # sequence694 >>> set_seed(18)695 >>> outputs = model.generate(**inputs, do_sample=True)696 >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0])697 1, 2, 3, 4, 5, 6, 7, 8, 9 and 10698 699 >>> # With `typical_p` set, the most obvious sequence is no longer produced, which may be good for your problem700 >>> set_seed(18)701 >>> outputs = model.generate(702 ... **inputs, do_sample=True, typical_p=0.1, return_dict_in_generate=True, output_scores=True703 ... )704 >>> print(tokenizer.batch_decode(outputs.sequences, skip_special_tokens=True)[0])705 1, 2, 3 and 5706 707 >>> # We can see that the token corresponding to "4" (token 934) in the second position, the most likely token708 >>> # as seen with greedy decoding, was entirely blocked out709 >>> print(outputs.scores[1][0, 934])710 tensor(-inf)711 ```712 """713 714 def __init__(self, mass: float = 0.9, filter_value: float = -float("Inf"), min_tokens_to_keep: int = 1):715 mass = float(mass)716 if not (mass > 0 and mass < 1):717 raise ValueError(f"`typical_p` has to be a float > 0 and < 1, but is {mass}")718 if not isinstance(min_tokens_to_keep, int) or (min_tokens_to_keep < 1):719 raise ValueError(f"`min_tokens_to_keep` has to be a positive integer, but is {min_tokens_to_keep}")720 721 self.filter_value = filter_value722 self.mass = mass723 self.min_tokens_to_keep = min_tokens_to_keep724 725 @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)726 def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:727 # calculate entropy728 normalized = torch.nn.functional.log_softmax(scores, dim=-1)729 p = torch.exp(normalized)730 ent = -(normalized * p).nansum(-1, keepdim=True)731 732 # shift and sort733 shifted_scores = torch.abs((-normalized) - ent)734 sorted_scores, sorted_indices = torch.sort(shifted_scores, descending=False)735 sorted_logits = scores.gather(-1, sorted_indices)736 cumulative_probs = sorted_logits.softmax(dim=-1).cumsum(dim=-1)737 738 # Remove tokens with cumulative mass above the threshold739 last_ind = (cumulative_probs < self.mass).sum(dim=1)740 last_ind.clamp_(max=sorted_scores.shape[-1] - 1)741 sorted_indices_to_remove = sorted_scores > sorted_scores.gather(1, last_ind.view(-1, 1))742 sorted_indices_to_remove[..., : self.min_tokens_to_keep] = 0743 indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)744 745 scores_processed = scores.masked_fill(indices_to_remove, self.filter_value)746 return scores_processed747 748 749class EpsilonLogitsWarper(LogitsProcessor):750 r"""751 [`LogitsProcessor`] that performs epsilon-sampling, i.e. restricting to tokens with `prob >= epsilon`. Takes the752 largest min_tokens_to_keep tokens if no tokens satisfy this constraint. See [Truncation Sampling as Language Model753 Desmoothing](https://huggingface.co/papers/2210.15191) for more information.754 755 Args:756 epsilon (`float`):757 If set to > 0, only the most tokens with probabilities `epsilon` or higher are kept for generation.758 filter_value (`float`, *optional*, defaults to -inf):759 All filtered values will be set to this float value.760 min_tokens_to_keep (`int`, *optional*, defaults to 1):761 Minimum number of tokens that cannot be filtered.762 763 Examples:764 ```python765 >>> from transformers import AutoTokenizer, AutoModelForCausalLM, set_seed766 767 >>> set_seed(1)768 >>> model = AutoModelForCausalLM.from_pretrained("distilbert/distilgpt2")769 >>> tokenizer = AutoTokenizer.from_pretrained("distilbert/distilgpt2")770 771 >>> inputs = tokenizer("A sequence: 1, 2", return_tensors="pt")772 773 >>> # With sampling, the output is unexpected -- sometimes too unexpected.774 >>> outputs = model.generate(**inputs, do_sample=True)775 >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0])776 A sequence: 1, 2, 3 | < 4 (left-hand pointer) ;777 <BLANKLINE>778 <BLANKLINE>779 780 >>> # With epsilon sampling, the output gets restricted to high-probability tokens. Note that this is similar to781 >>> # Top P sampling, which restricts tokens based on their cumulative probability.782 >>> # Pro tip: The paper recommends using `epsilon_cutoff` values between 3e-4 and 9e-4783 >>> outputs = model.generate(**inputs, do_sample=True, epsilon_cutoff=0.1)784 >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0])785 A sequence: 1, 2, 3, 4, 5, 6, 7, 8, 9786 ```787 """788 789 def __init__(self, epsilon: float, filter_value: float = -float("Inf"), min_tokens_to_keep: int = 1):790 epsilon = float(epsilon)791 if epsilon <= 0 or epsilon >= 1:792 raise ValueError(f"`epsilon_cutoff` has to be a float > 0 and < 1, but is {epsilon}")793 794 min_tokens_to_keep = int(min_tokens_to_keep)795 if min_tokens_to_keep < 1:796 raise ValueError(797 f"`min_tokens_to_keep` has to be a strictly positive integer, but is {min_tokens_to_keep}"798 )799 800 self.epsilon = epsilon801 self.filter_value = filter_value802 self.min_tokens_to_keep = min_tokens_to_keep803 804 @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)805 def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:806 # Determine which indices to remove807 probabilities = scores.softmax(dim=-1)808 indices_to_remove = probabilities < self.epsilon809 810 # Keep the words with the 'min_tokens_to_keep'-highest probabilities811 top_k = min(self.min_tokens_to_keep, scores.size(-1)) # Safety check812 indices_to_remove = indices_to_remove & (scores < torch.topk(scores, top_k)[0][..., -1, None])813 814 scores_processed = scores.masked_fill(indices_to_remove, self.filter_value)815 return scores_processed816 817 818class EtaLogitsWarper(LogitsProcessor):819 r"""820 [`LogitsProcessor`] that performs eta-sampling, a technique to filter out tokens with probabilities below a dynamic821 cutoff value, `eta`, which is calculated based on a combination of the hyperparameter `epsilon` and the entropy of822 the token probabilities, i.e. `eta := min(epsilon, sqrt(epsilon * e^-entropy(probabilities)))`. Takes the largest823 min_tokens_to_keep tokens if no tokens satisfy this constraint. It addresses the issue of poor quality in long824 samples of text generated by neural language models leading to more coherent and fluent text. See [Truncation825 Sampling as Language Model Desmoothing](https://huggingface.co/papers/2210.15191) for more information. Note: `do_sample`826 must be set to `True` for this `LogitsProcessor` to work.827 828 829 Args:830 epsilon (`float`):831 A float value in the range (0, 1). Hyperparameter used to calculate the dynamic cutoff value, `eta`. The832 suggested values from the paper ranges from 3e-4 to 4e-3 depending on the size of the model.833 filter_value (`float`, *optional*, defaults to -inf):834 All values that are found to be below the dynamic cutoff value, `eta`, are set to this float value. This835 parameter is useful when logits need to be modified for very low probability tokens that should be excluded836 from generation entirely.837 min_tokens_to_keep (`int`, *optional*, defaults to 1):838 Specifies the minimum number of tokens that must be kept for generation, regardless of their probabilities.839 For example, if `min_tokens_to_keep` is set to 1, at least one token will always be kept for generation,840 even if all tokens have probabilities below the cutoff `eta`.841 device (`str`, *optional*, defaults to `"cpu"`):842 The device to allocate the tensors.843 844 Examples:845 ```python846 >>> from transformers import AutoTokenizer, AutoModelForCausalLM, set_seed847 848 >>> set_seed(1)849 >>> model = AutoModelForCausalLM.from_pretrained("distilbert/distilgpt2")850 >>> tokenizer = AutoTokenizer.from_pretrained("distilbert/distilgpt2")851 852 >>> inputs = tokenizer("A sequence: 1, 2", return_tensors="pt")853 854 >>> # With sampling, the output is unexpected -- sometimes too unexpected.855 >>> outputs = model.generate(**inputs, do_sample=True)856 >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0])857 A sequence: 1, 2, 3 | < 4 (left-hand pointer) ;858 <BLANKLINE>859 <BLANKLINE>860 861 >>> # With eta sampling, the output gets restricted to high-probability tokens. You can see it as a dynamic form of862 >>> # epsilon sampling that adapts its cutoff probability based on the entropy (high entropy = lower cutoff).863 >>> # Pro tip: The paper recommends using `eta_cutoff` values between 3e-4 to 4e-3864 >>> outputs = model.generate(**inputs, do_sample=True, eta_cutoff=0.1)865 >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0])866 A sequence: 1, 2, 3, 4, 5, 6, 7, 8, 9867 ```868 """869 870 def __init__(871 self, epsilon: float, filter_value: float = -float("Inf"), min_tokens_to_keep: int = 1, device: str = "cpu"872 ):873 epsilon = float(epsilon)874 if epsilon <= 0 or epsilon >= 1:875 raise ValueError(f"`eta_cutoff` has to be a float > 0 and < 1, but is {epsilon}")876 877 min_tokens_to_keep = int(min_tokens_to_keep)878 if min_tokens_to_keep < 1:879 raise ValueError(880 f"`min_tokens_to_keep` has to be a strictly positive integer, but is {min_tokens_to_keep}"881 )882 883 self.epsilon = torch.tensor(epsilon, device=device)884 self.filter_value = filter_value885 self.min_tokens_to_keep = min_tokens_to_keep886 887 @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)888 def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:889 probabilities = scores.softmax(dim=-1)890 entropy = torch.distributions.Categorical(logits=scores).entropy()891 eta = torch.min(self.epsilon, torch.sqrt(self.epsilon) * torch.exp(-entropy))[..., None]892 indices_to_remove = probabilities < eta893 894 # Keep the words with the 'min_tokens_to_keep'-highest probabilities895 top_k = min(self.min_tokens_to_keep, scores.size(-1)) # Safety check896 indices_to_remove = indices_to_remove & (scores < torch.topk(scores, top_k)[0][..., -1, None])897 898 scores_processed = scores.masked_fill(indices_to_remove, self.filter_value)899 return scores_processed900 901 902def _get_ngrams(ngram_size: int, prev_input_ids: torch.Tensor, num_hypos: int):903 """904 Assume ngram_size=2 and prev_input_ids=tensor([[40, 2883, 2712, 4346]]). The output of generated ngrams look like905 this {(40,): [2883], (2883,): [2712], (2712,): [4346]}.906 907 Args:908 ngram_size (`int`):909 The number sequential tokens taken as a group which may only occur once before being banned.910 prev_input_ids (`torch.Tensor`):911 Generated token ids for the current hypothesis.912 num_hypos (`int`):913 The number of hypotheses for which n-grams need to be generated.914 915 Returns:916 generated_ngrams (`dict`):917 Dictionary of generated ngrams.918 """919 # Initialize an empty list of dictionaries, one for each hypothesis (index) in the range of num_hypos920 generated_ngrams = [{} for _ in range(num_hypos)]921 for idx in range(num_hypos):922 gen_tokens = prev_input_ids[idx].tolist()923 generated_ngram = generated_ngrams[idx]924 # Loop through each n-gram of size ngram_size in the list of tokens (gen_tokens)925 for ngram in zip(*[gen_tokens[i:] for i in range(ngram_size)]):926 prev_ngram_tuple = tuple(ngram[:-1])927 generated_ngram[prev_ngram_tuple] = generated_ngram.get(prev_ngram_tuple, []) + [ngram[-1]]928 return generated_ngrams929 930 931def _get_generated_ngrams(banned_ngrams, prev_input_ids, ngram_size, cur_len):932 """933 Determines the banned tokens for the current hypothesis based on previously generated n-grams.934 935 Args:936 banned_ngrams (`dict`):937 A dictionary containing previously generated n-grams for each hypothesis.938 prev_input_ids (`torch.Tensor`):939 Generated token ids for the current hypothesis.940 ngram_size (`int`):941 The number sequential tokens taken as a group which may only occur once before being banned.942 cur_len (`int`):943 The current length of the token sequences for which the n-grams are being checked.944 945 Returns:946 List of tokens that are banned.947 """948 # Before decoding the next token, prevent decoding of ngrams that have already appeared949 start_idx = cur_len + 1 - ngram_size950 ngram_idx = tuple(prev_input_ids[start_idx:cur_len].tolist())951 return banned_ngrams.get(ngram_idx, [])952 953 954def _calc_banned_ngram_tokens(955 ngram_size: int, prev_input_ids: torch.Tensor, num_hypos: int, cur_len: int956) -> list[Iterable[int]]:957 """Copied from fairseq for no_repeat_ngram in beam_search"""958 if cur_len + 1 < ngram_size:959 # return no banned tokens if we haven't generated no_repeat_ngram_size tokens yet960 return [[] for _ in range(num_hypos)]961 generated_ngrams = _get_ngrams(ngram_size, prev_input_ids, num_hypos)962 banned_tokens = [963 _get_generated_ngrams(generated_ngrams[hypo_idx], prev_input_ids[hypo_idx], ngram_size, cur_len)964 for hypo_idx in range(num_hypos)965 ]966 return banned_tokens967 968 969class NoRepeatNGramLogitsProcessor(LogitsProcessor):970 r"""971 N-grams are groups of "n" consecutive words, characters, or tokens taken from a sequence of text. Given the972 sentence: "She runs fast", the bi-grams (n=2) would be ("she", "runs") and ("runs", "fast"). In text generation,973 avoiding repetitions of word sequences provides a more diverse output. This [`LogitsProcessor`] enforces no974 repetition of n-grams by setting the scores of banned tokens to negative infinity which eliminates those tokens975 from consideration when further processing the scores. Note that, for decoder-only models like most LLMs, the976 prompt is also considered to obtain the n-grams.977 [Fairseq](https://github.com/pytorch/fairseq/blob/a07cb6f40480928c9e0548b737aadd36ee66ac76/fairseq/sequence_generator.py#L345).978 979 <Tip>980 981 Use n-gram penalties with care. For instance, penalizing 2-grams (bigrams) in an article about the city of New York982 might lead to undesirable outcomes where the city's name appears only once in the entire text.983 [Reference](https://huggingface.co/blog/how-to-generate)984 985 </Tip>986 987 Args:988 ngram_size (`int`):989 All ngrams of size `ngram_size` can only occur once.990 991 Examples:992 993 ```py994 >>> from transformers import AutoTokenizer, AutoModelForCausalLM995 996 >>> model = AutoModelForCausalLM.from_pretrained("distilbert/distilgpt2")997 >>> tokenizer = AutoTokenizer.from_pretrained("distilbert/distilgpt2")998 >>> inputs = tokenizer(["Today I"], return_tensors="pt")999 1000 >>> output = model.generate(**inputs)1001 >>> print(tokenizer.decode(output[0], skip_special_tokens=True))1002 Today I'm not sure if I'm going to be able to do it.1003 1004 >>> # Now let's add ngram size using `no_repeat_ngram_size`. This stops the repetitions ("I'm") in the output.1005 >>> output = model.generate(**inputs, no_repeat_ngram_size=2)1006 >>> print(tokenizer.decode(output[0], skip_special_tokens=True))1007 Today I'm not sure if I can get a better understanding of the nature of this issue1008 ```1009 """1010 1011 def __init__(self, ngram_size: int):1012 if not isinstance(ngram_size, int) or ngram_size <= 0:1013 raise ValueError(f"`ngram_size` has to be a strictly positive integer, but is {ngram_size}")1014 self.ngram_size = ngram_size1015 1016 @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)1017 def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:1018 num_batch_hypotheses = scores.shape[0]1019 cur_len = input_ids.shape[-1]1020 scores_processed = scores.clone()1021 banned_batch_tokens = _calc_banned_ngram_tokens(self.ngram_size, input_ids, num_batch_hypotheses, cur_len)1022 for i, banned_tokens in enumerate(banned_batch_tokens):1023 scores_processed[i, banned_tokens] = -float("inf")1024 1025 return scores_processed1026 1027 1028class EncoderNoRepeatNGramLogitsProcessor(LogitsProcessor):1029 r"""1030 [`LogitsProcessor`] that works similarly to [`NoRepeatNGramLogitsProcessor`], but applied exclusively to prevent1031 the repetition of n-grams present in the prompt.1032 1033 It was designed to promote chattiness in a language model, by preventing the generation of n-grams present in1034 previous conversation rounds.1035 1036 Args:1037 encoder_ngram_size (`int`):1038 All ngrams of size `ngram_size` can only occur within the encoder input ids.1039 encoder_input_ids (`int`):1040 The encoder_input_ids that should not be repeated within the decoder ids.1041 1042 Examples:1043 1044 ```py1045 >>> from transformers import AutoTokenizer, AutoModelForCausalLM1046 1047 >>> model = AutoModelForCausalLM.from_pretrained("bigscience/bloomz-560m")1048 >>> tokenizer = AutoTokenizer.from_pretrained("bigscience/bloomz-560m")1049 1050 >>> inputs = tokenizer("Alice: I love cats. What do you love?\nBob:", return_tensors="pt")1051 1052 >>> # With greedy decoding, we see Bob repeating Alice's opinion. If Bob was a chatbot, it would be a poor one.1053 >>> outputs = model.generate(**inputs)1054 >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0])1055 Alice: I love cats. What do you love?1056 Bob: I love cats. What do you1057 1058 >>> # With this logits processor, we can prevent Bob from repeating Alice's opinion.1059 >>> outputs = model.generate(**inputs, encoder_no_repeat_ngram_size=2)1060 >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0])1061 Alice: I love cats. What do you love?1062 Bob: My cats are very cute.1063 ```1064 """1065 1066 def __init__(self, encoder_ngram_size: int, encoder_input_ids: torch.LongTensor):1067 if not isinstance(encoder_ngram_size, int) or encoder_ngram_size <= 0:1068 raise ValueError(1069 f"`encoder_ngram_size` has to be a strictly positive integer, but is {encoder_ngram_size}"1070 )1071 self.ngram_size = encoder_ngram_size1072 if len(encoder_input_ids.shape) == 1:1073 encoder_input_ids = encoder_input_ids.unsqueeze(0)1074 self.batch_size = encoder_input_ids.shape[0]1075 self.generated_ngrams = _get_ngrams(encoder_ngram_size, encoder_input_ids, self.batch_size)1076 1077 @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)1078 def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:1079 # B x num_beams1080 num_hypos = scores.shape[0]1081 num_beams = num_hypos // self.batch_size1082 cur_len = input_ids.shape[-1]1083 scores_processed = scores.clone()1084 banned_batch_tokens = [1085 _get_generated_ngrams(1086 self.generated_ngrams[hypo_idx // num_beams], input_ids[hypo_idx], self.ngram_size, cur_len1087 )1088 for hypo_idx in range(num_hypos)1089 ]1090 1091 for i, banned_tokens in enumerate(banned_batch_tokens):1092 scores_processed[i, banned_tokens] = -float("inf")1093 1094 return scores_processed1095 1096 1097class SequenceBiasLogitsProcessor(LogitsProcessor):1098 """1099 [`LogitsProcessor`] that applies an additive bias on sequences. The bias is applied to the last token of a sequence1100 when the next generated token can complete it. Consequently, to take the most of biasing sequences with more than1101 one token, consider using beam methods (to gracefully work around partially completed sequences that have a1102 negative bias) and applying the bias to their prefixes (to ensure the bias is applied earlier).1103 1104 <Tip>1105 1106 At a token-level, biasing a word is different from biasing a word with a space before it. If you want to bias1107 "foo" mid-sentence, you'll likely want to add a prefix space and bias " foo" instead. Check the tokenizer section1108 of our NLP course to find out why: https://huggingface.co/learn/nlp-course/chapter2/4?fw=pt1109 1110 </Tip>1111 1112 Args:1113 sequence_bias (`list[list[Union[list[int], float]]]`):1114 List of lists that maps a sequence of tokens to its bias term (e.g. `[[[10, 45], -2.0],1115 [[64], -7.5]]`). Positive biases increase the odds of the1116 sequence being selected, while negative biases do the opposite. If a sequence has a length of 1, its bias1117 will always be applied. Otherwise, the bias will only be applied if the sequence in question is about to be1118 completed (in the token selection step after this processor is applied).1119 1120 Examples:1121 1122 ```python1123 >>> from transformers import AutoTokenizer, AutoModelForCausalLM1124 1125 >>> model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2.5-0.5B-Instruct")1126 >>> tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-0.5B-Instruct")1127 >>> inputs = tokenizer(["The full name of Donald is Donald"], return_tensors="pt")1128 1129 >>> summary_ids = model.generate(inputs["input_ids"], max_new_tokens=4, do_sample=False)1130 >>> print(tokenizer.batch_decode(summary_ids, skip_special_tokens=True)[0])1131 The full name of Donald is Donald John Trump Sr.1132 1133 >>> def get_tokens(word):1134 ... return tokenizer([word], add_special_tokens=False).input_ids[0]1135 1136 >>> # IMPORTANT: Remember our tip about adding spaces before words to bias them correctly.1137 >>> sequence_bias = [[get_tokens("Trump"), -10.0],] # will fail to apply bias1138 >>> biased_ids = model.generate(1139 ... inputs["input_ids"], max_new_tokens=4, do_sample=False, sequence_bias=sequence_bias1140 ... )1141 >>> print(tokenizer.batch_decode(biased_ids, skip_special_tokens=True)[0])1142 The full name of Donald is Donald John Trump Sr.1143 1144 >>> sequence_bias = [[get_tokens(" Trump"), -10.0],] # will work1145 >>> biased_ids = model.generate(1146 ... inputs["input_ids"], max_new_tokens=4, do_sample=False, sequence_bias=sequence_bias1147 ... )1148 >>> print(tokenizer.batch_decode(biased_ids, skip_special_tokens=True)[0])1149 The full name of Donald is Donald John Harper. He1150 1151 >>> # We can also add a positive bias to nudge the model towards specific tokens or continuations. This technique1152 >>> # is also more effective when paired up with beam search.1153 >>> sequence_bias = [[get_tokens(" Donald Duck"), 10.0],]1154 >>> biased_ids = model.generate(1155 ... inputs["input_ids"], max_new_tokens=4, num_beams=4, do_sample=False, sequence_bias=sequence_bias1156 ... )1157 >>> print(tokenizer.batch_decode(biased_ids, skip_special_tokens=True)[0])1158 The full name of Donald is Donald Duck. He is1159 ```1160 """1161 1162 def __init__(self, sequence_bias: list[list[Union[list[int], float]]]):1163 self.sequence_bias = sequence_bias1164 self._validate_arguments()1165 self._convert_list_arguments_into_dict()1166 1167 # Bias variables that will be populated on the first call (for retrocompatibility purposes, the vocabulary size1168 # is inferred in the first usage, which inhibits initializing here)1169 self.length_1_bias = None1170 self.prepared_bias_variables = False1171 1172 @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)1173 def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:1174 # 1 - Prepares the bias tensors. This is only needed the first time the logit processor is called.1175 if not self.prepared_bias_variables:1176 self._prepare_bias_variables(scores)1177 1178 # 2 - prepares an empty bias to add1179 bias = torch.zeros_like(scores)1180 1181 # 3 - include the bias from length = 11182 bias += self.length_1_bias1183 1184 # 4 - include the bias from length > 1, after determining which biased sequences may be completed.1185 for sequence_ids, sequence_bias in self.sequence_bias.items():1186 if len(sequence_ids) == 1: # the sequence is of length 1, already applied1187 continue1188 if len(sequence_ids) > input_ids.shape[1]: # the sequence is longer than the context, ignore1189 continue1190 prefix_length = len(sequence_ids) - 11191 last_token = sequence_ids[-1]1192 matching_rows = torch.eq(1193 input_ids[:, -prefix_length:],1194 torch.tensor(sequence_ids[:-1], dtype=input_ids.dtype, device=input_ids.device),1195 ).prod(dim=1)1196 bias[:, last_token] += torch.where(1197 matching_rows.bool(),1198 torch.tensor(sequence_bias, device=input_ids.device),1199 torch.tensor(0.0, device=input_ids.device),1200 )