Aluode/PerceptionLabPortable
0
1# coding=utf-82# Copyright 2021 The Facebook, Inc. and The HuggingFace Inc. team. All rights reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""PyTorch Blenderbot model."""16 17import math18import os19import warnings20from typing import Callable, Optional, Union21 22import torch23from torch import nn24from torch.nn import CrossEntropyLoss25 26from ...activations import ACT2FN27from ...cache_utils import Cache, DynamicCache, EncoderDecoderCache28from ...generation import GenerationMixin29from ...modeling_attn_mask_utils import (30 AttentionMaskConverter,31 _prepare_4d_attention_mask,32 _prepare_4d_attention_mask_for_sdpa,33)34from ...modeling_flash_attention_utils import FlashAttentionKwargs35from ...modeling_layers import GradientCheckpointingLayer36from ...modeling_outputs import (37 BaseModelOutput,38 BaseModelOutputWithPastAndCrossAttentions,39 CausalLMOutputWithCrossAttentions,40 Seq2SeqLMOutput,41 Seq2SeqModelOutput,42)43from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel44from ...processing_utils import Unpack45from ...utils import (46 auto_docstring,47 is_torch_flex_attn_available,48 is_torchdynamo_compiling,49 logging,50)51from ...utils.deprecation import deprecate_kwarg52from ..blenderbot_small import BlenderbotSmallForConditionalGeneration, BlenderbotSmallModel53from .configuration_blenderbot import BlenderbotConfig54 55 56if is_torch_flex_attn_available():57 from ...integrations.flex_attention import BlockMask, make_flex_block_causal_mask58 59 60logger = logging.get_logger(__name__)61 62 63# Copied from transformers.models.bart.modeling_bart.shift_tokens_right64def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int):65 """66 Shift input ids one token to the right.67 """68 shifted_input_ids = input_ids.new_zeros(input_ids.shape)69 shifted_input_ids[:, 1:] = input_ids[:, :-1].clone()70 shifted_input_ids[:, 0] = decoder_start_token_id71 72 if pad_token_id is None:73 raise ValueError("self.model.config.pad_token_id has to be defined.")74 # replace possible -100 values in labels by `pad_token_id`75 shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id)76 77 return shifted_input_ids78 79 80class BlenderbotLearnedPositionalEmbedding(nn.Embedding):81 """82 This module learns positional embeddings up to a fixed maximum size.83 """84 85 def __init__(self, num_embeddings: int, embedding_dim: int):86 super().__init__(num_embeddings, embedding_dim)87 88 def forward(89 self, input_ids_shape: torch.Size, past_key_values_length: int = 0, position_ids: Optional[torch.Tensor] = None90 ):91 """`input_ids_shape` is expected to be [bsz x seqlen]."""92 if position_ids is None:93 bsz, seq_len = input_ids_shape[:2]94 position_ids = torch.arange(95 past_key_values_length, past_key_values_length + seq_len, dtype=torch.long, device=self.weight.device96 )97 return super().forward(position_ids)98 99 100# Copied from transformers.models.bart.modeling_bart.BartScaledWordEmbedding with Bart->Blenderbot101class BlenderbotScaledWordEmbedding(nn.Embedding):102 """103 This module overrides nn.Embeddings' forward by multiplying with embeddings scale.104 """105 106 def __init__(self, num_embeddings: int, embedding_dim: int, padding_idx: int, embed_scale: Optional[float] = 1.0):107 super().__init__(num_embeddings, embedding_dim, padding_idx)108 self.embed_scale = embed_scale109 110 def forward(self, input_ids: torch.Tensor):111 return super().forward(input_ids) * self.embed_scale112 113 114# Copied from transformers.models.bart.modeling_bart.eager_attention_forward115def eager_attention_forward(116 module: nn.Module,117 query: torch.Tensor,118 key: torch.Tensor,119 value: torch.Tensor,120 attention_mask: Optional[torch.Tensor],121 scaling: Optional[float] = None,122 dropout: float = 0.0,123 head_mask: Optional[torch.Tensor] = None,124 **kwargs,125):126 if scaling is None:127 scaling = query.size(-1) ** -0.5128 129 attn_weights = torch.matmul(query, key.transpose(2, 3)) * scaling130 if attention_mask is not None:131 attn_weights = attn_weights + attention_mask132 133 attn_weights = nn.functional.softmax(attn_weights, dim=-1)134 135 if head_mask is not None:136 attn_weights = attn_weights * head_mask.view(1, -1, 1, 1)137 138 attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)139 attn_output = torch.matmul(attn_weights, value)140 attn_output = attn_output.transpose(1, 2).contiguous()141 142 return attn_output, attn_weights143 144 145# Copied from transformers.models.bart.modeling_bart.BartAttention with Bart->Blenderbot146class BlenderbotAttention(nn.Module):147 """Multi-headed attention from 'Attention Is All You Need' paper"""148 149 def __init__(150 self,151 embed_dim: int,152 num_heads: int,153 dropout: float = 0.0,154 is_decoder: bool = False,155 bias: bool = True,156 is_causal: bool = False,157 config: Optional[BlenderbotConfig] = None,158 layer_idx: Optional[int] = None,159 ):160 super().__init__()161 self.embed_dim = embed_dim162 self.num_heads = num_heads163 self.dropout = dropout164 self.head_dim = embed_dim // num_heads165 self.config = config166 167 if (self.head_dim * num_heads) != self.embed_dim:168 raise ValueError(169 f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}"170 f" and `num_heads`: {num_heads})."171 )172 self.scaling = self.head_dim**-0.5173 self.is_decoder = is_decoder174 self.is_causal = is_causal175 self.layer_idx = layer_idx176 if layer_idx is None and self.is_decoder:177 logger.warning_once(178 f"Instantiating a decoder {self.__class__.__name__} without passing `layer_idx` is not recommended and "179 "will lead to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` "180 "when creating this class."181 )182 183 self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias)184 self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias)185 self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias)186 self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)187 188 @deprecate_kwarg("past_key_value", new_name="past_key_values", version="4.58")189 def forward(190 self,191 hidden_states: torch.Tensor,192 key_value_states: Optional[torch.Tensor] = None,193 past_key_values: Optional[Cache] = None,194 attention_mask: Optional[torch.Tensor] = None,195 layer_head_mask: Optional[torch.Tensor] = None,196 output_attentions: bool = False,197 cache_position: Optional[torch.Tensor] = None,198 # TODO: we need a refactor so that the different attention modules can get their specific kwargs199 # ATM, we have mixed things encoder, decoder, and encoder-decoder attn200 **kwargs: Unpack[FlashAttentionKwargs],201 ) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[tuple[torch.Tensor]]]:202 """Input shape: Batch x Time x Channel"""203 204 # if key_value_states are provided this layer is used as a cross-attention layer205 # for the decoder206 is_cross_attention = key_value_states is not None207 208 # determine input shapes209 bsz, tgt_len = hidden_states.shape[:-1]210 src_len = key_value_states.shape[1] if is_cross_attention else tgt_len211 212 q_input_shape = (bsz, tgt_len, -1, self.head_dim)213 kv_input_shape = (bsz, src_len, -1, self.head_dim)214 215 # get query proj216 query_states = self.q_proj(hidden_states).view(*q_input_shape).transpose(1, 2)217 218 is_updated = False219 if past_key_values is not None:220 if isinstance(past_key_values, EncoderDecoderCache):221 is_updated = past_key_values.is_updated.get(self.layer_idx)222 if is_cross_attention:223 # after the first generated id, we can subsequently re-use all key/value_states from cache224 curr_past_key_value = past_key_values.cross_attention_cache225 else:226 curr_past_key_value = past_key_values.self_attention_cache227 else:228 curr_past_key_value = past_key_values229 230 current_states = key_value_states if is_cross_attention else hidden_states231 if is_cross_attention and past_key_values is not None and is_updated:232 # reuse k,v, cross_attentions233 key_states = curr_past_key_value.layers[self.layer_idx].keys234 value_states = curr_past_key_value.layers[self.layer_idx].values235 else:236 key_states = self.k_proj(current_states)237 value_states = self.v_proj(current_states)238 key_states = key_states.view(*kv_input_shape).transpose(1, 2)239 value_states = value_states.view(*kv_input_shape).transpose(1, 2)240 241 if past_key_values is not None:242 # save all key/value_states to cache to be re-used for fast auto-regressive generation243 cache_position = cache_position if not is_cross_attention else None244 key_states, value_states = curr_past_key_value.update(245 key_states, value_states, self.layer_idx, {"cache_position": cache_position}246 )247 # set flag that curr layer for cross-attn is already updated so we can re-use in subsequent calls248 if is_cross_attention and isinstance(past_key_values, EncoderDecoderCache):249 past_key_values.is_updated[self.layer_idx] = True250 251 attention_interface: Callable = eager_attention_forward252 if self.config._attn_implementation != "eager":253 attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]254 255 attn_output, attn_weights = attention_interface(256 self,257 query_states,258 key_states,259 value_states,260 attention_mask,261 dropout=0.0 if not self.training else self.dropout,262 scaling=self.scaling,263 output_attentions=output_attentions,264 head_mask=layer_head_mask,265 **kwargs,266 )267 268 attn_output = attn_output.reshape(bsz, tgt_len, -1).contiguous()269 attn_output = self.out_proj(attn_output)270 271 return attn_output, attn_weights272 273 274# Copied from transformers.models.mbart.modeling_mbart.MBartEncoderLayer with MBart->Blenderbot, MBART->BLENDERBOT275class BlenderbotEncoderLayer(GradientCheckpointingLayer):276 def __init__(self, config: BlenderbotConfig):277 super().__init__()278 self.embed_dim = config.d_model279 280 self.self_attn = BlenderbotAttention(281 embed_dim=self.embed_dim,282 num_heads=config.encoder_attention_heads,283 dropout=config.attention_dropout,284 config=config,285 )286 self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)287 self.dropout = config.dropout288 self.activation_fn = ACT2FN[config.activation_function]289 self.activation_dropout = config.activation_dropout290 self.fc1 = nn.Linear(self.embed_dim, config.encoder_ffn_dim)291 self.fc2 = nn.Linear(config.encoder_ffn_dim, self.embed_dim)292 self.final_layer_norm = nn.LayerNorm(self.embed_dim)293 294 def forward(295 self,296 hidden_states: torch.Tensor,297 attention_mask: torch.Tensor,298 layer_head_mask: torch.Tensor,299 output_attentions: bool = False,300 ) -> torch.Tensor:301 """302 Args:303 hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`304 attention_mask (`torch.FloatTensor`): attention mask of size305 `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.306 layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size307 `(encoder_attention_heads,)`.308 output_attentions (`bool`, *optional*):309 Whether or not to return the attentions tensors of all attention layers. See `attentions` under310 returned tensors for more detail.311 """312 residual = hidden_states313 hidden_states = self.self_attn_layer_norm(hidden_states)314 hidden_states, attn_weights = self.self_attn(315 hidden_states=hidden_states,316 attention_mask=attention_mask,317 layer_head_mask=layer_head_mask,318 output_attentions=output_attentions,319 )320 hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)321 hidden_states = residual + hidden_states322 323 residual = hidden_states324 hidden_states = self.final_layer_norm(hidden_states)325 hidden_states = self.activation_fn(self.fc1(hidden_states))326 hidden_states = nn.functional.dropout(hidden_states, p=self.activation_dropout, training=self.training)327 hidden_states = self.fc2(hidden_states)328 hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)329 hidden_states = residual + hidden_states330 331 if hidden_states.dtype == torch.float16:332 clamp_value = torch.finfo(hidden_states.dtype).max - 1000333 hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value)334 335 return hidden_states, attn_weights336 337 338# Copied from transformers.models.mbart.modeling_mbart.MBartDecoderLayer with MBart->Blenderbot, MBART->BLENDERBOT339class BlenderbotDecoderLayer(GradientCheckpointingLayer):340 def __init__(self, config: BlenderbotConfig, layer_idx: Optional[int] = None):341 super().__init__()342 self.embed_dim = config.d_model343 344 self.self_attn = BlenderbotAttention(345 embed_dim=self.embed_dim,346 num_heads=config.decoder_attention_heads,347 dropout=config.attention_dropout,348 is_decoder=True,349 is_causal=True,350 config=config,351 layer_idx=layer_idx,352 )353 self.dropout = config.dropout354 self.activation_fn = ACT2FN[config.activation_function]355 self.activation_dropout = config.activation_dropout356 357 self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)358 self.encoder_attn = BlenderbotAttention(359 self.embed_dim,360 config.decoder_attention_heads,361 dropout=config.attention_dropout,362 is_decoder=True,363 config=config,364 layer_idx=layer_idx,365 )366 self.encoder_attn_layer_norm = nn.LayerNorm(self.embed_dim)367 self.fc1 = nn.Linear(self.embed_dim, config.decoder_ffn_dim)368 self.fc2 = nn.Linear(config.decoder_ffn_dim, self.embed_dim)369 self.final_layer_norm = nn.LayerNorm(self.embed_dim)370 371 @deprecate_kwarg("past_key_value", new_name="past_key_values", version="4.58")372 def forward(373 self,374 hidden_states: torch.Tensor,375 attention_mask: Optional[torch.Tensor] = None,376 encoder_hidden_states: Optional[torch.Tensor] = None,377 encoder_attention_mask: Optional[torch.Tensor] = None,378 layer_head_mask: Optional[torch.Tensor] = None,379 cross_attn_layer_head_mask: Optional[torch.Tensor] = None,380 past_key_values: Optional[Cache] = None,381 output_attentions: Optional[bool] = False,382 use_cache: Optional[bool] = True,383 cache_position: Optional[torch.Tensor] = None,384 ) -> torch.Tensor:385 """386 Args:387 hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`388 attention_mask (`torch.FloatTensor`): attention mask of size389 `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.390 encoder_hidden_states (`torch.FloatTensor`):391 cross attention input to the layer of shape `(batch, seq_len, embed_dim)`392 encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size393 `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.394 layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size395 `(encoder_attention_heads,)`.396 cross_attn_layer_head_mask (`torch.FloatTensor`): mask for cross-attention heads in a given layer of397 size `(decoder_attention_heads,)`.398 past_key_values (`Cache`): cached past key and value projection states399 output_attentions (`bool`, *optional*):400 Whether or not to return the attentions tensors of all attention layers. See `attentions` under401 returned tensors for more detail.402 cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):403 Indices depicting the position of the input sequence tokens in the sequence. It is used to update the404 cache in the correct position and to infer the complete sequence length.405 """406 residual = hidden_states407 hidden_states = self.self_attn_layer_norm(hidden_states)408 409 # Self Attention410 hidden_states, self_attn_weights = self.self_attn(411 hidden_states=hidden_states,412 past_key_values=past_key_values,413 attention_mask=attention_mask,414 layer_head_mask=layer_head_mask,415 output_attentions=output_attentions,416 cache_position=cache_position,417 )418 hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)419 hidden_states = residual + hidden_states420 421 # Cross-Attention Block422 cross_attn_weights = None423 if encoder_hidden_states is not None:424 residual = hidden_states425 hidden_states = self.encoder_attn_layer_norm(hidden_states)426 427 hidden_states, cross_attn_weights = self.encoder_attn(428 hidden_states=hidden_states,429 key_value_states=encoder_hidden_states,430 attention_mask=encoder_attention_mask,431 layer_head_mask=cross_attn_layer_head_mask,432 past_key_values=past_key_values,433 output_attentions=output_attentions,434 )435 hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)436 hidden_states = residual + hidden_states437 438 # Fully Connected439 residual = hidden_states440 hidden_states = self.final_layer_norm(hidden_states)441 hidden_states = self.activation_fn(self.fc1(hidden_states))442 hidden_states = nn.functional.dropout(hidden_states, p=self.activation_dropout, training=self.training)443 hidden_states = self.fc2(hidden_states)444 hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)445 hidden_states = residual + hidden_states446 447 outputs = (hidden_states,)448 449 if output_attentions:450 outputs += (self_attn_weights, cross_attn_weights)451 452 return outputs453 454 455@auto_docstring456class BlenderbotPreTrainedModel(PreTrainedModel):457 config: BlenderbotConfig458 base_model_prefix = "model"459 supports_gradient_checkpointing = True460 _supports_flash_attn = True461 _supports_sdpa = True462 _supports_flex_attn = True463 464 _can_compile_fullgraph = True465 466 def _init_weights(self, module):467 std = self.config.init_std468 if isinstance(module, nn.Linear):469 module.weight.data.normal_(mean=0.0, std=std)470 if module.bias is not None:471 module.bias.data.zero_()472 elif isinstance(module, nn.Embedding):473 module.weight.data.normal_(mean=0.0, std=std)474 if module.padding_idx is not None:475 module.weight.data[module.padding_idx].zero_()476 elif isinstance(module, nn.LayerNorm):477 module.weight.data.fill_(1.0)478 module.bias.data.zero_()479 480 @property481 def dummy_inputs(self):482 pad_token = self.config.pad_token_id483 input_ids = torch.tensor([[0, 6, 10, 4, 2], [0, 8, 12, 2, pad_token]], device=self.device)484 dummy_inputs = {485 "attention_mask": input_ids.ne(pad_token),486 "input_ids": input_ids,487 "decoder_input_ids": input_ids,488 }489 return dummy_inputs490 491 # Copied from transformers.models.bart.modeling_bart.BartPreTrainedModel._update_full_mask492 def _update_full_mask(493 self,494 attention_mask: Union[torch.Tensor, None],495 inputs_embeds: torch.Tensor,496 ):497 if attention_mask is not None:498 if self.config._attn_implementation == "flash_attention_2":499 attention_mask = attention_mask if 0 in attention_mask else None500 elif self.config._attn_implementation == "sdpa":501 # output_attentions=True & head_mask can not be supported when using SDPA, fall back to502 # the manual implementation that requires a 4D causal mask in all cases.503 # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]504 attention_mask = _prepare_4d_attention_mask_for_sdpa(attention_mask, inputs_embeds.dtype)505 elif self.config._attn_implementation == "flex_attention":506 if isinstance(attention_mask, torch.Tensor):507 attention_mask = make_flex_block_causal_mask(attention_mask, is_causal=False)508 else:509 # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]510 attention_mask = _prepare_4d_attention_mask(attention_mask, inputs_embeds.dtype)511 512 return attention_mask513 514 # Copied from transformers.models.bart.modeling_bart.BartPreTrainedModel._update_causal_mask515 def _update_causal_mask(516 self,517 attention_mask: Optional[Union[torch.Tensor, "BlockMask"]],518 input_tensor: torch.Tensor,519 cache_position: torch.Tensor,520 past_key_values: Cache,521 ):522 if self.config._attn_implementation == "flex_attention":523 if isinstance(attention_mask, torch.Tensor):524 attention_mask = make_flex_block_causal_mask(attention_mask)525 # Other attention flavors support in-built causal (when `mask is None`)526 # while we need to create our specific block mask regardless527 elif attention_mask is None:528 attention_mask = make_flex_block_causal_mask(529 torch.ones(530 size=(input_tensor.shape[0], input_tensor.shape[1]),531 device=attention_mask.device,532 )533 )534 return attention_mask535 536 if self.config._attn_implementation == "flash_attention_2":537 if attention_mask is not None and (attention_mask == 0.0).any():538 return attention_mask539 return None540 541 # For SDPA, when possible, we will rely on its `is_causal` argument instead of its `attn_mask` argument, in542 # order to dispatch on Flash Attention 2. This feature is not compatible with static cache, as SDPA will fail543 # to infer the attention mask.544 past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0545 using_compilable_cache = past_key_values.is_compileable if past_key_values is not None else False546 547 # When output attentions is True, sdpa implementation's forward method calls the eager implementation's forward548 if self.config._attn_implementation == "sdpa" and not using_compilable_cache:549 if AttentionMaskConverter._ignore_causal_mask_sdpa(550 attention_mask,551 inputs_embeds=input_tensor,552 past_key_values_length=past_seen_tokens,553 is_training=self.training,554 ):555 return None556 557 dtype = input_tensor.dtype558 sequence_length = input_tensor.shape[1]559 if using_compilable_cache:560 target_length = past_key_values.get_max_cache_shape()561 else:562 target_length = (563 attention_mask.shape[-1]564 if isinstance(attention_mask, torch.Tensor)565 else past_seen_tokens + sequence_length + 1566 )567 568 # In case the provided `attention` mask is 2D, we generate a causal mask here (4D).569 causal_mask = self._prepare_4d_causal_attention_mask_with_cache_position(570 attention_mask,571 sequence_length=sequence_length,572 target_length=target_length,573 dtype=dtype,574 cache_position=cache_position,575 batch_size=input_tensor.shape[0],576 )577 578 if (579 self.config._attn_implementation == "sdpa"580 and attention_mask is not None581 and attention_mask.device.type in ["cuda", "xpu", "npu"]582 ):583 # Attend to all tokens in fully masked rows in the causal_mask, for example the relevant first rows when584 # using left padding. This is required by F.scaled_dot_product_attention memory-efficient attention path.585 # Details: https://github.com/pytorch/pytorch/issues/110213586 min_dtype = torch.finfo(dtype).min587 causal_mask = AttentionMaskConverter._unmask_unattended(causal_mask, min_dtype)588 589 return causal_mask590 591 @staticmethod592 # Copied from transformers.models.gptj.modeling_gptj.GPTJModel._prepare_4d_causal_attention_mask_with_cache_position593 def _prepare_4d_causal_attention_mask_with_cache_position(594 attention_mask: torch.Tensor,595 sequence_length: int,596 target_length: int,597 dtype: torch.dtype,598 cache_position: torch.Tensor,599 batch_size: int,600 **kwargs,601 ):602 """603 Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape604 `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.605 606 Args:607 attention_mask (`torch.Tensor`):608 A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape609 `(batch_size, 1, query_length, key_value_length)`.610 sequence_length (`int`):611 The sequence length being processed.612 target_length (`int`):613 The target length: when generating with static cache, the mask should be as long as the static cache,614 to account for the 0 padding, the part of the cache that is not filled yet.615 dtype (`torch.dtype`):616 The dtype to use for the 4D attention mask.617 cache_position (`torch.Tensor`):618 Indices depicting the position of the input sequence tokens in the sequence.619 batch_size (`torch.Tensor`):620 Batch size.621 """622 if attention_mask is not None and attention_mask.dim() == 4:623 # In this case we assume that the mask comes already in inverted form and requires no inversion or slicing.624 causal_mask = attention_mask625 else:626 min_dtype = torch.finfo(dtype).min627 causal_mask = torch.full(628 (sequence_length, target_length), fill_value=min_dtype, dtype=dtype, device=cache_position.device629 )630 if sequence_length != 1:631 causal_mask = torch.triu(causal_mask, diagonal=1)632 causal_mask *= torch.arange(target_length, device=cache_position.device) > cache_position.reshape(-1, 1)633 causal_mask = causal_mask[None, None, :, :].expand(batch_size, 1, -1, -1)634 if attention_mask is not None:635 causal_mask = causal_mask.clone() # copy to contiguous memory for in-place edit636 mask_length = attention_mask.shape[-1]637 padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :].to(638 causal_mask.device639 )640 padding_mask = padding_mask == 0641 causal_mask[:, :, :, :mask_length] = causal_mask[:, :, :, :mask_length].masked_fill(642 padding_mask, min_dtype643 )644 645 return causal_mask646 647 # Copied from transformers.models.bart.modeling_bart.BartPreTrainedModel._update_cross_attn_mask648 def _update_cross_attn_mask(649 self,650 encoder_hidden_states: Union[torch.Tensor, None],651 encoder_attention_mask: Union[torch.Tensor, None],652 input_shape: torch.Size,653 inputs_embeds: torch.Tensor,654 ):655 # expand encoder attention mask656 if encoder_hidden_states is not None and encoder_attention_mask is not None:657 if self.config._attn_implementation == "flash_attention_2":658 encoder_attention_mask = encoder_attention_mask if 0 in encoder_attention_mask else None659 elif self.config._attn_implementation == "sdpa":660 # output_attentions=True & cross_attn_head_mask can not be supported when using SDPA, and we fall back on661 # the manual implementation that requires a 4D causal mask in all cases.662 # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]663 encoder_attention_mask = _prepare_4d_attention_mask_for_sdpa(664 encoder_attention_mask,665 inputs_embeds.dtype,666 tgt_len=input_shape[-1],667 )668 elif self.config._attn_implementation == "flex_attention":669 if isinstance(encoder_attention_mask, torch.Tensor):670 encoder_attention_mask = make_flex_block_causal_mask(671 encoder_attention_mask,672 query_length=input_shape[-1],673 is_causal=False,674 )675 else:676 # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]677 encoder_attention_mask = _prepare_4d_attention_mask(678 encoder_attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]679 )680 681 return encoder_attention_mask682 683 684class BlenderbotEncoder(BlenderbotPreTrainedModel):685 """686 Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a687 [`BlenderbotEncoderLayer`].688 689 Args:690 config: BlenderbotConfig691 embed_tokens (nn.Embedding): output embedding692 """693 694 def __init__(self, config: BlenderbotConfig, embed_tokens: Optional[nn.Embedding] = None):695 super().__init__(config)696 697 self.dropout = config.dropout698 self.layerdrop = config.encoder_layerdrop699 700 embed_dim = config.d_model701 self.padding_idx = config.pad_token_id702 self.max_source_positions = config.max_position_embeddings703 embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0704 705 if embed_tokens is not None:706 self.embed_tokens = embed_tokens707 else:708 self.embed_tokens = BlenderbotScaledWordEmbedding(709 config.vocab_size, embed_dim, self.padding_idx, embed_scale=embed_scale710 )711 712 self.embed_positions = BlenderbotLearnedPositionalEmbedding(713 config.max_position_embeddings,714 embed_dim,715 )716 self.layers = nn.ModuleList([BlenderbotEncoderLayer(config) for _ in range(config.encoder_layers)])717 self.layer_norm = nn.LayerNorm(config.d_model)718 719 self.gradient_checkpointing = False720 # Initialize weights and apply final processing721 self.post_init()722 723 def forward(724 self,725 input_ids=None,726 attention_mask=None,727 head_mask=None,728 inputs_embeds=None,729 output_attentions=None,730 output_hidden_states=None,731 return_dict=None,732 ):733 r"""734 Args:735 input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):736 Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you737 provide it.738 739 Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and740 [`PreTrainedTokenizer.__call__`] for details.741 742 [What are input IDs?](../glossary#input-ids)743 attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):744 Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:745 746 - 1 for tokens that are **not masked**,747 - 0 for tokens that are **masked**.748 749 [What are attention masks?](../glossary#attention-mask)750 head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):751 Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:752 753 - 1 indicates the head is **not masked**,754 - 0 indicates the head is **masked**.755 756 inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):757 Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.758 This is useful if you want more control over how to convert `input_ids` indices into associated vectors759 than the model's internal embedding lookup matrix.760 output_attentions (`bool`, *optional*):761 Whether or not to return the attentions tensors of all attention layers. See `attentions` under762 returned tensors for more detail.763 output_hidden_states (`bool`, *optional*):764 Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors765 for more detail.766 return_dict (`bool`, *optional*):767 Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.768 """769 output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions770 output_hidden_states = (771 output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states772 )773 return_dict = return_dict if return_dict is not None else self.config.use_return_dict774 775 # retrieve input_ids and inputs_embeds776 if input_ids is not None and inputs_embeds is not None:777 raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")778 elif input_ids is not None:779 self.warn_if_padding_and_no_attention_mask(input_ids, attention_mask)780 input_shape = input_ids.size()781 input_ids = input_ids.view(-1, input_shape[-1])782 elif inputs_embeds is not None:783 input_shape = inputs_embeds.size()[:-1]784 else:785 raise ValueError("You have to specify either input_ids or inputs_embeds")786 787 if inputs_embeds is None:788 inputs_embeds = self.embed_tokens(input_ids)789 790 embed_pos = self.embed_positions(input_shape)791 792 hidden_states = inputs_embeds + embed_pos793 hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)794 795 attention_mask = self._update_full_mask(796 attention_mask,797 inputs_embeds,798 )799 800 encoder_states = () if output_hidden_states else None801 all_attentions = () if output_attentions else None802 803 # check if head_mask has a correct number of layers specified if desired804 if head_mask is not None:805 if head_mask.size()[0] != len(self.layers):806 raise ValueError(807 f"The head_mask should be specified for {len(self.layers)} layers, but it is for"808 f" {head_mask.size()[0]}."809 )810 for idx, encoder_layer in enumerate(self.layers):811 if output_hidden_states:812 encoder_states = encoder_states + (hidden_states,)813 # add LayerDrop (see https://huggingface.co/papers/1909.11556 for description)814 to_drop = False815 if self.training:816 dropout_probability = torch.rand([])817 if dropout_probability < self.layerdrop: # skip the layer818 to_drop = True819 820 if to_drop:821 layer_outputs = (None, None)822 else:823 layer_outputs = encoder_layer(824 hidden_states,825 attention_mask,826 layer_head_mask=(head_mask[idx] if head_mask is not None else None),827 output_attentions=output_attentions,828 )829 830 hidden_states = layer_outputs[0]831 832 if output_attentions:833 all_attentions = all_attentions + (layer_outputs[1],)834 835 # add final layer norm836 hidden_states = self.layer_norm(hidden_states)837 838 if output_hidden_states:839 encoder_states = encoder_states + (hidden_states,)840 841 if not return_dict:842 return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)843 return BaseModelOutput(844 last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions845 )846 847 848class BlenderbotDecoder(BlenderbotPreTrainedModel):849 """850 Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`BlenderbotDecoderLayer`]851 852 Args:853 config: BlenderbotConfig854 embed_tokens (nn.Embedding): output embedding855 """856 857 def __init__(self, config: BlenderbotConfig, embed_tokens: Optional[nn.Embedding] = None):858 super().__init__(config)859 self.dropout = config.dropout860 self.layerdrop = config.decoder_layerdrop861 self.padding_idx = config.pad_token_id862 self.max_target_positions = config.max_position_embeddings863 embed_scale = math.sqrt(config.d_model) if config.scale_embedding else 1.0864 865 if embed_tokens is not None:866 self.embed_tokens = embed_tokens867 else:868 self.embed_tokens = BlenderbotScaledWordEmbedding(869 config.vocab_size, config.d_model, self.padding_idx, embed_scale=embed_scale870 )871 872 self.embed_positions = BlenderbotLearnedPositionalEmbedding(873 config.max_position_embeddings,874 config.d_model,875 )876 self.layers = nn.ModuleList(877 [BlenderbotDecoderLayer(config, layer_idx=i) for i in range(config.decoder_layers)]878 )879 self.layer_norm = nn.LayerNorm(config.d_model)880 881 self.gradient_checkpointing = False882 # Initialize weights and apply final processing883 self.post_init()884 885 def forward(886 self,887 input_ids=None,888 attention_mask=None,889 encoder_hidden_states=None,890 encoder_attention_mask=None,891 head_mask=None,892 cross_attn_head_mask=None,893 past_key_values=None,894 inputs_embeds=None,895 use_cache=None,896 output_attentions=None,897 output_hidden_states=None,898 return_dict=None,899 cache_position: Optional[torch.Tensor] = None,900 ):901 r"""902 Args:903 input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):904 Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you905 provide it.906 907 Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and908 [`PreTrainedTokenizer.__call__`] for details.909 910 [What are input IDs?](../glossary#input-ids)911 attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):912 Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:913 914 - 1 for tokens that are **not masked**,915 - 0 for tokens that are **masked**.916 917 [What are attention masks?](../glossary#attention-mask)918 encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):919 Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention920 of the decoder.921 encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*):922 Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values923 selected in `[0, 1]`:924 925 - 1 for tokens that are **not masked**,926 - 0 for tokens that are **masked**.927 928 [What are attention masks?](../glossary#attention-mask)929 head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):930 Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0,931 1]`:932 933 - 1 indicates the head is **not masked**,934 - 0 indicates the head is **masked**.935 936 cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):937 Mask to nullify selected heads of the cross-attention modules in the decoder to avoid performing938 cross-attention on hidden heads. Mask values selected in `[0, 1]`:939 940 - 1 indicates the head is **not masked**,941 - 0 indicates the head is **masked**.942 943 past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):944 It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).945 946 Contains pre-computed hidden-states (key and values in the self-attention blocks and in the947 cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.948 949 If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those950 that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of951 all `decoder_input_ids` of shape `(batch_size, sequence_length)`.952 inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):953 Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.954 This is useful if you want more control over how to convert `input_ids` indices into associated vectors955 than the model's internal embedding lookup matrix.956 output_attentions (`bool`, *optional*):957 Whether or not to return the attentions tensors of all attention layers. See `attentions` under958 returned tensors for more detail.959 output_hidden_states (`bool`, *optional*):960 Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors961 for more detail.962 return_dict (`bool`, *optional*):963 Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.964 cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):965 Indices depicting the position of the input sequence tokens in the sequence. It is used to update the966 cache in the correct position and to infer the complete sequence length.967 """968 output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions969 output_hidden_states = (970 output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states971 )972 use_cache = use_cache if use_cache is not None else self.config.use_cache973 return_dict = return_dict if return_dict is not None else self.config.use_return_dict974 975 # retrieve input_ids and inputs_embeds976 if (input_ids is None) ^ (inputs_embeds is not None):977 raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time")978 elif input_ids is not None:979 input = input_ids980 input_shape = input.shape981 input_ids = input_ids.view(-1, input_shape[-1])982 elif inputs_embeds is not None:983 input_shape = inputs_embeds.size()[:-1]984 input = inputs_embeds[:, :, -1]985 else:986 raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds")987 988 if inputs_embeds is None:989 inputs_embeds = self.embed_tokens(input)990 991 if self.gradient_checkpointing and self.training:992 if use_cache:993 logger.warning_once(994 "`use_cache=True` is incompatible with gradient checkpointing`. Setting `use_cache=False`..."995 )996 use_cache = False997 998 # initialize `past_key_values`999 if use_cache and past_key_values is None:1000 past_key_values = (1001 EncoderDecoderCache(DynamicCache(config=self.config), DynamicCache(config=self.config))1002 if encoder_hidden_states is not None1003 else DynamicCache(config=self.config)1004 )1005 if use_cache and isinstance(past_key_values, tuple):1006 logger.warning_once(1007 "Passing a tuple of `past_key_values` is deprecated and will be removed in Transformers v4.58.0. "1008 "You should pass an instance of `EncoderDecoderCache` instead, e.g. "1009 "`past_key_values=EncoderDecoderCache.from_legacy_cache(past_key_values)`."1010 )1011 past_key_values = EncoderDecoderCache.from_legacy_cache(past_key_values)1012 1013 batch_size, seq_length = inputs_embeds.size()[:-1]1014 past_key_values_length = past_key_values.get_seq_length() if past_key_values is not None else 01015 if cache_position is None:1016 cache_position = torch.arange(1017 past_key_values_length, past_key_values_length + seq_length, device=inputs_embeds.device1018 )1019 1020 if attention_mask is None and not is_torchdynamo_compiling():1021 # required mask seq length can be calculated via length of past cache1022 mask_seq_length = past_key_values_length + seq_length1023 attention_mask = torch.ones(batch_size, mask_seq_length, device=inputs_embeds.device)1024 1025 self_attn_cache = (1026 past_key_values.self_attention_cache1027 if isinstance(past_key_values, EncoderDecoderCache)1028 else past_key_values1029 )1030 1031 causal_mask = self._update_causal_mask(1032 attention_mask,1033 inputs_embeds,1034 cache_position,1035 self_attn_cache,1036 )1037 encoder_attention_mask = self._update_cross_attn_mask(1038 encoder_hidden_states,1039 encoder_attention_mask,1040 input_shape,1041 inputs_embeds,1042 )1043 1044 # embed positions1045 position_ids = self.embed_positions(1046 (batch_size, seq_length), past_key_values_length, position_ids=cache_position1047 )1048 1049 hidden_states = inputs_embeds + position_ids1050 hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)1051 1052 # decoder layers1053 all_hidden_states = () if output_hidden_states else None1054 all_self_attns = () if output_attentions else None1055 all_cross_attentions = () if (output_attentions and encoder_hidden_states is not None) else None1056 1057 # check if head_mask/cross_attn_head_mask has a correct number of layers specified if desired1058 for attn_mask, mask_name in zip([head_mask, cross_attn_head_mask], ["head_mask", "cross_attn_head_mask"]):1059 if attn_mask is not None:1060 if attn_mask.size()[0] != len(self.layers):1061 raise ValueError(1062 f"The `{mask_name}` should be specified for {len(self.layers)} layers, but it is for"1063 f" {head_mask.size()[0]}."1064 )1065 for idx, decoder_layer in enumerate(self.layers):1066 # add LayerDrop (see https://huggingface.co/papers/1909.11556 for description)1067 if output_hidden_states:1068 all_hidden_states += (hidden_states,)1069 if self.training:1070 dropout_probability = torch.rand([])1071 if dropout_probability < self.layerdrop:1072 continue1073 1074 layer_outputs = decoder_layer(1075 hidden_states,1076 causal_mask,1077 encoder_hidden_states, # as a positional argument for gradient checkpointing1078 encoder_attention_mask=encoder_attention_mask,1079 layer_head_mask=(head_mask[idx] if head_mask is not None else None),1080 cross_attn_layer_head_mask=(cross_attn_head_mask[idx] if cross_attn_head_mask is not None else None),1081 past_key_values=past_key_values,1082 output_attentions=output_attentions,1083 use_cache=use_cache,1084 cache_position=cache_position,1085 )1086 hidden_states = layer_outputs[0]1087 1088 if output_attentions:1089 all_self_attns += (layer_outputs[1],)1090 1091 if encoder_hidden_states is not None:1092 all_cross_attentions += (layer_outputs[2],)1093 1094 # add final layer norm1095 hidden_states = self.layer_norm(hidden_states)1096 1097 # add hidden states from the last decoder layer1098 if output_hidden_states:1099 all_hidden_states += (hidden_states,)1100 1101 if not return_dict:1102 return tuple(1103 v1104 for v in [hidden_states, past_key_values, all_hidden_states, all_self_attns, all_cross_attentions]1105 if v is not None1106 )1107 return BaseModelOutputWithPastAndCrossAttentions(1108 last_hidden_state=hidden_states,1109 past_key_values=past_key_values,1110 hidden_states=all_hidden_states,1111 attentions=all_self_attns,1112 cross_attentions=all_cross_attentions,1113 )1114 1115 1116@auto_docstring1117class BlenderbotModel(BlenderbotPreTrainedModel):1118 _tied_weights_keys = ["decoder.embed_tokens.weight", "encoder.embed_tokens.weight"]1119 1120 def __init__(self, config: BlenderbotConfig):1121 super().__init__(config)1122 1123 padding_idx, vocab_size = config.pad_token_id, config.vocab_size1124 embed_scale = math.sqrt(config.d_model) if config.scale_embedding else 1.01125 self.shared = BlenderbotScaledWordEmbedding(vocab_size, config.d_model, padding_idx, embed_scale=embed_scale)1126 self.encoder = BlenderbotEncoder(config, self.shared)1127 self.decoder = BlenderbotDecoder(config, self.shared)1128 1129 # Initialize weights and apply final processing1130 self.post_init()1131 1132 @classmethod1133 def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], *model_args, **kwargs):1134 if pretrained_model_name_or_path == "facebook/blenderbot-90M":1135 warnings.warn(1136 "The checkpoint `facebook/blenderbot-90M` is deprecated. In the future, please use the identical"1137 " checkpoint `facebook/small_blenderbot-90M` with"1138 " `BlenderbotSmallModel.from_pretrained('facebook/small_blenderbot-90M')` instead.",1139 FutureWarning,1140 )1141 return BlenderbotSmallModel.from_pretrained(pretrained_model_name_or_path)1142 1143 return super().from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)1144 1145 def get_input_embeddings(self):1146 return self.shared1147 1148 def set_input_embeddings(self, value):1149 self.shared = value1150 self.encoder.embed_tokens = self.shared1151 self.decoder.embed_tokens = self.shared1152 1153 def get_encoder(self):1154 return self.encoder1155 1156 @auto_docstring1157 def forward(1158 self,1159 input_ids: Optional[torch.LongTensor] = None,1160 attention_mask: Optional[torch.Tensor] = None,1161 decoder_input_ids: Optional[torch.LongTensor] = None,1162 decoder_attention_mask: Optional[torch.LongTensor] = None,1163 head_mask: Optional[torch.Tensor] = None,1164 decoder_head_mask: Optional[torch.Tensor] = None,1165 cross_attn_head_mask: Optional[torch.Tensor] = None,1166 encoder_outputs: Optional[Union[tuple, BaseModelOutput]] = None,1167 past_key_values: Optional[Cache] = None,1168 inputs_embeds: Optional[torch.Tensor] = None,1169 decoder_inputs_embeds: Optional[torch.FloatTensor] = None,1170 use_cache: Optional[bool] = None,1171 output_attentions: Optional[bool] = None,1172 output_hidden_states: Optional[bool] = None,1173 return_dict: Optional[bool] = None,1174 cache_position: Optional[torch.Tensor] = None,1175 ) -> Union[tuple[torch.FloatTensor], Seq2SeqModelOutput]:1176 r"""1177 decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):1178 Indices of decoder input sequence tokens in the vocabulary.1179 1180 Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and1181 [`PreTrainedTokenizer.__call__`] for details.1182 1183 [What are decoder input IDs?](../glossary#decoder-input-ids)1184 1185 Blenderbot uses the `bos_token_id` as the starting token for `decoder_input_ids` generation. If1186 `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see1187 `past_key_values`).1188 decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):1189 Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also1190 be used by default.1191 cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):1192 Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0,1193 1]`:1194 1195 - 1 indicates the head is **not masked**,1196 - 0 indicates the head is **masked**.1197 1198 Example:1199 1200 ```python