Aluode/PerceptionLabPortable
0
1# coding=utf-82# Copyright 2021 The Facebook, Inc and The HuggingFace Inc. team. All rights reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""TF 2.0 Blenderbot model."""16 17from __future__ import annotations18 19import os20import random21import warnings22 23import tensorflow as tf24 25from ...activations_tf import get_tf_activation26from ...modeling_tf_outputs import (27 TFBaseModelOutput,28 TFBaseModelOutputWithPastAndCrossAttentions,29 TFSeq2SeqLMOutput,30 TFSeq2SeqModelOutput,31)32 33# Public API34from ...modeling_tf_utils import (35 TFCausalLanguageModelingLoss,36 TFPreTrainedModel,37 keras,38 keras_serializable,39 unpack_inputs,40)41from ...tf_utils import check_embeddings_within_bounds, shape_list, stable_softmax42from ...utils import (43 add_code_sample_docstrings,44 add_end_docstrings,45 add_start_docstrings,46 add_start_docstrings_to_model_forward,47 logging,48 replace_return_docstrings,49)50from .configuration_blenderbot import BlenderbotConfig51 52 53logger = logging.get_logger(__name__)54 55_CHECKPOINT_FOR_DOC = "facebook/blenderbot-400M-distill"56_CONFIG_FOR_DOC = "BlenderbotConfig"57 58 59LARGE_NEGATIVE = -1e860 61 62# Copied from transformers.models.bart.modeling_tf_bart.shift_tokens_right63def shift_tokens_right(input_ids: tf.Tensor, pad_token_id: int, decoder_start_token_id: int):64 pad_token_id = tf.cast(pad_token_id, input_ids.dtype)65 decoder_start_token_id = tf.cast(decoder_start_token_id, input_ids.dtype)66 start_tokens = tf.fill(67 (shape_list(input_ids)[0], 1), tf.convert_to_tensor(decoder_start_token_id, input_ids.dtype)68 )69 shifted_input_ids = tf.concat([start_tokens, input_ids[:, :-1]], -1)70 # replace possible -100 values in labels by `pad_token_id`71 shifted_input_ids = tf.where(72 shifted_input_ids == -100,73 tf.fill(shape_list(shifted_input_ids), tf.convert_to_tensor(pad_token_id, input_ids.dtype)),74 shifted_input_ids,75 )76 77 # "Verify that `labels` has only positive values and -100"78 assert_gte0 = tf.debugging.assert_greater_equal(shifted_input_ids, tf.constant(0, dtype=input_ids.dtype))79 80 # Make sure the assertion op is called by wrapping the result in an identity no-op81 with tf.control_dependencies([assert_gte0]):82 shifted_input_ids = tf.identity(shifted_input_ids)83 84 return shifted_input_ids85 86 87# Copied from transformers.models.bart.modeling_tf_bart._make_causal_mask88def _make_causal_mask(input_ids_shape: tf.TensorShape, past_key_values_length: int = 0):89 """90 Make causal mask used for bi-directional self-attention.91 """92 bsz = input_ids_shape[0]93 tgt_len = input_ids_shape[1]94 mask = tf.ones((tgt_len, tgt_len)) * LARGE_NEGATIVE95 mask_cond = tf.range(shape_list(mask)[-1])96 97 mask = tf.where(mask_cond < tf.reshape(mask_cond + 1, (shape_list(mask)[-1], 1)), 0.0, mask)98 99 if past_key_values_length > 0:100 mask = tf.concat([tf.zeros((tgt_len, past_key_values_length)), mask], axis=-1)101 102 return tf.tile(mask[None, None, :, :], (bsz, 1, 1, 1))103 104 105# Copied from transformers.models.bart.modeling_tf_bart._expand_mask106def _expand_mask(mask: tf.Tensor, tgt_len: int | None = None):107 """108 Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`.109 """110 src_len = shape_list(mask)[1]111 tgt_len = tgt_len if tgt_len is not None else src_len112 one_cst = tf.constant(1.0)113 mask = tf.cast(mask, dtype=one_cst.dtype)114 expanded_mask = tf.tile(mask[:, None, None, :], (1, 1, tgt_len, 1))115 116 return (one_cst - expanded_mask) * LARGE_NEGATIVE117 118 119class TFBlenderbotLearnedPositionalEmbedding(keras.layers.Embedding):120 """121 This module learns positional embeddings up to a fixed maximum size.122 """123 124 def __init__(self, num_embeddings: int, embedding_dim: int, **kwargs):125 super().__init__(num_embeddings, embedding_dim, **kwargs)126 127 def call(128 self, input_shape: tf.TensorShape, past_key_values_length: int = 0, position_ids: tf.Tensor | None = None129 ):130 """Input is expected to be of size [bsz x seqlen]."""131 if position_ids is None:132 seq_len = input_shape[1]133 position_ids = tf.range(seq_len, delta=1, name="range")134 position_ids += past_key_values_length135 136 return super().call(tf.cast(position_ids, dtype=tf.int32))137 138 139# Copied from transformers.models.bart.modeling_tf_bart.TFBartAttention with Bart->Blenderbot140class TFBlenderbotAttention(keras.layers.Layer):141 """Multi-headed attention from "Attention Is All You Need"""142 143 def __init__(144 self,145 embed_dim: int,146 num_heads: int,147 dropout: float = 0.0,148 is_decoder: bool = False,149 bias: bool = True,150 **kwargs,151 ):152 super().__init__(**kwargs)153 self.embed_dim = embed_dim154 155 self.num_heads = num_heads156 self.dropout = keras.layers.Dropout(dropout)157 self.head_dim = embed_dim // num_heads158 if (self.head_dim * num_heads) != self.embed_dim:159 raise ValueError(160 f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}"161 f" and `num_heads`: {num_heads})."162 )163 self.scaling = self.head_dim**-0.5164 self.is_decoder = is_decoder165 166 self.k_proj = keras.layers.Dense(embed_dim, use_bias=bias, name="k_proj")167 self.q_proj = keras.layers.Dense(embed_dim, use_bias=bias, name="q_proj")168 self.v_proj = keras.layers.Dense(embed_dim, use_bias=bias, name="v_proj")169 self.out_proj = keras.layers.Dense(embed_dim, use_bias=bias, name="out_proj")170 171 def _shape(self, tensor: tf.Tensor, seq_len: int, bsz: int):172 return tf.transpose(tf.reshape(tensor, (bsz, seq_len, self.num_heads, self.head_dim)), (0, 2, 1, 3))173 174 def call(175 self,176 hidden_states: tf.Tensor,177 key_value_states: tf.Tensor | None = None,178 past_key_value: tuple[tuple[tf.Tensor]] | None = None,179 attention_mask: tf.Tensor | None = None,180 layer_head_mask: tf.Tensor | None = None,181 training: bool | None = False,182 ) -> tuple[tf.Tensor, tf.Tensor | None]:183 """Input shape: Batch x Time x Channel"""184 185 # if key_value_states are provided this layer is used as a cross-attention layer186 # for the decoder187 is_cross_attention = key_value_states is not None188 bsz, tgt_len, embed_dim = shape_list(hidden_states)189 190 # get query proj191 query_states = self.q_proj(hidden_states) * self.scaling192 # get key, value proj193 if is_cross_attention and past_key_value is not None:194 # reuse k,v, cross_attentions195 key_states = past_key_value[0]196 value_states = past_key_value[1]197 elif is_cross_attention:198 # cross_attentions199 key_states = self._shape(self.k_proj(key_value_states), -1, bsz)200 value_states = self._shape(self.v_proj(key_value_states), -1, bsz)201 elif past_key_value is not None:202 # reuse k, v, self_attention203 key_states = self._shape(self.k_proj(hidden_states), -1, bsz)204 value_states = self._shape(self.v_proj(hidden_states), -1, bsz)205 key_states = tf.concat([past_key_value[0], key_states], axis=2)206 value_states = tf.concat([past_key_value[1], value_states], axis=2)207 else:208 # self_attention209 key_states = self._shape(self.k_proj(hidden_states), -1, bsz)210 value_states = self._shape(self.v_proj(hidden_states), -1, bsz)211 212 if self.is_decoder:213 # if cross_attention save Tuple(tf.Tensor, tf.Tensor) of all cross attention key/value_states.214 # Further calls to cross_attention layer can then reuse all cross-attention215 # key/value_states (first "if" case)216 # if uni-directional self-attention (decoder) save Tuple(tf.Tensor, tf.Tensor) of217 # all previous decoder key/value_states. Further calls to uni-directional self-attention218 # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case)219 # if encoder bi-directional self-attention `past_key_value` is always `None`220 past_key_value = (key_states, value_states)221 222 proj_shape = (bsz * self.num_heads, -1, self.head_dim)223 query_states = tf.reshape(self._shape(query_states, tgt_len, bsz), proj_shape)224 key_states = tf.reshape(key_states, proj_shape)225 value_states = tf.reshape(value_states, proj_shape)226 227 src_len = shape_list(key_states)[1]228 attn_weights = tf.matmul(query_states, key_states, transpose_b=True)229 230 tf.debugging.assert_equal(231 shape_list(attn_weights),232 [bsz * self.num_heads, tgt_len, src_len],233 message=(234 f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is"235 f" {shape_list(attn_weights)}"236 ),237 )238 239 if attention_mask is not None:240 tf.debugging.assert_equal(241 shape_list(attention_mask),242 [bsz, 1, tgt_len, src_len],243 message=(244 f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is"245 f" {shape_list(attention_mask)}"246 ),247 )248 249 attention_mask = tf.cast(attention_mask, dtype=attn_weights.dtype)250 attn_weights = tf.reshape(attn_weights, (bsz, self.num_heads, tgt_len, src_len)) + attention_mask251 attn_weights = tf.reshape(attn_weights, (bsz * self.num_heads, tgt_len, src_len))252 253 attn_weights = stable_softmax(attn_weights, axis=-1)254 255 if layer_head_mask is not None:256 tf.debugging.assert_equal(257 shape_list(layer_head_mask),258 [self.num_heads],259 message=(260 f"Head mask for a single layer should be of size {(self.num_heads)}, but is"261 f" {shape_list(layer_head_mask)}"262 ),263 )264 265 attn_weights = tf.reshape(layer_head_mask, (1, -1, 1, 1)) * tf.reshape(266 attn_weights, (bsz, self.num_heads, tgt_len, src_len)267 )268 attn_weights = tf.reshape(attn_weights, (bsz * self.num_heads, tgt_len, src_len))269 270 attn_probs = self.dropout(attn_weights, training=training)271 attn_output = tf.matmul(attn_probs, value_states)272 273 tf.debugging.assert_equal(274 shape_list(attn_output),275 [bsz * self.num_heads, tgt_len, self.head_dim],276 message=(277 f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is"278 f" {shape_list(attn_output)}"279 ),280 )281 282 attn_output = tf.transpose(283 tf.reshape(attn_output, (bsz, self.num_heads, tgt_len, self.head_dim)), (0, 2, 1, 3)284 )285 attn_output = tf.reshape(attn_output, (bsz, tgt_len, embed_dim))286 287 attn_output = self.out_proj(attn_output)288 attn_weights: tf.Tensor = tf.reshape(attn_weights, (bsz, self.num_heads, tgt_len, src_len))289 290 return attn_output, attn_weights, past_key_value291 292 def build(self, input_shape=None):293 if self.built:294 return295 self.built = True296 if getattr(self, "k_proj", None) is not None:297 with tf.name_scope(self.k_proj.name):298 self.k_proj.build([None, None, self.embed_dim])299 if getattr(self, "q_proj", None) is not None:300 with tf.name_scope(self.q_proj.name):301 self.q_proj.build([None, None, self.embed_dim])302 if getattr(self, "v_proj", None) is not None:303 with tf.name_scope(self.v_proj.name):304 self.v_proj.build([None, None, self.embed_dim])305 if getattr(self, "out_proj", None) is not None:306 with tf.name_scope(self.out_proj.name):307 self.out_proj.build([None, None, self.embed_dim])308 309 310# Copied from transformers.models.mbart.modeling_tf_mbart.TFMBartEncoderLayer with MBart->Blenderbot311class TFBlenderbotEncoderLayer(keras.layers.Layer):312 def __init__(self, config: BlenderbotConfig, **kwargs):313 super().__init__(**kwargs)314 self.embed_dim = config.d_model315 self.self_attn = TFBlenderbotAttention(316 self.embed_dim, config.encoder_attention_heads, dropout=config.attention_dropout, name="self_attn"317 )318 self.self_attn_layer_norm = keras.layers.LayerNormalization(epsilon=1e-5, name="self_attn_layer_norm")319 self.dropout = keras.layers.Dropout(config.dropout)320 self.activation_fn = get_tf_activation(config.activation_function)321 self.activation_dropout = keras.layers.Dropout(config.activation_dropout)322 self.fc1 = keras.layers.Dense(config.encoder_ffn_dim, name="fc1")323 self.fc2 = keras.layers.Dense(self.embed_dim, name="fc2")324 self.final_layer_norm = keras.layers.LayerNormalization(epsilon=1e-5, name="final_layer_norm")325 self.config = config326 327 def call(328 self,329 hidden_states: tf.Tensor,330 attention_mask: tf.Tensor,331 layer_head_mask: tf.Tensor,332 training: bool | None = False,333 ):334 """335 Args:336 hidden_states (`tf.Tensor`): input to the layer of shape *(batch, seq_len, embed_dim)*337 attention_mask (`tf.Tensor`): attention mask of size338 *(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.339 layer_head_mask (`tf.Tensor`): mask for attention heads in a given layer of size340 *(encoder_attention_heads,)*341 """342 residual = hidden_states343 hidden_states = self.self_attn_layer_norm(hidden_states)344 hidden_states, self_attn_weights, _ = self.self_attn(345 hidden_states=hidden_states, attention_mask=attention_mask, layer_head_mask=layer_head_mask346 )347 348 tf.debugging.assert_equal(349 shape_list(hidden_states),350 shape_list(residual),351 message=f"Self attn modified the shape of query {shape_list(residual)} to {shape_list(hidden_states)}",352 )353 354 hidden_states = self.dropout(hidden_states, training=training)355 hidden_states = residual + hidden_states356 357 residual = hidden_states358 hidden_states = self.final_layer_norm(hidden_states)359 hidden_states = self.activation_fn(self.fc1(hidden_states))360 hidden_states = self.activation_dropout(hidden_states, training=training)361 hidden_states = self.fc2(hidden_states)362 hidden_states = self.dropout(hidden_states, training=training)363 hidden_states = residual + hidden_states364 365 return hidden_states, self_attn_weights366 367 def build(self, input_shape=None):368 if self.built:369 return370 self.built = True371 if getattr(self, "self_attn", None) is not None:372 with tf.name_scope(self.self_attn.name):373 self.self_attn.build(None)374 if getattr(self, "self_attn_layer_norm", None) is not None:375 with tf.name_scope(self.self_attn_layer_norm.name):376 self.self_attn_layer_norm.build([None, None, self.embed_dim])377 if getattr(self, "fc1", None) is not None:378 with tf.name_scope(self.fc1.name):379 self.fc1.build([None, None, self.embed_dim])380 if getattr(self, "fc2", None) is not None:381 with tf.name_scope(self.fc2.name):382 self.fc2.build([None, None, self.config.encoder_ffn_dim])383 if getattr(self, "final_layer_norm", None) is not None:384 with tf.name_scope(self.final_layer_norm.name):385 self.final_layer_norm.build([None, None, self.embed_dim])386 387 388# Copied from transformers.models.mbart.modeling_tf_mbart.TFMBartDecoderLayer with MBart->Blenderbot389class TFBlenderbotDecoderLayer(keras.layers.Layer):390 def __init__(self, config: BlenderbotConfig, **kwargs):391 super().__init__(**kwargs)392 self.embed_dim = config.d_model393 self.self_attn = TFBlenderbotAttention(394 embed_dim=self.embed_dim,395 num_heads=config.decoder_attention_heads,396 dropout=config.attention_dropout,397 name="self_attn",398 is_decoder=True,399 )400 self.dropout = keras.layers.Dropout(config.dropout)401 self.activation_fn = get_tf_activation(config.activation_function)402 self.activation_dropout = keras.layers.Dropout(config.activation_dropout)403 404 self.self_attn_layer_norm = keras.layers.LayerNormalization(epsilon=1e-5, name="self_attn_layer_norm")405 self.encoder_attn = TFBlenderbotAttention(406 self.embed_dim,407 config.decoder_attention_heads,408 dropout=config.attention_dropout,409 name="encoder_attn",410 is_decoder=True,411 )412 self.encoder_attn_layer_norm = keras.layers.LayerNormalization(epsilon=1e-5, name="encoder_attn_layer_norm")413 self.fc1 = keras.layers.Dense(config.decoder_ffn_dim, name="fc1")414 self.fc2 = keras.layers.Dense(self.embed_dim, name="fc2")415 self.final_layer_norm = keras.layers.LayerNormalization(epsilon=1e-5, name="final_layer_norm")416 self.config = config417 418 def call(419 self,420 hidden_states: tf.Tensor,421 attention_mask: tf.Tensor | None = None,422 encoder_hidden_states: tf.Tensor | None = None,423 encoder_attention_mask: tf.Tensor | None = None,424 layer_head_mask: tf.Tensor | None = None,425 cross_attn_layer_head_mask: tf.Tensor | None = None,426 past_key_value: tuple[tf.Tensor] | None = None,427 training: bool | None = False,428 ) -> tuple[tf.Tensor, tf.Tensor, tuple[tuple[tf.Tensor]]]:429 """430 Args:431 hidden_states (`tf.Tensor`): input to the layer of shape *(batch, seq_len, embed_dim)*432 attention_mask (`tf.Tensor`): attention mask of size433 *(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.434 encoder_hidden_states (`tf.Tensor`):435 cross attention input to the layer of shape *(batch, seq_len, embed_dim)*436 encoder_attention_mask (`tf.Tensor`): encoder attention mask of size437 *(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.438 layer_head_mask (`tf.Tensor`): mask for attention heads in a given layer of size439 *(decoder_attention_heads,)*440 cross_attn_layer_head_mask (`tf.Tensor`): mask for heads of the cross-attention module.441 *(decoder_attention_heads,)*442 past_key_value (`Tuple(tf.Tensor)`): cached past key and value projection states443 """444 residual = hidden_states445 hidden_states = self.self_attn_layer_norm(hidden_states)446 447 # Self Attention448 # decoder uni-directional self-attention cached key/values tuple is at positions 1,2449 self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None450 # add present self-attn cache to positions 1,2 of present_key_value tuple451 hidden_states, self_attn_weights, present_key_value = self.self_attn(452 hidden_states=hidden_states,453 past_key_value=self_attn_past_key_value,454 attention_mask=attention_mask,455 layer_head_mask=layer_head_mask,456 )457 hidden_states = self.dropout(hidden_states, training=training)458 hidden_states = residual + hidden_states459 460 # Cross-Attention Block461 cross_attn_present_key_value = None462 cross_attn_weights = None463 if encoder_hidden_states is not None:464 residual = hidden_states465 hidden_states = self.encoder_attn_layer_norm(hidden_states)466 467 # cross_attn cached key/values tuple is at positions 3,4 of present_key_value tuple468 cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None469 hidden_states, cross_attn_weights, cross_attn_present_key_value = self.encoder_attn(470 hidden_states=hidden_states,471 key_value_states=encoder_hidden_states,472 attention_mask=encoder_attention_mask,473 layer_head_mask=cross_attn_layer_head_mask,474 past_key_value=cross_attn_past_key_value,475 )476 hidden_states = self.dropout(hidden_states, training=training)477 hidden_states = residual + hidden_states478 479 # add cross-attn to positions 3,4 of present_key_value tuple480 present_key_value = present_key_value + cross_attn_present_key_value481 482 # Fully Connected483 residual = hidden_states484 hidden_states = self.final_layer_norm(hidden_states)485 hidden_states = self.activation_fn(self.fc1(hidden_states))486 hidden_states = self.activation_dropout(hidden_states, training=training)487 hidden_states = self.fc2(hidden_states)488 hidden_states = self.dropout(hidden_states, training=training)489 hidden_states = residual + hidden_states490 491 return (492 hidden_states,493 self_attn_weights,494 cross_attn_weights,495 present_key_value,496 )497 498 def build(self, input_shape=None):499 if self.built:500 return501 self.built = True502 if getattr(self, "self_attn", None) is not None:503 with tf.name_scope(self.self_attn.name):504 self.self_attn.build(None)505 if getattr(self, "self_attn_layer_norm", None) is not None:506 with tf.name_scope(self.self_attn_layer_norm.name):507 self.self_attn_layer_norm.build([None, None, self.embed_dim])508 if getattr(self, "encoder_attn", None) is not None:509 with tf.name_scope(self.encoder_attn.name):510 self.encoder_attn.build(None)511 if getattr(self, "encoder_attn_layer_norm", None) is not None:512 with tf.name_scope(self.encoder_attn_layer_norm.name):513 self.encoder_attn_layer_norm.build([None, None, self.embed_dim])514 if getattr(self, "fc1", None) is not None:515 with tf.name_scope(self.fc1.name):516 self.fc1.build([None, None, self.embed_dim])517 if getattr(self, "fc2", None) is not None:518 with tf.name_scope(self.fc2.name):519 self.fc2.build([None, None, self.config.decoder_ffn_dim])520 if getattr(self, "final_layer_norm", None) is not None:521 with tf.name_scope(self.final_layer_norm.name):522 self.final_layer_norm.build([None, None, self.embed_dim])523 524 525class TFBlenderbotPreTrainedModel(TFPreTrainedModel):526 config_class = BlenderbotConfig527 base_model_prefix = "model"528 529 530BLENDERBOT_START_DOCSTRING = r"""531 This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the532 library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads533 etc.)534 535 This model is also a [keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it536 as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and537 behavior.538 539 <Tip>540 541 TensorFlow models and layers in `transformers` accept two formats as input:542 543 - having all inputs as keyword arguments (like PyTorch models), or544 - having all inputs as a list, tuple or dict in the first positional argument.545 546 The reason the second format is supported is that Keras methods prefer this format when passing inputs to models547 and layers. Because of this support, when using methods like `model.fit()` things should "just work" for you - just548 pass your inputs and labels in any format that `model.fit()` supports! If, however, you want to use the second549 format outside of Keras methods like `fit()` and `predict()`, such as when creating your own layers or models with550 the Keras `Functional` API, there are three possibilities you can use to gather all the input Tensors in the first551 positional argument:552 553 - a single Tensor with `input_ids` only and nothing else: `model(input_ids)`554 - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:555 `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])`556 - a dictionary with one or several input Tensors associated to the input names given in the docstring:557 `model({"input_ids": input_ids, "token_type_ids": token_type_ids})`558 559 Note that when creating models and layers with560 [subclassing](https://keras.io/guides/making_new_layers_and_models_via_subclassing/) then you don't need to worry561 about any of this, as you can just pass inputs like you would to any other Python function!562 563 </Tip>564 565 Args:566 config ([`BlenderbotConfig`]): Model configuration class with all the parameters of the model.567 Initializing with a config file does not load the weights associated with the model, only the568 configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights.569"""570 571BLENDERBOT_GENERATION_EXAMPLE = r"""572 Conversation example::573 574 ```py575 >>> from transformers import AutoTokenizer, TFBlenderbotForConditionalGeneration576 577 >>> mname = "facebook/blenderbot-400M-distill"578 >>> model = TFBlenderbotForConditionalGeneration.from_pretrained(mname)579 >>> tokenizer = AutoTokenizer.from_pretrained(mname)580 >>> UTTERANCE = "My friends are cool but they eat too many carbs."581 >>> print("Human: ", UTTERANCE)582 583 >>> inputs = tokenizer([UTTERANCE], return_tensors="tf")584 >>> reply_ids = model.generate(**inputs)585 >>> print("Bot: ", tokenizer.batch_decode(reply_ids, skip_special_tokens=True)[0])586 587 >>> REPLY = "I'm not sure"588 >>> print("Human: ", REPLY)589 >>> NEXT_UTTERANCE = (590 ... "My friends are cool but they eat too many carbs.</s> <s>That's unfortunate. "591 ... "Are they trying to lose weight or are they just trying to be healthier?</s> "592 ... "<s> I'm not sure."593 ... )594 >>> inputs = tokenizer([NEXT_UTTERANCE], return_tensors="tf")595 >>> next_reply_ids = model.generate(**inputs)596 >>> print("Bot: ", tokenizer.batch_decode(next_reply_ids, skip_special_tokens=True)[0])597 ```598"""599 600BLENDERBOT_INPUTS_DOCSTRING = r"""601 Args:602 input_ids (`tf.Tensor` of shape `({0})`):603 Indices of input sequence tokens in the vocabulary.604 605 Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and606 [`PreTrainedTokenizer.__call__`] for details.607 608 [What are input IDs?](../glossary#input-ids)609 attention_mask (`tf.Tensor` of shape `({0})`, *optional*):610 Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:611 612 - 1 for tokens that are **not masked**,613 - 0 for tokens that are **masked**.614 615 [What are attention masks?](../glossary#attention-mask)616 decoder_input_ids (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*):617 Indices of decoder input sequence tokens in the vocabulary.618 619 Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and620 [`PreTrainedTokenizer.__call__`] for details.621 622 [What are decoder input IDs?](../glossary#decoder-input-ids)623 624 Blenderbot uses the `bos_token_id` as the starting token for `decoder_input_ids` generation. If625 `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see626 `past_key_values`).627 decoder_attention_mask (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*):628 will be made by default and ignore pad tokens. It is not recommended to set this for most use cases.629 decoder_position_ids (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):630 Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the631 range `[0, config.max_position_embeddings - 1]`.632 head_mask (`tf.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):633 Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`:634 635 - 1 indicates the head is **not masked**,636 - 0 indicates the head is **masked**.637 638 decoder_head_mask (`tf.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):639 Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`:640 641 - 1 indicates the head is **not masked**,642 - 0 indicates the head is **masked**.643 644 cross_attn_head_mask (`tf.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):645 Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`:646 647 - 1 indicates the head is **not masked**,648 - 0 indicates the head is **masked**.649 650 encoder_outputs (`tf.FloatTensor`, *optional*):651 hidden states at the output of the last layer of the encoder. Used in the cross-attention of the decoder.652 of shape `(batch_size, sequence_length, hidden_size)` is a sequence of653 past_key_values (`tuple[tuple[tf.Tensor]]` of length `config.n_layers`)654 contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.655 If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that656 don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all657 `decoder_input_ids` of shape `(batch_size, sequence_length)`.658 use_cache (`bool`, *optional*, defaults to `True`):659 If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see660 `past_key_values`). Set to `False` during training, `True` during generation661 output_attentions (`bool`, *optional*):662 Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned663 tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the664 config will be used instead.665 output_hidden_states (`bool`, *optional*):666 Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for667 more detail. This argument can be used only in eager mode, in graph mode the value in the config will be668 used instead.669 return_dict (`bool`, *optional*):670 Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. This argument can be used in671 eager mode, in graph mode the value will always be set to True.672 training (`bool`, *optional*, defaults to `False`):673 Whether or not to use the model in training mode (some modules like dropout modules have different674 behaviors between training and evaluation).675"""676 677 678@keras_serializable679class TFBlenderbotEncoder(keras.layers.Layer):680 config_class = BlenderbotConfig681 """682 Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a683 [`TFBlenderbotEncoderLayer`].684 685 Args:686 config: BlenderbotConfig687 """688 689 def __init__(self, config: BlenderbotConfig, embed_tokens: keras.layers.Embedding | None = None, **kwargs):690 super().__init__(**kwargs)691 self.config = config692 self.dropout = keras.layers.Dropout(config.dropout)693 self.layerdrop = config.encoder_layerdrop694 self.padding_idx = config.pad_token_id695 self.max_source_positions = config.max_position_embeddings696 self.embed_scale = tf.math.sqrt(float(config.d_model)) if config.scale_embedding else 1.0697 698 self.embed_tokens = embed_tokens699 self.embed_positions = TFBlenderbotLearnedPositionalEmbedding(700 config.max_position_embeddings,701 config.d_model,702 name="embed_positions",703 )704 self.layers = [TFBlenderbotEncoderLayer(config, name=f"layers.{i}") for i in range(config.encoder_layers)]705 self.layer_norm = keras.layers.LayerNormalization(epsilon=1e-5, name="layer_norm")706 707 def get_embed_tokens(self):708 return self.embed_tokens709 710 def set_embed_tokens(self, embed_tokens):711 self.embed_tokens = embed_tokens712 713 @unpack_inputs714 def call(715 self,716 input_ids=None,717 inputs_embeds=None,718 attention_mask=None,719 head_mask=None,720 output_attentions=None,721 output_hidden_states=None,722 return_dict=None,723 training=False,724 ):725 """726 Args:727 input_ids (`tf.Tensor` of shape `(batch_size, sequence_length)`):728 Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you729 provide it.730 731 Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and732 [`PreTrainedTokenizer.__call__`] for details.733 734 [What are input IDs?](../glossary#input-ids)735 attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):736 Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:737 738 - 1 for tokens that are **not masked**,739 - 0 for tokens that are **masked**.740 741 [What are attention masks?](../glossary#attention-mask)742 head_mask (`tf.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, `optional):743 Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:744 745 - 1 indicates the head is **not masked**,746 - 0 indicates the head is **masked**.747 748 inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):749 Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.750 This is useful if you want more control over how to convert `input_ids` indices into associated vectors751 than the model's internal embedding lookup matrix.752 output_attentions (`bool`, *optional*):753 Whether or not to return the attentions tensors of all attention layers. See `attentions` under754 returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value755 in the config will be used instead.756 output_hidden_states (`bool`, *optional*):757 Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors758 for more detail. This argument can be used only in eager mode, in graph mode the value in the config759 will be used instead.760 return_dict (`bool`, *optional*):761 Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. This argument can be used762 in eager mode, in graph mode the value will always be set to True.763 training (`bool`, *optional*, defaults to `False`):764 Whether or not to use the model in training mode (some modules like dropout modules have different765 behaviors between training and evaluation).766 """767 if input_ids is not None and inputs_embeds is not None:768 raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")769 elif input_ids is not None:770 input_shape = shape_list(input_ids)771 elif inputs_embeds is not None:772 input_shape = shape_list(inputs_embeds)[:-1]773 else:774 raise ValueError("You have to specify either input_ids or inputs_embeds")775 776 if inputs_embeds is None:777 check_embeddings_within_bounds(input_ids, self.embed_tokens.input_dim)778 inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale779 780 embed_pos = self.embed_positions(input_shape)781 hidden_states = inputs_embeds + embed_pos782 hidden_states = self.dropout(hidden_states, training=training)783 784 # check attention mask and invert785 if attention_mask is not None:786 # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]787 attention_mask = _expand_mask(attention_mask)788 else:789 attention_mask = None790 791 encoder_states = () if output_hidden_states else None792 all_attentions = () if output_attentions else None793 794 # check if head_mask has a correct number of layers specified if desired795 if head_mask is not None:796 tf.debugging.assert_equal(797 shape_list(head_mask)[0],798 len(self.layers),799 message=(800 f"The head_mask should be specified for {len(self.layers)} layers, but it is for"801 f" {shape_list(head_mask)[0]}."802 ),803 )804 805 # encoder layers806 for idx, encoder_layer in enumerate(self.layers):807 if output_hidden_states:808 encoder_states = encoder_states + (hidden_states,)809 # add LayerDrop (see https://huggingface.co/papers/1909.11556 for description)810 dropout_probability = random.uniform(0, 1)811 if training and (dropout_probability < self.layerdrop): # skip the layer812 continue813 814 hidden_states, attn = encoder_layer(815 hidden_states,816 attention_mask,817 head_mask[idx] if head_mask is not None else None,818 )819 820 if output_attentions:821 all_attentions += (attn,)822 823 hidden_states = self.layer_norm(hidden_states)824 825 if output_hidden_states:826 encoder_states = encoder_states + (hidden_states,)827 828 if not return_dict:829 return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)830 return TFBaseModelOutput(831 last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions832 )833 834 def build(self, input_shape=None):835 if self.built:836 return837 self.built = True838 if getattr(self, "embed_positions", None) is not None:839 with tf.name_scope(self.embed_positions.name):840 self.embed_positions.build(None)841 if getattr(self, "layer_norm", None) is not None:842 with tf.name_scope(self.layer_norm.name):843 self.layer_norm.build([None, None, self.config.d_model])844 if getattr(self, "layers", None) is not None:845 for layer in self.layers:846 with tf.name_scope(layer.name):847 layer.build(None)848 849 850@keras_serializable851class TFBlenderbotDecoder(keras.layers.Layer):852 config_class = BlenderbotConfig853 """854 Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`TFBlenderbotDecoderLayer`]855 856 Args:857 config: BlenderbotConfig858 embed_tokens: output embedding859 """860 861 def __init__(self, config: BlenderbotConfig, embed_tokens: keras.layers.Embedding | None = None, **kwargs):862 super().__init__(**kwargs)863 self.config = config864 self.padding_idx = config.pad_token_id865 self.embed_tokens = embed_tokens866 self.layerdrop = config.decoder_layerdrop867 self.embed_positions = TFBlenderbotLearnedPositionalEmbedding(868 config.max_position_embeddings,869 config.d_model,870 name="embed_positions",871 )872 self.embed_scale = tf.math.sqrt(float(config.d_model)) if config.scale_embedding else 1.0873 self.layers = [TFBlenderbotDecoderLayer(config, name=f"layers.{i}") for i in range(config.decoder_layers)]874 self.layer_norm = keras.layers.LayerNormalization(epsilon=1e-5, name="layer_norm")875 876 self.dropout = keras.layers.Dropout(config.dropout)877 878 def get_embed_tokens(self):879 return self.embed_tokens880 881 def set_embed_tokens(self, embed_tokens):882 self.embed_tokens = embed_tokens883 884 @unpack_inputs885 def call(886 self,887 input_ids=None,888 inputs_embeds=None,889 attention_mask=None,890 position_ids=None,891 encoder_hidden_states=None,892 encoder_attention_mask=None,893 head_mask=None,894 cross_attn_head_mask=None,895 past_key_values=None,896 use_cache=None,897 output_attentions=None,898 output_hidden_states=None,899 return_dict=None,900 training=False,901 ):902 r"""903 Args:904 input_ids (`tf.Tensor` of shape `(batch_size, sequence_length)`):905 Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you906 provide it.907 908 Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and909 [`PreTrainedTokenizer.__call__`] for details.910 911 [What are input IDs?](../glossary#input-ids)912 attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):913 Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:914 915 - 1 for tokens that are **not masked**,916 - 0 for tokens that are **masked**.917 918 [What are attention masks?](../glossary#attention-mask)919 position_ids (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):920 Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the921 range `[0, config.max_position_embeddings - 1]`.922 encoder_hidden_states (`tf.Tensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):923 Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention924 of the decoder.925 encoder_attention_mask (`tf.Tensor` of shape `(batch_size, encoder_sequence_length)`, *optional*):926 Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values927 selected in `[0, 1]`:928 929 - 1 for tokens that are **not masked**,930 - 0 for tokens that are **masked**.931 932 [What are attention masks?](../glossary#attention-mask)933 head_mask (`tf.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):934 Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:935 936 - 1 indicates the head is **not masked**,937 - 0 indicates the head is **masked**.938 939 cross_attn_head_mask (`tf.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):940 Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`:941 942 - 1 indicates the head is **not masked**,943 - 0 indicates the head is **masked**.944 945 past_key_values (`tuple[tuple[tf.Tensor]]` of length `config.n_layers` with each tuple having 2 tuples each of which has 2 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):946 Contains precomputed key and value hidden-states of the attention blocks. Can be used to speed up947 decoding.948 949 If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those950 that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of951 all `decoder_input_ids` of shape `(batch_size, sequence_length)`.952 inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):953 Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.954 This is useful if you want more control over how to convert `input_ids` indices into associated vectors955 than the model's internal embedding lookup matrix.956 output_attentions (`bool`, *optional*):957 Whether or not to return the attentions tensors of all attention layers. See `attentions` under958 returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value959 in the config will be used instead.960 output_hidden_states (`bool`, *optional*):961 Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors962 for more detail. This argument can be used only in eager mode, in graph mode the value in the config963 will be used instead.964 return_dict (`bool`, *optional*):965 Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. This argument can be used966 in eager mode, in graph mode the value will always be set to True.967 training (`bool`, *optional*, defaults to `False`):968 Whether or not to use the model in training mode (some modules like dropout modules have different969 behaviors between training and evaluation).970 """971 if input_ids is not None and inputs_embeds is not None:972 raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time")973 elif input_ids is not None:974 input_shape = shape_list(input_ids)975 elif inputs_embeds is not None:976 input_shape = shape_list(inputs_embeds)[:-1]977 else:978 raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds")979 980 past_key_values_length = shape_list(past_key_values[0][0])[2] if past_key_values is not None else 0981 982 # embed positions983 if position_ids is None:984 positions = self.embed_positions(input_shape, past_key_values_length)985 else:986 positions = self.embed_positions(input_shape, position_ids=position_ids)987 988 if inputs_embeds is None:989 check_embeddings_within_bounds(input_ids, self.embed_tokens.input_dim)990 inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale991 992 hidden_states = inputs_embeds993 994 # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]995 if input_shape[-1] > 1:996 combined_attention_mask = _make_causal_mask(input_shape, past_key_values_length=past_key_values_length)997 else:998 combined_attention_mask = _expand_mask(999 tf.ones((input_shape[0], input_shape[1] + past_key_values_length)), tgt_len=input_shape[-1]1000 )1001 1002 if attention_mask is not None:1003 combined_attention_mask = combined_attention_mask + _expand_mask(attention_mask, tgt_len=input_shape[-1])1004 1005 if encoder_hidden_states is not None and encoder_attention_mask is not None:1006 # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]1007 encoder_attention_mask = _expand_mask(encoder_attention_mask, tgt_len=input_shape[-1])1008 1009 hidden_states = hidden_states + positions1010 hidden_states = self.dropout(hidden_states, training=training)1011 1012 # decoder layers1013 all_hidden_states = () if output_hidden_states else None1014 all_self_attns = () if output_attentions else None1015 all_cross_attns = () if (output_attentions and encoder_hidden_states is not None) else None1016 present_key_values = () if use_cache else None1017 1018 # check if head_mask and cross_attn_head_mask have a correct number of layers specified if desired1019 for attn_mask_name, attn_mask in [("head_mask", head_mask), ("cross_attn_head_mask", cross_attn_head_mask)]:1020 if attn_mask is not None:1021 tf.debugging.assert_equal(1022 shape_list(attn_mask)[0],1023 len(self.layers),1024 message=(1025 f"The {attn_mask_name} should be specified for {len(self.layers)} layers, but it is for"1026 f" {shape_list(attn_mask)[0]}."1027 ),1028 )1029 for idx, decoder_layer in enumerate(self.layers):1030 # add LayerDrop (see https://huggingface.co/papers/1909.11556 for description)1031 if output_hidden_states:1032 all_hidden_states += (hidden_states,)1033 dropout_probability = random.uniform(0, 1)1034 1035 if training and (dropout_probability < self.layerdrop):1036 continue1037 1038 past_key_value = past_key_values[idx] if past_key_values is not None else None1039 1040 hidden_states, layer_self_attn, layer_cross_attn, present_key_value = decoder_layer(1041 hidden_states,1042 attention_mask=combined_attention_mask,1043 encoder_hidden_states=encoder_hidden_states,1044 encoder_attention_mask=encoder_attention_mask,1045 layer_head_mask=head_mask[idx] if head_mask is not None else None,1046 cross_attn_layer_head_mask=cross_attn_head_mask[idx] if cross_attn_head_mask is not None else None,1047 past_key_value=past_key_value,1048 )1049 1050 if use_cache:1051 present_key_values += (present_key_value,)1052 1053 if output_attentions:1054 all_self_attns += (layer_self_attn,)1055 1056 if encoder_hidden_states is not None:1057 all_cross_attns += (layer_cross_attn,)1058 1059 hidden_states = self.layer_norm(hidden_states)1060 1061 if output_hidden_states:1062 all_hidden_states += (hidden_states,)1063 1064 if not return_dict:1065 return hidden_states, present_key_values, all_hidden_states, all_self_attns, all_cross_attns1066 else:1067 return TFBaseModelOutputWithPastAndCrossAttentions(1068 last_hidden_state=hidden_states,1069 past_key_values=present_key_values,1070 hidden_states=all_hidden_states,1071 attentions=all_self_attns,1072 cross_attentions=all_cross_attns,1073 )1074 1075 def build(self, input_shape=None):1076 if self.built:1077 return1078 self.built = True1079 if getattr(self, "embed_positions", None) is not None:1080 with tf.name_scope(self.embed_positions.name):1081 self.embed_positions.build(None)1082 if getattr(self, "layer_norm", None) is not None:1083 with tf.name_scope(self.layer_norm.name):1084 self.layer_norm.build([None, None, self.config.d_model])1085 if getattr(self, "layers", None) is not None:1086 for layer in self.layers:1087 with tf.name_scope(layer.name):1088 layer.build(None)1089 1090 1091@keras_serializable1092class TFBlenderbotMainLayer(keras.layers.Layer):1093 config_class = BlenderbotConfig1094 1095 def __init__(self, config: BlenderbotConfig, **kwargs):1096 super().__init__(**kwargs)1097 1098 self.config = config1099 self.shared = keras.layers.Embedding(1100 input_dim=config.vocab_size,1101 output_dim=config.d_model,1102 embeddings_initializer=keras.initializers.TruncatedNormal(stddev=self.config.init_std),1103 name="model.shared",1104 )1105 # Additional attribute to specify the expected name scope of the layer (for loading/storing weights)1106 self.shared.load_weight_prefix = "model.shared"1107 1108 self.encoder = TFBlenderbotEncoder(config, self.shared, name="encoder")1109 self.decoder = TFBlenderbotDecoder(config, self.shared, name="decoder")1110 1111 def get_input_embeddings(self):1112 return self.shared1113 1114 def set_input_embeddings(self, new_embeddings):1115 self.shared = new_embeddings1116 self.encoder.embed_tokens = self.shared1117 self.decoder.embed_tokens = self.shared1118 1119 @unpack_inputs1120 def call(1121 self,1122 input_ids=None,1123 attention_mask=None,1124 decoder_input_ids=None,1125 decoder_attention_mask=None,1126 decoder_position_ids=None,1127 head_mask=None,1128 decoder_head_mask=None,1129 cross_attn_head_mask=None,1130 encoder_outputs: tuple | TFBaseModelOutput | None = None,1131 past_key_values=None,1132 inputs_embeds=None,1133 decoder_inputs_embeds=None,1134 use_cache=None,1135 output_attentions=None,1136 output_hidden_states=None,1137 return_dict=None,1138 training=False,1139 **kwargs,1140 ):1141 output_hidden_states = (1142 output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states1143 )1144 1145 if encoder_outputs is None:1146 encoder_outputs = self.encoder(1147 input_ids=input_ids,1148 attention_mask=attention_mask,1149 head_mask=head_mask,1150 inputs_embeds=inputs_embeds,1151 output_attentions=output_attentions,1152 output_hidden_states=output_hidden_states,1153 return_dict=return_dict,1154 training=training,1155 )1156 # If the user passed a tuple for encoder_outputs, we wrap it in a TFBaseModelOutput when return_dict=True1157 elif return_dict and not isinstance(encoder_outputs, TFBaseModelOutput):1158 encoder_outputs = TFBaseModelOutput(1159 last_hidden_state=encoder_outputs[0],1160 hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None,1161 attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None,1162 )1163 # If the user passed a TFBaseModelOutput for encoder_outputs, we wrap it in a tuple when return_dict=False1164 elif not return_dict and not isinstance(encoder_outputs, tuple):1165 encoder_outputs = encoder_outputs.to_tuple()1166 1167 decoder_outputs = self.decoder(1168 decoder_input_ids,1169 attention_mask=decoder_attention_mask,1170 position_ids=decoder_position_ids,1171 encoder_hidden_states=encoder_outputs[0],1172 encoder_attention_mask=attention_mask,1173 head_mask=decoder_head_mask,1174 cross_attn_head_mask=cross_attn_head_mask,1175 past_key_values=past_key_values,1176 inputs_embeds=decoder_inputs_embeds,1177 use_cache=use_cache,1178 output_attentions=output_attentions,1179 output_hidden_states=output_hidden_states,1180 return_dict=return_dict,1181 training=training,1182 )1183 1184 if not return_dict:1185 return decoder_outputs + encoder_outputs1186 1187 return TFSeq2SeqModelOutput(1188 last_hidden_state=decoder_outputs.last_hidden_state,1189 past_key_values=decoder_outputs.past_key_values,1190 decoder_hidden_states=decoder_outputs.hidden_states,1191 decoder_attentions=decoder_outputs.attentions,1192 cross_attentions=decoder_outputs.cross_attentions,1193 encoder_last_hidden_state=encoder_outputs.last_hidden_state,1194 encoder_hidden_states=encoder_outputs.hidden_states,1195 encoder_attentions=encoder_outputs.attentions,1196 )1197 1198 def build(self, input_shape=None):1199 if self.built:1200 return