Aluode/PerceptionLabPortable
0
1# coding=utf-82# Copyright 2018 The OpenAI Team Authors and HuggingFace Inc. team.3# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.4#5# Licensed under the Apache License, Version 2.0 (the "License");6# you may not use this file except in compliance with the License.7# You may obtain a copy of the License at8#9# http://www.apache.org/licenses/LICENSE-2.010#11# Unless required by applicable law or agreed to in writing, software12# distributed under the License is distributed on an "AS IS" BASIS,13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.14# See the License for the specific language governing permissions and15# limitations under the License.16"""TF 2.0 ALBERT model."""17 18from __future__ import annotations19 20import math21from dataclasses import dataclass22 23import numpy as np24import tensorflow as tf25 26from ...activations_tf import get_tf_activation27from ...modeling_tf_outputs import (28 TFBaseModelOutput,29 TFBaseModelOutputWithPooling,30 TFMaskedLMOutput,31 TFMultipleChoiceModelOutput,32 TFQuestionAnsweringModelOutput,33 TFSequenceClassifierOutput,34 TFTokenClassifierOutput,35)36from ...modeling_tf_utils import (37 TFMaskedLanguageModelingLoss,38 TFModelInputType,39 TFMultipleChoiceLoss,40 TFPreTrainedModel,41 TFQuestionAnsweringLoss,42 TFSequenceClassificationLoss,43 TFTokenClassificationLoss,44 get_initializer,45 keras,46 keras_serializable,47 unpack_inputs,48)49from ...tf_utils import check_embeddings_within_bounds, shape_list, stable_softmax50from ...utils import (51 ModelOutput,52 add_code_sample_docstrings,53 add_start_docstrings,54 add_start_docstrings_to_model_forward,55 logging,56 replace_return_docstrings,57)58from .configuration_albert import AlbertConfig59 60 61logger = logging.get_logger(__name__)62 63_CHECKPOINT_FOR_DOC = "albert/albert-base-v2"64_CONFIG_FOR_DOC = "AlbertConfig"65 66 67class TFAlbertPreTrainingLoss:68 """69 Loss function suitable for ALBERT pretraining, that is, the task of pretraining a language model by combining SOP +70 MLM. .. note:: Any label of -100 will be ignored (along with the corresponding logits) in the loss computation.71 """72 73 def hf_compute_loss(self, labels: tf.Tensor, logits: tf.Tensor) -> tf.Tensor:74 loss_fn = keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction=keras.losses.Reduction.NONE)75 if self.config.tf_legacy_loss:76 # make sure only labels that are not equal to -10077 # are taken into account as loss78 masked_lm_active_loss = tf.not_equal(tf.reshape(tensor=labels["labels"], shape=(-1,)), -100)79 masked_lm_reduced_logits = tf.boolean_mask(80 tensor=tf.reshape(tensor=logits[0], shape=(-1, shape_list(logits[0])[2])),81 mask=masked_lm_active_loss,82 )83 masked_lm_labels = tf.boolean_mask(84 tensor=tf.reshape(tensor=labels["labels"], shape=(-1,)), mask=masked_lm_active_loss85 )86 sentence_order_active_loss = tf.not_equal(87 tf.reshape(tensor=labels["sentence_order_label"], shape=(-1,)), -10088 )89 sentence_order_reduced_logits = tf.boolean_mask(90 tensor=tf.reshape(tensor=logits[1], shape=(-1, 2)), mask=sentence_order_active_loss91 )92 sentence_order_label = tf.boolean_mask(93 tensor=tf.reshape(tensor=labels["sentence_order_label"], shape=(-1,)), mask=sentence_order_active_loss94 )95 masked_lm_loss = loss_fn(y_true=masked_lm_labels, y_pred=masked_lm_reduced_logits)96 sentence_order_loss = loss_fn(y_true=sentence_order_label, y_pred=sentence_order_reduced_logits)97 masked_lm_loss = tf.reshape(tensor=masked_lm_loss, shape=(-1, shape_list(sentence_order_loss)[0]))98 masked_lm_loss = tf.reduce_mean(input_tensor=masked_lm_loss, axis=0)99 100 return masked_lm_loss + sentence_order_loss101 102 # Clip negative labels to zero here to avoid NaNs and errors - those positions will get masked later anyway103 unmasked_lm_losses = loss_fn(y_true=tf.nn.relu(labels["labels"]), y_pred=logits[0])104 # make sure only labels that are not equal to -100105 # are taken into account for the loss computation106 lm_loss_mask = tf.cast(labels["labels"] != -100, dtype=unmasked_lm_losses.dtype)107 masked_lm_losses = unmasked_lm_losses * lm_loss_mask108 reduced_masked_lm_loss = tf.reduce_sum(masked_lm_losses) / tf.reduce_sum(lm_loss_mask)109 110 sop_logits = tf.reshape(logits[1], (-1, 2))111 # Clip negative labels to zero here to avoid NaNs and errors - those positions will get masked later anyway112 unmasked_sop_loss = loss_fn(y_true=tf.nn.relu(labels["sentence_order_label"]), y_pred=sop_logits)113 sop_loss_mask = tf.cast(labels["sentence_order_label"] != -100, dtype=unmasked_sop_loss.dtype)114 115 masked_sop_loss = unmasked_sop_loss * sop_loss_mask116 reduced_masked_sop_loss = tf.reduce_sum(masked_sop_loss) / tf.reduce_sum(sop_loss_mask)117 118 return tf.reshape(reduced_masked_lm_loss + reduced_masked_sop_loss, (1,))119 120 121class TFAlbertEmbeddings(keras.layers.Layer):122 """Construct the embeddings from word, position and token_type embeddings."""123 124 def __init__(self, config: AlbertConfig, **kwargs):125 super().__init__(**kwargs)126 127 self.config = config128 self.embedding_size = config.embedding_size129 self.max_position_embeddings = config.max_position_embeddings130 self.initializer_range = config.initializer_range131 self.LayerNorm = keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name="LayerNorm")132 self.dropout = keras.layers.Dropout(rate=config.hidden_dropout_prob)133 134 def build(self, input_shape=None):135 with tf.name_scope("word_embeddings"):136 self.weight = self.add_weight(137 name="weight",138 shape=[self.config.vocab_size, self.embedding_size],139 initializer=get_initializer(self.initializer_range),140 )141 142 with tf.name_scope("token_type_embeddings"):143 self.token_type_embeddings = self.add_weight(144 name="embeddings",145 shape=[self.config.type_vocab_size, self.embedding_size],146 initializer=get_initializer(self.initializer_range),147 )148 149 with tf.name_scope("position_embeddings"):150 self.position_embeddings = self.add_weight(151 name="embeddings",152 shape=[self.max_position_embeddings, self.embedding_size],153 initializer=get_initializer(self.initializer_range),154 )155 156 if self.built:157 return158 self.built = True159 if getattr(self, "LayerNorm", None) is not None:160 with tf.name_scope(self.LayerNorm.name):161 self.LayerNorm.build([None, None, self.config.embedding_size])162 163 # Copied from transformers.models.bert.modeling_tf_bert.TFBertEmbeddings.call164 def call(165 self,166 input_ids: tf.Tensor | None = None,167 position_ids: tf.Tensor | None = None,168 token_type_ids: tf.Tensor | None = None,169 inputs_embeds: tf.Tensor | None = None,170 past_key_values_length=0,171 training: bool = False,172 ) -> tf.Tensor:173 """174 Applies embedding based on inputs tensor.175 176 Returns:177 final_embeddings (`tf.Tensor`): output embedding tensor.178 """179 if input_ids is None and inputs_embeds is None:180 raise ValueError("Need to provide either `input_ids` or `input_embeds`.")181 182 if input_ids is not None:183 check_embeddings_within_bounds(input_ids, self.config.vocab_size)184 inputs_embeds = tf.gather(params=self.weight, indices=input_ids)185 186 input_shape = shape_list(inputs_embeds)[:-1]187 188 if token_type_ids is None:189 token_type_ids = tf.fill(dims=input_shape, value=0)190 191 if position_ids is None:192 position_ids = tf.expand_dims(193 tf.range(start=past_key_values_length, limit=input_shape[1] + past_key_values_length), axis=0194 )195 196 position_embeds = tf.gather(params=self.position_embeddings, indices=position_ids)197 token_type_embeds = tf.gather(params=self.token_type_embeddings, indices=token_type_ids)198 final_embeddings = inputs_embeds + position_embeds + token_type_embeds199 final_embeddings = self.LayerNorm(inputs=final_embeddings)200 final_embeddings = self.dropout(inputs=final_embeddings, training=training)201 202 return final_embeddings203 204 205class TFAlbertAttention(keras.layers.Layer):206 """Contains the complete attention sublayer, including both dropouts and layer norm."""207 208 def __init__(self, config: AlbertConfig, **kwargs):209 super().__init__(**kwargs)210 211 if config.hidden_size % config.num_attention_heads != 0:212 raise ValueError(213 f"The hidden size ({config.hidden_size}) is not a multiple of the number "214 f"of attention heads ({config.num_attention_heads})"215 )216 217 self.num_attention_heads = config.num_attention_heads218 self.attention_head_size = int(config.hidden_size / config.num_attention_heads)219 self.all_head_size = self.num_attention_heads * self.attention_head_size220 self.sqrt_att_head_size = math.sqrt(self.attention_head_size)221 self.output_attentions = config.output_attentions222 223 self.query = keras.layers.Dense(224 units=self.all_head_size, kernel_initializer=get_initializer(config.initializer_range), name="query"225 )226 self.key = keras.layers.Dense(227 units=self.all_head_size, kernel_initializer=get_initializer(config.initializer_range), name="key"228 )229 self.value = keras.layers.Dense(230 units=self.all_head_size, kernel_initializer=get_initializer(config.initializer_range), name="value"231 )232 self.dense = keras.layers.Dense(233 units=config.hidden_size, kernel_initializer=get_initializer(config.initializer_range), name="dense"234 )235 self.LayerNorm = keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name="LayerNorm")236 # Two different dropout probabilities; see https://github.com/google-research/albert/blob/master/modeling.py#L971-L993237 self.attention_dropout = keras.layers.Dropout(rate=config.attention_probs_dropout_prob)238 self.output_dropout = keras.layers.Dropout(rate=config.hidden_dropout_prob)239 self.config = config240 241 def transpose_for_scores(self, tensor: tf.Tensor, batch_size: int) -> tf.Tensor:242 # Reshape from [batch_size, seq_length, all_head_size] to [batch_size, seq_length, num_attention_heads, attention_head_size]243 tensor = tf.reshape(tensor=tensor, shape=(batch_size, -1, self.num_attention_heads, self.attention_head_size))244 245 # Transpose the tensor from [batch_size, seq_length, num_attention_heads, attention_head_size] to [batch_size, num_attention_heads, seq_length, attention_head_size]246 return tf.transpose(tensor, perm=[0, 2, 1, 3])247 248 def call(249 self,250 input_tensor: tf.Tensor,251 attention_mask: tf.Tensor,252 head_mask: tf.Tensor,253 output_attentions: bool,254 training: bool = False,255 ) -> tuple[tf.Tensor]:256 batch_size = shape_list(input_tensor)[0]257 mixed_query_layer = self.query(inputs=input_tensor)258 mixed_key_layer = self.key(inputs=input_tensor)259 mixed_value_layer = self.value(inputs=input_tensor)260 query_layer = self.transpose_for_scores(mixed_query_layer, batch_size)261 key_layer = self.transpose_for_scores(mixed_key_layer, batch_size)262 value_layer = self.transpose_for_scores(mixed_value_layer, batch_size)263 264 # Take the dot product between "query" and "key" to get the raw attention scores.265 # (batch size, num_heads, seq_len_q, seq_len_k)266 attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True)267 dk = tf.cast(self.sqrt_att_head_size, dtype=attention_scores.dtype)268 attention_scores = tf.divide(attention_scores, dk)269 270 if attention_mask is not None:271 # Apply the attention mask is (precomputed for all layers in TFAlbertModel call() function)272 attention_scores = tf.add(attention_scores, attention_mask)273 274 # Normalize the attention scores to probabilities.275 attention_probs = stable_softmax(logits=attention_scores, axis=-1)276 277 # This is actually dropping out entire tokens to attend to, which might278 # seem a bit unusual, but is taken from the original Transformer paper.279 attention_probs = self.attention_dropout(inputs=attention_probs, training=training)280 281 # Mask heads if we want to282 if head_mask is not None:283 attention_probs = tf.multiply(attention_probs, head_mask)284 285 context_layer = tf.matmul(attention_probs, value_layer)286 context_layer = tf.transpose(context_layer, perm=[0, 2, 1, 3])287 288 # (batch_size, seq_len_q, all_head_size)289 context_layer = tf.reshape(tensor=context_layer, shape=(batch_size, -1, self.all_head_size))290 self_outputs = (context_layer, attention_probs) if output_attentions else (context_layer,)291 hidden_states = self_outputs[0]292 hidden_states = self.dense(inputs=hidden_states)293 hidden_states = self.output_dropout(inputs=hidden_states, training=training)294 attention_output = self.LayerNorm(inputs=hidden_states + input_tensor)295 296 # add attentions if we output them297 outputs = (attention_output,) + self_outputs[1:]298 299 return outputs300 301 def build(self, input_shape=None):302 if self.built:303 return304 self.built = True305 if getattr(self, "query", None) is not None:306 with tf.name_scope(self.query.name):307 self.query.build([None, None, self.config.hidden_size])308 if getattr(self, "key", None) is not None:309 with tf.name_scope(self.key.name):310 self.key.build([None, None, self.config.hidden_size])311 if getattr(self, "value", None) is not None:312 with tf.name_scope(self.value.name):313 self.value.build([None, None, self.config.hidden_size])314 if getattr(self, "dense", None) is not None:315 with tf.name_scope(self.dense.name):316 self.dense.build([None, None, self.config.hidden_size])317 if getattr(self, "LayerNorm", None) is not None:318 with tf.name_scope(self.LayerNorm.name):319 self.LayerNorm.build([None, None, self.config.hidden_size])320 321 322class TFAlbertLayer(keras.layers.Layer):323 def __init__(self, config: AlbertConfig, **kwargs):324 super().__init__(**kwargs)325 326 self.attention = TFAlbertAttention(config, name="attention")327 self.ffn = keras.layers.Dense(328 units=config.intermediate_size, kernel_initializer=get_initializer(config.initializer_range), name="ffn"329 )330 331 if isinstance(config.hidden_act, str):332 self.activation = get_tf_activation(config.hidden_act)333 else:334 self.activation = config.hidden_act335 336 self.ffn_output = keras.layers.Dense(337 units=config.hidden_size, kernel_initializer=get_initializer(config.initializer_range), name="ffn_output"338 )339 self.full_layer_layer_norm = keras.layers.LayerNormalization(340 epsilon=config.layer_norm_eps, name="full_layer_layer_norm"341 )342 self.dropout = keras.layers.Dropout(rate=config.hidden_dropout_prob)343 self.config = config344 345 def call(346 self,347 hidden_states: tf.Tensor,348 attention_mask: tf.Tensor,349 head_mask: tf.Tensor,350 output_attentions: bool,351 training: bool = False,352 ) -> tuple[tf.Tensor]:353 attention_outputs = self.attention(354 input_tensor=hidden_states,355 attention_mask=attention_mask,356 head_mask=head_mask,357 output_attentions=output_attentions,358 training=training,359 )360 ffn_output = self.ffn(inputs=attention_outputs[0])361 ffn_output = self.activation(ffn_output)362 ffn_output = self.ffn_output(inputs=ffn_output)363 ffn_output = self.dropout(inputs=ffn_output, training=training)364 hidden_states = self.full_layer_layer_norm(inputs=ffn_output + attention_outputs[0])365 366 # add attentions if we output them367 outputs = (hidden_states,) + attention_outputs[1:]368 369 return outputs370 371 def build(self, input_shape=None):372 if self.built:373 return374 self.built = True375 if getattr(self, "attention", None) is not None:376 with tf.name_scope(self.attention.name):377 self.attention.build(None)378 if getattr(self, "ffn", None) is not None:379 with tf.name_scope(self.ffn.name):380 self.ffn.build([None, None, self.config.hidden_size])381 if getattr(self, "ffn_output", None) is not None:382 with tf.name_scope(self.ffn_output.name):383 self.ffn_output.build([None, None, self.config.intermediate_size])384 if getattr(self, "full_layer_layer_norm", None) is not None:385 with tf.name_scope(self.full_layer_layer_norm.name):386 self.full_layer_layer_norm.build([None, None, self.config.hidden_size])387 388 389class TFAlbertLayerGroup(keras.layers.Layer):390 def __init__(self, config: AlbertConfig, **kwargs):391 super().__init__(**kwargs)392 393 self.albert_layers = [394 TFAlbertLayer(config, name=f"albert_layers_._{i}") for i in range(config.inner_group_num)395 ]396 397 def call(398 self,399 hidden_states: tf.Tensor,400 attention_mask: tf.Tensor,401 head_mask: tf.Tensor,402 output_attentions: bool,403 output_hidden_states: bool,404 training: bool = False,405 ) -> TFBaseModelOutput | tuple[tf.Tensor]:406 layer_hidden_states = () if output_hidden_states else None407 layer_attentions = () if output_attentions else None408 409 for layer_index, albert_layer in enumerate(self.albert_layers):410 if output_hidden_states:411 layer_hidden_states = layer_hidden_states + (hidden_states,)412 413 layer_output = albert_layer(414 hidden_states=hidden_states,415 attention_mask=attention_mask,416 head_mask=head_mask[layer_index],417 output_attentions=output_attentions,418 training=training,419 )420 hidden_states = layer_output[0]421 422 if output_attentions:423 layer_attentions = layer_attentions + (layer_output[1],)424 425 # Add last layer426 if output_hidden_states:427 layer_hidden_states = layer_hidden_states + (hidden_states,)428 429 return tuple(v for v in [hidden_states, layer_hidden_states, layer_attentions] if v is not None)430 431 def build(self, input_shape=None):432 if self.built:433 return434 self.built = True435 if getattr(self, "albert_layers", None) is not None:436 for layer in self.albert_layers:437 with tf.name_scope(layer.name):438 layer.build(None)439 440 441class TFAlbertTransformer(keras.layers.Layer):442 def __init__(self, config: AlbertConfig, **kwargs):443 super().__init__(**kwargs)444 445 self.num_hidden_layers = config.num_hidden_layers446 self.num_hidden_groups = config.num_hidden_groups447 # Number of layers in a hidden group448 self.layers_per_group = int(config.num_hidden_layers / config.num_hidden_groups)449 self.embedding_hidden_mapping_in = keras.layers.Dense(450 units=config.hidden_size,451 kernel_initializer=get_initializer(config.initializer_range),452 name="embedding_hidden_mapping_in",453 )454 self.albert_layer_groups = [455 TFAlbertLayerGroup(config, name=f"albert_layer_groups_._{i}") for i in range(config.num_hidden_groups)456 ]457 self.config = config458 459 def call(460 self,461 hidden_states: tf.Tensor,462 attention_mask: tf.Tensor,463 head_mask: tf.Tensor,464 output_attentions: bool,465 output_hidden_states: bool,466 return_dict: bool,467 training: bool = False,468 ) -> TFBaseModelOutput | tuple[tf.Tensor]:469 hidden_states = self.embedding_hidden_mapping_in(inputs=hidden_states)470 all_attentions = () if output_attentions else None471 all_hidden_states = (hidden_states,) if output_hidden_states else None472 473 for i in range(self.num_hidden_layers):474 # Index of the hidden group475 group_idx = int(i / (self.num_hidden_layers / self.num_hidden_groups))476 layer_group_output = self.albert_layer_groups[group_idx](477 hidden_states=hidden_states,478 attention_mask=attention_mask,479 head_mask=head_mask[group_idx * self.layers_per_group : (group_idx + 1) * self.layers_per_group],480 output_attentions=output_attentions,481 output_hidden_states=output_hidden_states,482 training=training,483 )484 hidden_states = layer_group_output[0]485 486 if output_attentions:487 all_attentions = all_attentions + layer_group_output[-1]488 489 if output_hidden_states:490 all_hidden_states = all_hidden_states + (hidden_states,)491 492 if not return_dict:493 return tuple(v for v in [hidden_states, all_hidden_states, all_attentions] if v is not None)494 495 return TFBaseModelOutput(496 last_hidden_state=hidden_states, hidden_states=all_hidden_states, attentions=all_attentions497 )498 499 def build(self, input_shape=None):500 if self.built:501 return502 self.built = True503 if getattr(self, "embedding_hidden_mapping_in", None) is not None:504 with tf.name_scope(self.embedding_hidden_mapping_in.name):505 self.embedding_hidden_mapping_in.build([None, None, self.config.embedding_size])506 if getattr(self, "albert_layer_groups", None) is not None:507 for layer in self.albert_layer_groups:508 with tf.name_scope(layer.name):509 layer.build(None)510 511 512class TFAlbertPreTrainedModel(TFPreTrainedModel):513 """514 An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained515 models.516 """517 518 config_class = AlbertConfig519 base_model_prefix = "albert"520 521 522class TFAlbertMLMHead(keras.layers.Layer):523 def __init__(self, config: AlbertConfig, input_embeddings: keras.layers.Layer, **kwargs):524 super().__init__(**kwargs)525 526 self.config = config527 self.embedding_size = config.embedding_size528 self.dense = keras.layers.Dense(529 config.embedding_size, kernel_initializer=get_initializer(config.initializer_range), name="dense"530 )531 if isinstance(config.hidden_act, str):532 self.activation = get_tf_activation(config.hidden_act)533 else:534 self.activation = config.hidden_act535 536 self.LayerNorm = keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name="LayerNorm")537 538 # The output weights are the same as the input embeddings, but there is539 # an output-only bias for each token.540 self.decoder = input_embeddings541 542 def build(self, input_shape=None):543 self.bias = self.add_weight(shape=(self.config.vocab_size,), initializer="zeros", trainable=True, name="bias")544 self.decoder_bias = self.add_weight(545 shape=(self.config.vocab_size,), initializer="zeros", trainable=True, name="decoder/bias"546 )547 548 if self.built:549 return550 self.built = True551 if getattr(self, "dense", None) is not None:552 with tf.name_scope(self.dense.name):553 self.dense.build([None, None, self.config.hidden_size])554 if getattr(self, "LayerNorm", None) is not None:555 with tf.name_scope(self.LayerNorm.name):556 self.LayerNorm.build([None, None, self.config.embedding_size])557 558 def get_output_embeddings(self) -> keras.layers.Layer:559 return self.decoder560 561 def set_output_embeddings(self, value: tf.Variable):562 self.decoder.weight = value563 self.decoder.vocab_size = shape_list(value)[0]564 565 def get_bias(self) -> dict[str, tf.Variable]:566 return {"bias": self.bias, "decoder_bias": self.decoder_bias}567 568 def set_bias(self, value: tf.Variable):569 self.bias = value["bias"]570 self.decoder_bias = value["decoder_bias"]571 self.config.vocab_size = shape_list(value["bias"])[0]572 573 def call(self, hidden_states: tf.Tensor) -> tf.Tensor:574 hidden_states = self.dense(inputs=hidden_states)575 hidden_states = self.activation(hidden_states)576 hidden_states = self.LayerNorm(inputs=hidden_states)577 seq_length = shape_list(tensor=hidden_states)[1]578 hidden_states = tf.reshape(tensor=hidden_states, shape=[-1, self.embedding_size])579 hidden_states = tf.matmul(a=hidden_states, b=self.decoder.weight, transpose_b=True)580 hidden_states = tf.reshape(tensor=hidden_states, shape=[-1, seq_length, self.config.vocab_size])581 hidden_states = tf.nn.bias_add(value=hidden_states, bias=self.decoder_bias)582 583 return hidden_states584 585 586@keras_serializable587class TFAlbertMainLayer(keras.layers.Layer):588 config_class = AlbertConfig589 590 def __init__(self, config: AlbertConfig, add_pooling_layer: bool = True, **kwargs):591 super().__init__(**kwargs)592 593 self.config = config594 595 self.embeddings = TFAlbertEmbeddings(config, name="embeddings")596 self.encoder = TFAlbertTransformer(config, name="encoder")597 self.pooler = (598 keras.layers.Dense(599 units=config.hidden_size,600 kernel_initializer=get_initializer(config.initializer_range),601 activation="tanh",602 name="pooler",603 )604 if add_pooling_layer605 else None606 )607 608 def get_input_embeddings(self) -> keras.layers.Layer:609 return self.embeddings610 611 def set_input_embeddings(self, value: tf.Variable):612 self.embeddings.weight = value613 self.embeddings.vocab_size = shape_list(value)[0]614 615 def _prune_heads(self, heads_to_prune):616 """617 Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base618 class PreTrainedModel619 """620 raise NotImplementedError621 622 @unpack_inputs623 def call(624 self,625 input_ids: TFModelInputType | None = None,626 attention_mask: np.ndarray | tf.Tensor | None = None,627 token_type_ids: np.ndarray | tf.Tensor | None = None,628 position_ids: np.ndarray | tf.Tensor | None = None,629 head_mask: np.ndarray | tf.Tensor | None = None,630 inputs_embeds: np.ndarray | tf.Tensor | None = None,631 output_attentions: bool | None = None,632 output_hidden_states: bool | None = None,633 return_dict: bool | None = None,634 training: bool = False,635 ) -> TFBaseModelOutputWithPooling | tuple[tf.Tensor]:636 if input_ids is not None and inputs_embeds is not None:637 raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")638 elif input_ids is not None:639 input_shape = shape_list(input_ids)640 elif inputs_embeds is not None:641 input_shape = shape_list(inputs_embeds)[:-1]642 else:643 raise ValueError("You have to specify either input_ids or inputs_embeds")644 645 if attention_mask is None:646 attention_mask = tf.fill(dims=input_shape, value=1)647 648 if token_type_ids is None:649 token_type_ids = tf.fill(dims=input_shape, value=0)650 651 embedding_output = self.embeddings(652 input_ids=input_ids,653 position_ids=position_ids,654 token_type_ids=token_type_ids,655 inputs_embeds=inputs_embeds,656 training=training,657 )658 659 # We create a 3D attention mask from a 2D tensor mask.660 # Sizes are [batch_size, 1, 1, to_seq_length]661 # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length]662 # this attention mask is more simple than the triangular masking of causal attention663 # used in OpenAI GPT, we just need to prepare the broadcast dimension here.664 extended_attention_mask = tf.reshape(attention_mask, (input_shape[0], 1, 1, input_shape[1]))665 666 # Since attention_mask is 1.0 for positions we want to attend and 0.0 for667 # masked positions, this operation will create a tensor which is 0.0 for668 # positions we want to attend and -10000.0 for masked positions.669 # Since we are adding it to the raw scores before the softmax, this is670 # effectively the same as removing these entirely.671 extended_attention_mask = tf.cast(extended_attention_mask, dtype=embedding_output.dtype)672 one_cst = tf.constant(1.0, dtype=embedding_output.dtype)673 ten_thousand_cst = tf.constant(-10000.0, dtype=embedding_output.dtype)674 extended_attention_mask = tf.multiply(tf.subtract(one_cst, extended_attention_mask), ten_thousand_cst)675 676 # Prepare head mask if needed677 # 1.0 in head_mask indicate we keep the head678 # attention_probs has shape bsz x n_heads x N x N679 # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]680 # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]681 if head_mask is not None:682 raise NotImplementedError683 else:684 head_mask = [None] * self.config.num_hidden_layers685 686 encoder_outputs = self.encoder(687 hidden_states=embedding_output,688 attention_mask=extended_attention_mask,689 head_mask=head_mask,690 output_attentions=output_attentions,691 output_hidden_states=output_hidden_states,692 return_dict=return_dict,693 training=training,694 )695 696 sequence_output = encoder_outputs[0]697 pooled_output = self.pooler(inputs=sequence_output[:, 0]) if self.pooler is not None else None698 699 if not return_dict:700 return (701 sequence_output,702 pooled_output,703 ) + encoder_outputs[1:]704 705 return TFBaseModelOutputWithPooling(706 last_hidden_state=sequence_output,707 pooler_output=pooled_output,708 hidden_states=encoder_outputs.hidden_states,709 attentions=encoder_outputs.attentions,710 )711 712 def build(self, input_shape=None):713 if self.built:714 return715 self.built = True716 if getattr(self, "embeddings", None) is not None:717 with tf.name_scope(self.embeddings.name):718 self.embeddings.build(None)719 if getattr(self, "encoder", None) is not None:720 with tf.name_scope(self.encoder.name):721 self.encoder.build(None)722 if getattr(self, "pooler", None) is not None:723 with tf.name_scope(self.pooler.name):724 self.pooler.build([None, None, self.config.hidden_size])725 726 727@dataclass728class TFAlbertForPreTrainingOutput(ModelOutput):729 """730 Output type of [`TFAlbertForPreTraining`].731 732 Args:733 prediction_logits (`tf.Tensor` of shape `(batch_size, sequence_length, config.vocab_size)`):734 Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).735 sop_logits (`tf.Tensor` of shape `(batch_size, 2)`):736 Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation737 before SoftMax).738 hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):739 Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape740 `(batch_size, sequence_length, hidden_size)`.741 742 Hidden-states of the model at the output of each layer plus the initial embedding outputs.743 attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):744 Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,745 sequence_length)`.746 747 Attentions weights after the attention softmax, used to compute the weighted average in the self-attention748 heads.749 """750 751 loss: tf.Tensor | None = None752 prediction_logits: tf.Tensor | None = None753 sop_logits: tf.Tensor | None = None754 hidden_states: tuple[tf.Tensor] | None = None755 attentions: tuple[tf.Tensor] | None = None756 757 758ALBERT_START_DOCSTRING = r"""759 760 This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the761 library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads762 etc.)763 764 This model is also a [keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it765 as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and766 behavior.767 768 <Tip>769 770 TensorFlow models and layers in `transformers` accept two formats as input:771 772 - having all inputs as keyword arguments (like PyTorch models), or773 - having all inputs as a list, tuple or dict in the first positional argument.774 775 The reason the second format is supported is that Keras methods prefer this format when passing inputs to models776 and layers. Because of this support, when using methods like `model.fit()` things should "just work" for you - just777 pass your inputs and labels in any format that `model.fit()` supports! If, however, you want to use the second778 format outside of Keras methods like `fit()` and `predict()`, such as when creating your own layers or models with779 the Keras `Functional` API, there are three possibilities you can use to gather all the input Tensors in the first780 positional argument:781 782 - a single Tensor with `input_ids` only and nothing else: `model(input_ids)`783 - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:784 `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])`785 - a dictionary with one or several input Tensors associated to the input names given in the docstring:786 `model({"input_ids": input_ids, "token_type_ids": token_type_ids})`787 788 Note that when creating models and layers with789 [subclassing](https://keras.io/guides/making_new_layers_and_models_via_subclassing/) then you don't need to worry790 about any of this, as you can just pass inputs like you would to any other Python function!791 792 </Tip>793 794 Args:795 config ([`AlbertConfig`]): Model configuration class with all the parameters of the model.796 Initializing with a config file does not load the weights associated with the model, only the797 configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.798"""799 800ALBERT_INPUTS_DOCSTRING = r"""801 Args:802 input_ids (`Numpy array` or `tf.Tensor` of shape `({0})`):803 Indices of input sequence tokens in the vocabulary.804 805 Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and806 [`PreTrainedTokenizer.encode`] for details.807 808 [What are input IDs?](../glossary#input-ids)809 attention_mask (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):810 Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:811 812 - 1 for tokens that are **not masked**,813 - 0 for tokens that are **masked**.814 815 [What are attention masks?](../glossary#attention-mask)816 token_type_ids (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):817 Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,818 1]`:819 820 - 0 corresponds to a *sentence A* token,821 - 1 corresponds to a *sentence B* token.822 823 [What are token type IDs?](../glossary#token-type-ids)824 position_ids (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):825 Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,826 config.max_position_embeddings - 1]`.827 828 [What are position IDs?](../glossary#position-ids)829 head_mask (`Numpy array` or `tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):830 Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:831 832 - 1 indicates the head is **not masked**,833 - 0 indicates the head is **masked**.834 835 inputs_embeds (`tf.Tensor` of shape `({0}, hidden_size)`, *optional*):836 Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This837 is useful if you want more control over how to convert `input_ids` indices into associated vectors than the838 model's internal embedding lookup matrix.839 output_attentions (`bool`, *optional*):840 Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned841 tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the842 config will be used instead.843 output_hidden_states (`bool`, *optional*):844 Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for845 more detail. This argument can be used only in eager mode, in graph mode the value in the config will be846 used instead.847 return_dict (`bool`, *optional*):848 Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. This argument can be used in849 eager mode, in graph mode the value will always be set to True.850 training (`bool`, *optional*, defaults to `False`):851 Whether or not to use the model in training mode (some modules like dropout modules have different852 behaviors between training and evaluation).853"""854 855 856@add_start_docstrings(857 "The bare Albert Model transformer outputting raw hidden-states without any specific head on top.",858 ALBERT_START_DOCSTRING,859)860class TFAlbertModel(TFAlbertPreTrainedModel):861 def __init__(self, config: AlbertConfig, *inputs, **kwargs):862 super().__init__(config, *inputs, **kwargs)863 864 self.albert = TFAlbertMainLayer(config, name="albert")865 866 @unpack_inputs867 @add_start_docstrings_to_model_forward(ALBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length"))868 @add_code_sample_docstrings(869 checkpoint=_CHECKPOINT_FOR_DOC,870 output_type=TFBaseModelOutputWithPooling,871 config_class=_CONFIG_FOR_DOC,872 )873 def call(874 self,875 input_ids: TFModelInputType | None = None,876 attention_mask: np.ndarray | tf.Tensor | None = None,877 token_type_ids: np.ndarray | tf.Tensor | None = None,878 position_ids: np.ndarray | tf.Tensor | None = None,879 head_mask: np.ndarray | tf.Tensor | None = None,880 inputs_embeds: np.ndarray | tf.Tensor | None = None,881 output_attentions: bool | None = None,882 output_hidden_states: bool | None = None,883 return_dict: bool | None = None,884 training: bool | None = False,885 ) -> TFBaseModelOutputWithPooling | tuple[tf.Tensor]:886 outputs = self.albert(887 input_ids=input_ids,888 attention_mask=attention_mask,889 token_type_ids=token_type_ids,890 position_ids=position_ids,891 head_mask=head_mask,892 inputs_embeds=inputs_embeds,893 output_attentions=output_attentions,894 output_hidden_states=output_hidden_states,895 return_dict=return_dict,896 training=training,897 )898 899 return outputs900 901 def build(self, input_shape=None):902 if self.built:903 return904 self.built = True905 if getattr(self, "albert", None) is not None:906 with tf.name_scope(self.albert.name):907 self.albert.build(None)908 909 910@add_start_docstrings(911 """912 Albert Model with two heads on top for pretraining: a `masked language modeling` head and a `sentence order913 prediction` (classification) head.914 """,915 ALBERT_START_DOCSTRING,916)917class TFAlbertForPreTraining(TFAlbertPreTrainedModel, TFAlbertPreTrainingLoss):918 # names with a '.' represents the authorized unexpected/missing layers when a TF model is loaded from a PT model919 _keys_to_ignore_on_load_unexpected = [r"predictions.decoder.weight"]920 921 def __init__(self, config: AlbertConfig, *inputs, **kwargs):922 super().__init__(config, *inputs, **kwargs)923 924 self.num_labels = config.num_labels925 926 self.albert = TFAlbertMainLayer(config, name="albert")927 self.predictions = TFAlbertMLMHead(config, input_embeddings=self.albert.embeddings, name="predictions")928 self.sop_classifier = TFAlbertSOPHead(config, name="sop_classifier")929 930 def get_lm_head(self) -> keras.layers.Layer:931 return self.predictions932 933 @unpack_inputs934 @add_start_docstrings_to_model_forward(ALBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length"))935 @replace_return_docstrings(output_type=TFAlbertForPreTrainingOutput, config_class=_CONFIG_FOR_DOC)936 def call(937 self,938 input_ids: TFModelInputType | None = None,939 attention_mask: np.ndarray | tf.Tensor | None = None,940 token_type_ids: np.ndarray | tf.Tensor | None = None,941 position_ids: np.ndarray | tf.Tensor | None = None,942 head_mask: np.ndarray | tf.Tensor | None = None,943 inputs_embeds: np.ndarray | tf.Tensor | None = None,944 output_attentions: bool | None = None,945 output_hidden_states: bool | None = None,946 return_dict: bool | None = None,947 labels: np.ndarray | tf.Tensor | None = None,948 sentence_order_label: np.ndarray | tf.Tensor | None = None,949 training: bool | None = False,950 ) -> TFAlbertForPreTrainingOutput | tuple[tf.Tensor]:951 r"""952 Return:953 954 Example:955 956 ```python957 >>> import tensorflow as tf958 >>> from transformers import AutoTokenizer, TFAlbertForPreTraining959 960 >>> tokenizer = AutoTokenizer.from_pretrained("albert/albert-base-v2")961 >>> model = TFAlbertForPreTraining.from_pretrained("albert/albert-base-v2")962 963 >>> input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :]964 >>> # Batch size 1965 >>> outputs = model(input_ids)966 967 >>> prediction_logits = outputs.prediction_logits968 >>> sop_logits = outputs.sop_logits969 ```"""970 971 outputs = self.albert(972 input_ids=input_ids,973 attention_mask=attention_mask,974 token_type_ids=token_type_ids,975 position_ids=position_ids,976 head_mask=head_mask,977 inputs_embeds=inputs_embeds,978 output_attentions=output_attentions,979 output_hidden_states=output_hidden_states,980 return_dict=return_dict,981 training=training,982 )983 sequence_output, pooled_output = outputs[:2]984 prediction_scores = self.predictions(hidden_states=sequence_output)985 sop_scores = self.sop_classifier(pooled_output=pooled_output, training=training)986 total_loss = None987 988 if labels is not None and sentence_order_label is not None:989 d_labels = {"labels": labels}990 d_labels["sentence_order_label"] = sentence_order_label991 total_loss = self.hf_compute_loss(labels=d_labels, logits=(prediction_scores, sop_scores))992 993 if not return_dict:994 output = (prediction_scores, sop_scores) + outputs[2:]995 return ((total_loss,) + output) if total_loss is not None else output996 997 return TFAlbertForPreTrainingOutput(998 loss=total_loss,999 prediction_logits=prediction_scores,1000 sop_logits=sop_scores,1001 hidden_states=outputs.hidden_states,1002 attentions=outputs.attentions,1003 )1004 1005 def build(self, input_shape=None):1006 if self.built:1007 return1008 self.built = True1009 if getattr(self, "albert", None) is not None:1010 with tf.name_scope(self.albert.name):1011 self.albert.build(None)1012 if getattr(self, "predictions", None) is not None:1013 with tf.name_scope(self.predictions.name):1014 self.predictions.build(None)1015 if getattr(self, "sop_classifier", None) is not None:1016 with tf.name_scope(self.sop_classifier.name):1017 self.sop_classifier.build(None)1018 1019 1020class TFAlbertSOPHead(keras.layers.Layer):1021 def __init__(self, config: AlbertConfig, **kwargs):1022 super().__init__(**kwargs)1023 1024 self.dropout = keras.layers.Dropout(rate=config.classifier_dropout_prob)1025 self.classifier = keras.layers.Dense(1026 units=config.num_labels,1027 kernel_initializer=get_initializer(config.initializer_range),1028 name="classifier",1029 )1030 self.config = config1031 1032 def call(self, pooled_output: tf.Tensor, training: bool) -> tf.Tensor:1033 dropout_pooled_output = self.dropout(inputs=pooled_output, training=training)1034 logits = self.classifier(inputs=dropout_pooled_output)1035 1036 return logits1037 1038 def build(self, input_shape=None):1039 if self.built:1040 return1041 self.built = True1042 if getattr(self, "classifier", None) is not None:1043 with tf.name_scope(self.classifier.name):1044 self.classifier.build([None, None, self.config.hidden_size])1045 1046 1047@add_start_docstrings("""Albert Model with a `language modeling` head on top.""", ALBERT_START_DOCSTRING)1048class TFAlbertForMaskedLM(TFAlbertPreTrainedModel, TFMaskedLanguageModelingLoss):1049 # names with a '.' represents the authorized unexpected/missing layers when a TF model is loaded from a PT model1050 _keys_to_ignore_on_load_unexpected = [r"pooler", r"predictions.decoder.weight"]1051 1052 def __init__(self, config: AlbertConfig, *inputs, **kwargs):1053 super().__init__(config, *inputs, **kwargs)1054 1055 self.albert = TFAlbertMainLayer(config, add_pooling_layer=False, name="albert")1056 self.predictions = TFAlbertMLMHead(config, input_embeddings=self.albert.embeddings, name="predictions")1057 1058 def get_lm_head(self) -> keras.layers.Layer:1059 return self.predictions1060 1061 @unpack_inputs1062 @add_start_docstrings_to_model_forward(ALBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length"))1063 @replace_return_docstrings(output_type=TFMaskedLMOutput, config_class=_CONFIG_FOR_DOC)1064 def call(1065 self,1066 input_ids: TFModelInputType | None = None,1067 attention_mask: np.ndarray | tf.Tensor | None = None,1068 token_type_ids: np.ndarray | tf.Tensor | None = None,1069 position_ids: np.ndarray | tf.Tensor | None = None,1070 head_mask: np.ndarray | tf.Tensor | None = None,1071 inputs_embeds: np.ndarray | tf.Tensor | None = None,1072 output_attentions: bool | None = None,1073 output_hidden_states: bool | None = None,1074 return_dict: bool | None = None,1075 labels: np.ndarray | tf.Tensor | None = None,1076 training: bool | None = False,1077 ) -> TFMaskedLMOutput | tuple[tf.Tensor]:1078 r"""1079 labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):1080 Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,1081 config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the1082 loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`1083 1084 Returns:1085 1086 Example:1087 1088 ```python1089 >>> import tensorflow as tf1090 >>> from transformers import AutoTokenizer, TFAlbertForMaskedLM1091 1092 >>> tokenizer = AutoTokenizer.from_pretrained("albert/albert-base-v2")1093 >>> model = TFAlbertForMaskedLM.from_pretrained("albert/albert-base-v2")1094 1095 >>> # add mask_token1096 >>> inputs = tokenizer(f"The capital of [MASK] is Paris.", return_tensors="tf")1097 >>> logits = model(**inputs).logits1098 1099 >>> # retrieve index of [MASK]1100 >>> mask_token_index = tf.where(inputs.input_ids == tokenizer.mask_token_id)[0][1]1101 >>> predicted_token_id = tf.math.argmax(logits[0, mask_token_index], axis=-1)1102 >>> tokenizer.decode(predicted_token_id)1103 'france'1104 ```1105 1106 ```python1107 >>> labels = tokenizer("The capital of France is Paris.", return_tensors="tf")["input_ids"]1108 >>> labels = tf.where(inputs.input_ids == tokenizer.mask_token_id, labels, -100)1109 >>> outputs = model(**inputs, labels=labels)1110 >>> round(float(outputs.loss), 2)1111 0.811112 ```1113 """1114 outputs = self.albert(1115 input_ids=input_ids,1116 attention_mask=attention_mask,1117 token_type_ids=token_type_ids,1118 position_ids=position_ids,1119 head_mask=head_mask,1120 inputs_embeds=inputs_embeds,1121 output_attentions=output_attentions,1122 output_hidden_states=output_hidden_states,1123 return_dict=return_dict,1124 training=training,1125 )1126 sequence_output = outputs[0]1127 prediction_scores = self.predictions(hidden_states=sequence_output, training=training)1128 loss = None if labels is None else self.hf_compute_loss(labels=labels, logits=prediction_scores)1129 1130 if not return_dict:1131 output = (prediction_scores,) + outputs[2:]1132 1133 return ((loss,) + output) if loss is not None else output1134 1135 return TFMaskedLMOutput(1136 loss=loss,1137 logits=prediction_scores,1138 hidden_states=outputs.hidden_states,1139 attentions=outputs.attentions,1140 )1141 1142 def build(self, input_shape=None):1143 if self.built:1144 return1145 self.built = True1146 if getattr(self, "albert", None) is not None:1147 with tf.name_scope(self.albert.name):1148 self.albert.build(None)1149 if getattr(self, "predictions", None) is not None:1150 with tf.name_scope(self.predictions.name):1151 self.predictions.build(None)1152 1153 1154@add_start_docstrings(1155 """1156 Albert Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled1157 output) e.g. for GLUE tasks.1158 """,1159 ALBERT_START_DOCSTRING,1160)1161class TFAlbertForSequenceClassification(TFAlbertPreTrainedModel, TFSequenceClassificationLoss):1162 # names with a '.' represents the authorized unexpected/missing layers when a TF model is loaded from a PT model1163 _keys_to_ignore_on_load_unexpected = [r"predictions"]1164 _keys_to_ignore_on_load_missing = [r"dropout"]1165 1166 def __init__(self, config: AlbertConfig, *inputs, **kwargs):1167 super().__init__(config, *inputs, **kwargs)1168 1169 self.num_labels = config.num_labels1170 1171 self.albert = TFAlbertMainLayer(config, name="albert")1172 self.dropout = keras.layers.Dropout(rate=config.classifier_dropout_prob)1173 self.classifier = keras.layers.Dense(1174 units=config.num_labels, kernel_initializer=get_initializer(config.initializer_range), name="classifier"1175 )1176 self.config = config1177 1178 @unpack_inputs1179 @add_start_docstrings_to_model_forward(ALBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length"))1180 @add_code_sample_docstrings(1181 checkpoint="vumichien/albert-base-v2-imdb",1182 output_type=TFSequenceClassifierOutput,1183 config_class=_CONFIG_FOR_DOC,1184 expected_output="'LABEL_1'",1185 expected_loss=0.12,1186 )1187 def call(1188 self,1189 input_ids: TFModelInputType | None = None,1190 attention_mask: np.ndarray | tf.Tensor | None = None,1191 token_type_ids: np.ndarray | tf.Tensor | None = None,1192 position_ids: np.ndarray | tf.Tensor | None = None,1193 head_mask: np.ndarray | tf.Tensor | None = None,1194 inputs_embeds: np.ndarray | tf.Tensor | None = None,1195 output_attentions: bool | None = None,1196 output_hidden_states: bool | None = None,1197 return_dict: bool | None = None,1198 labels: np.ndarray | tf.Tensor | None = None,1199 training: bool | None = False,1200 ) -> TFSequenceClassifierOutput | tuple[tf.Tensor]: