Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2018 The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""Classes to support Encoder-Decoder architectures"""16 17import gc18import inspect19import os20import tempfile21import warnings22from typing import Optional, Union23 24import torch25from torch import nn26from torch.nn import CrossEntropyLoss27 28from ...cache_utils import Cache29from ...configuration_utils import PretrainedConfig30from ...generation import GenerationMixin31from ...modeling_outputs import BaseModelOutput, Seq2SeqLMOutput32from ...modeling_utils import PreTrainedModel33from ...utils import auto_docstring, logging34from ..auto.configuration_auto import AutoConfig35from ..auto.modeling_auto import AutoModel, AutoModelForCausalLM36from .configuration_encoder_decoder import EncoderDecoderConfig37 38 39logger = logging.get_logger(__name__)40 41 42DEPRECATION_WARNING = (43    "Version v4.12.0 introduces a better way to train encoder-decoder models by computing the loss inside the"44    " encoder-decoder framework rather than in the decoder itself. You may observe training discrepancies if"45    " fine-tuning a model trained with versions anterior to 4.12.0. The decoder_input_ids are now created based on the"46    " labels, no need to pass them yourself anymore."47)48 49 50def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int):51    """52    Shift input ids one token to the right.53    """54    shifted_input_ids = input_ids.new_zeros(input_ids.shape)55    shifted_input_ids[:, 1:] = input_ids[:, :-1].clone()56    if decoder_start_token_id is None:57        raise ValueError("Make sure to set the decoder_start_token_id attribute of the model's configuration.")58    shifted_input_ids[:, 0] = decoder_start_token_id59 60    if pad_token_id is None:61        raise ValueError("Make sure to set the pad_token_id attribute of the model's configuration.")62    # replace possible -100 values in labels by `pad_token_id`63    shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id)64 65    return shifted_input_ids66 67 68@auto_docstring69class EncoderDecoderModel(PreTrainedModel, GenerationMixin):70    r"""71    [`EncoderDecoderModel`] is a generic model class that will be instantiated as a transformer architecture with one72    of the base model classes of the library as encoder and another one as decoder when created with the73    :meth*~transformers.AutoModel.from_pretrained* class method for the encoder and74    :meth*~transformers.AutoModelForCausalLM.from_pretrained* class method for the decoder.75    """76 77    config: EncoderDecoderConfig78    base_model_prefix = "encoder_decoder"79    main_input_name = "input_ids"80    supports_gradient_checkpointing = True81    _supports_param_buffer_assignment = False82    _supports_flash_attn = True83    _supports_sdpa = True84 85    def __init__(86        self,87        config: Optional[PretrainedConfig] = None,88        encoder: Optional[PreTrainedModel] = None,89        decoder: Optional[PreTrainedModel] = None,90    ):91        r"""92        encoder (`PreTrainedModel`, *optional*):93            The encoder model to use.94        decoder (`PreTrainedModel`, *optional*):95            The decoder model to use.96        """97        if config is None and (encoder is None or decoder is None):98            raise ValueError("Either a configuration or an encoder and a decoder has to be provided.")99        if config is None:100            config = EncoderDecoderConfig.from_encoder_decoder_configs(encoder.config, decoder.config)101        else:102            if not isinstance(config, self.config_class):103                raise ValueError(f"Config: {config} has to be of type {self.config_class}")104 105        if config.decoder.cross_attention_hidden_size is not None:106            if config.decoder.cross_attention_hidden_size != config.encoder.hidden_size:107                raise ValueError(108                    "If `cross_attention_hidden_size` is specified in the decoder's configuration, it has to be equal"109                    f" to the encoder's `hidden_size`. Got {config.decoder.cross_attention_hidden_size} for"110                    f" `config.decoder.cross_attention_hidden_size` and {config.encoder.hidden_size} for"111                    " `config.encoder.hidden_size`."112                )113 114        # initialize with config115        super().__init__(config)116 117        if encoder is None:118            from ..auto.modeling_auto import AutoModel119 120            encoder = AutoModel.from_config(config.encoder)121 122        if decoder is None:123            from ..auto.modeling_auto import AutoModelForCausalLM124 125            decoder = AutoModelForCausalLM.from_config(config.decoder)126 127        self.encoder = encoder128        self.decoder = decoder129 130        if self.encoder.config.to_dict() != self.config.encoder.to_dict():131            logger.warning(132                f"Config of the encoder: {self.encoder.__class__} is overwritten by shared encoder config:"133                f" {self.config.encoder}"134            )135        if self.decoder.config.to_dict() != self.config.decoder.to_dict():136            logger.warning(137                f"Config of the decoder: {self.decoder.__class__} is overwritten by shared decoder config:"138                f" {self.config.decoder}"139            )140 141        # make sure that the individual model's config refers to the shared config142        # so that the updates to the config will be synced143        # update `_attn_implementation` because the attn is set in a deepcopied config within PreTrainedModel144        self.config.encoder._attn_implementation = self.encoder.config._attn_implementation145        self.config.decoder._attn_implementation = self.decoder.config._attn_implementation146        self.encoder.config = self.config.encoder147        self.decoder.config = self.config.decoder148 149        # encoder outputs might need to be projected to different dimension for decoder150        if (151            self.encoder.config.hidden_size != self.decoder.config.hidden_size152            and self.decoder.config.cross_attention_hidden_size is None153        ):154            self.enc_to_dec_proj = nn.Linear(self.encoder.config.hidden_size, self.decoder.config.hidden_size)155 156        if self.encoder.get_output_embeddings() is not None:157            raise ValueError(158                f"The encoder {self.encoder} should not have a LM Head. Please use a model without LM Head"159            )160 161        decoder_signature = set(inspect.signature(self.decoder.forward).parameters.keys())162        if "encoder_hidden_states" not in decoder_signature:163            raise ValueError(164                "The selected decoder is not prepared for the encoder hidden states to be passed. Please see the "165                "following discussion on GitHub: https://github.com/huggingface/transformers/issues/23350"166            )167 168        # tie encoder, decoder weights if config set accordingly169        self.tie_weights()170 171    def tie_weights(self):172        self.encoder.tie_weights()173        self.decoder.tie_weights()174        # tie encoder & decoder if needed175        if self.config.tie_encoder_decoder:176            # tie encoder and decoder base model177            decoder_base_model_prefix = self.decoder.base_model_prefix178            tied_weights = self._tie_encoder_decoder_weights(179                self.encoder,180                self.decoder._modules[decoder_base_model_prefix],181                self.decoder.base_model_prefix,182                "encoder",183            )184            # Setting a dynamic variable instead of `_tied_weights_keys` because it's a class185            # attributed not an instance member, therefore modifying it will modify the entire class186            # Leading to issues on subsequent calls by different tests or subsequent calls.187            self._dynamic_tied_weights_keys = tied_weights188 189    def _init_weights(self, module):190        if module in self.encoder.modules():191            self.encoder._init_weights(module)192        elif module in self.decoder.modules():193            self.decoder._init_weights(module)194 195    def get_encoder(self):196        return self.encoder197 198    def get_input_embeddings(self):199        return self.encoder.get_input_embeddings()200 201    def get_output_embeddings(self):202        return self.decoder.get_output_embeddings()203 204    def set_output_embeddings(self, new_embeddings):205        return self.decoder.set_output_embeddings(new_embeddings)206 207    @classmethod208    def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):209        r"""210        Example:211 212        ```python213        >>> from transformers import EncoderDecoderModel214 215        >>> model = EncoderDecoderModel.from_pretrained("patrickvonplaten/bert2bert-cnn_dailymail-fp16")216        ```"""217 218        from_tf = kwargs.pop("from_tf", False)219        if from_tf:220            from transformers import TFEncoderDecoderModel221 222            # a workaround to load from tensorflow checkpoint223            # Using `_tf_model` won't work, because the weight names in the encoder/decoder of `_tf_model` get224            # extended before saving those components. For example, The name of `_tf_model.encoder.vit` is225            # `[top model name]/encoder/vit`, but the name of `tf_model.encoder.vit` is `[top model name]/vit`. The226            # [top model name] is handled (stripped) by the conversion method, and the former case gets extra `encoder`,227            # which should not occur when we want to save the components alone.228            # There was a (very) ugly potential fix, which wasn't integrated to `transformers`: see229            #   https://github.com/huggingface/transformers/pull/13222/commits/dbb3c9de76eee235791d2064094654637c99f36d#r697304245230            #   (the change in `src/transformers/modeling_tf_utils.py`)231            _tf_model = TFEncoderDecoderModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)232            config = _tf_model.config233 234            # Using `tf_model` instead235            encoder = _tf_model.encoder.__class__(_tf_model.config.encoder)236            decoder = _tf_model.decoder.__class__(_tf_model.config.decoder)237            # Make sure models are built238            encoder(encoder.dummy_inputs)239            decoder(decoder.dummy_inputs)240 241            # Get the variable correspondence between `_tf_model` and `encoder` and `decoder`242            encoder_variables = {}243            for v in encoder.trainable_variables + encoder.non_trainable_variables:244                encoder_variables["/".join(v.name.split("/")[1:])] = v245            decoder_variables = {}246            for v in decoder.trainable_variables + decoder.non_trainable_variables:247                decoder_variables["/".join(v.name.split("/")[1:])] = v248 249            _encoder_variables = {}250            for v in _tf_model.encoder.trainable_variables + _tf_model.encoder.non_trainable_variables:251                _encoder_variables["/".join(v.name.split("/")[2:])] = v252            _decoder_variables = {}253            for v in _tf_model.decoder.trainable_variables + _tf_model.decoder.non_trainable_variables:254                _decoder_variables["/".join(v.name.split("/")[2:])] = v255 256            # assign weight values to `encoder` and `decoder` from `_tf_model`257            for name, v in encoder_variables.items():258                v.assign(_encoder_variables[name])259            for name, v in decoder_variables.items():260                v.assign(_decoder_variables[name])261 262            tf_model = TFEncoderDecoderModel(encoder=encoder, decoder=decoder)263 264            # Deal with `enc_to_dec_proj`265            if hasattr(_tf_model, "enc_to_dec_proj"):266                tf_model(tf_model.dummy_inputs)267                tf_model.enc_to_dec_proj.kernel.assign(_tf_model.enc_to_dec_proj.kernel)268                tf_model.enc_to_dec_proj.bias.assign(_tf_model.enc_to_dec_proj.bias)269 270            with tempfile.TemporaryDirectory() as tmpdirname:271                encoder_dir = os.path.join(tmpdirname, "encoder")272                decoder_dir = os.path.join(tmpdirname, "decoder")273                tf_model.encoder.save_pretrained(encoder_dir)274                tf_model.decoder.save_pretrained(decoder_dir)275 276                if hasattr(tf_model, "enc_to_dec_proj"):277                    enc_to_dec_proj_weight = torch.transpose(278                        torch.from_numpy(tf_model.enc_to_dec_proj.kernel.numpy()), 1, 0279                    )280                    enc_to_dec_proj_bias = torch.from_numpy(tf_model.enc_to_dec_proj.bias.numpy())281 282                del _tf_model283                del tf_model284                gc.collect()285 286                model = EncoderDecoderModel.from_encoder_decoder_pretrained(287                    encoder_dir, decoder_dir, encoder_from_tf=True, decoder_from_tf=True288                )289                # This is only for copying some specific attributes of this particular model.290                model.config = config291 292                if hasattr(model, "enc_to_dec_proj"):293                    model.enc_to_dec_proj.weight.data = enc_to_dec_proj_weight.contiguous()294                    model.enc_to_dec_proj.bias.data = enc_to_dec_proj_bias.contiguous()295 296                return model297 298        return super().from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)299 300    @classmethod301    def from_encoder_decoder_pretrained(302        cls,303        encoder_pretrained_model_name_or_path: Optional[str] = None,304        decoder_pretrained_model_name_or_path: Optional[str] = None,305        *model_args,306        **kwargs,307    ) -> PreTrainedModel:308        r"""309        Instantiate an encoder and a decoder from one or two base classes of the library from pretrained model310        checkpoints.311 312 313        The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated). To train314        the model, you need to first set it back in training mode with `model.train()`.315 316        Params:317            encoder_pretrained_model_name_or_path (`str`, *optional*):318                Information necessary to initiate the encoder. Can be either:319 320                    - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.321                    - A path to a *directory* containing model weights saved using322                      [`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.323                    - A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In324                      this case, `from_tf` should be set to `True` and a configuration object should be provided as325                      `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a326                      PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.327 328            decoder_pretrained_model_name_or_path (`str`, *optional*, defaults to `None`):329                Information necessary to initiate the decoder. Can be either:330 331                    - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.332                    - A path to a *directory* containing model weights saved using333                      [`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.334                    - A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In335                      this case, `from_tf` should be set to `True` and a configuration object should be provided as336                      `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a337                      PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.338 339            model_args (remaining positional arguments, *optional*):340                All remaining positional arguments will be passed to the underlying model's `__init__` method.341 342            kwargs (remaining dictionary of keyword arguments, *optional*):343                Can be used to update the configuration object (after it being loaded) and initiate the model (e.g.,344                `output_attentions=True`).345 346                - To update the encoder configuration, use the prefix *encoder_* for each configuration parameter.347                - To update the decoder configuration, use the prefix *decoder_* for each configuration parameter.348                - To update the parent model configuration, do not use a prefix for each configuration parameter.349 350                Behaves differently depending on whether a `config` is provided or automatically loaded.351 352        Example:353 354        ```python355        >>> from transformers import EncoderDecoderModel356 357        >>> # initialize a bert2bert from two pretrained BERT models. Note that the cross-attention layers will be randomly initialized358        >>> model = EncoderDecoderModel.from_encoder_decoder_pretrained("google-bert/bert-base-uncased", "google-bert/bert-base-uncased")359        >>> # saving model after fine-tuning360        >>> model.save_pretrained("./bert2bert")361        >>> # load fine-tuned model362        >>> model = EncoderDecoderModel.from_pretrained("./bert2bert")363        ```"""364 365        kwargs_encoder = {366            argument[len("encoder_") :]: value for argument, value in kwargs.items() if argument.startswith("encoder_")367        }368 369        kwargs_decoder = {370            argument[len("decoder_") :]: value for argument, value in kwargs.items() if argument.startswith("decoder_")371        }372 373        # remove encoder, decoder kwargs from kwargs374        for key in kwargs_encoder:375            del kwargs["encoder_" + key]376        for key in kwargs_decoder:377            del kwargs["decoder_" + key]378 379        # Load and initialize the encoder and decoder380        # The distinction between encoder and decoder at the model level is made381        # by the value of the flag `is_decoder` that we need to set correctly.382        encoder = kwargs_encoder.pop("model", None)383        if encoder is None:384            if encoder_pretrained_model_name_or_path is None:385                raise ValueError(386                    "If `encoder_model` is not defined as an argument, a `encoder_pretrained_model_name_or_path` has "387                    "to be defined."388                )389 390            if "config" not in kwargs_encoder:391                encoder_config, kwargs_encoder = AutoConfig.from_pretrained(392                    encoder_pretrained_model_name_or_path, **kwargs_encoder, return_unused_kwargs=True393                )394 395                if encoder_config.is_decoder is True or encoder_config.add_cross_attention is True:396                    logger.info(397                        f"Initializing {encoder_pretrained_model_name_or_path} as a encoder model "398                        "from a decoder model. Cross-attention and causal mask are disabled."399                    )400                    encoder_config.is_decoder = False401                    encoder_config.add_cross_attention = False402 403                kwargs_encoder["config"] = encoder_config404 405            encoder = AutoModel.from_pretrained(encoder_pretrained_model_name_or_path, *model_args, **kwargs_encoder)406 407        decoder = kwargs_decoder.pop("model", None)408        if decoder is None:409            if decoder_pretrained_model_name_or_path is None:410                raise ValueError(411                    "If `decoder_model` is not defined as an argument, a `decoder_pretrained_model_name_or_path` has "412                    "to be defined."413                )414 415            if "config" not in kwargs_decoder:416                decoder_config, kwargs_decoder = AutoConfig.from_pretrained(417                    decoder_pretrained_model_name_or_path, **kwargs_decoder, return_unused_kwargs=True418                )419 420                if decoder_config.is_decoder is False or decoder_config.add_cross_attention is False:421                    logger.info(422                        f"Initializing {decoder_pretrained_model_name_or_path} as a decoder model. Cross attention"423                        f" layers are added to {decoder_pretrained_model_name_or_path} and randomly initialized if"424                        f" {decoder_pretrained_model_name_or_path}'s architecture allows for cross attention layers."425                    )426                    decoder_config.is_decoder = True427                    decoder_config.add_cross_attention = True428 429                kwargs_decoder["config"] = decoder_config430 431            if kwargs_decoder["config"].is_decoder is False or kwargs_decoder["config"].add_cross_attention is False:432                logger.warning(433                    f"Decoder model {decoder_pretrained_model_name_or_path} is not initialized as a decoder. "434                    f"In order to initialize {decoder_pretrained_model_name_or_path} as a decoder, "435                    "make sure that the attributes `is_decoder` and `add_cross_attention` of `decoder_config` "436                    "passed to `.from_encoder_decoder_pretrained(...)` are set to `True` or do not pass a "437                    "`decoder_config` to `.from_encoder_decoder_pretrained(...)`"438                )439 440            decoder = AutoModelForCausalLM.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs_decoder)441 442        # instantiate config with corresponding kwargs443        config = EncoderDecoderConfig.from_encoder_decoder_configs(encoder.config, decoder.config, **kwargs)444        return cls(encoder=encoder, decoder=decoder, config=config)445 446    @auto_docstring447    def forward(448        self,449        input_ids: Optional[torch.LongTensor] = None,450        attention_mask: Optional[torch.FloatTensor] = None,451        decoder_input_ids: Optional[torch.LongTensor] = None,452        decoder_attention_mask: Optional[torch.BoolTensor] = None,453        encoder_outputs: Optional[tuple[torch.FloatTensor]] = None,454        past_key_values: Optional[Cache] = None,455        inputs_embeds: Optional[torch.FloatTensor] = None,456        decoder_inputs_embeds: Optional[torch.FloatTensor] = None,457        labels: Optional[torch.LongTensor] = None,458        use_cache: Optional[bool] = None,459        output_attentions: Optional[bool] = None,460        output_hidden_states: Optional[bool] = None,461        return_dict: Optional[bool] = None,462        **kwargs,463    ) -> Union[tuple, Seq2SeqLMOutput]:464        r"""465        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):466            Indices of decoder input sequence tokens in the vocabulary.467 468            Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and469            [`PreTrainedTokenizer.__call__`] for details.470 471            [What are input IDs?](../glossary#input-ids)472 473            If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see474            `past_key_values`).475 476            For training, `decoder_input_ids` are automatically created by the model by shifting the `labels` to the477            right, replacing -100 by the `pad_token_id` and prepending them with the `decoder_start_token_id`.478        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):479            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also480            be used by default.481        decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*):482            Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded483            representation. This is useful if you want more control over how to convert `decoder_input_ids` indices484            into associated vectors than the model's internal embedding lookup matrix.485        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):486            Labels for computing the masked language modeling loss for the decoder. Indices should be in `[-100, 0,487            ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored488            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`489 490        Examples:491 492        ```python493        >>> from transformers import EncoderDecoderModel, BertTokenizer494        >>> import torch495 496        >>> tokenizer = BertTokenizer.from_pretrained("google-bert/bert-base-uncased")497        >>> model = EncoderDecoderModel.from_encoder_decoder_pretrained(498        ...     "google-bert/bert-base-uncased", "google-bert/bert-base-uncased"499        ... )  # initialize Bert2Bert from pre-trained checkpoints500 501        >>> # training502        >>> model.config.decoder_start_token_id = tokenizer.cls_token_id503        >>> model.config.pad_token_id = tokenizer.pad_token_id504        >>> model.config.vocab_size = model.config.decoder.vocab_size505 506        >>> input_ids = tokenizer("This is a really long text", return_tensors="pt").input_ids507        >>> labels = tokenizer("This is the corresponding summary", return_tensors="pt").input_ids508        >>> outputs = model(input_ids=input_ids, labels=labels)509        >>> loss, logits = outputs.loss, outputs.logits510 511        >>> # save and load from pretrained512        >>> model.save_pretrained("bert2bert")513        >>> model = EncoderDecoderModel.from_pretrained("bert2bert")514 515        >>> # generation516        >>> generated = model.generate(input_ids)517        ```"""518        return_dict = return_dict if return_dict is not None else self.config.use_return_dict519 520        kwargs_encoder = {argument: value for argument, value in kwargs.items() if not argument.startswith("decoder_")}521 522        kwargs_decoder = {523            argument[len("decoder_") :]: value for argument, value in kwargs.items() if argument.startswith("decoder_")524        }525        if "num_items_in_batch" in kwargs_encoder:526            kwargs_decoder["num_items_in_batch"] = kwargs_encoder.pop("num_items_in_batch", None)527 528        if encoder_outputs is None:529            encoder_outputs = self.encoder(530                input_ids=input_ids,531                attention_mask=attention_mask,532                inputs_embeds=inputs_embeds,533                output_attentions=output_attentions,534                output_hidden_states=output_hidden_states,535                return_dict=return_dict,536                **kwargs_encoder,537            )538        elif isinstance(encoder_outputs, tuple):539            encoder_outputs = BaseModelOutput(*encoder_outputs)540 541        encoder_hidden_states = encoder_outputs[0]542 543        # optionally project encoder_hidden_states544        if (545            self.encoder.config.hidden_size != self.decoder.config.hidden_size546            and self.decoder.config.cross_attention_hidden_size is None547        ):548            encoder_hidden_states = self.enc_to_dec_proj(encoder_hidden_states)549 550        if (labels is not None) and (decoder_input_ids is None and decoder_inputs_embeds is None):551            decoder_input_ids = shift_tokens_right(552                labels, self.config.pad_token_id, self.config.decoder_start_token_id553            )554            if decoder_attention_mask is None:555                decoder_attention_mask = decoder_input_ids.new_tensor(decoder_input_ids != self.config.pad_token_id)556 557        # Decode558        decoder_outputs = self.decoder(559            input_ids=decoder_input_ids,560            attention_mask=decoder_attention_mask,561            encoder_hidden_states=encoder_hidden_states,562            encoder_attention_mask=attention_mask,563            inputs_embeds=decoder_inputs_embeds,564            output_attentions=output_attentions,565            output_hidden_states=output_hidden_states,566            use_cache=use_cache,567            past_key_values=past_key_values,568            return_dict=return_dict,569            **kwargs_decoder,570        )571 572        # Compute loss independent from decoder (as some shift the logits inside them)573        loss = None574        if labels is not None:575            warnings.warn(DEPRECATION_WARNING, FutureWarning)576            logits = decoder_outputs.logits if return_dict else decoder_outputs[0]577            loss_fct = CrossEntropyLoss()578            loss = loss_fct(logits.reshape(-1, self.decoder.config.vocab_size), labels.view(-1))579 580        if not return_dict:581            if loss is not None:582                return (loss,) + decoder_outputs + encoder_outputs583            else:584                return decoder_outputs + encoder_outputs585 586        return Seq2SeqLMOutput(587            loss=loss,588            logits=decoder_outputs.logits,589            past_key_values=decoder_outputs.past_key_values,590            decoder_hidden_states=decoder_outputs.hidden_states,591            decoder_attentions=decoder_outputs.attentions,592            cross_attentions=decoder_outputs.cross_attentions,593            encoder_last_hidden_state=encoder_outputs.last_hidden_state,594            encoder_hidden_states=encoder_outputs.hidden_states,595            encoder_attentions=encoder_outputs.attentions,596        )597 598    def prepare_decoder_input_ids_from_labels(self, labels: torch.Tensor):599        return shift_tokens_right(labels, self.config.pad_token_id, self.config.decoder_start_token_id)600 601    def resize_token_embeddings(self, *args, **kwargs):602        raise NotImplementedError(603            "Resizing the embedding layers via the EncoderDecoderModel directly is not supported. Please use the"604            " respective methods of the wrapped objects (model.encoder.resize_token_embeddings(...) or"605            " model.decoder.resize_token_embeddings(...))"606        )607 608 609__all__ = ["EncoderDecoderModel"]610