Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2020 The Microsoft Authors and The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""ProphetNet model configuration"""16 17from typing import Callable, Optional, Union18 19from ...configuration_utils import PretrainedConfig20from ...utils import logging21 22 23logger = logging.get_logger(__name__)24 25 26class ProphetNetConfig(PretrainedConfig):27    r"""28    This is the configuration class to store the configuration of a [`ProphetNetModel`]. It is used to instantiate a29    ProphetNet model according to the specified arguments, defining the model architecture. Instantiating a30    configuration with the defaults will yield a similar configuration to that of the ProphetNet31    [microsoft/prophetnet-large-uncased](https://huggingface.co/microsoft/prophetnet-large-uncased) architecture.32 33    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the34    documentation from [`PretrainedConfig`] for more information.35 36    Args:37        activation_dropout (`float`, *optional*, defaults to 0.1):38            The dropout ratio for activations inside the fully connected layer.39        activation_function (`str` or `function`, *optional*, defaults to `"gelu"`):40            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,41            `"relu"`, `"silu"` and `"gelu_new"` are supported.42        vocab_size (`int`, *optional*, defaults to 30522):43            Vocabulary size of the ProphetNET model. Defines the number of different tokens that can be represented by44            the `inputs_ids` passed when calling [`ProphetNetModel`].45        hidden_size (`int`, *optional*, defaults to 1024):46            Dimensionality of the layers and the pooler layer.47        encoder_ffn_dim (`int`, *optional*, defaults to 4096):48            Dimensionality of the "intermediate" (often named feed-forward) layer in decoder.49        num_encoder_layers (`int`, *optional*, defaults to 12):50            Number of encoder layers.51        num_encoder_attention_heads (`int`, *optional*, defaults to 16):52            Number of attention heads for each attention layer in the Transformer encoder.53        decoder_ffn_dim (`int`, *optional*, defaults to 4096):54            Dimensionality of the `intermediate` (often named feed-forward) layer in decoder.55        num_decoder_layers (`int`, *optional*, defaults to 12):56            Number of decoder layers.57        num_decoder_attention_heads (`int`, *optional*, defaults to 16):58            Number of attention heads for each attention layer in the Transformer decoder.59        attention_dropout (`float`, *optional*, defaults to 0.1):60            The dropout ratio for the attention probabilities.61        dropout (`float`, *optional*, defaults to 0.1):62            The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.63        max_position_embeddings (`int`, *optional*, defaults to 512):64            The maximum sequence length that this model might ever be used with. Typically set this to something large65            just in case (e.g., 512 or 1024 or 2048).66        init_std (`float`, *optional*, defaults to 0.02):67            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.68        add_cross_attention (`bool`, *optional*, defaults to `True`):69            Whether cross-attention layers should be added to the model.70        is_encoder_decoder (`bool`, *optional*, defaults to `True`):71            Whether this is an encoder/decoder model.72        pad_token_id (`int`, *optional*, defaults to 1)73            Padding token id.74        bos_token_id (`int`, *optional*, defaults to 0)75            Beginning of stream token id.76        eos_token_id (`int`, *optional*, defaults to 2)77            End of stream token id.78        ngram (`int`, *optional*, defaults to 2)79            Number of future tokens to predict. Set to 1 to be same as traditional Language model to predict next first80            token.81        num_buckets (`int`, *optional*, defaults to 32)82            The number of buckets to use for each attention layer. This is for relative position calculation. See the83            [T5 paper](see https://huggingface.co/papers/1910.10683) for more details.84        relative_max_distance (`int`, *optional*, defaults to 128)85            Relative distances greater than this number will be put into the last same bucket. This is for relative86            position calculation. See the [T5 paper](see https://huggingface.co/papers/1910.10683) for more details.87        disable_ngram_loss (`bool`, *optional*, defaults to `False`):88            Whether be trained predicting only the next first token.89        eps (`float`, *optional*, defaults to 0.0):90            Controls the `epsilon` parameter value for label smoothing in the loss calculation. If set to 0, no label91            smoothing is performed.92        use_cache (`bool`, *optional*, defaults to `True`):93            Whether or not the model should return the last key/values attentions (not used by all models).94    """95 96    model_type = "prophetnet"97    keys_to_ignore_at_inference = ["past_key_values"]98    attribute_map = {99        "num_attention_heads": "num_encoder_attention_heads",100    }101 102    def __init__(103        self,104        activation_dropout: Optional[float] = 0.1,105        activation_function: Optional[Union[str, Callable]] = "gelu",106        vocab_size: Optional[int] = 30522,107        hidden_size: Optional[int] = 1024,108        encoder_ffn_dim: Optional[int] = 4096,109        num_encoder_layers: Optional[int] = 12,110        num_encoder_attention_heads: Optional[int] = 16,111        decoder_ffn_dim: Optional[int] = 4096,112        num_decoder_layers: Optional[int] = 12,113        num_decoder_attention_heads: Optional[int] = 16,114        attention_dropout: Optional[float] = 0.1,115        dropout: Optional[float] = 0.1,116        max_position_embeddings: Optional[int] = 512,117        init_std: Optional[float] = 0.02,118        is_encoder_decoder: Optional[bool] = True,119        add_cross_attention: Optional[bool] = True,120        decoder_start_token_id: Optional[int] = 0,121        ngram: Optional[int] = 2,122        num_buckets: Optional[int] = 32,123        relative_max_distance: Optional[int] = 128,124        disable_ngram_loss: Optional[bool] = False,125        eps: Optional[float] = 0.0,126        use_cache: Optional[bool] = True,127        pad_token_id: Optional[int] = 0,128        bos_token_id: Optional[int] = 1,129        eos_token_id: Optional[int] = 2,130        **kwargs,131    ):132        self.vocab_size = vocab_size133        self.hidden_size = hidden_size134        self.encoder_ffn_dim = encoder_ffn_dim135        self.num_encoder_layers = num_encoder_layers136        self.num_encoder_attention_heads = num_encoder_attention_heads137        self.decoder_ffn_dim = decoder_ffn_dim138        self.num_decoder_layers = num_decoder_layers139        self.num_decoder_attention_heads = num_decoder_attention_heads140        self.max_position_embeddings = max_position_embeddings141        self.init_std = init_std  # Normal(0, this parameter)142        self.activation_function = activation_function143 144        # parameters for prophetnet145        self.ngram = ngram146        self.num_buckets = num_buckets147        self.relative_max_distance = relative_max_distance148        self.disable_ngram_loss = disable_ngram_loss149        self.eps = eps150 151        # 3 Types of Dropout152        self.attention_dropout = attention_dropout153        self.activation_dropout = activation_dropout154        self.dropout = dropout155 156        self.use_cache = use_cache157 158        super().__init__(159            pad_token_id=pad_token_id,160            bos_token_id=bos_token_id,161            eos_token_id=eos_token_id,162            is_encoder_decoder=is_encoder_decoder,163            add_cross_attention=add_cross_attention,164            decoder_start_token_id=decoder_start_token_id,165            **kwargs,166        )167 168    @property169    def num_hidden_layers(self) -> int:170        return self.num_encoder_layers171 172    @num_hidden_layers.setter173    def num_hidden_layers(self, value):174        raise NotImplementedError(175            "This model does not support the setting of `num_hidden_layers`. Please set `num_encoder_layers` and"176            " `num_decoder_layers`."177        )178 179 180__all__ = ["ProphetNetConfig"]181