Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2022 EleutherAI and The HuggingFace Inc. team. All rights reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""GPTNeoX model configuration"""16 17from ...configuration_utils import PretrainedConfig18from ...modeling_rope_utils import rope_config_validation19from ...utils import logging20 21 22logger = logging.get_logger(__name__)23 24 25class GPTNeoXConfig(PretrainedConfig):26    r"""27    This is the configuration class to store the configuration of a [`GPTNeoXModel`]. It is used to instantiate an28    GPTNeoX model according to the specified arguments, defining the model architecture. Instantiating a configuration29    with the defaults will yield a similar configuration to that of the GPTNeoX30    [EleutherAI/gpt-neox-20b](https://huggingface.co/EleutherAI/gpt-neox-20b) architecture.31 32    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the33    documentation from [`PretrainedConfig`] for more information.34 35 36    Args:37        vocab_size (`int`, *optional*, defaults to 50432):38            Vocabulary size of the GPTNeoX model. Defines the number of different tokens that can be represented by the39            `inputs_ids` passed when calling [`GPTNeoXModel`].40        hidden_size (`int`, *optional*, defaults to 6144):41            Dimension of the encoder layers and the pooler layer.42        num_hidden_layers (`int`, *optional*, defaults to 44):43            Number of hidden layers in the Transformer encoder.44        num_attention_heads (`int`, *optional*, defaults to 64):45            Number of attention heads for each attention layer in the Transformer encoder.46        intermediate_size (`int`, *optional*, defaults to 24576):47            Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.48        hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):49            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,50            `"relu"`, `"selu"` and `"gelu_new"` are supported.51        rotary_pct (`float`, *optional*, defaults to 0.25):52            percentage of hidden dimensions to allocate to rotary embeddings53        rotary_emb_base (`int`, *optional*, defaults to 10000)54            base for computing rotary embeddings frequency55        attention_dropout (`float`, *optional*, defaults to 0.0):56            The dropout ratio probability of the attention score.57        hidden_dropout (`float`, *optional*, defaults to 0.0):58            The dropout ratio of (1) the word embeddings, (2) the post-attention hidden states, and (3) the post-mlp59            hidden states.60        classifier_dropout (`float`, *optional*, defaults to 0.1):61            Argument used when doing token classification, used in the model [`GPTNeoXForTokenClassification`].62 63            The dropout ratio for the hidden layer.64        max_position_embeddings (`int`, *optional*, defaults to 2048):65            The maximum sequence length that this model might ever be used with. Typically set this to something large66            just in case (e.g., 512 or 1024 or 2048).67        initializer_range (`float`, *optional*, defaults to 1e-5):68            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.69        layer_norm_eps (`float`, *optional*, defaults to 1e-12):70            The epsilon used by the layer normalization layers.71        use_cache (`bool`, *optional*, defaults to `True`):72            Whether or not the model should return the last key/values attentions (not used by all models). Only73            relevant if `config.is_decoder=True`.74        use_parallel_residual (`bool`, *optional*, defaults to `True`):75            Whether to use a "parallel" formulation in each Transformer layer, which can provide a slight training76            speedup at large scales (e.g. 20B).77        rope_scaling (`Dict`, *optional*):78            Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type79            and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value80            accordingly.81            Expected contents:82                `rope_type` (`str`):83                    The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',84                    'llama3'], with 'default' being the original RoPE implementation.85                `factor` (`float`, *optional*):86                    Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In87                    most scaling types, a `factor` of x will enable the model to handle sequences of length x *88                    original maximum pre-trained length.89                `original_max_position_embeddings` (`int`, *optional*):90                    Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during91                    pretraining.92                `attention_factor` (`float`, *optional*):93                    Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention94                    computation. If unspecified, it defaults to value recommended by the implementation, using the95                    `factor` field to infer the suggested value.96                `beta_fast` (`float`, *optional*):97                    Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear98                    ramp function. If unspecified, it defaults to 32.99                `beta_slow` (`float`, *optional*):100                    Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear101                    ramp function. If unspecified, it defaults to 1.102                `short_factor` (`list[float]`, *optional*):103                    Only used with 'longrope'. The scaling factor to be applied to short contexts (<104                    `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden105                    size divided by the number of attention heads divided by 2106                `long_factor` (`list[float]`, *optional*):107                    Only used with 'longrope'. The scaling factor to be applied to long contexts (<108                    `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden109                    size divided by the number of attention heads divided by 2110                `low_freq_factor` (`float`, *optional*):111                    Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE112                `high_freq_factor` (`float`, *optional*):113                    Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE114        attention_bias (`bool`, *optional*, defaults to `True`):115            Whether to use a bias in the query, key, value and output projection layers during self-attention.116 117        Example:118 119    ```python120    >>> from transformers import GPTNeoXConfig, GPTNeoXModel121 122    >>> # Initializing a GPTNeoX gpt-neox-20b style configuration123    >>> configuration = GPTNeoXConfig()124 125    >>> # Initializing a model (with random weights) from the gpt-neox-20b style configuration126    >>> model = GPTNeoXModel(configuration)  # doctest: +SKIP127 128    >>> # Accessing the model configuration129    >>> configuration = model.config  # doctest: +SKIP130    ```"""131 132    model_type = "gpt_neox"133    keys_to_ignore_at_inference = ["past_key_values"]134    base_model_tp_plan = {135        "layers.*.attention.query_key_value": "colwise",136        "layers.*.attention.dense": "rowwise",137        "layers.*.mlp.dense_h_to_4h": "colwise",138        "layers.*.mlp.dense_4h_to_h": "rowwise",139    }140    base_model_pp_plan = {141        "embed_in": (["input_ids"], ["inputs_embeds"]),142        "emb_dropout": (["inputs_embeds"], ["hidden_states"]),143        "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),144        "final_layer_norm": (["hidden_states"], ["hidden_states"]),145    }146 147    def __init__(148        self,149        vocab_size=50432,150        hidden_size=6144,151        num_hidden_layers=44,152        num_attention_heads=64,153        intermediate_size=24576,154        hidden_act="gelu",155        rotary_pct=0.25,156        rotary_emb_base=10000,157        attention_dropout=0.0,158        hidden_dropout=0.0,159        classifier_dropout=0.1,160        max_position_embeddings=2048,161        initializer_range=0.02,162        layer_norm_eps=1e-5,163        use_cache=True,164        bos_token_id=0,165        eos_token_id=2,166        tie_word_embeddings=False,167        use_parallel_residual=True,168        rope_scaling=None,169        attention_bias=True,170        **kwargs,171    ):172        super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)173        self.vocab_size = vocab_size174        self.max_position_embeddings = max_position_embeddings175        self.hidden_size = hidden_size176        self.num_hidden_layers = num_hidden_layers177        self.num_attention_heads = num_attention_heads178        self.intermediate_size = intermediate_size179        self.hidden_act = hidden_act180        self.rotary_pct = rotary_pct181        self.partial_rotary_factor = rotary_pct182        self.rotary_emb_base = rotary_emb_base183        self.rope_theta = rotary_emb_base184        self.attention_dropout = attention_dropout185        self.hidden_dropout = hidden_dropout186        self.classifier_dropout = classifier_dropout187        self.initializer_range = initializer_range188        self.layer_norm_eps = layer_norm_eps189        self.use_cache = use_cache190        self.tie_word_embeddings = tie_word_embeddings191        self.use_parallel_residual = use_parallel_residual192        self.rope_scaling = rope_scaling193        self.attention_bias = attention_bias194        # Validate the correctness of rotary position embeddings parameters195        # BC: if there is a 'type' field, move it to 'rope_type'.196        if self.rope_scaling is not None and "type" in self.rope_scaling:197            self.rope_scaling["rope_type"] = self.rope_scaling["type"]198        rope_config_validation(self)199 200        if self.hidden_size % self.num_attention_heads != 0:201            raise ValueError(202                "The hidden size is not divisible by the number of attention heads! Make sure to update them!"203            )204 205 206__all__ = ["GPTNeoXConfig"]207