Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2021 The HuggingFace Inc. team. All rights reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""GPT Neo model configuration"""16 17from collections import OrderedDict18from collections.abc import Mapping19from typing import Any, Optional20 21from ... import PreTrainedTokenizer, TensorType, is_torch_available22from ...configuration_utils import PretrainedConfig23from ...onnx import OnnxConfigWithPast24from ...utils import logging25 26 27logger = logging.get_logger(__name__)28 29 30class GPTNeoConfig(PretrainedConfig):31    r"""32    This is the configuration class to store the configuration of a [`GPTNeoModel`]. It is used to instantiate a GPT33    Neo model according to the specified arguments, defining the model architecture. Instantiating a configuration with34    the defaults will yield a similar configuration to that of the GPTNeo35    [EleutherAI/gpt-neo-1.3B](https://huggingface.co/EleutherAI/gpt-neo-1.3B) architecture.36 37    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the38    documentation from [`PretrainedConfig`] for more information.39 40 41    Args:42        vocab_size (`int`, *optional*, defaults to 50257):43            Vocabulary size of the GPT Neo model. Defines the number of different tokens that can be represented by the44            `inputs_ids` passed when calling [`GPTNeoModel`]. Vocabulary size of the model. Defines the different45            tokens that can be represented by the *inputs_ids* passed to the forward method of [`GPTNeoModel`].46        max_position_embeddings (`int`, *optional*, defaults to 2048):47            The maximum sequence length that this model might ever be used with. Typically set this to something large48            just in case (e.g., 512 or 1024 or 2048).49        hidden_size (`int`, *optional*, defaults to 2048):50            Dimensionality of the encoder layers and the pooler layer.51        num_layers (`int`, *optional*, defaults to 24):52            Number of hidden layers in the Transformer encoder.53        attention_types (`List`, *optional*, defaults to `[[['global', 'local'], 12]]`):54            The type of attention for each layer in a `List` of the following format `[[["attention_type"],55            num_layerss]]` e.g. for a 24 layer model `[[["global"], 24]]` or `[[["global", "local"], 12]]` Choose the56            value of `attention_type` from `["global", "local"]`57        num_heads (`int`, *optional*, defaults to 16):58            Number of attention heads for each attention layer in the Transformer encoder.59        intermediate_size (`int`, *optional*, defaults to 8192):60            Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.61        window_size (`int`, *optional*, defaults to 256):62            The size of the sliding window for local attention.63        activation_function (`str` or `function`, *optional*, defaults to `"gelu_new"`):64            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,65            `"relu"`, `"selu"` and `"gelu_new"` are supported.66        resid_dropout (`float`, *optional*, defaults to 0.0):67            Residual dropout used in the attention pattern.68        embed_dropout (`float`, *optional*, defaults to 0.0):69            The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.70        attention_dropout (`float`, *optional*, defaults to 0.0):71            The dropout ratio for the attention probabilities.72        classifier_dropout (`float`, *optional*, defaults to 0.1):73            Argument used when doing token classification, used in the model [`GPTNeoForTokenClassification`]. The74            dropout ratio for the hidden layer.75        layer_norm_epsilon (`float`, *optional*, defaults to 1e-05):76            The epsilon used by the layer normalization layers.77        initializer_range (`float`, *optional*, defaults to 0.02):78            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.79        use_cache (`bool`, *optional*, defaults to `True`):80            Whether or not the model should return the last key/values attentions (not used by all models). Only81            relevant if `config.is_decoder=True`.82        bos_token_id (`int`, *optional*, defaults to 50256):83            The id of the beginning of sentence token in the vocabulary.84        eos_token_id (`int`, *optional*, defaults to 50256):85            The id of the end of sentence token in the vocabulary.86 87    Example:88 89    ```python90    >>> from transformers import GPTNeoConfig, GPTNeoModel91 92    >>> # Initializing a GPTNeo EleutherAI/gpt-neo-1.3B style configuration93    >>> configuration = GPTNeoConfig()94 95    >>> # Initializing a model (with random weights) from the EleutherAI/gpt-neo-1.3B style configuration96    >>> model = GPTNeoModel(configuration)97 98    >>> # Accessing the model configuration99    >>> configuration = model.config100    ```"""101 102    model_type = "gpt_neo"103    keys_to_ignore_at_inference = ["past_key_values"]104    attribute_map = {"num_attention_heads": "num_heads", "num_hidden_layers": "num_layers"}105 106    def __init__(107        self,108        vocab_size=50257,109        max_position_embeddings=2048,110        hidden_size=2048,111        num_layers=24,112        attention_types=[[["global", "local"], 12]],113        num_heads=16,114        intermediate_size=None,115        window_size=256,116        activation_function="gelu_new",117        resid_dropout=0.0,118        embed_dropout=0.0,119        attention_dropout=0.0,120        classifier_dropout=0.1,121        layer_norm_epsilon=1e-5,122        initializer_range=0.02,123        use_cache=True,124        bos_token_id=50256,125        eos_token_id=50256,126        **kwargs,127    ):128        self.vocab_size = vocab_size129        self.max_position_embeddings = max_position_embeddings130        self.hidden_size = hidden_size131        self.num_layers = num_layers132        self.num_heads = num_heads133        self.intermediate_size = intermediate_size134        self.window_size = window_size135        self.activation_function = activation_function136        self.resid_dropout = resid_dropout137        self.embed_dropout = embed_dropout138        self.attention_dropout = attention_dropout139        self.classifier_dropout = classifier_dropout140        self.layer_norm_epsilon = layer_norm_epsilon141        self.initializer_range = initializer_range142        self.use_cache = use_cache143 144        self.bos_token_id = bos_token_id145        self.eos_token_id = eos_token_id146 147        self.attention_types = attention_types148        self.attention_layers = self.expand_attention_types_params(attention_types)149 150        if len(self.attention_layers) != self.num_layers:151            raise ValueError(152                "Configuration for convolutional module is incorrect. "153                "It is required that `len(config.attention_layers)` == `config.num_layers` "154                f"but is `len(config.attention_layers) = {len(self.attention_layers)}`, "155                f"`config.num_layers = {self.num_layers}`. "156                "`config.attention_layers` is prepared using `config.attention_types`. "157                "Please verify the value of `config.attention_types` argument."158            )159 160        super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)161 162    @staticmethod163    def expand_attention_types_params(attention_types):164        attentions = []165        for item in attention_types:166            for _ in range(item[1]):167                attentions.extend(item[0])168        return attentions169 170 171def custom_unfold(input, dimension, size, step):172    """Custom torch.Tensor.unfold implementation to enable the export to ONNX."""173    import torch174 175    shape = input.size()176    rank = len(shape)177    sizedim = shape[dimension]178 179    low_indices = torch.arange(0, sizedim, step)180    min_length = torch.div(sizedim - size, step, rounding_mode="floor") + 1181    indices = torch.arange(size) + low_indices[:min_length][:, None]182 183    s = [slice(None)] * rank184    s[dimension] = indices185    sliced = input[s]186 187    perm = list(range(0, rank + 1))188    perm.append(perm.pop(dimension + 1))189 190    return sliced.permute(perm)191 192 193def custom_get_block_length_and_num_blocks(seq_length, window_size):194    """195    Custom implementation for GPTNeoAttentionMixin._get_block_length_and_num_blocks to enable the export to ONNX as196    original implementation uses Python variables and control flow.197    """198    import torch199 200    candidates = torch.arange(1, window_size)201    remainders = torch.remainder(seq_length, candidates)202    divisor_indices = remainders == 0203    divisors = candidates[divisor_indices]204    largest_divisor = torch.max(divisors)205    return largest_divisor, torch.div(seq_length, largest_divisor, rounding_mode="floor")206 207 208class GPTNeoOnnxConfig(OnnxConfigWithPast):209    @property210    def inputs(self) -> Mapping[str, Mapping[int, str]]:211        common_inputs = OrderedDict({"input_ids": {0: "batch", 1: "sequence"}})212        if self.use_past:213            self.fill_with_past_key_values_(common_inputs, direction="inputs")214            common_inputs["attention_mask"] = {0: "batch", 1: "past_sequence + sequence"}215        else:216            common_inputs["attention_mask"] = {0: "batch", 1: "sequence"}217 218        return common_inputs219 220    @property221    def num_attention_heads(self) -> int:222        return self._config.num_heads223 224    def generate_dummy_inputs(225        self,226        tokenizer: PreTrainedTokenizer,227        batch_size: int = -1,228        seq_length: int = -1,229        is_pair: bool = False,230        framework: Optional[TensorType] = None,231    ) -> Mapping[str, Any]:232        common_inputs = super(OnnxConfigWithPast, self).generate_dummy_inputs(233            tokenizer, batch_size=batch_size, seq_length=seq_length, is_pair=is_pair, framework=framework234        )235 236        # We need to order the input in the way they appears in the forward()237        ordered_inputs = OrderedDict({"input_ids": common_inputs["input_ids"]})238 239        # Need to add the past_keys240        if self.use_past:241            if not is_torch_available():242                raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed.")243            else:244                import torch245 246                batch, seqlen = common_inputs["input_ids"].shape247                # Not using the same length for past_key_values248                past_key_values_length = seqlen + 2249                past_shape = (250                    batch,251                    self.num_attention_heads,252                    past_key_values_length,253                    self._config.hidden_size // self.num_attention_heads,254                )255                ordered_inputs["past_key_values"] = [256                    (torch.zeros(past_shape), torch.zeros(past_shape)) for _ in range(self.num_layers)257                ]258 259        ordered_inputs["attention_mask"] = common_inputs["attention_mask"]260        if self.use_past:261            mask_dtype = ordered_inputs["attention_mask"].dtype262            ordered_inputs["attention_mask"] = torch.cat(263                [ordered_inputs["attention_mask"], torch.ones(batch, past_key_values_length, dtype=mask_dtype)], dim=1264            )265 266        return ordered_inputs267 268    @property269    def default_onnx_opset(self) -> int:270        return 13271 272 273__all__ = ["GPTNeoConfig", "GPTNeoOnnxConfig"]274