Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2020 The Allen Institute for AI team and The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""Longformer configuration"""16 17from collections import OrderedDict18from collections.abc import Mapping19from typing import TYPE_CHECKING, Any, Optional, Union20 21from ...configuration_utils import PretrainedConfig22from ...onnx import OnnxConfig23from ...utils import TensorType, logging24 25 26if TYPE_CHECKING:27    from ...onnx.config import PatchingSpec28    from ...tokenization_utils_base import PreTrainedTokenizerBase29 30 31logger = logging.get_logger(__name__)32 33 34class LongformerConfig(PretrainedConfig):35    r"""36    This is the configuration class to store the configuration of a [`LongformerModel`] or a [`TFLongformerModel`]. It37    is used to instantiate a Longformer model according to the specified arguments, defining the model architecture.38 39    This is the configuration class to store the configuration of a [`LongformerModel`]. It is used to instantiate an40    Longformer model according to the specified arguments, defining the model architecture. Instantiating a41    configuration with the defaults will yield a similar configuration to that of the LongFormer42    [allenai/longformer-base-4096](https://huggingface.co/allenai/longformer-base-4096) architecture with a sequence43    length 4,096.44 45    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the46    documentation from [`PretrainedConfig`] for more information.47 48    Args:49        vocab_size (`int`, *optional*, defaults to 30522):50            Vocabulary size of the Longformer model. Defines the number of different tokens that can be represented by51            the `inputs_ids` passed when calling [`LongformerModel`] or [`TFLongformerModel`].52        hidden_size (`int`, *optional*, defaults to 768):53            Dimensionality of the encoder layers and the pooler layer.54        num_hidden_layers (`int`, *optional*, defaults to 12):55            Number of hidden layers in the Transformer encoder.56        num_attention_heads (`int`, *optional*, defaults to 12):57            Number of attention heads for each attention layer in the Transformer encoder.58        intermediate_size (`int`, *optional*, defaults to 3072):59            Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder.60        hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu"`):61            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,62            `"relu"`, `"silu"` and `"gelu_new"` are supported.63        hidden_dropout_prob (`float`, *optional*, defaults to 0.1):64            The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.65        attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):66            The dropout ratio for the attention probabilities.67        max_position_embeddings (`int`, *optional*, defaults to 512):68            The maximum sequence length that this model might ever be used with. Typically set this to something large69            just in case (e.g., 512 or 1024 or 2048).70        type_vocab_size (`int`, *optional*, defaults to 2):71            The vocabulary size of the `token_type_ids` passed when calling [`LongformerModel`] or72            [`TFLongformerModel`].73        initializer_range (`float`, *optional*, defaults to 0.02):74            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.75        layer_norm_eps (`float`, *optional*, defaults to 1e-12):76            The epsilon used by the layer normalization layers.77        attention_window (`int` or `list[int]`, *optional*, defaults to 512):78            Size of an attention window around each token. If an `int`, use the same size for all layers. To specify a79            different window size for each layer, use a `list[int]` where `len(attention_window) == num_hidden_layers`.80 81    Example:82 83    ```python84    >>> from transformers import LongformerConfig, LongformerModel85 86    >>> # Initializing a Longformer configuration87    >>> configuration = LongformerConfig()88 89    >>> # Initializing a model from the configuration90    >>> model = LongformerModel(configuration)91 92    >>> # Accessing the model configuration93    >>> configuration = model.config94    ```"""95 96    model_type = "longformer"97 98    def __init__(99        self,100        attention_window: Union[list[int], int] = 512,101        sep_token_id: int = 2,102        pad_token_id: int = 1,103        bos_token_id: int = 0,104        eos_token_id: int = 2,105        vocab_size: int = 30522,106        hidden_size: int = 768,107        num_hidden_layers: int = 12,108        num_attention_heads: int = 12,109        intermediate_size: int = 3072,110        hidden_act: str = "gelu",111        hidden_dropout_prob: float = 0.1,112        attention_probs_dropout_prob: float = 0.1,113        max_position_embeddings: int = 512,114        type_vocab_size: int = 2,115        initializer_range: float = 0.02,116        layer_norm_eps: float = 1e-12,117        onnx_export: bool = False,118        **kwargs,119    ):120        """Constructs LongformerConfig."""121        super().__init__(pad_token_id=pad_token_id, **kwargs)122 123        self.attention_window = attention_window124        self.sep_token_id = sep_token_id125        self.bos_token_id = bos_token_id126        self.eos_token_id = eos_token_id127        self.vocab_size = vocab_size128        self.hidden_size = hidden_size129        self.num_hidden_layers = num_hidden_layers130        self.num_attention_heads = num_attention_heads131        self.hidden_act = hidden_act132        self.intermediate_size = intermediate_size133        self.hidden_dropout_prob = hidden_dropout_prob134        self.attention_probs_dropout_prob = attention_probs_dropout_prob135        self.max_position_embeddings = max_position_embeddings136        self.type_vocab_size = type_vocab_size137        self.initializer_range = initializer_range138        self.layer_norm_eps = layer_norm_eps139        self.onnx_export = onnx_export140 141 142class LongformerOnnxConfig(OnnxConfig):143    def __init__(144        self, config: "PretrainedConfig", task: str = "default", patching_specs: "Optional[list[PatchingSpec]]" = None145    ):146        super().__init__(config, task, patching_specs)147        config.onnx_export = True148 149    @property150    def inputs(self) -> Mapping[str, Mapping[int, str]]:151        if self.task == "multiple-choice":152            dynamic_axis = {0: "batch", 1: "choice", 2: "sequence"}153        else:154            dynamic_axis = {0: "batch", 1: "sequence"}155        return OrderedDict(156            [157                ("input_ids", dynamic_axis),158                ("attention_mask", dynamic_axis),159                ("global_attention_mask", dynamic_axis),160            ]161        )162 163    @property164    def outputs(self) -> Mapping[str, Mapping[int, str]]:165        outputs = super().outputs166        if self.task == "default":167            outputs["pooler_output"] = {0: "batch"}168        return outputs169 170    @property171    def atol_for_validation(self) -> float:172        """173        What absolute tolerance value to use during model conversion validation.174 175        Returns:176            Float absolute tolerance value.177        """178        return 1e-4179 180    @property181    def default_onnx_opset(self) -> int:182        # needs to be >= 14 to support tril operator183        return max(super().default_onnx_opset, 14)184 185    def generate_dummy_inputs(186        self,187        tokenizer: "PreTrainedTokenizerBase",188        batch_size: int = -1,189        seq_length: int = -1,190        is_pair: bool = False,191        framework: Optional[TensorType] = None,192    ) -> Mapping[str, Any]:193        inputs = super().generate_dummy_inputs(194            preprocessor=tokenizer, batch_size=batch_size, seq_length=seq_length, is_pair=is_pair, framework=framework195        )196        import torch197 198        # for some reason, replacing this code by inputs["global_attention_mask"] = torch.randint(2, inputs["input_ids"].shape, dtype=torch.int64)199        # makes the export fail randomly200        inputs["global_attention_mask"] = torch.zeros_like(inputs["input_ids"])201        # make every second token global202        inputs["global_attention_mask"][:, ::2] = 1203 204        return inputs205 206 207__all__ = ["LongformerConfig", "LongformerOnnxConfig"]208