Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2023 The HuggingFace Inc. team. All rights reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""Pix2Struct model configuration"""16 17from ...configuration_utils import PretrainedConfig18from ...utils import logging19 20 21logger = logging.get_logger(__name__)22 23 24class Pix2StructTextConfig(PretrainedConfig):25    r"""26    This is the configuration class to store the configuration of a [`Pix2StructTextModel`]. It is used to instantiate27    a Pix2Struct text model according to the specified arguments, defining the model architecture. Instantiating a28    configuration with the defaults will yield a similar configuration to that of the Pix2Struct text decoder used by29    the [google/pix2struct-base](https://huggingface.co/google/pix2struct-base) architecture.30 31    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the32    documentation from [`PretrainedConfig`] for more information.33 34    Args:35        vocab_size (`int`, *optional*, defaults to 50244):36            Vocabulary size of the `Pix2Struct` text model. Defines the number of different tokens that can be37            represented by the `inputs_ids` passed when calling [`Pix2StructTextModel`].38        hidden_size (`int`, *optional*, defaults to 768):39            Dimensionality of the encoder layers and the pooler layer.40        d_kv (`int`, *optional*, defaults to 64):41            Dimensionality of the key, query, value projections in each attention head.42        d_ff (`int`, *optional*, defaults to 2048):43            Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.44        num_layers (`int`, *optional*, defaults to 12):45            Number of hidden layers in the Transformer encoder.46        num_heads (`int`, *optional*, defaults to 12):47            Number of attention heads for each attention layer in the Transformer encoder.48        relative_attention_num_buckets (`int`, *optional*, defaults to 32):49            The number of buckets to use for each attention layer.50        relative_attention_max_distance (`int`, *optional*, defaults to 128):51            The maximum distance of the longer sequences for the bucket separation.52        dropout_rate (`float`, *optional*, defaults to 0.1):53            The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.54        layer_norm_epsilon (`float`, *optional*, defaults to 1e-6):55            The epsilon used by the layer normalization layers.56        initializer_factor (`float`, *optional*, defaults to 1.0):57            A factor for initializing all weight matrices (should be kept to 1, used internally for initialization58            testing).59        dense_act_fn (`Union[Callable, str]`, *optional*, defaults to `"gelu_new"`):60            The non-linear activation function (function or string).61        decoder_start_token_id (`int`, *optional*, defaults to 0):62            The id of the `decoder_start_token_id` token.63        use_cache (`bool`, *optional*, defaults to `False`):64            Whether or not the model should return the last key/values attentions (not used by all models).65        pad_token_id (`int`, *optional*, defaults to 0):66            The id of the `padding` token.67        eos_token_id (`int`, *optional*, defaults to 1):68            The id of the `end-of-sequence` token.69 70    Example:71 72    ```python73    >>> from transformers import Pix2StructTextConfig, Pix2StructTextModel74 75    >>> # Initializing a Pix2StructTextConfig with google/pix2struct-base style configuration76    >>> configuration = Pix2StructTextConfig()77 78    >>> # Initializing a Pix2StructTextModel (with random weights) from the google/pix2struct-base style configuration79    >>> model = Pix2StructTextModel(configuration)80 81    >>> # Accessing the model configuration82    >>> configuration = model.config83    ```"""84 85    model_type = "pix2struct_text_model"86    keys_to_ignore_at_inference = ["past_key_values"]87    attribute_map = {88        "hidden_size": "hidden_size",89        "num_attention_heads": "num_heads",90        "num_hidden_layers": "num_layers",91        "decoder_attention_heads": "num_heads",92        "encoder_attention_heads": "num_heads",93        "encoder_layers": "num_layers",94        "decoder_layers": "num_layers",95    }96 97    def __init__(98        self,99        vocab_size=50244,100        hidden_size=768,101        d_kv=64,102        d_ff=2048,103        num_layers=12,104        num_heads=12,105        relative_attention_num_buckets=32,106        relative_attention_max_distance=128,107        dropout_rate=0.1,108        layer_norm_epsilon=1e-6,109        initializer_factor=1.0,110        dense_act_fn="gelu_new",111        decoder_start_token_id=0,112        use_cache=False,113        pad_token_id=0,114        eos_token_id=1,115        tie_word_embeddings=False,116        is_decoder=True,117        **kwargs,118    ):119        self.vocab_size = vocab_size120        self.hidden_size = hidden_size121        self.d_kv = d_kv122        self.d_ff = d_ff123        self.num_layers = num_layers124        self.num_heads = num_heads125        self.relative_attention_num_buckets = relative_attention_num_buckets126        self.relative_attention_max_distance = relative_attention_max_distance127        self.dropout_rate = dropout_rate128        self.layer_norm_epsilon = layer_norm_epsilon129        self.initializer_factor = initializer_factor130        self.use_cache = use_cache131 132        self.eos_token_id = eos_token_id133        self.decoder_start_token_id = decoder_start_token_id134 135        # for backwards compatibility136        self.dense_act_fn = dense_act_fn137 138        super().__init__(139            pad_token_id=pad_token_id,140            eos_token_id=eos_token_id,141            decoder_start_token_id=decoder_start_token_id,142            tie_word_embeddings=tie_word_embeddings,143            is_decoder=is_decoder,144            **kwargs,145        )146 147 148class Pix2StructVisionConfig(PretrainedConfig):149    r"""150    This is the configuration class to store the configuration of a [`Pix2StructVisionModel`]. It is used to151    instantiate a Pix2Struct vision model according to the specified arguments, defining the model architecture.152    Instantiating a configuration defaults will yield a similar configuration to that of the Pix2Struct-base153    [google/pix2struct-base](https://huggingface.co/google/pix2struct-base) architecture.154 155    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the156    documentation from [`PretrainedConfig`] for more information.157 158    Args:159        hidden_size (`int`, *optional*, defaults to 768):160            Dimensionality of the encoder layers and the pooler layer.161        patch_embed_hidden_size (`int`, *optional*, defaults to 768):162            Dimensionality of the input patch_embedding layer in the Transformer encoder.163        d_ff (`int`, *optional*, defaults to 2048):164            Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.165        d_kv (`int`, *optional*, defaults to 64):166            Dimensionality of the key, query, value projections per attention head.167        num_hidden_layers (`int`, *optional*, defaults to 12):168            Number of hidden layers in the Transformer encoder.169        num_attention_heads (`int`, *optional*, defaults to 12):170            Number of attention heads for each attention layer in the Transformer encoder.171        dense_act_fn (`str` or `function`, *optional*, defaults to `"gelu_new"`):172            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,173            `"relu"`, `"selu"` and `"gelu_new"` `"gelu"` are supported.174        layer_norm_eps (`float`, *optional*, defaults to 1e-06):175            The epsilon used by the layer normalization layers.176        dropout_rate (`float`, *optional*, defaults to 0.0):177            The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.178        attention_dropout (`float`, *optional*, defaults to 0.0):179            The dropout ratio for the attention probabilities.180        initializer_range (`float`, *optional*, defaults to 1e-10):181            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.182        initializer_factor (`float`, *optional*, defaults to 1.0):183            A factor for initializing all weight matrices (should be kept to 1, used internally for initialization184            testing).185        seq_len (`int`, *optional*, defaults to 4096):186            Maximum sequence length (here number of patches) supported by the model.187        relative_attention_num_buckets (`int`, *optional*, defaults to 32):188            The number of buckets to use for each attention layer.189        relative_attention_max_distance (`int`, *optional*, defaults to 128):190            The maximum distance (in tokens) to use for each attention layer.191 192    Example:193 194    ```python195    >>> from transformers import Pix2StructVisionConfig, Pix2StructVisionModel196 197    >>> # Initializing a Pix2StructVisionConfig with google/pix2struct-base style configuration198    >>> configuration = Pix2StructVisionConfig()199 200    >>> # Initializing a Pix2StructVisionModel (with random weights) from the google/pix2struct-base style configuration201    >>> model = Pix2StructVisionModel(configuration)202 203    >>> # Accessing the model configuration204    >>> configuration = model.config205    ```"""206 207    model_type = "pix2struct_vision_model"208 209    def __init__(210        self,211        hidden_size=768,212        patch_embed_hidden_size=768,213        d_ff=2048,214        d_kv=64,215        num_hidden_layers=12,216        num_attention_heads=12,217        dense_act_fn="gelu_new",218        layer_norm_eps=1e-6,219        dropout_rate=0.0,220        attention_dropout=0.0,221        initializer_range=1e-10,222        initializer_factor=1.0,223        seq_len=4096,224        relative_attention_num_buckets=32,225        relative_attention_max_distance=128,226        **kwargs,227    ):228        super().__init__(**kwargs)229 230        self.hidden_size = hidden_size231        self.patch_embed_hidden_size = patch_embed_hidden_size232        self.d_ff = d_ff233        self.dropout_rate = dropout_rate234        self.num_hidden_layers = num_hidden_layers235        self.num_attention_heads = num_attention_heads236        self.initializer_range = initializer_range237        self.initializer_factor = initializer_factor238        self.attention_dropout = attention_dropout239        self.layer_norm_eps = layer_norm_eps240        self.dense_act_fn = dense_act_fn241        self.seq_len = seq_len242        self.relative_attention_num_buckets = relative_attention_num_buckets243        self.relative_attention_max_distance = relative_attention_max_distance244        self.d_kv = d_kv245 246 247class Pix2StructConfig(PretrainedConfig):248    r"""249    [`Pix2StructConfig`] is the configuration class to store the configuration of a250    [`Pix2StructForConditionalGeneration`]. It is used to instantiate a Pix2Struct model according to the specified251    arguments, defining the text model and vision model configs. Instantiating a configuration with the defaults will252    yield a similar configuration to that of the Pix2Struct-base253    [google/pix2struct-base](https://huggingface.co/google/pix2struct-base) architecture.254 255    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the256    documentation from [`PretrainedConfig`] for more information.257 258    Args:259        text_config (`dict`, *optional*):260            Dictionary of configuration options used to initialize [`Pix2StructTextConfig`].261        vision_config (`dict`, *optional*):262            Dictionary of configuration options used to initialize [`Pix2StructVisionConfig`].263        initializer_factor (`float`, *optional*, defaults to 1.0):264            Factor to multiply the initialization range with.265        initializer_range (`float`, *optional*, defaults to 0.02):266            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.267        is_vqa (`bool`, *optional*, defaults to `False`):268            Whether the model has been fine-tuned for VQA or not.269        kwargs (*optional*):270            Dictionary of keyword arguments.271 272    Example:273 274    ```python275    >>> from transformers import Pix2StructConfig, Pix2StructForConditionalGeneration276 277    >>> # Initializing a Pix2StructConfig with google/pix2struct-base style configuration278    >>> configuration = Pix2StructConfig()279 280    >>> # Initializing a Pix2StructForConditionalGeneration (with random weights) from the google/pix2struct-base style configuration281    >>> model = Pix2StructForConditionalGeneration(configuration)282 283    >>> # Accessing the model configuration284    >>> configuration = model.config285 286    >>> # We can also initialize a Pix2StructConfig from a Pix2StructTextConfig and a Pix2StructVisionConfig287 288    >>> # Initializing a Pix2Struct text and Pix2Struct vision configuration289    >>> config_text = Pix2StructTextConfig()290    >>> config_vision = Pix2StructVisionConfig()291 292    >>> config = Pix2StructConfig.from_text_vision_configs(config_text, config_vision)293    ```"""294 295    model_type = "pix2struct"296    sub_configs = {"text_config": Pix2StructTextConfig, "vision_config": Pix2StructVisionConfig}297 298    def __init__(299        self,300        text_config=None,301        vision_config=None,302        initializer_factor=1.0,303        initializer_range=0.02,304        is_vqa=False,305        tie_word_embeddings=False,306        is_encoder_decoder=True,307        **kwargs,308    ):309        super().__init__(tie_word_embeddings=tie_word_embeddings, is_encoder_decoder=is_encoder_decoder, **kwargs)310 311        if text_config is None:312            text_config = {}313            logger.info("text_config is None. Initializing the Pix2StructTextConfig with default values.")314 315        if vision_config is None:316            vision_config = {}317            logger.info("vision_config is None. Initializing the Pix2StructVisionConfig with default values.")318 319        text_config["is_encoder_decoder"] = is_encoder_decoder320        text_config["tie_word_embeddings"] = tie_word_embeddings321        self.text_config = Pix2StructTextConfig(**text_config)322        self.vision_config = Pix2StructVisionConfig(**vision_config)323 324        self.decoder_start_token_id = self.text_config.decoder_start_token_id325        self.pad_token_id = self.text_config.pad_token_id326        self.eos_token_id = self.text_config.eos_token_id327 328        self.initializer_factor = initializer_factor329        self.initializer_range = initializer_range330 331        self.text_config.initializer_range = self.initializer_range332        self.vision_config.initializer_range = self.initializer_range333 334        self.is_vqa = is_vqa335 336 337__all__ = ["Pix2StructConfig", "Pix2StructTextConfig", "Pix2StructVisionConfig"]338