Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.3#4# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX5# and OPT implementations in this library. It has been modified from its6# original forms to accommodate minor architectural differences compared7# to GPT-NeoX and OPT used by the Meta AI team that trained the model.8#9# Licensed under the Apache License, Version 2.0 (the "License");10# you may not use this file except in compliance with the License.11# You may obtain a copy of the License at12#13#     http://www.apache.org/licenses/LICENSE-2.014#15# Unless required by applicable law or agreed to in writing, software16# distributed under the License is distributed on an "AS IS" BASIS,17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.18# See the License for the specific language governing permissions and19# limitations under the License.20"""Idefics model configuration"""21 22from ...configuration_utils import PretrainedConfig23from ...utils import logging24 25 26logger = logging.get_logger(__name__)27 28 29class IdeficsVisionConfig(PretrainedConfig):30    r"""31    This is the configuration class to store the configuration of a [`IdeficsModel`]. It is used to instantiate an32    Idefics model according to the specified arguments, defining the model architecture. Instantiating a configuration33    with the defaults will yield a similar configuration to that of the Idefics-9B.34 35    e.g. [HuggingFaceM4/idefics-9b](https://huggingface.co/HuggingFaceM4/idefics-9b)36 37    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the38    documentation from [`PretrainedConfig`] for more information.39 40    Args:41        embed_dim (`int`, *optional*, defaults to 768):42            Dimensionality of the encoder layers and the pooler layer. (elsewhere referred to as `hidden_size`)43        image_size (`int`, *optional*, defaults to 224):44            The size (resolution) of each image.45        intermediate_size (`int`, *optional*, defaults to 5120):46            Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.47        patch_size (`int`, *optional*, defaults to 14):48            The size (resolution) of each patch.49        num_hidden_layers (`int`, *optional*, defaults to 32):50            Number of hidden layers in the Transformer encoder.51        num_attention_heads (`int`, *optional*, defaults to 16):52            Number of attention heads for each attention layer in the Transformer encoder.53        num_channels (`int`, *optional*, defaults to 3):54            Number of image channels.55        hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):56            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,57            `"relu"`, `"selu"` and `"gelu_new"` `"quick_gelu"` are supported.58        layer_norm_eps (`float`, *optional*, defaults to 1e-05):59            The epsilon used by the layer normalization layers.60        attention_dropout (`float`, *optional*, defaults to 0.0):61            The dropout ratio for the attention probabilities.62        initializer_range (`float`, *optional*, defaults to 0.02):63            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.64        initializer_factor (`float`, *optional*, defaults to 1.0):65            A factor for initializing all weight matrices (should be kept to 1.0, used internally for initialization66            testing).67    """68 69    model_type = "idefics_vision"70    attribute_map = {71        "hidden_size": "embed_dim",72    }73 74    def __init__(75        self,76        embed_dim=768,77        image_size=224,78        intermediate_size=5120,79        patch_size=14,80        num_hidden_layers=32,81        num_attention_heads=16,82        num_channels=3,83        hidden_act="gelu",84        layer_norm_eps=1e-5,85        attention_dropout=0.0,86        initializer_range=0.02,87        initializer_factor=1.0,88        **kwargs,89    ):90        self.embed_dim = embed_dim91        self.image_size = image_size92        self.intermediate_size = intermediate_size93        self.patch_size = patch_size94        self.num_hidden_layers = num_hidden_layers95        self.num_attention_heads = num_attention_heads96        self.num_channels = num_channels97        self.layer_norm_eps = layer_norm_eps98        self.attention_dropout = attention_dropout99        self.initializer_range = initializer_range100        self.initializer_factor = initializer_factor101        self.hidden_act = hidden_act102 103        super().__init__(**kwargs)104 105 106class IdeficsPerceiverConfig(PretrainedConfig):107    r"""108    This is the configuration class to store the configuration of a [`IdeficsModel`]. It is used to instantiate an109    Idefics model according to the specified arguments, defining the model architecture. Instantiating a configuration110    with the defaults will yield a similar configuration to that of the Idefics-9B.111 112    e.g. [HuggingFaceM4/idefics-9b](https://huggingface.co/HuggingFaceM4/idefics-9b)113 114    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the115    documentation from [`PretrainedConfig`] for more information.116 117    Args:118        use_resampler (`bool`, *optional*, defaults to `False`):119            Whether or not to use the resampler120        resampler_n_latents (`int`, *optional*, defaults to 64):121            Number of latent embeddings to resample ("compress") the input sequence to (usually < 128).122        resampler_depth (`int`, *optional*, defaults to 6):123            Depth of the Perceiver Resampler (Transformer w/ cross attention). Should be shallow (< 3).124        resampler_n_heads (`int`, *optional*, defaults to 16):125            Number of heads in each Transformer block (for multi-headed self-attention).126        resampler_head_dim (`int`, *optional*, defaults to 96):127            Dimensionality of each head projection in the Transformer block.128        qk_layer_norms_perceiver (`bool`, *optional*, defaults to `False`):129            Whether or not to use qk layer norms in perceiver130    """131 132    model_type = "idefics_perciever"133 134    def __init__(135        self,136        use_resampler=False,137        resampler_n_latents=64,138        resampler_depth=6,139        resampler_n_heads=16,140        resampler_head_dim=96,141        qk_layer_norms_perceiver=False,142        **kwargs,143    ):144        self.use_resampler = use_resampler145        self.resampler_n_latents = resampler_n_latents146        self.resampler_depth = resampler_depth147        self.resampler_n_heads = resampler_n_heads148        self.resampler_head_dim = resampler_head_dim149        self.qk_layer_norms_perceiver = qk_layer_norms_perceiver150 151        super().__init__(**kwargs)152 153 154class IdeficsConfig(PretrainedConfig):155    r"""156    This is the configuration class to store the configuration of a [`IdeficsModel`]. It is used to instantiate an157    Idefics model according to the specified arguments, defining the model architecture. Instantiating a configuration158    with the defaults will yield a similar configuration to that of the Idefics-9B.159 160    e.g. [HuggingFaceM4/idefics-9b](https://huggingface.co/HuggingFaceM4/idefics-9b)161 162    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the163    documentation from [`PretrainedConfig`] for more information.164 165    Args:166        additional_vocab_size (`int`, *optional*, defaults to 0):167            Additional vocabulary size of the model, typically for the special "<img>" token. Additional vocab tokens168            are always trainable whereas regular vocab tokens can be frozen or not.169        vocab_size (`int`, *optional*, defaults to 32000):170            Vocabulary size of the Idefics model. Defines the number of different tokens that can be represented by the171            `inputs_ids` passed when calling [`~IdeficsModel`]172        hidden_size (`int`, *optional*, defaults to 4096):173            Dimension of the hidden representations.174        intermediate_size (`int`, *optional*, defaults to 11008):175            Dimension of the MLP representations.176        num_hidden_layers (`int`, *optional*, defaults to 32):177            Number of hidden layers in the Transformer encoder.178        num_attention_heads (`int`, *optional*, defaults to 32):179            Number of attention heads for each attention layer in the Transformer encoder.180        dropout (`float`, *optional*, defaults to 0.0):181            The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.182        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):183            The non-linear activation function (function or string) in the decoder.184        initializer_range (`float`, *optional*, defaults to 0.02):185            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.186        alpha_initializer (`str`, *optional*, defaults to `"zeros"`):187            Initialization type for the alphas.188        alphas_initializer_range (`float`, *optional*, defaults to 0.0):189            The standard deviation of the truncated_normal_initializer for initializing the alphas in the Gated Cross190            Attention.191        alpha_type (`str`, *optional*, defaults to `"float"`):192            Whether the gating alphas should be vectors or single floats.193        rms_norm_eps (`float`, *optional*, defaults to 1e-6):194            The epsilon used by the rms normalization layers.195        use_cache (`bool`, *optional*, defaults to `True`):196            Whether or not the model should return the last key/values attentions (not used by all models). Only197            relevant if `config.is_decoder=True`.198        pad_token_id (`int`, *optional*, defaults to 0)199            Padding token id.200        bos_token_id (`int`, *optional*, defaults to 1)201            Beginning of stream token id.202        eos_token_id (`int`, *optional*, defaults to 2)203            End of stream token id.204        tie_word_embeddings(`bool`, *optional*, defaults to `False`):205            Whether to tie weight embeddings206        cross_layer_interval (`int`, *optional*, default to 1)207            Interval for cross attention (from text to image) layers.208        qk_layer_norms (`bool`, *optional*, defaults to `False`): Whether to add layer norm after q and k209        freeze_text_layers (`bool`, *optional*, defaults to `True`): Whether to freeze text layers210        freeze_text_module_exceptions (`bool`, *optional*, defaults to `[]`):211            Exceptions to freezing text layers when `freeze_text_layers` is `True`212        freeze_lm_head (`bool`, *optional*, defaults to `False`): Whether to freeze lm head213        freeze_vision_layers (`bool`, *optional*, defaults to `True`):  Whether to freeze vision layers214        freeze_vision_module_exceptions (`bool`, *optional*, defaults to `[]`):215            Exceptions to freezing vision layers when `freeze_vision_layers` is `True`216        use_resampler (`bool`, *optional*, defaults to `False`): Whether to use the Resampler217        vision_config (`IdeficsVisionConfig`,  *optional*): Custom vision config or dict218        perceiver_config (`IdeficsPerceiverConfig`,  *optional*): Custom perceiver config or dict219 220    Example:221 222    ```python223    >>> from transformers import IdeficsModel, IdeficsConfig224 225    >>> # Initializing a Idefics idefics-9b style configuration226    >>> configuration = IdeficsConfig()227 228    >>> # Initializing a model from the idefics-9b style configuration229    >>> model = IdeficsModel(configuration)230 231    >>> # Accessing the model configuration232    >>> configuration = model.config233    ```"""234 235    model_type = "idefics"236    sub_configs = {"perceiver_config": IdeficsPerceiverConfig, "vision_config": IdeficsVisionConfig}237 238    def __init__(239        self,240        vocab_size=32000,241        additional_vocab_size=0,242        hidden_size=4096,243        intermediate_size=11008,244        num_hidden_layers=32,245        num_attention_heads=32,246        dropout=0.0,247        hidden_act="silu",248        initializer_range=0.02,249        alpha_initializer="zeros",250        alphas_initializer_range=0.0,251        alpha_type="float",252        rms_norm_eps=1e-6,253        use_cache=True,254        pad_token_id=0,255        bos_token_id=1,256        eos_token_id=2,257        tie_word_embeddings=False,258        cross_layer_interval=1,259        qk_layer_norms=False,260        freeze_text_layers=True,261        freeze_text_module_exceptions=[],262        freeze_lm_head=False,263        freeze_vision_layers=True,264        freeze_vision_module_exceptions=[],265        use_resampler=False,266        vision_config=None,267        perceiver_config=None,268        **kwargs,269    ):270        self.vocab_size = vocab_size271        self.additional_vocab_size = additional_vocab_size272        self.hidden_size = hidden_size273        self.intermediate_size = intermediate_size274        self.num_hidden_layers = num_hidden_layers275        self.num_attention_heads = num_attention_heads276        self.dropout = dropout277        self.hidden_act = hidden_act278        self.initializer_range = initializer_range279        self.alpha_initializer = alpha_initializer280        self.alphas_initializer_range = alphas_initializer_range281        self.alpha_type = alpha_type282        self.rms_norm_eps = rms_norm_eps283        self.use_cache = use_cache284 285        self.cross_layer_interval = cross_layer_interval286        self.qk_layer_norms = qk_layer_norms287        self.freeze_vision_layers = freeze_vision_layers288 289        self.freeze_text_layers = freeze_text_layers290        self.freeze_text_module_exceptions = freeze_text_module_exceptions291        self.freeze_vision_module_exceptions = freeze_vision_module_exceptions292        self.freeze_lm_head = freeze_lm_head293 294        self.use_resampler = use_resampler295 296        if perceiver_config is None:297            self.perceiver_config = IdeficsPerceiverConfig()298        elif isinstance(perceiver_config, dict):299            self.perceiver_config = IdeficsPerceiverConfig(**perceiver_config)300        elif isinstance(perceiver_config, IdeficsPerceiverConfig):301            self.perceiver_config = perceiver_config302 303        if vision_config is None:304            self.vision_config = IdeficsVisionConfig()305        elif isinstance(vision_config, dict):306            self.vision_config = IdeficsVisionConfig(**vision_config)307        elif isinstance(vision_config, IdeficsVisionConfig):308            self.vision_config = vision_config309 310        super().__init__(311            pad_token_id=pad_token_id,312            bos_token_id=bos_token_id,313            eos_token_id=eos_token_id,314            tie_word_embeddings=tie_word_embeddings,315            **kwargs,316        )317 318        # IMPORTANT: Do not do any __init__ args-based checks in the constructor, since319        # PretrainedConfig.from_dict first instantiates the class with the config dict and only then320        # updates the config object with `kwargs` from from_pretrained, so during the instantiation321        # of this object many attributes have default values and haven't yet been overridden.322        # Do any required checks inside `from_pretrained` once the superclass' `from_pretrained` was run.323 324 325__all__ = ["IdeficsConfig"]326