Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2023 The HuggingFace Inc. team. All rights reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""BLIP-2 model configuration"""16 17from typing import Optional18 19from ...configuration_utils import PretrainedConfig20from ...models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES21from ...utils import logging22from ..auto import CONFIG_MAPPING, AutoConfig23 24 25logger = logging.get_logger(__name__)26 27 28class Blip2VisionConfig(PretrainedConfig):29    r"""30    This is the configuration class to store the configuration of a [`Blip2VisionModel`]. It is used to instantiate a31    BLIP-2 vision encoder according to the specified arguments, defining the model architecture. Instantiating a32    configuration defaults will yield a similar configuration to that of the BLIP-233    [Salesforce/blip2-opt-2.7b](https://huggingface.co/Salesforce/blip2-opt-2.7b) architecture.34 35    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the36    documentation from [`PretrainedConfig`] for more information.37 38    Args:39        hidden_size (`int`, *optional*, defaults to 1408):40            Dimensionality of the encoder layers and the pooler layer.41        intermediate_size (`int`, *optional*, defaults to 6144):42            Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.43        num_hidden_layers (`int`, *optional*, defaults to 39):44            Number of hidden layers in the Transformer encoder.45        num_attention_heads (`int`, *optional*, defaults to 16):46            Number of attention heads for each attention layer in the Transformer encoder.47        image_size (`int`, *optional*, defaults to 224):48            The size (resolution) of each image.49        patch_size (`int`, *optional*, defaults to 14):50            The size (resolution) of each patch.51        hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):52            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,53            `"relu"`, `"selu"` and `"gelu_new"` `"gelu"` are supported. layer_norm_eps (`float`, *optional*, defaults54            to 1e-5): The epsilon used by the layer normalization layers.55        attention_dropout (`float`, *optional*, defaults to 0.0):56            The dropout ratio for the attention probabilities.57        initializer_range (`float`, *optional*, defaults to 0.02):58            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.59        qkv_bias (`bool`, *optional*, defaults to `True`):60            Whether to add a bias to the queries and values in the self-attention layers.61 62    Example:63 64    ```python65    >>> from transformers import Blip2VisionConfig, Blip2VisionModel66 67    >>> # Initializing a Blip2VisionConfig with Salesforce/blip2-opt-2.7b style configuration68    >>> configuration = Blip2VisionConfig()69 70    >>> # Initializing a Blip2VisionModel (with random weights) from the Salesforce/blip2-opt-2.7b style configuration71    >>> model = Blip2VisionModel(configuration)72 73    >>> # Accessing the model configuration74    >>> configuration = model.config75    ```"""76 77    model_type = "blip_2_vision_model"78    base_config_key = "vision_config"79 80    def __init__(81        self,82        hidden_size=1408,83        intermediate_size=6144,84        num_hidden_layers=39,85        num_attention_heads=16,86        image_size=224,87        patch_size=14,88        hidden_act="gelu",89        layer_norm_eps=1e-6,90        attention_dropout=0.0,91        initializer_range=1e-10,92        qkv_bias=True,93        **kwargs,94    ):95        super().__init__(**kwargs)96 97        self.hidden_size = hidden_size98        self.intermediate_size = intermediate_size99        self.num_hidden_layers = num_hidden_layers100        self.num_attention_heads = num_attention_heads101        self.patch_size = patch_size102        self.image_size = image_size103        self.initializer_range = initializer_range104        self.attention_dropout = attention_dropout105        self.layer_norm_eps = layer_norm_eps106        self.hidden_act = hidden_act107        self.qkv_bias = qkv_bias108 109 110class Blip2QFormerConfig(PretrainedConfig):111    r"""112    This is the configuration class to store the configuration of a [`Blip2QFormerModel`]. It is used to instantiate a113    BLIP-2 Querying Transformer (Q-Former) model according to the specified arguments, defining the model architecture.114    Instantiating a configuration with the defaults will yield a similar configuration to that of the BLIP-2115    [Salesforce/blip2-opt-2.7b](https://huggingface.co/Salesforce/blip2-opt-2.7b) architecture. Configuration objects116    inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the documentation from117    [`PretrainedConfig`] for more information.118 119    Note that [`Blip2QFormerModel`] is very similar to [`BertLMHeadModel`] with interleaved cross-attention.120 121    Args:122        vocab_size (`int`, *optional*, defaults to 30522):123            Vocabulary size of the Q-Former model. Defines the number of different tokens that can be represented by124            the `inputs_ids` passed when calling the model.125        hidden_size (`int`, *optional*, defaults to 768):126            Dimensionality of the encoder layers and the pooler layer.127        num_hidden_layers (`int`, *optional*, defaults to 12):128            Number of hidden layers in the Transformer encoder.129        num_attention_heads (`int`, *optional*, defaults to 12):130            Number of attention heads for each attention layer in the Transformer encoder.131        intermediate_size (`int`, *optional*, defaults to 3072):132            Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder.133        hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu"`):134            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,135            `"relu"`, `"silu"` and `"gelu_new"` are supported.136        hidden_dropout_prob (`float`, *optional*, defaults to 0.1):137            The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.138        attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):139            The dropout ratio for the attention probabilities.140        max_position_embeddings (`int`, *optional*, defaults to 512):141            The maximum sequence length that this model might ever be used with. Typically set this to something large142            just in case (e.g., 512 or 1024 or 2048).143        initializer_range (`float`, *optional*, defaults to 0.02):144            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.145        layer_norm_eps (`float`, *optional*, defaults to 1e-12):146            The epsilon used by the layer normalization layers.147        pad_token_id (`int`, *optional*, defaults to 0):148            Index to be used for padding token.149        position_embedding_type (`str`, *optional*, defaults to `"absolute"`):150            Type of position embedding. Choose one of `"absolute"`, `"relative_key"`, `"relative_key_query"`. For151            positional embeddings use `"absolute"`. For more information on `"relative_key"`, please refer to152            [Self-Attention with Relative Position Representations (Shaw et al.)](https://huggingface.co/papers/1803.02155).153            For more information on `"relative_key_query"`, please refer to *Method 4* in [Improve Transformer Models154            with Better Relative Position Embeddings (Huang et al.)](https://huggingface.co/papers/2009.13658).155        cross_attention_frequency (`int`, *optional*, defaults to 2):156            The frequency of adding cross-attention to the Transformer layers.157        encoder_hidden_size (`int`, *optional*, defaults to 1408):158            The hidden size of the hidden states for cross-attention.159        use_qformer_text_input (`bool`, *optional*, defaults to `False`):160            Whether to use BERT-style embeddings.161 162    Examples:163 164    ```python165    >>> from transformers import Blip2QFormerConfig, Blip2QFormerModel166 167    >>> # Initializing a BLIP-2 Salesforce/blip2-opt-2.7b style configuration168    >>> configuration = Blip2QFormerConfig()169 170    >>> # Initializing a model (with random weights) from the Salesforce/blip2-opt-2.7b style configuration171    >>> model = Blip2QFormerModel(configuration)172    >>> # Accessing the model configuration173    >>> configuration = model.config174    ```"""175 176    model_type = "blip_2_qformer"177    base_config_key = "qformer_config"178 179    def __init__(180        self,181        vocab_size=30522,182        hidden_size=768,183        num_hidden_layers=12,184        num_attention_heads=12,185        intermediate_size=3072,186        hidden_act="gelu",187        hidden_dropout_prob=0.1,188        attention_probs_dropout_prob=0.1,189        max_position_embeddings=512,190        initializer_range=0.02,191        layer_norm_eps=1e-12,192        pad_token_id=0,193        position_embedding_type="absolute",194        cross_attention_frequency=2,195        encoder_hidden_size=1408,196        use_qformer_text_input=False,197        **kwargs,198    ):199        super().__init__(pad_token_id=pad_token_id, **kwargs)200 201        self.vocab_size = vocab_size202        self.hidden_size = hidden_size203        self.num_hidden_layers = num_hidden_layers204        self.num_attention_heads = num_attention_heads205        self.hidden_act = hidden_act206        self.intermediate_size = intermediate_size207        self.hidden_dropout_prob = hidden_dropout_prob208        self.attention_probs_dropout_prob = attention_probs_dropout_prob209        self.max_position_embeddings = max_position_embeddings210        self.initializer_range = initializer_range211        self.layer_norm_eps = layer_norm_eps212        self.position_embedding_type = position_embedding_type213        self.cross_attention_frequency = cross_attention_frequency214        self.encoder_hidden_size = encoder_hidden_size215        self.use_qformer_text_input = use_qformer_text_input216 217 218class Blip2Config(PretrainedConfig):219    r"""220    [`Blip2Config`] is the configuration class to store the configuration of a [`Blip2ForConditionalGeneration`]. It is221    used to instantiate a BLIP-2 model according to the specified arguments, defining the vision model, Q-Former model222    and language model configs. Instantiating a configuration with the defaults will yield a similar configuration to223    that of the BLIP-2 [Salesforce/blip2-opt-2.7b](https://huggingface.co/Salesforce/blip2-opt-2.7b) architecture.224 225    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the226    documentation from [`PretrainedConfig`] for more information.227 228    Args:229        vision_config (`dict`, *optional*):230            Dictionary of configuration options used to initialize [`Blip2VisionConfig`].231        qformer_config (`dict`, *optional*):232            Dictionary of configuration options used to initialize [`Blip2QFormerConfig`].233        text_config (`dict`, *optional*):234            Dictionary of configuration options used to initialize any [`PretrainedConfig`].235        num_query_tokens (`int`, *optional*, defaults to 32):236            The number of query tokens passed through the Transformer.237        image_text_hidden_size (`int`, *optional*, defaults to 256):238            Dimensionality of the hidden state of the image-text fusion layer.239 240        image_token_index (`int`, *optional*):241            Token index of special image token.242        kwargs (*optional*):243            Dictionary of keyword arguments.244 245    Example:246 247    ```python248    >>> from transformers import (249    ...     Blip2VisionConfig,250    ...     Blip2QFormerConfig,251    ...     OPTConfig,252    ...     Blip2Config,253    ...     Blip2ForConditionalGeneration,254    ... )255 256    >>> # Initializing a Blip2Config with Salesforce/blip2-opt-2.7b style configuration257    >>> configuration = Blip2Config()258 259    >>> # Initializing a Blip2ForConditionalGeneration (with random weights) from the Salesforce/blip2-opt-2.7b style configuration260    >>> model = Blip2ForConditionalGeneration(configuration)261 262    >>> # Accessing the model configuration263    >>> configuration = model.config264 265    >>> # We can also initialize a Blip2Config from a Blip2VisionConfig, Blip2QFormerConfig and any PretrainedConfig266 267    >>> # Initializing BLIP-2 vision, BLIP-2 Q-Former and language model configurations268    >>> vision_config = Blip2VisionConfig()269    >>> qformer_config = Blip2QFormerConfig()270    >>> text_config = OPTConfig()271 272    >>> config = Blip2Config.from_text_vision_configs(vision_config, qformer_config, text_config)273    ```"""274 275    model_type = "blip-2"276    attribute_map = {277        "image_token_id": "image_token_index",278    }279    sub_configs = {"text_config": AutoConfig, "qformer_config": Blip2QFormerConfig, "vision_config": Blip2VisionConfig}280 281    def __init__(282        self,283        vision_config=None,284        qformer_config=None,285        text_config=None,286        num_query_tokens=32,287        image_text_hidden_size=256,288        image_token_index=None,289        **kwargs,290    ):291        super().__init__(**kwargs)292 293        if vision_config is None:294            vision_config = {}295            logger.info("vision_config is None. initializing the Blip2VisionConfig with default values.")296 297        if qformer_config is None:298            qformer_config = {}299            logger.info("qformer_config is None. Initializing the Blip2QFormerConfig with default values.")300 301        if text_config is None:302            text_config = {}303            logger.info("text_config is None. Initializing the text config with default values (`OPTConfig`).")304 305        self.vision_config = Blip2VisionConfig(**vision_config)306        self.qformer_config = Blip2QFormerConfig(**qformer_config)307        text_model_type = text_config.get("model_type", "opt")308        self.text_config = CONFIG_MAPPING[text_model_type](**text_config)309 310        self.num_query_tokens = num_query_tokens311        self.image_text_hidden_size = image_text_hidden_size312        self.image_token_index = image_token_index313        self.qformer_config.encoder_hidden_size = self.vision_config.hidden_size314        self.use_decoder_only_language_model = self.text_config.model_type in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES315        self.is_encoder_decoder = self.text_config.is_encoder_decoder316        self.initializer_factor = 1.0317        self.initializer_range = 0.02318 319    @classmethod320    def from_vision_qformer_text_configs(321        cls,322        vision_config: Blip2VisionConfig,323        qformer_config: Blip2QFormerConfig,324        text_config: Optional[PretrainedConfig] = None,325        **kwargs,326    ):327        r"""328        Instantiate a [`Blip2Config`] (or a derived class) from a BLIP-2 vision model, Q-Former and language model329        configurations.330 331        Args:332            vision_config (`dict`):333                Dictionary of configuration options used to initialize [`Blip2VisionConfig`].334            qformer_config (`dict`):335                Dictionary of configuration options used to initialize [`Blip2QFormerConfig`].336            text_config (`dict`, *optional*):337                Dictionary of configuration options used to initialize any [`PretrainedConfig`].338 339        Returns:340            [`Blip2Config`]: An instance of a configuration object341        """342 343        return cls(344            vision_config=vision_config.to_dict(),345            qformer_config=qformer_config.to_dict(),346            text_config=text_config.to_dict() if text_config is not None else None,347            **kwargs,348        )349 350 351__all__ = ["Blip2Config", "Blip2QFormerConfig", "Blip2VisionConfig"]352