Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2024 HuggingFace Inc. team. All rights reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15 16 17from ...configuration_utils import PretrainedConfig18from ...utils import (19    logging,20)21from ..auto import CONFIG_MAPPING, AutoConfig22 23 24logger = logging.get_logger(__name__)25 26 27class LlavaOnevisionConfig(PretrainedConfig):28    r"""29    This is the configuration class to store the configuration of a [`LlavaOnevisionForConditionalGeneration`]. It is used to instantiate an30    Llava-NeXT model according to the specified arguments, defining the model architecture. Instantiating a configuration31    with the defaults will yield a similar configuration to that of the [llava-hf/llava-onevision-qwen2-7b-ov-hf](https://huggingface.co/llava-hf/llava-onevision-qwen2-7b-ov-hf)32    model.33 34    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the35    documentation from [`PretrainedConfig`] for more information.36 37    Args:38        vision_config (`Union[AutoConfig, dict]`,  *optional*, defaults to `SiglipVisionConfig`):39            The config object or dictionary of the vision backbone.40        text_config (`Union[AutoConfig, dict]`, *optional*, defaults to `Qwen2Config`):41            The config object or dictionary of the text backbone.42        image_token_index (`int`, *optional*, defaults to 151646):43            The image token index to encode the image prompt.44        video_token_index (`int`, *optional*, defaults to 151647):45            The video token index to encode the video prompt.46        projector_hidden_act (`str`, *optional*, defaults to `"gelu"`):47            The activation function used by the multimodal projector.48        vision_feature_select_strategy (`str`, *optional*, defaults to `"full"`):49            The feature selection strategy used to select the vision feature from the vision backbone.50            Can be one of `"default"` or `"full"`. If `"default"`, the CLS token is removed from the vision features.51            If `"full"`, the full vision features are used.52        vision_feature_layer (`Union[int, list[int]]`, *optional*, defaults to -1):53            The index of the layer to select the vision feature. If multiple indices are provided,54            the vision feature of the corresponding indices will be concatenated to form the55            vision features.56        vision_aspect_ratio (`str`, *optional*, defaults to `"anyres_max_9"`):57            Aspect ratio used when processong image features. The default value is "anyres_max_9".58        image_grid_pinpoints (`List`, *optional*):59            A list of possible resolutions to use for processing high resolution images. Each item in the list should be a tuple or list60            of the form `(height, width)`.61        tie_word_embeddings (`bool`, *optional*, defaults to `False`):62            Whether the model's input and output word embeddings should be tied.63        multimodal_projector_bias (`bool`, *optional*, defaults to `True`):64            Whether to use bias in the multimodal projector.65 66    Example:67 68    ```python69    >>> from transformers import LlavaOnevisionForConditionalGeneration, LlavaOnevisionConfig, SiglipVisionConfig, Qwen2Config70 71    >>> # Initializing a CLIP-vision config72    >>> vision_config = SiglipVisionConfig()73 74    >>> # Initializing a Llama config75    >>> text_config = Qwen2Config()76 77    >>> # Initializing a Llava-Next llava-hf/llava-onevision-qwen2-7b-ov-hf style configuration78    >>> configuration = LlavaOnevisionConfig(vision_config, text_config)79 80    >>> # Initializing a model from the llava-hf/llava-onevision-qwen2-7b-ov-hf style configuration81    >>> model = LlavaOnevisionForConditionalGeneration(configuration)82 83    >>> # Accessing the model configuration84    >>> configuration = model.config85    ```"""86 87    model_type = "llava_onevision"88    attribute_map = {89        "image_token_id": "image_token_index",90        "video_token_id": "video_token_index",91    }92    sub_configs = {"text_config": AutoConfig, "vision_config": AutoConfig}93 94    def __init__(95        self,96        vision_config=None,97        text_config=None,98        image_token_index=151646,99        video_token_index=151647,100        projector_hidden_act="gelu",101        vision_feature_select_strategy="full",102        vision_feature_layer=-1,103        vision_aspect_ratio="anyres_max_9",104        image_grid_pinpoints=None,105        tie_word_embeddings=False,106        multimodal_projector_bias=True,107        **kwargs,108    ):109        self.image_token_index = image_token_index110        self.video_token_index = video_token_index111        self.projector_hidden_act = projector_hidden_act112        self.multimodal_projector_bias = multimodal_projector_bias113 114        if vision_feature_select_strategy not in ["default", "full"]:115            raise ValueError(116                "vision_feature_select_strategy should be one of 'default', 'full'."117                f"Got: {vision_feature_select_strategy}"118            )119 120        self.vision_feature_select_strategy = vision_feature_select_strategy121        self.vision_feature_layer = vision_feature_layer122        self.vision_aspect_ratio = vision_aspect_ratio123        image_grid_pinpoints = (124            image_grid_pinpoints125            if image_grid_pinpoints is not None126            else [127                [384, 384],128                [384, 768],129                [384, 1152],130                [384, 1536],131                [384, 1920],132                [384, 2304],133                [768, 384],134                [768, 768],135                [768, 1152],136                [768, 1536],137                [768, 1920],138                [768, 2304],139                [1152, 384],140                [1152, 768],141                [1152, 1152],142                [1152, 1536],143                [1152, 1920],144                [1152, 2304],145                [1536, 384],146                [1536, 768],147                [1536, 1152],148                [1536, 1536],149                [1536, 1920],150                [1536, 2304],151                [1920, 384],152                [1920, 768],153                [1920, 1152],154                [1920, 1536],155                [1920, 1920],156                [1920, 2304],157                [2304, 384],158                [2304, 768],159                [2304, 1152],160                [2304, 1536],161                [2304, 1920],162                [2304, 2304],163            ]164        )165        self.image_grid_pinpoints = image_grid_pinpoints166 167        if isinstance(vision_config, dict):168            vision_config["model_type"] = vision_config.get("model_type", "siglip_vision_model")169            vision_config = CONFIG_MAPPING[vision_config["model_type"]](**vision_config)170        elif vision_config is None:171            vision_config = CONFIG_MAPPING["siglip_vision_model"](172                hidden_size=1152,173                intermediate_size=4304,174                patch_size=14,175                image_size=384,176                num_hidden_layers=26,177                num_attention_heads=16,178                vision_use_head=False,179            )180 181        self.vision_config = vision_config182 183        if isinstance(text_config, dict):184            text_config["model_type"] = text_config.get("model_type", "qwen2")185            text_config = CONFIG_MAPPING[text_config["model_type"]](**text_config)186        elif text_config is None:187            text_config = CONFIG_MAPPING["qwen2"]()188 189        self.text_config = text_config190 191        super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)192 193 194__all__ = ["LlavaOnevisionConfig"]195