Midea-AIRC/ECHO_block8

1#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨2#           This file was automatically generated from src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py.3#               Do NOT edit this file manually as any edits will be overwritten by the generation of4#             the file from the modular. If any change should be done, please apply the change to the5#                          modular_qwen2_5_vl.py file directly. One of our CI enforces this.6#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨7# coding=utf-88# Copyright 2025 The Qwen Team and The HuggingFace Inc. team. All rights reserved.9#10# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX11# and OPT implementations in this library. It has been modified from its12# original forms to accommodate minor architectural differences compared13# to GPT-NeoX and OPT used by the Meta AI team that trained the model.14#15# Licensed under the Apache License, Version 2.0 (the "License");16# you may not use this file except in compliance with the License.17# You may obtain a copy of the License at18#19#     http://www.apache.org/licenses/LICENSE-2.020#21# Unless required by applicable law or agreed to in writing, software22# distributed under the License is distributed on an "AS IS" BASIS,23# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.24# See the License for the specific language governing permissions and25# limitations under the License.26from transformers.configuration_utils import PretrainedConfig, layer_type_validation27from transformers.modeling_rope_utils import rope_config_validation28 29 30class EchoVisionConfig(PretrainedConfig):31    model_type = "echo"32    base_config_key = "vision_config"33 34    def __init__(35        self,36        depth=32,37        hidden_size=3584,38        hidden_act="silu",39        intermediate_size=3420,40        num_heads=16,41        in_channels=3,42        patch_size=14,43        spatial_merge_size=2,44        temporal_patch_size=2,45        tokens_per_second=4,46        window_size=112,47        out_hidden_size=3584,48        fullatt_block_indexes=[7, 15, 23, 31],49        initializer_range=0.02,50        **kwargs,51    ):52        super().__init__(**kwargs)53 54        self.depth = depth55        self.hidden_size = hidden_size56        self.hidden_act = hidden_act57        self.intermediate_size = intermediate_size58        self.num_heads = num_heads59        self.in_channels = in_channels60        self.patch_size = patch_size61        self.spatial_merge_size = spatial_merge_size62        self.temporal_patch_size = temporal_patch_size63        self.tokens_per_second = tokens_per_second64        self.window_size = window_size65        self.fullatt_block_indexes = fullatt_block_indexes66        self.out_hidden_size = out_hidden_size67        self.initializer_range = initializer_range68 69 70class EchoTextConfig(PretrainedConfig):71    r"""72    This is the configuration class to store the configuration of a [`EchoTextModel`]. It is used to instantiate a73    Qwen2-VL model according to the specified arguments, defining the model architecture. Instantiating a configuration74    with the defaults will yield a similar configuration to that of75    Qwen2-VL-7B-Instruct [Qwen/Qwen2-VL-7B-Instruct](https://huggingface.co/Qwen/Qwen2-VL-7B-Instruct).76 77    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the78    documentation from [`PretrainedConfig`] for more information.79 80    Args:81        vocab_size (`int`, *optional*, defaults to 152064):82            Vocabulary size of the Echo model. Defines the number of different tokens that can be represented by the83            `inputs_ids` passed when calling [`EchoModel`]84        hidden_size (`int`, *optional*, defaults to 8192):85            Dimension of the hidden representations.86        intermediate_size (`int`, *optional*, defaults to 29568):87            Dimension of the MLP representations.88        num_hidden_layers (`int`, *optional*, defaults to 80):89            Number of hidden layers in the Transformer encoder.90        num_attention_heads (`int`, *optional*, defaults to 64):91            Number of attention heads for each attention layer in the Transformer encoder.92        num_key_value_heads (`int`, *optional*, defaults to 8):93            This is the number of key_value heads that should be used to implement Grouped Query Attention. If94            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if95            `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When96            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed97            by meanpooling all the original heads within that group. For more details, check out [this98            paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to `32`.99        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):100            The non-linear activation function (function or string) in the decoder.101        max_position_embeddings (`int`, *optional*, defaults to 32768):102            The maximum sequence length that this model might ever be used with.103        initializer_range (`float`, *optional*, defaults to 0.02):104            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.105        rms_norm_eps (`float`, *optional*, defaults to 1e-05):106            The epsilon used by the rms normalization layers.107        use_cache (`bool`, *optional*, defaults to `True`):108            Whether or not the model should return the last key/values attentions (not used by all models). Only109            relevant if `config.is_decoder=True`.110        tie_word_embeddings (`bool`, *optional*, defaults to `False`):111            Whether the model's input and output word embeddings should be tied.112        rope_theta (`float`, *optional*, defaults to 1000000.0):113            The base period of the RoPE embeddings.114        use_sliding_window (`bool`, *optional*, defaults to `False`):115            Whether to use sliding window attention.116        sliding_window (`int`, *optional*, defaults to 4096):117            Sliding window attention (SWA) window size. If not specified, will default to `4096`.118        max_window_layers (`int`, *optional*, defaults to 80):119            The number of layers using full attention. The first `max_window_layers` layers will use full attention, while any120            additional layer afterwards will use SWA (Sliding Window Attention).121        layer_types (`list`, *optional*):122            Attention pattern for each layer.123        attention_dropout (`float`, *optional*, defaults to 0.0):124            The dropout ratio for the attention probabilities.125        rope_scaling (`Dict`, *optional*):126            Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type127            and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value128            accordingly.129            Expected contents:130                `rope_type` (`str`):131                    The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',132                    'llama3'], with 'default' being the original RoPE implementation.133                `factor` (`float`, *optional*):134                    Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In135                    most scaling types, a `factor` of x will enable the model to handle sequences of length x *136                    original maximum pre-trained length.137                `original_max_position_embeddings` (`int`, *optional*):138                    Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during139                    pretraining.140                `attention_factor` (`float`, *optional*):141                    Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention142                    computation. If unspecified, it defaults to value recommended by the implementation, using the143                    `factor` field to infer the suggested value.144                `beta_fast` (`float`, *optional*):145                    Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear146                    ramp function. If unspecified, it defaults to 32.147                `beta_slow` (`float`, *optional*):148                    Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear149                    ramp function. If unspecified, it defaults to 1.150                `short_factor` (`list[float]`, *optional*):151                    Only used with 'longrope'. The scaling factor to be applied to short contexts (<152                    `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden153                    size divided by the number of attention heads divided by 2154                `long_factor` (`list[float]`, *optional*):155                    Only used with 'longrope'. The scaling factor to be applied to long contexts (<156                    `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden157                    size divided by the number of attention heads divided by 2158                `low_freq_factor` (`float`, *optional*):159                    Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE160                `high_freq_factor` (`float`, *optional*):161                    Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE162        image_token_id (`int`, *optional*):163            Token index used as placeholder for image embeddings.164        video_token_id (`int`, *optional*):165            Token index used as placeholder for video embeddings.166 167    ```python168    >>> from transformers import EchoTextModel, EchoConfig169 170    >>> # Initializing a Echo style configuration171    >>> configuration = EchoConfig()172 173    >>> # Initializing a model from the Qwen2-VL-7B style configuration174    >>> model = EchoTextModel(configuration)175 176    >>> # Accessing the model configuration177    >>> configuration = model.config178    ```"""179 180    model_type = "echo_text"181    base_config_key = "text_config"182    keys_to_ignore_at_inference = ["past_key_values"]183    # Default tensor parallel plan for base model `Echo`184    base_model_tp_plan = {185        "layers.*.self_attn.q_proj": "colwise",186        "layers.*.self_attn.k_proj": "colwise",187        "layers.*.self_attn.v_proj": "colwise",188        "layers.*.self_attn.o_proj": "rowwise",189        "layers.*.mlp.gate_proj": "colwise",190        "layers.*.mlp.up_proj": "colwise",191        "layers.*.mlp.down_proj": "rowwise",192    }193    base_model_pp_plan = {194        "embed_tokens": (["input_ids"], ["inputs_embeds"]),195        "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),196        "norm": (["hidden_states"], ["hidden_states"]),197    }198 199    def __init__(200        self,201        vocab_size=152064,202        hidden_size=8192,203        intermediate_size=29568,204        num_hidden_layers=80,205        num_attention_heads=64,206        num_key_value_heads=8,207        hidden_act="silu",208        max_position_embeddings=32768,209        initializer_range=0.02,210        rms_norm_eps=1e-05,211        use_cache=True,212        tie_word_embeddings=False,213        rope_theta=1000000.0,214        use_sliding_window=False,215        sliding_window=4096,216        max_window_layers=80,217        layer_types=None,218        attention_dropout=0.0,219        rope_scaling=None,220        image_token_id=None,221        video_token_id=None,222        **kwargs,223    ):224        self.vocab_size = vocab_size225        self.max_position_embeddings = max_position_embeddings226        self.hidden_size = hidden_size227        self.intermediate_size = intermediate_size228        self.num_hidden_layers = num_hidden_layers229        self.num_attention_heads = num_attention_heads230        self.use_sliding_window = use_sliding_window231        self.sliding_window = sliding_window if self.use_sliding_window else None232        self.max_window_layers = max_window_layers233 234        # for backward compatibility235        if num_key_value_heads is None:236            num_key_value_heads = num_attention_heads237 238        self.num_key_value_heads = num_key_value_heads239        self.hidden_act = hidden_act240        self.initializer_range = initializer_range241        self.rms_norm_eps = rms_norm_eps242        self.use_cache = use_cache243        self.rope_theta = rope_theta244        self.attention_dropout = attention_dropout245        self.rope_scaling = rope_scaling246 247        self.layer_types = layer_types248        if self.layer_types is None:249            self.layer_types = [250                "sliding_attention"251                if self.sliding_window is not None and i >= self.max_window_layers252                else "full_attention"253                for i in range(self.num_hidden_layers)254            ]255        layer_type_validation(self.layer_types)256 257        # Validate the correctness of rotary position embeddings parameters258        # BC: if there is a 'type' field, move it to 'rope_type'.259        # and change type from 'mrope' to 'default' because `mrope` does default RoPE calculations260        # one can set it to "linear"/"dynamic" etc. to have scaled RoPE261        # TODO: @raushan update config in the hub262        if self.rope_scaling is not None and "type" in self.rope_scaling:263            if self.rope_scaling["type"] == "mrope":264                self.rope_scaling["type"] = "default"265            self.rope_scaling["rope_type"] = self.rope_scaling["type"]266        rope_config_validation(self, ignore_keys={"mrope_section"})267        self.image_token_id = image_token_id268        self.video_token_id = video_token_id269 270        super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)271 272 273class EchoConfig(PretrainedConfig):274    r"""275    This is the configuration class to store the configuration of a [`EchoModel`]. It is used to instantiate a276    Qwen2-VL model according to the specified arguments, defining the model architecture. Instantiating a configuration277    with the defaults will yield a similar configuration to that of278    Qwen2-VL-7B-Instruct [Qwen/Qwen2-VL-7B-Instruct](https://huggingface.co/Qwen/Qwen2-VL-7B-Instruct).279 280    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the281    documentation from [`PretrainedConfig`] for more information.282 283 284    Args:285        text_config (`Union[PreTrainedConfig, dict]`, *optional*, defaults to `EchoTextConfig`):286            The config object or dictionary of the text backbone.287        vision_config (`Union[PreTrainedConfig, dict]`,  *optional*, defaults to `EchoVisionConfig`):288            The config object or dictionary of the vision backbone.289        image_token_id (`int`, *optional*, defaults to 151655):290            The image token index to encode the image prompt.291        video_token_id (`int`, *optional*, defaults to 151656):292            The video token index to encode the image prompt.293 294    ```python295    >>> from transformers import EchoForConditionalGeneration, EchoConfig296 297    >>> # Initializing a Echo style configuration298    >>> configuration = EchoConfig()299 300    >>> # Initializing a model from the Qwen2-VL-7B style configuration301    >>> model = EchoForConditionalGeneration(configuration)302 303    >>> # Accessing the model configuration304    >>> configuration = model.config305    ```"""306 307    model_type = "echo"308    sub_configs = {"vision_config": EchoVisionConfig, "text_config": EchoTextConfig}309    keys_to_ignore_at_inference = ["past_key_values"]310 311    def __init__(312        self,313        text_config=None,314        vision_config=None,315        image_token_id=151655,316        video_token_id=151656,317        **kwargs,318    ):319        if isinstance(vision_config, dict):320            self.vision_config = self.sub_configs["vision_config"](**vision_config)321        elif vision_config is None:322            self.vision_config = self.sub_configs["vision_config"]()323 324        if isinstance(text_config, dict):325            self.text_config = self.sub_configs["text_config"](**text_config)326        elif text_config is None:327            # For BC use all kwargs to init `TextConfig`328            self.text_config = self.sub_configs["text_config"](**kwargs)329 330        self.image_token_id = image_token_id331        self.video_token_id = video_token_id332 333        super().__init__(**kwargs)334 335 336__all__ = ["EchoConfig", "EchoTextConfig"]337