MiniMaxAI/MiniMax-VL-01

1"""MiniMaxVL01 model configuration"""2 3from transformers.configuration_utils import PretrainedConfig4from transformers.utils import logging5from transformers.models.auto import CONFIG_MAPPING, AutoConfig6from .configuration_minimax_text_01 import MiniMaxText01Config7 8 9class MiniMaxVL01Config(PretrainedConfig):10    r"""11    This is the configuration class to store the configuration of a [`MiniMaxVL01ForConditionalGeneration`]. It is used to instantiate an12    MiniMaxVL01 model according to the specified arguments, defining the model architecture. Instantiating a configuration13    with the defaults will yield a similar configuration to that of the MiniMaxVL01.14 15    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the16    documentation from [`PretrainedConfig`] for more information.17 18    Args:19        vision_config (`Union[AutoConfig, dict]`,  *optional*, defaults to `CLIPVisionConfig`):20            The config object or dictionary of the vision backbone.21        text_config (`Union[AutoConfig, dict]`, *optional*, defaults to `MiniMaxText01Config`):22            The config object or dictionary of the text backbone.23        ignore_index (`int`, *optional*, defaults to -100):24            The ignore index for the loss function.25        image_token_index (`int`, *optional*, defaults to 32000):26            The image token index to encode the image prompt.27        projector_hidden_act (`str`, *optional*, defaults to `"gelu"`):28            The activation function used by the multimodal projector.29        vision_feature_select_strategy (`str`, *optional*, defaults to `"default"`):30            The feature selection strategy used to select the vision feature from the vision backbone.31            Can be one of `"default"` or `"full"`. If `"default"`, the CLS token is removed from the vision features.32            If `"full"`, the full vision features are used.33        vision_feature_layer (`int`, *optional*, defaults to -2):34            The index of the layer to select the vision feature.35        image_grid_pinpoints (`List`, *optional*, defaults to `[[336, 672], [672, 336], [672, 672], [1008, 336], [336, 1008]]`):36            A list of possible resolutions to use for processing high resolution images. Each item in the list should be a tuple or list37            of the form `(height, width)`.38        tie_word_embeddings (`bool`, *optional*, defaults to `False`):39            Whether the model's input and output word embeddings should be tied.40        image_seq_length (`int`, *optional*, defaults to 576):41            Sequence length of one image embedding.42 43    Example:44 45    ```python46    >>> from transformers import MiniMaxVL01ForConditionalGeneration, MiniMaxVL01Config, CLIPVisionConfig, MiniMaxText01Config47 48    >>> # Initializing a CLIP-vision config49    >>> vision_config = CLIPVisionConfig()50 51    >>> # Initializing a MiniMaxText01 config52    >>> text_config = MiniMaxText01Config()53 54    >>> # Initializing a MiniMaxVL01 style configuration55    >>> configuration = MiniMaxVL01Config(vision_config, text_config)56 57    >>> # Initializing a model from the MiniMaxVL01 style configuration58    >>> model = MiniMaxVL01ForConditionalGeneration(configuration)59 60    >>> # Accessing the model configuration61    >>> configuration = model.config62    ```"""63 64    model_type = "minimax_vl_01"65 66    def __init__(67        self,68        vision_config=None,69        text_config=None,70        ignore_index=-100,71        image_token_index=32000,72        projector_hidden_act="gelu",73        vision_feature_select_strategy="default",74        vision_feature_layer=-2,75        image_grid_pinpoints=None,76        tie_word_embeddings=False,77        image_seq_length=576,78        **kwargs,79    ):80        self.ignore_index = ignore_index81        self.image_token_index = image_token_index82        self.projector_hidden_act = projector_hidden_act83        self.image_seq_length = image_seq_length84 85        if vision_feature_select_strategy not in ["default", "full"]:86            raise ValueError(87                "vision_feature_select_strategy should be one of 'default', 'full'."88                f"Got: {vision_feature_select_strategy}"89            )90 91        self.vision_feature_select_strategy = vision_feature_select_strategy92        self.vision_feature_layer = vision_feature_layer93        image_grid_pinpoints = (94            image_grid_pinpoints95            if image_grid_pinpoints is not None96            else [[336, 672], [672, 336], [672, 672], [1008, 336], [336, 1008]]97        )98        self.image_grid_pinpoints = image_grid_pinpoints99 100        if isinstance(vision_config, dict):101            vision_config["model_type"] = (102                vision_config["model_type"] if "model_type" in vision_config else "clip_vision_model"103            )104            vision_config = CONFIG_MAPPING[vision_config["model_type"]](**vision_config)105        elif vision_config is None:106            vision_config = CONFIG_MAPPING["clip_vision_model"](107                intermediate_size=4096,108                hidden_size=1024,109                patch_size=14,110                image_size=336,111                num_hidden_layers=24,112                num_attention_heads=16,113                vocab_size=32000,114                projection_dim=768,115            )116 117        self.vision_config = vision_config118 119        if text_config is not None:120            assert "model_type" in text_config, "text_config model_type is not specified"121            text_config = MiniMaxText01Config(**text_config)122        else:123            text_config = MiniMaxText01Config()124 125        self.text_config = text_config126 127        super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)128