allenai/MolmoPoint-GUI-8B

1"""2Molmo2 configuration3"""4 5from typing import Optional6 7from transformers import PretrainedConfig, LogitsProcessor8from transformers.utils import logging9 10from .configuration_molmo2 import Molmo2TextConfig, Molmo2VitConfig, \11    Molmo2AdapterConfig12 13logger = logging.get_logger(__name__)14 15 16class MolmoPointAdapterConfig(PretrainedConfig):17    r"""18    This is the configuration class to store the configuration of Molmo2Adapter. With Molmo2VitConfig,19    It is used to instantiate an Molmo2VisionBackbone according to the specified arguments,20    defining the model architecture.21 22    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the23    documentation from [`PretrainedConfig`] for more information.24 25    Example:26 27    ```python28    >>> from transformers import Molmo2VitConfig, Molmo2AdapterConfig, Molmo2VisionBackbone29 30    >>> # Initializing a Molmo2VitConfig and a Molmo2AdapterConfig31    >>> vit_config = Molmo2VitConfig()32    >>> adapter_config = MolmoPoolingConfig()33 34    >>> # Initializing a Molmo2VisionBackbone (with random weights)35    >>> model = Molmo2VisionBackbone(vit_config, adapter_config)36 37    >>> # Accessing the model configuration38    >>> vit_configuration = model.vit_config39    >>> adapter_configuration = model.adapter_config40    ```"""41 42    model_type = "molmo_point"43    base_config_key = "adapter_config"44 45    def __init__(46        self,47        vit_layers: tuple = (-3, -9),48        pooling_attention_mask: bool = False,49        hidden_size: int = 1152,50        num_attention_heads: int = 16,51        num_key_value_heads: int = 16,52        head_dim: int = 72,53        float32_attention: bool = True,54        attention_dropout: float = 0.0,55        residual_dropout: float = 0.0,56        hidden_act: str = "silu",57        intermediate_size: int = 18944,58        text_hidden_size: int = 3584,59        image_feature_dropout: float = 0.0,60        initializer_range: float = 0.02,61        attn_implementation: str = "eager",62        positional_embeddings: int = 16,63        attention_pooling_out_layer: bool = False,64        **kwargs,65    ):66        self.attn_implementation = attn_implementation67        super().__init__(68            attn_implementation=attn_implementation,69            **kwargs70        )71        self.vit_layers = vit_layers72        self.pooling_attention_mask = pooling_attention_mask73        self.hidden_size = hidden_size74        self.num_attention_heads = num_attention_heads75        self.num_key_value_heads = num_key_value_heads76        self.head_dim = head_dim77        self.float32_attention = float32_attention78        self.attention_dropout = attention_dropout79        self.residual_dropout = residual_dropout80        self.hidden_act = hidden_act81        self.intermediate_size = intermediate_size82        self.text_hidden_size = text_hidden_size83        self.image_feature_dropout = image_feature_dropout84        self.initializer_range = initializer_range85        self.positional_embeddings = positional_embeddings86        self.attention_pooling_out_layer = attention_pooling_out_layer87 88 89class MolmoPointConfig(PretrainedConfig):90    r"""91    This is the configuration class to store the configuration of a [`MolmoPointForConditionalGeneration`].92    It is used to instantiate an Molmo2 model according to the specified arguments, defining the model architecture.93 94    Example:95 96    ```python97    >>> from transformers import Molmo2Config, Molmo2VitConfig, Molmo2AdapterConfig, Molmo2TextConfig98 99    >>> # Initializing a Molmo2VitConfig100    >>> vit_config = Molmo2VitConfig()101 102    >>> # Initializing a Molmo2AdapterConfig103    >>> adapter_config = MolmoPointAdapterConfig()104 105    >>> # Initializing a Molmo2TextConfig106    >>> text_config = Molmo2TextConfig()107 108    >>> # Initializing a Molmo2Config109    >>> configuration = MolmoPointConfig(110    >>>     vit_config=vit_config,111    >>>     adapter_config=adapter_config,112    >>>     text_config=text_config,113    >>>     image_start_token_id=151936,114    >>>     image_end_token_id=151937,115    >>>     image_patch_id=151938,116    >>>     image_col_id=151939,117    >>>     low_res_image_start_token_id=151940,118    >>>     image_low_res_id=151942,119    >>>     frame_start_token_id=151943,120    >>>     frame_end_token_id=151944,121    >>> )122 123    >>> # Initializing a model124    >>> model = MolmoPointForConditionalGeneration(configuration)125 126    >>> # Accessing the model configuration127    >>> configuration = model.config128    ```"""129 130    model_type = "molmo_point"131    sub_configs = {132        "text_config": Molmo2TextConfig,133        "vit_config": Molmo2VitConfig,134        "adapter_config": MolmoPointAdapterConfig,135    }136 137    def __init__(138        self,139        vit_config: Molmo2VitConfig = None,140        adapter_config: MolmoPointAdapterConfig = None,141        text_config: Molmo2TextConfig = None,142        image_start_token_id: int = None,143        low_res_image_start_token_id: int = None,144        image_end_token_id: int = None,145        image_patch_id: int = None,146        image_non_indexable_patch_id: int = None,147        image_col_id: int = None,148        frame_start_token_id: int = None,149        frame_end_token_id: int = None,150        patch_token_id: int = None,151        subpatch_token_id: int = None,152        location_token_id: int = None,153        use_frame_special_tokens: bool = True,154        initializer_range: float = 0.02,155 156        # point config157        patch_location: Optional[str]="3x3",158        no_more_points_class: bool=False,159        patch_embed_dim: int=256,160        patch_embedding_kind: str="linear",161        embed_selected_vit_patch: Optional[str]="linear",162        embed_location: bool=False,163        layer_norm_x: bool=True,164        norm_logits: bool=True,165        # FIXME figure out how infernce params work166        mask_patches: Optional[str]="always",167        mask_subpatches: str="inference",168        mask_repeats: Optional[str]="inference",169        token_prediction_rotary: bool=True,170        token_prediction_rotary_theta: Optional[float]=50000,171        **kwargs,172    ):173        super().__init__(**kwargs)174        if vit_config is None:175            self.vit_config = Molmo2VitConfig()176        elif isinstance(vit_config, dict):177            self.vit_config = Molmo2VitConfig(**vit_config)178        else:179            self.vit_config = vit_config180        if adapter_config is None:181            self.adapter_config = Molmo2AdapterConfig()182        elif isinstance(adapter_config, dict):183            self.adapter_config = Molmo2AdapterConfig(**adapter_config)184        else:185            self.adapter_config = adapter_config186        if text_config is None:187            self.text_config = Molmo2TextConfig()188        elif isinstance(text_config, dict):189            self.text_config = Molmo2TextConfig(**text_config)190        else:191            self.text_config = text_config192        self.image_start_token_id = image_start_token_id193        self.low_res_image_start_token_id = low_res_image_start_token_id194        self.image_end_token_id = image_end_token_id195        self.image_high_res_id = image_patch_id196        self.image_non_indexable_patch_id = image_non_indexable_patch_id197        self.image_patch_id = image_patch_id198        self.image_col_id = image_col_id199        self.frame_start_token_id = frame_start_token_id200        self.frame_end_token_id = frame_end_token_id201        self.patch_token_id = patch_token_id202        self.subpatch_token_id = subpatch_token_id203        self.location_token_id = location_token_id204        self.use_frame_special_tokens = use_frame_special_tokens205        self.initializer_range = initializer_range206        self.patch_location = patch_location207        self.no_more_points_class = no_more_points_class208        self.patch_embed_dim = patch_embed_dim209        self.patch_embedding_kind = patch_embedding_kind210        self.embed_selected_vit_patch = embed_selected_vit_patch211        self.embed_location = embed_location212        self.layer_norm_x = layer_norm_x213        self.norm_logits = norm_logits214        self.mask_patches = mask_patches215        self.mask_subpatches = mask_subpatches216        self.mask_repeats = mask_repeats217        self.token_prediction_rotary = token_prediction_rotary218        self.token_prediction_rotary_theta = token_prediction_rotary_theta219 220    @property221    def image_num_patch(self):222        assert self.vit_config is not None223        return self.vit_config.image_num_patch224    225    @property226    def num_attention_heads(self):227        return self.text_config.num_attention_heads228    229    @property230    def num_key_value_heads(self):231        return self.text_config.num_key_value_heads232 233    @property234    def head_dim(self):235        return self.text_config.head_dim236 237    @property238    def num_hidden_layers(self):239        return self.text_config.num_hidden_layers240    241    @property242    def hidden_size(self):243        return self.text_config.hidden_size244    245    @property246    def vocab_size(self):247        return self.text_config.vocab_size248    249    @property250    def max_position_embeddings(self):251        return self.text_config.max_position_embeddings252 253 254MolmoPointAdapterConfig.register_for_auto_class()255MolmoPointConfig.register_for_auto_class()