Aluode/PerceptionLabPortable

1from typing import Optional2 3import torch4from torch import nn5 6from ...modeling_attn_mask_utils import _create_4d_causal_attention_mask, _prepare_4d_attention_mask7from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling8from ...modeling_utils import PreTrainedModel9from ...processing_utils import Unpack10from ...utils import TransformersKwargs, auto_docstring, logging11from ...utils.generic import check_model_inputs12from ..clip.configuration_clip import CLIPConfig, CLIPTextConfig, CLIPVisionConfig13from ..clip.modeling_clip import (14    CLIPMLP,15    CLIPAttention,16    CLIPForImageClassification,17    CLIPModel,18    CLIPTextEmbeddings,19    CLIPTextModel,20    CLIPTextModelWithProjection,21    CLIPTextTransformer,22    CLIPVisionEmbeddings,23    CLIPVisionModel,24    CLIPVisionModelWithProjection,25)26 27 28logger = logging.get_logger(__name__)29 30 31_CHECKPOINT_FOR_DOC = "facebook/metaclip-2-worldwide-huge-quickgelu"32_CONFIG_FOR_DOC = "MetaClip2Config"33 34 35class MetaClip2TextConfig(CLIPTextConfig):36    r"""37    This is the configuration class to store the configuration of a [`MetaClip2TextModel`]. It is used to instantiate38    a MetaClip2 text encoder according to the specified arguments, defining the model architecture. Instantiating a39    configuration with the defaults will yield a similar configuration to that of the MetaClip240    [facebook/metaclip-2-worldwide-huge-quickgelu](https://huggingface.co/facebook/metaclip-2-worldwide-huge-quickgelu) architecture.41 42    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the43    documentation from [`PretrainedConfig`] for more information.44 45    Args:46        vocab_size (`int`, *optional*, defaults to 49408):47            Vocabulary size of the MetaClip2 text model. Defines the number of different tokens that can be represented by48            the `inputs_ids` passed when calling [`MetaClip2TextModel`].49        hidden_size (`int`, *optional*, defaults to 512):50            Dimensionality of the encoder layers and the pooler layer.51        intermediate_size (`int`, *optional*, defaults to 2048):52            Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.53        projection_dim (`int`, *optional*, defaults to 512):54            Dimensionality of text and vision projection layers.55        num_hidden_layers (`int`, *optional*, defaults to 12):56            Number of hidden layers in the Transformer encoder.57        num_attention_heads (`int`, *optional*, defaults to 8):58            Number of attention heads for each attention layer in the Transformer encoder.59        max_position_embeddings (`int`, *optional*, defaults to 77):60            The maximum sequence length that this model might ever be used with. Typically set this to something large61            just in case (e.g., 512 or 1024 or 2048).62        hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`):63            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,64            `"relu"`, `"selu"` and `"gelu_new"` `"quick_gelu"` are supported.65        layer_norm_eps (`float`, *optional*, defaults to 1e-05):66            The epsilon used by the layer normalization layers.67        attention_dropout (`float`, *optional*, defaults to 0.0):68            The dropout ratio for the attention probabilities.69        initializer_range (`float`, *optional*, defaults to 0.02):70            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.71        initializer_factor (`float`, *optional*, defaults to 1.0):72            A factor for initializing all weight matrices (should be kept to 1, used internally for initialization73            testing).74        pad_token_id (`int`, *optional*, defaults to 1):75            Padding token id.76        bos_token_id (`int`, *optional*, defaults to 49406):77            Beginning of stream token id.78        eos_token_id (`int`, *optional*, defaults to 49407):79            End of stream token id.80 81    Example:82 83    ```python84    >>> from transformers import MetaClip2TextConfig, MetaClip2TextModel85 86    >>> # Initializing a MetaClip2TextConfig with facebook/metaclip-2-worldwide-huge-quickgelu style configuration87    >>> configuration = MetaClip2TextConfig()88 89    >>> # Initializing a MetaClip2TextModel (with random weights) from the facebook/metaclip-2-worldwide-huge-quickgelu style configuration90    >>> model = MetaClip2TextModel(configuration)91 92    >>> # Accessing the model configuration93    >>> configuration = model.config94    ```"""95 96    pass97 98 99class MetaClip2VisionConfig(CLIPVisionConfig):100    r"""101    This is the configuration class to store the configuration of a [`MetaClip2VisionModel`]. It is used to instantiate a MetaClip2102    vision encoder according to the specified arguments, defining the model architecture. Instantiating a configuration103    with the defaults will yield a similar configuration to that of the vision encoder of the MetaClip2104    [facebook/metaclip-2-worldwide-huge-quickgelu](https://huggingface.co/facebook/metaclip-2-worldwide-huge-quickgelu) architecture.105 106    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the107    documentation from [`PretrainedConfig`] for more information.108 109    Args:110        hidden_size (`int`, *optional*, defaults to 768):111            Dimensionality of the encoder layers and the pooler layer.112        intermediate_size (`int`, *optional*, defaults to 3072):113            Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.114        projection_dim (`int`, *optional*, defaults to 512):115            Dimensionality of text and vision projection layers.116        num_hidden_layers (`int`, *optional*, defaults to 12):117            Number of hidden layers in the Transformer encoder.118        num_attention_heads (`int`, *optional*, defaults to 12):119            Number of attention heads for each attention layer in the Transformer encoder.120        num_channels (`int`, *optional*, defaults to 3):121            The number of input channels.122        image_size (`int`, *optional*, defaults to 224):123            The size (resolution) of each image.124        patch_size (`int`, *optional*, defaults to 32):125            The size (resolution) of each patch.126        hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`):127            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,128            `"relu"`, `"selu"` and `"gelu_new"` `"quick_gelu"` are supported.129        layer_norm_eps (`float`, *optional*, defaults to 1e-05):130            The epsilon used by the layer normalization layers.131        attention_dropout (`float`, *optional*, defaults to 0.0):132            The dropout ratio for the attention probabilities.133        initializer_range (`float`, *optional*, defaults to 0.02):134            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.135        initializer_factor (`float`, *optional*, defaults to 1.0):136            A factor for initializing all weight matrices (should be kept to 1, used internally for initialization137            testing).138 139    Example:140 141    ```python142    >>> from transformers import MetaClip2VisionConfig, MetaClip2VisionModel143 144    >>> # Initializing a MetaClip2VisionConfig with facebook/metaclip-2-worldwide-huge-quickgelu style configuration145    >>> configuration = MetaClip2VisionConfig()146 147    >>> # Initializing a MetaClip2VisionModel (with random weights) from the facebook/metaclip-2-worldwide-huge-quickgelu style configuration148    >>> model = MetaClip2VisionModel(configuration)149 150    >>> # Accessing the model configuration151    >>> configuration = model.config152    ```"""153 154    pass155 156 157class MetaClip2Config(CLIPConfig):158    r"""159    [`MetaClip2Config`] is the configuration class to store the configuration of a [`MetaClip2Model`]. It is used to160    instantiate a MetaClip2 model according to the specified arguments, defining the text model and vision model configs.161    Instantiating a configuration with the defaults will yield a similar configuration to that of the MetaClip2162    [facebook/metaclip-2-worldwide-huge-quickgelu](https://huggingface.co/facebook/metaclip-2-worldwide-huge-quickgelu) architecture.163 164    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the165    documentation from [`PretrainedConfig`] for more information.166 167    Args:168        text_config (`dict`, *optional*):169            Dictionary of configuration options used to initialize [`MetaClip2TextConfig`].170        vision_config (`dict`, *optional*):171            Dictionary of configuration options used to initialize [`MetaClip2VisionConfig`].172        projection_dim (`int`, *optional*, defaults to 512):173            Dimensionality of text and vision projection layers.174        logit_scale_init_value (`float`, *optional*, defaults to 2.6592):175            The initial value of the *logit_scale* parameter. Default is used as per the original MetaClip2 implementation.176        kwargs (*optional*):177            Dictionary of keyword arguments.178 179    Example:180 181    ```python182    >>> from transformers import MetaClip2Config, MetaClip2Model183 184    >>> # Initializing a MetaClip2Config with facebook/metaclip-2-worldwide-huge-quickgelu style configuration185    >>> configuration = MetaClip2Config()186 187    >>> # Initializing a MetaClip2Model (with random weights) from the facebook/metaclip-2-worldwide-huge-quickgelu style configuration188    >>> model = MetaClip2Model(configuration)189 190    >>> # Accessing the model configuration191    >>> configuration = model.config192 193    >>> # We can also initialize a MetaClip2Config from a MetaClip2TextConfig and a MetaClip2VisionConfig194    >>> from transformers import MetaClip2TextConfig, MetaClip2VisionConfig195 196    >>> # Initializing a MetaClip2Text and MetaClip2Vision configuration197    >>> config_text = MetaClip2TextConfig()198    >>> config_vision = MetaClip2VisionConfig()199 200    >>> config = MetaClip2Config.from_text_vision_configs(config_text, config_vision)201    ```"""202 203    pass204 205 206class MetaClip2TextEmbeddings(CLIPTextEmbeddings):207    pass208 209 210class MetaClip2VisionEmbeddings(CLIPVisionEmbeddings):211    pass212 213 214class MetaClip2Attention(CLIPAttention):215    pass216 217 218class MetaClip2MLP(CLIPMLP):219    pass220 221 222@auto_docstring223class MetaClip2PreTrainedModel(PreTrainedModel):224    config: MetaClip2Config225    base_model_prefix = "metaclip_2"226    supports_gradient_checkpointing = True227    _supports_sdpa = True228    _supports_flash_attn = True229    _supports_flex_attn = True230    _supports_attention_backend = True231 232    def _init_weights(self, module):233        """Initialize the weights"""234        factor = self.config.initializer_factor235        if isinstance(module, MetaClip2TextEmbeddings):236            module.token_embedding.weight.data.normal_(mean=0.0, std=factor * 0.02)237            module.position_embedding.weight.data.normal_(mean=0.0, std=factor * 0.02)238        elif isinstance(module, MetaClip2VisionEmbeddings):239            factor = self.config.initializer_factor240            nn.init.normal_(module.class_embedding, mean=0.0, std=module.embed_dim**-0.5 * factor)241            nn.init.normal_(module.patch_embedding.weight, std=module.config.initializer_range * factor)242            nn.init.normal_(module.position_embedding.weight, std=module.config.initializer_range * factor)243        elif isinstance(module, MetaClip2Attention):244            factor = self.config.initializer_factor245            in_proj_std = (module.embed_dim**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor246            out_proj_std = (module.embed_dim**-0.5) * factor247            nn.init.normal_(module.q_proj.weight, std=in_proj_std)248            nn.init.normal_(module.k_proj.weight, std=in_proj_std)249            nn.init.normal_(module.v_proj.weight, std=in_proj_std)250            nn.init.normal_(module.out_proj.weight, std=out_proj_std)251        elif isinstance(module, MetaClip2MLP):252            factor = self.config.initializer_factor253            in_proj_std = (module.config.hidden_size**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor254            fc_std = (2 * module.config.hidden_size) ** -0.5 * factor255            nn.init.normal_(module.fc1.weight, std=fc_std)256            nn.init.normal_(module.fc2.weight, std=in_proj_std)257        elif isinstance(module, MetaClip2Model):258            nn.init.normal_(259                module.text_projection.weight,260                std=module.text_embed_dim**-0.5 * self.config.initializer_factor,261            )262            nn.init.normal_(263                module.visual_projection.weight,264                std=module.vision_embed_dim**-0.5 * self.config.initializer_factor,265            )266        elif isinstance(module, MetaClip2VisionModelWithProjection):267            nn.init.normal_(268                module.visual_projection.weight,269                std=self.config.hidden_size**-0.5 * self.config.initializer_factor,270            )271        elif isinstance(module, MetaClip2TextModelWithProjection):272            nn.init.normal_(273                module.text_projection.weight,274                std=self.config.hidden_size**-0.5 * self.config.initializer_factor,275            )276        elif isinstance(module, MetaClip2ForImageClassification):277            nn.init.normal_(278                module.classifier.weight,279                std=self.config.vision_config.hidden_size**-0.5 * self.config.initializer_factor,280            )281 282        if isinstance(module, nn.LayerNorm):283            module.bias.data.zero_()284            module.weight.data.fill_(1.0)285        if isinstance(module, nn.Linear) and module.bias is not None:286            module.bias.data.zero_()287 288 289class MetaClip2TextTransformer(CLIPTextTransformer):290    @check_model_inputs(tie_last_hidden_states=False)291    @auto_docstring292    def forward(293        self,294        input_ids,295        attention_mask: Optional[torch.Tensor] = None,296        position_ids: Optional[torch.Tensor] = None,297        use_cache: Optional[bool] = None,298        **kwargs: Unpack[TransformersKwargs],299    ) -> BaseModelOutputWithPooling:300        input_shape = input_ids.size()301        input_ids = input_ids.view(-1, input_shape[-1])302 303        hidden_states = self.embeddings(input_ids=input_ids, position_ids=position_ids)304 305        # CLIP's text model uses causal mask, prepare it here.306        # https://github.com/openai/CLIP/blob/cfcffb90e69f37bf2ff1e988237a0fbe41f33c04/clip/model.py#L324307        causal_attention_mask = _create_4d_causal_attention_mask(308            input_shape, hidden_states.dtype, device=hidden_states.device309        )310 311        # expand attention_mask312        if attention_mask is not None and self.config._attn_implementation != "flash_attention_2":313            # [batch_size, seq_len] -> [batch_size, 1, tgt_seq_len, src_seq_len]314            attention_mask = _prepare_4d_attention_mask(attention_mask, hidden_states.dtype)315 316        encoder_outputs: BaseModelOutput = self.encoder(317            inputs_embeds=hidden_states,318            attention_mask=attention_mask,319            causal_attention_mask=causal_attention_mask,320            **kwargs,321        )322 323        last_hidden_state = encoder_outputs.last_hidden_state324        last_hidden_state = self.final_layer_norm(last_hidden_state)325 326        # Use robust pooling like CLIP - finds the first EOS token position per sequence327        pooled_output = last_hidden_state[328            torch.arange(last_hidden_state.shape[0], device=last_hidden_state.device),329            (input_ids.to(dtype=torch.int, device=last_hidden_state.device) == self.eos_token_id).int().argmax(dim=-1),330        ]331 332        return BaseModelOutputWithPooling(333            last_hidden_state=last_hidden_state,334            pooler_output=pooled_output,335            hidden_states=encoder_outputs.hidden_states,336            attentions=encoder_outputs.attentions,337        )338 339 340class MetaClip2TextModel(CLIPTextModel):341    """342    The text model from MetaClip2 without any head or projection on top.343    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the344    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads345    etc.)346 347    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.348    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage349    and behavior.350 351    Args:352        config ([`MetaClip2TextConfig`]): Model configuration class with all the parameters of the model.353            Initializing with a config file does not load the weights associated with the model, only the354            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.355 356    Examples:357 358    ```python359    >>> from transformers import AutoTokenizer, MetaClip2TextModel360 361    >>> model = MetaClip2TextModel.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")362    >>> tokenizer = AutoTokenizer.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")363 364    >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")365 366    >>> outputs = model(**inputs)367    >>> last_hidden_state = outputs.last_hidden_state368    >>> pooled_output = outputs.pooler_output  # pooled (EOS token) states369    ```"""370 371    def __init__(self, config: MetaClip2TextConfig):372        super().__init__(config)373        self.text_model = MetaClip2TextTransformer(config)374        # Initialize weights and apply final processing375        self.post_init()376 377    def forward(378        self,379        input_ids: Optional[torch.Tensor] = None,380        attention_mask: Optional[torch.Tensor] = None,381        position_ids: Optional[torch.Tensor] = None,382        output_attentions: Optional[bool] = None,383        output_hidden_states: Optional[bool] = None,384    ):385        r"""386        Examples:387 388        ```python389        >>> from transformers import AutoTokenizer, MetaClip2TextModel390 391        >>> model = MetaClip2TextModel.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")392        >>> tokenizer = AutoTokenizer.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")393 394        >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")395 396        >>> outputs = model(**inputs)397        >>> last_hidden_state = outputs.last_hidden_state398        >>> pooled_output = outputs.pooler_output  # pooled (EOS token) states399        ```"""400        return super().forward(401            input_ids=input_ids,402            attention_mask=attention_mask,403            position_ids=position_ids,404            output_attentions=output_attentions,405            output_hidden_states=output_hidden_states,406        )407 408 409class MetaClip2TextModelWithProjection(CLIPTextModelWithProjection):410    """411    MetaClip2 text model with a projection layer on top (a linear layer on top of the pooled output).412 413    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the414    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads415    etc.)416 417    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.418    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage419    and behavior.420 421    Args:422        config ([`MetaClip2TextConfig`]): Model configuration class with all the parameters of the model.423            Initializing with a config file does not load the weights associated with the model, only the424            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.425 426    Examples:427 428    ```python429    >>> from transformers import AutoTokenizer, MetaClip2TextModelWithProjection430 431    >>> model = MetaClip2TextModelWithProjection.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")432    >>> tokenizer = AutoTokenizer.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")433 434    >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")435 436    >>> outputs = model(**inputs)437    >>> text_embeds = outputs.text_embeds438    ```"""439 440    def __init__(self, config: MetaClip2TextConfig):441        super().__init__(config)442 443        text_model = MetaClip2TextModel._from_config(config)444        self.text_model = text_model.text_model445 446        self.text_projection = nn.Linear(config.hidden_size, config.projection_dim, bias=False)447 448        # Initialize weights and apply final processing449        self.post_init()450 451    def forward(452        self,453        input_ids: Optional[torch.Tensor] = None,454        attention_mask: Optional[torch.Tensor] = None,455        position_ids: Optional[torch.Tensor] = None,456        output_attentions: Optional[bool] = None,457        output_hidden_states: Optional[bool] = None,458    ):459        r"""460        Examples:461 462        ```python463        >>> from transformers import AutoTokenizer, MetaClip2TextModelWithProjection464 465        >>> model = MetaClip2TextModelWithProjection.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")466        >>> tokenizer = AutoTokenizer.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")467 468        >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")469 470        >>> outputs = model(**inputs)471        >>> text_embeds = outputs.text_embeds472        ```"""473        return super().forward(474            input_ids=input_ids,475            attention_mask=attention_mask,476            position_ids=position_ids,477            output_attentions=output_attentions,478            output_hidden_states=output_hidden_states,479        )480 481 482class MetaClip2Model(CLIPModel):483    """484    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the485    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads486    etc.)487 488    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.489    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage490    and behavior.491 492    Args:493        config ([`MetaClip2Config`]): Model configuration class with all the parameters of the model.494            Initializing with a config file does not load the weights associated with the model, only the495            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.496 497    Examples:498 499    ```python500    >>> from PIL import Image501    >>> import requests502    >>> from transformers import AutoProcessor, MetaClip2Model503 504    >>> model = MetaClip2Model.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")505    >>> processor = AutoProcessor.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")506 507    >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"508    >>> image = Image.open(requests.get(url, stream=True).raw)509 510    >>> inputs = processor(511    ...     text=["a photo of a cat", "a photo of a dog"], images=image, return_tensors="pt", padding=True512    ... )513 514    >>> outputs = model(**inputs)515    >>> logits_per_image = outputs.logits_per_image  # this is the image-text similarity score516    >>> probs = logits_per_image.softmax(dim=1)  # we can take the softmax to get the label probabilities517    ```"""518 519    def __init__(self, config: MetaClip2Config):520        super().__init__(config)521 522        text_config = config.text_config523        vision_config = config.vision_config524 525        self.projection_dim = config.projection_dim526        self.text_embed_dim = text_config.hidden_size527        self.vision_embed_dim = vision_config.hidden_size528 529        text_model = MetaClip2TextModel._from_config(text_config)530        self.text_model = text_model.text_model531 532        vision_model = MetaClip2VisionModel._from_config(vision_config)533        self.vision_model = vision_model.vision_model534 535        self.visual_projection = nn.Linear(self.vision_embed_dim, self.projection_dim, bias=False)536        self.text_projection = nn.Linear(self.text_embed_dim, self.projection_dim, bias=False)537        self.logit_scale = nn.Parameter(torch.tensor(self.config.logit_scale_init_value))538 539        # Initialize weights and apply final processing540        self.post_init()541 542    def forward(543        self,544        input_ids: Optional[torch.LongTensor] = None,545        pixel_values: Optional[torch.FloatTensor] = None,546        attention_mask: Optional[torch.Tensor] = None,547        position_ids: Optional[torch.LongTensor] = None,548        return_loss: Optional[bool] = None,549        output_attentions: Optional[bool] = None,550        output_hidden_states: Optional[bool] = None,551        interpolate_pos_encoding: bool = False,552    ):553        r"""554        return_loss (`bool`, *optional*):555            Whether or not to return the contrastive loss.556 557        Examples:558 559        ```python560        >>> from PIL import Image561        >>> import requests562        >>> from transformers import AutoProcessor, MetaClip2Model563 564        >>> model = MetaClip2Model.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")565        >>> processor = AutoProcessor.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")566 567        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"568        >>> image = Image.open(requests.get(url, stream=True).raw)569 570        >>> inputs = processor(571        ...     text=["a photo of a cat", "a photo of a dog"], images=image, return_tensors="pt", padding=True572        ... )573 574        >>> outputs = model(**inputs)575        >>> logits_per_image = outputs.logits_per_image  # this is the image-text similarity score576        >>> probs = logits_per_image.softmax(dim=1)  # we can take the softmax to get the label probabilities577        ```"""578        return super().forward(579            input_ids=input_ids,580            pixel_values=pixel_values,581            attention_mask=attention_mask,582            position_ids=position_ids,583            return_loss=return_loss,584            output_attentions=output_attentions,585            output_hidden_states=output_hidden_states,586            interpolate_pos_encoding=interpolate_pos_encoding,587        )588 589    def get_text_features(590        self,591        input_ids: Optional[torch.Tensor] = None,592        attention_mask: Optional[torch.Tensor] = None,593        position_ids: Optional[torch.Tensor] = None,594        output_attentions: Optional[bool] = None,595        output_hidden_states: Optional[bool] = None,596    ):597        r"""598        Returns:599            text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by600            applying the projection layer to the pooled output of [`MetaClip2TextModel`].601 602        Examples:603 604        ```python605        >>> from transformers import AutoTokenizer, MetaClip2Model606 607        >>> model = MetaClip2Model.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")608        >>> tokenizer = AutoTokenizer.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")609 610        >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")611        >>> text_features = model.get_text_features(**inputs)612        ```"""613        return super().get_text_features(614            input_ids=input_ids,615            attention_mask=attention_mask,616            position_ids=position_ids,617            output_attentions=output_attentions,618            output_hidden_states=output_hidden_states,619        )620 621    def get_image_features(622        self,623        pixel_values: Optional[torch.FloatTensor] = None,624        output_attentions: Optional[bool] = None,625        output_hidden_states: Optional[bool] = None,626        interpolate_pos_encoding: bool = False,627    ):628        r"""629        Returns:630            image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by631            applying the projection layer to the pooled output of [`MetaClip2VisionModel`].632 633        Examples:634 635        ```python636        >>> from PIL import Image637        >>> import requests638        >>> from transformers import AutoProcessor, MetaClip2Model639 640        >>> model = MetaClip2Model.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")641        >>> processor = AutoProcessor.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")642 643        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"644        >>> image = Image.open(requests.get(url, stream=True).raw)645 646        >>> inputs = processor(images=image, return_tensors="pt")647 648        >>> image_features = model.get_image_features(**inputs)649        ```"""650        return super().get_image_features(651            pixel_values=pixel_values,652            output_attentions=output_attentions,653            output_hidden_states=output_hidden_states,654            interpolate_pos_encoding=interpolate_pos_encoding,655        )656 657 658class MetaClip2VisionModel(CLIPVisionModel):659    """660    The vision model from MetaClip2 without any head or projection on top.661 662    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the663    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads664    etc.)665 666    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.667    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage668    and behavior.669 670    Args:671        config ([`MetaClip2VisionConfig`]): Model configuration class with all the parameters of the model.672            Initializing with a config file does not load the weights associated with the model, only the673            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.674 675    Examples:676 677    ```python678    >>> from PIL import Image679    >>> import requests680    >>> from transformers import AutoProcessor, MetaClip2VisionModel681 682    >>> model = MetaClip2VisionModel.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")683    >>> processor = AutoProcessor.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")684 685    >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"686    >>> image = Image.open(requests.get(url, stream=True).raw)687 688    >>> inputs = processor(images=image, return_tensors="pt")689 690    >>> outputs = model(**inputs)691    >>> last_hidden_state = outputs.last_hidden_state692    >>> pooled_output = outputs.pooler_output  # pooled CLS states693    ```"""694 695    def forward(696        self,697        pixel_values: Optional[torch.FloatTensor] = None,698        output_attentions: Optional[bool] = None,699        output_hidden_states: Optional[bool] = None,700        interpolate_pos_encoding: bool = False,701    ):702        r"""703        Examples:704 705        ```python706        >>> from PIL import Image707        >>> import requests708        >>> from transformers import AutoProcessor, MetaClip2VisionModel709 710        >>> model = MetaClip2VisionModel.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")711        >>> processor = AutoProcessor.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")712 713        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"714        >>> image = Image.open(requests.get(url, stream=True).raw)715 716        >>> inputs = processor(images=image, return_tensors="pt")717 718        >>> outputs = model(**inputs)719        >>> last_hidden_state = outputs.last_hidden_state720        >>> pooled_output = outputs.pooler_output  # pooled CLS states721        ```"""722        return super().forward(723            pixel_values=pixel_values,724            output_attentions=output_attentions,725            output_hidden_states=output_hidden_states,726            interpolate_pos_encoding=interpolate_pos_encoding,727        )728 729 730class MetaClip2VisionModelWithProjection(CLIPVisionModelWithProjection):731    """732    MetaClip2 vision model with a projection layer on top (a linear layer on top of the pooled output).733 734    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the735    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads736    etc.)737 738    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.739    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage740    and behavior.741 742    Args:743        config ([`MetaClip2VisionConfig`]): Model configuration class with all the parameters of the model.744            Initializing with a config file does not load the weights associated with the model, only the745            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.746 747    Examples:748 749    ```python750    >>> from PIL import Image751    >>> import requests752    >>> from transformers import AutoProcessor, MetaClip2VisionModelWithProjection753 754    >>> model = MetaClip2VisionModelWithProjection.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")755    >>> processor = AutoProcessor.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")756 757    >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"758    >>> image = Image.open(requests.get(url, stream=True).raw)759 760    >>> inputs = processor(images=image, return_tensors="pt")761 762    >>> outputs = model(**inputs)763    >>> image_embeds = outputs.image_embeds764    ```"""765 766    def forward(767        self,768        pixel_values: Optional[torch.FloatTensor] = None,769        output_attentions: Optional[bool] = None,770        output_hidden_states: Optional[bool] = None,771        interpolate_pos_encoding: bool = False,772    ):773        r"""774        Examples:775 776        ```python777        >>> from PIL import Image778        >>> import requests779        >>> from transformers import AutoProcessor, MetaClip2VisionModelWithProjection780 781        >>> model = MetaClip2VisionModelWithProjection.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")782        >>> processor = AutoProcessor.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")783 784        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"785        >>> image = Image.open(requests.get(url, stream=True).raw)786 787        >>> inputs = processor(images=image, return_tensors="pt")788 789        >>> outputs = model(**inputs)790        >>> image_embeds = outputs.image_embeds791        ```"""792        return super().forward(793            pixel_values=pixel_values,794            output_attentions=output_attentions,795            output_hidden_states=output_hidden_states,796            interpolate_pos_encoding=interpolate_pos_encoding,797        )798 799 800class MetaClip2ForImageClassification(CLIPForImageClassification):801    pass802 803 804__all__ = [805    "MetaClip2Config",806    "MetaClip2TextConfig",807    "MetaClip2VisionConfig",808    "MetaClip2Model",809    "MetaClip2PreTrainedModel",810    "MetaClip2TextModel",811    "MetaClip2TextModelWithProjection",812    "MetaClip2VisionModel",813    "MetaClip2VisionModelWithProjection",814    "MetaClip2ForImageClassification",815]816