BAAI/BGE-VL-large
252.4k
1# coding=utf-82# Copyright 2021 The OpenAI Team Authors and The HuggingFace Team. All rights reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""PyTorch CLIP model."""16 17from dataclasses import dataclass18from typing import Any, Optional, Tuple, Union19 20import torch21import torch.utils.checkpoint22from torch import nn23from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss24from PIL import Image25from transformers.activations import ACT2FN26from transformers.modeling_attn_mask_utils import _create_4d_causal_attention_mask, _prepare_4d_attention_mask27from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ImageClassifierOutput28from transformers.modeling_utils import PreTrainedModel29from transformers.pytorch_utils import is_torch_greater_or_equal_than_2_230from transformers.utils import (31 ModelOutput,32 add_code_sample_docstrings,33 add_start_docstrings,34 add_start_docstrings_to_model_forward,35 is_flash_attn_2_available,36 is_flash_attn_greater_or_equal_2_10,37 logging,38 replace_return_docstrings,39)40from transformers.models.clip.configuration_clip import CLIPConfig, CLIPTextConfig, CLIPVisionConfig41from transformers import CLIPProcessor42 43if is_flash_attn_2_available():44 from transformers.modeling_flash_attention_utils import _flash_attention_forward45 46 47logger = logging.get_logger(__name__)48 49# General docstring50_CONFIG_FOR_DOC = "MMRet_CLIP"51 52# Image classification docstring53_IMAGE_CLASS_CHECKPOINT = "JUNJIE99/MMRet-large"54_IMAGE_CLASS_EXPECTED_OUTPUT = "LABEL_0"55 56 57# contrastive loss function, adapted from58# https://sachinruk.github.io/blog/2021-03-07-clip.html59def contrastive_loss(logits: torch.Tensor) -> torch.Tensor:60 return nn.functional.cross_entropy(logits, torch.arange(len(logits), device=logits.device))61 62 63def clip_loss(similarity: torch.Tensor) -> torch.Tensor:64 caption_loss = contrastive_loss(similarity)65 image_loss = contrastive_loss(similarity.t())66 return (caption_loss + image_loss) / 2.067 68 69def _get_vector_norm(tensor: torch.Tensor) -> torch.Tensor:70 """71 This method is equivalent to tensor.norm(p=2, dim=-1, keepdim=True) and used to make72 model `executorch` exportable. See issue https://github.com/pytorch/executorch/issues/356673 """74 square_tensor = torch.pow(tensor, 2)75 sum_tensor = torch.sum(square_tensor, dim=-1, keepdim=True)76 normed_tensor = torch.pow(sum_tensor, 0.5)77 return normed_tensor78 79 80@dataclass81class CLIPVisionModelOutput(ModelOutput):82 """83 Base class for vision model's outputs that also contains image embeddings of the pooling of the last hidden states.84 85 Args:86 image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`):87 The image embeddings obtained by applying the projection layer to the pooler_output.88 last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):89 Sequence of hidden-states at the output of the last layer of the model.90 hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):91 Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +92 one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.93 94 Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.95 attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):96 Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,97 sequence_length)`.98 99 Attentions weights after the attention softmax, used to compute the weighted average in the self-attention100 heads.101 """102 103 image_embeds: Optional[torch.FloatTensor] = None104 last_hidden_state: torch.FloatTensor = None105 hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None106 attentions: Optional[Tuple[torch.FloatTensor, ...]] = None107 108 109@dataclass110class CLIPTextModelOutput(ModelOutput):111 """112 Base class for text model's outputs that also contains a pooling of the last hidden states.113 114 Args:115 text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`):116 The text embeddings obtained by applying the projection layer to the pooler_output.117 last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):118 Sequence of hidden-states at the output of the last layer of the model.119 hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):120 Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +121 one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.122 123 Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.124 attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):125 Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,126 sequence_length)`.127 128 Attentions weights after the attention softmax, used to compute the weighted average in the self-attention129 heads.130 """131 132 text_embeds: Optional[torch.FloatTensor] = None133 last_hidden_state: torch.FloatTensor = None134 hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None135 attentions: Optional[Tuple[torch.FloatTensor, ...]] = None136 137 138@dataclass139class CLIPOutput(ModelOutput):140 """141 Args:142 loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`):143 Contrastive loss for image-text similarity.144 logits_per_image (`torch.FloatTensor` of shape `(image_batch_size, text_batch_size)`):145 The scaled dot product scores between `image_embeds` and `text_embeds`. This represents the image-text146 similarity scores.147 logits_per_text (`torch.FloatTensor` of shape `(text_batch_size, image_batch_size)`):148 The scaled dot product scores between `text_embeds` and `image_embeds`. This represents the text-image149 similarity scores.150 text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`):151 The text embeddings obtained by applying the projection layer to the pooled output of [`CLIPTextModel`].152 image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`):153 The image embeddings obtained by applying the projection layer to the pooled output of [`CLIPVisionModel`].154 text_model_output (`BaseModelOutputWithPooling`):155 The output of the [`CLIPTextModel`].156 vision_model_output (`BaseModelOutputWithPooling`):157 The output of the [`CLIPVisionModel`].158 """159 160 loss: Optional[torch.FloatTensor] = None161 logits_per_image: torch.FloatTensor = None162 logits_per_text: torch.FloatTensor = None163 text_embeds: torch.FloatTensor = None164 image_embeds: torch.FloatTensor = None165 text_model_output: BaseModelOutputWithPooling = None166 vision_model_output: BaseModelOutputWithPooling = None167 168 def to_tuple(self) -> Tuple[Any]:169 return tuple(170 self[k] if k not in ["text_model_output", "vision_model_output"] else getattr(self, k).to_tuple()171 for k in self.keys()172 )173 174 175class CLIPVisionEmbeddings(nn.Module):176 def __init__(self, config: CLIPVisionConfig):177 super().__init__()178 self.config = config179 self.embed_dim = config.hidden_size180 self.image_size = config.image_size181 self.patch_size = config.patch_size182 183 self.class_embedding = nn.Parameter(torch.randn(self.embed_dim))184 185 self.patch_embedding = nn.Conv2d(186 in_channels=config.num_channels,187 out_channels=self.embed_dim,188 kernel_size=self.patch_size,189 stride=self.patch_size,190 bias=False,191 )192 193 self.num_patches = (self.image_size // self.patch_size) ** 2194 self.num_positions = self.num_patches + 1195 self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim)196 self.register_buffer("position_ids", torch.arange(self.num_positions).expand((1, -1)), persistent=False)197 198 def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor:199 batch_size = pixel_values.shape[0]200 target_dtype = self.patch_embedding.weight.dtype201 patch_embeds = self.patch_embedding(pixel_values.to(dtype=target_dtype)) # shape = [*, width, grid, grid]202 patch_embeds = patch_embeds.flatten(2).transpose(1, 2)203 204 class_embeds = self.class_embedding.expand(batch_size, 1, -1)205 embeddings = torch.cat([class_embeds, patch_embeds], dim=1)206 embeddings = embeddings + self.position_embedding(self.position_ids)207 return embeddings208 209 210class CLIPTextEmbeddings(nn.Module):211 def __init__(self, config: CLIPTextConfig):212 super().__init__()213 embed_dim = config.hidden_size214 215 self.token_embedding = nn.Embedding(config.vocab_size, embed_dim)216 self.position_embedding = nn.Embedding(config.max_position_embeddings, embed_dim)217 218 # position_ids (1, len position emb) is contiguous in memory and exported when serialized219 self.register_buffer(220 "position_ids", torch.arange(config.max_position_embeddings).expand((1, -1)), persistent=False221 )222 223 def forward(224 self,225 input_ids: Optional[torch.LongTensor] = None,226 position_ids: Optional[torch.LongTensor] = None,227 inputs_embeds: Optional[torch.FloatTensor] = None,228 ) -> torch.Tensor:229 seq_length = input_ids.shape[-1] if input_ids is not None else inputs_embeds.shape[-2]230 231 if position_ids is None:232 position_ids = self.position_ids[:, :seq_length]233 234 if inputs_embeds is None:235 inputs_embeds = self.token_embedding(input_ids)236 237 position_embeddings = self.position_embedding(position_ids)238 embeddings = inputs_embeds + position_embeddings239 240 return embeddings241 242 243class CLIPAttention(nn.Module):244 """Multi-headed attention from 'Attention Is All You Need' paper"""245 246 def __init__(self, config):247 super().__init__()248 self.config = config249 self.embed_dim = config.hidden_size250 self.num_heads = config.num_attention_heads251 self.head_dim = self.embed_dim // self.num_heads252 if self.head_dim * self.num_heads != self.embed_dim:253 raise ValueError(254 f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:"255 f" {self.num_heads})."256 )257 self.scale = self.head_dim**-0.5258 self.dropout = config.attention_dropout259 260 self.k_proj = nn.Linear(self.embed_dim, self.embed_dim)261 self.v_proj = nn.Linear(self.embed_dim, self.embed_dim)262 self.q_proj = nn.Linear(self.embed_dim, self.embed_dim)263 self.out_proj = nn.Linear(self.embed_dim, self.embed_dim)264 265 def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):266 return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous()267 268 def forward(269 self,270 hidden_states: torch.Tensor,271 attention_mask: Optional[torch.Tensor] = None,272 causal_attention_mask: Optional[torch.Tensor] = None,273 output_attentions: Optional[bool] = False,274 ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:275 """Input shape: Batch x Time x Channel"""276 277 bsz, tgt_len, embed_dim = hidden_states.size()278 279 # get query proj280 query_states = self.q_proj(hidden_states) * self.scale281 key_states = self._shape(self.k_proj(hidden_states), -1, bsz)282 value_states = self._shape(self.v_proj(hidden_states), -1, bsz)283 284 proj_shape = (bsz * self.num_heads, -1, self.head_dim)285 query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape)286 key_states = key_states.view(*proj_shape)287 value_states = value_states.view(*proj_shape)288 289 src_len = key_states.size(1)290 attn_weights = torch.bmm(query_states, key_states.transpose(1, 2))291 292 if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len):293 raise ValueError(294 f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is"295 f" {attn_weights.size()}"296 )297 298 # apply the causal_attention_mask first299 if causal_attention_mask is not None:300 if causal_attention_mask.size() != (bsz, 1, tgt_len, src_len):301 raise ValueError(302 f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is"303 f" {causal_attention_mask.size()}"304 )305 attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + causal_attention_mask306 attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)307 308 if attention_mask is not None:309 if attention_mask.size() != (bsz, 1, tgt_len, src_len):310 raise ValueError(311 f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}"312 )313 attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask314 attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)315 316 attn_weights = nn.functional.softmax(attn_weights, dim=-1)317 318 if output_attentions:319 # this operation is a bit akward, but it's required to320 # make sure that attn_weights keeps its gradient.321 # In order to do so, attn_weights have to reshaped322 # twice and have to be reused in the following323 attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len)324 attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len)325 else:326 attn_weights_reshaped = None327 328 attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training)329 330 attn_output = torch.bmm(attn_probs, value_states)331 332 if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim):333 raise ValueError(334 f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is"335 f" {attn_output.size()}"336 )337 338 attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim)339 attn_output = attn_output.transpose(1, 2)340 attn_output = attn_output.reshape(bsz, tgt_len, embed_dim)341 342 attn_output = self.out_proj(attn_output)343 344 return attn_output, attn_weights_reshaped345 346 347class CLIPFlashAttention2(CLIPAttention):348 """349 CLIPAttention flash attention module. This module inherits from `CLIPAttention` as the weights of the module stays350 untouched. The only required change would be on the forward pass where it needs to correctly call the public API of351 flash attention and deal with padding tokens in case the input contains any of them.352 """353 354 # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2.__init__355 def __init__(self, *args, **kwargs):356 super().__init__(*args, **kwargs)357 358 # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1.359 # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0.360 # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left).361 self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10()362 363 # Adapted from transformers.models.llama.modeling_llama.LlamaFlashAttention2.forward364 def forward(365 self,366 hidden_states: torch.Tensor,367 attention_mask: Optional[torch.Tensor] = None,368 causal_attention_mask: Optional[torch.Tensor] = None,369 output_attentions: Optional[bool] = False,370 ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:371 output_attentions = False372 373 batch_size, q_len, _ = hidden_states.size()374 375 query_states = self.q_proj(hidden_states)376 key_states = self.k_proj(hidden_states)377 value_states = self.v_proj(hidden_states)378 379 # Flash attention requires the input to have the shape380 # batch_size x seq_length x head_dim x hidden_dim381 # therefore we just need to keep the original shape382 query_states = query_states.view(batch_size, q_len, self.num_heads, self.head_dim)383 key_states = key_states.view(batch_size, q_len, self.num_heads, self.head_dim)384 value_states = value_states.view(batch_size, q_len, self.num_heads, self.head_dim)385 386 dropout_rate = self.dropout if self.training else 0.0387 388 # In PEFT, usually we cast the layer norms in float32 for training stability reasons389 # therefore the input hidden states gets silently casted in float32. Hence, we need390 # cast them back in the correct dtype just to be sure everything works as expected.391 # This might slowdown training & inference so it is recommended to not cast the LayerNorms392 # in fp32.393 394 input_dtype = query_states.dtype395 if input_dtype == torch.float32:396 if torch.is_autocast_enabled():397 target_dtype = torch.get_autocast_gpu_dtype()398 # Handle the case where the model is quantized399 elif hasattr(self.config, "_pre_quantization_dtype"):400 target_dtype = self.config._pre_quantization_dtype401 else:402 target_dtype = self.q_proj.weight.dtype403 404 logger.warning_once(405 f"The input hidden states seems to be silently casted in float32, this might be related to"406 f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in"407 f" {target_dtype}."408 )409 410 query_states = query_states.to(target_dtype)411 key_states = key_states.to(target_dtype)412 value_states = value_states.to(target_dtype)413 414 attn_output = _flash_attention_forward(415 query_states,416 key_states,417 value_states,418 attention_mask,419 q_len,420 dropout=dropout_rate,421 is_causal=causal_attention_mask is not None,422 use_top_left_mask=self._flash_attn_uses_top_left_mask,423 )424 425 attn_output = attn_output.reshape(batch_size, q_len, self.embed_dim).contiguous()426 attn_output = self.out_proj(attn_output)427 428 if not output_attentions:429 attn_weights = None430 431 return attn_output, attn_weights432 433 434class CLIPSdpaAttention(CLIPAttention):435 """436 SDPA attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from437 `CLIPAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to438 SDPA API.439 """440 441 # Adapted from CLIPAttention.forward442 def forward(443 self,444 hidden_states: torch.Tensor,445 attention_mask: Optional[torch.Tensor] = None,446 causal_attention_mask: Optional[torch.Tensor] = None,447 output_attentions: Optional[bool] = False,448 ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:449 if output_attentions:450 # TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"` once this is implemented.451 logger.warning_once(452 "CLIPModel is using CLIPSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not "453 "support `output_attentions=True`. Falling back to the manual attention implementation, but specifying "454 "the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can "455 'be removed using the argument `attn_implementation="eager"` when loading the model.'456 )457 return super().forward(458 hidden_states=hidden_states,459 attention_mask=attention_mask,460 causal_attention_mask=causal_attention_mask,461 output_attentions=output_attentions,462 )463 464 # CLIP text model uses both `causal_attention_mask` and `attention_mask`465 if attention_mask is not None and causal_attention_mask is not None:466 attn_mask = attention_mask + causal_attention_mask467 elif causal_attention_mask is not None:468 attn_mask = causal_attention_mask469 else:470 attn_mask = attention_mask471 472 bsz, tgt_len, embed_dim = hidden_states.size()473 474 query_states = self.q_proj(hidden_states)475 key_states = self.k_proj(hidden_states)476 value_states = self.v_proj(hidden_states)477 478 query_states = query_states.view(bsz, -1, self.num_heads, self.head_dim).transpose(1, 2)479 key_states = key_states.view(bsz, -1, self.num_heads, self.head_dim).transpose(1, 2)480 value_states = value_states.view(bsz, -1, self.num_heads, self.head_dim).transpose(1, 2)481 482 # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask,483 # Reference: https://github.com/pytorch/pytorch/issues/112577.484 if not is_torch_greater_or_equal_than_2_2 and query_states.device.type == "cuda" and attn_mask is not None:485 query_states = query_states.contiguous()486 key_states = key_states.contiguous()487 value_states = value_states.contiguous()488 489 # CLIP text model uses both `causal_attention_mask` and `attention_mask` sequentially.490 attn_output = torch.nn.functional.scaled_dot_product_attention(491 query_states,492 key_states,493 value_states,494 attn_mask=attn_mask,495 dropout_p=self.dropout if self.training else 0.0,496 scale=self.scale,497 )498 499 attn_output = attn_output.transpose(1, 2)500 attn_output = attn_output.reshape(bsz, tgt_len, embed_dim)501 502 attn_output = self.out_proj(attn_output)503 504 return attn_output, None505 506 507CLIP_ATTENTION_CLASSES = {508 "eager": CLIPAttention,509 "sdpa": CLIPSdpaAttention,510 "flash_attention_2": CLIPFlashAttention2,511}512 513 514class CLIPMLP(nn.Module):515 def __init__(self, config):516 super().__init__()517 self.config = config518 self.activation_fn = ACT2FN[config.hidden_act]519 self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)520 self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)521 522 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:523 hidden_states = self.fc1(hidden_states)524 hidden_states = self.activation_fn(hidden_states)525 hidden_states = self.fc2(hidden_states)526 return hidden_states527 528 529class CLIPEncoderLayer(nn.Module):530 def __init__(self, config: CLIPConfig):531 super().__init__()532 self.embed_dim = config.hidden_size533 self.self_attn = CLIP_ATTENTION_CLASSES[config._attn_implementation](config)534 self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)535 self.mlp = CLIPMLP(config)536 self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)537 538 def forward(539 self,540 hidden_states: torch.Tensor,541 attention_mask: torch.Tensor,542 causal_attention_mask: torch.Tensor,543 output_attentions: Optional[bool] = False,544 ) -> Tuple[torch.FloatTensor]:545 """546 Args:547 hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`548 attention_mask (`torch.FloatTensor`): attention mask of size549 `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.550 `(config.encoder_attention_heads,)`.551 output_attentions (`bool`, *optional*):552 Whether or not to return the attentions tensors of all attention layers. See `attentions` under553 returned tensors for more detail.554 """555 residual = hidden_states556 557 hidden_states = self.layer_norm1(hidden_states)558 hidden_states, attn_weights = self.self_attn(559 hidden_states=hidden_states,560 attention_mask=attention_mask,561 causal_attention_mask=causal_attention_mask,562 output_attentions=output_attentions,563 )564 hidden_states = residual + hidden_states565 566 residual = hidden_states567 hidden_states = self.layer_norm2(hidden_states)568 hidden_states = self.mlp(hidden_states)569 hidden_states = residual + hidden_states570 571 outputs = (hidden_states,)572 573 if output_attentions:574 outputs += (attn_weights,)575 576 return outputs577 578 579class CLIPPreTrainedModel(PreTrainedModel):580 """581 An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained582 models.583 """584 585 config_class = CLIPConfig586 base_model_prefix = "clip"587 supports_gradient_checkpointing = True588 _supports_sdpa = True589 _supports_flash_attn_2 = True590 591 def _init_weights(self, module):592 """Initialize the weights"""593 factor = self.config.initializer_factor594 if isinstance(module, CLIPTextEmbeddings):595 module.token_embedding.weight.data.normal_(mean=0.0, std=factor * 0.02)596 module.position_embedding.weight.data.normal_(mean=0.0, std=factor * 0.02)597 elif isinstance(module, CLIPVisionEmbeddings):598 factor = self.config.initializer_factor599 nn.init.normal_(module.class_embedding, mean=0.0, std=module.embed_dim**-0.5 * factor)600 nn.init.normal_(module.patch_embedding.weight, std=module.config.initializer_range * factor)601 nn.init.normal_(module.position_embedding.weight, std=module.config.initializer_range * factor)602 elif isinstance(module, CLIPAttention):603 factor = self.config.initializer_factor604 in_proj_std = (module.embed_dim**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor605 out_proj_std = (module.embed_dim**-0.5) * factor606 nn.init.normal_(module.q_proj.weight, std=in_proj_std)607 nn.init.normal_(module.k_proj.weight, std=in_proj_std)608 nn.init.normal_(module.v_proj.weight, std=in_proj_std)609 nn.init.normal_(module.out_proj.weight, std=out_proj_std)610 elif isinstance(module, CLIPMLP):611 factor = self.config.initializer_factor612 in_proj_std = (module.config.hidden_size**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor613 fc_std = (2 * module.config.hidden_size) ** -0.5 * factor614 nn.init.normal_(module.fc1.weight, std=fc_std)615 nn.init.normal_(module.fc2.weight, std=in_proj_std)616 elif isinstance(module, CLIPModel):617 nn.init.normal_(618 module.text_projection.weight,619 std=module.text_embed_dim**-0.5 * self.config.initializer_factor,620 )621 nn.init.normal_(622 module.visual_projection.weight,623 std=module.vision_embed_dim**-0.5 * self.config.initializer_factor,624 )625 elif isinstance(module, CLIPVisionModelWithProjection):626 nn.init.normal_(627 module.visual_projection.weight,628 std=self.config.hidden_size**-0.5 * self.config.initializer_factor,629 )630 elif isinstance(module, CLIPTextModelWithProjection):631 nn.init.normal_(632 module.text_projection.weight,633 std=self.config.hidden_size**-0.5 * self.config.initializer_factor,634 )635 elif isinstance(module, CLIPForImageClassification):636 nn.init.normal_(637 module.classifier.weight,638 std=self.config.vision_config.hidden_size**-0.5 * self.config.initializer_factor,639 )640 641 if isinstance(module, nn.LayerNorm):642 module.bias.data.zero_()643 module.weight.data.fill_(1.0)644 if isinstance(module, nn.Linear) and module.bias is not None:645 module.bias.data.zero_()646 647 648CLIP_START_DOCSTRING = r"""649 This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the650 library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads651 etc.)652 653 This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.654 Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage655 and behavior.656 657 Parameters:658 config ([`CLIPConfig`]): Model configuration class with all the parameters of the model.659 Initializing with a config file does not load the weights associated with the model, only the660 configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.661"""662 663CLIP_TEXT_INPUTS_DOCSTRING = r"""664 Args:665 input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):666 Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide667 it.668 669 Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and670 [`PreTrainedTokenizer.__call__`] for details.671 672 [What are input IDs?](../glossary#input-ids)673 attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):674 Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:675 676 - 1 for tokens that are **not masked**,677 - 0 for tokens that are **masked**.678 679 [What are attention masks?](../glossary#attention-mask)680 position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):681 Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,682 config.max_position_embeddings - 1]`.683 684 [What are position IDs?](../glossary#position-ids)685 output_attentions (`bool`, *optional*):686 Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned687 tensors for more detail.688 output_hidden_states (`bool`, *optional*):689 Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for690 more detail.691 return_dict (`bool`, *optional*):692 Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.693"""694 695CLIP_VISION_INPUTS_DOCSTRING = r"""696 Args:697 pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):698 Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using699 [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details.700 output_attentions (`bool`, *optional*):701 Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned702 tensors for more detail.703 output_hidden_states (`bool`, *optional*):704 Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for705 more detail.706 return_dict (`bool`, *optional*):707 Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.708"""709 710CLIP_INPUTS_DOCSTRING = r"""711 Args:712 input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):713 Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide714 it.715 716 Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and717 [`PreTrainedTokenizer.__call__`] for details.718 719 [What are input IDs?](../glossary#input-ids)720 attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):721 Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:722 723 - 1 for tokens that are **not masked**,724 - 0 for tokens that are **masked**.725 726 [What are attention masks?](../glossary#attention-mask)727 position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):728 Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,729 config.max_position_embeddings - 1]`.730 731 [What are position IDs?](../glossary#position-ids)732 pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):733 Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using734 [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details.735 return_loss (`bool`, *optional*):736 Whether or not to return the contrastive loss.737 output_attentions (`bool`, *optional*):738 Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned739 tensors for more detail.740 output_hidden_states (`bool`, *optional*):741 Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for742 more detail.743 return_dict (`bool`, *optional*):744 Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.745"""746 747 748class CLIPEncoder(nn.Module):749 """750 Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a751 [`CLIPEncoderLayer`].752 753 Args:754 config: CLIPConfig755 """756 757 def __init__(self, config: CLIPConfig):758 super().__init__()759 self.config = config760 self.layers = nn.ModuleList([CLIPEncoderLayer(config) for _ in range(config.num_hidden_layers)])761 self.gradient_checkpointing = False762 763 def forward(764 self,765 inputs_embeds,766 attention_mask: Optional[torch.Tensor] = None,767 causal_attention_mask: Optional[torch.Tensor] = None,768 output_attentions: Optional[bool] = None,769 output_hidden_states: Optional[bool] = None,770 return_dict: Optional[bool] = None,771 ) -> Union[Tuple, BaseModelOutput]:772 r"""773 Args:774 inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):775 Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.776 This is useful if you want more control over how to convert `input_ids` indices into associated vectors777 than the model's internal embedding lookup matrix.778 attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):779 Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:780 781 - 1 for tokens that are **not masked**,782 - 0 for tokens that are **masked**.783 784 [What are attention masks?](../glossary#attention-mask)785 causal_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):786 Causal mask for the text model. Mask values selected in `[0, 1]`:787 788 - 1 for tokens that are **not masked**,789 - 0 for tokens that are **masked**.790 791 [What are attention masks?](../glossary#attention-mask)792 output_attentions (`bool`, *optional*):793 Whether or not to return the attentions tensors of all attention layers. See `attentions` under794 returned tensors for more detail.795 output_hidden_states (`bool`, *optional*):796 Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors797 for more detail.798 return_dict (`bool`, *optional*):799 Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.800 """801 output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions802 output_hidden_states = (803 output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states804 )805 return_dict = return_dict if return_dict is not None else self.config.use_return_dict806 807 encoder_states = () if output_hidden_states else None808 all_attentions = () if output_attentions else None809 810 hidden_states = inputs_embeds811 for idx, encoder_layer in enumerate(self.layers):812 if output_hidden_states:813 encoder_states = encoder_states + (hidden_states,)814 if self.gradient_checkpointing and self.training:815 layer_outputs = self._gradient_checkpointing_func(816 encoder_layer.__call__,817 hidden_states,818 attention_mask,819 causal_attention_mask,820 output_attentions,821 )822 else:823 layer_outputs = encoder_layer(824 hidden_states,825 attention_mask,826 causal_attention_mask,827 output_attentions=output_attentions,828 )829 830 hidden_states = layer_outputs[0]831 832 if output_attentions:833 all_attentions = all_attentions + (layer_outputs[1],)834 835 if output_hidden_states:836 encoder_states = encoder_states + (hidden_states,)837 838 if not return_dict:839 return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)840 return BaseModelOutput(841 last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions842 )843 844 845class CLIPTextTransformer(nn.Module):846 def __init__(self, config: CLIPTextConfig):847 super().__init__()848 self.config = config849 embed_dim = config.hidden_size850 self.embeddings = CLIPTextEmbeddings(config)851 self.encoder = CLIPEncoder(config)852 self.final_layer_norm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)853 854 # For `pooled_output` computation855 self.eos_token_id = config.eos_token_id856 857 # For attention mask, it differs between `flash_attention_2` and other attention implementations858 self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2"859 860 @add_start_docstrings_to_model_forward(CLIP_TEXT_INPUTS_DOCSTRING)861 @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPTextConfig)862 def forward(863 self,864 input_ids: Optional[torch.Tensor] = None,865 attention_mask: Optional[torch.Tensor] = None,866 position_ids: Optional[torch.Tensor] = None,867 output_attentions: Optional[bool] = None,868 output_hidden_states: Optional[bool] = None,869 return_dict: Optional[bool] = None,870 ) -> Union[Tuple, BaseModelOutputWithPooling]:871 r"""872 Returns:873 874 """875 output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions876 output_hidden_states = (877 output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states878 )879 return_dict = return_dict if return_dict is not None else self.config.use_return_dict880 881 if input_ids is None:882 raise ValueError("You have to specify input_ids")883 884 input_shape = input_ids.size()885 input_ids = input_ids.view(-1, input_shape[-1])886 887 hidden_states = self.embeddings(input_ids=input_ids, position_ids=position_ids)888 889 # CLIP's text model uses causal mask, prepare it here.890 # https://github.com/openai/CLIP/blob/cfcffb90e69f37bf2ff1e988237a0fbe41f33c04/clip/model.py#L324891 causal_attention_mask = _create_4d_causal_attention_mask(892 input_shape, hidden_states.dtype, device=hidden_states.device893 )894 895 # expand attention_mask896 if attention_mask is not None and not self._use_flash_attention_2:897 # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]898 attention_mask = _prepare_4d_attention_mask(attention_mask, hidden_states.dtype)899 900 encoder_outputs = self.encoder(901 inputs_embeds=hidden_states,902 attention_mask=attention_mask,903 causal_attention_mask=causal_attention_mask,904 output_attentions=output_attentions,905 output_hidden_states=output_hidden_states,906 return_dict=return_dict,907 )908 909 last_hidden_state = encoder_outputs[0]910 last_hidden_state = self.final_layer_norm(last_hidden_state)911 912 if self.eos_token_id == 2:913 # The `eos_token_id` was incorrect before PR #24773: Let's keep what have been done here.914 # A CLIP model with such `eos_token_id` in the config can't work correctly with extra new tokens added915 # ------------------------------------------------------------916 # text_embeds.shape = [batch_size, sequence_length, transformer.width]917 # take features from the eot embedding (eot_token is the highest number in each sequence)918 # casting to torch.int for onnx compatibility: argmax doesn't support int64 inputs with opset 14919 pooled_output = last_hidden_state[920 torch.arange(last_hidden_state.shape[0], device=last_hidden_state.device),921 input_ids.to(dtype=torch.int, device=last_hidden_state.device).argmax(dim=-1),922 ]923 else:924 # The config gets updated `eos_token_id` from PR #24773 (so the use of exta new tokens is possible)925 pooled_output = last_hidden_state[926 torch.arange(last_hidden_state.shape[0], device=last_hidden_state.device),927 # We need to get the first position of `eos_token_id` value (`pad_token_ids` might equal to `eos_token_id`)928 # Note: we assume each sequence (along batch dim.) contains an `eos_token_id` (e.g. prepared by the tokenizer)929 (input_ids.to(dtype=torch.int, device=last_hidden_state.device) == self.eos_token_id)930 .int()931 .argmax(dim=-1),932 ]933 934 if not return_dict:935 return (last_hidden_state, pooled_output) + encoder_outputs[1:]936 937 return BaseModelOutputWithPooling(938 last_hidden_state=last_hidden_state,939 pooler_output=pooled_output,940 hidden_states=encoder_outputs.hidden_states,941 attentions=encoder_outputs.attentions,942 )943 944 945@add_start_docstrings(946 """The text model from CLIP without any head or projection on top.""",947 CLIP_START_DOCSTRING,948)949class CLIPTextModel(CLIPPreTrainedModel):950 config_class = CLIPTextConfig951 952 _no_split_modules = ["CLIPTextEmbeddings", "CLIPEncoderLayer"]953 954 def __init__(self, config: CLIPTextConfig):955 super().__init__(config)956 self.text_model = CLIPTextTransformer(config)957 # Initialize weights and apply final processing958 self.post_init()959 960 def get_input_embeddings(self) -> nn.Module:961 return self.text_model.embeddings.token_embedding962 963 def set_input_embeddings(self, value):964 self.text_model.embeddings.token_embedding = value965 966 @add_start_docstrings_to_model_forward(CLIP_TEXT_INPUTS_DOCSTRING)967 @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPTextConfig)968 def forward(969 self,970 input_ids: Optional[torch.Tensor] = None,971 attention_mask: Optional[torch.Tensor] = None,972 position_ids: Optional[torch.Tensor] = None,973 output_attentions: Optional[bool] = None,974 output_hidden_states: Optional[bool] = None,975 return_dict: Optional[bool] = None,976 ) -> Union[Tuple, BaseModelOutputWithPooling]:977 r"""978 Returns:979 980 Examples:981 982 ```python983 >>> from transformers import AutoTokenizer, CLIPTextModel984 985 >>> model = CLIPTextModel.from_pretrained("openai/clip-vit-base-patch32")986 >>> tokenizer = AutoTokenizer.from_pretrained("openai/clip-vit-base-patch32")987 988 >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")989 990 >>> outputs = model(**inputs)991 >>> last_hidden_state = outputs.last_hidden_state992 >>> pooled_output = outputs.pooler_output # pooled (EOS token) states993 ```"""994 return_dict = return_dict if return_dict is not None else self.config.use_return_dict995 996 return self.text_model(997 input_ids=input_ids,998 attention_mask=attention_mask,999 position_ids=position_ids,1000 output_attentions=output_attentions,1001 output_hidden_states=output_hidden_states,1002 return_dict=return_dict,1003 )1004 1005 1006class CLIPVisionTransformer(nn.Module):1007 def __init__(self, config: CLIPVisionConfig):1008 super().__init__()1009 self.config = config1010 embed_dim = config.hidden_size1011 1012 self.embeddings = CLIPVisionEmbeddings(config)1013 self.pre_layrnorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)1014 self.encoder = CLIPEncoder(config)1015 self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)1016 1017 @add_start_docstrings_to_model_forward(CLIP_VISION_INPUTS_DOCSTRING)1018 @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPVisionConfig)1019 def forward(1020 self,1021 pixel_values: Optional[torch.FloatTensor] = None,1022 output_attentions: Optional[bool] = None,1023 output_hidden_states: Optional[bool] = None,1024 return_dict: Optional[bool] = None,1025 ) -> Union[Tuple, BaseModelOutputWithPooling]:1026 r"""1027 Returns:1028 1029 """1030 output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions1031 output_hidden_states = (1032 output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states1033 )1034 return_dict = return_dict if return_dict is not None else self.config.use_return_dict1035 1036 if pixel_values is None:1037 raise ValueError("You have to specify pixel_values")1038 1039 hidden_states = self.embeddings(pixel_values)1040 hidden_states = self.pre_layrnorm(hidden_states)1041 1042 encoder_outputs = self.encoder(1043 inputs_embeds=hidden_states,1044 output_attentions=output_attentions,1045 output_hidden_states=output_hidden_states,1046 return_dict=return_dict,1047 )1048 1049 last_hidden_state = encoder_outputs[0]1050 pooled_output = last_hidden_state[:, 0, :]1051 pooled_output = self.post_layernorm(pooled_output)1052 1053 if not return_dict:1054 return (last_hidden_state, pooled_output) + encoder_outputs[1:]1055 1056 return BaseModelOutputWithPooling(1057 last_hidden_state=last_hidden_state,1058 pooler_output=pooled_output,1059 hidden_states=encoder_outputs.hidden_states,1060 attentions=encoder_outputs.attentions,1061 )1062 1063 1064@add_start_docstrings(1065 """The vision model from CLIP without any head or projection on top.""",1066 CLIP_START_DOCSTRING,1067)1068class CLIPVisionModel(CLIPPreTrainedModel):1069 config_class = CLIPVisionConfig1070 main_input_name = "pixel_values"1071 _no_split_modules = ["CLIPEncoderLayer"]1072 1073 def __init__(self, config: CLIPVisionConfig):1074 super().__init__(config)1075 self.vision_model = CLIPVisionTransformer(config)1076 # Initialize weights and apply final processing1077 self.post_init()1078 1079 def get_input_embeddings(self) -> nn.Module:1080 return self.vision_model.embeddings.patch_embedding1081 1082 @add_start_docstrings_to_model_forward(CLIP_VISION_INPUTS_DOCSTRING)1083 @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPVisionConfig)1084 def forward(1085 self,1086 pixel_values: Optional[torch.FloatTensor] = None,1087 output_attentions: Optional[bool] = None,1088 output_hidden_states: Optional[bool] = None,1089 return_dict: Optional[bool] = None,1090 ) -> Union[Tuple, BaseModelOutputWithPooling]:1091 r"""1092 Returns:1093 1094 Examples:1095 1096 ```python1097 >>> from PIL import Image1098 >>> import requests1099 >>> from transformers import AutoProcessor, CLIPVisionModel1100 1101 >>> model = CLIPVisionModel.from_pretrained("openai/clip-vit-base-patch32")1102 >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32")1103 1104 >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"1105 >>> image = Image.open(requests.get(url, stream=True).raw)1106 1107 >>> inputs = processor(images=image, return_tensors="pt")1108 1109 >>> outputs = model(**inputs)1110 >>> last_hidden_state = outputs.last_hidden_state1111 >>> pooled_output = outputs.pooler_output # pooled CLS states1112 ```"""1113 return_dict = return_dict if return_dict is not None else self.config.use_return_dict1114 1115 return self.vision_model(1116 pixel_values=pixel_values,1117 output_attentions=output_attentions,1118 output_hidden_states=output_hidden_states,1119 return_dict=return_dict,1120 )1121 1122 1123@add_start_docstrings(CLIP_START_DOCSTRING)1124class CLIPModel(CLIPPreTrainedModel):1125 config_class = CLIPConfig1126 _no_split_modules = ["CLIPTextEmbeddings", "CLIPEncoderLayer", "CLIPVisionEmbeddings"]1127 1128 def __init__(self, config: CLIPConfig):1129 super().__init__(config)1130 1131 if not isinstance(config.text_config, CLIPTextConfig):1132 raise TypeError(1133 "config.text_config is expected to be of type CLIPTextConfig but is of type"1134 f" {type(config.text_config)}."1135 )1136 1137 if not isinstance(config.vision_config, CLIPVisionConfig):1138 raise TypeError(1139 "config.vision_config is expected to be of type CLIPVisionConfig but is of type"1140 f" {type(config.vision_config)}."1141 )1142 1143 text_config = config.text_config1144 vision_config = config.vision_config1145 1146 self.projection_dim = config.projection_dim1147 self.text_embed_dim = text_config.hidden_size1148 self.vision_embed_dim = vision_config.hidden_size1149 1150 text_model = CLIPTextModel._from_config(text_config, attn_implementation=config._attn_implementation)1151 self.text_model = text_model.text_model1152 1153 vision_model = CLIPVisionModel._from_config(vision_config, attn_implementation=config._attn_implementation)1154 self.vision_model = vision_model.vision_model1155 1156 self.visual_projection = nn.Linear(self.vision_embed_dim, self.projection_dim, bias=False)1157 self.text_projection = nn.Linear(self.text_embed_dim, self.projection_dim, bias=False)1158 self.logit_scale = nn.Parameter(torch.tensor(self.config.logit_scale_init_value))1159 1160 # Initialize weights and apply final processing1161 self.post_init()1162 1163 def set_processor(self, model_name):1164 self.processor = CLIPProcessor.from_pretrained(model_name)1165 1166 @add_start_docstrings_to_model_forward(CLIP_TEXT_INPUTS_DOCSTRING)1167 def get_text_features(1168 self,1169 input_ids: Optional[torch.Tensor] = None,1170 attention_mask: Optional[torch.Tensor] = None,1171 position_ids: Optional[torch.Tensor] = None,1172 output_attentions: Optional[bool] = None,1173 output_hidden_states: Optional[bool] = None,1174 return_dict: Optional[bool] = None,1175 ) -> torch.FloatTensor:1176 r"""1177 Returns:1178 text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by1179 applying the projection layer to the pooled output of [`CLIPTextModel`].1180 1181 Examples:1182 1183 ```python1184 >>> from transformers import AutoTokenizer, CLIPModel1185 1186 >>> model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")1187 >>> tokenizer = AutoTokenizer.from_pretrained("openai/clip-vit-base-patch32")1188 1189 >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")1190 >>> text_features = model.get_text_features(**inputs)1191 ```"""1192 # Use CLIP model's config for some fields (if specified) instead of those of vision & text components.1193 output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions1194 output_hidden_states = (1195 output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states1196 )1197 return_dict = return_dict if return_dict is not None else self.config.use_return_dict1198 1199 text_outputs = self.text_model(1200 input_ids=input_ids,