Aluode/PerceptionLabPortable
0
1# coding=utf-82# Copyright 2024 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved.3#4# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX5# and OPT implementations in this library. It has been modified from its6# original forms to accommodate minor architectural differences compared7# to GPT-NeoX and OPT used by the Meta AI team that trained the model.8#9# Licensed under the Apache License, Version 2.0 (the "License");10# you may not use this file except in compliance with the License.11# You may obtain a copy of the License at12#13# http://www.apache.org/licenses/LICENSE-2.014#15# Unless required by applicable law or agreed to in writing, software16# distributed under the License is distributed on an "AS IS" BASIS,17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.18# See the License for the specific language governing permissions and19# limitations under the License.20"""PyTorch Qwen2-VL model."""21 22from dataclasses import dataclass23from typing import Any, Callable, Optional, Union24 25import torch26import torch.nn as nn27import torch.nn.functional as F28from torch.nn import LayerNorm29 30from ...activations import ACT2FN31from ...cache_utils import Cache, DynamicCache32from ...generation import GenerationMixin33from ...masking_utils import create_causal_mask, create_sliding_window_causal_mask34from ...modeling_flash_attention_utils import FlashAttentionKwargs35from ...modeling_layers import GradientCheckpointingLayer36from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput37from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update38from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel39from ...processing_utils import Unpack40from ...utils import (41 TransformersKwargs,42 auto_docstring,43 can_return_tuple,44 is_torchdynamo_compiling,45 logging,46)47from ...utils.deprecation import deprecate_kwarg48from ..qwen2.modeling_qwen2 import (49 Qwen2RMSNorm,50)51from .configuration_qwen2_vl import Qwen2VLConfig, Qwen2VLTextConfig, Qwen2VLVisionConfig52 53 54logger = logging.get_logger(__name__)55 56 57@dataclass58@auto_docstring(59 custom_intro="""60 Base class for Llava outputs, with hidden states and attentions.61 """62)63class Qwen2VLModelOutputWithPast(ModelOutput):64 r"""65 past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):66 It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).67 68 Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see69 `past_key_values` input) to speed up sequential decoding.70 rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*):71 The rope index difference between sequence length and multimodal rope.72 """73 74 last_hidden_state: Optional[torch.FloatTensor] = None75 past_key_values: Optional[Cache] = None76 hidden_states: Optional[tuple[torch.FloatTensor]] = None77 attentions: Optional[tuple[torch.FloatTensor]] = None78 rope_deltas: Optional[torch.LongTensor] = None79 80 81@dataclass82@auto_docstring(83 custom_intro="""84 Base class for Qwen2VL causal language model (or autoregressive) outputs.85 """86)87class Qwen2VLCausalLMOutputWithPast(ModelOutput):88 r"""89 loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):90 Language modeling loss (for next-token prediction).91 logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):92 Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).93 past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):94 It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).95 96 Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see97 `past_key_values` input) to speed up sequential decoding.98 rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*):99 The rope index difference between sequence length and multimodal rope.100 """101 102 loss: Optional[torch.FloatTensor] = None103 logits: Optional[torch.FloatTensor] = None104 past_key_values: Optional[Cache] = None105 hidden_states: Optional[tuple[torch.FloatTensor]] = None106 attentions: Optional[tuple[torch.FloatTensor]] = None107 rope_deltas: Optional[torch.LongTensor] = None108 109 110class Qwen2VLRotaryEmbedding(nn.Module):111 inv_freq: torch.Tensor # fix linting for `register_buffer`112 113 def __init__(self, config: Qwen2VLTextConfig, device=None):114 super().__init__()115 # BC: "rope_type" was originally "type"116 if hasattr(config, "rope_scaling") and config.rope_scaling is not None:117 self.rope_type = config.rope_scaling.get("rope_type", config.rope_scaling.get("type"))118 else:119 self.rope_type = "default"120 self.max_seq_len_cached = config.max_position_embeddings121 self.original_max_seq_len = config.max_position_embeddings122 123 self.config = config124 self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]125 126 inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)127 self.register_buffer("inv_freq", inv_freq, persistent=False)128 self.original_inv_freq = self.inv_freq129 130 @torch.no_grad()131 @dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope)132 def forward(self, x, position_ids):133 # In contrast to other models, Qwen2_VL has different position ids for the grids134 # So we expand the inv_freq to shape (3, ...)135 inv_freq_expanded = self.inv_freq[None, None, :, None].float().expand(3, position_ids.shape[1], -1, 1)136 position_ids_expanded = position_ids[:, :, None, :].float() # shape (3, bs, 1, positions)137 138 device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"139 with torch.autocast(device_type=device_type, enabled=False): # Force float32140 freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(2, 3)141 emb = torch.cat((freqs, freqs), dim=-1)142 cos = emb.cos() * self.attention_scaling143 sin = emb.sin() * self.attention_scaling144 145 return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)146 147 148# Copied from transformers.models.llama.modeling_llama.rotate_half149def rotate_half(x):150 """Rotates half the hidden dims of the input."""151 x1 = x[..., : x.shape[-1] // 2]152 x2 = x[..., x.shape[-1] // 2 :]153 return torch.cat((-x2, x1), dim=-1)154 155 156def apply_multimodal_rotary_pos_emb(q, k, cos, sin, mrope_section, unsqueeze_dim=1):157 """Applies Rotary Position Embedding with Multimodal Sections to the query and key tensors (https://qwenlm.github.io/blog/qwen2-vl/).158 159 Explanation:160 Multimodal 3D rotary position embedding is an extension to 1D rotary position embedding. The input embedding161 sequence contains vision (images / videos) embedding and text embedding or just contains text embedding. For162 vision embedding part, we apply rotary position embedding on temporal, height and width dimension separately.163 Here we split the channel dimension to 3 chunks for the temporal, height and width rotary position embedding.164 For text embedding part, we just apply 1D rotary position embedding. The three rotary position index (temporal,165 height and width) of text embedding is always the same, so the text embedding rotary position embedding has no166 difference with modern LLMs.167 168 Args:169 q (`torch.Tensor`): The query tensor.170 k (`torch.Tensor`): The key tensor.171 cos (`torch.Tensor`): The cosine part of the rotary embedding.172 sin (`torch.Tensor`): The sine part of the rotary embedding.173 position_ids (`torch.Tensor`):174 The position indices of the tokens corresponding to the query and key tensors. For example, this can be175 used to pass offsetted position ids when working with a KV-cache.176 mrope_section(`List(int)`):177 Multimodal rope section is for channel dimension of temporal, height and width in rope calculation.178 unsqueeze_dim (`int`, *optional*, defaults to 1):179 The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and180 sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note181 that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and182 k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes183 cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have184 the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.185 Returns:186 `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.187 """188 mrope_section = mrope_section * 2189 cos = torch.cat([m[i % 3] for i, m in enumerate(cos.split(mrope_section, dim=-1))], dim=-1).unsqueeze(190 unsqueeze_dim191 )192 sin = torch.cat([m[i % 3] for i, m in enumerate(sin.split(mrope_section, dim=-1))], dim=-1).unsqueeze(193 unsqueeze_dim194 )195 196 q_embed = (q * cos) + (rotate_half(q) * sin)197 k_embed = (k * cos) + (rotate_half(k) * sin)198 return q_embed, k_embed199 200 201def apply_rotary_pos_emb_vision(202 q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor203) -> tuple[torch.Tensor, torch.Tensor]:204 orig_q_dtype = q.dtype205 orig_k_dtype = k.dtype206 q, k = q.float(), k.float()207 cos, sin = cos.unsqueeze(-2).float(), sin.unsqueeze(-2).float()208 q_embed = (q * cos) + (rotate_half(q) * sin)209 k_embed = (k * cos) + (rotate_half(k) * sin)210 q_embed = q_embed.to(orig_q_dtype)211 k_embed = k_embed.to(orig_k_dtype)212 return q_embed, k_embed213 214 215class VisionRotaryEmbedding(nn.Module):216 inv_freq: torch.Tensor # fix linting for `register_buffer`217 218 def __init__(self, dim: int, theta: float = 10000.0) -> None:219 super().__init__()220 inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float) / dim))221 self.register_buffer("inv_freq", inv_freq, persistent=False)222 223 def forward(self, seqlen: int) -> torch.Tensor:224 seq = torch.arange(seqlen, device=self.inv_freq.device, dtype=self.inv_freq.dtype)225 freqs = torch.outer(seq, self.inv_freq)226 return freqs227 228 229class PatchEmbed(nn.Module):230 def __init__(231 self,232 patch_size: int = 14,233 temporal_patch_size: int = 2,234 in_channels: int = 3,235 embed_dim: int = 1152,236 ) -> None:237 super().__init__()238 self.patch_size = patch_size239 self.temporal_patch_size = temporal_patch_size240 self.in_channels = in_channels241 self.embed_dim = embed_dim242 243 kernel_size = [temporal_patch_size, patch_size, patch_size]244 self.proj = nn.Conv3d(in_channels, embed_dim, kernel_size=kernel_size, stride=kernel_size, bias=False)245 246 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:247 target_dtype = self.proj.weight.dtype248 hidden_states = hidden_states.view(249 -1, self.in_channels, self.temporal_patch_size, self.patch_size, self.patch_size250 )251 hidden_states = self.proj(hidden_states.to(dtype=target_dtype)).view(-1, self.embed_dim)252 return hidden_states253 254 255class PatchMerger(nn.Module):256 def __init__(self, dim: int, context_dim: int, spatial_merge_size: int = 2) -> None:257 super().__init__()258 self.hidden_size = context_dim * (spatial_merge_size**2)259 self.ln_q = LayerNorm(context_dim, eps=1e-6)260 self.mlp = nn.Sequential(261 nn.Linear(self.hidden_size, self.hidden_size),262 nn.GELU(),263 nn.Linear(self.hidden_size, dim),264 )265 266 def forward(self, x: torch.Tensor) -> torch.Tensor:267 x = self.mlp(self.ln_q(x).view(-1, self.hidden_size))268 return x269 270 271class VisionMlp(nn.Module):272 def __init__(self, dim: int, hidden_dim: int, hidden_act: str) -> None:273 super().__init__()274 self.fc1 = nn.Linear(dim, hidden_dim)275 self.act = ACT2FN[hidden_act]276 self.fc2 = nn.Linear(hidden_dim, dim)277 278 def forward(self, x) -> torch.Tensor:279 return self.fc2(self.act(self.fc1(x)))280 281 282# Copied from transformers.models.llama.modeling_llama.repeat_kv283def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:284 """285 This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,286 num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)287 """288 batch, num_key_value_heads, slen, head_dim = hidden_states.shape289 if n_rep == 1:290 return hidden_states291 hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)292 return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)293 294 295def eager_attention_forward(296 module: nn.Module,297 query: torch.Tensor,298 key: torch.Tensor,299 value: torch.Tensor,300 attention_mask: Optional[torch.Tensor],301 scaling: float,302 dropout: float = 0.0,303 **kwargs,304):305 key_states = repeat_kv(key, module.num_key_value_groups)306 value_states = repeat_kv(value, module.num_key_value_groups)307 308 attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling309 if attention_mask is not None:310 causal_mask = attention_mask[:, :, :, : key_states.shape[-2]]311 attn_weights = attn_weights + causal_mask312 313 attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)314 attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)315 attn_output = torch.matmul(attn_weights, value_states)316 attn_output = attn_output.transpose(1, 2).contiguous()317 318 return attn_output, attn_weights319 320 321class VisionAttention(nn.Module):322 def __init__(self, config: Qwen2VLVisionConfig) -> None:323 super().__init__()324 self.dim = config.embed_dim325 self.num_heads = config.num_heads326 self.head_dim = self.dim // self.num_heads327 self.num_key_value_groups = 1 # needed for eager attention328 self.qkv = nn.Linear(self.dim, self.dim * 3, bias=True)329 self.proj = nn.Linear(self.dim, self.dim)330 self.scaling = self.head_dim**-0.5331 self.config = config332 self.attention_dropout = 0.0333 self.is_causal = False334 335 def forward(336 self,337 hidden_states: torch.Tensor,338 cu_seqlens: torch.Tensor,339 rotary_pos_emb: Optional[torch.Tensor] = None,340 position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None,341 **kwargs,342 ) -> torch.Tensor:343 seq_length = hidden_states.shape[0]344 query_states, key_states, value_states = (345 self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0)346 )347 cos, sin = position_embeddings348 query_states, key_states = apply_rotary_pos_emb_vision(query_states, key_states, cos, sin)349 350 query_states = query_states.transpose(0, 1).unsqueeze(0)351 key_states = key_states.transpose(0, 1).unsqueeze(0)352 value_states = value_states.transpose(0, 1).unsqueeze(0)353 354 attention_interface: Callable = eager_attention_forward355 if self.config._attn_implementation != "eager":356 attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]357 358 if self.config._attn_implementation == "flash_attention_2":359 # Flash Attention 2: Use cu_seqlens for variable length attention360 max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max()361 attn_output, _ = attention_interface(362 self,363 query_states,364 key_states,365 value_states,366 attention_mask=None,367 scaling=self.scaling,368 dropout=0.0 if not self.training else self.attention_dropout,369 cu_seq_lens_q=cu_seqlens,370 cu_seq_lens_k=cu_seqlens,371 max_length_q=max_seqlen,372 max_length_k=max_seqlen,373 is_causal=False,374 **kwargs,375 )376 else:377 # Other implementations: Process each chunk separately378 lengths = cu_seqlens[1:] - cu_seqlens[:-1]379 splits = [380 torch.split(tensor, lengths.tolist(), dim=2) for tensor in (query_states, key_states, value_states)381 ]382 383 attn_outputs = [384 attention_interface(385 self,386 q,387 k,388 v,389 attention_mask=None,390 scaling=self.scaling,391 dropout=0.0 if not self.training else self.attention_dropout,392 is_causal=False,393 **kwargs,394 )[0]395 for q, k, v in zip(*splits)396 ]397 attn_output = torch.cat(attn_outputs, dim=1)398 399 attn_output = attn_output.reshape(seq_length, -1).contiguous()400 attn_output = self.proj(attn_output)401 return attn_output402 403 404class Qwen2VLVisionBlock(GradientCheckpointingLayer):405 def __init__(self, config, attn_implementation: str = "sdpa") -> None:406 super().__init__()407 self.norm1 = LayerNorm(config.embed_dim, eps=1e-6)408 self.norm2 = LayerNorm(config.embed_dim, eps=1e-6)409 mlp_hidden_dim = int(config.embed_dim * config.mlp_ratio)410 411 self.attn = VisionAttention(config=config)412 self.mlp = VisionMlp(dim=config.embed_dim, hidden_dim=mlp_hidden_dim, hidden_act=config.hidden_act)413 414 def forward(415 self,416 hidden_states: torch.Tensor,417 cu_seqlens: torch.Tensor,418 rotary_pos_emb: Optional[torch.Tensor] = None,419 position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None,420 **kwargs,421 ) -> torch.Tensor:422 hidden_states = hidden_states + self.attn(423 self.norm1(hidden_states),424 cu_seqlens=cu_seqlens,425 rotary_pos_emb=rotary_pos_emb,426 position_embeddings=position_embeddings,427 **kwargs,428 )429 hidden_states = hidden_states + self.mlp(self.norm2(hidden_states))430 return hidden_states431 432 433# Copied from transformers.models.qwen2.modeling_qwen2.Qwen2MLP434class Qwen2MLP(nn.Module):435 def __init__(self, config):436 super().__init__()437 self.config = config438 self.hidden_size = config.hidden_size439 self.intermediate_size = config.intermediate_size440 self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)441 self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)442 self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)443 self.act_fn = ACT2FN[config.hidden_act]444 445 def forward(self, x):446 down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))447 return down_proj448 449 450class Qwen2VLAttention(nn.Module):451 """452 Multi-headed attention from 'Attention Is All You Need' paper. Modified to use sliding window attention: Longformer453 and "Generating Long Sequences with Sparse Transformers".454 """455 456 def __init__(self, config: Qwen2VLTextConfig, layer_idx: Optional[int] = None):457 super().__init__()458 self.config = config459 self.layer_idx = layer_idx460 if layer_idx is None:461 logger.warning_once(462 f"Instantiating {self.__class__.__name__} without passing `layer_idx` is not recommended and will "463 "to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` "464 "when creating this class."465 )466 467 self.hidden_size = config.hidden_size468 self.num_heads = config.num_attention_heads469 self.head_dim = self.hidden_size // self.num_heads470 self.num_key_value_heads = config.num_key_value_heads471 self.num_key_value_groups = self.num_heads // self.num_key_value_heads472 self.is_causal = True473 self.attention_dropout = config.attention_dropout474 self.rope_scaling = config.rope_scaling475 self.scaling = self.head_dim**-0.5476 477 if (self.head_dim * self.num_heads) != self.hidden_size:478 raise ValueError(479 f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"480 f" and `num_heads`: {self.num_heads})."481 )482 self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=True)483 self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True)484 self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True)485 self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False)486 self.sliding_window = config.sliding_window if config.layer_types[layer_idx] == "sliding_attention" else None487 488 self.rotary_emb = Qwen2VLRotaryEmbedding(config=config)489 490 @deprecate_kwarg("past_key_value", new_name="past_key_values", version="4.58")491 def forward(492 self,493 hidden_states: torch.Tensor,494 attention_mask: Optional[torch.Tensor] = None,495 position_ids: Optional[torch.LongTensor] = None,496 past_key_values: Optional[Cache] = None,497 output_attentions: bool = False,498 use_cache: bool = False,499 cache_position: Optional[torch.LongTensor] = None,500 position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, # necessary, but kept here for BC501 **kwargs: Unpack[FlashAttentionKwargs],502 ) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[tuple[torch.Tensor]]]:503 bsz, q_len, _ = hidden_states.size()504 505 query_states = self.q_proj(hidden_states)506 key_states = self.k_proj(hidden_states)507 value_states = self.v_proj(hidden_states)508 509 query_states = query_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2)510 key_states = key_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2)511 value_states = value_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2)512 513 cos, sin = position_embeddings514 query_states, key_states = apply_multimodal_rotary_pos_emb(515 query_states, key_states, cos, sin, self.rope_scaling["mrope_section"]516 )517 518 if past_key_values is not None:519 cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} # Specific to RoPE models520 key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs)521 522 attention_interface: Callable = eager_attention_forward523 if self.config._attn_implementation != "eager":524 attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]525 526 attn_output, attn_weights = attention_interface(527 self,528 query_states,529 key_states,530 value_states,531 attention_mask,532 dropout=0.0 if not self.training else self.attention_dropout,533 scaling=self.scaling,534 sliding_window=self.sliding_window,535 position_ids=position_ids, # pass positions for FA2536 **kwargs,537 )538 539 attn_output = attn_output.reshape(bsz, q_len, -1).contiguous()540 attn_output = self.o_proj(attn_output)541 return attn_output, attn_weights542 543 544class Qwen2VLDecoderLayer(GradientCheckpointingLayer):545 def __init__(self, config: Qwen2VLTextConfig, layer_idx: int):546 super().__init__()547 self.hidden_size = config.hidden_size548 549 if config.use_sliding_window and config._attn_implementation != "flash_attention_2":550 logger.warning_once(551 f"Sliding Window Attention is enabled but not implemented for `{config._attn_implementation}`; "552 "unexpected results may be encountered."553 )554 self.self_attn = Qwen2VLAttention(config, layer_idx)555 556 self.mlp = Qwen2MLP(config)557 self.input_layernorm = Qwen2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)558 self.post_attention_layernorm = Qwen2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)559 self.attention_type = config.layer_types[layer_idx]560 561 @deprecate_kwarg("past_key_value", new_name="past_key_values", version="4.58")562 def forward(563 self,564 hidden_states: torch.Tensor,565 attention_mask: Optional[torch.Tensor] = None,566 position_ids: Optional[torch.LongTensor] = None,567 past_key_values: Optional[Cache] = None,568 output_attentions: Optional[bool] = False,569 use_cache: Optional[bool] = False,570 cache_position: Optional[torch.LongTensor] = None,571 position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, # necessary, but kept here for BC572 **kwargs: Unpack[FlashAttentionKwargs],573 ) -> tuple[torch.FloatTensor, Optional[tuple[torch.FloatTensor, torch.FloatTensor]]]:574 """575 Args:576 hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`577 attention_mask (`torch.FloatTensor`, *optional*): attention mask of size578 `(batch, sequence_length)` where padding elements are indicated by 0.579 output_attentions (`bool`, *optional*):580 Whether or not to return the attentions tensors of all attention layers. See `attentions` under581 returned tensors for more detail.582 use_cache (`bool`, *optional*):583 If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding584 (see `past_key_values`).585 past_key_values (`Cache`, *optional*): cached past key and value projection states586 cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):587 Indices depicting the position of the input sequence tokens in the sequence.588 position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):589 Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,590 with `head_dim` being the embedding dimension of each attention head.591 kwargs (`dict`, *optional*):592 Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code593 into the model594 """595 596 residual = hidden_states597 598 hidden_states = self.input_layernorm(hidden_states)599 600 # Self Attention601 hidden_states, self_attn_weights = self.self_attn(602 hidden_states=hidden_states,603 attention_mask=attention_mask,604 position_ids=position_ids,605 past_key_values=past_key_values,606 output_attentions=output_attentions,607 use_cache=use_cache,608 cache_position=cache_position,609 position_embeddings=position_embeddings,610 **kwargs,611 )612 hidden_states = residual + hidden_states613 614 # Fully Connected615 residual = hidden_states616 hidden_states = self.post_attention_layernorm(hidden_states)617 hidden_states = self.mlp(hidden_states)618 hidden_states = residual + hidden_states619 620 outputs = (hidden_states,)621 622 if output_attentions:623 outputs += (self_attn_weights,)624 625 return outputs626 627 628@auto_docstring629class Qwen2VLPreTrainedModel(PreTrainedModel):630 config: Qwen2VLConfig631 base_model_prefix = "model"632 supports_gradient_checkpointing = True633 _no_split_modules = ["Qwen2VLDecoderLayer", "Qwen2VLVisionBlock"]634 _skip_keys_device_placement = "past_key_values"635 _supports_flash_attn = True636 _supports_sdpa = True637 638 _can_compile_fullgraph = True639 _supports_attention_backend = True640 641 642@auto_docstring643class Qwen2VisionTransformerPretrainedModel(Qwen2VLPreTrainedModel):644 config: Qwen2VLVisionConfig645 _no_split_modules = ["Qwen2VLVisionBlock"]646 647 def __init__(self, config) -> None:648 super().__init__(config)649 self.spatial_merge_size = config.spatial_merge_size650 651 self.patch_embed = PatchEmbed(652 patch_size=config.patch_size,653 temporal_patch_size=config.temporal_patch_size,654 in_channels=config.in_channels,655 embed_dim=config.embed_dim,656 )657 658 head_dim = config.embed_dim // config.num_heads659 self.rotary_pos_emb = VisionRotaryEmbedding(head_dim // 2)660 661 self.blocks = nn.ModuleList([Qwen2VLVisionBlock(config) for _ in range(config.depth)])662 self.merger = PatchMerger(663 dim=config.hidden_size, context_dim=config.embed_dim, spatial_merge_size=config.spatial_merge_size664 )665 self.gradient_checkpointing = False666 667 def get_dtype(self) -> torch.dtype:668 return self.blocks[0].mlp.fc2.weight.dtype669 670 def get_device(self) -> torch.device:671 return self.blocks[0].mlp.fc2.weight.device672 673 def rot_pos_emb(self, grid_thw):674 pos_ids = []675 for t, h, w in grid_thw:676 hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)677 hpos_ids = hpos_ids.reshape(678 h // self.spatial_merge_size,679 self.spatial_merge_size,680 w // self.spatial_merge_size,681 self.spatial_merge_size,682 )683 hpos_ids = hpos_ids.permute(0, 2, 1, 3)684 hpos_ids = hpos_ids.flatten()685 686 wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)687 wpos_ids = wpos_ids.reshape(688 h // self.spatial_merge_size,689 self.spatial_merge_size,690 w // self.spatial_merge_size,691 self.spatial_merge_size,692 )693 wpos_ids = wpos_ids.permute(0, 2, 1, 3)694 wpos_ids = wpos_ids.flatten()695 pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1))696 pos_ids = torch.cat(pos_ids, dim=0)697 max_grid_size = grid_thw[:, 1:].max()698 rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size)699 rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1)700 return rotary_pos_emb701 702 @auto_docstring703 def forward(704 self,705 hidden_states: torch.Tensor,706 grid_thw: torch.Tensor,707 **kwargs,708 ) -> torch.Tensor:709 r"""710 grid_thw (`torch.LongTensor` of shape `(num_images, 3)`):711 The temporal, height and width dimensions of feature shape for each image. Each row contains [t, h, w] values.712 """713 hidden_states = self.patch_embed(hidden_states)714 rotary_pos_emb = self.rot_pos_emb(grid_thw)715 emb = torch.cat((rotary_pos_emb, rotary_pos_emb), dim=-1)716 position_embeddings = (emb.cos(), emb.sin())717 718 cu_seqlens = torch.repeat_interleave(grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]).cumsum(719 dim=0,720 # Select dtype based on the following factors:721 # - FA2 requires that cu_seqlens_q must have dtype int32722 # - torch.onnx.export requires that cu_seqlens_q must have same dtype as grid_thw723 # See https://github.com/huggingface/transformers/pull/34852 for more information724 dtype=grid_thw.dtype if torch.jit.is_tracing() else torch.int32,725 )726 cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0)727 728 for blk in self.blocks:729 hidden_states = blk(730 hidden_states,731 cu_seqlens=cu_seqlens,732 position_embeddings=position_embeddings,733 **kwargs,734 )735 736 return self.merger(hidden_states)737 738 739@auto_docstring740class Qwen2VLTextModel(Qwen2VLPreTrainedModel):741 config: Qwen2VLTextConfig742 743 def __init__(self, config: Qwen2VLTextConfig):744 super().__init__(config)745 self.padding_idx = config.pad_token_id746 self.vocab_size = config.vocab_size747 748 self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)749 self.layers = nn.ModuleList(750 [Qwen2VLDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]751 )752 self._attn_implementation = config._attn_implementation753 self.norm = Qwen2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)754 self.rotary_emb = Qwen2VLRotaryEmbedding(config=config)755 self.has_sliding_layers = "sliding_attention" in self.config.layer_types756 757 self.gradient_checkpointing = False758 # Initialize weights and apply final processing759 self.post_init()760 761 @auto_docstring762 def forward(763 self,764 input_ids: Optional[torch.LongTensor] = None,765 attention_mask: Optional[torch.Tensor] = None,766 position_ids: Optional[torch.LongTensor] = None,767 past_key_values: Optional[Cache] = None,768 inputs_embeds: Optional[torch.FloatTensor] = None,769 use_cache: Optional[bool] = None,770 output_attentions: Optional[bool] = None,771 output_hidden_states: Optional[bool] = None,772 return_dict: Optional[bool] = None,773 cache_position: Optional[torch.LongTensor] = None,774 **kwargs: Unpack[FlashAttentionKwargs],775 ) -> Union[tuple, BaseModelOutputWithPast]:776 output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions777 output_hidden_states = (778 output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states779 )780 use_cache = use_cache if use_cache is not None else self.config.use_cache781 782 return_dict = return_dict if return_dict is not None else self.config.use_return_dict783 784 if (input_ids is None) ^ (inputs_embeds is not None):785 raise ValueError("You must specify exactly one of input_ids or inputs_embeds")786 787 if self.gradient_checkpointing and self.training:788 if use_cache:789 logger.warning_once(790 "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."791 )792 use_cache = False793 794 # torch.jit.trace() doesn't support cache objects in the output795 if use_cache and past_key_values is None and not torch.jit.is_tracing():796 past_key_values = DynamicCache(config=self.config)797 798 if inputs_embeds is None:799 inputs_embeds = self.embed_tokens(input_ids)800 801 if cache_position is None:802 past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0803 cache_position = torch.arange(804 past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device805 )806 807 # the hard coded `3` is for temporal, height and width.808 if position_ids is None:809 position_ids = cache_position.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1)810 elif position_ids.ndim == 2:811 position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1)812 813 # NOTE: we need to pass text position ids for packing. Qwen2-VL uses 3D positions814 # where each dim indicates visual spatial positions for temporal/height/width grids.815 # There are two scenarios when FA2-like packed masking might be activated.816 # 1. User specifically passed packed `position_ids` and no attention mask.817 # In this case we expect the useer to create correct position ids for all 3 grids818 # and prepend text-only position ids to it. The final tensor will be [4, bs, seq-len]819 # 2. User runs forward with no attention mask and no position ids. In this case, position ids820 # are prepared by the model (`get_rope_index`) as `[4, bs, seq-len]` tensor. Text-only positions are821 # prepended by us when creating positions so that the mask is constructed correctly. NOTE: failing to pass822 # text-only positions will cause incorrect mask construction, do not change `prepare_input_for_generation`823 if position_ids.ndim == 3 and position_ids.shape[0] == 4:824 text_position_ids = position_ids[0]825 position_ids = position_ids[1:]826 else:827 # If inputs are not packed (usual 3D positions), do not prepare mask from position_ids828 text_position_ids = None829 830 # It may already have been prepared by e.g. `generate`831 if not isinstance(causal_mask_mapping := attention_mask, dict):832 # Prepare mask arguments833 mask_kwargs = {834 "config": self.config,835 "input_embeds": inputs_embeds,836 "attention_mask": attention_mask,837 "cache_position": cache_position,838 "past_key_values": past_key_values,839 "position_ids": text_position_ids,840 }841 # Create the masks842 causal_mask_mapping = {843 "full_attention": create_causal_mask(**mask_kwargs),844 }845 # The sliding window alternating layers are not always activated depending on the config846 if self.has_sliding_layers:847 causal_mask_mapping["sliding_attention"] = create_sliding_window_causal_mask(**mask_kwargs)848 849 hidden_states = inputs_embeds850 851 # create position embeddings to be shared across the decoder layers852 position_embeddings = self.rotary_emb(hidden_states, position_ids)853 854 # decoder layers855 all_hidden_states = () if output_hidden_states else None856 all_self_attns = () if output_attentions else None857 858 for decoder_layer in self.layers:859 if output_hidden_states:860 all_hidden_states += (hidden_states,)861 862 layer_outputs = decoder_layer(863 hidden_states,864 attention_mask=causal_mask_mapping[decoder_layer.attention_type],865 position_ids=text_position_ids,866 past_key_values=past_key_values,867 output_attentions=output_attentions,868 use_cache=use_cache,869 cache_position=cache_position,870 position_embeddings=position_embeddings,871 **kwargs,872 )873 874 hidden_states = layer_outputs[0]875 876 if output_attentions:877 all_self_attns += (layer_outputs[1],)878 879 hidden_states = self.norm(hidden_states)880 881 # add hidden states from the last decoder layer882 if output_hidden_states:883 all_hidden_states += (hidden_states,)884 885 if not return_dict:886 return tuple(887 v for v in [hidden_states, past_key_values, all_hidden_states, all_self_attns] if v is not None888 )889 return BaseModelOutputWithPast(890 last_hidden_state=hidden_states,891 past_key_values=past_key_values,892 hidden_states=all_hidden_states,893 attentions=all_self_attns,894 )895 896 897@auto_docstring898class Qwen2VLModel(Qwen2VLPreTrainedModel):899 base_model_prefix = ""900 _checkpoint_conversion_mapping = {"^model": "language_model"}901 # Reference: fix gemma3 grad acc #37208902 accepts_loss_kwargs = False903 904 def __init__(self, config: Qwen2VLConfig):905 super().__init__(config)906 self.visual = Qwen2VisionTransformerPretrainedModel._from_config(config.vision_config)907 self.language_model = Qwen2VLTextModel._from_config(config.text_config)908 self.rope_deltas = None # cache rope_deltas here909 910 # Initialize weights and apply final processing911 self.post_init()912 913 def get_input_embeddings(self):914 return self.language_model.get_input_embeddings()915 916 def set_input_embeddings(self, value):917 self.language_model.set_input_embeddings(value)918 919 def set_decoder(self, decoder):920 self.language_model = decoder921 922 def get_decoder(self):923 return self.language_model924 925 def get_rope_index(926 self,927 input_ids: Optional[torch.LongTensor] = None,928 image_grid_thw: Optional[torch.LongTensor] = None,929 video_grid_thw: Optional[torch.LongTensor] = None,930 attention_mask: Optional[torch.Tensor] = None,931 ) -> tuple[torch.Tensor, torch.Tensor]:932 """933 Calculate the 3D rope index based on image and video's temporal, height and width in LLM.934 935 Explanation:936 Each embedding sequence contains vision embedding and text embedding or just contains text embedding.937 938 For pure text embedding sequence, the rotary position embedding has no difference with modern LLMs.939 Examples:940 input_ids: [T T T T T], here T is for text.941 temporal position_ids: [0, 1, 2, 3, 4]942 height position_ids: [0, 1, 2, 3, 4]943 width position_ids: [0, 1, 2, 3, 4]944 945 For vision and text embedding sequence, we calculate 3D rotary position embedding for vision part946 and 1D rotary position embedding for text part.947 Examples:948 Assume we have a video input with 3 temporal patches, 2 height patches and 2 width patches.949 input_ids: [V V V V V V V V V V V V T T T T T], here V is for vision.950 vision temporal position_ids: [0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2]951 vision height position_ids: [0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1]952 vision width position_ids: [0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1]953 text temporal position_ids: [3, 4, 5, 6, 7]954 text height position_ids: [3, 4, 5, 6, 7]955 text width position_ids: [3, 4, 5, 6, 7]956 Here we calculate the text start position_ids as the max vision position_ids plus 1.957 958 Args:959 input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):960 Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide961 it.962 image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*):963 The temporal, height and width of feature shape of each image in LLM.964 video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*):965 The temporal, height and width of feature shape of each video in LLM.966 attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):967 Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:968 969 - 1 for tokens that are **not masked**,970 - 0 for tokens that are **masked**.971 972 Returns:973 position_ids (`torch.LongTensor` of shape `(3, batch_size, sequence_length)`)974 mrope_position_deltas (`torch.Tensor` of shape `(batch_size)`)975 """976 spatial_merge_size = self.config.vision_config.spatial_merge_size977 image_token_id = self.config.image_token_id978 video_token_id = self.config.video_token_id979 vision_start_token_id = self.config.vision_start_token_id980 mrope_position_deltas = []981 if input_ids is not None and (image_grid_thw is not None or video_grid_thw is not None):982 total_input_ids = input_ids983 if attention_mask is None:984 attention_mask = torch.ones_like(total_input_ids)985 position_ids = torch.ones(986 3, input_ids.shape[0], input_ids.shape[1], dtype=input_ids.dtype, device=input_ids.device987 )988 image_index, video_index = 0, 0989 for i, input_ids in enumerate(total_input_ids):990 input_ids = input_ids[attention_mask[i].to(input_ids.device) == 1]991 image_nums, video_nums = 0, 0992 vision_start_indices = torch.argwhere(input_ids == vision_start_token_id).squeeze(1)993 vision_tokens = input_ids[vision_start_indices + 1]994 image_nums = (vision_tokens == image_token_id).sum()995 video_nums = (vision_tokens == video_token_id).sum()996 input_tokens = input_ids.tolist()997 llm_pos_ids_list: list = []998 st = 0999 remain_images, remain_videos = image_nums, video_nums1000 for _ in range(image_nums + video_nums):1001 if image_token_id in input_tokens and remain_images > 0:1002 ed_image = input_tokens.index(image_token_id, st)1003 else:1004 ed_image = len(input_tokens) + 11005 if video_token_id in input_tokens and remain_videos > 0:1006 ed_video = input_tokens.index(video_token_id, st)1007 else:1008 ed_video = len(input_tokens) + 11009 if ed_image < ed_video:1010 t, h, w = (1011 image_grid_thw[image_index][0],1012 image_grid_thw[image_index][1],1013 image_grid_thw[image_index][2],1014 )1015 image_index += 11016 remain_images -= 11017 ed = ed_image1018 else:1019 t, h, w = (1020 video_grid_thw[video_index][0],1021 video_grid_thw[video_index][1],1022 video_grid_thw[video_index][2],1023 )1024 video_index += 11025 remain_videos -= 11026 ed = ed_video1027 llm_grid_t, llm_grid_h, llm_grid_w = (1028 t.item(),1029 h.item() // spatial_merge_size,1030 w.item() // spatial_merge_size,1031 )1032 text_len = ed - st1033 1034 st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 01035 llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx)1036 1037 t_index = torch.arange(llm_grid_t).view(-1, 1).expand(-1, llm_grid_h * llm_grid_w).flatten()1038 h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(llm_grid_t, -1, llm_grid_w).flatten()1039 w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(llm_grid_t, llm_grid_h, -1).flatten()1040 llm_pos_ids_list.append(torch.stack([t_index, h_index, w_index]) + text_len + st_idx)1041 st = ed + llm_grid_t * llm_grid_h * llm_grid_w1042 1043 if st < len(input_tokens):1044 st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 01045 text_len = len(input_tokens) - st1046 llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx)1047 1048 llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)1049 position_ids[..., i, attention_mask[i] == 1] = llm_positions.to(position_ids.device)1050 mrope_position_deltas.append(llm_positions.max() + 1 - len(total_input_ids[i]))1051 mrope_position_deltas = torch.tensor(mrope_position_deltas, device=input_ids.device).unsqueeze(1)1052 return position_ids, mrope_position_deltas1053 else:1054 if attention_mask is not None:1055 position_ids = attention_mask.long().cumsum(-1) - 11056 position_ids.masked_fill_(attention_mask == 0, 1)1057 position_ids = position_ids.unsqueeze(0).expand(3, -1, -1).to(attention_mask.device)1058 max_position_ids = position_ids.max(0, keepdim=False)[0].max(-1, keepdim=True)[0]1059 mrope_position_deltas = max_position_ids + 1 - attention_mask.shape[-1]1060 else:1061 position_ids = (1062 torch.arange(input_ids.shape[1], device=input_ids.device)1063 .view(1, 1, -1)1064 .expand(3, input_ids.shape[0], -1)1065 )1066 mrope_position_deltas = torch.zeros(1067 [input_ids.shape[0], 1],1068 device=input_ids.device,1069 dtype=input_ids.dtype,1070 )1071 1072 return position_ids, mrope_position_deltas1073 1074 def get_video_features(1075 self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None1076 ):1077 """1078 Encodes videos into continuous embeddings that can be forwarded to the language model.1079 1080 Args:1081 pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):1082 The tensors corresponding to the input videos.1083 video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*):1084 The temporal, height and width of feature shape of each video in LLM.1085 """1086 pixel_values_videos = pixel_values_videos.type(self.visual.dtype)1087 video_embeds = self.visual(pixel_values_videos, grid_thw=video_grid_thw)1088 split_sizes = (video_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist()1089 video_embeds = torch.split(video_embeds, split_sizes)1090 return video_embeds1091 1092 def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None):1093 """1094 Encodes images into continuous embeddings that can be forwarded to the language model.1095 1096 Args:1097 pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):1098 The tensors corresponding to the input images.1099 image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*):1100 The temporal, height and width of feature shape of each image in LLM.1101 """1102 pixel_values = pixel_values.type(self.visual.dtype)1103 image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw)1104 split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist()1105 image_embeds = torch.split(image_embeds, split_sizes)1106 return image_embeds1107 1108 def get_placeholder_mask(1109 self,1110 input_ids: torch.LongTensor,1111 inputs_embeds: torch.FloatTensor,1112 image_features: Optional[torch.FloatTensor] = None,1113 video_features: Optional[torch.FloatTensor] = None,1114 ):1115 """1116 Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is1117 equal to the length of multimodal features. If the lengths are different, an error is raised.1118 """1119 if input_ids is None:1120 special_image_mask = inputs_embeds == self.get_input_embeddings()(1121 torch.tensor(self.config.image_token_id, dtype=torch.long, device=inputs_embeds.device)1122 )1123 special_image_mask = special_image_mask.all(-1)1124 special_video_mask = inputs_embeds == self.get_input_embeddings()(1125 torch.tensor(self.config.video_token_id, dtype=torch.long, device=inputs_embeds.device)1126 )1127 special_video_mask = special_video_mask.all(-1)1128 else:1129 special_image_mask = input_ids == self.config.image_token_id1130 special_video_mask = input_ids == self.config.video_token_id1131 1132 n_image_tokens = special_image_mask.sum()1133 special_image_mask = special_image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)1134 if image_features is not None and inputs_embeds[special_image_mask].numel() != image_features.numel():1135 raise ValueError(1136 f"Image features and image tokens do not match: tokens: {n_image_tokens}, features {image_features.shape[0]}"1137 )1138 1139 n_video_tokens = special_video_mask.sum()1140 special_video_mask = special_video_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)1141 if video_features is not None and inputs_embeds[special_video_mask].numel() != video_features.numel():1142 raise ValueError(1143 f"Videos features and video tokens do not match: tokens: {n_video_tokens}, features {video_features.shape[0]}"1144 )1145 1146 return special_image_mask, special_video_mask1147 1148 @auto_docstring1149 def forward(1150 self,1151 input_ids: Optional[torch.LongTensor] = None,1152 attention_mask: Optional[torch.Tensor] = None,1153 position_ids: Optional[torch.LongTensor] = None,1154 past_key_values: Optional[Cache] = None,1155 inputs_embeds: Optional[torch.FloatTensor] = None,1156 use_cache: Optional[bool] = None,1157 output_attentions: Optional[bool] = None,1158 output_hidden_states: Optional[bool] = None,1159 return_dict: Optional[bool] = None,1160 pixel_values: Optional[torch.Tensor] = None,1161 pixel_values_videos: Optional[torch.FloatTensor] = None,1162 image_grid_thw: Optional[torch.LongTensor] = None,1163 video_grid_thw: Optional[torch.LongTensor] = None,1164 rope_deltas: Optional[torch.LongTensor] = None,1165 cache_position: Optional[torch.LongTensor] = None,1166 **kwargs: Unpack[TransformersKwargs],1167 ) -> Union[tuple, Qwen2VLModelOutputWithPast]:1168 r"""1169 image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*):1170 The temporal, height and width of feature shape of each image in LLM.1171 video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*):1172 The temporal, height and width of feature shape of each video in LLM.1173 rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*):1174 The rope index difference between sequence length and multimodal rope.1175 """1176 1177 output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions1178 output_hidden_states = (1179 output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states1180 )1181 return_dict = return_dict if return_dict is not None else self.config.use_return_dict1182 1183 if inputs_embeds is None:1184 inputs_embeds = self.get_input_embeddings()(input_ids)1185 1186 if pixel_values is not None:1187 image_embeds = self.get_image_features(pixel_values, image_grid_thw)1188 image_embeds = torch.cat(image_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype)1189 image_mask, _ = self.get_placeholder_mask(1190 input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds1191 )1192 inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds)1193 1194 if pixel_values_videos is not None:1195 video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw)1196 video_embeds = torch.cat(video_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype)1197 _, video_mask = self.get_placeholder_mask(1198 input_ids, inputs_embeds=inputs_embeds, video_features=video_embeds1199 )1200 inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds)