yanziang/InternVideo3-8B-Instruct
101.5k
1# coding=utf-82# Copyright 2025 The InternVideo Team. All rights reserved.3# #4# # Licensed under the Apache License, Version 2.0 (the "License");5# # you may not use this file except in compliance with the License.6# # You may obtain a copy of the License at7# #8# # http://www.apache.org/licenses/LICENSE-2.09# #10# # Unless required by applicable law or agreed to in writing, software11# # distributed under the License is distributed on an "AS IS" BASIS,12# # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# # See the License for the specific language governing permissions and14# # limitations under the License.15 16# from dataclasses import dataclass17# from typing import Any, Callable, Optional, Union18 19# import torch20# import torch.nn as nn21# import torch.nn.functional as F22 23# from transformers.activations import ACT2FN24# from transformers.cache_utils import Cache, DynamicCache25# from transformers.generation import GenerationMixin26# from transformers.integrations import use_kernel_forward_from_hub27# from transformers.masking_utils import create_causal_mask28# from transformers.modeling_flash_attention_utils import FlashAttentionKwargs29# from transformers.modeling_layers import GradientCheckpointingLayer30# from transformers.modeling_outputs import BaseModelOutputWithPast, ModelOutput31# from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update32# from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel33# from transformers.processing_utils import Unpack34# from transformers.utils import TransformersKwargs, auto_docstring, is_torchdynamo_compiling35# from transformers.utils.deprecation import deprecate_kwarg36# from transformers.utils.generic import check_model_inputs37# from transformers.models.qwen3_vl.configuration_qwen3_vl import InternVideo3Config, InternVideo3TextConfig, InternVideo3VisionConfig38 39 40# from transformers.models.deepseek_v3.modeling_deepseek_v3 import (41# apply_rotary_pos_emb_interleave42# )43 44 45# class InternVideo3VisionMLP(nn.Module):46# def __init__(self, config):47# super().__init__()48# self.hidden_size = config.hidden_size49# self.intermediate_size = config.intermediate_size50# self.linear_fc1 = nn.Linear(self.hidden_size, self.intermediate_size, bias=True)51# self.linear_fc2 = nn.Linear(self.intermediate_size, self.hidden_size, bias=True)52# self.act_fn = ACT2FN[config.hidden_act]53 54# def forward(self, hidden_state):55# return self.linear_fc2(self.act_fn(self.linear_fc1(hidden_state)))56 57 58# class InternVideo3VisionPatchEmbed(nn.Module):59# def __init__(self, config) -> None:60# super().__init__()61# self.patch_size = config.patch_size62# self.temporal_patch_size = config.temporal_patch_size63# self.in_channels = config.in_channels64# self.embed_dim = config.hidden_size65 66# kernel_size = [self.temporal_patch_size, self.patch_size, self.patch_size]67# self.proj = nn.Conv3d(self.in_channels, self.embed_dim, kernel_size=kernel_size, stride=kernel_size, bias=True)68 69# def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:70# target_dtype = self.proj.weight.dtype71# hidden_states = hidden_states.view(72# -1, self.in_channels, self.temporal_patch_size, self.patch_size, self.patch_size73# )74# hidden_states = self.proj(hidden_states.to(dtype=target_dtype)).view(-1, self.embed_dim)75# return hidden_states76 77 78# class InternVideo3VisionRotaryEmbedding(nn.Module):79# inv_freq: torch.Tensor # fix linting for `register_buffer`80 81# def __init__(self, dim: int, theta: float = 10000.0) -> None:82# super().__init__()83# inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float) / dim))84# self.register_buffer("inv_freq", inv_freq, persistent=False)85 86# def forward(self, seqlen: int) -> torch.Tensor:87# seq = torch.arange(seqlen, device=self.inv_freq.device, dtype=self.inv_freq.dtype)88# freqs = torch.outer(seq, self.inv_freq)89# return freqs90 91 92# class InternVideo3VisionPatchMerger(nn.Module):93# def __init__(self, config: InternVideo3VisionConfig, use_postshuffle_norm=False) -> None:94# super().__init__()95# self.hidden_size = config.hidden_size * (config.spatial_merge_size**2)96# self.use_postshuffle_norm = use_postshuffle_norm97# self.norm = nn.LayerNorm(self.hidden_size if use_postshuffle_norm else config.hidden_size, eps=1e-6)98# self.linear_fc1 = nn.Linear(self.hidden_size, self.hidden_size)99# self.act_fn = nn.GELU()100# self.linear_fc2 = nn.Linear(self.hidden_size, config.out_hidden_size)101 102# def forward(self, x: torch.Tensor) -> torch.Tensor:103# x = self.norm(x.view(-1, self.hidden_size) if self.use_postshuffle_norm else x).view(-1, self.hidden_size)104# x = self.linear_fc2(self.act_fn(self.linear_fc1(x)))105# return x106 107 108# def rotate_half(x):109# """Rotates half the hidden dims of the input."""110# x1 = x[..., : x.shape[-1] // 2]111# x2 = x[..., x.shape[-1] // 2 :]112# return torch.cat((-x2, x1), dim=-1)113 114 115# def apply_rotary_pos_emb_vision(116# q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor117# ) -> tuple[torch.Tensor, torch.Tensor]:118# orig_q_dtype = q.dtype119# orig_k_dtype = k.dtype120# q, k = q.float(), k.float()121# cos, sin = cos.unsqueeze(-2).float(), sin.unsqueeze(-2).float()122# q_embed = (q * cos) + (rotate_half(q) * sin)123# k_embed = (k * cos) + (rotate_half(k) * sin)124# q_embed = q_embed.to(orig_q_dtype)125# k_embed = k_embed.to(orig_k_dtype)126# return q_embed, k_embed127 128 129# def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:130# """131# This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,132# num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)133# """134# batch, num_key_value_heads, slen, head_dim = hidden_states.shape135# if n_rep == 1:136# return hidden_states137# hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)138# return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)139 140 141# def eager_attention_forward(142# module: nn.Module,143# query: torch.Tensor,144# key: torch.Tensor,145# value: torch.Tensor,146# attention_mask: Optional[torch.Tensor],147# scaling: float,148# dropout: float = 0.0,149# **kwargs: Unpack[TransformersKwargs],150# ):151# key_states = repeat_kv(key, module.num_key_value_groups)152# value_states = repeat_kv(value, module.num_key_value_groups)153 154# attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling155# if attention_mask is not None:156# causal_mask = attention_mask[:, :, :, : key_states.shape[-2]]157# attn_weights = attn_weights + causal_mask158 159# attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)160# attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)161# attn_output = torch.matmul(attn_weights, value_states)162# attn_output = attn_output.transpose(1, 2).contiguous()163 164# return attn_output, attn_weights165 166 167# class InternVideo3VisionAttention(nn.Module):168# def __init__(self, config: InternVideo3VisionConfig) -> None:169# super().__init__()170# self.dim = config.hidden_size171# self.num_heads = config.num_heads172# self.head_dim = self.dim // self.num_heads173# self.num_key_value_groups = 1 # needed for eager attention174# self.qkv = nn.Linear(self.dim, self.dim * 3, bias=True)175# self.proj = nn.Linear(self.dim, self.dim)176# self.scaling = self.head_dim**-0.5177# self.config = config178# self.attention_dropout = 0.0179# self.is_causal = False180 181# def forward(182# self,183# hidden_states: torch.Tensor,184# cu_seqlens: torch.Tensor,185# rotary_pos_emb: Optional[torch.Tensor] = None,186# position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None,187# **kwargs,188# ) -> torch.Tensor:189# seq_length = hidden_states.shape[0]190# query_states, key_states, value_states = (191# self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0)192# )193# cos, sin = position_embeddings194# query_states, key_states = apply_rotary_pos_emb_vision(query_states, key_states, cos, sin)195 196# query_states = query_states.transpose(0, 1).unsqueeze(0)197# key_states = key_states.transpose(0, 1).unsqueeze(0)198# value_states = value_states.transpose(0, 1).unsqueeze(0)199 200# attention_interface: Callable = eager_attention_forward201# if self.config._attn_implementation != "eager":202# attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]203 204# if self.config._attn_implementation == "flash_attention_2":205# # Flash Attention 2: Use cu_seqlens for variable length attention206# max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max()207# attn_output, _ = attention_interface(208# self,209# query_states,210# key_states,211# value_states,212# attention_mask=None,213# scaling=self.scaling,214# dropout=0.0 if not self.training else self.attention_dropout,215# cu_seq_lens_q=cu_seqlens,216# cu_seq_lens_k=cu_seqlens,217# max_length_q=max_seqlen,218# max_length_k=max_seqlen,219# is_causal=False,220# **kwargs,221# )222# else:223# # Other implementations: Process each chunk separately224# lengths = cu_seqlens[1:] - cu_seqlens[:-1]225# splits = [226# torch.split(tensor, lengths.tolist(), dim=2) for tensor in (query_states, key_states, value_states)227# ]228 229# attn_outputs = [230# attention_interface(231# self,232# q,233# k,234# v,235# attention_mask=None,236# scaling=self.scaling,237# dropout=0.0 if not self.training else self.attention_dropout,238# is_causal=False,239# **kwargs,240# )[0]241# for q, k, v in zip(*splits)242# ]243# attn_output = torch.cat(attn_outputs, dim=1)244 245# attn_output = attn_output.reshape(seq_length, -1).contiguous()246# attn_output = self.proj(attn_output)247# return attn_output248 249 250# class InternVideo3VisionBlock(GradientCheckpointingLayer):251# def __init__(self, config, attn_implementation: str = "sdpa") -> None:252# super().__init__()253# self.norm1 = nn.LayerNorm(config.hidden_size, eps=1e-6)254# self.norm2 = nn.LayerNorm(config.hidden_size, eps=1e-6)255# self.attn = InternVideo3VisionAttention(config=config)256# self.mlp = InternVideo3VisionMLP(config=config)257 258# def forward(259# self,260# hidden_states: torch.Tensor,261# cu_seqlens: torch.Tensor,262# rotary_pos_emb: Optional[torch.Tensor] = None,263# position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None,264# **kwargs,265# ) -> torch.Tensor:266# hidden_states = hidden_states + self.attn(267# self.norm1(hidden_states),268# cu_seqlens=cu_seqlens,269# rotary_pos_emb=rotary_pos_emb,270# position_embeddings=position_embeddings,271# **kwargs,272# )273# hidden_states = hidden_states + self.mlp(self.norm2(hidden_states))274# return hidden_states275 276 277# class InternVideo3TextRotaryEmbedding(nn.Module):278# inv_freq: torch.Tensor # fix linting for `register_buffer`279 280# def __init__(self, config: InternVideo3TextConfig, device=None):281# super().__init__()282# if hasattr(config, "rope_scaling") and config.rope_scaling is not None:283# self.rope_type = config.rope_scaling.get("rope_type", "default")284# else:285# self.rope_type = "default"286# self.max_seq_len_cached = config.max_position_embeddings287# self.original_max_seq_len = config.max_position_embeddings288 289# self.config = config290# self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]291 292# inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)293# self.register_buffer("inv_freq", inv_freq, persistent=False)294# self.original_inv_freq = self.inv_freq295 296# self.mrope_section = config.rope_scaling.get("mrope_section", [24, 20, 20])297 298# def apply_interleaved_mrope(self, freqs, mrope_section):299# """Apply interleaved MRoPE to 3D rotary embeddings.300# Reorganizes frequency layout from chunked [TTT...HHH...WWW] to301# interleaved [THTHWHTHW...TT], preserving frequency continuity.302# args:303# x: (3, bs, seq_len, head_dim // 2)304# mrope_section: (3,)305# returns:306# x_t: (bs, seq_len, head_dim // 2)307# """308# freqs_t = freqs[0] # just overwrite the first dimension T309# for dim, offset in enumerate((1, 2), start=1): # H, W310# length = mrope_section[dim] * 3311# idx = slice(offset, length, 3)312# freqs_t[..., idx] = freqs[dim, ..., idx]313# return freqs_t314 315# @torch.no_grad()316# @dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope)317# def forward(self, x, position_ids):318# # In contrast to other models, InternVideo3 has different position ids for the grids319# # So we expand the inv_freq to shape (3, ...)320# if position_ids.ndim == 2:321# position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1)322# inv_freq_expanded = self.inv_freq[None, None, :, None].float().expand(3, position_ids.shape[1], -1, 1)323# position_ids_expanded = position_ids[:, :, None, :].float() # shape (3, bs, 1, positions)324 325# device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"326# with torch.autocast(device_type=device_type, enabled=False): # Force float32327# freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(2, 3)328# freqs = self.apply_interleaved_mrope(freqs, self.mrope_section)329# emb = torch.cat((freqs, freqs), dim=-1)330# cos = emb.cos() * self.attention_scaling331# sin = emb.sin() * self.attention_scaling332 333# return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)334 335 336# @use_kernel_forward_from_hub("RMSNorm")337# class InternVideo3TextRMSNorm(nn.Module):338# def __init__(self, hidden_size, eps: float = 1e-6) -> None:339# """340# InternVideo3TextRMSNorm is equivalent to T5LayerNorm341# """342# super().__init__()343# self.weight = nn.Parameter(torch.ones(hidden_size))344# self.variance_epsilon = eps345 346# def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:347# input_dtype = hidden_states.dtype348# hidden_states = hidden_states.to(torch.float32)349# variance = hidden_states.pow(2).mean(-1, keepdim=True)350# hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)351# return self.weight * hidden_states.to(input_dtype)352 353# def extra_repr(self):354# return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"355 356 357# def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):358# """Applies Rotary Position Embedding to the query and key tensors.359 360# Args:361# q (`torch.Tensor`): The query tensor.362# k (`torch.Tensor`): The key tensor.363# cos (`torch.Tensor`): The cosine part of the rotary embedding.364# sin (`torch.Tensor`): The sine part of the rotary embedding.365# position_ids (`torch.Tensor`, *optional*):366# Deprecated and unused.367# unsqueeze_dim (`int`, *optional*, defaults to 1):368# The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and369# sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note370# that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and371# k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes372# cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have373# the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.374# Returns:375# `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.376# """377# cos = cos.unsqueeze(unsqueeze_dim)378# sin = sin.unsqueeze(unsqueeze_dim)379# q_embed = (q * cos) + (rotate_half(q) * sin)380# k_embed = (k * cos) + (rotate_half(k) * sin)381# return q_embed, k_embed382 383 384 385# class InternVideo3TextAttentionMLA(nn.Module):386# """Multi-headed attention from 'Attention Is All You Need' paper"""387 388# def __init__(self, config: InternVideo3TextConfig, layer_idx: int):389# super().__init__()390# self.config = config391# self.layer_idx = layer_idx392# self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)393# self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads394# # self.scaling = self.head_dim**-0.5395# self.attention_dropout = config.attention_dropout396# self.is_causal = True397 398# # self.q_proj = nn.Linear(399# # config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias400# # )401# # self.k_proj = nn.Linear(402# # config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias403# # )404# # self.v_proj = nn.Linear(405# # config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias406# # )407# # self.o_proj = nn.Linear(408# # config.num_attention_heads * self.head_dim, config.hidden_size, bias=False409# # )410 411# self.num_heads = config.num_attention_heads412# self.rope_theta = config.rope_theta413# self.q_lora_rank = config.q_lora_rank414# # 支持按层的 kv rank 覆盖415# if getattr(config, "kv_lora_rank_list", None) is not None:416# self.kv_lora_rank = config.kv_lora_rank_list[layer_idx]417# else:418# self.kv_lora_rank = config.kv_lora_rank419# self.qk_rope_head_dim = config.qk_rope_head_dim420# self.qk_nope_head_dim = config.qk_nope_head_dim421# self.v_head_dim = config.v_head_dim422# self.qk_head_dim = config.qk_head_dim423 424# self.scaling = self.qk_head_dim**-0.5425 426# self.q_proj = nn.Linear(config.hidden_size, self.num_heads * self.qk_head_dim, bias=config.attention_bias)427 428# self.kv_a_proj_with_mqa = nn.Linear(429# config.hidden_size, self.kv_lora_rank + self.qk_rope_head_dim,430# bias=config.attention_bias,431# )432 433# self.kv_b_proj = nn.Linear(434# self.kv_lora_rank, self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),435# bias=False,436# )437 438# self.o_proj = nn.Linear(439# self.num_heads * self.v_head_dim, config.hidden_size,440# bias=False,441# )442 443# @deprecate_kwarg("past_key_value", new_name="past_key_values", version="4.58")444# def forward(445# self,446# hidden_states: torch.Tensor,447# position_embeddings: tuple[torch.Tensor, torch.Tensor],448# attention_mask: Optional[torch.Tensor],449# past_key_values: Optional[Cache] = None,450# cache_position: Optional[torch.LongTensor] = None,451# **kwargs: Unpack[FlashAttentionKwargs],452# ) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[tuple[torch.Tensor]]]:453# batch_size, seq_length = hidden_states.shape[:-1]454# query_shape = (batch_size, seq_length, -1, self.qk_head_dim)455# key_shape = (batch_size, seq_length, -1, self.qk_nope_head_dim + self.v_head_dim)456 457# if self.q_lora_rank is None:458# q_states = self.q_proj(hidden_states)459# else:460# q_states = self.q_b_proj(self.q_a_layernorm(self.q_a_proj(hidden_states)))461# q_states = q_states.view(query_shape).transpose(1, 2)462# q_pass, q_rot = torch.split(q_states, [self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1)463 464# compressed_kv = self.kv_a_proj_with_mqa(hidden_states)465# k_pass, k_rot = torch.split(compressed_kv, [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)466 467# k_pass = self.kv_b_proj(k_pass).view(key_shape).transpose(1, 2)468# k_pass, value_states = torch.split(k_pass, [self.qk_nope_head_dim, self.v_head_dim], dim=-1)469 470# k_rot = k_rot.view(batch_size, 1, seq_length, self.qk_rope_head_dim)471 472# cos, sin = position_embeddings473# if self.config.rope_interleave: # support using interleaved weights for efficiency474# q_rot, k_rot = apply_rotary_pos_emb_interleave(q_rot, k_rot, cos, sin)475# else:476# q_rot, k_rot = apply_rotary_pos_emb(q_rot, k_rot, cos, sin)477# k_rot = k_rot.expand(*k_pass.shape[:-1], -1)478 479# query_states = torch.cat((q_pass, q_rot), dim=-1)480# key_states = torch.cat((k_pass, k_rot), dim=-1)481 482# if past_key_values is not None:483# # sin and cos are specific to RoPE models; cache_position needed for the static cache484# cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}485# key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs)486 487# if self.config._attn_implementation == "flash_attention_2" and self.qk_head_dim != self.v_head_dim:488# value_states = F.pad(value_states, [0, self.qk_head_dim - self.v_head_dim])489 490# attention_interface: Callable = eager_attention_forward491# if self.config._attn_implementation != "eager":492# attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]493 494# attn_output, attn_weights = attention_interface(495# self,496# query_states,497# key_states,498# value_states,499# attention_mask,500# dropout=0.0 if not self.training else self.attention_dropout,501# scaling=self.scaling,502# **kwargs,503# )504 505# if self.config._attn_implementation == "flash_attention_2" and self.qk_head_dim != self.v_head_dim:506# attn_output = attn_output[:, :, :, : self.v_head_dim]507 508# attn_output = attn_output.reshape(batch_size, seq_length, -1).contiguous()509# attn_output = self.o_proj(attn_output)510# return attn_output, attn_weights511 512 513 514# class InternVideo3TextMLP(nn.Module):515# def __init__(self, config):516# super().__init__()517# self.config = config518# self.hidden_size = config.hidden_size519# self.intermediate_size = config.intermediate_size520# self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)521# self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)522# self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)523# self.act_fn = ACT2FN[config.hidden_act]524 525# def forward(self, x):526# down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))527# return down_proj528 529 530# class InternVideo3TextDecoderLayer(GradientCheckpointingLayer):531# def __init__(self, config: InternVideo3TextConfig, layer_idx: int):532# super().__init__()533# self.hidden_size = config.hidden_size534 535# self.self_attn = InternVideo3TextAttentionMLA(config=config, layer_idx=layer_idx)536 537# self.mlp = InternVideo3TextMLP(config)538# self.input_layernorm = InternVideo3TextRMSNorm(config.hidden_size, eps=config.rms_norm_eps)539# self.post_attention_layernorm = InternVideo3TextRMSNorm(config.hidden_size, eps=config.rms_norm_eps)540 541# @deprecate_kwarg("past_key_value", new_name="past_key_values", version="4.58")542# def forward(543# self,544# hidden_states: torch.Tensor,545# position_embeddings: tuple[torch.Tensor, torch.Tensor],546# attention_mask: Optional[torch.Tensor] = None,547# position_ids: Optional[torch.LongTensor] = None,548# past_key_values: Optional[Cache] = None,549# use_cache: Optional[bool] = False,550# cache_position: Optional[torch.LongTensor] = None,551# **kwargs: Unpack[TransformersKwargs],552# ) -> torch.Tensor:553# residual = hidden_states554# hidden_states = self.input_layernorm(hidden_states)555# # Self Attention556# hidden_states, _ = self.self_attn(557# hidden_states=hidden_states,558# attention_mask=attention_mask,559# position_ids=position_ids,560# past_key_values=past_key_values,561# use_cache=use_cache,562# cache_position=cache_position,563# position_embeddings=position_embeddings,564# **kwargs,565# )566# hidden_states = residual + hidden_states567 568# # Fully Connected569# residual = hidden_states570# hidden_states = self.post_attention_layernorm(hidden_states)571# hidden_states = self.mlp(hidden_states)572# hidden_states = residual + hidden_states573# return hidden_states574 575 576# @dataclass577# @auto_docstring(578# custom_intro="""579# Base class for Llava outputs, with hidden states and attentions.580# """581# )582# class InternVideo3ModelOutputWithPast(ModelOutput):583# r"""584# past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):585# It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).586 587# Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see588# `past_key_values` input) to speed up sequential decoding.589# rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*):590# The rope index difference between sequence length and multimodal rope.591# """592 593# last_hidden_state: Optional[torch.FloatTensor] = None594# past_key_values: Optional[Cache] = None595# hidden_states: Optional[tuple[torch.FloatTensor]] = None596# attentions: Optional[tuple[torch.FloatTensor]] = None597# rope_deltas: Optional[torch.LongTensor] = None598 599 600# @auto_docstring601# class InternVideo3PreTrainedModel(PreTrainedModel):602# config: InternVideo3Config603# base_model_prefix = "model"604# supports_gradient_checkpointing = True605# _no_split_modules = ["InternVideo3TextDecoderLayer", "InternVideo3VisionBlock"]606# _skip_keys_device_placement = "past_key_values"607# _supports_flash_attn = True608# _supports_sdpa = True609 610# _can_compile_fullgraph = True611# _supports_attention_backend = True612# _can_record_outputs = {613# "hidden_states": InternVideo3TextDecoderLayer,614# "attentions": InternVideo3TextAttentionMLA,615# }616 617 618# class InternVideo3VisionModel(InternVideo3PreTrainedModel):619# config: InternVideo3VisionConfig620# _no_split_modules = ["InternVideo3VisionBlock"]621 622# def __init__(self, config, *inputs, **kwargs) -> None:623# super().__init__(config, *inputs, **kwargs)624# self.spatial_merge_size = config.spatial_merge_size625# self.patch_size = config.patch_size626# self.spatial_merge_unit = self.spatial_merge_size * self.spatial_merge_size627 628# self.patch_embed = InternVideo3VisionPatchEmbed(629# config=config,630# )631 632# self.pos_embed = nn.Embedding(config.num_position_embeddings, config.hidden_size)633# self.num_grid_per_side = int(config.num_position_embeddings**0.5)634 635# head_dim = config.hidden_size // config.num_heads636# self.rotary_pos_emb = InternVideo3VisionRotaryEmbedding(head_dim // 2)637 638# self.blocks = nn.ModuleList([InternVideo3VisionBlock(config) for _ in range(config.depth)])639# self.merger = InternVideo3VisionPatchMerger(640# config=config,641# use_postshuffle_norm=False,642# )643 644# self.deepstack_visual_indexes = config.deepstack_visual_indexes645# self.deepstack_merger_list = nn.ModuleList(646# [647# InternVideo3VisionPatchMerger(648# config=config,649# use_postshuffle_norm=True,650# )651# for _ in range(len(config.deepstack_visual_indexes))652# ]653# )654 655# self.gradient_checkpointing = False656 657# def rot_pos_emb(self, grid_thw: torch.Tensor) -> torch.Tensor:658# merge_size = self.spatial_merge_size659 660# max_hw = int(grid_thw[:, 1:].max().item())661# freq_table = self.rotary_pos_emb(max_hw) # (max_hw, dim // 2)662# device = freq_table.device663 664# total_tokens = int(torch.prod(grid_thw, dim=1).sum().item())665# pos_ids = torch.empty((total_tokens, 2), dtype=torch.long, device=device)666 667# offset = 0668# for num_frames, height, width in grid_thw:669# merged_h, merged_w = height // merge_size, width // merge_size670 671# block_rows = torch.arange(merged_h, device=device) # block row indices672# block_cols = torch.arange(merged_w, device=device) # block col indices673# intra_row = torch.arange(merge_size, device=device) # intra-block row offsets674# intra_col = torch.arange(merge_size, device=device) # intra-block col offsets675 676# # Compute full-resolution positions677# row_idx = block_rows[:, None, None, None] * merge_size + intra_row[None, None, :, None]678# col_idx = block_cols[None, :, None, None] * merge_size + intra_col[None, None, None, :]679 680# row_idx = row_idx.expand(merged_h, merged_w, merge_size, merge_size).reshape(-1)681# col_idx = col_idx.expand(merged_h, merged_w, merge_size, merge_size).reshape(-1)682 683# coords = torch.stack((row_idx, col_idx), dim=-1)684 685# if num_frames > 1:686# coords = coords.repeat(num_frames, 1)687 688# num_tokens = coords.shape[0]689# pos_ids[offset : offset + num_tokens] = coords690# offset += num_tokens691 692# embeddings = freq_table[pos_ids] # lookup rotary embeddings693# embeddings = embeddings.flatten(1)694# return embeddings695 696# def fast_pos_embed_interpolate(self, grid_thw):697# grid_ts, grid_hs, grid_ws = grid_thw[:, 0], grid_thw[:, 1], grid_thw[:, 2]698 699# idx_list = [[] for _ in range(4)]700# weight_list = [[] for _ in range(4)]701 702# for t, h, w in zip(grid_ts, grid_hs, grid_ws):703# h_idxs = torch.linspace(0, self.num_grid_per_side - 1, h)704# w_idxs = torch.linspace(0, self.num_grid_per_side - 1, w)705 706# h_idxs_floor = h_idxs.int()707# w_idxs_floor = w_idxs.int()708# h_idxs_ceil = (h_idxs.int() + 1).clip(max=self.num_grid_per_side - 1)709# w_idxs_ceil = (w_idxs.int() + 1).clip(max=self.num_grid_per_side - 1)710 711# dh = h_idxs - h_idxs_floor712# dw = w_idxs - w_idxs_floor713 714# base_h = h_idxs_floor * self.num_grid_per_side715# base_h_ceil = h_idxs_ceil * self.num_grid_per_side716 717# indices = [718# (base_h[None].T + w_idxs_floor[None]).flatten(),719# (base_h[None].T + w_idxs_ceil[None]).flatten(),720# (base_h_ceil[None].T + w_idxs_floor[None]).flatten(),721# (base_h_ceil[None].T + w_idxs_ceil[None]).flatten(),722# ]723 724# weights = [725# ((1 - dh)[None].T * (1 - dw)[None]).flatten(),726# ((1 - dh)[None].T * dw[None]).flatten(),727# (dh[None].T * (1 - dw)[None]).flatten(),728# (dh[None].T * dw[None]).flatten(),729# ]730 731# for i in range(4):732# idx_list[i].extend(indices[i].tolist())733# weight_list[i].extend(weights[i].tolist())734 735# idx_tensor = torch.tensor(idx_list, dtype=torch.long, device=self.pos_embed.weight.device)736# weight_tensor = torch.tensor(737# weight_list, dtype=self.pos_embed.weight.dtype, device=self.pos_embed.weight.device738# )739# pos_embeds = self.pos_embed(idx_tensor) * weight_tensor[:, :, None]740# patch_pos_embeds = pos_embeds[0] + pos_embeds[1] + pos_embeds[2] + pos_embeds[3]741 742# patch_pos_embeds = patch_pos_embeds.split([h * w for h, w in zip(grid_hs, grid_ws)])743 744# patch_pos_embeds_permute = []745# merge_size = self.config.spatial_merge_size746# for pos_embed, t, h, w in zip(patch_pos_embeds, grid_ts, grid_hs, grid_ws):747# pos_embed = pos_embed.repeat(t, 1)748# pos_embed = (749# pos_embed.view(t, h // merge_size, merge_size, w // merge_size, merge_size, -1)750# .permute(0, 1, 3, 2, 4, 5)751# .flatten(0, 4)752# )753# patch_pos_embeds_permute.append(pos_embed)754# patch_pos_embeds = torch.cat(patch_pos_embeds_permute)755# return patch_pos_embeds756 757# def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs) -> torch.Tensor:758# """759# Args:760# hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`):761# The final hidden states of the model.762# grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`):763# The temporal, height and width of feature shape of each image in LLM.764 765# Returns:766# `torch.Tensor`: hidden_states.767# """768# hidden_states = self.patch_embed(hidden_states)769 770# pos_embeds = self.fast_pos_embed_interpolate(grid_thw)771# hidden_states = hidden_states + pos_embeds772 773# rotary_pos_emb = self.rot_pos_emb(grid_thw)774 775# seq_len, _ = hidden_states.size()776# hidden_states = hidden_states.reshape(seq_len, -1)777# rotary_pos_emb = rotary_pos_emb.reshape(seq_len, -1)778# emb = torch.cat((rotary_pos_emb, rotary_pos_emb), dim=-1)779# position_embeddings = (emb.cos(), emb.sin())780 781# cu_seqlens = torch.repeat_interleave(grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]).cumsum(782# dim=0,783# # Select dtype based on the following factors:784# # - FA2 requires that cu_seqlens_q must have dtype int32785# # - torch.onnx.export requires that cu_seqlens_q must have same dtype as grid_thw786# # See https://github.com/huggingface/transformers/pull/34852 for more information787# dtype=grid_thw.dtype if torch.jit.is_tracing() else torch.int32,788# )789# cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0)790 791# deepstack_feature_lists = []792# for layer_num, blk in enumerate(self.blocks):793# hidden_states = blk(794# hidden_states,795# cu_seqlens=cu_seqlens,796# position_embeddings=position_embeddings,797# **kwargs,798# )799# if layer_num in self.deepstack_visual_indexes:800# deepstack_feature = self.deepstack_merger_list[self.deepstack_visual_indexes.index(layer_num)](801# hidden_states802# )803# deepstack_feature_lists.append(deepstack_feature)804 805# hidden_states = self.merger(hidden_states)806 807# return hidden_states, deepstack_feature_lists808 809 810# @auto_docstring(811# custom_intro=(812# "Text part of InternVideo3, "813# "not a pure text-only model, as DeepStack integrates visual features into the early hidden states."814# )815# )816# class InternVideo3TextModel(InternVideo3PreTrainedModel):817# config: InternVideo3TextConfig818# _no_split_modules = ["InternVideo3TextDecoderLayer"]819 820# def __init__(self, config: InternVideo3TextConfig):821# super().__init__(config)822# self.padding_idx = config.pad_token_id823# self.vocab_size = config.vocab_size824 825# self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)826# self.layers = nn.ModuleList(827# [InternVideo3TextDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]828# )829# self.norm = InternVideo3TextRMSNorm(config.hidden_size, eps=config.rms_norm_eps)830# self.rotary_emb = InternVideo3TextRotaryEmbedding(config=config)831# self.gradient_checkpointing = False832 833# # Initialize weights and apply final processing834# self.post_init()835 836# @check_model_inputs()837# @auto_docstring838# def forward(839# self,840# input_ids: Optional[torch.LongTensor] = None,841# attention_mask: Optional[torch.Tensor] = None,842# position_ids: Optional[torch.LongTensor] = None,843# past_key_values: Optional[Cache] = None,844# inputs_embeds: Optional[torch.FloatTensor] = None,845# use_cache: Optional[bool] = None,846# cache_position: Optional[torch.LongTensor] = None,847# # args for deepstack848# visual_pos_masks: Optional[torch.Tensor] = None,849# deepstack_visual_embeds: Optional[list[torch.Tensor]] = None,850# **kwargs: Unpack[FlashAttentionKwargs],851# ) -> Union[tuple, BaseModelOutputWithPast]:852# r"""853# visual_pos_masks (`torch.Tensor` of shape `(batch_size, seqlen)`, *optional*):854# The mask of the visual positions.855# deepstack_visual_embeds (`list[torch.Tensor]`, *optional*):856# The deepstack visual embeddings. The shape is (num_layers, visual_seqlen, embed_dim).857# The feature is extracted from the different visual encoder layers, and fed to the decoder858# hidden states. It's from the paper DeepStack(https://arxiv.org/abs/2406.04334).859# """860# if (input_ids is None) ^ (inputs_embeds is not None):861# raise ValueError("You must specify exactly one of input_ids or inputs_embeds")862 863# # torch.jit.trace() doesn't support cache objects in the output864# if use_cache and past_key_values is None and not torch.jit.is_tracing():865# past_key_values = DynamicCache(config=self.config)866 867# if inputs_embeds is None:868# inputs_embeds = self.embed_tokens(input_ids)869 870# if cache_position is None:871# past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0872# cache_position = torch.arange(873# past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device874# )875 876# # the hard coded `3` is for temporal, height and width.877# if position_ids is None:878# position_ids = cache_position.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1)879# elif position_ids.ndim == 2:880# position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1)881 882# if position_ids.ndim == 3 and position_ids.shape[0] == 4:883# text_position_ids = position_ids[0]884# position_ids = position_ids[1:]885# else:886# text_position_ids = position_ids[0]887 888# attention_mask = create_causal_mask(889# config=self.config,890# input_embeds=inputs_embeds,891# attention_mask=attention_mask,892# cache_position=cache_position,893# past_key_values=past_key_values,894# position_ids=text_position_ids,895# )896 897# hidden_states = inputs_embeds898 899# # create position embeddings to be shared across the decoder layers900# position_embeddings = self.rotary_emb(hidden_states, position_ids)901 902# # decoder layers903# for layer_idx, decoder_layer in enumerate(self.layers):904# layer_outputs = decoder_layer(905# hidden_states,906# attention_mask=attention_mask,907# position_ids=text_position_ids,908# past_key_values=past_key_values,909# cache_position=cache_position,910# position_embeddings=position_embeddings,911# **kwargs,912# )913# hidden_states = layer_outputs914 915# # add visual features to the hidden states of first several layers916# if deepstack_visual_embeds is not None and layer_idx in range(len(deepstack_visual_embeds)):917# hidden_states = self._deepstack_process(918# hidden_states,919# visual_pos_masks,920# deepstack_visual_embeds[layer_idx],921# )922 923# hidden_states = self.norm(hidden_states)924 925# return BaseModelOutputWithPast(926# last_hidden_state=hidden_states,927# past_key_values=past_key_values,928# )929 930# def _deepstack_process(931# self, hidden_states: torch.Tensor, visual_pos_masks: torch.Tensor, visual_embeds: torch.Tensor932# ):933# visual_pos_masks = visual_pos_masks.to(hidden_states.device)934# visual_embeds = visual_embeds.to(hidden_states.device, hidden_states.dtype)935# local_this = hidden_states[visual_pos_masks, :].clone() + visual_embeds936# hidden_states[visual_pos_masks, :] = local_this937# return hidden_states938 939 940# @auto_docstring941# class InternVideo3Model(InternVideo3PreTrainedModel):942# base_model_prefix = ""943# _checkpoint_conversion_mapping = {}944# # Reference: fix gemma3 grad acc #37208945# accepts_loss_kwargs = False946# config: InternVideo3Config947# _no_split_modules = ["InternVideo3TextDecoderLayer", "InternVideo3VisionBlock"]948 949# def __init__(self, config):950# super().__init__(config)951# self.visual = InternVideo3VisionModel._from_config(config.vision_config)952# self.language_model = InternVideo3TextModel._from_config(config.text_config)953# self.rope_deltas = None # cache rope_deltas here954 955# # Initialize weights and apply final processing956# self.post_init()957 958# def get_input_embeddings(self):959# return self.language_model.get_input_embeddings()960 961# def set_input_embeddings(self, value):962# self.language_model.set_input_embeddings(value)963 964# def set_decoder(self, decoder):965# self.language_model = decoder966 967# def get_decoder(self):968# return self.language_model969 970# def get_rope_index(971# self,972# input_ids: Optional[torch.LongTensor] = None,973# image_grid_thw: Optional[torch.LongTensor] = None,974# video_grid_thw: Optional[torch.LongTensor] = None,975# attention_mask: Optional[torch.Tensor] = None,976# ) -> tuple[torch.Tensor, torch.Tensor]:977# """Different from the original implementation, InternVideo3 use timestamps rather than absolute time position ids."""978 979# # Since we use timestamps to seperate videos, like <t1> <vision_start> <frame1> <vision_end> <t2> <vision_start> <frame2> <vision_end>, the video_grid_thw should also be split980# if video_grid_thw is not None:981# video_grid_thw = torch.repeat_interleave(video_grid_thw, video_grid_thw[:, 0], dim=0)982# video_grid_thw[:, 0] = 1983 984# spatial_merge_size = self.config.vision_config.spatial_merge_size985# image_token_id = self.config.image_token_id986# video_token_id = self.config.video_token_id987# vision_start_token_id = self.config.vision_start_token_id988# mrope_position_deltas = []989# if input_ids is not None and (image_grid_thw is not None or video_grid_thw is not None):990# total_input_ids = input_ids991# if attention_mask is None:992# attention_mask = torch.ones_like(total_input_ids)993# position_ids = torch.ones(994# 3,995# input_ids.shape[0],996# input_ids.shape[1],997# dtype=input_ids.dtype,998# device=input_ids.device,999# )1000# image_index, video_index = 0, 01001# attention_mask = attention_mask.to(total_input_ids.device)1002# for i, input_ids in enumerate(total_input_ids):1003# input_ids = input_ids[attention_mask[i] == 1]1004# image_nums, video_nums = 0, 01005# vision_start_indices = torch.argwhere(input_ids == vision_start_token_id).squeeze(1)1006# vision_tokens = input_ids[vision_start_indices + 1]1007# image_nums = (vision_tokens == image_token_id).sum()1008# video_nums = (vision_tokens == video_token_id).sum()1009# input_tokens = input_ids.tolist()1010# llm_pos_ids_list: list = []1011# st = 01012# remain_images, remain_videos = image_nums, video_nums1013# for _ in range(image_nums + video_nums):1014# if image_token_id in input_tokens and remain_images > 0:1015# ed_image = input_tokens.index(image_token_id, st)1016# else:1017# ed_image = len(input_tokens) + 11018# if video_token_id in input_tokens and remain_videos > 0:1019# ed_video = input_tokens.index(video_token_id, st)1020# else:1021# ed_video = len(input_tokens) + 11022# if ed_image < ed_video:1023# t, h, w = (1024# image_grid_thw[image_index][0],1025# image_grid_thw[image_index][1],1026# image_grid_thw[image_index][2],1027# )1028# image_index += 11029# remain_images -= 11030# ed = ed_image1031 1032# else:1033# t, h, w = (1034# video_grid_thw[video_index][0],1035# video_grid_thw[video_index][1],1036# video_grid_thw[video_index][2],1037# )1038# video_index += 11039# remain_videos -= 11040# ed = ed_video1041# llm_grid_t, llm_grid_h, llm_grid_w = (1042# t.item(),1043# h.item() // spatial_merge_size,1044# w.item() // spatial_merge_size,1045# )1046# text_len = ed - st1047 1048# st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 01049# llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx)1050 1051# # t_index is always 0 because llm_grid_t is always 1 (we use timestamps to encode the temporal information for videos)1052# t_index = torch.arange(llm_grid_t).view(-1, 1).expand(-1, llm_grid_h * llm_grid_w).flatten()1053# h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(llm_grid_t, -1, llm_grid_w).flatten()1054# w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(llm_grid_t, llm_grid_h, -1).flatten()1055# llm_pos_ids_list.append(torch.stack([t_index, h_index, w_index]) + text_len + st_idx)1056# st = ed + llm_grid_t * llm_grid_h * llm_grid_w1057 1058# if st < len(input_tokens):1059# st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 01060# text_len = len(input_tokens) - st1061# llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx)1062 1063# llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)1064# position_ids[..., i, attention_mask[i] == 1] = llm_positions.to(position_ids.device)1065# mrope_position_deltas.append(llm_positions.max() + 1 - len(total_input_ids[i]))1066# mrope_position_deltas = torch.tensor(mrope_position_deltas, device=input_ids.device).unsqueeze(1)1067# return position_ids, mrope_position_deltas1068# else:1069# if attention_mask is not None:1070# position_ids = attention_mask.long().cumsum(-1) - 11071# position_ids.masked_fill_(attention_mask == 0, 1)1072# position_ids = position_ids.unsqueeze(0).expand(3, -1, -1).to(attention_mask.device)1073# max_position_ids = position_ids.max(0, keepdim=False)[0].max(-1, keepdim=True)[0]1074# mrope_position_deltas = max_position_ids + 1 - attention_mask.shape[-1]1075# else:1076# position_ids = (1077# torch.arange(input_ids.shape[1], device=input_ids.device)1078# .view(1, 1, -1)1079# .expand(3, input_ids.shape[0], -1)1080# )1081# mrope_position_deltas = torch.zeros(1082# [input_ids.shape[0], 1],1083# device=input_ids.device,1084# dtype=input_ids.dtype,1085# )1086 1087# return position_ids, mrope_position_deltas1088 1089# def get_video_features(1090# self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None1091# ):1092# """1093# Encodes videos into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned.1094 1095# Args:1096# pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):1097# The tensors corresponding to the input videos.1098# video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*):1099# The temporal, height and width of feature shape of each video in LLM.1100# """1101# # Same implementation as for images1102# return self.get_image_features(pixel_values_videos, video_grid_thw)1103 1104# def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None):1105# """1106# Encodes images into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned.1107 1108# Args:1109# pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):1110# The tensors corresponding to the input images.1111# image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*):1112# The temporal, height and width of feature shape of each image in LLM.1113# """1114# pixel_values = pixel_values.type(self.visual.dtype)1115# image_embeds, deepstack_image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw)1116# split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist()1117# image_embeds = torch.split(image_embeds, split_sizes)1118# return image_embeds, deepstack_image_embeds1119 1120# def get_placeholder_mask(1121# self,1122# input_ids: torch.LongTensor,1123# inputs_embeds: torch.FloatTensor,1124# image_features: Optional[torch.FloatTensor] = None,1125# video_features: Optional[torch.FloatTensor] = None,1126# ):1127# """1128# Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is1129# equal to the length of multimodal features. If the lengths are different, an error is raised.1130# """1131# if input_ids is None:1132# special_image_mask = inputs_embeds == self.get_input_embeddings()(1133# torch.tensor(self.config.image_token_id, dtype=torch.long, device=inputs_embeds.device)1134# )1135# special_image_mask = special_image_mask.all(-1)1136# special_video_mask = inputs_embeds == self.get_input_embeddings()(1137# torch.tensor(self.config.video_token_id, dtype=torch.long, device=inputs_embeds.device)1138# )1139# special_video_mask = special_video_mask.all(-1)1140# else:1141# special_image_mask = input_ids == self.config.image_token_id1142# special_video_mask = input_ids == self.config.video_token_id1143 1144# n_image_tokens = special_image_mask.sum()1145# special_image_mask = special_image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)1146# if image_features is not None and inputs_embeds[special_image_mask].numel() != image_features.numel():1147# raise ValueError(1148# f"Image features and image tokens do not match: tokens: {n_image_tokens}, features {image_features.shape[0]}"1149# )1150 1151# n_video_tokens = special_video_mask.sum()1152# special_video_mask = special_video_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)1153# if video_features is not None and inputs_embeds[special_video_mask].numel() != video_features.numel():1154# raise ValueError(1155# f"Videos features and video tokens do not match: tokens: {n_video_tokens}, features {video_features.shape[0]}"1156# )1157 1158# return special_image_mask, special_video_mask1159 1160# @auto_docstring1161# @check_model_inputs()1162# def forward(1163# self,1164# input_ids: torch.LongTensor = None,1165# attention_mask: Optional[torch.Tensor] = None,1166# position_ids: Optional[torch.LongTensor] = None,1167# past_key_values: Optional[Cache] = None,1168# inputs_embeds: Optional[torch.FloatTensor] = None,1169# pixel_values: Optional[torch.Tensor] = None,1170# pixel_values_videos: Optional[torch.FloatTensor] = None,1171# image_grid_thw: Optional[torch.LongTensor] = None,1172# video_grid_thw: Optional[torch.LongTensor] = None,1173# cache_position: Optional[torch.LongTensor] = None,1174# **kwargs: Unpack[TransformersKwargs],1175# ) -> Union[tuple, InternVideo3ModelOutputWithPast]:1176# r"""1177# image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*):1178# The temporal, height and width of feature shape of each image in LLM.1179# video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*):1180# The temporal, height and width of feature shape of each video in LLM.1181# """1182# if (input_ids is None) ^ (inputs_embeds is not None):1183# raise ValueError("You must specify exactly one of input_ids or inputs_embeds")1184 1185# if inputs_embeds is None:1186# inputs_embeds = self.get_input_embeddings()(input_ids)1187 1188# image_mask = None1189# video_mask = None1190 1191# if pixel_values is not None:1192# image_embeds, deepstack_image_embeds = self.get_image_features(pixel_values, image_grid_thw)1193# image_embeds = torch.cat(image_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype)1194# image_mask, _ = self.get_placeholder_mask(1195# input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds1196# )1197# inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds)1198 1199# if pixel_values_videos is not None:1200# video_embeds, deepstack_video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw)