OpenGVLab/VideoChat-Flash-Qwen2-7B_res448
131.1k
1from typing import Optional, Tuple, Union, Dict2from dataclasses import dataclass3from functools import partial, reduce4from PIL import Image5import os6from transformers.image_processing_utils import BatchFeature, get_size_dict7from transformers.image_transforms import (8 convert_to_rgb,9 normalize,10 rescale,11 resize,12 to_channel_dimension_format,13)14from transformers.image_utils import (15 ChannelDimension,16 PILImageResampling,17 to_numpy_array,18)19import numpy as np20import torch21import torch.nn as nn22import torch.nn.functional as F23import torch.utils.checkpoint as checkpoint24from functools import partial25try:26 from flash_attn import flash_attn_qkvpacked_func27 use_flash_attn = True28except:29 use_flash_attn = False30 print("You need to install flash_attn to be faster!")31 32try:33 from timm.layers import drop_path, to_2tuple, trunc_normal_34except:35 from timm.models.layers import drop_path, trunc_normal_, to_2tuple36 37 38 39class DropPath(nn.Module):40 """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).41 """42 def __init__(self, drop_prob=None):43 super(DropPath, self).__init__()44 self.drop_prob = drop_prob45 46 def forward(self, x):47 return drop_path(x, self.drop_prob, self.training)48 49 def extra_repr(self) -> str:50 return 'p={}'.format(self.drop_prob)51 52 53class Mlp(nn.Module):54 def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.):55 super().__init__()56 out_features = out_features or in_features57 hidden_features = hidden_features or in_features58 self.fc1 = nn.Linear(in_features, hidden_features)59 self.act = act_layer()60 self.fc2 = nn.Linear(hidden_features, out_features)61 self.drop = nn.Dropout(drop)62 63 def forward(self, x):64 x = self.fc1(x)65 x = self.act(x)66 x = self.drop(x)67 x = self.fc2(x)68 x = self.drop(x)69 return x70 71class Attention(nn.Module):72 def __init__(73 self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0.,74 proj_drop=0., attn_head_dim=None,75 attn_type='flash_v2'):76 77 if use_flash_attn:78 attn_type = attn_type79 else:80 attn_type = 'origin'81 82 print(attn_type)83 84 super().__init__()85 self.num_heads = num_heads86 head_dim = dim // num_heads87 if attn_head_dim is not None:88 head_dim = attn_head_dim89 all_head_dim = head_dim * self.num_heads90 self.scale = qk_scale or head_dim ** -0.591 92 self.qkv = nn.Linear(dim, all_head_dim * 3, bias=False)93 if qkv_bias:94 self.q_bias = nn.Parameter(torch.zeros(all_head_dim))95 self.v_bias = nn.Parameter(torch.zeros(all_head_dim))96 else:97 self.q_bias = None98 self.v_bias = None99 100 if attn_type not in ['origin', 'flash_v2']:101 raise NotImplementedError(f"Not support attn_type: {attn_type}")102 103 # print('umt:', f'attn_type: {attn_type}')104 105 self.attn_type = attn_type106 if attn_type == 'flash_v2':107 self.attn_drop = attn_drop108 else:109 self.attn_drop = nn.Dropout(attn_drop)110 self.proj = nn.Linear(all_head_dim, dim)111 self.proj_drop = nn.Dropout(proj_drop)112 113 def forward(self, x):114 B, N, C = x.shape115 qkv_bias = None116 if self.q_bias is not None:117 qkv_bias = torch.cat((self.q_bias, torch.zeros_like(self.v_bias, requires_grad=False), self.v_bias))118 # qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)119 qkv = F.linear(input=x, weight=self.qkv.weight, bias=qkv_bias)120 121 if self.attn_type == 'flash_v2':122 qkv = qkv.reshape(B, N, 3, self.num_heads, -1)123 x = flash_attn_qkvpacked_func(qkv, dropout_p=self.attn_drop, softmax_scale=self.scale, causal=False).reshape(B, N, -1)124 else:125 qkv = qkv.reshape(B, N, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4)126 q, k, v = qkv[0], qkv[1], qkv[127 2] # make torchscript happy (cannot use tensor as tuple)128 # B num_heads N head_dim129 130 q = q * self.scale131 attn = (q @ k.transpose(-2, -1))132 133 attn = attn.softmax(dim=-1)134 attn = self.attn_drop(attn)135 136 x = (attn @ v).transpose(1, 2).reshape(B, N, -1)137 138 x = self.proj(x)139 x = self.proj_drop(x)140 return x141 142 143 144 145class Block(nn.Module):146 def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0.,147 drop_path=0., init_values=None, act_layer=nn.GELU, norm_layer=nn.LayerNorm,148 attn_head_dim=None):149 super().__init__()150 self.norm1 = norm_layer(dim)151 self.attn = Attention(152 dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale,153 attn_drop=attn_drop, proj_drop=drop, attn_head_dim=attn_head_dim)154 # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here155 self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()156 self.norm2 = norm_layer(dim)157 mlp_hidden_dim = int(dim * mlp_ratio)158 self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)159 160 if init_values > 0:161 self.gamma_1 = nn.Parameter(init_values * torch.ones((dim)),requires_grad=True)162 self.gamma_2 = nn.Parameter(init_values * torch.ones((dim)),requires_grad=True)163 else:164 self.gamma_1, self.gamma_2 = None, None165 166 def forward(self, x):167 if self.gamma_1 is None:168 x = x + self.drop_path(self.attn(self.norm1(x)))169 x = x + self.drop_path(self.mlp(self.norm2(x)))170 else:171 x = x + self.drop_path(self.gamma_1 * self.attn(self.norm1(x)))172 x = x + self.drop_path(self.gamma_2 * self.mlp(self.norm2(x)))173 return x174 175 176class PatchEmbed(nn.Module):177 """ Image to Patch Embedding178 """179 def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768, num_frames=16, tubelet_size=2):180 super().__init__()181 img_size = to_2tuple(img_size)182 patch_size = to_2tuple(patch_size)183 self.tubelet_size = int(tubelet_size)184 num_patches = (img_size[1] // patch_size[1]) * (img_size[0] // patch_size[0]) * (num_frames // self.tubelet_size)185 self.img_size = img_size186 self.patch_size = patch_size187 self.num_patches = num_patches188 self.proj = nn.Conv3d(189 in_channels=in_chans, out_channels=embed_dim, 190 kernel_size=(self.tubelet_size, patch_size[0], patch_size[1]), 191 stride=(self.tubelet_size, patch_size[0], patch_size[1])192 )193 # print('umt:', f'Num of patches: {num_patches}')194 195 def forward(self, x, **kwargs):196 B, C, T, H, W = x.shape197 # FIXME look at relaxing size constraints198 # assert H == self.img_size[0] and W == self.img_size[1], \199 # f"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]})."200 x = self.proj(x).flatten(2).transpose(1, 2)201 return x202 203# sin-cos position encoding204# https://github.com/jadore801120/attention-is-all-you-need-pytorch/blob/master/transformer/Models.py#L31205def get_sinusoid_encoding_table(n_position, d_hid, ckpt_num_frame=-1, cur_frame=12): 206 ''' Sinusoid position encoding table ''' 207 # TODO: make it with torch instead of numpy 208 def get_position_angle_vec(position): 209 return [position / np.power(10000, 2 * (hid_j // 2) / d_hid) for hid_j in range(d_hid)] 210 211 if ckpt_num_frame != -1 and ckpt_num_frame != cur_frame:212 # print('umt:', f"Interpolate position embedding")213 # print('umt:', f"Testing frame: {cur_frame}")214 # print('umt:', f"Checkpoint frame: {ckpt_num_frame}")215 216 T = ckpt_num_frame # checkpoint frame217 new_T = cur_frame # testing frame218 n_position = n_position // new_T * T # generate checkpoint position embedding219 sinusoid_table = np.array([get_position_angle_vec(pos_i) for pos_i in range(n_position)]) 220 sinusoid_table[:, 0::2] = np.sin(sinusoid_table[:, 0::2]) # dim 2i 221 sinusoid_table[:, 1::2] = np.cos(sinusoid_table[:, 1::2]) # dim 2i+1 222 sinusoid_table = torch.tensor(sinusoid_table, dtype=torch.float, requires_grad=False).unsqueeze(0)223 # interpolate224 P = int((n_position // T) ** 0.5)225 C = d_hid226 sinusoid_table = sinusoid_table.reshape(-1, T, P, P, C)227 sinusoid_table = sinusoid_table.permute(0, 2, 3, 4, 1).reshape(-1, C, T) # BHW, C, T228 sinusoid_table = torch.nn.functional.interpolate(sinusoid_table, size=new_T, mode='linear')229 sinusoid_table = sinusoid_table.reshape(1, P, P, C, new_T).permute(0, 4, 1, 2, 3) # B, T, H, W, C230 sinusoid_table = sinusoid_table.flatten(1, 3)231 return sinusoid_table232 else:233 sinusoid_table = np.array([get_position_angle_vec(pos_i) for pos_i in range(n_position)]) 234 sinusoid_table[:, 0::2] = np.sin(sinusoid_table[:, 0::2]) # dim 2i 235 sinusoid_table[:, 1::2] = np.cos(sinusoid_table[:, 1::2]) # dim 2i+1 236 return torch.tensor(sinusoid_table, dtype=torch.float, requires_grad=False).unsqueeze(0) 237 238 239def get_sinusoid_encoding_table2(n_position=784, d_hid=1024, cur_frame=8, ckpt_num_frame=4, pre_n_position=784): 240 ''' Sinusoid position encoding table ''' 241 # TODO: make it with torch instead of numpy 242 def get_position_angle_vec(position): 243 return [position / np.power(10000, 2 * (hid_j // 2) / d_hid) for hid_j in range(d_hid)] 244 245 # generate checkpoint position embedding246 sinusoid_table = np.array([get_position_angle_vec(pos_i) for pos_i in range(pre_n_position)]) 247 sinusoid_table[:, 0::2] = np.sin(sinusoid_table[:, 0::2]) # dim 2i 248 sinusoid_table[:, 1::2] = np.cos(sinusoid_table[:, 1::2]) # dim 2i+1 249 sinusoid_table = torch.tensor(sinusoid_table, dtype=torch.float, requires_grad=False).unsqueeze(0)250 251 # print(f"n_position: {n_position}")252 # print(f"pre_n_position: {pre_n_position}")253 254 if n_position != pre_n_position:255 T = ckpt_num_frame # checkpoint frame256 P = 14 # checkpoint size257 C = d_hid258 new_P = int((n_position // cur_frame) ** 0.5) # testing size259 # print(f'Pretraining uses 14x14, but current version is {new_P}x{new_P}')260 # print(f'Interpolate the position embedding')261 sinusoid_table = sinusoid_table.reshape(-1, T, P, P, C)262 sinusoid_table = sinusoid_table.reshape(-1, P, P, C).permute(0, 3, 1, 2)263 sinusoid_table = torch.nn.functional.interpolate(264 sinusoid_table, size=(new_P, new_P), mode='bicubic', align_corners=False)265 # BT, C, H, W -> BT, H, W, C -> B, T, H, W, C266 sinusoid_table = sinusoid_table.permute(0, 2, 3, 1).reshape(-1, T, new_P, new_P, C)267 sinusoid_table = sinusoid_table.flatten(1, 3) # B, THW, C268 269 if cur_frame != ckpt_num_frame:270 # print(f'Pretraining uses 4 frames, but current frame is {cur_frame}')271 # print(f'Interpolate the position embedding')272 T = ckpt_num_frame # checkpoint frame273 new_T = cur_frame # testing frame274 # interpolate275 P = int((n_position // cur_frame) ** 0.5) # testing size276 C = d_hid277 sinusoid_table = sinusoid_table.reshape(-1, T, P, P, C)278 sinusoid_table = sinusoid_table.permute(0, 2, 3, 4, 1).reshape(-1, C, T) # BHW, C, T279 sinusoid_table = torch.nn.functional.interpolate(sinusoid_table, size=new_T, mode='linear')280 sinusoid_table = sinusoid_table.reshape(1, P, P, C, new_T).permute(0, 4, 1, 2, 3) # B, T, H, W, C281 sinusoid_table = sinusoid_table.flatten(1, 3) # B, THW, C282 283 return sinusoid_table284 285 286class PretrainVisionTransformerEncoder(nn.Module):287 """ Vision Transformer with support for patch or hybrid CNN input stage288 """289 def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768, depth=12,290 num_heads=12, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop_rate=0., attn_drop_rate=0.,291 drop_path_rate=0., norm_layer=nn.LayerNorm, init_values=None, num_frames=8, tubelet_size=1,292 use_learnable_pos_emb=False,293 use_checkpoint=False, checkpoint_num=0, 294 ckpt_num_frame=-1, with_ln=True, return_index=-1295 ):296 super().__init__()297 self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models298 self.patch_embed = PatchEmbed(299 img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim, 300 num_frames=num_frames, tubelet_size=tubelet_size301 )302 num_patches = self.patch_embed.num_patches303 self.depth = depth + return_index + 1304 self.use_checkpoint = use_checkpoint305 self.checkpoint_num = checkpoint_num306 # print('umt:', f"Use checkpoint: {use_checkpoint}")307 # print('umt:', f"Checkpoint number: {checkpoint_num}")308 # print('UMT:', f"Real runing depth: {self.depth}")309 310 # TODO: Add the cls token311 if use_learnable_pos_emb:312 self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + 1, embed_dim))313 self.img_pos_embed = nn.Parameter(torch.zeros(1, num_patches//(num_frames//tubelet_size) + 1, embed_dim))314 else:315 # sine-cosine positional embeddings 316 if img_size != 224:317 self.pos_embed = get_sinusoid_encoding_table2(num_patches, embed_dim, ckpt_num_frame=ckpt_num_frame, cur_frame=num_frames//tubelet_size)318 self.img_pos_embed = get_sinusoid_encoding_table2(num_patches//(num_frames//tubelet_size), embed_dim, cur_frame=1, ckpt_num_frame=1, pre_n_position=14*14)319 else:320 self.pos_embed = get_sinusoid_encoding_table(num_patches, embed_dim, ckpt_num_frame=ckpt_num_frame, cur_frame=num_frames//tubelet_size)321 self.img_pos_embed = get_sinusoid_encoding_table(num_patches//(num_frames//tubelet_size), embed_dim)322 323 dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule324 self.blocks = nn.ModuleList([325 Block(326 dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale,327 drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer,328 init_values=init_values)329 for i in range(self.depth)])330 331 if with_ln:332 self.vision_layernorm = nn.LayerNorm(embed_dim, eps=1e-12)333 else:334 self.vision_layernorm = nn.Identity()335 336 if use_learnable_pos_emb:337 trunc_normal_(self.pos_embed, std=.02)338 339 @torch.jit.ignore340 def no_weight_decay(self):341 return {'pos_embed', 'cls_token'}342 343 def forward_features(self, x, use_image=False):344 x = self.patch_embed(x)345 346 if use_image:347 x = x + self.img_pos_embed.type_as(x).to(x.device).clone().detach()348 else:349 x = x + self.pos_embed.type_as(x).to(x.device).clone().detach()350 351 B, _, C = x.shape352 x_vis = x353 354 for idx, blk in enumerate(self.blocks):355 if self.use_checkpoint and idx < self.checkpoint_num:356 x_vis = checkpoint.checkpoint(blk, x_vis)357 else:358 x_vis = blk(x_vis)359 360 # with ln ot not361 x_vis = self.vision_layernorm(x_vis)362 return x_vis363 364 def forward(self, x, use_image=False):365 x_vis = self.forward_features(x, use_image)366 return x_vis367 368 369class PretrainVisionTransformer(nn.Module):370 """ Vision Transformer with support for patch or hybrid CNN input stage371 """372 def __init__(self,373 img_size=224, 374 patch_size=16, 375 encoder_in_chans=3, 376 encoder_embed_dim=768, 377 encoder_depth=12,378 encoder_num_heads=12, 379 mlp_ratio=4., 380 qkv_bias=True, 381 qk_scale=None, 382 drop_rate=0., 383 attn_drop_rate=0.,384 drop_path_rate=0., 385 norm_layer=partial(nn.LayerNorm, eps=1e-6), 386 init_values=0.,387 use_learnable_pos_emb=False,388 num_frames=8,389 tubelet_size=1,390 use_checkpoint=False,391 checkpoint_num=0,392 ckpt_num_frame=4, # the pretrained model uses 4 frames393 return_index=-1,394 with_ln=False395 ):396 super().__init__()397 398 self.encoder = PretrainVisionTransformerEncoder(399 img_size=img_size, 400 patch_size=patch_size, 401 in_chans=encoder_in_chans, 402 embed_dim=encoder_embed_dim, 403 depth=encoder_depth,404 num_heads=encoder_num_heads, 405 mlp_ratio=mlp_ratio, 406 qkv_bias=qkv_bias, 407 qk_scale=qk_scale, 408 drop_rate=drop_rate, 409 attn_drop_rate=attn_drop_rate,410 drop_path_rate=drop_path_rate, 411 norm_layer=norm_layer, 412 init_values=init_values,413 num_frames=num_frames,414 tubelet_size=tubelet_size,415 use_learnable_pos_emb=use_learnable_pos_emb,416 use_checkpoint=use_checkpoint,417 checkpoint_num=checkpoint_num,418 ckpt_num_frame=ckpt_num_frame,419 with_ln=with_ln,420 return_index=return_index421 )422 # print('umt:', f'With LN: {with_ln}')423 # print('UMT:', f'Total {encoder_depth} layer')424 # print('UMT:', f'Return {encoder_depth+return_index+1}-th layer')425 426 self.apply(self._init_weights)427 428 def _init_weights(self, m):429 if isinstance(m, nn.Linear):430 nn.init.xavier_uniform_(m.weight)431 if isinstance(m, nn.Linear) and m.bias is not None:432 nn.init.constant_(m.bias, 0)433 elif isinstance(m, nn.LayerNorm):434 nn.init.constant_(m.bias, 0)435 nn.init.constant_(m.weight, 1.0)436 437 @torch.jit.ignore438 def no_weight_decay(self):439 return {'pos_embed', 'cls_token', 'clip_pos_embed'}440 441 def forward(self, x, use_image=False):442 T = x.shape[2]443 x_vis = self.encoder(x, use_image) # [B, N_vis, C_e]444 B, TL, C = x_vis.shape445 x_vis = x_vis.view(B, T, TL // T, C)446 447 return x_vis448 449 450 451 452 453 454 455class UMTImageProcessor:456 def __init__(self, image_mean=(0.485, 0.456, 0.406), image_std=(0.229, 0.224, 0.225), size=(224, 224), crop_size: Dict[str, int] = None, resample=PILImageResampling.BICUBIC, rescale_factor=1 / 255, data_format=ChannelDimension.FIRST):457 crop_size = crop_size if crop_size is not None else {"height": 224, "width": 224}458 crop_size = get_size_dict(crop_size, default_to_square=True, param_name="crop_size")459 460 self.image_mean = image_mean461 self.image_std = image_std462 self.size = size463 self.resample = resample464 self.rescale_factor = rescale_factor465 self.data_format = data_format466 self.crop_size = crop_size467 468 def preprocess(self, images, return_tensors, target_size=None):469 if isinstance(images, Image.Image):470 images = [images]471 else:472 # to adapt video data473 images = [to_numpy_array(image) for image in images]474 assert isinstance(images, list)475 476 if target_size is None:477 target_size = self.size478 479 transforms = [480 convert_to_rgb,481 to_numpy_array,482 partial(resize, size=target_size, resample=self.resample, data_format=self.data_format),483 partial(rescale, scale=self.rescale_factor, data_format=self.data_format),484 partial(normalize, mean=self.image_mean, std=self.image_std, data_format=self.data_format),485 partial(to_channel_dimension_format, channel_dim=self.data_format, input_channel_dim=self.data_format),486 ]487 488 images = reduce(lambda x, f: [*map(f, x)], transforms, images)489 data = {"pixel_values": images}490 491 return BatchFeature(data=data, tensor_type=return_tensors)492 493 494class UMTVisionConfig:495 model_type = "umt_vision_model"496 497 def __init__(498 self,499 num_frames=4,500 hidden_size=1024,501 num_hidden_layers=24,502 num_attention_heads=16,503 num_channels=3,504 image_size=224,505 patch_size=16,506 return_idx=-2507 # **kwargs,508 ):509 # super().__init__(**kwargs)510 self.num_frames = num_frames511 self.hidden_size = hidden_size512 self.num_hidden_layers = num_hidden_layers513 self.num_attention_heads = num_attention_heads514 self.num_channels = num_channels515 self.patch_size = patch_size516 self.image_size = image_size517 self.return_idx = return_idx518 519 520def build_vit(config, pt_type='origin'):521 model = PretrainVisionTransformer(522 img_size=config.image_size, 523 patch_size=16, 524 encoder_embed_dim=1024, 525 encoder_depth=24,526 encoder_num_heads=16, 527 drop_path_rate=0., 528 num_frames=config.num_frames,529 tubelet_size=1,530 use_checkpoint=False,531 checkpoint_num=24,532 return_index=config.return_idx,533 with_ln=True, # merge vision_layernorm in it534 )535 536 # no need to load pt537 538 return model539 540 541 542class UMTVisionTower(nn.Module):543 def __init__(self, vision_tower, vision_tower_cfg, delay_load=False, pt_type='origin', image_size=224):544 super().__init__()545 546 self.is_loaded = False547 self.pt_type = pt_type548 549 self.config = UMTVisionConfig(num_frames=vision_tower_cfg.mm_local_num_frames, return_idx=vision_tower_cfg.mm_vision_select_layer, image_size=image_size)550 551 self.vision_tower_name = vision_tower552 553 self.image_processor = UMTImageProcessor(size=(image_size, image_size))554 555 if not delay_load:556 print(f"Loading vision tower: {vision_tower}")557 self.load_model()558 elif getattr(vision_tower_cfg, "unfreeze_mm_vision_tower", False):559 # TODO: better detector is needed.560 print(f"The checkpoint seems to contain `vision_tower` weights: `unfreeze_mm_vision_tower`: True.")561 self.load_model()562 elif hasattr(vision_tower_cfg, "mm_tunable_parts") and "mm_vision_tower" in vision_tower_cfg.mm_tunable_parts:563 print(f"The checkpoint seems to contain `vision_tower` weights: `mm_tunable_parts` contains `mm_vision_tower`.")564 self.load_model()565 else:566 self.cfg_only = self.config567 568 def load_model(self, device_map=None):569 if self.is_loaded:570 print("{} is already loaded, `load_model` called again, skipping.".format(self.vision_tower_name))571 return572 573 self.vision_tower = build_vit(self.config, pt_type=self.pt_type)574 self.vision_tower.requires_grad_(False)575 576 self.is_loaded = True577 578 def forward(self, images):579 if type(images) is list:580 raise NotImplementedError581 else:582 # input: B T C H W583 # output: B T*L C584 T = images.shape[1]585 images = images.permute(0, 2, 1, 3, 4)586 image_embeds = self.vision_tower(images, use_image=(T == 1))587 B, T, L, C = image_embeds.shape588 image_embeds = image_embeds.reshape(B, -1, C)589 590 return image_embeds591 592 @property593 def dummy_feature(self):594 return torch.zeros(1, self.hidden_size, device=self.device, dtype=self.dtype)595 596 @property597 def dtype(self):598 for p in self.vision_tower.parameters():599 return p.dtype600 601 @property602 def device(self):603 for p in self.vision_tower.parameters():604 return p.device605 606 @property607 def hidden_size(self):608 return self.config.hidden_size609 610 @property611 def num_patches(self):612 return (self.config.image_size // self.config.patch_size) ** 2613 614 @property615 def num_patches_per_side(self):616 return self.config.image_size // self.config.patch_size617 618 @property619 def image_size(self):620 return self.config.image_size621 622 623def build_vision_tower(vision_tower_cfg, **kwargs):624 vision_tower = getattr(vision_tower_cfg, "mm_vision_tower", getattr(vision_tower_cfg, "vision_tower", None))625 626 627 if "umt-hd" in vision_tower:628 return UMTVisionTower(vision_tower, vision_tower_cfg=vision_tower_cfg, image_size=448, **kwargs)629 elif "umt" in vision_tower:630 return UMTVisionTower(vision_tower, vision_tower_cfg=vision_tower_cfg, **kwargs)631 632 raise ValueError(f"Unknown vision tower: {vision_tower}")