algoryn/dots.ocr
153
1import math2 3import torch4import torch.nn as nn5import torch.nn.functional as F6import torch.utils.checkpoint7 8flash_attn_available = True9npu_available = True10 11try:12 from flash_attn import flash_attn_varlen_func13except ImportError:14 flash_attn_available = False15 16from torch.nn import LayerNorm17from transformers.modeling_utils import PreTrainedModel18from .configuration_dots import DotsVisionConfig19 20try:21 import torch_npu22except ImportError:23 npu_available = False24 25 26def rotate_half(x):27 """Rotates half the hidden dims of the input."""28 x1 = x[..., : x.shape[-1] // 2]29 x2 = x[..., x.shape[-1] // 2:]30 return torch.cat((-x2, x1), dim=-1)31 32 33def apply_rotary_pos_emb_vision(tensor: torch.Tensor, freqs: torch.Tensor) -> torch.Tensor:34 orig_dtype = tensor.dtype35 tensor = tensor.float()36 37 cos = freqs.cos()38 sin = freqs.sin()39 40 cos = cos.unsqueeze(1).repeat(1, 1, 2).unsqueeze(0).float()41 sin = sin.unsqueeze(1).repeat(1, 1, 2).unsqueeze(0).float()42 43 output = (tensor * cos) + (rotate_half(tensor) * sin)44 45 output = output.to(orig_dtype)46 47 return output48 49 50class VisionRotaryEmbedding(nn.Module):51 def __init__(self, dim: int, theta: float = 10000.0) -> None:52 super().__init__()53 inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float) / dim))54 self.register_buffer("inv_freq", inv_freq, persistent=False)55 56 def forward(self, seqlen: int) -> torch.Tensor:57 seq = torch.arange(seqlen, device=self.inv_freq.device, dtype=self.inv_freq.dtype)58 freqs = torch.outer(seq, self.inv_freq)59 return freqs60 61 62class PatchMerger(nn.Module):63 def __init__(64 self,65 dim: int,66 context_dim: int,67 spatial_merge_size: int = 2,68 pre_norm="layernorm",69 init_merger_std=None,70 ) -> None:71 super().__init__()72 self.hidden_size = context_dim * (spatial_merge_size ** 2)73 self.pre_norm = pre_norm74 if self.pre_norm == "layernorm":75 self.ln_q = LayerNorm(context_dim, eps=1e-6)76 elif self.pre_norm == "rmsnorm":77 self.ln_q = RMSNorm(context_dim, eps=1e-6)78 else:79 print("no norm in patch merger")80 81 self.mlp = nn.Sequential(82 nn.Linear(self.hidden_size, self.hidden_size),83 nn.GELU(),84 nn.Linear(self.hidden_size, dim),85 )86 87 if init_merger_std is not None:88 nn.init.normal_(self.mlp[0].weight, mean=0.0, std=init_merger_std)89 nn.init.zeros_(self.mlp[0].bias)90 nn.init.normal_(self.mlp[2].weight, mean=0.0, std=init_merger_std)91 nn.init.zeros_(self.mlp[2].bias)92 93 def forward(self, x: torch.Tensor) -> torch.Tensor:94 if self.pre_norm:95 x = self.mlp(self.ln_q(x).view(-1, self.hidden_size))96 else:97 x = self.mlp(x.view(-1, self.hidden_size))98 return x99 100 101class VisionAttention(nn.Module):102 def __init__(self, config, dim: int, num_heads: int = 16, bias=True) -> None:103 super().__init__()104 self.num_heads = num_heads105 self.head_dim = dim // num_heads106 self.qkv = nn.Linear(dim, dim * 3, bias=bias)107 self.proj = nn.Linear(dim, dim, bias=bias)108 109 def forward(110 self,111 hidden_states: torch.Tensor,112 cu_seqlens: torch.Tensor,113 rotary_pos_emb: torch.Tensor = None,114 ) -> torch.Tensor:115 seq_length = hidden_states.shape[0]116 117 q, k, v = self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0)118 q = apply_rotary_pos_emb_vision(q.unsqueeze(0), rotary_pos_emb).squeeze(0)119 k = apply_rotary_pos_emb_vision(k.unsqueeze(0), rotary_pos_emb).squeeze(0)120 121 attention_mask = torch.full(122 [1, seq_length, seq_length], torch.finfo(q.dtype).min, device=q.device, dtype=q.dtype123 )124 for i in range(1, len(cu_seqlens)):125 attention_mask[..., cu_seqlens[i - 1]: cu_seqlens[i], cu_seqlens[i - 1]: cu_seqlens[i]] = 0126 127 q = q.transpose(0, 1)128 k = k.transpose(0, 1)129 v = v.transpose(0, 1)130 attn_weights = torch.matmul(q, k.transpose(1, 2)) / math.sqrt(self.head_dim)131 attn_weights = attn_weights + attention_mask132 attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(q.dtype)133 attn_output = torch.matmul(attn_weights, v)134 attn_output = attn_output.transpose(0, 1)135 attn_output = attn_output.reshape(seq_length, -1)136 attn_output = self.proj(attn_output)137 return attn_output138 139 140class VisionFlashAttention2(nn.Module):141 def __init__(self, config, dim: int, num_heads: int = 16, bias=True) -> None:142 super().__init__()143 self.num_heads = num_heads144 self.qkv = nn.Linear(dim, dim * 3, bias=bias)145 self.proj = nn.Linear(dim, dim, bias=bias)146 self.config = config147 self.is_causal = config.is_causal148 149 def forward(150 self,151 hidden_states: torch.Tensor,152 cu_seqlens: torch.Tensor,153 rotary_pos_emb: torch.Tensor = None,154 ) -> torch.Tensor:155 seq_length = hidden_states.shape[0]156 q, k, v = (157 self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0)158 ) # 'shd'159 q = apply_rotary_pos_emb_vision(q.unsqueeze(0), rotary_pos_emb).squeeze(0)160 k = apply_rotary_pos_emb_vision(k.unsqueeze(0), rotary_pos_emb).squeeze(0)161 max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item()162 attn_output = flash_attn_varlen_func(163 q, k, v, cu_seqlens, cu_seqlens, max_seqlen, max_seqlen, causal=self.is_causal164 ).reshape(seq_length, -1)165 attn_output = self.proj(attn_output)166 167 return attn_output168 169 170class VisionAttentionV2(nn.Module):171 def __init__(self, config, dim: int, num_heads: int = 16, bias=True) -> None:172 super().__init__()173 self.num_heads = num_heads174 self.head_dim = dim // num_heads175 self.qkv = nn.Linear(dim, dim * 3, bias=bias)176 self.proj = nn.Linear(dim, dim, bias=bias)177 178 def forward(179 self,180 hidden_states: torch.Tensor,181 cu_seqlens: torch.Tensor,182 rotary_pos_emb: torch.Tensor = None,183 ) -> torch.Tensor:184 seq_length = hidden_states.shape[0]185 186 q, k, v = self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0)187 q = apply_rotary_pos_emb_vision(q.unsqueeze(0), rotary_pos_emb).squeeze(0)188 k = apply_rotary_pos_emb_vision(k.unsqueeze(0), rotary_pos_emb).squeeze(0)189 190 seqlens = torch.diff(cu_seqlens).tolist()191 192 q_list = torch.split(q, seqlens, 0)193 k_list = torch.split(k, seqlens, 0)194 v_list = torch.split(v, seqlens, 0)195 # eager attention 空间复杂度为 O(n^2) , n 为 b*s（batch_size * seq_len）, 序列太长容易OOM， 这个实现 更具batch 切分 seq196 # 减少内存需求， 计算相对 continus batching 较慢。197 outputs = []198 for q_i, k_i, v_i in zip(q_list, k_list, v_list):199 q_i = q_i.transpose(0, 1)200 k_i = k_i.transpose(0, 1)201 v_i = v_i.transpose(0, 1)202 out = torch.matmul(q_i, k_i.transpose(1, 2)) / math.sqrt(self.head_dim)203 out = nn.functional.softmax(out, dim=-1, dtype=torch.float32).to(q.dtype)204 out = torch.matmul(out, v_i)205 out = out.transpose(0, 1)206 outputs.append(out)207 208 attn_output = torch.concat(outputs, dim=0)209 attn_output = attn_output.reshape(seq_length, -1)210 attn_output = self.proj(attn_output)211 return attn_output212 213 214class VisionAscendAttention(nn.Module):215 def __init__(self, config, dim: int, num_heads: int = 16, bias=True) -> None:216 super().__init__()217 self.num_heads = num_heads218 self.head_dim = dim // num_heads219 self.qkv = nn.Linear(dim, dim * 3, bias=bias)220 self.proj = nn.Linear(dim, dim, bias=bias)221 self.config = config222 223 def forward(224 self,225 hidden_states: torch.Tensor,226 cu_seqlens: torch.Tensor,227 rotary_pos_emb: torch.Tensor = None,228 ) -> torch.Tensor:229 seq_length = hidden_states.shape[0]230 q, k, v = self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0)231 232 q = apply_rotary_pos_emb_vision(q.unsqueeze(0), rotary_pos_emb).squeeze(0)233 k = apply_rotary_pos_emb_vision(k.unsqueeze(0), rotary_pos_emb).squeeze(0)234 235 attention_mask = torch.ones([1, seq_length, seq_length], device=q.device, dtype=torch.bool)236 for i in range(1, len(cu_seqlens)):237 attention_mask[..., cu_seqlens[i - 1]: cu_seqlens[i], cu_seqlens[i - 1]: cu_seqlens[i]] = False238 239 q = q.transpose(0, 1).unsqueeze(0)240 k = k.transpose(0, 1).unsqueeze(0)241 v = v.transpose(0, 1).unsqueeze(0)242 243 attn_output = torch_npu.npu_prompt_flash_attention(q, k, v,244 atten_mask=attention_mask,245 num_heads=self.num_heads, input_layout="BNSD",246 scale_value=self.head_dim ** -0.5)247 attn_output = attn_output.squeeze(0).transpose(0, 1)248 attn_output = attn_output.reshape(seq_length, -1)249 attn_output = self.proj(attn_output)250 return attn_output251 252 253class VisionSdpaAttention(nn.Module):254 def __init__(self, config, dim: int, num_heads: int = 16, bias=True) -> None:255 super().__init__()256 self.num_heads = num_heads257 self.qkv = nn.Linear(dim, dim * 3, bias=bias)258 self.proj = nn.Linear(dim, dim, bias=bias)259 self.config = config260 261 def forward(262 self,263 hidden_states: torch.Tensor,264 cu_seqlens: torch.Tensor,265 rotary_pos_emb: torch.Tensor = None,266 ) -> torch.Tensor:267 seq_length = hidden_states.shape[0]268 q, k, v = self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0)269 270 q = apply_rotary_pos_emb_vision(q.unsqueeze(0), rotary_pos_emb).squeeze(0)271 k = apply_rotary_pos_emb_vision(k.unsqueeze(0), rotary_pos_emb).squeeze(0)272 273 attention_mask = torch.zeros([1, seq_length, seq_length], device=q.device, dtype=torch.bool)274 for i in range(1, len(cu_seqlens)):275 attention_mask[..., cu_seqlens[i - 1]: cu_seqlens[i], cu_seqlens[i - 1]: cu_seqlens[i]] = True276 277 # Convert q, k, v to 4D to enable : (1, num_heads, seq_length, head_dim)278 q = q.transpose(0, 1).unsqueeze(0) # (1, num_heads, seq_length, head_dim)279 k = k.transpose(0, 1).unsqueeze(0)280 v = v.transpose(0, 1).unsqueeze(0)281 282 # See: https://github.com/pytorch/pytorch/issues/127523283 if attention_mask.stride(-1) != 1:284 attention_mask = torch.empty_like(attention_mask, memory_format=torch.contiguous_format).copy_(attention_mask)285 286 # use memory efficient backend287 from torch.nn.attention import SDPBackend, sdpa_kernel288 with sdpa_kernel(SDPBackend.EFFICIENT_ATTENTION):289 attn_output = F.scaled_dot_product_attention(q, k, v, attention_mask, dropout_p=0.0)290 291 attn_output = attn_output.squeeze(0).transpose(0, 1) # (seq_length, num_heads, head_dim)292 attn_output = attn_output.reshape(seq_length, -1)293 294 attn_output = self.proj(attn_output)295 return attn_output296 297 298DOTS_VISION_ATTENTION_CLASSES = {299 "eager": VisionAttention,300 "eager_v2": VisionAttentionV2, # 内存更少301 "flash_attention_2": VisionFlashAttention2,302 "sdpa": VisionSdpaAttention,303 "ascend_fa": VisionAscendAttention, # ascend， 长序列精度下降严重。304}305 306 307class RMSNorm(nn.Module):308 def __init__(self, dim: int, eps: float = 1e-6):309 super().__init__()310 self.weight = nn.Parameter(torch.ones(dim))311 self.eps = eps312 313 def forward(self, x: torch.Tensor) -> torch.Tensor:314 output = self._norm(x.float()).type_as(x)315 return output * self.weight316 317 def extra_repr(self) -> str:318 return f"{tuple(self.weight.shape)}, eps={self.eps}"319 320 def _norm(self, x: torch.Tensor) -> torch.Tensor:321 return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)322 323 324class DotsSwiGLUFFN(nn.Module):325 def __init__(self, config):326 super().__init__()327 hidden_features = config.intermediate_size328 in_features = config.embed_dim329 bias = config.use_bias330 331 self.fc1 = nn.Linear(in_features, hidden_features, bias=bias)332 self.fc2 = nn.Linear(hidden_features, in_features, bias=bias)333 self.fc3 = nn.Linear(in_features, hidden_features, bias=bias)334 335 def forward(self, x: torch.Tensor) -> torch.Tensor:336 x = F.silu(self.fc1(x)) * self.fc3(x)337 x = self.fc2(x)338 return x339 340 341class DotsPatchEmbed(nn.Module):342 def __init__(self, config):343 super().__init__()344 self.num_channels = config.num_channels345 self.patch_size = config.patch_size346 self.temporal_patch_size = config.temporal_patch_size347 self.embed_dim = config.embed_dim348 self.config = config349 self.proj = nn.Conv2d(350 config.num_channels,351 config.embed_dim,352 kernel_size=(config.patch_size, config.patch_size),353 stride=(config.patch_size, config.patch_size),354 )355 self.norm = RMSNorm(config.embed_dim, eps=config.rms_norm_eps)356 357 def forward(self, x: torch.Tensor, grid_thw=None) -> torch.Tensor:358 x = x.view(-1, self.num_channels, self.temporal_patch_size, self.patch_size, self.patch_size)[:, :, 0]359 x = self.proj(x).view(-1, self.embed_dim)360 x = self.norm(x)361 return x362 363 364class DotsViTPreprocessor(nn.Module):365 def __init__(self, config):366 super().__init__()367 self.patch_h = config.patch_size368 self.patch_w = config.patch_size369 self.embed_dim = config.embed_dim370 self.config = config371 self.patchifier = DotsPatchEmbed(config)372 373 def forward(self, x: torch.Tensor, grid_thw=None) -> torch.Tensor:374 tokens = self.patchifier(x, grid_thw)375 return tokens376 377 378class DotsVisionBlock(nn.Module):379 def __init__(self, config, attn_implementation: str = "flash_attention_2"):380 super().__init__()381 382 if attn_implementation == "flash_attention_2" and not flash_attn_available:383 # fallback to eager384 attn_implementation = "eager"385 print("flash attention not available! fallback to eager implementation ")386 387 if attn_implementation == "ascend_fa" and not npu_available:388 attn_implementation = "eager"389 print("flash attention not available! fallback to eager implementation ")390 391 self.attn = DOTS_VISION_ATTENTION_CLASSES[attn_implementation](392 config, config.embed_dim, num_heads=config.num_attention_heads, bias=config.use_bias393 )394 self.norm1 = RMSNorm(config.embed_dim, eps=config.rms_norm_eps)395 self.mlp = DotsSwiGLUFFN(config)396 self.norm2 = RMSNorm(config.embed_dim, eps=config.rms_norm_eps)397 398 def forward(self, hidden_states, cu_seqlens, rotary_pos_emb) -> torch.Tensor:399 hidden_states = hidden_states + self.attn(400 self.norm1(hidden_states), cu_seqlens=cu_seqlens, rotary_pos_emb=rotary_pos_emb401 )402 hidden_states = hidden_states + self.mlp(self.norm2(hidden_states))403 return hidden_states404 405 406class DotsVisionTransformer(PreTrainedModel):407 def __init__(self, config: DotsVisionConfig) -> None:408 super().__init__(config)409 self.config = config410 self.spatial_merge_size = config.spatial_merge_size411 412 self.patch_embed = DotsViTPreprocessor(config)413 self._init_weights(self.patch_embed.patchifier.proj)414 415 head_dim = config.embed_dim // config.num_attention_heads416 417 self.rotary_pos_emb = VisionRotaryEmbedding(head_dim // 2)418 419 _num_hidden_layers = config.num_hidden_layers420 self.blocks = nn.ModuleList(421 [DotsVisionBlock(config, config.attn_implementation) for _ in range(_num_hidden_layers)]422 )423 424 if self.config.post_norm:425 self.post_trunk_norm = RMSNorm(config.embed_dim, eps=config.rms_norm_eps)426 427 self.merger = PatchMerger(428 dim=config.hidden_size,429 context_dim=config.embed_dim,430 spatial_merge_size=config.spatial_merge_size,431 init_merger_std=self.config.init_merger_std,432 )433 434 self.gradient_checkpointing = False435 self._gradient_checkpointing_func = torch.utils.checkpoint.checkpoint436 437 def _init_weights(self, module):438 std = self.config.initializer_range439 if isinstance(module, (nn.Linear, nn.Conv3d)):440 module.weight.data.normal_(mean=0.0, std=std)441 if module.bias is not None:442 module.bias.data.zero_()443 elif isinstance(module, nn.Embedding):444 module.weight.data.normal_(mean=0.0, std=std)445 if module.padding_idx is not None:446 module.weight.data[module.padding_idx].zero_()447 448 @property449 def dtype(self) -> torch.dtype:450 return self.blocks[0].mlp.fc2.weight.dtype451 452 @property453 def device(self) -> torch.device:454 return self.blocks[0].mlp.fc2.weight.device455 456 def get_pos_ids_by_grid(self, grid_thw):457 pos_ids = []458 for t, h, w in grid_thw:459 hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)460 hpos_ids = hpos_ids.reshape(461 h // self.spatial_merge_size,462 self.spatial_merge_size,463 w // self.spatial_merge_size,464 self.spatial_merge_size,465 )466 hpos_ids = hpos_ids.permute(0, 2, 1, 3)467 hpos_ids = hpos_ids.flatten()468 469 wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)470 wpos_ids = wpos_ids.reshape(471 h // self.spatial_merge_size,472 self.spatial_merge_size,473 w // self.spatial_merge_size,474 self.spatial_merge_size,475 )476 wpos_ids = wpos_ids.permute(0, 2, 1, 3)477 wpos_ids = wpos_ids.flatten()478 pos_ids.append(479 torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1)480 )481 482 return pos_ids483 484 def rot_pos_emb(self, grid_thw):485 pos_ids = self.get_pos_ids_by_grid(grid_thw)486 pos_ids = torch.cat(pos_ids, dim=0)487 max_grid_size = grid_thw[:, 1:].max()488 rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size)489 rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1)490 return rotary_pos_emb491 492 def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, bf16=True) -> torch.Tensor:493 if bf16:494 hidden_states = hidden_states.bfloat16()495 hidden_states = self.patch_embed(hidden_states, grid_thw)496 497 rotary_pos_emb = self.rot_pos_emb(grid_thw)498 499 cu_seqlens = torch.repeat_interleave(grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]).cumsum(500 dim=0,501 dtype=grid_thw.dtype if torch.jit.is_tracing() else torch.int32,502 )503 cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0)504 505 for blk in self.blocks:506 if self.gradient_checkpointing and self.training:507 hidden_states = self._gradient_checkpointing_func(508 blk.__call__,509 hidden_states,510 cu_seqlens,511 rotary_pos_emb,512 )513 else:514 hidden_states = blk(hidden_states, cu_seqlens=cu_seqlens, rotary_pos_emb=rotary_pos_emb)515 516 if self.config.post_norm:517 hidden_states = self.post_trunk_norm(hidden_states)518 519 hidden_states = self.merger(hidden_states)520 return hidden_states521