optimum-intel-internal-testing/tiny-random-minicpmv-2_6
129k
1import warnings2from functools import partial3from typing import Optional, Tuple, List4 5import numpy as np6import torch7import torch.nn.functional as F8from torch import Tensor, nn9from torch.nn.functional import *10from torch.nn.init import trunc_normal_11from torch.nn.modules.activation import *12from transformers.integrations import is_deepspeed_zero3_enabled13 14 15def get_2d_sincos_pos_embed(embed_dim, image_size):16 """17 image_size: image_size or (image_height, image_width)18 return:19 pos_embed: [image_height, image_width, embed_dim]20 """21 if isinstance(image_size, int):22 grid_h_size, grid_w_size = image_size, image_size23 else:24 grid_h_size, grid_w_size = image_size[0], image_size[1]25 26 grid_h = np.arange(grid_h_size, dtype=np.float32)27 grid_w = np.arange(grid_w_size, dtype=np.float32)28 grid = np.meshgrid(grid_w, grid_h) # here w goes first29 grid = np.stack(grid, axis=0)30 31 pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)32 return pos_embed33 34 35def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):36 assert embed_dim % 2 == 037 38 # use half of dimensions to encode grid_h39 emb_h = get_1d_sincos_pos_embed_from_grid_new(embed_dim // 2, grid[0]) # (H, W, D/2)40 emb_w = get_1d_sincos_pos_embed_from_grid_new(embed_dim // 2, grid[1]) # (H, W, D/2)41 42 emb = np.concatenate([emb_h, emb_w], axis=-1) # (H, W, D)43 return emb44 45 46def get_1d_sincos_pos_embed_from_grid_new(embed_dim, pos):47 """48 embed_dim: output dimension for each position49 pos: a list of positions to be encoded: size (H, W)50 out: (H, W, D)51 """52 assert embed_dim % 2 == 053 omega = np.arange(embed_dim // 2, dtype=np.float32)54 omega /= embed_dim / 2.055 omega = 1.0 / 10000**omega # (D/2,)56 57 out = np.einsum("hw,d->hwd", pos, omega) # (H, W, D/2), outer product58 59 emb_sin = np.sin(out) # (H, W, D/2)60 emb_cos = np.cos(out) # (H, W, D/2)61 62 emb = np.concatenate([emb_sin, emb_cos], axis=-1) # (H, W, D)63 return emb64 65 66class Resampler(nn.Module):67 """68 A 2D perceiver-resampler network with one cross attention layers by69 given learnable queries and 2d sincos pos_emb70 Outputs:71 A tensor with the shape of (batch_size, num_queries, embed_dim)72 """73 74 def __init__(75 self,76 num_queries,77 embed_dim,78 num_heads,79 kv_dim=None,80 norm_layer=partial(nn.LayerNorm, eps=1e-6),81 adaptive=False,82 max_size=(70, 70),83 ):84 super().__init__()85 self.num_queries = num_queries86 self.embed_dim = embed_dim87 self.num_heads = num_heads88 self.adaptive = adaptive89 self.max_size = max_size90 91 self.query = nn.Parameter(torch.zeros(self.num_queries, embed_dim))92 93 if kv_dim is not None and kv_dim != embed_dim:94 self.kv_proj = nn.Linear(kv_dim, embed_dim, bias=False)95 else:96 self.kv_proj = nn.Identity()97 98 self.attn = MultiheadAttention(embed_dim, num_heads)99 self.ln_q = norm_layer(embed_dim)100 self.ln_kv = norm_layer(embed_dim)101 102 self.ln_post = norm_layer(embed_dim)103 self.proj = nn.Parameter((embed_dim**-0.5) * torch.randn(embed_dim, embed_dim))104 105 self._set_2d_pos_cache(self.max_size)106 107 def _set_2d_pos_cache(self, max_size, device="cpu"):108 if is_deepspeed_zero3_enabled():109 device = "cuda"110 pos_embed = torch.from_numpy(get_2d_sincos_pos_embed(self.embed_dim, max_size)).float().to(device)111 self.register_buffer("pos_embed", pos_embed, persistent=False)112 113 def _adjust_pos_cache(self, tgt_sizes, device):114 max_h = torch.max(tgt_sizes[:, 0])115 max_w = torch.max(tgt_sizes[:, 1])116 if max_h > self.max_size[0] or max_w > self.max_size[1]:117 self.max_size = [max(max_h, self.max_size[0]), max(max_w, self.max_size[1])]118 self._set_2d_pos_cache(self.max_size, device)119 120 def _initialize_weights(self, module):121 """122 Initialize the weights if they are not already initialized.123 """124 if getattr(module, "_is_hf_initialized", False):125 return126 self._init_weights(module)127 module._is_hf_initialized = True128 129 def _init_weights(self, m):130 if isinstance(m, nn.Linear):131 trunc_normal_(m.weight, std=0.02)132 if isinstance(m, nn.Linear) and m.bias is not None:133 nn.init.constant_(m.bias, 0)134 elif isinstance(m, nn.LayerNorm):135 nn.init.constant_(m.bias, 0)136 nn.init.constant_(m.weight, 1.0)137 138 def forward(self, x, tgt_sizes=None):139 assert x.shape[0] == tgt_sizes.shape[0]140 bs = x.shape[0]141 142 device = x.device143 dtype = x.dtype144 145 patch_len = tgt_sizes[:, 0] * tgt_sizes[:, 1]146 147 self._adjust_pos_cache(tgt_sizes, device=device)148 149 max_patch_len = torch.max(patch_len)150 key_padding_mask = torch.zeros((bs, max_patch_len), dtype=torch.bool, device=device)151 152 pos_embed = []153 for i in range(bs):154 tgt_h, tgt_w = tgt_sizes[i]155 pos_embed.append(self.pos_embed[:tgt_h, :tgt_w, :].reshape((tgt_h * tgt_w, -1)).to(dtype)) # patches * D156 key_padding_mask[i, patch_len[i] :] = True157 158 pos_embed = torch.nn.utils.rnn.pad_sequence(pos_embed, batch_first=True, padding_value=0.0).permute(159 1, 0, 2160 ) # BLD => L * B * D161 162 x = self.kv_proj(x) # B * L * D163 x = self.ln_kv(x).permute(1, 0, 2) # L * B * D164 165 q = self.ln_q(self.query) # Q * D166 167 out = self.attn(168 self._repeat(q, bs), # Q * B * D169 x + pos_embed, # L * B * D + L * B * D170 x,171 key_padding_mask=key_padding_mask,172 )[0]173 # out: Q * B * D174 x = out.permute(1, 0, 2) # B * Q * D175 176 x = self.ln_post(x)177 x = x @ self.proj178 return x179 180 def _repeat(self, query, N: int):181 return query.unsqueeze(1).repeat(1, N, 1)182 183 184class MultiheadAttention(nn.MultiheadAttention):185 def __init__(186 self,187 embed_dim,188 num_heads,189 dropout=0.0,190 bias=True,191 add_bias_kv=False,192 add_zero_attn=False,193 kdim=None,194 vdim=None,195 batch_first=False,196 device=None,197 dtype=None,198 ):199 super().__init__(200 embed_dim, num_heads, dropout, bias, add_bias_kv, add_zero_attn, kdim, vdim, batch_first, device, dtype201 )202 203 # rewrite out_proj layer，with nn.Linear204 self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias, device=device, dtype=dtype)205 206 def forward(207 self,208 query: Tensor,209 key: Tensor,210 value: Tensor,211 key_padding_mask: Optional[Tensor] = None,212 need_weights: bool = True,213 attn_mask: Optional[Tensor] = None,214 average_attn_weights: bool = True,215 is_causal: bool = False,216 ) -> Tuple[Tensor, Optional[Tensor]]:217 why_not_fast_path = ""218 if (219 (attn_mask is not None and torch.is_floating_point(attn_mask))220 or (key_padding_mask is not None)221 and torch.is_floating_point(key_padding_mask)222 ):223 why_not_fast_path = "floating-point masks are not supported for fast path."224 225 is_batched = query.dim() == 3226 227 key_padding_mask = _canonical_mask(228 mask=key_padding_mask,229 mask_name="key_padding_mask",230 other_type=F._none_or_dtype(attn_mask),231 other_name="attn_mask",232 target_type=query.dtype,233 )234 235 attn_mask = _canonical_mask(236 mask=attn_mask,237 mask_name="attn_mask",238 other_type=None,239 other_name="",240 target_type=query.dtype,241 check_other=False,242 )243 244 if not is_batched:245 why_not_fast_path = f"input not batched; expected query.dim() of 3 but got {query.dim()}"246 elif query is not key or key is not value:247 # When lifting this restriction, don't forget to either248 # enforce that the dtypes all match or test cases where249 # they don't!250 why_not_fast_path = "non-self attention was used (query, key, and value are not the same Tensor)"251 elif self.in_proj_bias is not None and query.dtype != self.in_proj_bias.dtype:252 why_not_fast_path = (253 f"dtypes of query ({query.dtype}) and self.in_proj_bias ({self.in_proj_bias.dtype}) don't match"254 )255 elif self.in_proj_weight is None:256 why_not_fast_path = "in_proj_weight was None"257 elif query.dtype != self.in_proj_weight.dtype:258 # this case will fail anyway, but at least they'll get a useful error message.259 why_not_fast_path = (260 f"dtypes of query ({query.dtype}) and self.in_proj_weight ({self.in_proj_weight.dtype}) don't match"261 )262 elif self.training:263 why_not_fast_path = "training is enabled"264 elif (self.num_heads % 2) != 0:265 why_not_fast_path = "self.num_heads is not even"266 elif not self.batch_first:267 why_not_fast_path = "batch_first was not True"268 elif self.bias_k is not None:269 why_not_fast_path = "self.bias_k was not None"270 elif self.bias_v is not None:271 why_not_fast_path = "self.bias_v was not None"272 elif self.add_zero_attn:273 why_not_fast_path = "add_zero_attn was enabled"274 elif not self._qkv_same_embed_dim:275 why_not_fast_path = "_qkv_same_embed_dim was not True"276 elif query.is_nested and (key_padding_mask is not None or attn_mask is not None):277 why_not_fast_path = "supplying both src_key_padding_mask and src_mask at the same time \278 is not supported with NestedTensor input"279 elif torch.is_autocast_enabled():280 why_not_fast_path = "autocast is enabled"281 282 if not why_not_fast_path:283 tensor_args = (284 query,285 key,286 value,287 self.in_proj_weight,288 self.in_proj_bias,289 self.out_proj.weight,290 self.out_proj.bias,291 )292 # We have to use list comprehensions below because TorchScript does not support293 # generator expressions.294 if torch.overrides.has_torch_function(tensor_args):295 why_not_fast_path = "some Tensor argument has_torch_function"296 elif _is_make_fx_tracing():297 why_not_fast_path = "we are running make_fx tracing"298 elif not all(_check_arg_device(x) for x in tensor_args):299 why_not_fast_path = (300 "some Tensor argument's device is neither one of "301 f"cpu, cuda or {torch.utils.backend_registration._privateuse1_backend_name}"302 )303 elif torch.is_grad_enabled() and any(_arg_requires_grad(x) for x in tensor_args):304 why_not_fast_path = (305 "grad is enabled and at least one of query or the "306 "input/output projection weights or biases requires_grad"307 )308 if not why_not_fast_path:309 merged_mask, mask_type = self.merge_masks(attn_mask, key_padding_mask, query)310 311 if self.in_proj_bias is not None and self.in_proj_weight is not None:312 return torch._native_multi_head_attention(313 query,314 key,315 value,316 self.embed_dim,317 self.num_heads,318 self.in_proj_weight,319 self.in_proj_bias,320 self.out_proj.weight,321 self.out_proj.bias,322 merged_mask,323 need_weights,324 average_attn_weights,325 mask_type,326 )327 328 any_nested = query.is_nested or key.is_nested or value.is_nested329 assert not any_nested, (330 "MultiheadAttention does not support NestedTensor outside of its fast path. "331 + f"The fast path was not hit because {why_not_fast_path}"332 )333 334 if self.batch_first and is_batched:335 # make sure that the transpose op does not affect the "is" property336 if key is value:337 if query is key:338 query = key = value = query.transpose(1, 0)339 else:340 query, key = (x.transpose(1, 0) for x in (query, key))341 value = key342 else:343 query, key, value = (x.transpose(1, 0) for x in (query, key, value))344 345 if not self._qkv_same_embed_dim:346 attn_output, attn_output_weights = self.multi_head_attention_forward(347 query,348 key,349 value,350 self.embed_dim,351 self.num_heads,352 self.in_proj_weight,353 self.in_proj_bias,354 self.bias_k,355 self.bias_v,356 self.add_zero_attn,357 self.dropout,358 self.out_proj.weight,359 self.out_proj.bias,360 training=self.training,361 key_padding_mask=key_padding_mask,362 need_weights=need_weights,363 attn_mask=attn_mask,364 use_separate_proj_weight=True,365 q_proj_weight=self.q_proj_weight,366 k_proj_weight=self.k_proj_weight,367 v_proj_weight=self.v_proj_weight,368 average_attn_weights=average_attn_weights,369 is_causal=is_causal,370 )371 else:372 attn_output, attn_output_weights = self.multi_head_attention_forward(373 query,374 key,375 value,376 self.embed_dim,377 self.num_heads,378 self.in_proj_weight,379 self.in_proj_bias,380 self.bias_k,381 self.bias_v,382 self.add_zero_attn,383 self.dropout,384 self.out_proj.weight,385 self.out_proj.bias,386 training=self.training,387 key_padding_mask=key_padding_mask,388 need_weights=need_weights,389 attn_mask=attn_mask,390 average_attn_weights=average_attn_weights,391 is_causal=is_causal,392 )393 if self.batch_first and is_batched:394 return attn_output.transpose(1, 0), attn_output_weights395 else:396 return attn_output, attn_output_weights397 398 def multi_head_attention_forward(399 self,400 query: Tensor,401 key: Tensor,402 value: Tensor,403 embed_dim_to_check: int,404 num_heads: int,405 in_proj_weight: Optional[Tensor],406 in_proj_bias: Optional[Tensor],407 bias_k: Optional[Tensor],408 bias_v: Optional[Tensor],409 add_zero_attn: bool,410 dropout_p: float,411 out_proj_weight: Tensor,412 out_proj_bias: Optional[Tensor],413 training: bool = True,414 key_padding_mask: Optional[Tensor] = None,415 need_weights: bool = True,416 attn_mask: Optional[Tensor] = None,417 use_separate_proj_weight: bool = False,418 q_proj_weight: Optional[Tensor] = None,419 k_proj_weight: Optional[Tensor] = None,420 v_proj_weight: Optional[Tensor] = None,421 static_k: Optional[Tensor] = None,422 static_v: Optional[Tensor] = None,423 average_attn_weights: bool = True,424 is_causal: bool = False,425 ) -> Tuple[Tensor, Optional[Tensor]]:426 tens_ops = (query, key, value, in_proj_weight, in_proj_bias, bias_k, bias_v, out_proj_weight, out_proj_bias)427 428 is_batched = _mha_shape_check(query, key, value, key_padding_mask, attn_mask, num_heads)429 430 # For unbatched input, we unsqueeze at the expected batch-dim to pretend that the input431 # is batched, run the computation and before returning squeeze the432 # batch dimension so that the output doesn't carry this temporary batch dimension.433 if not is_batched:434 # unsqueeze if the input is unbatched435 query = query.unsqueeze(1)436 key = key.unsqueeze(1)437 value = value.unsqueeze(1)438 if key_padding_mask is not None:439 key_padding_mask = key_padding_mask.unsqueeze(0)440 441 # set up shape vars442 tgt_len, bsz, embed_dim = query.shape443 src_len, _, _ = key.shape444 445 key_padding_mask = _canonical_mask(446 mask=key_padding_mask,447 mask_name="key_padding_mask",448 other_type=_none_or_dtype(attn_mask),449 other_name="attn_mask",450 target_type=query.dtype,451 )452 453 if is_causal and attn_mask is None:454 raise RuntimeError(455 "Need attn_mask if specifying the is_causal hint. "456 "You may use the Transformer module method "457 "`generate_square_subsequent_mask` to create this mask."458 )459 460 if is_causal and key_padding_mask is None and not need_weights:461 # when we have a kpm or need weights, we need attn_mask462 # Otherwise, we use the is_causal hint go as is_causal463 # indicator to SDPA.464 attn_mask = None465 else:466 attn_mask = _canonical_mask(467 mask=attn_mask,468 mask_name="attn_mask",469 other_type=None,470 other_name="",471 target_type=query.dtype,472 check_other=False,473 )474 475 if key_padding_mask is not None:476 # We have the attn_mask, and use that to merge kpm into it.477 # Turn off use of is_causal hint, as the merged mask is no478 # longer causal.479 is_causal = False480 481 assert (482 embed_dim == embed_dim_to_check483 ), f"was expecting embedding dimension of {embed_dim_to_check}, but got {embed_dim}"484 if isinstance(embed_dim, torch.Tensor):485 # embed_dim can be a tensor when JIT tracing486 head_dim = embed_dim.div(num_heads, rounding_mode="trunc")487 else:488 head_dim = embed_dim // num_heads489 assert head_dim * num_heads == embed_dim, f"embed_dim {embed_dim} not divisible by num_heads {num_heads}"490 if use_separate_proj_weight:491 # allow MHA to have different embedding dimensions when separate projection weights are used492 assert (493 key.shape[:2] == value.shape[:2]494 ), f"key's sequence and batch dims {key.shape[:2]} do not match value's {value.shape[:2]}"495 else:496 assert key.shape == value.shape, f"key shape {key.shape} does not match value shape {value.shape}"497 498 #499 # compute in-projection500 #501 if not use_separate_proj_weight:502 assert in_proj_weight is not None, "use_separate_proj_weight is False but in_proj_weight is None"503 q, k, v = _in_projection_packed(query, key, value, in_proj_weight, in_proj_bias)504 else:505 assert q_proj_weight is not None, "use_separate_proj_weight is True but q_proj_weight is None"506 assert k_proj_weight is not None, "use_separate_proj_weight is True but k_proj_weight is None"507 assert v_proj_weight is not None, "use_separate_proj_weight is True but v_proj_weight is None"508 if in_proj_bias is None:509 b_q = b_k = b_v = None510 else:511 b_q, b_k, b_v = in_proj_bias.chunk(3)512 q, k, v = _in_projection(query, key, value, q_proj_weight, k_proj_weight, v_proj_weight, b_q, b_k, b_v)513 514 # prep attention mask515 516 if attn_mask is not None:517 # ensure attn_mask's dim is 3518 if attn_mask.dim() == 2:519 correct_2d_size = (tgt_len, src_len)520 if attn_mask.shape != correct_2d_size:521 raise RuntimeError(522 f"The shape of the 2D attn_mask is {attn_mask.shape}, but should be {correct_2d_size}."523 )524 attn_mask = attn_mask.unsqueeze(0)525 elif attn_mask.dim() == 3:526 correct_3d_size = (bsz * num_heads, tgt_len, src_len)527 if attn_mask.shape != correct_3d_size:528 raise RuntimeError(529 f"The shape of the 3D attn_mask is {attn_mask.shape}, but should be {correct_3d_size}."530 )531 else:532 raise RuntimeError(f"attn_mask's dimension {attn_mask.dim()} is not supported")533 534 # add bias along batch dimension (currently second)535 if bias_k is not None and bias_v is not None:536 assert static_k is None, "bias cannot be added to static key."537 assert static_v is None, "bias cannot be added to static value."538 k = torch.cat([k, bias_k.repeat(1, bsz, 1)])539 v = torch.cat([v, bias_v.repeat(1, bsz, 1)])540 if attn_mask is not None:541 attn_mask = pad(attn_mask, (0, 1))542 if key_padding_mask is not None:543 key_padding_mask = pad(key_padding_mask, (0, 1))544 else:545 assert bias_k is None546 assert bias_v is None547 548 #549 # reshape q, k, v for multihead attention and make em batch first550 #551 q = q.view(tgt_len, bsz * num_heads, head_dim).transpose(0, 1)552 if static_k is None:553 k = k.view(k.shape[0], bsz * num_heads, head_dim).transpose(0, 1)554 else:555 # TODO finish disentangling control flow so we don't do in-projections when statics are passed556 assert (557 static_k.size(0) == bsz * num_heads558 ), f"expecting static_k.size(0) of {bsz * num_heads}, but got {static_k.size(0)}"559 assert (560 static_k.size(2) == head_dim561 ), f"expecting static_k.size(2) of {head_dim}, but got {static_k.size(2)}"562 k = static_k563 if static_v is None:564 v = v.view(v.shape[0], bsz * num_heads, head_dim).transpose(0, 1)565 else:566 # TODO finish disentangling control flow so we don't do in-projections when statics are passed567 assert (568 static_v.size(0) == bsz * num_heads569 ), f"expecting static_v.size(0) of {bsz * num_heads}, but got {static_v.size(0)}"570 assert (571 static_v.size(2) == head_dim572 ), f"expecting static_v.size(2) of {head_dim}, but got {static_v.size(2)}"573 v = static_v574 575 # add zero attention along batch dimension (now first)576 if add_zero_attn:577 zero_attn_shape = (bsz * num_heads, 1, head_dim)578 k = torch.cat([k, torch.zeros(zero_attn_shape, dtype=k.dtype, device=k.device)], dim=1)579 v = torch.cat([v, torch.zeros(zero_attn_shape, dtype=v.dtype, device=v.device)], dim=1)580 if attn_mask is not None:581 attn_mask = pad(attn_mask, (0, 1))582 if key_padding_mask is not None:583 key_padding_mask = pad(key_padding_mask, (0, 1))584 585 # update source sequence length after adjustments586 src_len = k.size(1)587 588 # merge key padding and attention masks589 if key_padding_mask is not None:590 assert key_padding_mask.shape == (591 bsz,592 src_len,593 ), f"expecting key_padding_mask shape of {(bsz, src_len)}, but got {key_padding_mask.shape}"594 key_padding_mask = (595 key_padding_mask.view(bsz, 1, 1, src_len)596 .expand(-1, num_heads, -1, -1)597 .reshape(bsz * num_heads, 1, src_len)598 )599 if attn_mask is None:600 attn_mask = key_padding_mask601 else:602 attn_mask = attn_mask + key_padding_mask603 604 # adjust dropout probability605 if not training:606 dropout_p = 0.0607 608 #609 # (deep breath) calculate attention and out projection610 #611 612 if need_weights:613 B, Nt, E = q.shape614 q_scaled = q / math.sqrt(E)615 616 assert not (is_causal and attn_mask is None), "FIXME: is_causal not implemented for need_weights"617 618 if attn_mask is not None:619 attn_output_weights = torch.baddbmm(attn_mask, q_scaled, k.transpose(-2, -1))620 else:621 attn_output_weights = torch.bmm(q_scaled, k.transpose(-2, -1))622 attn_output_weights = softmax(attn_output_weights, dim=-1)623 if dropout_p > 0.0:624 attn_output_weights = dropout(attn_output_weights, p=dropout_p)625 626 attn_output = torch.bmm(attn_output_weights, v)627 628 attn_output = attn_output.transpose(0, 1).contiguous().view(tgt_len * bsz, embed_dim)629 attn_output = self.out_proj(attn_output)630 attn_output = attn_output.view(tgt_len, bsz, attn_output.size(1))631 632 # optionally average attention weights over heads633 attn_output_weights = attn_output_weights.view(bsz, num_heads, tgt_len, src_len)634 if average_attn_weights:635 attn_output_weights = attn_output_weights.mean(dim=1)636 637 if not is_batched:638 # squeeze the output if input was unbatched639 attn_output = attn_output.squeeze(1)640 attn_output_weights = attn_output_weights.squeeze(0)641 return attn_output, attn_output_weights642 else:643 # attn_mask can be either (L,S) or (N*num_heads, L, S)644 # if attn_mask's shape is (1, L, S) we need to unsqueeze to (1, 1, L, S)645 # in order to match the input for SDPA of (N, num_heads, L, S)646 if attn_mask is not None:647 if attn_mask.size(0) == 1 and attn_mask.dim() == 3:648 attn_mask = attn_mask.unsqueeze(0)649 else:650 attn_mask = attn_mask.view(bsz, num_heads, -1, src_len)651 652 q = q.view(bsz, num_heads, tgt_len, head_dim)653 k = k.view(bsz, num_heads, src_len, head_dim)654 v = v.view(bsz, num_heads, src_len, head_dim)655 656 attn_output = F.scaled_dot_product_attention(q, k, v, attn_mask, dropout_p, is_causal)657 attn_output = attn_output.permute(2, 0, 1, 3).contiguous().view(bsz * tgt_len, embed_dim)658 659 attn_output = self.out_proj(attn_output)660 attn_output = attn_output.view(tgt_len, bsz, attn_output.size(1))661 if not is_batched:662 # squeeze the output if input was unbatched663 attn_output = attn_output.squeeze(1)664 return attn_output, None665 666 667def _mha_shape_check(668 query: Tensor,669 key: Tensor,670 value: Tensor,671 key_padding_mask: Optional[Tensor],672 attn_mask: Optional[Tensor],673 num_heads: int,674):675 # Verifies the expected shape for `query, `key`, `value`, `key_padding_mask` and `attn_mask`676 # and returns if the input is batched or not.677 # Raises an error if `query` is not 2-D (unbatched) or 3-D (batched) tensor.678 679 # Shape check.680 if query.dim() == 3:681 # Batched Inputs682 is_batched = True683 assert key.dim() == 3 and value.dim() == 3, (684 "For batched (3-D) `query`, expected `key` and `value` to be 3-D"685 f" but found {key.dim()}-D and {value.dim()}-D tensors respectively"686 )687 if key_padding_mask is not None:688 assert key_padding_mask.dim() == 2, (689 "For batched (3-D) `query`, expected `key_padding_mask` to be `None` or 2-D"690 f" but found {key_padding_mask.dim()}-D tensor instead"691 )692 if attn_mask is not None:693 assert attn_mask.dim() in (2, 3), (694 "For batched (3-D) `query`, expected `attn_mask` to be `None`, 2-D or 3-D"695 f" but found {attn_mask.dim()}-D tensor instead"696 )697 elif query.dim() == 2:698 # Unbatched Inputs699 is_batched = False700 assert key.dim() == 2 and value.dim() == 2, (701 "For unbatched (2-D) `query`, expected `key` and `value` to be 2-D"702 f" but found {key.dim()}-D and {value.dim()}-D tensors respectively"703 )704 705 if key_padding_mask is not None:706 assert key_padding_mask.dim() == 1, (707 "For unbatched (2-D) `query`, expected `key_padding_mask` to be `None` or 1-D"708 f" but found {key_padding_mask.dim()}-D tensor instead"709 )710 711 if attn_mask is not None:712 assert attn_mask.dim() in (2, 3), (713 "For unbatched (2-D) `query`, expected `attn_mask` to be `None`, 2-D or 3-D"714 f" but found {attn_mask.dim()}-D tensor instead"715 )716 if attn_mask.dim() == 3:717 expected_shape = (num_heads, query.shape[0], key.shape[0])718 assert (719 attn_mask.shape == expected_shape720 ), f"Expected `attn_mask` shape to be {expected_shape} but got {attn_mask.shape}"721 else:722 raise AssertionError(723 f"query should be unbatched 2D or batched 3D tensor but received {query.dim()}-D query tensor"724 )725 726 return is_batched727 728 729def _canonical_mask(730 mask: Optional[Tensor],731 mask_name: str,732 other_type: Optional[DType],733 other_name: str,734 target_type: DType,735 check_other: bool = True,736) -> Optional[Tensor]:737 if mask is not None:738 _mask_dtype = mask.dtype739 _mask_is_float = torch.is_floating_point(mask)740 if _mask_dtype != torch.bool and not _mask_is_float:741 raise AssertionError(f"only bool and floating types of {mask_name} are supported")742 if check_other and other_type is not None:743 if _mask_dtype != other_type:744 warnings.warn(745 f"Support for mismatched {mask_name} and {other_name} "746 "is deprecated. Use same type for both instead."747 )748 if not _mask_is_float:749 mask = torch.zeros_like(mask, dtype=target_type).masked_fill_(mask, float("-inf"))750 return mask751 752 753def _none_or_dtype(input: Optional[Tensor]) -> Optional[DType]:754 if input is None:755 return None756 elif isinstance(input, torch.Tensor):757 return input.dtype758 raise RuntimeError("input to _none_or_dtype() must be None or torch.Tensor")759 760 761def _in_projection_packed(762 q: Tensor,763 k: Tensor,764 v: Tensor,765 w: Tensor,766 b: Optional[Tensor] = None,767) -> List[Tensor]:768 r"""769 Performs the in-projection step of the attention operation, using packed weights.770 Output is a triple containing projection tensors for query, key and value.771 Args:772 q, k, v: query, key and value tensors to be projected. For self-attention,773 these are typically the same tensor; for encoder-decoder attention,774 k and v are typically the same tensor. (We take advantage of these775 identities for performance if they are present.) Regardless, q, k and v776 must share a common embedding dimension; otherwise their shapes may vary.777 w: projection weights for q, k and v, packed into a single tensor. Weights778 are packed along dimension 0, in q, k, v order.779 b: optional projection biases for q, k and v, packed into a single tensor780 in q, k, v order.781 Shape:782 Inputs:783 - q: :math:`(..., E)` where E is the embedding dimension784 - k: :math:`(..., E)` where E is the embedding dimension785 - v: :math:`(..., E)` where E is the embedding dimension786 - w: :math:`(E * 3, E)` where E is the embedding dimension787 - b: :math:`E * 3` where E is the embedding dimension788 Output:789 - in output list :math:`[q', k', v']`, each output tensor will have the790 same shape as the corresponding input tensor.791 """792 E = q.size(-1)793 if k is v:794 if q is k:795 # self-attention796 proj = linear(q, w, b)797 # reshape to 3, E and not E, 3 is deliberate for better memory coalescing and keeping same order as chunk()798 proj = proj.unflatten(-1, (3, E)).unsqueeze(0).transpose(0, -2).squeeze(-2).contiguous()799 return proj[0], proj[1], proj[2]800 else:801 # encoder-decoder attention802 w_q, w_kv = w.split([E, E * 2])803 if b is None:804 b_q = b_kv = None805 else:806 b_q, b_kv = b.split([E, E * 2])807 q_proj = linear(q, w_q, b_q)808 kv_proj = linear(k, w_kv, b_kv)809 # reshape to 2, E and not E, 2 is deliberate for better memory coalescing and keeping same order as chunk()810 kv_proj = kv_proj.unflatten(-1, (2, E)).unsqueeze(0).transpose(0, -2).squeeze(-2).contiguous()811 return (q_proj, kv_proj[0], kv_proj[1])812 else:813 w_q, w_k, w_v = w.chunk(3)814 if b is None:815 b_q = b_k = b_v = None816 else:817 b_q, b_k, b_v = b.chunk(3)818 return linear(q, w_q, b_q), linear(k, w_k, b_k), linear(v, w_v, b_v)819 820 821def _in_projection(822 q: Tensor,823 k: Tensor,824 v: Tensor,825 w_q: Tensor,826 w_k: Tensor,827 w_v: Tensor,828 b_q: Optional[Tensor] = None,829 b_k: Optional[Tensor] = None,830 b_v: Optional[Tensor] = None,831) -> Tuple[Tensor, Tensor, Tensor]:832 r"""833 Performs the in-projection step of the attention operation. This is simply834 a triple of linear projections, with shape constraints on the weights which835 ensure embedding dimension uniformity in the projected outputs.836 Output is a triple containing projection tensors for query, key and value.837 Args:838 q, k, v: query, key and value tensors to be projected.839 w_q, w_k, w_v: weights for q, k and v, respectively.840 b_q, b_k, b_v: optional biases for q, k and v, respectively.841 Shape:842 Inputs:843 - q: :math:`(Qdims..., Eq)` where Eq is the query embedding dimension and Qdims are any844 number of leading dimensions.845 - k: :math:`(Kdims..., Ek)` where Ek is the key embedding dimension and Kdims are any846 number of leading dimensions.847 - v: :math:`(Vdims..., Ev)` where Ev is the value embedding dimension and Vdims are any848 number of leading dimensions.849 - w_q: :math:`(Eq, Eq)`850 - w_k: :math:`(Eq, Ek)`851 - w_v: :math:`(Eq, Ev)`852 - b_q: :math:`(Eq)`853 - b_k: :math:`(Eq)`854 - b_v: :math:`(Eq)`855 Output: in output triple :math:`(q', k', v')`,856 - q': :math:`[Qdims..., Eq]`857 - k': :math:`[Kdims..., Eq]`858 - v': :math:`[Vdims..., Eq]`859 """860 Eq, Ek, Ev = q.size(-1), k.size(-1), v.size(-1)861 assert w_q.shape == (Eq, Eq), f"expecting query weights shape of {(Eq, Eq)}, but got {w_q.shape}"862 assert w_k.shape == (Eq, Ek), f"expecting key weights shape of {(Eq, Ek)}, but got {w_k.shape}"863 assert w_v.shape == (Eq, Ev), f"expecting value weights shape of {(Eq, Ev)}, but got {w_v.shape}"864 assert b_q is None or b_q.shape == (Eq,), f"expecting query bias shape of {(Eq,)}, but got {b_q.shape}"865 assert b_k is None or b_k.shape == (Eq,), f"expecting key bias shape of {(Eq,)}, but got {b_k.shape}"866 assert b_v is None or b_v.shape == (Eq,), f"expecting value bias shape of {(Eq,)}, but got {b_v.shape}"867 return linear(q, w_q, b_q), linear(k, w_k, b_k), linear(v, w_v, b_v)868