WishArdently/InternVideo2Stage2-VisionEncoder

1import numpy as np2import torch3import logging4 5logger = logging.getLogger(__name__)6 7# --------------------------------------------------------8# 3D sine-cosine position embedding9# References:10# MVD: https://github.com/ruiwang2021/mvd/blob/main/modeling_finetune.py11# --------------------------------------------------------12def get_3d_sincos_pos_embed(embed_dim, grid_size, t_size, cls_token=False):13    """14    grid_size: int of the grid height and width15    t_size: int of the temporal size16    return:17    pos_embed: [t_size*grid_size*grid_size, embed_dim] or [1+t_size*grid_size*grid_size, embed_dim] (w/ or w/o cls_token)18    """19    assert embed_dim % 4 == 020    embed_dim_spatial = embed_dim // 4 * 321    embed_dim_temporal = embed_dim // 422 23    # spatial24    grid_h = np.arange(grid_size, dtype=np.float32)25    grid_w = np.arange(grid_size, dtype=np.float32)26    grid = np.meshgrid(grid_w, grid_h)  # here w goes first27    grid = np.stack(grid, axis=0)28 29    grid = grid.reshape([2, 1, grid_size, grid_size])30    pos_embed_spatial = get_2d_sincos_pos_embed_from_grid(31        embed_dim_spatial, grid32    )33 34    # temporal35    grid_t = np.arange(t_size, dtype=np.float32)36    pos_embed_temporal = get_1d_sincos_pos_embed_from_grid(37        embed_dim_temporal, grid_t38    )39 40    # concate: [T, H, W] order41    pos_embed_temporal = pos_embed_temporal[:, np.newaxis, :]42    pos_embed_temporal = np.repeat(43        pos_embed_temporal, grid_size**2, axis=144    )  # [T, H*W, D // 4]45    pos_embed_spatial = pos_embed_spatial[np.newaxis, :, :]46    pos_embed_spatial = np.repeat(47        pos_embed_spatial, t_size, axis=048    )  # [T, H*W, D // 4 * 3]49 50    pos_embed = np.concatenate([pos_embed_temporal, pos_embed_spatial], axis=-1)51    pos_embed = pos_embed.reshape([-1, embed_dim])  # [T*H*W, D]52 53    if cls_token:54        pos_embed = np.concatenate(55            [np.zeros([1, embed_dim]), pos_embed], axis=056        )57    return pos_embed58 59 60# --------------------------------------------------------61# 2D sine-cosine position embedding62# References:63# Transformer: https://github.com/tensorflow/models/blob/master/official/nlp/transformer/model_utils.py64# MoCo v3: https://github.com/facebookresearch/moco-v365# --------------------------------------------------------66def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False):67    """68    grid_size: int of the grid height and width69    return:70    pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)71    """72    grid_h = np.arange(grid_size, dtype=np.float32)73    grid_w = np.arange(grid_size, dtype=np.float32)74    grid = np.meshgrid(grid_w, grid_h)  # here w goes first75    grid = np.stack(grid, axis=0)76 77    grid = grid.reshape([2, 1, grid_size, grid_size])78    pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)79    if cls_token:80        pos_embed = np.concatenate(81            [np.zeros([1, embed_dim]), pos_embed], axis=082        )83    return pos_embed84 85 86def get_1d_sincos_pos_embed(embed_dim, t_size, cls_token=False):87    """88    t_size: int of the temporal size89    return:90    pos_embed: [t_size, embed_dim] or [1+t_size, embed_dim] (w/ or w/o cls_token)91    """92    grid_t = np.arange(t_size, dtype=np.float32)93    pos_embed = get_1d_sincos_pos_embed_from_grid(embed_dim, grid_t)94    if cls_token:95        pos_embed = np.concatenate(96            [np.zeros([1, embed_dim]), pos_embed], axis=097        )98    return pos_embed99 100 101def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):102    assert embed_dim % 2 == 0103 104    # use half of dimensions to encode grid_h105    emb_h = get_1d_sincos_pos_embed_from_grid(106        embed_dim // 2, grid[0]107    )  # (H*W, D/2)108    emb_w = get_1d_sincos_pos_embed_from_grid(109        embed_dim // 2, grid[1]110    )  # (H*W, D/2)111 112    emb = np.concatenate([emb_h, emb_w], axis=1)  # (H*W, D)113    return emb114 115 116def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):117    """118    embed_dim: output dimension for each position119    pos: a list of positions to be encoded: size (M,)120    out: (M, D)121    """122    assert embed_dim % 2 == 0123    omega = np.arange(embed_dim // 2, dtype=np.float32)124    omega /= embed_dim / 2.0125    omega = 1.0 / 10000**omega  # (D/2,)126 127    pos = pos.reshape(-1)  # (M,)128    out = np.einsum("m,d->md", pos, omega)  # (M, D/2), outer product129 130    emb_sin = np.sin(out)  # (M, D/2)131    emb_cos = np.cos(out)  # (M, D/2)132 133    emb = np.concatenate([emb_sin, emb_cos], axis=1)  # (M, D)134    return emb135 136 137def interpolate_pos_embed(checkpoint_model, model, orig_t_size=4, pos_name='vision_encoder.pos_embed'):138    if pos_name in checkpoint_model:139        pos_embed_checkpoint = checkpoint_model[pos_name]140        embedding_size = pos_embed_checkpoint.shape[-1] # channel dim141        num_patches = model.patch_embed.num_patches # 142        num_extra_tokens = model.pos_embed.shape[-2] - num_patches # 0/1143 144        # we use 4 frames for pretraining145        new_t_size = model.T146        # height (== width) for the checkpoint position embedding147        orig_size = int(((pos_embed_checkpoint.shape[-2] - num_extra_tokens)//(orig_t_size)) ** 0.5)148        # height (== width) for the new position embedding149        new_size = int((num_patches // (new_t_size))** 0.5)150        151        # class_token and dist_token are kept unchanged152        if orig_t_size != new_t_size:153            logger.info(f"Temporal interpolate from {orig_t_size} to {new_t_size} ({pos_name})")154            extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens]155            # only the position tokens are interpolated156            pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:]157            # B, L, C -> B， T, HW, C -> BHW, C, T  (B = 1)158            pos_tokens = pos_tokens.view(1, orig_t_size, -1, embedding_size)159            pos_tokens = pos_tokens.permute(0, 2, 3, 1).reshape(-1, embedding_size, orig_t_size)160            pos_tokens = torch.nn.functional.interpolate(pos_tokens, size=new_t_size, mode='linear')161            pos_tokens = pos_tokens.view(1, -1, embedding_size, new_t_size)162            pos_tokens = pos_tokens.permute(0, 3, 1, 2).reshape(1, -1, embedding_size)163            new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1)164            checkpoint_model[pos_name] = new_pos_embed165            pos_embed_checkpoint = new_pos_embed166 167        # class_token and dist_token are kept unchanged168        if orig_size != new_size:169            logger.info(f"Position interpolate from {orig_size}x{orig_size} to {new_size}x{new_size} ({pos_name})")170            extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens]171            # only the position tokens are interpolated172            pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:]173            # B, L, C -> BT, H, W, C -> BT, C, H, W174            pos_tokens = pos_tokens.reshape(-1, new_t_size, orig_size, orig_size, embedding_size)175            pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size, embedding_size).permute(0, 3, 1, 2)176            pos_tokens = torch.nn.functional.interpolate(177                pos_tokens, size=(new_size, new_size), mode='bicubic', align_corners=False)178            # BT, C, H, W -> BT, H, W, C ->  B, T, H, W, C179            pos_tokens = pos_tokens.permute(0, 2, 3, 1).reshape(-1, new_t_size, new_size, new_size, embedding_size) 180            pos_tokens = pos_tokens.flatten(1, 3) # B, L, C181            new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1)182            checkpoint_model[pos_name] = new_pos_embed183 184 185def interpolate_pos_embed_internvideo2(checkpoint_model, model, orig_t_size = 8):186    # interpolate position embedding187    for pos_name in ['pos_embed', 'clip_pos_embed']:188        if pos_name in checkpoint_model:189            pos_embed_checkpoint = checkpoint_model[pos_name]190            embedding_size = pos_embed_checkpoint.shape[-1] # channel dim191            num_patches = model.patch_embed.num_patches # 192            num_extra_tokens = model.pos_embed.shape[-2] - num_patches # 0/1193 194            # we use 8 frames for pretraining195            # new_t_size = args.num_frames * args.num_segments // model.patch_embed.tubelet_size196            new_t_size = model.num_frames // model.tubelet_size197            # height (== width) for the checkpoint position embedding198            orig_size = int(((pos_embed_checkpoint.shape[-2] - num_extra_tokens)//(orig_t_size)) ** 0.5)199            # height (== width) for the new position embedding200            new_size = int((num_patches // (new_t_size))** 0.5)201            202            # class_token and dist_token are kept unchanged203            if orig_t_size != new_t_size:204                logger.info(f"Temporal interpolate from {orig_t_size} to {new_t_size} ({pos_name})")205                extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens]206                # only the position tokens are interpolated207                pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:]208                # B, L, C -> B， T, HW, C -> BHW, C, T  (B = 1)209                pos_tokens = pos_tokens.view(1, orig_t_size, -1, embedding_size)210                pos_tokens = pos_tokens.permute(0, 2, 3, 1).reshape(-1, embedding_size, orig_t_size)211                pos_tokens = torch.nn.functional.interpolate(pos_tokens, size=new_t_size, mode='linear')212                pos_tokens = pos_tokens.view(1, -1, embedding_size, new_t_size)213                pos_tokens = pos_tokens.permute(0, 3, 1, 2).reshape(1, -1, embedding_size)214                new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1)215                checkpoint_model[pos_name] = new_pos_embed216                pos_embed_checkpoint = new_pos_embed217 218            # class_token and dist_token are kept unchanged219            if orig_size != new_size:220                logger.info(f"Position interpolate from {orig_size}x{orig_size} to {new_size}x{new_size} ({pos_name})")221                extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens]222                # only the position tokens are interpolated223                pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:]224                # B, L, C -> BT, H, W, C -> BT, C, H, W225                pos_tokens = pos_tokens.reshape(-1, new_t_size, orig_size, orig_size, embedding_size)226                pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size, embedding_size).permute(0, 3, 1, 2)227                pos_tokens = torch.nn.functional.interpolate(228                    pos_tokens, size=(new_size, new_size), mode='bicubic', align_corners=False)229                # BT, C, H, W -> BT, H, W, C ->  B, T, H, W, C230                pos_tokens = pos_tokens.permute(0, 2, 3, 1).reshape(-1, new_t_size, new_size, new_size, embedding_size) 231                pos_tokens = pos_tokens.flatten(1, 3) # B, L, C232                new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1)233                checkpoint_model[pos_name] = new_pos_embed234    235    if 'pos_embed_spatial' in checkpoint_model or 'pos_embed_temporal' in checkpoint_model:236        raise NotImplementedError237 238 239def interpolate_pos_embed_internvideo2_new(checkpoint_model, model, orig_t_size = 8):240    pos_names = []241    for k in checkpoint_model.keys():242        if ('pos_embed' in k or 'clip_pos_embed' in k) and 'img_pos_embed' not in k:243            pos_names.append(k)244    245    logger.info(f"pos names list for interpolating: {pos_names}")246 247    assert len(pos_names) > 0, checkpoint_model.keys()248 249    if 'pos_embed_spatial' in checkpoint_model.keys() or 'pos_embed_temporal' in checkpoint_model.keys():250        raise NotImplementedError251    252    # interpolate position embedding253    for pos_name in pos_names:254 255        pos_embed_checkpoint = checkpoint_model[pos_name]256        embedding_size = pos_embed_checkpoint.shape[-1] # channel dim257        num_patches = model.patch_embed.num_patches # 258        num_extra_tokens = model.pos_embed.shape[-2] - num_patches # 0/1259 260        # we use 8 frames for pretraining261        # new_t_size = args.num_frames * args.num_segments // model.patch_embed.tubelet_size262        new_t_size = model.num_frames // model.tubelet_size263        # height (== width) for the checkpoint position embedding264        orig_size = int(((pos_embed_checkpoint.shape[-2] - num_extra_tokens)//(orig_t_size)) ** 0.5)265        # height (== width) for the new position embedding266        new_size = int((num_patches // (new_t_size))** 0.5)267        268        # class_token and dist_token are kept unchanged269        if orig_t_size != new_t_size:270            logger.info(f"Temporal interpolate from {orig_t_size} to {new_t_size} ({pos_name})")271            extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens]272            # only the position tokens are interpolated273            pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:]274            # B, L, C -> B， T, HW, C -> BHW, C, T  (B = 1)275            pos_tokens = pos_tokens.view(1, orig_t_size, -1, embedding_size)276            pos_tokens = pos_tokens.permute(0, 2, 3, 1).reshape(-1, embedding_size, orig_t_size)277            pos_tokens = torch.nn.functional.interpolate(pos_tokens, size=new_t_size, mode='linear')278            pos_tokens = pos_tokens.view(1, -1, embedding_size, new_t_size)279            pos_tokens = pos_tokens.permute(0, 3, 1, 2).reshape(1, -1, embedding_size)280            new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1)281            checkpoint_model[pos_name] = new_pos_embed282            pos_embed_checkpoint = new_pos_embed283 284        # class_token and dist_token are kept unchanged285        if orig_size != new_size:286            logger.info(f"Position interpolate from {orig_size}x{orig_size} to {new_size}x{new_size} ({pos_name})")287            extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens]288            # only the position tokens are interpolated289            pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:]290            # B, L, C -> BT, H, W, C -> BT, C, H, W291            pos_tokens = pos_tokens.reshape(-1, new_t_size, orig_size, orig_size, embedding_size)292            pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size, embedding_size).permute(0, 3, 1, 2)293            pos_tokens = torch.nn.functional.interpolate(294                pos_tokens, size=(new_size, new_size), mode='bicubic', align_corners=False)295            # BT, C, H, W -> BT, H, W, C ->  B, T, H, W, C296            pos_tokens = pos_tokens.permute(0, 2, 3, 1).reshape(-1, new_t_size, new_size, new_size, embedding_size) 297            pos_tokens = pos_tokens.flatten(1, 3) # B, L, C298            new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1)299            checkpoint_model[pos_name] = new_pos_embed