multimodalart/EchoMimic-zero

1# Adapted from https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/resnet.py2 3import torch4import torch.nn as nn5import torch.nn.functional as F6from einops import rearrange7 8 9class InflatedConv3d(nn.Conv2d):10    def forward(self, x):11        video_length = x.shape[2]12 13        x = rearrange(x, "b c f h w -> (b f) c h w")14        x = super().forward(x)15        x = rearrange(x, "(b f) c h w -> b c f h w", f=video_length)16 17        return x18 19 20class InflatedGroupNorm(nn.GroupNorm):21    def forward(self, x):22        video_length = x.shape[2]23 24        x = rearrange(x, "b c f h w -> (b f) c h w")25        x = super().forward(x)26        x = rearrange(x, "(b f) c h w -> b c f h w", f=video_length)27 28        return x29 30 31class Upsample3D(nn.Module):32    def __init__(33        self,34        channels,35        use_conv=False,36        use_conv_transpose=False,37        out_channels=None,38        name="conv",39    ):40        super().__init__()41        self.channels = channels42        self.out_channels = out_channels or channels43        self.use_conv = use_conv44        self.use_conv_transpose = use_conv_transpose45        self.name = name46 47        conv = None48        if use_conv_transpose:49            raise NotImplementedError50        elif use_conv:51            self.conv = InflatedConv3d(self.channels, self.out_channels, 3, padding=1)52 53    def forward(self, hidden_states, output_size=None):54        assert hidden_states.shape[1] == self.channels55 56        if self.use_conv_transpose:57            raise NotImplementedError58 59        # Cast to float32 to as 'upsample_nearest2d_out_frame' op does not support bfloat1660        dtype = hidden_states.dtype61        if dtype == torch.bfloat16:62            hidden_states = hidden_states.to(torch.float32)63 64        # upsample_nearest_nhwc fails with large batch sizes. see https://github.com/huggingface/diffusers/issues/98465        if hidden_states.shape[0] >= 64:66            hidden_states = hidden_states.contiguous()67 68        # if `output_size` is passed we force the interpolation output69        # size and do not make use of `scale_factor=2`70        if output_size is None:71            hidden_states = F.interpolate(72                hidden_states, scale_factor=[1.0, 2.0, 2.0], mode="nearest"73            )74        else:75            hidden_states = F.interpolate(76                hidden_states, size=output_size, mode="nearest"77            )78 79        # If the input is bfloat16, we cast back to bfloat1680        if dtype == torch.bfloat16:81            hidden_states = hidden_states.to(dtype)82 83        # if self.use_conv:84        #     if self.name == "conv":85        #         hidden_states = self.conv(hidden_states)86        #     else:87        #         hidden_states = self.Conv2d_0(hidden_states)88        hidden_states = self.conv(hidden_states)89 90        return hidden_states91 92 93class Downsample3D(nn.Module):94    def __init__(95        self, channels, use_conv=False, out_channels=None, padding=1, name="conv"96    ):97        super().__init__()98        self.channels = channels99        self.out_channels = out_channels or channels100        self.use_conv = use_conv101        self.padding = padding102        stride = 2103        self.name = name104 105        if use_conv:106            self.conv = InflatedConv3d(107                self.channels, self.out_channels, 3, stride=stride, padding=padding108            )109        else:110            raise NotImplementedError111 112    def forward(self, hidden_states):113        assert hidden_states.shape[1] == self.channels114        if self.use_conv and self.padding == 0:115            raise NotImplementedError116 117        assert hidden_states.shape[1] == self.channels118        hidden_states = self.conv(hidden_states)119 120        return hidden_states121 122 123class ResnetBlock3D(nn.Module):124    def __init__(125        self,126        *,127        in_channels,128        out_channels=None,129        conv_shortcut=False,130        dropout=0.0,131        temb_channels=512,132        groups=32,133        groups_out=None,134        pre_norm=True,135        eps=1e-6,136        non_linearity="swish",137        time_embedding_norm="default",138        output_scale_factor=1.0,139        use_in_shortcut=None,140        use_inflated_groupnorm=None,141    ):142        super().__init__()143        self.pre_norm = pre_norm144        self.pre_norm = True145        self.in_channels = in_channels146        out_channels = in_channels if out_channels is None else out_channels147        self.out_channels = out_channels148        self.use_conv_shortcut = conv_shortcut149        self.time_embedding_norm = time_embedding_norm150        self.output_scale_factor = output_scale_factor151 152        if groups_out is None:153            groups_out = groups154 155        assert use_inflated_groupnorm != None156        if use_inflated_groupnorm:157            self.norm1 = InflatedGroupNorm(158                num_groups=groups, num_channels=in_channels, eps=eps, affine=True159            )160        else:161            self.norm1 = torch.nn.GroupNorm(162                num_groups=groups, num_channels=in_channels, eps=eps, affine=True163            )164 165        self.conv1 = InflatedConv3d(166            in_channels, out_channels, kernel_size=3, stride=1, padding=1167        )168 169        if temb_channels is not None:170            if self.time_embedding_norm == "default":171                time_emb_proj_out_channels = out_channels172            elif self.time_embedding_norm == "scale_shift":173                time_emb_proj_out_channels = out_channels * 2174            else:175                raise ValueError(176                    f"unknown time_embedding_norm : {self.time_embedding_norm} "177                )178 179            self.time_emb_proj = torch.nn.Linear(180                temb_channels, time_emb_proj_out_channels181            )182        else:183            self.time_emb_proj = None184 185        if use_inflated_groupnorm:186            self.norm2 = InflatedGroupNorm(187                num_groups=groups_out, num_channels=out_channels, eps=eps, affine=True188            )189        else:190            self.norm2 = torch.nn.GroupNorm(191                num_groups=groups_out, num_channels=out_channels, eps=eps, affine=True192            )193        self.dropout = torch.nn.Dropout(dropout)194        self.conv2 = InflatedConv3d(195            out_channels, out_channels, kernel_size=3, stride=1, padding=1196        )197 198        if non_linearity == "swish":199            self.nonlinearity = lambda x: F.silu(x)200        elif non_linearity == "mish":201            self.nonlinearity = Mish()202        elif non_linearity == "silu":203            self.nonlinearity = nn.SiLU()204 205        self.use_in_shortcut = (206            self.in_channels != self.out_channels207            if use_in_shortcut is None208            else use_in_shortcut209        )210 211        self.conv_shortcut = None212        if self.use_in_shortcut:213            self.conv_shortcut = InflatedConv3d(214                in_channels, out_channels, kernel_size=1, stride=1, padding=0215            )216 217    def forward(self, input_tensor, temb):218        hidden_states = input_tensor219 220        hidden_states = self.norm1(hidden_states)221        hidden_states = self.nonlinearity(hidden_states)222 223        hidden_states = self.conv1(hidden_states)224 225        if temb is not None:226            temb = self.time_emb_proj(self.nonlinearity(temb))[:, :, None, None, None]227 228        if temb is not None and self.time_embedding_norm == "default":229            hidden_states = hidden_states + temb230 231        hidden_states = self.norm2(hidden_states)232 233        if temb is not None and self.time_embedding_norm == "scale_shift":234            scale, shift = torch.chunk(temb, 2, dim=1)235            hidden_states = hidden_states * (1 + scale) + shift236 237        hidden_states = self.nonlinearity(hidden_states)238 239        hidden_states = self.dropout(hidden_states)240        hidden_states = self.conv2(hidden_states)241 242        if self.conv_shortcut is not None:243            input_tensor = self.conv_shortcut(input_tensor)244 245        output_tensor = (input_tensor + hidden_states) / self.output_scale_factor246 247        return output_tensor248 249 250class Mish(torch.nn.Module):251    def forward(self, hidden_states):252        return hidden_states * torch.tanh(torch.nn.functional.softplus(hidden_states))253