declare-lab/tango2

1# Copyright 2023 The HuggingFace Team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14from dataclasses import dataclass15from typing import Optional16 17import torch18from torch import nn19 20from ..configuration_utils import ConfigMixin, register_to_config21from ..utils import BaseOutput22from .attention import BasicTransformerBlock23from .modeling_utils import ModelMixin24 25 26@dataclass27class TransformerTemporalModelOutput(BaseOutput):28    """29    Args:30        sample (`torch.FloatTensor` of shape `(batch_size x num_frames, num_channels, height, width)`)31            Hidden states conditioned on `encoder_hidden_states` input.32    """33 34    sample: torch.FloatTensor35 36 37class TransformerTemporalModel(ModelMixin, ConfigMixin):38    """39    Transformer model for video-like data.40 41    Parameters:42        num_attention_heads (`int`, *optional*, defaults to 16): The number of heads to use for multi-head attention.43        attention_head_dim (`int`, *optional*, defaults to 88): The number of channels in each head.44        in_channels (`int`, *optional*):45            Pass if the input is continuous. The number of channels in the input and output.46        num_layers (`int`, *optional*, defaults to 1): The number of layers of Transformer blocks to use.47        dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.48        cross_attention_dim (`int`, *optional*): The number of encoder_hidden_states dimensions to use.49        sample_size (`int`, *optional*): Pass if the input is discrete. The width of the latent images.50            Note that this is fixed at training time as it is used for learning a number of position embeddings. See51            `ImagePositionalEmbeddings`.52        activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward.53        attention_bias (`bool`, *optional*):54            Configure if the TransformerBlocks' attention should contain a bias parameter.55        double_self_attention (`bool`, *optional*):56            Configure if each TransformerBlock should contain two self-attention layers57    """58 59    @register_to_config60    def __init__(61        self,62        num_attention_heads: int = 16,63        attention_head_dim: int = 88,64        in_channels: Optional[int] = None,65        out_channels: Optional[int] = None,66        num_layers: int = 1,67        dropout: float = 0.0,68        norm_num_groups: int = 32,69        cross_attention_dim: Optional[int] = None,70        attention_bias: bool = False,71        sample_size: Optional[int] = None,72        activation_fn: str = "geglu",73        norm_elementwise_affine: bool = True,74        double_self_attention: bool = True,75    ):76        super().__init__()77        self.num_attention_heads = num_attention_heads78        self.attention_head_dim = attention_head_dim79        inner_dim = num_attention_heads * attention_head_dim80 81        self.in_channels = in_channels82 83        self.norm = torch.nn.GroupNorm(num_groups=norm_num_groups, num_channels=in_channels, eps=1e-6, affine=True)84        self.proj_in = nn.Linear(in_channels, inner_dim)85 86        # 3. Define transformers blocks87        self.transformer_blocks = nn.ModuleList(88            [89                BasicTransformerBlock(90                    inner_dim,91                    num_attention_heads,92                    attention_head_dim,93                    dropout=dropout,94                    cross_attention_dim=cross_attention_dim,95                    activation_fn=activation_fn,96                    attention_bias=attention_bias,97                    double_self_attention=double_self_attention,98                    norm_elementwise_affine=norm_elementwise_affine,99                )100                for d in range(num_layers)101            ]102        )103 104        self.proj_out = nn.Linear(inner_dim, in_channels)105 106    def forward(107        self,108        hidden_states,109        encoder_hidden_states=None,110        timestep=None,111        class_labels=None,112        num_frames=1,113        cross_attention_kwargs=None,114        return_dict: bool = True,115    ):116        """117        Args:118            hidden_states ( When discrete, `torch.LongTensor` of shape `(batch size, num latent pixels)`.119                When continous, `torch.FloatTensor` of shape `(batch size, channel, height, width)`): Input120                hidden_states121            encoder_hidden_states ( `torch.LongTensor` of shape `(batch size, encoder_hidden_states dim)`, *optional*):122                Conditional embeddings for cross attention layer. If not given, cross-attention defaults to123                self-attention.124            timestep ( `torch.long`, *optional*):125                Optional timestep to be applied as an embedding in AdaLayerNorm's. Used to indicate denoising step.126            class_labels ( `torch.LongTensor` of shape `(batch size, num classes)`, *optional*):127                Optional class labels to be applied as an embedding in AdaLayerZeroNorm. Used to indicate class labels128                conditioning.129            return_dict (`bool`, *optional*, defaults to `True`):130                Whether or not to return a [`models.unet_2d_condition.UNet2DConditionOutput`] instead of a plain tuple.131 132        Returns:133            [`~models.transformer_2d.TransformerTemporalModelOutput`] or `tuple`:134            [`~models.transformer_2d.TransformerTemporalModelOutput`] if `return_dict` is True, otherwise a `tuple`.135            When returning a tuple, the first element is the sample tensor.136        """137        # 1. Input138        batch_frames, channel, height, width = hidden_states.shape139        batch_size = batch_frames // num_frames140 141        residual = hidden_states142 143        hidden_states = hidden_states[None, :].reshape(batch_size, num_frames, channel, height, width)144        hidden_states = hidden_states.permute(0, 2, 1, 3, 4)145 146        hidden_states = self.norm(hidden_states)147        hidden_states = hidden_states.permute(0, 3, 4, 2, 1).reshape(batch_size * height * width, num_frames, channel)148 149        hidden_states = self.proj_in(hidden_states)150 151        # 2. Blocks152        for block in self.transformer_blocks:153            hidden_states = block(154                hidden_states,155                encoder_hidden_states=encoder_hidden_states,156                timestep=timestep,157                cross_attention_kwargs=cross_attention_kwargs,158                class_labels=class_labels,159            )160 161        # 3. Output162        hidden_states = self.proj_out(hidden_states)163        hidden_states = (164            hidden_states[None, None, :]165            .reshape(batch_size, height, width, channel, num_frames)166            .permute(0, 3, 4, 1, 2)167            .contiguous()168        )169        hidden_states = hidden_states.reshape(batch_frames, channel, height, width)170 171        output = hidden_states + residual172 173        if not return_dict:174            return (output,)175 176        return TransformerTemporalModelOutput(sample=output)177