declare-lab/tango2
92
1a
2 ��'d!a � @ s� d dl Z d dlZd dlm mZ d dlmZ ddlmZmZm Z m3Z4 G dd� dej �Z G dd� dej �Z
G d d5� d6ej �ZG d d � d ej �ZG d
d� dej �ZG dd� dej �Zg d�g d�g d�d�ZG dd� dej �ZG dd� dej �ZG dd� dej �ZG dd � d ej �ZG d d � d ej �ZG d d � d ej �ZG d!d"� d"ej �ZG d#d$� d$ej �ZG d%d&� d&ej �ZG d'd(� d(ej �Z G d)d*� d*ej �Z d+d,� Z d-d.� Z d/d0� Z d1d2� Z!dS )3� N)�nn� )� Downsample1D�ResidualTemporalBlock1D�7Upsample1D�rearrange_dimsc 8 s( e Zd Zd � fdd � Zd
d9d �Z� ZS )�DownResnetBlock1DNr F� �default� �?Tc s� t � �� || _|d u r |n|}|| _|| _| | _| | _|10| _|d u rJ|}t|||d�g} t |�D ]}
| �11t|||d�� qbt � | �| _
|dkr�dd� | _n.|dkr�t �� | _n|dkr�t �� | _nd | _d | _| r�t|ddd �| _d S )12N�� embed_dim�swishc S s13 t �| �S �N��F�silu��x� r �T/home/deep/Projects/audio_diffusion/diffusers/src/diffusers/models/unet_1d_blocks.py�<lambda>; � z,DownResnetBlock1D.__init__.<locals>.<lambda>�mishr Tr )�use_conv�padding)�super�__init__� in_channels� out_channels�use_conv_shortcut�time_embedding_norm�add_downsample�output_scale_factorr �range�appendr �14ModuleList�resnets� nonlinearity�Mish�SiLU�15downsampler )�selfr r �16num_layers�
conv_shortcut�
temb_channels�groups�17groups_out�
non_linearityr! r# r" r' �_�� __class__r r r s0 18 zDownResnetBlock1D.__init__c C sl d}| j d ||�}| j dd � D ]}|||�}q"||f7 }| jd urP| �|�}| jd urd| �|�}||fS )Nr r r )r' r( r+ )r, �
hidden_states�temb�
output_states�resnetr r r �forwardG s 1920212223zDownResnetBlock1D.forward)24Nr Fr r NNr25 r T)N��__name__�26__module__� __qualname__r r: �
__classcell__r r r4 r r s �/r c s( e Zd Zd � fdd� Zd d d27�Z� ZS )
�UpResnetBlock1DNr r r28 r Tc
s� t � �� || _|d u r |n|}|| _|| _|29| _| | _|d u rD|}td| ||d�g} t|�D ]} | � t|||d�� q`t30� | �| _ |dkr�dd� | _
n.|dkr�t31�� | _
n|dkr�t32�� | _
nd | _
d | _|33r�t|dd �| _d S )34N� r r c S s35 t �| �S r r r r r r r { r z*UpResnetBlock1D.__init__.<locals>.<lambda>r r T)�use_conv_transpose)r r r r r! � add_upsampler# r r$ r% r r& r' r( r) r* �upsampler )
r, r r r- r/ r0 r1 r2 r! r# rC r' r3 r4 r r r Z s.
36 zUpResnetBlock1D.__init__c C s| |d ur"|d }t j||fdd�}| jd ||�}| jdd � D ]}|||�}q@| jd urd| �|�}| jd urx| �|�}|S )N�����r ��dimr )�torch�catr' r( rD �r, r6 �res_hidden_states_tupler7 �res_hidden_statesr9 r r r r: � s 37383940zUpResnetBlock1D.forward) Nr r r NNr41 r T)NNr; r r r4 r r@ Y s �-r@ c s& e Zd Z� fdd�Zddd�Z� ZS )�ValueFunctionMidBlock1Dc sp t � �� || _|| _|| _t||d |d�| _t|d dd�| _t|d |d |d�| _ t|d dd�| _42d S )NrA r T�r � ) r r r r r
r �res1r �down1�res2�down2)r, r r r
r4 r r r � s 43z ValueFunctionMidBlock1D.__init__Nc C s0 | � ||�}| �|�}| �||�}| �|�}|S r )rP rQ rR rS )r, r r7 r r r r: � s44 45 46z ValueFunctionMidBlock1D.forward)Nr; r r r4 r rM � s rM c s0 e Zd Zd eeed�� fdd�
Zdd� Z� ZS )47�MidResTemporalBlock1Dr FN)r- r" rC c 48 s� t � �� || _|| _|| _t|||d�g}t|�D ]} |�t|||d�� q4t� |�| _49|dkrndd� | _ n.|dkr�t� � | _ n|dkr�t�
� | _ nd | _ d | _|r�t|dd�| _d | _|r�t|dd�| _| jr�| jr�td ��d S )50Nr r c S s51 t �| �S r r r r r r r � r z0MidResTemporalBlock1D.__init__.<locals>.<lambda>r r TrN z$Block cannot downsample and upsample)r r r r r" r r$ r% r r& r' r( r) r* rD r r+ �52ValueError)53r, r r r
r- r" rC r2 r' r3 r4 r r r � s. 54 55 z MidResTemporalBlock1D.__init__c C sT | j d ||�}| j dd � D ]}|||�}q | jr>| �|�}| jrP| �|�| _|S )Nr r )r' rD r+ �r, r6 r7 r9 r r r r: � s 56 z MidResTemporalBlock1D.forward)r FFN)r< r= r> �int�boolr r: r? r r r4 r rT � s ��+rT c s& e Zd Z� fdd�Zddd�Z� ZS )�OutConv1DBlockc sd t � �� tj||ddd�| _t�||�| _|dkr>t�� | _|dkrPt� � | _t�||d�| _57d S )N� rA �r r r r ) r r r �Conv1d�final_conv1d_1� GroupNorm�final_conv1d_gnr* �final_conv1d_actr) �final_conv1d_2)r, �num_groups_outr r
�act_fnr4 r r r � s 585960zOutConv1DBlock.__init__Nc C s< | � |�}t|�}| �|�}t|�}| �|�}| �|�}|S r )r] r r_ r` ra )r, r6 r7 r r r r: � s 61626364zOutConv1DBlock.forward)Nr; r r r4 r rY � s 65rY c s$ e Zd Z� fdd�Zdd� Z� ZS )�OutValueFunctionBlockc s@ t � �� t�t�|| |d �t�� t�|d d�g�| _d S )NrA r )r r r r& �Linearr) � final_block)r, �fc_dimr
r4 r r r � s 66��z OutValueFunctionBlock.__init__c C s< |� |jd d�}tj||fdd�}| jD ] }||�}q*|S )Nr rE rF )�view�shaperH rI rf )r, r6 r7 �layerr r r r: s67 6869z OutValueFunctionBlock.forwardr; r r r4 r rd � s 70rd )� �?� �?rl rk )� ��� ��� �?� ��?rp ro rn rm ) � �8n?� ��Վ?� @Dh��� �$��� �<X�?� ����?rv ru rt rs rr rq )�linear�cubicZlanczos3c s& e Zd Zd� fdd� Zdd� Z� ZS )� Downsample1drw �reflectc sB t � �� || _t�t| �}|jd d d | _| �d|� d S )Nr rA r �kernel� r r �pad_moderH �tensor�_kernelsri �pad�register_buffer�r, r{ r} � kernel_1dr4 r r r $ s71 72zDownsample1d.__init__c C sv t �|| jfd | j�}|�|jd |jd | jjd g�}tj|jd |jd�}| j� |�|||f< t j73||dd�S )NrA r r ��device)�stride) r r� r} � new_zerosri r{ rH �aranger� �to�conv1d)r, r6 �weight�indicesr r r r: + s74 $zDownsample1d.forward)rw rz r; r r r4 r ry # s ry c s( e Zd Zd� fdd� Zd dd�Z� ZS )75�76Upsample1drw rz c sF t � �� || _t�t| �d }|jd d d | _| �d|� d S )NrA r r r{ r| r� r4 r r r 4 s77 78zUpsample1d.__init__Nc C s� t �|| jd d fd | j�}|�|jd |jd | jjd g�}tj|jd |jd�}| j� |�|||f< t j79||d| jd d d�S )Nr rA r r� )r� r ) r r� r} r� ri r{ rH r� r� r� �conv_transpose1d)r, r6 r7 r� r� r r r r: ; s80 $zUpsample1d.forward)rw rz )Nr; r r r4 r r� 3 s r� c s: e Zd Zd81� fdd� Zejejd�dd�Zdd � Z� ZS ) �SelfAttention1dr � c s� t � �� || _tjd|d�| _|| _t�| j| j�| _t�| j| j�| _ t�| j| j�| _82tj| j| jdd�| _ tj |dd�| _
d S )Nr )� num_channelsT��bias)�inplace)r r �channelsr r^ �83group_norm� num_headsre �query�key�value� proj_attn�Dropout�dropout)r, r �n_head� dropout_rater4 r r r D s 84zSelfAttention1d.__init__)�85projection�returnc C s4 |� � d d� | jdf }|�|��dddd�}|S )NrE r rA r � )�sizer� rh �permute)r, r� Znew_projection_shapeZnew_projectionr r r �transpose_for_scoresR s z$SelfAttention1d.transpose_for_scoresc C s |}|j \}}}| �|�}|�dd�}| �|�}| �|�}| �|�}| �|�} | �|�}86| �|�} dt�t�|87j d �� } t �88| | |89�dd�| �}
t j |
dd�}t �90|| �}|� dddd��
� }|�� d d� | jf }|�|�}| �|�}|�dd�}| �|�}|| }|S )Nr rA rE �����rF r r� )ri r� � transposer� r� r� r� �math�sqrtrH �matmul�softmaxr� �91contiguousr� r� rh r� r� )r, r6 �residual�batch� channel_dim�seq�92query_proj�key_proj�93value_projZ query_statesZ94key_statesZ value_states�scale�attention_scores�attention_probsZnew_hidden_states_shape�outputr r r r: X s, 95 96979899100101 102103 104zSelfAttention1d.forward)r r� ) r< r= r> r rH �Tensorr� r: r? r r r4 r r� C s r� c s& e Zd Zd� fdd� Zdd� Z� ZS )� ResConvBlockFc s� t � �� || _||k| _| jr4tj||ddd�| _tj||ddd�| _t�d|�| _ t�105� | _ tj||ddd�| _ | js�t�d|�| _
t�106� | _d S )Nr Fr� rZ rA r[ )r r �is_last�
has_conv_skipr r\ � conv_skip�conv_1r^ � group_norm_1�GELU�gelu_1�conv_2� group_norm_2�gelu_2)r, r � mid_channelsr r� r4 r r r ~ s 107108109zResConvBlock.__init__c C sb | j r| �|�n|}| �|�}| �|�}| �|�}| �|�}| jsV| �|�}| �|�}|| }|S r ) r� r� r� r� r� r� r� r� r� )r, r6 r� r� r r r r: � s 110111112113114115zResConvBlock.forward)Fr; r r r4 r r� } s r� c s( e Zd Zd� fdd� Zddd�Z� ZS )�UNetMidBlock1DNc s� t � �� |d u r|n|}td�| _t|||�t|||�t|||�t|||�t|||�t|||�g}t||d �t||d �t||d �t||d �t||d �t||d �g}tdd�| _t� |�| _116t� |�| _ d S )Nrx r �r{ ) r r ry �downr� r� r� �upr r& �117attentionsr' )r, r� r r r' r� r4 r r r � s( 118119120121122123124125� � zUNetMidBlock1D.__init__c C s@ | � |�}t| j| j�D ]\}}||�}||�}q| �|�}|S r )r� �zipr� r' r� )r, r6 r7 �attnr9 r r r r: � s 126127128zUNetMidBlock1D.forward)N)Nr; r r r4 r r� � s r� c s( e Zd Zd� fdd� Zddd�Z� ZS )�AttnDownBlock1DNc s� t � �� |d u r|n|}td�| _t|||�t|||�t|||�g}t||d �t||d �t||d �g}t�|�| _t�|�| _ d S )Nrx r )129r r ry r� r� r� r r& r� r' )r, r r r� r' r� r4 r r r � s 130131132133134� � zAttnDownBlock1D.__init__c C s< | � |�}t| j| j�D ]\}}||�}||�}q||ffS r )r� r� r' r� )r, r6 r7 r9 r� r r r r: � s135 136137zAttnDownBlock1D.forward)N)Nr; r r r4 r r� � s r� c s( e Zd Zd� fdd� Zddd�Z� ZS )� DownBlock1DNc sV t � �� |d u r|n|}td�| _t|||�t|||�t|||�g}t�|�| _d S )Nrx )r r ry r� r� r r& r' �r, r r r� r' r4 r r r � s 138139140141142�zDownBlock1D.__init__c C s( | � |�}| jD ] }||�}q||ffS r )r� r' rV r r r r: � s 143144145zDownBlock1D.forward)N)Nr; r r r4 r r� � s
r� c s( e Zd Zd� fdd� Zddd�Z� ZS )�DownBlock1DNoSkipNc sL t � �� |d u r|n|}t|||�t|||�t|||�g}t�|�| _d S r �r r r� r r& r' r� r4 r r r � s 146147148149�zDownBlock1DNoSkip.__init__c C s0 t j||gdd�}| jD ] }||�}q||ffS )Nr rF �rH rI r' rV r r r r: s 150151zDownBlock1DNoSkip.forward)N)Nr; r r r4 r r� � s r� c s( e Zd Zd� fdd� Zddd�Z� ZS )�
AttnUpBlock1DNc s� t � �� |d u r|n|}td| ||�t|||�t|||�g}t||d �t||d �t||d �g}t�|�| _t�|�| _tdd�| _ d S )NrA r rx r� )152r r r� r� r r& r� r' r� r� )r, r r r� r' r� r4 r r r s 153154155� � zAttnUpBlock1D.__init__c C sP |d }t j||gdd�}t| j| j�D ]\}}||�}||�}q(| �|�}|S �NrE r rF )rH rI r� r' r� r� )r, r6 rK r7 rL r9 r� r r r r: % s 156157zAttnUpBlock1D.forward)N)Nr; r r r4 r r� s r� c s( e Zd Zd� fdd� Zddd�Z� ZS )� UpBlock1DNc s\ t � �� |d u r|n|}td| ||�t|||�t|||�g}t�|�| _tdd�| _d S )NrA rx r� )r r r� r r& r' r� r� �r, r r r� r' r4 r r r 3 s 158159160� zUpBlock1D.__init__c C s< |d }t j||gdd�}| jD ] }||�}q | �|�}|S r� )rH rI r' r� rJ r r r r: @ s 161162163zUpBlock1D.forward)N)Nr; r r r4 r r� 2 s
r� c s( e Zd Zd� fdd� Zddd�Z� ZS )�UpBlock1DNoSkipNc sT t � �� |d u r|n|}td| ||�t|||�t|||dd�g}t�|�| _d S )NrA T)r� r� r� r4 r r r M s 164165�zUpBlock1DNoSkip.__init__c C s2 |d }t j||gdd�}| jD ] }||�}q |S r� r� rJ r r r r: Y s166 167168zUpBlock1DNoSkip.forward)N)Nr; r r r4 r r� L s r� c C sh | dkrt |||||d�S | dkr.t||d�S | dkrBt||d�S | dkrVt||d�S t| � d���d S )Nr )r r- r r/ r" r� )r r r� r� � does not exist.)r r� r� r� rU )�down_block_typer- r r r/ r" r r r �get_down_blockc s � r� c C sh | dkrt |||||d�S | dkr.t||d�S | dkrBt||d�S | dkrVt||d�S t| � d���d S )Nr@ )r r- r r/ rC r� )r r r� r� r� )r@ r� r� r� rU )�
up_block_typer- r r r/ rC r r r � get_up_blocku s � r� c C sX | dkrt |||||d�S | dkr0t|||d�S | dkrFt|||d�S t| � d���d S )NrT )r- r r r
r" rM )r r r
r� )r r� r r� )rT rM r� rU )�mid_block_typer- r r� r r
r" r r r �
get_mid_block� s �r� c C s, | dkrt ||||�S | dkr(t||�S d S )NrY Z
ValueFunction)rY rd )�out_block_typerb r
r rc rg r r r �
get_out_block� s169 170r� )"r� rH Ztorch.nn.functionalr �171functionalr r9 r r r r �Moduler r@ rM rT rY rd r ry r� r� r� r� r� r� r� r� r� r� r� r� r� r� r r r r �<module> s: B@9�:"( !