CoolFace
Apppublic

cocktailpeanut/DiffRhythm

sourceHugging Faceapache-2.0updated 1y agoView on Hugging Face
10likes
modules.cpython-310.pyc187 linesDownload Raw Back to __pycache__
1o

2�0�g�O�@s�dZddlmZddlmZddlZddlZddlmZddlZddlmm	Z3ddlZddlm
Z
Gdd�dej�ZGd	d4�d5ej�ZGdd�dej�ZGd
d�dej�Zd0d1dd�Zd2dd�ZGdd�dej�ZGdd�dej�ZGdd�dej�ZGd d!�d!ej�ZGd"d#�d#ej�ZGd$d%�d%ej�ZGd&d'�d'�ZGd(d)�d)�ZGd*d+�d+ej�ZGd,d-�d-ej�ZGd.d/�d/ej�ZdS)3z\6ein notation:7b - batch8n - sequence9nt - text sequence10nw - raw wave length11d - dimension12�)�annotations)�OptionalN)�nn)�apply_rotary_pos_embcs(eZdZdZ�fdd�Zdd�Z�ZS)�	FiLMLayerze13    Feature-wise Linear Modulation (FiLM) layer14    Reference: https://arxiv.org/abs/1709.0787115    cs,tt|���||_t�||dd�|_dS)N��)�superr�__init__�in_channelsr�Conv1d�film)�selfrZ
cond_channels��	__class__��N/home/node59_tmpdata3/hkchen/DiffRhythm/diffrhythm/diffrhythm/model/modules.pyr16szFiLMLayer.__init__cCsDtj|�|�d��ddd�\}}|�dd�}|�dd�}|||S)Nrr)�chunks�dim)�torch�chunkr
�	unsqueeze�	transpose)r�x�c�gamma�betarrr�forward#s zFiLMLayer.forward��__name__�17__module__�__qualname__�__doc__r18r�
__classcell__rrrrrsrcs8eZdZ									d
�fd	d19�	Zdd�Z�ZS)�MelSpec���d��]FrNTc2021sJt���||_tjj|||||||	||d�	|_|jdt�	d�dd�dS)N)	�sample_rate�n_fft�22win_length�23hop_length�n_mels�power�center�24normalized�norm�dummyrF)�25persistent)26r	r27�n_mel_channels�28torchaudio�29transforms�MelSpectrogram�mel_stft�register_bufferr�tensor)30rZ
filter_lengthr,r+r4�target_sample_rate�	normalizer.r1r/rrrr31.s32�zMelSpec.__init__cCsbt|j�dkr|�d�}t|j�dksJ�|jj|jkr"|�|j�|�|�}|jdd���}|S)N�rrg�h㈵��>)�min)	�len�shape�squeezer2�device�tor8�clamp�log)r�inp�melrrrrKs3334zMelSpec.forward)	r%r&r%r'r(FrNT�rr r!r35rr#rrrrr$-s�r$c�&eZdZ�fdd�Zddd�Z�ZS)�SinusPositionEmbeddingcst���||_dS�N)r	r36r�rrrrrr37]s3839zSinusPositionEmbedding.__init__��cCsv|j}|jd}t�d�|d}t�tj||d���|�}||�d�|�d�}tj	|�40�|��fdd�}|S)Nri'r�rBr������r)rBr�mathrEr�exp�arange�floatr�cat�sin�cos)rr�scalerBZhalf_dim�embrrrras41zSinusPositionEmbedding.forward)rMrHrrrrrJ\srJcs*eZdZd�fdd�	Zd
dd42d�Z�ZS)�ConvPositionEmbedding��cs`t���|ddks
J�t�tj|||||dd�t��tj|||||dd�t���|_dS)Nrr)�groups�padding)r	r43r�44Sequentialr�Mish�conv1d)rr�kernel_sizer]rrrr45os4647�zConvPositionEmbedding.__init__Nr�float['b n d']�mask�bool['b n'] | NonecCs^|dur|d}|�|d�}|�ddd�}|�|�}|�ddd�}|dur-|�|d�}|S)N).N�rrr)�masked_fill�permutera)rrrd�outrrrrys48zConvPositionEmbedding.forward)r[r\rK)rrcrdrerHrrrrrZns49rZ���@��?r�int�end�thetarTcCs�||||d9}d|t�d|d�d|d���|}tj||jd�}t�||���}t�|�}t�|�}tj||gdd�S)NrrkrrNrOrP)rrSrTrB�outerrWrVrU)rrmrnZtheta_rescale_factor�freqs�tZ	freqs_cosZ	freqs_sinrrr�precompute_freqs_cis�s*5051rrcCs`|tj|tjd�}|�d�tj||jtjd��d�|�d���}t�||k||d�}|S)N)�dtyper)rBrsr)r�	ones_like�float32rrSrB�long�where)�start�length�max_posrX�posrrr�get_pos_embed_indices�s&��r|c�$eZdZ�fdd�Zdd�Z�ZS)�GRNcs:t���t�t�dd|��|_t�t�dd|��|_dS)Nr)r	r52r�	Parameterr�zerosrrrLrrrr53�s54zGRN.__init__cCs@tj|dddd�}||jddd�d}|j|||j|S)NrrT)�pr�keepdimrO)rr������ư>)rr1�meanrr)rrZGx�Nxrrrr�szGRN.forwardrHrrrrr~�sr~cs,eZdZ	d
d�fdd�
Zddd�Z�ZS)�ConvNeXtV2Blockrrrl�intermediate_dim�dilationcsrt���|dd}tj||d|||d�|_tj|dd�|_t�||�|_t�	�|_55t|�|_t�||�|_
dS)N�r�)rbr^r]r�r�)�eps)r	r56rr�dwconv�	LayerNormr1�Linear�pwconv1�GELU�actr~�grn�pwconv2)rrr�r�r^rrrr57�s58�5960zConvNeXtV2Block.__init__r�torch.Tensor�returncCs`|}|�dd�}|�|�}|�dd�}|�|�}|�|�}|�|�}|�|�}|�|�}||S)Nrr)rr�r1r�r�r�r�)rr�residualrrrr�s616263646566zConvNeXtV2Block.forward)r)rrlr�rlr�rl)rr�r�r�rHrrrrr��s�r�crI)�AdaLayerNormZerocs<t���t��|_t�||d�|_tj|ddd�|_dS)Nr�Fr���elementwise_affiner��	r	r67r�SiLU�silur��linearr�r1rLrrrr68��6970zAdaLayerNormZero.__init__Nc	Csh|�|�|��}tj|ddd�\}}}}}}|�|�d|dd�df|dd�df}|||||fS)Nr�rrP�r�r�rrr1)	rrrYZ	shift_msaZ	scale_msa�gate_msa�	shift_mlp�	scale_mlp�gate_mlprrrr�s.zAdaLayerNormZero.forwardrKrHrrrrr��sr�cr})�AdaLayerNormZero_Finalcs<t���t��|_t�||d�|_tj|ddd�|_dS)NrFr�r�r�)rrZcond_dimrrrr71�r�zAdaLayerNormZero_Final.__init__cCsb|�|�|��}tj|ddd�\}}|�|�d|dd�ddd�f|dd�ddd�f}|S)NrrrPr�)rrrYrX�shiftrrrr�s:zAdaLayerNormZero_Final.forwardrHrrrrr��sr�cs(eZdZdd�fdd�
Zd	d72�Z�ZS)
�FeedForwardN�rf�none�approximate�strc	sht���t||�}|dur|n|}tj|d�}t�t�||�|�}t�|t�|�t�||��|_dS)N)r�)	r	r73rlrr�r_r��Dropout�ff)	rrZdim_out�mult�dropoutr��	inner_dim�74activationZ75project_inrrrr76s77"zFeedForward.__init__cCs78|�|�SrK)r�)rrrrrrs79zFeedForward.forward)Nr�rfr�)r�r�rHrrrrr�s80r�cs>eZdZ					dd�fdd�
Z				dddd�Z�ZS)�	Attention��@rfN�	processor�"JointAttnProcessor | AttnProcessorrrl�heads�dim_headr�rT�context_dim�
Optional[int]cst���ttd�std��||_||_||_|||_||_	||_81||_t�
||j�|_t�
||j�|_t�
||j�|_|j82dur_t�
||j�|_t�
||j�|_|jdur_t�
||j�|_t�g�|_|j�t�
|j|��|j�t�|��|jdur�|js�t�
|j|�|_dSdSdS)N�scaled_dot_product_attentionzHAttention equires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.)r	r83�hasattr�F�ImportErrorr�rr�r�r�r��context_pre_onlyrr��to_q�to_k�to_v�to_k_c�to_v_c�to_q_c�84ModuleList�to_out�appendr��to_out_c)rr�rr�r�r�r�r�rrrr85s086 87 88899091�zAttention.__init__rrcrrdrer�r�cCs0|dur|j||||||d�S|j||||d�S)N)rrd�rope�c_rope)rdr�)r�)rrrrdr�r�rrrrBszAttention.forward)r�r�rfNN)r�r�rrlr�rlr�rlr�rTr�r��NNNN)rrcrrcrdrer�r�rHrrrrr�s�-�r�c@s$eZdZdd�Z		dddd
�ZdS)�
AttnProcessorcC�dSrKr�rrrrr92T�zAttnProcessor.__init__N�attnr�rrcrdrer��torch.FloatTensorcCs�|jd}|�|�}|�|�}|�|�}|dur6|\}	}93|94dur&|95|96dfnd\}}t||	|�}t||	|�}|jd}
|
|j}|�|d|j|��dd�}|�|d|j|��dd�}|�|d|j|��dd�}|dur�|}|�d��d�}|�	||j|jd|jd�}nd}t97j||||dd	d98�}|�dd��|d|j|�}|�
|j�}|jd|�}|jd|�}|dur�|�d�}|�|d�}|S)Nr��rkrkrOrr�����rfF��	attn_mask�	dropout_p�	is_causal)r@r�r�r�rr��viewrr�expandr�r��reshaperCrsr�rg)rr�rrdr��99batch_size�query�key�valuerp�100xpos_scale�q_xpos_scale�k_xpos_scaler��head_dimr�rrr�__call__Ws8101102103104105106 107zAttnProcessor.__call__)NN)r�r�rrcrdrer�r��rr r!r108r�rrrrr�Ss109�r�c@s(eZdZdd�Z				dddd�ZdS)�JointAttnProcessorcCr�rKrr�rrrr110�r�zJointAttnProcessor.__init__Nr�r�rrcr�float['b nt d']rdrer�r�cCs�|}|jd}|�|�}	|�|�}111|�|�}|�|�}|�|�}
|�|�}|durG|\}}|dur7||dfnd\}}t|	||�}	t|112||�}113|duri|\}}|durY||dfnd\}}t|||�}t|
||�}
tj	|	|gdd�}	tj	|114|
gdd�}115tj	||gdd�}|116jd}||j117}|	�|d|j118|��dd�}	|119�|d|j120|��dd�}121|�|d|j122|��dd�}|dur�t
j|d|jdfdd	�}|�d��d�}|�||j123|	jd124|125jd126�}nd}t
j|	|127||ddd
�}|�dd��|d|j128|�}|�|	j�}|dd�d|jd�f|dd�|jdd�f}}|jd|�}|jd|�}|j�s1|�|�}|du�rB|�d�}|�|d�}||fS)Nrr�r�rrPrOrT)r�r�rfFr�)r@r�r�r�r�r�r�rrrUr�r�rr��padrr�r�r�rCrsr�r�r�rg)rr�rrrdr�r�r�r�r�r�r�Zc_queryZc_keyZc_valuerpr�r�r�r�r�r�rrrr��sZ	129130131132133134135136137 �138139140zJointAttnProcessor.__call__r�)141r�r�rrcrr�rdrer�r�r�rrrrr��s�r�cs(eZdZd	�fdd�	Zd142dd�Z�ZS)�DiTBlockr�皙�����?Fcsjt���t|�|_tt�||||d�|_tj|ddd�|_	t143|||dd�|_||_|r3t
|�|_dSdS)N)r�rr�r�r�Fr�r��tanh�rr�r�r�)r	r144r��	attn_normr�r�r�rr��ff_normr�r��use_style_promptr��prompt_norm)rrr�r��ff_multr�r�rrrr145�s146147��zDiTBlock.__init__Nc
Cs�|dur
|jr
|�||�}|j||d�\}}}}	}148|j|||d�}||�d�|}|�|�d|	dd�df|dd�df}|�|�}||149�d�|}|S)N�rY)rrdr�r)r�r�r�r�rr�r�)
rrrqrrdr�r1r�r�r�r��attn_outputZ	ff_outputrrrr�s.150zDiTBlock.forward�r�r�F�NNNrHrrrrr��sr�cs,eZdZdZd151�fdd�	Zddd	�Z�ZS)�152MMDiTBlocka153    modified from diffusers/src/diffusers/models/attention.py154 155    notes.156    _c: context related. text, cond, etc. (left part in sd3 fig2.b)157    _x: noised input related. (right part)158    context_pre_only: last layer only do prenorm + modulation cuz no more ffn159    r�r�Fc	s�t���||_|rt|�nt|�|_t|�|_tt�||||||d�|_	|s:t160j|ddd�|_t
|||dd�|_nd|_d|_t161j|ddd�|_t
|||dd�|_dS)N)r�rr�r�r�r�r�Fr�r�r�r�)r	r162r�r�r��attn_norm_c�attn_norm_xr�r�r�rr��	ff_norm_cr��ff_c�	ff_norm_x�ff_x)rrr�r�r�r�r�rrrr163s(164165�166zMMDiTBlock.__init__NcCs(|jr167|�||�}n|j||d�\}}}	}168}|j||d�\}}
}}}|j|||||d�\}}|jr4d}n.||�d�|}|�|�d|169dd�df|	dd�df}|�|�}||�d�|}||
�d�|}|�|�d|dd�df|dd�df}|�|�}||�d�|}||fS)Nr�)rrrdr�r�r)	r�r�r�r�rr�r�r�r�)rrrrqrdr�r�Znorm_cZ170c_gate_msaZc_shift_mlpZc_scale_mlpZ171c_gate_mlp�norm_xZ172x_gate_msaZx_shift_mlpZx_scale_mlpZ173x_gate_mlpZ
x_attn_outputZ
c_attn_outputZc_ff_outputZx_ff_outputrrrr7s .174.175zMMDiTBlock.forwardr�r�rrrrrr�s	r�cs(eZdZd�fdd�	Zd	dd�Z�ZS)176�TimestepEmbeddingr&cs<t���t|�|_t�t�||�t��t�||��|_dSrK)	r	r177rJ�178time_embedrr_r�r��time_mlp)rrZfreq_embed_dimrrrr179Zs180181(zTimestepEmbedding.__init__�timestep�182float['b']cCs$|�|�}|�|j�}|�|�}|SrK)r�rCrsr)rrZtime_hidden�timerrrr_s183184zTimestepEmbedding.forward)r&)rrrHrrrrr�Ysr�)rjrk)rrlrmrlrnrT)rk) r"�185__future__r�typingrrQrr�torch.nn.functional�186functionalr�r5Zx_transformers.x_transformersr�Modulerr$rJrZrrr|r~r�r�r�r�r�r�r�r�r�r�rrrr�<module>s6	/187#<=V-F