CoolFace
Apppublic

declare-lab/tango2

sourceHugging Faceupdated 2y agoView on Hugging Face
92likes
attention_processor.cpython-39.pyc200 linesDownload Raw Back to __pycache__
1a

2�0d�s�@s2ddlmZmZmZddlZddlmmZddlmZddl	m3Z4mZddlm
Z
e�e�Ze
�rxddlZddlZndZGdd�dej�ZGd	d5�d6�ZGdd�dej�ZGd
d�dej�ZGdd�d�ZGdd�d�ZGdd�d�ZGdd�dej�ZGdd�d�ZGdd�d�ZeeeeeeeefZdS)�)�Callable�Optional�UnionN)�nn�)�	deprecate�logging)�is_xformers_availablecs�eZdZdZdeeeeeeeeeeeeeeeedd	�
�fd7d�
Zdeee	d�d
d�Z8dd�Zdd�dd�Zd dd�Z
dd�Zdd�Zd!dd�Zd"dd�Z�ZS)#�	Attentiona�9    A cross attention layer.10 11    Parameters:12        query_dim (`int`): The number of channels in the query.13        cross_attention_dim (`int`, *optional*):14            The number of channels in the encoder_hidden_states. If not given, defaults to `query_dim`.15        heads (`int`,  *optional*, defaults to 8): The number of heads to use for multi-head attention.16        dim_head (`int`,  *optional*, defaults to 64): The number of channels in each head.17        dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.18        bias (`bool`, *optional*, defaults to False):19            Set to `True` for the query, key, and value linear layers to contain a bias parameter.20    N��@�FT�
AttnProcessor)
�	query_dim�cross_attention_dim�heads�dim_head�dropout�upcast_attention�upcast_softmax�cross_attention_norm�added_kv_proj_dim�norm_num_groups�out_bias�scale_qk�	processorcsTt���||}|dur|n|}||_||_|	|_|
r@|dnd|_||_||_|21|_|durvt	j22||ddd�|_nd|_|	r�t	�|�|_
t	j|||d�|_t	j|||d�|_t	j|||d�|_|jdur�t	�|23|�|_t	�|24|�|_t	�g�|_|j�t	j|||d��|j�t	�|��|du�rFttd��r@|
�r@t�nt�}|�|�dS)Ng���?g�h㈵��>T)�num_channels�25num_groups�eps�affine��bias�scaled_dot_product_attention)�super�__init__rrr�scaler�sliceable_head_dimrr�	GroupNorm�26group_norm�	LayerNorm�27norm_cross�Linear�to_q�to_k�to_v�28add_k_proj�29add_v_proj�30ModuleList�to_out�append�Dropout�hasattr�F�AttnProcessor2_0r�
set_processor)�selfrrrrrr"rrrrrrrr�	inner_dim��	__class__��Y/home/deep/Projects/audio_diffusion/diffusers/src/diffusers/models/attention_processor.pyr%1s8313233�zAttention.__init__)�'use_memory_efficient_attention_xformers�attention_opc34Csdt|d�ot|jttf�}|�r35|jdur4td��n�t�sHtddd��npt	j36��s\td��n\z2t
j�t	jddd	�t	jddd	�t	jddd	��}Wn(ty�}z|�WYd}~n37d}~00|r�t|jj|jj|jj|d38�}|�|j���|�|jjjjj�n39t|d�}nL|�rPt|jj|jj|jjd�}|�|j���|�|jjjjj�nt�}|�|�dS)
NrzoMemory efficient attention with `xformers` is currently not supported when `self.added_kv_proj_dim` is defined.zeRefer to https://github.com/facebookresearch/xformers for more information on how to install xformers�xformers)�namezvtorch.cuda.is_available() should be True but is False. xformers' memory efficient attention is only available for GPU )�r�(�cuda)�device)�hidden_sizer�rankrA�rA)rHrrI)r6�40isinstancer�LoRAAttnProcessor�LoRAXFormersAttnProcessorr�NotImplementedErrorr	�ModuleNotFoundError�torchrF�is_available�41ValueErrorrB�ops�memory_efficient_attention�randn�	ExceptionrHrrI�load_state_dict�42state_dict�to�	to_q_lora�up�weightrG�XFormersAttnProcessorrr9)r:r@rAZis_lora�_�err>r>r?�+set_use_memory_efficient_attention_xformersqsZ43�44��45����z5Attention.set_use_memory_efficient_attention_xformerscCs~|dur*||jkr*td|�d|j�d���|durF|jdurFt|�}n*|durXt|�}n|jdurjt�}nt�}|�|�dS)Nzslice_size z has to be smaller or equal to �.)r'rRr�SlicedAttnAddedKVProcessor�SlicedAttnProcessor�AttnAddedKVProcessorrr9)r:�46slice_sizerr>r>r?�set_attention_slice�s474849zAttention.set_attention_slice�rcCsVt|d�rLt|jtjj�rLt|tjj�sLt�d|j�d|���|j�	d�||_dS)Nrz-You are removing possibly trained weights of z with )50r6rKrrPr�Module�logger�info�_modules�pop)r:rr>r>r?r9�s���zAttention.set_processorcKs|j||f||d�|��S)N)�encoder_hidden_states�attention_maskrg)r:�
hidden_statesrmrn�cross_attention_kwargsr>r>r?�forward�s���zAttention.forwardcCsL|j}|j\}}}|�|||||�}|�dddd��|||||�}|S�NrrrD��r�shape�reshape�permute�r:�tensor�	head_size�51batch_size�seq_len�dimr>r>r?�batch_to_head_dim�s52"zAttention.batch_to_head_dimcCsL|j}|j\}}}|�|||||�}|�dddd��|||||�}|Srrrtrxr>r>r?�head_to_batch_dim�s53"zAttention.head_to_batch_dimc	Cs�|j}|jr|��}|��}|durTtj|jd|jd|jd|j|jd�}d}n|}d}tj|||�dd�||j	d�}|j54r�|��}|jdd�}|�|�}|S)NrrD��dtyperG����������)�beta�alpha�r})
r�r�floatrP�emptyrurG�baddbmm�	transposer&r�softmaxrY)	r:�query�keyrnr�Z
baddbmm_inputr��attention_scores�attention_probsr>r>r?�get_attention_scores�s. �55�56zAttention.get_attention_scoresc	Cs�|durtddd�d}|j}|dur*|S|jd}||krTtd|�d|�d���nn||kr�|jjd	kr�|jd57|jd|f}tj||j|jd�}tj	||gdd
�}n||}t58j|d59|fdd�}|jd60||kr�|j|d61d
�}|S)Nzbatch_size=Nonez0.0.15z�Not passing the `batch_size` parameter to `prepare_attention_mask` can lead to incorrect attention mask preparation and is deprecated behavior. Please make sure to pass `batch_size` to `prepare_attention_mask` when preparing the attention_mask.rDr�zmask's length (z) exceeds the sequence length (z).�mpsrr�rr�r
)�value)
rrrurRrG�typerP�zerosr��catr7�pad�repeat_interleave)	r:rn�
target_lengthr{rzZcurrent_lengthZ
padding_shape�paddingZremaining_lengthr>r>r?�prepare_attention_masks.�	62z Attention.prepare_attention_mask)
Nrrr
FFFFNNTTN)N)NN)N)N)�__name__�63__module__�__qualname__�__doc__�intrr��boolr%rr`rfr9rqr~rr�r��
__classcell__r>r>r<r?r64"sP��A��A
6566r67c@seZdZded�dd�ZdS)rN��attncCs�|dur|jn|j\}}}|�|||�}|�|�}|dur@|}n|jrP|�|�}|�|�}	|�|�}68|�|�}|�|	�}	|�|69�}70|�||	|�}t	�71||72�}|�|�}|jd|�}|jd|�}|S�NrrD)
rur�r-rr+r.r/rr�rP�bmmr~r3)r:r�rormrnr{�sequence_lengthr^r�r�r�r�r>r>r?�__call__/s&�7374757677787980zAttnProcessor.__call__)NN�r�r�r�r81r�r>r>r>r?r.s82��rcs&eZdZd�fdd�	Zdd�Z�ZS)�LoRALinearLayer�cs�t���|t||�kr2td|�dt||�����tj||dd�|_tj||dd�|_tjj	|jj83d|d�tj�|jj84�dS)Nz85LoRA rank z must be less or equal than Fr!rD)�std)r$r%�minrRrr,�downr[�init�normal_r\�zeros_)r:�in_features�out_featuresrIr<r>r?r%Us86zLoRALinearLayer.__init__cCs4|j}|jjj}|�|�|��}|�|�}|�|�S�N)r�r�r\rYr[)r:ro�87orig_dtyper�Zdown_hidden_statesZup_hidden_statesr>r>r?rqas888990zLoRALinearLayer.forward)r�)r�r�r�r%rqr�r>r>r<r?r�Tsr�cs.eZdZd	�fdd�	Zd91ed�dd�Z�ZS)rLNr�cs`t���||_||_||_t|||�|_t|p2|||�|_t|pD|||�|_t|||�|_	dSr�)92r$r%rHrrIr�rZ�	to_k_lora�	to_v_lora�to_out_lora)r:rHrrIr<r>r?r%ls93zLoRAAttnProcessor.__init__rr�c
Cs�|dur|jn|j\}}}|�|||�}|�|�||�|�}	|�|	�}	|durV|n|}|�|�||�|�}94|�|�||�|�}|�|95�}96|�|�}|�	|	|97|�}t98�||�}|�|�}|j
d|�||�|�}|j
d|�}|Sr�)rur�r-rZrr.r�r/r�r�rPr�r~r3r�)
r:r�rormrnr&r{r�r^r�r�r�r�r>r>r?r�xs �99100101102zLoRAAttnProcessor.__call__)Nr�)NNr)r�r�r�r%r103r�r�r>r>r<r?rLksrLc@seZdZded�dd�ZdS)rdNr�cCsN|}|�|jd|jdd��dd�}|j\}}}|�dd�}|�|||�}|�|�dd���dd�}|�|�}	|�|	�}	|�|�}104|�|�}|�|105�}106|�|�}|�	|�}|�107|�}
|�|�}|�|
�}
tj||108gdd�}109tj|
|gdd�}|�
|	|110|�}t�||�}|�|�}|jd|�}|jd|�}|�dd��|j�}||}|S)NrrDr�rr�r�)�viewrur�r�r)r-rr.r/r0r1rPr�r�r�r~r3rv)r:r�rormrn�residualr{r�r^r�r�r��encoder_hidden_states_key_proj� encoder_hidden_states_value_projr�r>r>r?r��s4"111112113114115116117118119120121zAttnAddedKVProcessor.__call__)NNr�r>r>r>r?rd�srdc@sDeZdZdeed�dd�Zd	eejeejeejd�dd�Z	dS)122r]NrJcCs123||_dSr�rJ)r:rAr>r>r?r%�szXFormersAttnProcessor.__init__)r�rormrncCs|dur|jn|j\}}}|�|||�}|durJ|j\}}}|�d|d�}|�|�}	|durb|}n|jrr|�|�}|�|�}124|�|�}|�|	��	�}	|�|125��	�}126|�|��	�}t127jj|	|128|||j
|jd�}|�|	j�}|�|�}|jd|�}|jd|�}|S)Nr���	attn_bias�opr&rrD)rur��expandr-rr+r.r/r�129contiguousrBrSrTrAr&rYr�r~r3)r:r�rormrnr{Z130key_tokensr^�query_tokensr�r�r�r>r>r?r��s0�131132133134�135zXFormersAttnProcessor.__call__)N)NN)136r�r�r�rrr%r137rP�FloatTensorr�r>r>r>r?r]�s��r]c@s$eZdZdd�Zded�dd�ZdS)r8cCsttd�std��dS)Nr#zPAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.)r6r7�ImportError)r:r>r>r?r%�s138zAttnProcessor2_0.__init__Nr�c
CsD|dur|jn|j\}}}|jd}|durR|�|||�}|�||jd|jd�}|�|�}	|durj|}n|jrz|�|�}|�|�}139|�|�}||j}|	�|d|j|��	dd�}	|140�|d|j|��	dd�}141|�|d|j|��	dd�}t142j|	|143||ddd�}|�	dd��|d|j|�}|�
|	j�}|jd|�}|jd|�}|S)Nr�rDrr
F)�	attn_mask�	dropout_pZ	is_causalr)rur�r�rr-rr+r.r/r�r7r#rvrYr�r3)
r:r�rormrnr{r�r^r;r�r�r��head_dimr>r>r?r��s2�144145146147148149�zAttnProcessor2_0.__call__)NN�r�r�r�r%r150r�r>r>r>r?r8�sr8cs8eZdZd151eed��fdd�
Zded�dd	�Z�ZS)rMr�NrJcsft���||_||_||_||_t|||�|_t|p8|||�|_t|pJ|||�|_	t|||�|_152dSr�)r$r%rHrrIrAr�rZr�r�r�)r:rHrrIrAr<r>r?r% s153z"LoRAXFormersAttnProcessor.__init__rr�cCs�|dur|jn|j\}}}|�|||�}|�|�||�|�}	|�|	���}	|durZ|n|}|�|�||�|�}154|�|�||�	|�}|�|155���}156|�|���}t157jj|	|158|||j
|jd�}|�|�}|jd|�||�|�}|jd|�}|S)Nr�rrD)rur�r-rZrr�r.r�r/r�rBrSrTrAr&r~r3r�)r:r�rormrnr&r{r�r^r�r�r�r>r>r?r�-s"��159z"LoRAXFormersAttnProcessor.__call__)r�N)NNr)	r�r�r�rrr%r160r�r�r>r>r<r?rMs
rMc@s$eZdZdd�Zded�dd�ZdS)rccCs161||_dSr��re�r:rer>r>r?r%LszSlicedAttnProcessor.__init__Nr�cCsn|dur|jn|j\}}}|�|||�}|�|�}|jd}	|�|�}|durT|}n|jrd|�|�}|�|�}162|�|�}|�|163�}164|�|�}|j\}}
}tj	||
|	|j165f|j|jd�}t
||j�D]|}||j}|d|j}|||�}|166||�}|du�r|||�nd}|�|||�}t�||||��}||||�<q�|�|�}|jd|�}|jd|�}|S)Nr��rGr�rDr)rur�r-rrr+r.r/rPr�rrGr��rangerer�r�r~r3)r:r�rormrnr{r�r^r�r}r�r��batch_size_attentionr��i�	start_idx�end_idx�query_slice�	key_slice�attn_mask_slice�167attn_slicer>r>r?r�Os>�168169170171172173174175�176177zSlicedAttnProcessor.__call__)NNr�r>r>r>r?rcKsrcc@s$eZdZdd�Zddd�dd�ZdS)	rbcCs178||_dSr�r�r�r>r>r?r%�sz#SlicedAttnAddedKVProcessor.__init__Nr179r�cCs�|}|�|jd|jdd��dd�}|�dd�}|j\}}}|�|||�}|�|�dd���dd�}|�|�}	|	jd}180|�|	�}	|�|�}|�|�}|�	|�}
|�181|�}|�|�}|�|�}|�|
�}
|�|�}tj|
|gdd�}tj||gdd�}|	j\}}}tj
|||182|jf|	j|	jd�}t||j�D]~}||j}|d|j}|	||�}|||�}|du�r||||�nd}|�|||�}t�||||��}||||�<�q2|�|�}|jd|�}|jd|�}|�dd��|j�}||}|S)NrrDr�rr�r�r�)r�rur�r�r)r-rr.r/r0r1rPr�r�rrGr�r�rer�r�r~r3rv)r:r�rormrnr�r{r�r^r�r}r�r�r�r�r�r�r�r�r�r�r�r�r�r>r>r?r��sL"183184185186187188189190191192193�194195z#SlicedAttnAddedKVProcessor.__call__)NN)r�r�r�r%r�r>r>r>r?rb�srb)�typingrrrrP�torch.nn.functionalr�196functionalr7�utilsrr�utils.import_utilsr	�197get_loggerr�rirB�xformers.opsrhr198rr�rLrdr]r8rMrcrbZAttentionProcessorr>r>r>r?�<module>s@199200&**00,5@��