CoolFace
Apppublic

declare-lab/tango2

sourceHugging Faceupdated 2y agoView on Hugging Face
92likes
rootdiffuserssrcdiffusersmodels__pycache__controlnet.cpython-39.pyc
controlnet.cpython-39.pyc129 linesDownload Raw Back to __pycache__
1a

2��'d�a�@s3ddlmZddlmZmZmZmZmZmZddl	Z	ddl	m4Z5ddlmZ
ddlmZmZddlmZmZd	d6lmZmZd	dlmZmZd	dlmZd	d
lmZmZmZm Z d	dl!m"Z"e�#e$�Z%eGdd�de��Z&Gdd�de7j'�Z(Gdd�dee�Z)dd�Z*dS)�)�	dataclass)�Any�Dict�List�Optional�Tuple�UnionN)�nn)�8functional�)�ConfigMixin�register_to_config)�9BaseOutput�logging�)�AttentionProcessor�
AttnProcessor)�TimestepEmbedding�	Timesteps)�10ModelMixin)�CrossAttnDownBlock2D�DownBlock2D�UNetMidBlock2DCrossAttn�get_down_block)�UNet2DConditionModelc@s&eZdZUeejed<ejed<dS)�ControlNetOutput�down_block_res_samples�mid_block_res_sampleN)�__name__�11__module__�__qualname__r�torch�Tensor�__annotations__�r$r$�P/home/deep/Projects/audio_diffusion/diffusers/src/diffusers/models/controlnet.pyr&s12rcs8eZdZdZd	eeeed��fdd�
Zdd�Z�ZS)13�ControlNetConditioningEmbeddingu�14    Quoting from https://arxiv.org/abs/2302.05543: "Stable Diffusion uses a pre-processing method similar to VQ-GAN15    [11] to convert the entire dataset of 512 × 512 images into smaller 64 × 64 “latent images” for stabilized16    training. This requires ControlNets to convert image-based conditions to 64 × 64 feature space to match the17    convolution size. We use a tiny network E(·) of four convolution layers with 4 × 4 kernels and 2 × 2 strides18    (activated by ReLU, channels are 16, 32, 64, 128, initialized with Gaussian weights, trained jointly with the full19    model) to encode image-space conditions ... into feature maps ..."20    ���� �`�)�conditioning_embedding_channels�conditioning_channels�block_out_channelsc21s�t���tj||dddd�|_t�g�|_tt|�d�D]N}||}||d}|j�	tj||ddd��|j�	tj||dddd��q>t22tj|d|ddd��|_dS)Nrr'r��kernel_size�paddingr)r1r2�stride�����)�super�__init__r	�Conv2d�conv_in�23ModuleList�blocks�range�len�append�zero_module�conv_out)�selfr-r.r/�iZ24channel_inZchannel_out��	__class__r$r%r66s25�z(ControlNetConditioningEmbedding.__init__cCs@|�|�}t�|�}|jD]}||�}t�|�}q|�|�}|S�N)r8�F�silur:r?)r@�conditioning�	embedding�blockr$r$r%�forwardLs26272829z'ControlNetConditioningEmbedding.forward)r'r()	rrr �__doc__�intrr6rJ�
__classcell__r$r$rBr%r&,s��r&cs^eZdZdZed(eeeeee	eeefeeeee30eeee31ee	eeefeeeeeeeeeeeeed��fdd�
�Ze
d)eeeeeed�dd��Zeeeefd�dd��Ze	eeeeffd�dd�Zdd�Zd d!�Zd*d"d#�Zd+eje	eje32efejeje33eejeejeejeeeefee	eefd%�d&d'�Z�ZS),�ControlNetModelT�r�rrrrF�i@i��rRrrrFr*��h㈵��>rR�N�default�rgbr(��in_channels�flip_sin_to_cos�34freq_shift�down_block_types�only_cross_attentionr/�layers_per_block�downsample_padding�mid_block_scale_factor�act_fn�norm_num_groups�norm_eps�cross_attention_dim�attention_head_dim�use_linear_projection�class_embed_type�num_class_embeds�upcast_attention�resnet_time_scale_shift�%projection_class_embeddings_input_dim�%controlnet_conditioning_channel_order�#conditioning_embedding_out_channelsc$st���t|�t|�kr0td|�d|�d���t|t�s`t|�t|�kr`td|�d|�d���t|t�s�t|�t|�kr�td|�d|�d���d}|dd}tj||d	||d35�|_	|d	d}t36|d	||�|_|d	}t|||37d�|_
|du�r|du�rt�||�|_nb|d
k�r*t||�|_nJ|dk�rDt�||�|_n0|dk�rn|du�r`td��t||�|_nd|_t|d	|d�|_t�g�|_t�g�|_t|t��r�|gt|�}t|t��r�|ft|�}|d	}tj||dd�}t|�}|j�|�t|�D]�\}}|}||}|t|�dk} t|||||| ||38||
||||||||d�}!|j�|!�t|�D]*}"tj||dd�}t|�}|j�|��qp| �stj||dd�}t|�}|j�|��q|d}#tj|#|#dd�}t|�}||_t|#|||39|	||
|d|||d�|_dS)NzbMust provide the same number of `block_out_channels` as `down_block_types`. `block_out_channels`: z. `down_block_types`: �.zfMust provide the same number of `only_cross_attention` as `down_block_types`. `only_cross_attention`: zbMust provide the same number of `attention_head_dim` as `down_block_types`. `attention_head_dim`: r'rrrr0rO)r`�timestep�identity�40projectionzX`class_embed_type`: 'projection' requires `projection_class_embeddings_input_dim` be set)r-r/)r1)�41num_layersrX�out_channels�
temb_channels�add_downsample�42resnet_eps�
resnet_act_fn�
resnet_groupsrc�attn_num_head_channelsr^rer\rhrir4)rXrsrurv�output_scale_factorrircrxrwrerh)r5r6r<�43ValueError�44isinstance�boolrLr	r7r8r�	time_projr�time_embedding�	Embedding�class_embedding�Identityr&�controlnet_cond_embeddingr9�down_blocks�controlnet_down_blocksr>r=�	enumeraterr;�controlnet_mid_blockr�	mid_block)$r@rXrYrZr[r\r/r]r^r_r`rarbrcrdrerfrgrhrirjrkrlZconv_in_kernelZconv_in_padding�time_embed_dimZtimestep_input_dim�output_channel�controlnet_blockrA�down_block_type�
input_channel�is_final_block�45down_block�_Zmid_block_channelrBr$r%r6\s�46�����47484950�51���zControlNetModel.__init__)�unetrkrl�load_weights_from_unetcCs�||jj|jj|jj|jj|jj|jj|jj|jj|jj	|jj52|jj|jj|jj
|jj|jj|jj|jj|jj|jj|jj||d�}|r�|j�|j���|j�|j���|j�|j���|jr�|j�|j���|j�|j���|j�|j���|S)a&53        Instantiate Controlnet class from UNet2DConditionModel.54 55        Parameters:56            unet (`UNet2DConditionModel`):57                UNet model which weights are copied to the ControlNet. Note that all configuration options are also58                copied where applicable.59        rW)�configrXrYrZr[r\r/r]r^r_r`rarbrcrdrerfrgrhrirjr8�load_state_dict�60state_dictr}r~r�r�r�)�clsr�rkrlr��61controlnetr$r$r%�	from_unetsB�zControlNetModel.from_unet)�returncsHi}ttjjtttfd��fdd��|��D]\}}�|||�q.|S)z�62        Returns:63            `dict` of attention processors: A dictionary containing all attention processors used in the model with64            indexed by its weight name.65        )�name�module�66processorscsFt|d�r|j||�d�<|��D]\}}�|�d|��||�q"|S�N�
set_processorz67.processorrm)�hasattr�	processor�named_children)r�r�r��sub_name�child��fn_recursive_add_processorsr$r%r�Fs6869zDControlNetModel.attn_processors.<locals>.fn_recursive_add_processors)�strr!r	�Modulerrr�)r@r�r�r�r$r�r%�attn_processors;s70	"	zControlNetModel.attn_processors)r�cs~t|j���}t|t�rDt|�|krDtdt|��d|�d|�d���ttjj	d��fdd��|�71�D]\}}�|||�qddS)	a�72        Parameters:73            `processor (`dict` of `AttentionProcessor` or `AttentionProcessor`):74                The instantiated processor class or a dictionary of processor classes that will be set as the processor75                of **all** `Attention` layers.76            In case `processor` is a dict, the key needs to define the path to the corresponding cross attention processor. This is strongly recommended when setting trainable attention processors.:77 78        z>A dict of processors was passed, but the number of processors z0 does not match the number of attention layers: z. Please make sure to pass z processor classes.)r�r�csbt|d�r6t|t�s |�|�n|�|�|�d���|��D]\}}�|�d|��||�q>dSr�)r�r{�dictr��popr�)r�r�r�r�r���fn_recursive_attn_processorr$r%r�fs7980zGControlNetModel.set_attn_processor.<locals>.fn_recursive_attn_processorN)r<r��keysr{r�rzr�r!r	r�r�)r@r��countr�r�r$r�r%�set_attn_processorUs	���81z"ControlNetModel.set_attn_processorcCs|�t��dS)ze82        Disables custom attention processors and sets the default attention implementation.83        N)r�r)r@r$r$r%�set_default_attn_processortsz*ControlNetModel.set_default_attn_processorc84sJg�tjjd���fdd��|��D]}�|�q$t��}|dkrRdd��D�}n|dkrd|dg}t|t�sx||gn|}t|�t��kr�td	t|��d85|j�dt���dt���d
�	��t	t|��D]:}||}�|}|dur�||kr�td|�d|�d
���q�tjjt86td��fdd��tt|��}|��D]}�||��q4dS)a87        Enable sliced attention computation.88 89        When this option is enabled, the attention module will split the input tensor in slices, to compute attention90        in several steps. This is useful to save some memory in exchange for a small speed decrease.91 92        Args:93            slice_size (`str` or `int` or `list(int)`, *optional*, defaults to `"auto"`):94                When `"auto"`, halves the input to the attention heads, so attention will be computed in two steps. If95                `"max"`, maximum amount of memory will be saved by running only one slice at a time. If a number is96                provided, uses as many slices as `attention_head_dim // slice_size`. In this case, `attention_head_dim`97                must be a multiple of `slice_size`.98        )r�cs0t|d�r��|j�|��D]}�|�qdS�N�set_attention_slice)r�r=�sliceable_head_dim�children)r�r�)�$fn_recursive_retrieve_sliceable_dims�sliceable_head_dimsr$r%r��s99zQControlNetModel.set_attention_slice.<locals>.fn_recursive_retrieve_sliceable_dims�autocSsg|]}|d�qS)rr$)�.0�dimr$r$r%�100<listcomp>��z7ControlNetModel.set_attention_slice.<locals>.<listcomp>�maxrzYou have provided z, but z has zH different attention layers. Make sure to match `len(slice_size)` to be rmNzsize z has to be smaller or equal to )r��101slice_sizecs4t|d�r|�|���|��D]}�||�q dSr�)r�r�r�r�)r�r�r�)� fn_recursive_set_attention_slicer$r%r��s102zMControlNetModel.set_attention_slice.<locals>.fn_recursive_set_attention_slice)
r!r	r�r�r<r{�listrzr�r;rrL�reversed)r@r�r�Znum_sliceable_layersrA�sizer�Zreversed_slice_sizer$)r�r�r�r%r�{s2103104��z#ControlNetModel.set_attention_slicecCst|ttf�r||_dSrD)r{rr�gradient_checkpointing)r@r��valuer$r$r%�_set_gradient_checkpointing�sz+ControlNetModel._set_gradient_checkpointing��?)�samplern�encoder_hidden_states�controlnet_cond�conditioning_scale�class_labels�
timestep_cond�attention_mask�cross_attention_kwargs�return_dictr�csd|jj}|dkrn(|dkr,tj|dgd�}ntd|����|dur`d|�|j�d}|�d�}|}t�|�s�|j	j105dk}
t|t�r�|
r�tj
ntj}n|
r�tjntj}tj|g||j	d�}nt|j�d	kr�|d�|j	�}|�|jd	�}|�|�}|j|jd106�}|�||�}|jdu�r`|du�r,td��|jjdk�rD|�|�}|�|�j|jd107�}||}|�|�}|�|�}||7}|f}|jD]J}t|d
��r�|j�r�||||||	d�\}}n|||d�\}}||7}�q�|jdu�r�|j|||||	d�}d}t||j �D]\}}||�}||f7}�q|}|�!|�}�fdd�|D�}|�9}|108�sX||fSt"||d�S)NrVZbgrr)�dimsz1unknown `controlnet_conditioning_channel_order`: g����mps)�dtype�devicer)r�z9class_labels should be provided when num_class_embeds > 0rn�has_cross_attention)�
hidden_states�tembr�r�r�)r�r�)r�r�r�r$csg|]}|��qSr$r$)r�r��r�r$r%r�/r�z+ControlNetModel.forward.<locals>.<listcomp>)rr)#r�rkr!�fliprz�tor��	unsqueeze�	is_tensorr��typer{�float�float32�float64�int32�int64�tensorr<�shape�expandr}r~r�rfr8r�r�r�r�r��zipr�r�r)r@r�rnr�r�r�r�r�r�r�r�Z
channel_order�	timesteps�is_mpsr��t_emb�embZ	class_embrZdownsample_blockZres_samplesZ!controlnet_down_block_res_samplesZdown_block_res_sampler�rr$r�r%rJ�s�109110111112113114115116117��118119�zControlNetModel.forward)rOTrrPFrQrrrrFr*rSrRrTFNNFrUNrVr()rVr(T)F)r�NNNNT)rrr � _supports_gradient_checkpointingr
rLr|rr�rr�rr6�classmethodrr��propertyrrr�r�r�r�r�r!�FloatTensorr"rrrJrMr$r$rBr%rNYs��120�)�121�5A122 123�124�rNcCs|��D]}tj�|�q|SrD)�125parametersr	�init�zeros_)r��pr$r$r%r>:sr>)+�dataclassesr�typingrrrrrrr!r	�torch.nnr126rE�configuration_utilsrr
�utilsrr�attention_processorrr�127embeddingsrr�modeling_utilsr�unet_2d_blocksrrrr�unet_2d_conditionr�128get_loggerr�loggerrr�r&rNr>r$r$r$r%�<module>s( 129-d