CoolFace
Apppublic

fred-dev/comfy_ui_ali

sourceHugging Facemitupdated 1y agoView on Hugging Face
0likes
controlnet.cpython-312.pyc197 linesDownload Raw Back to __pycache__
1�

2���g������dZddlZddlmZddlZddlZddlZddlZddl	Zddl3ZddlZddlZddl
ZddlZddlZddlZddlZddlZddlZddlZddlmZerddlmZd�ZGd�de�ZGd	�d4�ZGd�de�ZGd
�d�ZGd�de�Zifd�Zd�Z ifd�Z!Gd�de�Z"ifd�Z#ifd�Z$difd�Z%ifd�Z&d�Z'difd�Z(difd�Z)Gd�de�Z*ifd �Z+y)!a�5    This file is part of ComfyUI.6    Copyright (C) 2024 Comfy7 8    This program is free software: you can redistribute it and/or modify9    it under the terms of the GNU General Public License as published by10    the Free Software Foundation, either version 3 of the License, or11    (at your option) any later version.12 13    This program is distributed in the hope that it will be useful,14    but WITHOUT ANY WARRANTY; without even the implied warranty of15    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the16    GNU General Public License for more details.17 18    You should have received a copy of the GNU General Public License19    along with this program.  If not, see <https://www.gnu.org/licenses/>.20�N)�Enum)�
TYPE_CHECKING)�	HookGroupc�L�|jd}|dk(r|S||z}|d|}||jdkDrBtj|g||jdzz|d||jdzgzd��}|jd}||k(r|Stj|g|zd��S)Nr���dim)�shape�torch�cat)�tensor�target_batch_size�batched_number�current_batch_size�	per_batchs     �[/Users/fredrodrigues/Library/CloudStorage/Dropbox/Code/comfyui_hardware/comfy/controlnet.py�broadcast_image_tor,s������a����Q���
�!�^�3�I�
�J�Y�
�F��6�<�<��?�"����F�8�y�F�L�L��O�'C�D��Om�QZ�]c�]i�]i�jk�]l�Ql�Hn�Go�o�uv�w�����a����.�.��
��y�y�&��N�2��:�:�c��eZdZdZdZy)�StrengthTyper�N)�__name__�21__module__�__qualname__�CONSTANT�	LINEAR_UP�rrrr>s���H��Irrc�Z�eZdZd�Zdddgfd�Zd�Zd�Zd�Zd	�Zd22�Z	d�Z23d�Zd
�Zdd�Z
y)�ControlBasec�,�d|_d|_d|_d|_d|_d|_d|_d|_d|_d|_	i|_24d|_g|_tj|_d|_g|_d|_d|_d�|_y)N��?�gr!F��
nearest-exactc��|S�Nr��as r�<lambda>z&ControlBase.__init__.<locals>.<lambda>Vs��!r)�cond_hint_original�	cond_hint�strength�timestep_percent_range�
latent_format�vae�global_average_pooling�timestep_range�compression_ratio�upscale_algorithm�25extra_args�previous_controlnet�extra_condsrr�
strength_type�concat_mask�extra_concat_orig�extra_concat�extra_hooks�preprocess_image��selfs r�__init__zControlBase.__init__Cs���"&��������
�&0��#�!������&+��#�"���!"���!0������#'�� ����)�2�2��� ���!#��� ���&*��� +��rr!r"Nc�Z�||_||_||_|j�|�t	j26d�||_|j�|_|jrJt|j�dk(r2|jjtjdgggg��|S)NzXWARNING: no VAE provided to the controlnet apply node when this controlnet requires one.rr!)r*r,r-r.�logging�warningr/�copyr9r8�len�appendrr
)r>r+r,r-r/r:s      r�
set_cond_hintzControlBase.set_cond_hintXs���"+��� ��
�&<��#����)��{���� z�{��D�H�!-�!2�!2�!4�������D�$:�$:� ;�q� @��"�"�)�)�%�,�,�3�%��	�{�*C�D��rc��||jd�||jd�f|_|j�|jj||�yy)Nrr)r-r1r5�pre_run)r>�model�percent_to_timestep_functions   rrHzControlBase.pre_runesr��;�D�<W�<W�XY�<Z�[�]y�z~�{V�{V�WX�{Y�^Z�[����#�#�/��$�$�,�,�U�4P�Q�0rc��||_|Sr&)r5)r>�27controlnets  r�set_previous_controlnetz#ControlBase.set_previous_controlnetjs��#-�� ��rc�z�|j�|jj�d|_d|_d|_yr&)r5�cleanupr+r:r1r=s rrOzControlBase.cleanupns7���#�#�/��$�$�,�,�.���� ���"��rc�\�g}|j�||jj�z
}|Sr&)r5�28get_models�r>�outs  rrQzControlBase.get_modelsvs1�����#�#�/��4�+�+�6�6�8�8�C��29rc��g}|j�|j|j�|j�||jj�z
}|Sr&)r;rEr5�get_extra_hooksrRs  rrUzControlBase.get_extra_hooks|sO�������'��J�J�t�'�'�(��#�#�/��4�+�+�;�;�=�=�C��30rc��|j|_|j|_|j|_|j|_|j|_|j31|_|j|_|jj�|_|j|_	|jj�|_32|j|_|j|_|jj�|_
|jr|jj�nd|_|j |_yr&)r*r,r-r0r2r3r.r4rCr/r6r7r8r9r;�cloner<�r>�cs  r�copy_tozControlBase.copy_to�s���#�6�6����]�]��33�#'�#>�#>�� �#'�#>�#>�� �"�4�4���"�4�4����,�,������+�+�-���������(�(�-�-�/��
��,�,����(�(��
�"�4�4�9�9�;���48�4D�4D��(�(�.�.�0�$��
�!�2�2��rc�R�|j�|jj|�Sy)Nr)r5�inference_memory_requirements�r>�dtypes  rr\z)ControlBase.inference_memory_requirements�s(���#�#�/��+�+�I�I�%�P�P�rc	���gggd�}|D�]@}||}t�}tt|��D�]}||}	|	��|jrDt	j34|	dd��j
dd|	jd|	jd�}	|	|vr�|j|	�|jtjk(r|	|jz}	nD|jtjk(r'|	|jtt|�|z35�zz}	|� |	j|k7r|	j!|�}	||j#|	�����C|��dD]�}	||	}36tt||	��D]q}||	|}|t|37�k\r|38j#|��+|��.|39|�||40|<�9|41|jd|jdkr||42|z|43|<�g||44|z|45|<�s��|S)	N)�input�middle�output)r�T)r	�keepdimrrrcr)�set�rangerDr0r�mean�repeatr46�addr7rrr,r�floatr^�torE)r>�control�control_prev�output_dtyperS�key�control_output�47applied_to�i�x�o�prev_vals            r�
control_mergezControlBase.control_merge�s����B�"�5���C�$�S�\�N���J��3�~�.�/��"�1�%���=��2�2�!�J�J�q�f�d�C�J�J�1�a�QR�QX�QX�YZ�Q[�]^�]d�]d�ef�]g�h���48�*�"���q�)��-�-��1F�1F�F�����.�A�!�/�/�<�3I�3I�I��$�-�-�5��^�9L�q�9P�3Q�"Q�R�A�#�/�A�G�G�|�4K��D�D��.���C�����"�!0��*�#�2����F���s�<��?�3�4�A�+�A��q�1�H��C��F�{�����*�!�-��Q�4�<�#+�A�a�D� ��t�z�z�!�}�x�~�~�a�/@�@�'/�!�A�$���!��'/�!�A�$���!��5�3��49rc�"�||j|<yr&)r4)r>�argument�values   r�
set_extra_argzControlBase.set_extra_arg�s��$)�����!rr&)rrrr?rFrHrMrOrQrUrZr\rvrzrrrrrBsH��,�*14�J�\`�oq��R�50�#���3�"�51'�R*rrc52�v��eZdZdddddddgejdd�f53�fd�	Zd�Zd�Z�fd	�Z�fd54�Z	�fd�Z55�xZS)�56ControlNetNFr#�yc��|Sr&rr's rr)zControlNet.<lambda>�s	��CDrc�v��t�|��||_||_|�Mtj57j
|j|tjj���|_	||_58||_d|_||_
||_|xj|z
c_||_|	|_|59|_y)N)�load_device�offload_device)�superr?�
control_modelr��comfy�
model_patcher�ModelPatcher�model_management�unet_offload_device�control_model_wrappedr2r0�model_sampling_current�manual_cast_dtyper.r6r7r8r<)r>r�r0r2r.r�r�r6r7r8r<�	__class__s           �rr?zControlNet.__init__�s����
����*���&����$�).�)<�)<�)I�)I�$�J\�J\�ju�GL�G]�G]�Gq�Gq�Gs�)I�*t�D�&�!2���&<��#�&*��#�!2���*������K�'��*���&��� 0��rc	�60�d}|j�|jj|||||�}|j�/|d|jdkDs|d|jdkr|�|Sy|jj}|j61�|j62}|j�m|jd|jz|jjdk7s7|jd|jz|jjdk7�r�|j�|`d|_|j}|j�||jjz}n|j�td��tjj|j |jd|z|jd|z|j"d�|_|j%|j�|_|j�ytj&j)d��}	|jj+|jj-dd	��|_tj&j/|	�|j�*|jj1|j�|_t3|j4�dkD�rg}63|j4D]�}|j7|jj8�}tjj||jjd|jjd|j"d�}|64j;tjj=||jjd����t?j@|jg|65zd�66�|_|jj7|j8|��|_|jd|jjdk7r)tC|j|jd|�|_|jEd|d
�}|jFjI�}
|jJD]+}|jE|d�}|��|j7|�|
|<�-|jLjO|�}|jLjQ||�}|jd|j7|�|j|j7|�|j7|�d�|
��}|jS||d��S)NrrrrczuThis Controlnet needs a VAE but none was provided, please use a ControlNetApply node with a VAE input and connect it.�centerT)�only_currently_used�����r)�devicer^�crossattn_controlnet�c_crossattn)rs�hint�	timesteps�context)rnr)*r5�get_controlr1r�r^r�r+r67r2r/�downscale_ratior.�68ValueErrorr��utils�common_upscaler*r3r<r��
loaded_models�encode�movedim�load_models_gpu�69process_inrDr9rkr�rE�repeat_to_batch_sizerrr�getr4rCr6r��timestep�calculate_inputrv)r>�x_noisy�t�condr�transformer_optionsrmr^r2r��	to_concatrYr��extra�tempr�rls                 rr�zControlNet.get_control�s������#�#�/��3�3�?�?���D�R`�bu�v�L����*���t�d�)�)�!�,�,��!��t�7J�7J�1�7M�0M��+�'�'���"�"�(�(���!�!�-��*�*�E��>�>�!�W�]�]�1�%5��8N�8N�%N�RV�R`�R`�Rf�Rf�gh�Ri�%i�mt�mz�mz�{|�m}�AE�AW�AW�nW�[_�[i�[i�[o�[o�pq�[r�nr��~�~�)��N�!�D�N� $� 6� 6���x�x�#�!�T�X�X�%=�%=�=�!��%�%�1�$�&]�^�^�"�[�[�7�7��8O�8O�QX�Q^�Q^�_`�Qa�du�Qu�w~�xE�xE�FG�xH�K\�x\�^b�^t�^t�v~��D�N�!�2�2�4�>�>�B�D�N��x�x�#� %� 6� 6� D� D�Y]� D� ^�
�!%��������1G�1G��2�1N�!O����&�&�6�6�}�E��!�!�-�!%�!3�!3�!>�!>�t�~�~�!N����4�)�)�*�Q�.��	��/�/�A����T�^�^�2�2�3�A����2�2�1�d�n�n�6J�6J�1�6M�t�~�~�Oc�Oc�de�Of�hl�h~�h~�AI�J�A��$�$�U�[�[�%E�%E�a����I]�I]�^_�I`�%a�b�0�"'���D�N�N�+;�i�+G�Q�!O���!�^�^�.�.�g�n�n�E�.�R�D�N��=�=���t�~�~�3�3�A�6�6�/�����
�
�a�@P�R`�a�D�N��(�(�1�4�
�3F�G�����$�$�&���!�!�A��8�8�A�t�$�D����7�7�5�>��a��"�70�.�.�7�7��:���-�-�=�=�a��I��$�$�$�$�Q�w�z�z�%�'8�t�~�~�Ya�Yd�Yd�ej�Yk�u|�u�u�AF�vG�Q�KP�Q���!�!�'�<�d�!�K�Krc���td|j|j|j��}|j|_|j71|_|j
|�|S�N�r0r�r�)r|r0r�r�r�r�rZrXs  rrCzControlNet.copys`���t�D�4O�4O�]a�]m�]m�BF�BX�BX�
Y���,�,���"&�"<�"<������Q���rc�Z��t�|��}|j|j�|Sr&)r�rQrEr�)r>rSr�s  �rrQzControlNet.get_modelss'����g� �"���72�73�4�-�-�.��74rc�H��t�|�||�|j|_yr&)r�rH�model_samplingr�)r>rIrJr�s   �rrHzControlNet.pre_run s ���
����;�<�&+�&:�&:��#rc�0��d|_t�|�	�yr&)r�r�rO�r>r�s �rrOzControlNet.cleanup$s���&*��#�
���r)rrrrrr?r�rCrQrHrO�
__classcell__�r�s@rr|r|�sr���%)�%�[\�lp�C�W[�jm�in�~J�~S�~S�af�yD�1�"9L�v��75;��rr|c���eZdZGd�dejj76ejj�Z	Gd�dejj77ejj�Z78y)�ControlLoraOpsc	�<��eZdZ		ddedededdf�fd�
Zd�Z�xZS)	�ControlLoraOps.LinearN�in_features�out_features�bias�returnc�v��t�|��||_||_d|_d|_d|_d|_yr&)r�r?r�r��weight�up�downr�)r>r�r�r�r�r^r�s      �rr?zControlLoraOps.Linear.__init__*s:����G���*�D�� ,�D���D�K��D�G��D�I��D�Irc79��tjj||�\}}|j��tj80jj||t	j|jjd��|jjd���j|jj�j|j�z|�Stj81jj|||�S�Nr)�	start_dim)r��ops�cast_bias_weightr�r�nn�82functional�linear�mm�flattenr��reshaper�r83�typer^�r>r`r�r�s    r�forwardzControlLoraOps.Linear.forward4s�� �9�9�5�5�d�E�B�L�F�D��w�w�"��x�x�*�*�1�1�%��5�8�8�D�G�G�O�O�fg�O�Lh�jn�js�js�j{�j{�GH�j{�kI�DJ�CS�CS�TX�T_�T_�Te�Te�Cf�Ck�Ck�lq�lw�lw�Cx�:x�z~����x�x�*�*�1�1�%���F�Fr)TNN)rrr�int�boolr?r�r�r�s@r�Linearr�)s1���MQ�'+�	��	�3�	�d�	�04�	�	Grr�c�6��eZdZ								d�fd�	Zd�Z�xZS)�ControlLoraOps.Conv2dc����t�|��||_||_||_||_||_||_d|_d|_	||_84|	|_d|_d|_
d|_d|_y)NFr)r�r?�in_channels�out_channels�kernel_size�stride�padding�dilation�85transposed�output_padding�groups�padding_moder�r�r�r�)
r>r�r�r�r�r�r�r�r�r�r�r^r�s
            �rr?zControlLoraOps.Conv2d.__init__<sw���
�G���*�D�� ,�D��*�D�� �D�K�"�D�L�$�D�M�#�D�O�"#�D�� �D�K� ,�D���D�K��D�I��D�G��D�Irc86���tjj||�\}}|j��tj87jj||t	j|jjd��|jjd���j|jj�j|j�z||j |j"|j$|j&�Stj88jj||||j |j"|j$|j&�Sr�)r�r�r�r�rr�r��conv2dr�r�r�r�r�r89r�r^r�r�r�r�r�s    rr�zControlLoraOps.Conv2d.forward\s��� �9�9�5�5�d�E�B�L�F�D��w�w�"��x�x�*�*�1�1�%��5�8�8�D�G�G�O�O�fg�O�Lh�jn�js�js�j{�j{�GH�j{�kI�DJ�CS�CS�TX�T_�T_�Te�Te�Cf�Ck�Ck�lq�lw�lw�Cx�:x�z~�@D�@K�@K�MQ�MY�MY�[_�[h�[h�jn�ju�ju�v�v��x�x�*�*�1�1�%���t�{�{�TX�T`�T`�bf�bo�bo�qu�q|�q|�}�}r)rrrrT�zerosNN)rrrr?r�r�r�s@r�Conv2dr�;s)�������� ���	�@	~rr�N)rrrrr��Moduler�r��CastWeightBiasOpr�r�rrrr�r�(sF��G������%�)�)�"<�"<�G�$&~������%�)�)�"<�"<�&~rr�c�F��eZdZdifd�Z�fd�Zd�Z�fd�Zd�Zd�Z�xZ	S)�ControlLoraFc�v�tj|�||_||_|xjdgz
c_y)Nr})rr?�control_weightsr0r6)r>r�r0�
model_optionss    rr?zControlLora.__init__es3�����T�"�.���&<��#����S�E�!�rc	���t�90|�||�|jjj	�}|jd�|jdjd|d<|j|_|j�}|j�)Gd�dttjj�}n4Gd�dttjj�}|j}||d<||d	<tjjj di|��|_|j"j%tj&j)��|j*}|j-�}|D]3}||}		tj.j1|j"||	��5|jD]y}|d91vs�tj.j1|j"||j|j%|�j%tj&j)����{y#Y��xYw)Nr�zinput_hint_block.0.weightr�
hint_channelsc��eZdZy��-ControlLora.pre_run.<locals>.control_lora_opsN�rrrrrr�control_lora_opsr�s���rr�c��eZdZyr�r�rrrr�z-ControlLora.pre_run.<locals>.control_lora_opsvr�r�92operationsr^>�lora_controlnetr)r�rH�model_config�unet_configrC�popr�r93r��	get_dtyper�r�r��disable_weight_init�manual_cast�cldmr|r�rkr��get_torch_device�diffusion_model�94state_dictr��set_attr_param)r>rIrJ�controlnet_configr^r�r��sd�kr�r�s          �rrHzControlLora.pre_runks����
����;�<�!�.�.�:�:�?�?�A�����n�-�-1�-A�-A�B]�-^�-d�-d�ef�-g��/�*�!&�!8�!8������!���!�!�)�
�>�5�9�9�3P�3P�
�
�>�5�9�9�3H�3H�
��*�*�E�*:��,�'�%*��'�"�"�Z�Z�_�_�7�7�L�:K�L��������e�4�4�E�E�G�H��/�/��
�
'�
'�
)���A���U�F�
����*�*�4�+=�+=�q�&�I���%�%�A��+�+����*�*�4�+=�+=�q�$�BV�BV�WX�BY�B\�B\�]b�Bc�Bf�Bf�gl�g}�g}�hO�hO�hQ�CR�S�&��
��s�+H<�<Ic�j�t|j|j��}|j|�|S)N)r0)r�r�r0rZrXs  rrCzControlLora.copy�s*����,�,�T�E`�E`�a�����Q���rc�4��|`d|_t�|�	�yr&)r�r�rOr�s �rrOzControlLora.cleanup�s�����!���
���rc�0�tj|�}|Sr&)rrQrRs  rrQzControlLora.get_models�s���$�$�T�*���95rc���tjj|j�tjj|�ztj||�zSr&)r�r��calculate_parametersr�r��96dtype_sizerr\r]s  rr\z)ControlLora.inference_memory_requirements�sc���{�{�/�/��0D�0D�E��H^�H^�Hi�Hi�jo�Hp�p�s~�t]�t]�^b�di�tj�j�	jr)97rrrr?rHrCrOrQr\r�r�s@rr�r�ds*���?D�TV�"�S�B�98�99�jrr�c�L�tjj|dd�}|jdd�}|�Vtjj|�}t
|j�}tjjd||��}tjj�}tjj||�}|jdd�}|�"tjj||d��}tjj�}	||||||	fS)N�Tr^r���model_params�supported_dtypes�weight_dtype�custom_operations��disable_fast_fp8)r��model_detection�model_config_from_unetr�r�r�list�supported_inference_dtypesr��100unet_dtyper��unet_manual_castr��pick_operationsr�)101rr�r�rrrr�r�r�r�s102          rrr�s���(�(�?�?��B��M�L��"�"�7�D�1�J����{�{�/�/��3��%)�,�*Q�*Q�%R�"��+�+�6�6�B�Ys�CO�6�P�103��(�(�9�9�;�K��.�.�?�?�104�K�X���"�"�#6��=�J����Y�Y�.�.�z�;L�_c�.�d�105��+�+�?�?�A�N���[�*�>O�Q_�_�_rc���|j|d��\}}t|�dkDr$tjdj	|��t|�dkDr$tj106dj	|��|S)NF��strictr�missing controlnet keys: {}�unexpected controlnet keys: {})�load_state_dictrDrArB�format�debug)r�r�missing�107unexpecteds    r�controlnet_load_state_dictr$�sj��'�7�7��5�7�I��G�Z�108�7�|�a�����5�<�<�W�E�F�109�:�����
�
�6�=�=�j�I�J��rc	��tjj|d�}t||��\}}}}}}tjj	|d�}	|D]110}111||112||113<�d}|jd�jd}|dk(rd}tjjjd|	||||d	�|j��}
t|
|�}
tjj�}d114|_t|
d||||��}|S)
Nr�r�zjoint_blocks.{}.F�pos_embed_input.proj.weightr�T)�115num_blocks�control_latent_channelsr�r�r^r)r2r.r8r�r�r)r�r�convert_diffusers_mmditr�count_blocksr�r116r��mmditr|r�r$�latent_formats�SD3�shift_factor)rr��new_sdr�r�r�rr�r�r)rr8r*r�r.rls                r�load_controlnet_mmditr2�sB��
�
"�
"�
:�
:�2�r�
B�F�[l�ms�DQ�\R�X�L�*�k�:�7H�.��&�&�3�3�F�<N�O�J�
���q�E��q�	���K�$�j�j�)F�G�M�M�a�P���"�$����J�J�$�$�/�/�T�:�_v�DN�We�mw�T�{G�{S�{S�T�M�.�}�f�E�M��(�(�,�,�.�M�!"�M����!�=�fq�@K�_p�q�G��Nrc�$��eZdZ�fd�Zd�Z�xZS)�ControlNetSD35c���|jjrM|jjj|jj117j
�d��\}}nL|jjj|jjj
�d��\}}t�|�%||�y)NFr)118r��double_y_emb�orig_y_embedderrr��119y_embedderr��120x_embedderr�rH)r>rIrJr"r#r�s     �rrHzControlNetSD35.pre_run�s�������*�*�"&�"4�"4�"D�"D�"T�"T�UZ�Uj�Uj�Uu�Uu�VA�VA�VC�LQ�"T�#R��G�Z�"&�"4�"4�"?�"?�"O�"O�PU�Pe�Pe�Pp�Pp�P{�P{�P}�GL�"O�#M��G�Z�
����;�<rc���td|j|j|j��}|j|_|j121|_|j
|�|Sr�)r4r0r�r�r�r�rZrXs  rrCzControlNetSD35.copy�s`���4��8S�8S�ae�aq�aq�FJ�F\�F\�
]���,�,���"&�"<�"<������Q���r)rrrrHrCr�r�s@rr4r4�s
���=�rr4c��d}d|vr(t|jd�j��}|dk(}|dk(}i}tjj122D]$}|d|vs�|j|d�||d<�&|D]123}||||<�|}|dj}|ddz}d|z}	|}124|	|125z}tjj|d�}tjj�}
tjj�}tjjd�	�}tjj||
�}|jd126d�}|�"tjj!||d��}tj"j$j'ddd
|||d|dk(||127d|||��}t)||�}tj*j-�}d�}|rd�}n|rd�}t/|d||
||��}|S)Nr��control_typerrrzy_embedder.mlp.0.weight�@ztransformer_blocks.{}.)r
rTr�i)�img_size�128patch_size�in_chans�129num_layers�main_model_doubler6�attention_head_dim�num_attention_heads�adm_in_channelsr�r^r�c��|Sr&rr's rr)z&load_controlnet_sd35.<locals>.<lambda>s��rc��|dzdzdzS)N��g�?rr's rr)z&load_controlnet_sd35.<locals>.<lambda>s��a�#�g��m�c�&9rc��d|z130S)Nr!rr's rr)z&load_controlnet_sd35.<locals>.<lambda>s��S�1�Wr)r2r.r�r�r<)�roundr��itemr�r��MMDIT_MAP_BASICr131rr,r�r�r�rrr�r�rr��dit_embedder�ControlNetEmbedderr$r.r/r4)rr�r<�132canny_cnet�133depth_cnetr1r�y_emb_shape�depth�hidden_size�	num_heads�head_dimr)r�r�rr�r�r�r.r<rls                      r�load_controlnet_sd35rW�s1���L�����R�V�V�N�3�8�8�:�;����"�J���"�J�
�F�
�[�[�
(�
(���Q�4�2�:��6�6�!�A�$�<�F�1�Q�4�L�)����q�E��q�	��	�B��.�/�5�5�K���N�b� �E��u�*�K��I��i�'�H��&�&�3�3�F�<T�U�J��(�(�9�9�;�K��+�+�?�?�A�N��'�'�2�2��2�C�J��.�.�?�?�134�K�X���"�"�#6��=�J����Y�Y�.�.�z�;L�_c�.�d�135��J�J�+�+�>�>��JK�HJ�JT�QV�LW�XY�N�^i�jk�^l�Ll�RZ�S\�OS�FT�EO�JT�?�V�M�/�}�b�A�M��(�(�,�,�.�M�"���:��	�,���]�a�}�ju�J[�n~��G��Nrc	�6�t||��\}}}}}}tjjjj|||��}t
||�}tjj�}	gd�}136t|d|	|||137tj��}|S)Nr&)r�r�r^)�text_embedding_mask�encoder_hidden_states_t5�text_embedding_mask_t5�image_meta_size�style�cos_cis_img�sin_cis_imgr)r2r.r�r�r6r7)rr��ldm�hyditrL�HunYuanControlNetr$r.�SDXLr|rr)�controlnet_datar�r�r�r�rr�r�r�r.r6rls            r�load_controlnet_hunyuanditres���[l�m|�MZ�\[�X�L�*�k�:�7H�.��I�I�O�O�.�.�@�@�J�_m�u�@�A�M�.�}�o�N�M��(�(�-�-�/�M�Z�K���!�=�fq�FW�ep�@L�@U�@U�V�G��NrFc���t||��\}}}}}}tjjjj138d||||d�|j��}	t|	|�}	ddg}139t|	|||140��}|S)Nr&)�	mistoliner�r�r^r}�guidance)r�r�r6r)	rr�r`�fluxrL�ControlNetFluxr�r$r|)rrgr�r�r�r�rr�r�r�r6rls            r�$load_controlnet_flux_xlabs_mistolinerk(s���[l�mo�@M�\N�X�L�*�k�:�7H�.��I�I�N�N�-�-�<�<�n�y�]g�p~�GQ�n�Ua�Um�Um�n�M�.�}�b�A�M��141�#�K���K�Sd�r}�~�G��Nrc142��tjj|d�}t||��\}}}}}}|D]143}	||	||	<�d}144d}||vr||jd}145|jd�jddz}d}
|d	k(rd146}
tjjjjdd147|148||||d�|j��}t||�}tjj�}dd
g}t|d||
|||��}|S)Nrr&rzcontrolnet_mode_embedder.weightzpos_embed_input.weightr�Fr(T)�latent_input�num_union_modesr*r�r�r^r}rh)r2r.r8r�r�r6r)r�rr+rr149r�r`rirLrjr�r$r.�Fluxr|)rr�r1r�r�r�rr�r�rro�150union_cnetr*r8r�r.r6rls                  r�load_controlnet_flux_instantxrr0si��
�
"�
"�
:�
:�2�r�
B�F�[l�ms�DQ�\R�X�L�*�k�:�7H�.�
���q�E��q�	���O�2�J��V�� ��,�2�2�1�5��$�j�j�)A�B�H�H��K�q�P���K��"�$����I�I�N�N�-�-�<�<�~�$�`o�Ja�nx�AO�Wa�~�eq�e}�e}�~�M�.�}�f�E�M��(�(�-�-�/�M��151�#�K���!�=�fq�@K�_p�~I�J�G��Nrc�F�tjj|ddi�S)Nzsingle_controlnet_blocks.zcontrolnet_single_blocks.)r�r��state_dict_prefix_replace)rs r�convert_mistolineruHs ���;�;�0�0��6Q�Sn�5o�p�prc�"�|}d|j�vr
t||��Sd|vr
t||��Sd}d}d|v�r�tjj|�}tjj|�}d|d<d|d<d	}d152}|rDddg}	|	D]3}153d
j||154�}dj||155�}||vrd}n|||<�5|dz
}|r�Dd	}d156}|rpddg}	|	D]_}157|d	k(rdj|158�}ndj|dz159|160�}dj|dz|161�}||vrdj|162�}d}|||<�a|dz
}|r�pi}
|D]}||vs�|j|�|
||<� d|vrY|djd	|d<t|j��D](}|jdd�}|j|�|
|<�*|j�}t|�d	kDr$tjdj|��|
}ncd|vrDd|vr
t!||��Sd|vrd|vr
t#||��St%||��Sd |vr(t'||��Sd!|vrt!t)|�d163|�"�Sd#}d}d$}||vrd164}|}d%}n-||vrd&}n&t+||��}|�tj,d'�|S|�Btjj/||d165�}t|j0�}|j2}|j5d(d�}|�btjj7|�}|�tj8j;�g}tj8j;d)||�*�}tj8j=�}tj8j?||�}|j5d+d�}|� tj@jC||�}||d,<||d(<tj8jE�|d-<|jd.�|d/j|�jd|d0<tjFjFjHd;i|��}|�rd1|vr�|��tj8jK|g�|jM�}|D]w}d%}|jO|�s�d2j|t|�d�} | |vs�9||}!|!|| jQ|!jR�jU|!jV�z
}!�yntjd3�Gd4�d5tXjZj\�}"|"�}#||#_/|#ja|d�6�\}$}%n|ja|d�6�\}$}%t|$�d	kDr$tjd7j|$��t|%�d	kDr$tjbd8j|%��|j5d9d�}&tI||&||�:�}'|'S)<Nzafter_proj_list.18.biasr&r�z(controlnet_cond_embedding.conv_in.weightzmiddle_block_out.0.weightzcontrolnet_mid_block.weightzmiddle_block_out.0.biaszcontrolnet_mid_block.biasrTz.weightz.biaszcontrolnet_down_blocks.{}{}zzero_convs.{}.0{}Frz#controlnet_cond_embedding.conv_in{}z%controlnet_cond_embedding.blocks.{}{}zinput_hint_block.{}{}rz$controlnet_cond_embedding.conv_out{}z#control_add_embedding.linear_1.bias�task_embedding�!union_controlnet_num_control_typez.attn.in_proj_z.attn.in_proj.zleftover keys: {}zcontrolnet_blocks.0.weightz,double_blocks.0.img_attn.norm.key_norm.scaler'z,transformer_blocks.0.adaLN_modulation.1.biaszcontrolnet_x_embedder.weightz!controlnet_blocks.0.linear.weight)rgr�z#control_model.zero_convs.0.0.weightzzero_convs.0.0.weightzcontrol_model.rz*error could not detect control model type.r^r�rrr�r�r�z{}input_hint_block.0.weightr��166differencezdiffusion_model.{}zPWARNING: Loaded a diff controlnet without a model. It will very likely not work.c��eZdZy)�1load_controlnet_state_dict.<locals>.WeightsLoaderNr�rrr�
WeightsLoaderr{�s��rr|rrrr0r�r)2�keysrer�r�r�unet_config_from_diffusers_unetr��unet_to_diffusersr r�r167r�replacerDrArBrkrWr2rrru�load_t2i_adapter�errorrrr�r�rr�rr�rr�rr�r�r|r��model_state_dict�168startswithr�r^rkr�rr�r�r�rr!)(r�rIr�rdrr�diffusers_keys�count�loop�suffix�s�k_in�k_outr1r�new_k�
leftover_keys�pth_key�pthro�prefix�netr�rrr�r�r�r��model_sdrs�c_m�sd_key�cdr|�wr"r#r0rls(                                        r�load_controlnet_state_dictr�Lsh�� �O� �O�$8�$8�$:�:�)�/��W�W��O�+��?�-�H�H���!%��1�_�D�!�1�1�Q�Q�Ra�b�����6�6�7H�I��8S��4�5�6O��2�3��������)�F���4�;�;�E�1�E��+�2�2�5�!�<����.� �D��',��t�$�
�
�Q�J�E���������)�F����A�:�@�G�G��J�D�B�I�I�%�RS�)�UV�W�D�/�6�6�u�q�y�!�D����.�A�H�H��K�D� �D�',��t�$��
�Q�J�E�����A��O�#�,;�,?�,?��,B��~�a�(�)� �1�O�C�ET�Ue�Ef�El�El�mn�Eo��A�B��/�.�.�0�1���	�	�"2�4D�E�� /� 3� 3�A� 6��u�
�2�(�,�,�.�
��}���!��O�O�/�6�6�}�E�F� ��	%��	8�9�_�L�7��Wd�e�e�
*�o�
=�=��P�+�O�=�Y�Y�,�_�M�Z�Z�
+��
>�0��P]�^�^�	,��	?�3�4E�o�4V�bf�wD�E�	E�3�G�169�C�170!�C��/�!�����!��	��	�����m�L���;��M�M�F�G��171�� ��,�,�C�C�O�U[�]a�b��%)�,�*Q�*Q�%R�"�(�4�4���"�"�7�D�1�J����{�{�/�/��@��%�-�*/�*@�*@�*K�*K�*M�)N�&��+�+�6�6�B�Ys�CO�6�P�172��(�(�9�9�;�K��.�.�?�?�173�K�X���"�"�#6��=�J����Y�Y�.�.�z�;L�M�174�&0��l�#�!+��g��"'�"8�"8�"L�"L�"N��h�����.�)�)8�9V�9]�9]�^d�9e�)f�)l�)l�mn�)o��o�&��J�J�O�O�.�.�C�1B�C�M�175��?�*�� ��&�&�6�6��w�?� �1�1�3��(�A�*�C��|�|�C�(�!5�!<�!<�Q�s�3�x�y�\�!J��!�X�-�!0��!3�B��(�6�"2�"7�"7����"A�"D�"D�R�Y�Y�"O�O�B�
)���� r�s�	�E�H�H�O�O�	��O��'����/�/���/�N����+�;�;�O�TY�;�Z����176�7�|�a�����5�<�<�W�E�F�177�:�����
�
�6�=�=�j�I�J�*�.�.�/G��O����?U�cn�CT�U�G��Nrc�F�d|vrItjj|�d}|jd�s|jd�rd|d<t	t178jj|d��||��}|�$tjdj|��|S)	Nr0r�_shuffle�
_shuffle_fp16T)�	safe_load)rIr�zCerror checkpoint does not contain controlnet or t2i adapter data {})�os�path�splitext�endswithr�r�r��load_torch_filerAr�r )�	ckpt_pathrIr��filename�cnets     r�load_controlnetr��s����}�4��7�7�#�#�I�.�q�1�����Z�(�H�,=�,=�o�,N�6:�M�2�3�%�e�k�k�&A�&A�)�W[�&A�&\�di�zG�H�D��|��
�
�[�b�b�cl�m�n��Krc�2��eZdZd�fd�	Zd�Zd�Zd�Z�xZS)�179T2IAdapterc���t�|��||_||_d|_||_||_|�tjj�}||_180yr&)r�r?�	t2i_model�channels_in�
control_inputr2r3r�r�r�r�)r>r�r�r2r3r�r�s      �rr?zT2IAdapter.__init__�sU���
����"���&���!���!2���!2����>��+�+�<�<�>�F���rc��|jj}tj||z�|z}tj||z�|z}||fSr&)r��unshuffle_amount�math�ceil)r>�width�heightr�s    r�scale_image_tozT2IAdapter.scale_image_to�sP���>�>�:�:���	�	�%�"2�2�3�6F�F�����6�$4�4�5�8H�H���f�}�rc��d}|j�|jj|||||�}|j�/|d|jdkDs|d|jdkr|�|Sy|j�m|jd|j181z|jjdk7s7|jd|j182z|jjdk7�r|j�|`d|_d|_|j|jd|j183z|jd|j184z�\}}tjj|j|||jd�j�j|j�|_|j dk(rC|jjddkDr't#j$|jdd��|_|jd|jjdk7r)t'|j|jd|�|_|j��|j(j|j*�|j(j|j�|j)|jj|j*��|_|j(j-�i}	|jD](}185t/t1d�|j|186��|	|187<�*|j3|	||j*�S)	Nrrrrcr�T)rdc�*�|�dS|j�Sr&)rWr's rr)z(T2IAdapter.get_control.<locals>.<lambda>s��!�)�$�2R�����2Rr)r5r�r1r+r188r2r�r�r�r�r�r*r3rjrkr�r�rrgrr�r^�cpur�maprv)r>r�r�r�rr�rmr�r�r�rs           rr�zT2IAdapter.get_control�s.�����#�#�/��3�3�?�?���D�R`�bu�v�L����*���t�d�)�)�!�,�,��!��t�7J�7J�1�7M�0M��+�'�'���>�>�!�W�]�]�1�%5��8N�8N�%N�RV�R`�R`�Rf�Rf�gh�Ri�%i�mt�mz�mz�{|�m}�AE�AW�AW�nW�[_�[i�[i�[o�[o�pq�[r�nr��~�~�)��N�!%�D��!�D�N� �/�/��
�
�a�0@�4�CY�CY�0Y�[b�[h�[h�ij�[k�nr�oE�oE�\E�F�M�E�6�"�[�[�7�7��8O�8O�QV�X^�`d�`v�`v�yA�B�H�H�J�M�M�NR�NY�NY�Z�D�N����1�$����)=�)=�a�)@�1�)D�!&���D�N�N�A�t�!L����=�=���t�~�~�3�3�A�6�6�/�����
�
�a�@P�R`�a�D�N����%��N�N���g�m�m�,��N�N���d�k�k�*�!%������0A�0A�'�-�-�0P�!Q�D���N�N��� ��
��#�#�A�#�C�(R�TX�Tf�Tf�gh�Ti�$j�k�M�!��$��!�!�-��w�}�}�M�Mrc��t|j|j|j|j�}|j|�|Sr&)r�r�r�r2r3rZrXs  rrCzT2IAdapter.copy"s9���t�~�~�t�'7�'7��9O�9O�QU�Qg�Qg�h�����Q���rr&)rrrr?r�r�rCr�r�s@rr�r��s���	��!N�Frr�c	���d}d}d|vr|d}d|vr�i}td�D]b}td�D],}dj|dz|z�|dj||�<�.dj|dz�|d	j|�<�dd189|d<tjj	||�}|j�}d|vrB|djd
}tjjj|gd�d��}	�nSd|vr�|djd
}|djd}190|djd}d}ttd�|��}
t|
�dkDrd}d}|dk(s|dk(rd}tjjj||191|192dz|193dz|194dzgddd|d||��}	n�d|vrLtjjj j#|djd
gd���}	d}d}nRd|vrMtjjj j#|djd
d gd��!�}	d
}d}ny|	j%|�\}}t|�dkDr$t'j(d"j|��t|�dkDr$t'j*d#j|��t-|	|	j.||�S)$Nr#r$�adapterz&adapter.body.0.resnets.0.block1.weightrmrzbody.{}.zadapter.body.{}.resnets.{}.zadapter.body.{}.rzadapter.zbody.0.in_conv.weightr)i@i��r�)�cin�channels�nums_rbzconv_in.weightrzbody.0.block2.weightFc�$�|jd�S)Nzdown_opt.op.weight)r�r's rr)z"load_t2i_adapter.<locals>.<lambda>?s��!�*�*�5I�*JrT�i)r�r�r��ksize�sk�use_conv�xlzbackbone.0.0.weight)rrmr#��3�7�;�?)�c_in�proj_blocks� �bilinearzbackbone.10.blocks.0.weightzbackbone.0.weight�large)r��bottleneck_moder�zt2i missing {}zt2i unexpected {})rfr r�r�rtr}r195�t2i_adapterr��
Adapter_lightr�filterrD�Adapterr`�cascaderLr|rrArBr!r��input_channels)�t2i_datar�r2r3�prefix_replacerr�jr}r��model_ad�channelr�r��	down_optsr�r"r#s                 rr�r�'s(����'���H���I�&��/�8�;����q��A��1�X��MW�M^�M^�_`�cd�_d�gh�_h�Mi��<�C�C�A�q�I�J��=G�=N�=N�q�ST�u�=U�N�-�4�4�Q�9�:��&(��z�"��;�;�8�8��>�R���=�=�?�D��$�&��.�/�5�5�a�8���$�$�,�,�:�:�s�Mc�mn�:�o��	�T�	!��'�(�.�.�q�1���+�,�2�2�1�5���/�0�6�6�q�9������ J�D�Q�R�	��y�>�A���H�
���#�:�����B��$�$�,�,�4�4���QX�YZ�QZ�\c�de�\e�gn�op�gp�Gq�rt�st�Gu�@A�IN�SW�bj�oq�4�r��	�$�	&��9�9�$�$�/�/�:�:��I^�@_�@e�@e�fg�@h�wT�:�U����&��	&�$�	.��9�9�$�$�/�/�:�:��I\�@]�@c�@c�de�@f�x�Nk�:�l����+���"�2�2�8�<��G�Z�196�7�|�a�����(�/�/��8�9�197�:�����
�
�)�0�0��<�=��h�� 7� 7�9J�L]�^�^r),�__doc__r�enumrr�r�rA�comfy.utilsr��comfy.model_management�comfy.model_detection�comfy.model_patcher�	comfy.ops�comfy.latent_formats�comfy.cldm.cldm�comfy.t2i_adapter.adapter�comfy.ldm.cascade.controlnet�comfy.cldm.mmdit�comfy.ldm.hydit.controlnet�comfy.ldm.flux.controlnet�comfy.cldm.dit_embedder�typingr�comfy.hooksrrrrr|r�r�rr$r2r4rWrerkrrrur�r�r�r�rrr�<module>r�s���&
���	��������� �#��!� �� ��%�;�$�4��B*�B*�J_��_�B9~�9~�x7j�*�7j�r)+�`�(�-/��*
�Z�
�,.�9�z?A�	�8=�B��57��0q�26�R�T�l&*��	�8��8�t.0�1_r