CoolFace
Apppublic

fred-dev/comfy_ui_ali

sourceHugging Facemitupdated 1y agoView on Hugging Face
0likes
lora.cpython-312.pyc161 linesDownload Raw Back to __pycache__
1�

2���g	x���dZddlmZddlZddlZddlZddlZddlZdddddd	d3�Z	dd�Z4ifd�Zifd
�Zd�Z
dd�Zejdfd�Zy)a�5    This file is part of ComfyUI.6    Copyright (C) 2024 Comfy7 8    This program is free software: you can redistribute it and/or modify9    it under the terms of the GNU General Public License as published by10    the Free Software Foundation, either version 3 of the License, or11    (at your option) any later version.12 13    This program is distributed in the hope that it will be useful,14    but WITHOUT ANY WARRANTY; without even the implied warranty of15    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the16    GNU General Public License for more details.17 18    You should have received a copy of the GNU General Public License19    along with this program.  If not, see <https://www.gnu.org/licenses/>.20�)�annotationsN�mlp_fc1�mlp_fc2�self_attn_k_proj�self_attn_q_proj�self_attn_v_proj�self_attn_out_proj)zmlp.fc1zmlp.fc2zself_attn.k_projzself_attn.q_projzself_attn.v_projzself_attn.out_projc���i}t�}|D�]�}dj|�}d}||j�vr$||j�}|j	|�dj|�}d}	||j�vr||}	|j	|�dj|�}21d}|22|j�vr%	||23j�}|j	|24�dj|�}dj|�}
dj|�}dj|�}dj|�}d	j|�}d}||j�vr%|}d25j|�}dj|�}n�|
|j�vr|
}dj|�}d}n�||j�vr|}d
j|�}d}nw||j�vr|}dj|�}d}nO||j�vr|}dj|�}d}n'||j�vr|}dj|�}d}|�dd}�(||j�vr||}|j	|�d||||||	|ff|||<|j	|�|j	|�dj|�}dj|�}dj|�}dj|�}dj|�}dj|�}||j�vr�d}d}||j�vr,||}||}|j	|�|j	|�d||||||||||||	ff|||<|j	|�|j	|�|j	|�|j	|�dj|�}dj|�}dj|�} dj|�}!dj|�}"dj|�}#dj|�}$d}%||j�vr||}%|j	|�d}&||j�vr||}&|j	|�d}'| |j�vr|| }'|j	| �d}(|!|j�vr||!}(|j	|!�d})|#|j�vr||#})|j	|#�d}*|$|j�vr||$}*|j	|$�d}+|"|j�vr||"}+|j	|"�|%�|&�|'�|)�d |%|&||'|(|)|*|+|	f	f|||<d!j|�},d"j|�}-d#j|�}.d$j|�}/|,|vr`d%||,||-||.||/||	ff|||<|j	|,�|j	|-�|j	|.�|j	|/�d&j|�}0d'j|�}1|j
|0d�}2|j
|1d�}3|2�V|j	|0�d(|2ff|||<|3�8|j	|1�d(|3ff|d)j||dtd*��<d+j|�}4|j
|4d�}5|5�d(|5ff|||<|j	|4�d,j|�}6|j
|6d�}7|7�8d(|7ff|d)j||dtd*��<|j	|6�d-j|�}8|j
|8d�}9|9���}d.|9ff|||<|j	|8����|r>|j�D]+}||vs�tjd/j|���-|S#Y��xYw)0Nz{}.alphaz
{}.dora_scalez{}.reshape_weightz{}.lora_up.weightz{}_lora.up.weightz{}.lora_B.weightz{}.lora.up.weightz	{}.lora_Bz{}.lora_linear_layer.up.weightz{}.lora_down.weightz{}.lora_mid.weightz{}_lora.down.weightz{}.lora_A.weightz{}.lora.down.weightz	{}.lora_Az {}.lora_linear_layer.down.weight�loraz{}.hada_w1_az{}.hada_w1_bz{}.hada_w2_az{}.hada_w2_bz26{}.hada_t1z27{}.hada_t2�lohaz28{}.lokr_w1z29{}.lokr_w2z{}.lokr_w1_az{}.lokr_w1_bz30{}.lokr_t2z{}.lokr_w2_az{}.lokr_w2_b�lokrz{}.a1.weightz{}.a2.weightz{}.b1.weightz{}.b2.weight�gloraz	{}.w_normz	{}.b_norm�diffz{}.bias�.weightz{}.diffz	{}.diff_bz
{}.set_weight�setzlora key not loaded: {})31r�format�keys�item�add�tolist�get�len�logging�warning):r�to_load�log_missing�32patch_dict�loaded_keys�x�33alpha_name�alpha�dora_scale_name�34dora_scale�reshape_name�reshape�regular_lora�diffusers_lora�diffusers2_lora�diffusers3_lora�35mochi_lora�transformers_lora�A_name�B_name�mid_name�mid�hada_w1_a_name�hada_w1_b_name�hada_w2_a_name�hada_w2_b_name�hada_t1_name�hada_t2_name�hada_t1�hada_t2�lokr_w1_name�lokr_w2_name�lokr_w1_a_name�lokr_w1_b_name�lokr_t2_name�lokr_w2_a_name�lokr_w2_b_name�lokr_w1�lokr_w2�	lokr_w1_a�	lokr_w1_b�	lokr_w2_a�	lokr_w2_b�lokr_t2�a1_name�a2_name�b1_name�b2_name�w_norm_name�b_norm_name�w_norm�b_norm�	diff_name�diff_weight�diff_bias_name�	diff_bias�set_weight_name�36set_weights:                                                          �U/Users/fredrodrigues/Library/CloudStorage/Dropbox/Code/comfyui_hardware/comfy/lora.py�	load_lorarU$s_���J��%�K�
���&�&�q�)�37��������$���$�)�)�+�E��O�O�J�'�)�0�0��3���38��d�i�i�k�)��o�.�J��O�O�O�,�*�1�1�!�4�����4�9�9�;�&�
��|�,�3�3�5������-�+�1�1�!�4��,�3�3�A�6��,�3�3�A�6��-�4�4�Q�7�� �'�'��*�39�<�C�C�A�F�����4�9�9�;�&�!�F�*�1�1�!�4�F�+�2�2�1�5�H�
�t�y�y�{�
*�#�F�*�1�1�!�4�F��H�
��	�	��
+�$�F�'�.�.�q�1�F��H�
��	�	��
+�$�F�*�1�1�!�4�F��H�
�4�9�9�;�
&��F� �'�'��*�F��H�
�$�)�)�+�
-�&�F�6�=�=�a�@�F��H����C��#��D�I�I�K�(?��8�n������)�&,�t�F�|�T�&�\�5�RU�Wa�cj�.k�%l�J�w�q�z�"��O�O�F�#��O�O�F�#�(�.�.�q�1��'�.�.�q�1��'�.�.�q�1��'�.�.�q�1��#�*�*�1�-��#�*�*�1�-���T�Y�Y�[�(��G��G��t�y�y�{�*��|�,���|�,������-�����-�&,�t�N�/C�T�.�EY�[`�bf�gu�bv�x|�~L�yM�OV�X_�ak�/l�&m�J�w�q�z�"��O�O�N�+��O�O�N�+��O�O�N�+��O�O�N�+�$�*�*�1�-��#�*�*�1�-��'�.�.�q�1��'�.�.�q�1��#�*�*�1�-��'�.�.�q�1��'�.�.�q�1�����4�9�9�;�&��<�(�G��O�O�L�)����4�9�9�;�&��<�(�G��O�O�L�)��	��T�Y�Y�[�(��^�,�I��O�O�N�+��	��T�Y�Y�[�(��^�,�I��O�O�N�+��	��T�Y�Y�[�(��^�,�I��O�O�N�+��	��T�Y�Y�[�(��^�,�I��O�O�N�+����4�9�9�;�&��<�(�G��O�O�L�)���W�%8�i�>S�Yb�Yn�&,�w���	�S\�^g�ir�t{�~H�/I�&J�J�w�q�z�"�!�'�'��*�� �'�'��*�� �'�'��*�� �'�'��*���d�?�&-��W�
�t�G�}�d�SZ�m�]a�bi�]j�lq�s}�/~�%�J�w�q�z�"��O�O�G�$��O�O�G�$��O�O�G�$��O�O�G�$�!�(�(��+��!�(�(��+�����+�t�,�����+�t�,�����O�O�K�(�&,�v�i�%8�J�w�q�z�"��!�����,�NT�W]�V_�M`�40�9�+�+�G�A�J�7G��Y���,H�I�J��$�$�Q�'�	��h�h�y�$�/���"�&,�{�n�%=�J�w�q�z�"��O�O�I�&�$�+�+�A�.���H�H�^�T�2�	�� �JP�S\�R^�I_�J�y�'�'���41�3C�S��^�O�(D�E�F��O�O�N�+�)�0�0��3���X�X�o�t�4�42��!�&+�j�]�%;�J�w�q�z�"��O�O�O�,�_�b�����A���#���� 9� @� @�� C�D�����G
��s�=$_-�-_2c43��|j�j�}|D]5}|jd�s�||dj|dt	d��<�7d}d}d}td�D�]�}tD�]|}dj||�}||vrS|j|t|�}	|||	<dj|t|�}	|||	<dj||�}	|||	<d	j||�}||vrU|j|t|�}	|||	<dj|t|�}	|||	<d44}dj||�}	|||	<dj||�}||vs��d45}|r7dj|t|�}	|||	<d
j||�}	|||	<��*dj|t|�}	|||	<dj||�}	|||	<dj|t|�}	|||	<�����|D]�}|jd�s�|jd�r}|t	d�t	d�}46d}|r|dz
}|r4|dz
}|dk(r*||dj||47jdd��<|dz
}||dj||48jdd��<��|jd�s��|t	d�t	d�}49dj|50jdd��}	|||	<��d}||vr51||d<||d<d}||vr||d<|S)Nrztext_encoders.{}z'lora_te_text_model_encoder_layers_{}_{}F� z9clip_h.transformer.text_model.encoder.layers.{}.{}.weightz(lora_te1_text_model_encoder_layers_{}_{}z,text_encoder.text_model.encoder.layers.{}.{}z9clip_l.transformer.text_model.encoder.layers.{}.{}.weightTz9clip_g.transformer.text_model.encoder.layers.{}.{}.weightz(lora_te2_text_model_encoder_layers_{}_{}z.text_encoder_2.text_model.encoder.layers.{}.{}z-lora_prior_te_text_model_encoder_layers_{}_{}zt5xxl.transformer.��zlora_te{}_{}�.�_zhydit_clip.transformer.bert.zlora_te1_{}z)clip_g.transformer.text_projection.weight�lora_prior_te_text_projection�lora_te2_text_projectionz)clip_l.transformer.text_projection.weight�lora_te1_text_projection)	�52state_dictr�endswithrr�range�
LORA_CLIP_MAP�53startswith�replace)�model�key_map�sdk�k�text_model_lora_key�clip_l_present�clip_g_present�b�c�lora_key�l_key�t5_indexs            rT�model_lora_keys_cliprq�s���54�55�56�57�58!�59!�60#�C�
���:�:�i� �FG�G�&�-�-�a�0@�#�i�.��.A�B�C��D���N��N�
�2�Y����A�K�R�R�ST�VW�X�A��C�x�.�5�5�a��q�9I�J��$%���!�E�L�L�Q�P]�^_�P`�a��$%���!�I�P�P�QR�TU�V��$%���!�K�R�R�ST�VW�X�A��C�x�.�5�5�a��q�9I�J��$%���!�E�L�L�Q�P]�^_�P`�a��$%���!�!%��I�P�P�QR�TU�V��$%���!�K�R�R�ST�VW�X�A��C�x�!%��!�I�P�P�QR�Ta�bc�Td�e�H�()�G�H�%�O�V�V�WX�Z[�\�H�()�G�H�%�H�O�O�PQ�S`�ab�Sc�d�H�()�G�H�%�M�T�T�UV�XY�Z�H�()�G�H�%�N�U�U�VW�Yf�gh�Yi�j�H�()�G�H�%�E��J���:�:�i� ��|�|�0�1��#�2�3�S��^�O�D����!���M�H�!���M�H��1�}�\]��� 5� 5�h��
�
�c�SV�@W� X�Y� �A�
��TU���-�-�h��
�
�c�3�8O�P�Q����<�=��#�<�=�s�9�~�o�N��(�/�/��
�
�c�3�0G�H��$%���!�#�(	4�A��C�x�34��/�0�./��*�+�3�A��C�x�./��*�+��N�c61��|j�}|j�}|D]�}|jd�s�|jd�r^|t	d�t	d�jdd�}||dj
|�<||dj
|dt	d��<��||dj
|�<��tjj|jj�}|D]�}|jd�s�dj
||�}|dt	d�jdd�}||dj
|�<||dj
|�<d	d62g}|D]L}	dj
|	|dt	d�jdd
��}63|64jd�r|65dd}66|||67<�N��t|tjj�rg|D]b}|jd�s�|jd�s�'|t	d�t	d�jdd�}||dj
|�<�dt|tjj�r�tjj!|jjd��}|D]�}|jd�s�||}dj
|dt	d��}|||<dj
|dt	d��}|||<dj
|dt	d�jdd��}|||<dj
|dt	d�jdd��}|||<��t|tjj"�rvtjj%|jjd��}|D]<}|jd�s�||}dj
|dt	d��}|||<�>t|tjj&�r�tjj)|jjd��}|D]�}|jd�s�||}dj
|dt	d��}|||<dj
|dt	d��}|||<dj
|dt	d��}|||<��t|tjj*�rW|D]R}|jd�s�|jd�s�'|t	d�t	d�}||dj
|�<�Tt|tjj,�r�tjj/|jjd��}|D]�}|jd�s�||}||dj
|dt	d��<||dj
|dt	d�jdd��<||dj
|dt	d�jdd��<��t|tjj0�rW|D]R}|jd�s�|jd�s�'|t	d�t	d�}||dj
|�<�Tt|tjj2�r�|D]�}|jd�s�|jd�s�'|}|jdd�jdd�jdd�}|jdd�jdd�}|jd d!�}|t	d�t	d�}||dj
|�<||dj
|�<��|S)"Nzdiffusion_model.rrZr[zlora_unet_{}z{}zdiffusion_model.{}z68lycoris_{}�zunet.z{}{}z.to_z.processor.to_z	.to_out.0�����zlora_prior_unet_{})�
output_prefixztransformer.{}zbase_model.model.{}zlora_transformer_{}zunet.base_model.model.{}z	_mod.lin.z_mod.linear.z69_attn.qkv.z70_attn_qkv.z_attn.proj.z_attn_proj.zmlp.0.zmlp.fc1.zmlp.2.zmlp.fc2.z.modulation.lin.z.modulation.linear.)r_rrcr`rrdr�comfy�utils�unet_to_diffusers�model_config�unet_config�71isinstance�72model_base�StableCascade_C�SD3�mmdit_to_diffusers�AuraFlow�auraflow_to_diffusers�PixArt�pixart_to_diffusers�73HunyuanDiT�Flux�flux_to_diffusers�74GenmoMochi�HunyuanVideo)rerf�sdrgrh�key_lora�diffusers_keys�unet_key�diffusers_lora_prefix�p�diffusers_lora_key�tos            rT�model_lora_keys_unetr�-sE��	�	�	�	�B�75�'�'�)�C�
���<�<�*�+��z�z�)�$��S�!3�4�c�)�n�_�E�M�M�c�SV�W��;<���-�-�h�7�8�<=�����A�&6��I���$7�8�9�*+�����A��'���[�[�2�2�5�3E�3E�3Q�3Q�R�N�
���:�:�i� �+�2�2�>�!�3D�E�H��)�3�y�>�/�*�2�2�3��<�H�7?�G�N�)�)�(�3�4�5=�G�L�'�'��1�2�%'��M�!�*��%+�]�]�1�a�8H�#�i�.��6I�6Q�6Q�RX�Zj�6k�%l�"�%�.�.�{�;�);�C�R�)@�&�.6��*�+�	+���%��)�)�9�9�:��A��|�|�.�/��:�:�i�(� ��%7�!8�#�i�.��I�Q�Q�RU�WZ�[�H�EF�G�0�7�7��A�B�	��%��)�)�-�-�.����7�7��8J�8J�8V�8V�fx�7�y���A��z�z�)�$�#�A�&��+�2�2�1�5E�s�9�~�o�3F�G��$&���!�0�7�7��:J�C�	�N�?�8K�L��$&���!�0�7�7��:J�C�	�N�?�8K�8S�8S�TW�Y\�8]�^��$&���!�'�.�.�q�1A�3�y�>�/�/B�/J�/J�3�PS�/T�U��$&���!� ��%��)�)�2�2�3����:�:�5�;M�;M�;Y�;Y�i{�:�|���A��z�z�)�$�#�A�&��+�2�2�1�5E�s�9�~�o�3F�G��$&���!�	 ��%��)�)�0�0�1����8�8��9K�9K�9W�9W�gy�8�z���A��z�z�)�$�#�A�&��+�2�2�1�5E�s�9�~�o�3F�G��$&���!�0�7�7��:J�C�	�N�?�8K�L��$&���!�5�<�<�Q�?O��Y���=P�Q��$&���!� ��%��)�)�4�4�5��A��|�|�.�/�A�J�J�y�4I��S�!3�4�c�)�n�_�E��BC��-�4�4�X�>�?��76�%��)�)�.�.�/����6�6�u�7I�7I�7U�7U�ew�6�x���A��z�z�)�$�#�A�&��HJ��(�/�/��2B�C�	�N�?�0C�D�E�VX���+�+�A�.>��I���,?�,G�,G��S�,Q�R�S�_a��-�4�4�Q�7G��Y���5H�5P�5P�QT�VY�5Z�[�\� ��%��)�)�4�4�5��A��|�|�.�/�A�J�J�y�4I��S�!3�4�c�)�n�_�E��12�����H�-�.��77�%��)�)�6�6�7��A��|�|�.�/�A�J�J�y�4I���#�+�+�K��H�P�P�Q]�_k�l�t�t�vC�ER�S��#�+�+�H�j�A�I�I�(�T^�_��#�+�+�,>�@U�V��#�C�(:�$;�S��^�O�L��=>��(�/�/��9�:�AB��,�3�3�H�=�>���Nrrc�(�tjj||j|�}||z}|||�j	|j78�z}|j
dd�j|jdd�jdd��j|jdgdg|j�dz79z���j
dd�}|||zj	|j80�z}|dk7r||z}|||zz
}|S||dd|S)NrrX�����T)�dim�keepdim��?)rw�model_management�cast_to_device�device�type�dtype�	transposer%�shape�normr�)	r#�weight�	lora_diffr!�strength�intermediate_dtype�function�weight_calc�weight_norms	         rT�weight_decomposer��s!���'�'�6�6�z�6�=�=�Rd�e�J�
���I��8�I�.�3�3�F�L�L�A�A�K�	����a��#�	���"�"�1�%�r�	*�	
��!�T��	"�	���"�"�1�%�	G�*+��{���/@�1�/D�(E�	G�81��1�a����J��,�2�2�6�<�<�@�@�K��3���v����(�k�*�*���M� ��q�	��Mrrc���ttt|��D�cgc]}|||j|k��c}�rt	d��t|�t|j�k7rt	d��tj||j|j��}td�|jD��}td�|jD��}||||<|Scc}w)a�82    Pad a tensor to a new shape with zeros.83 84    Args:85        tensor (torch.Tensor): The original tensor to be padded.86        new_shape (List[int]): The desired shape of the padded tensor.87 88    Returns:89        torch.Tensor: A new tensor padded with zeros to the specified shape.90 91    Note:92        If the new shape is smaller than the original tensor in any dimension,93        the original tensor will be truncated in that dimension.94    zGThe new shape must be larger than the original tensor in all dimensionszLThe new shape must have the same number of dimensions as the original tensor)r�r�c3�4K�|]}td|����y�w�rN��slice��.0r�s  rT�	<genexpr>z&pad_tensor_to_shape.<locals>.<genexpr>�s����>��#��a��
����c3�4K�|]}td|����y�wr�r�r�s  rTr�z&pad_tensor_to_shape.<locals>.<genexpr>�s����=���u�Q��}��r�)95�anyrarr��96ValueError�torch�zerosr�r��tuple)�tensor�	new_shape�i�
padded_tensor�orig_slices�97new_slicess      rT�pad_tensor_to_shaper��s����E�#�i�.�4I�J�4I�q�I�a�L�6�<�<��?�*�4I�J�K��b�c�c�98�9�~��V�\�\�*�*��g�h�h��K�K�	����f�m�m�T�M��>����>�>�K��=����=�=�J�!'�{� 3�M�*�����!Ks�C$c�t�|D�]A}|d}|d}|d}|d}	|d}99|100�d�}101d}|	�|}|j|	d|	d|	d�}|dk7r||z}t|t�rRt|dd|ddtj102j
|dd|j|d�	�d�103�||��f}t|�dk(rd}nt|�dk(r104|d}|d}dk(�r&|d}
t|�dkDxr|dd
}|ri|
j|jk7rPtjdj||j|
j��t||
j�}|dk7�r�|
j|jk7r<tjdj||
j|j���nz||105|tj106j
|
|j|j�z�z
}�n7|dk(r|j!|d��n|dk(r�|d}tj107j
||j|�tj108j
||dd|j|�z109}||110|tj111j
||j|j�z�z
}�no|dk(�rtj112j
|d|j|�}tj113j
|d|j|�}|d}|d}|�t||�}|d�|d|jdz}nd}|d��tj114j
|d|j|�}|jd|jd|jd|jdg}t#j$|j'dd�j)d��|j'dd�j)d���j+|�j'dd�}	t#j$|j)d��|j)d���j+|j�}|�t-|||||||115�}n*||116||z|zj/|j��z
}�	nf|dk(�r�|d}|d}|d}|d}|d}|d}|d} |d}d}!|�x|jd}!t#j$tj117j
||j|�tj118j
||j|��}n+tj119j
||j|�}|��|jd}!| �it#j$tj120j
||j|�tj121j
||j|��}n�t#j4dtj122j
| |j|�tj123j
||j|�tj124j
||j|��}n+tj125j
||j|�}t|j�dk(r |j7d�j7d�}|d�|!�	|d|!z}nd}	t#j8||�j+|j�}|�t-|||||||126�}n*||127||z|zj/|j��z
}�n�|dk(�r�|d}"|d}#|d�|d|#jdz}nd}|d}$|d}%|d}|d��1|d}&|d} t#j4dtj128j
|&|j|�tj129j
|#|j|�tj130j
|"|j|��}'t#j4dtj131j
| |j|�tj132j
|%|j|�tj133j
|$|j|��}(n�t#j$tj134j
|"|j|�tj135j
|#|j|��}'t#j$tj136j
|$|j|�tj137j
|%|j|��}(	|'|(zj+|j�}|�t-|||||||138�}n*||139||z|zj/|j��z
}�n�|dk(�r�|d}d})|djd|djdcxk(r+|djdcxk(r|djdk(rnn|djd}*d})|djd|djdcxk(r+|djdcxk(r|djdk(r[nnX|)rB|djd|jdk(r |jd|jdk(rnd})|djd}*tj140j
|dj)d��|j|�}+tj141j
|dj)d��|j|�},tj142j
|dj)d��|j|�}-tj143j
|dj)d��|j|�}.|d�	|d*z}nd}	|)r{t#j$|.|-�t#j$t#j$|j)d��j;|��|,�|+�zj+|j�}n�|j=�dkDrVt#j4d t#j4d |j;|��|+�|,�j+|j�}nSt#j$t#j$|j;|��|+�|,�j+|j�}|t#j$|-|.�j+|j�z
}|�t-|||||||144�}n*||145||z|zj/|j��z
}n%tjd!j||��|���@|}��D|S#t0$r0}tj2dj|||��Yd}~�?d}~wwxYw#t0$r0}tj2dj|||��Yd}~�{d}~wwxYw#t0$r0}tj2dj|||��Yd}~��d}~wwxYw#t0$r0}tj2dj|||��Yd}~��d}~wwxYw)"NrrXrY��c��|S)N�)�as rT�<lambda>z"calculate_weight.<locals>.<lambda>�s��rrr�T)�copy)�inplace)r�r�146pad_weightz"Pad weight {} from {} to shape: {}gz4WARNING SHAPE MISMATCH {} WEIGHT NOT MERGED {} != {}r�
model_as_lorar�)�	start_dimzERROR {} {} {}r
���zi j k l, j r, i p -> p r k lrrF)r�zo i ..., i j -> o j ...zpatch type not recognized {} {})�narrowr|�list�calculate_weightrwr�r�r�rr�r�inforr�rr��copy_r��mmr��flattenr%r�r��	Exception�error�einsum�	unsqueeze�kronr�r�)/�patchesr��keyr��original_weightsr�r��v�strength_model�offsetr��147old_weight�148patch_typer�
do_pad_weight�
target_weightrO�mat1�mat2r#r%r!�mat3�final_shaper��e�w1�w2�w1_a�w1_b�w2_a�w2_b�t2r��w1a�w1b�w2a�w2b�t1�m1�m2�	old_glora�rank�a1�a2�b1�b2s/                                               rTr�r��s-��
���Q�4��
�a�D���1����1����Q�4����"�H��149����J��]�]�6�!�9�f�Q�i����C�F��S� ��n�$�F��a���!�!�A�B�%���1��a���1G�1G�1V�1V�WX�YZ�W[�\]�W^�`f�`m�`m�pB�IM�1V�2N�X\�*]�_b�wI�J�M�A��q�6�Q�;��J�
��V�q�[��1��J��!��A����!"�1��D���F�Q�J�=�1�Q�4��+=�M�����v�|�|�!;����A�H�H��f�l�l�\`�\f�\f�g�h�,�V�T�Z�Z�@���3���:�:����-��O�O�$Z�$a�$a�be�gk�gq�gq�sy�s�s�%A�B��h�x�%�2H�2H�2W�2W�X\�^d�^k�^k�ms�my�my�2z�'z�{�{�F�
�5�
 ��L�L��1���
�?�
*�*+�A�$�M��0�0�?�?�
�v�}�}�^p�q��0�0�?�?�@P�QT�@U�VW�@X�YZ�@[�]c�]j�]j�l~��@�K��h�x�%�*@�*@�*O�*O�P[�]c�]j�]j�lr�lx�lx�*y�y�z�z�F�
�6�
!��)�)�8�8��1��v�}�}�N`�a�D��)�)�8�8��1��v�}�}�N`�a�D��1��J���d�G��"�,�V�W�=����t���!��t�z�z�!�}�,������t���-�-�<�<�Q�q�T�6�=�=�Rd�e��#�z�z�!�}�d�j�j��m�T�Z�Z��]�D�J�J�WX�M�Z���x�x����q�!� 4� <� <�q� <� I�4�>�>�Z[�]^�K_�Kg�Kg�rs�Kg�Kt�u�}�}�J�K�U�U�VW�YZ�[��
K�!�H�H�T�\�\�A�\�%>����WX��@Y�Z�b�b�ci�co�co�p�	��)�-�j�&�)�U�T\�^p�rz�{�F��h��E�)9�Y�(F�'L�'L�V�\�\�'Z�[�[�F���6�
!��1��B��1��B��Q�4�D��Q�4�D��Q�4�D��Q�4�D��1��B��1��J��C��z��j�j��m���X�X�e�4�4�C�C�D�&�-�-�Yk�l� %� 6� 6� E� E�d�F�M�M�[m� n�p���+�+�:�:�2�v�}�}�N`�a���z��j�j��m���:����%�"8�"8�"G�"G��f�m�m�]o�"p�$)�$:�$:�$I�$I�$�PV�P]�P]�_q�$r�t�B����&D�(-�(>�(>�(M�(M�b�RX�R_�R_�as�(t�(-�(>�(>�(M�(M�d�TZ�Ta�Ta�cu�(v�(-�(>�(>�(M�(M�d�TZ�Ta�Ta�cu�(v�x�B�150�+�+�:�:�2�v�}�}�N`�a���2�8�8�}��!��\�\�!�_�.�.�q�1����t��C�O��!��s�151����
K�!�J�J�r�2�.�6�6�v�|�|�D�	��)�-�j�&�)�U�T\�^p�rz�{�F��h��E�)9�Y�(F�'L�'L�V�\�\�'Z�[�[�F���6�
!��A�$�C��A�$�C���t���!��s�y�y��|�+�����A�$�C��A�$�C��1��J���t���q�T���q�T���\�\�"@�$)�$:�$:�$I�$I�"�f�m�m�]o�$p�$)�$:�$:�$I�$I�#�v�}�}�^p�$q�$)�$:�$:�$I�$I�#�v�}�}�^p�$q�s��152�\�\�"@�$)�$:�$:�$I�$I�"�f�m�m�]o�$p�$)�$:�$:�$I�$I�#�v�}�}�^p�$q�$)�$:�$:�$I�$I�#�v�}�}�^p�$q�s��153�X�X�e�4�4�C�C�C����Xj�k� %� 6� 6� E� E�c�6�=�=�Zl� m�o���X�X�e�4�4�C�C�C����Xj�k� %� 6� 6� E� E�c�6�=�=�Zl� m�o��
K��"�W�-�-�f�l�l�;�	��)�-�j�&�)�U�T\�^p�rz�{�F��h��E�)9�Y�(F�'L�'L�V�\�\�'Z�[�[�F���7�
"��1��J��I���t�z�z�!�}��!��154�155�1�
�O��1����A��O�!�A�$�*�*�Q�-�O���t�z�z�!�}�� �	���t�z�z�!�}��!��156�157�1�
�O��1����A��O�!�A�$�*�*�Q�-�O���1����A��&�,�,�q�/�!A�f�l�l�ST�o�Y_�Ye�Ye�fg�Yh�Fh�� %�I��Q�4�:�:�a�=�D��'�'�6�6�q��t�|�|�a�|�7P�RX�R_�R_�as�t�B��'�'�6�6�q��t�|�|�a�|�7P�RX�R_�R_�as�t�B��'�'�6�6�q��t�|�|�a�|�7P�RX�R_�R_�as�t�B��'�'�6�6�q��t�|�|�a�|�7P�RX�R_�R_�as�t�B���t���!��t�����
K��!&���"�b�!1�E�H�H�U�X�X�f�n�n�_`�n�Fa�Fd�Fd�k}�Fd�F~�AC�>D�FH�5I�"I�!R�!R�SY�S_�S_�!`�I��z�z�|�a�'�$)�L�L�1J�E�L�L�Yr�tz�t}�t}�EW�t}�uX�Z\�M]�_a�%b�%j�%j�kq�kw�kw�%x�	�$)�H�H�U�X�X�f�i�i�FX�i�6Y�[]�-^�`b�$c�$k�$k�lr�lx�lx�$y�	�����"�b�!1�!9�!9�&�,�,�!G�G�I��)�-�j�&�)�U�T\�^p�rz�{�F��h��E�)9�Y�(F�'L�'L�V�\�\�'Z�[�[�F��
�O�O�=�D�D�Z�QT�U�V��!��F�S�V�M��I�
K��
�
�.�5�5�j�#�q�I�J�J��
K��\�
K��
�
�.�5�5�j�#�q�I�J�J��
K��J�
K��
�
�.�5�5�j�#�q�I�J�J��
K��R�
K��
�
�.�5�5�j�#�q�I�J�J��
K�sd�158B
y159�5A-z�(A{�0F){>�160	z�&y>�>z�	z?�&z:�:z?�	{;�&{6�6{;�>	|7�&|2�2|7)T)r��torch.Tensorr�z	list[int]�returnr�)�__doc__�161__future__r�comfy.utilsrw�comfy.model_management�comfy.model_baserr�rbrUrqr�r�r��float32r�r�rrrT�<module>rsv���$#��������*�*�*�.�
�
�y�v)+�L�\)+�k�\�(�B?D�m�m�^b�Lrr