CoolFace
Apppublic

fred-dev/comfy_ui_ali

sourceHugging Facemitupdated 1y agoView on Hugging Face
0likes
clip_vision.cpython-312.pyc42 linesDownload Raw Back to __pycache__
1�

2���g'���ddlmZmZmZddlZddlZddlZddlZddlZ	ddl3Z	ddlZ	ddlZ	ddl
Z	Gd�d�Zdgd�gd�d	fd4�ZGd�d�Zd
�Zdd�Zd�Zy)�)�load_torch_file�transformers_convert�state_dict_prefix_replace�Nc��eZdZd�Zd�Zy)�Outputc��t||�S�N)�getattr)�self�keys  �\/Users/fredrodrigues/Library/CloudStorage/Dropbox/Code/comfyui_hardware/comfy/clip_vision.py�__getitem__zOutput.__getitem__s���t�S�!�!�c��t|||�yr5)�setattr)rr
�items   r�__setitem__zOutput.__setitem__s����c�4� rN)�__name__�6__module__�__qualname__rr�rrrr
s��"�!rr���g3<�4'��?gwg��M�?gy{�� �?�g��B�91�?g�w�t.��?g��	U��?Tc�p�tj||j|j��}tj||j|j��}|j	dd�}|j7d|k(r|j8d|k(s�|r`|t
|j9d|j10d�z}t||j11dz�t||j12dz�f}n||f}tjjj||dd��}|j13d|z14dz}|j15d|z16dz}|dd�dd�|||z�|||z�f}tjd	|zd17d�j�d	z}||jgd��z18|jgd��zS)
N)�device�dtype�����r���bicubicT)�size�mode�	antialiasg�o@r��)r!rr)
�torch�tensorrr�movedim�shape�min�round�nn�19functional�interpolate�clip�view)	�imager#�mean�std�crop�scale�20scale_size�h�ws	         r�clip_preprocessr:s|���<�<��U�\�\����E�D�21�,�,�s�5�<�<�u�{�{�22C�C��M�M�"�a� �E��K�K��N�d�"�u�{�{�1�~��'=���C����A�����A��?�?�E������A�� 6�7��u�u�{�{�ST�~�?U�9V�W�J����J����#�#�/�/��J�Y�bf�/�g��
�[�[��^�d�
"�Q�&��
�[�[��^�d�
"�Q�&���a��!�A�d�F�(�1�Q�t�V�8�+�,���J�J��u��q�#�.�4�4�6��>�E��D�I�I�g�&�&�#�(�(�7�*;�;�;rc�&�eZdZd�Zd�Zd�Zdd�Zy)�ClipVisionModelc��t|�5}tj|�}ddd�jdd�|_|jdgd��|_|jdgd��|_tjj�|_23tjj�}tjj|j�|_
tjj||j|tj j"�|_|j$j'�tj(j+|j$|j|��|_y#1swY��axYw)N�24image_sizer�25image_meanr�	image_stdr)�load_device�offload_device)�open�json�load�getr>r?r@�comfy�model_management�text_encoder_devicerA�text_encoder_offload_device�text_encoder_dtyper�26clip_model�CLIPVisionModelProjection�ops�manual_cast�model�eval�
model_patcher�ModelPatcher�patcher)r�json_config�f�configrBs     r�__init__zClipVisionModel.__init__&s��
�+�
�!��Y�Y�q�\�F��!�*�*�\�3�7��� �*�*�\�3V�W������K�1U�V��� �1�1�E�E�G����/�/�K�K�M���+�+�>�>�t�?O�?O�P��27��%�%�?�?���28�29�Tb�di�dm�dm�dy�dy�z��30��31�32�����*�*�7�7��33�34�PT�P`�P`�q�7�A����
�s�F�Fc�<�|jj|d��S)NF)�strict)rP�load_state_dict)r�sds  r�load_sdzClipVisionModel.load_sd5s���z�z�)�)�"�U�)�;�;rc�6�|jj�Sr35)rP�36state_dict)rs r�get_sdzClipVisionModel.get_sd8s���z�z�$�$�&�&rc�~�tjj|j�t	|j|j�|j|j|j|��j�}|j|d��}t�}|djtjj��|d<|djtjj��|d<|djtjj��|d	<|d37|d<|S)N)r#r3r4r5�����)�pixel_values�intermediate_outputr�last_hidden_stater �image_embedsr�penultimate_hidden_statesr!�mm_projected)rGrH�load_model_gpurTr:�torAr>r?r@�floatrPr�intermediate_device)rr2r5rc�out�outputss      r�encode_imagezClipVisionModel.encode_image;s��
���-�-�d�l�l�;�&�u�x�x��0@�0@�'A����^b�^m�^m�sw�tB�tB�IM�N�T�T�V���j�j�l��j�K���(��'*�1�v�y�y��1G�1G�1[�1[�1]�'^��#�$�"%�a�&�)�)�E�,B�,B�,V�,V�,X�"Y����/2�1�v�y�y��9O�9O�9c�9c�9e�/f��+�,�"%�a�&�����rN)T)rrrrXr]r`rorrrr<r<%s��
A�<�'�38rr<c�P�|j�}dj|�|vr�dj|�ddj|�ddj|�ddj|�d	d39j|�ddj|�d
dj|�di}|D]}||vs�|j|�|||<� dj|�|vr3|jdj|��jdd�|d<t	||dd�}|S|di}t||�}|S)Nz-{}transformer.resblocks.0.attn.in_proj_weightz{}class_embeddingz'vision_model.embeddings.class_embeddingz{}conv1.weightz.vision_model.embeddings.patch_embedding.weightz{}positional_embedding�1vision_model.embeddings.position_embedding.weightz{}ln_post.biasz vision_model.post_layernorm.biasz{}ln_post.weightz"vision_model.post_layernorm.weightz
{}ln_pre.biaszvision_model.pre_layrnorm.biasz{}ln_pre.weightz vision_model.pre_layrnorm.weightz{}projrrzvisual_projection.weightz
vision_model.�0�)�keys�format�pop�	transposerr)r\�prefix�sd_k�keys_to_replace�x�replace_prefixs      r�convert_to_transformersr}Gs:��
�7�7�9�D�6�=�=�f�E��M��&�&�v�.�0Y��#�#�F�+�-]�$�+�+�F�3�5h��#�#�F�+�-O��%�%�f�-�/S��"�"�6�*�,L��$�$�V�,�.P�40��!�A��D�y�)+������?�1�%�&�!��?�?�6�"�d�*�-/�V�V�H�O�O�F�4K�-L�-V�-V�WX�Z[�-\�B�)�*�
!�"�f�o�r�
B���I�!�"���
&�r�>�
:��
�Irc�8�|rt||�}d|vr`tjjtjj	tjjt��d�}�nd|vr`tjjtjj	tjjt��d�}�n�d|v�r�|djddk(r`tjjtjj	tjjt��d	�}�n7|d41jddk(r�d|vr_tjjtjj	tjjt��d
�}n�tjjtjj	tjjt��d�}n`tjjtjj	tjjt��d�}nyt|�}|j|�\}}t|�dkDr$tjdj|��t|�}t|j!��}|D]}||vs�|j#|��|S)Nz1vision_model.encoder.layers.47.layer_norm1.weightzclip_vision_config_g.jsonz1vision_model.encoder.layers.30.layer_norm1.weightzclip_vision_config_h.jsonz1vision_model.encoder.layers.22.layer_norm1.weightz0vision_model.encoder.layers.0.layer_norm1.weightri�zclip_vision_siglip_384.jsonrqiAz#multi_modal_projector.linear_1.biasz&clip_vision_config_vitl_336_llava.jsonz clip_vision_config_vitl_336.jsonzclip_vision_config_vitl.jsonzmissing clip vision: {})r}�os�path�join�dirname�realpath�__file__r*r<r]�len�logging�warningru�set�listrtrv)	r\rx�convert_keysrUr0�m�urt�ks	         r�load_clipvision_from_sdr�as���
$�R��
0��:�b�@��g�g�l�l�2�7�7�?�?�2�7�7�3C�3C�H�3M�#N�Pk�l��	<��	B��g�g�l�l�2�7�7�?�?�2�7�7�3C�3C�H�3M�#N�Pk�l��	<��	B�
�@�A�G�G��J�d�R��'�'�,�,�r�w�w���r�w�w�7G�7G��7Q�'R�Tq�r�K�
�C�
D�
J�
J�1�
M�QT�
T�4��:� �g�g�l�l�2�7�7�?�?�2�7�7�;K�;K�H�;U�+V�YA�B�� �g�g�l�l�2�7�7�?�?�2�7�7�;K�;K�H�;U�+V�Xz�{���'�'�,�,�r�w�w���r�w�w�7G�7G��7Q�'R�Tr�s�K���;�'�D��<�<���D�A�q�42�1�v��z����1�8�8��;�<��A��A�����	�?�D�
���A�:��F�F�1�I���Krc�R�t|�}d|vrt|dd��St|�S)Nz2visual.transformer.resblocks.0.attn.in_proj_weightzvisual.T)rxr�)rr�)�	ckpt_pathr\s  rrErE�s.��	��	#�B�;�r�A�&�r�)�$�O�O�&�r�*�*r)rsF)�utilsrrrrr'rDr��	comfy.opsrG�comfy.model_patcher�comfy.model_management�comfy.utils�comfy.clip_modelrr:r<r}r�rErrr�<module>r�sa��S�S�	���������!�!�!$�*M�Sw�C�<�$ � �D�4�>+r