CoolFace
Apppublic

fred-dev/comfy_ui_ali

sourceHugging Facemitupdated 1y agoView on Hugging Face
0likes
clip_model.cpython-312.pyc94 linesDownload Raw Back to __pycache__
1�

2���gz-���ddlZddlmZddlZGd�dej3j�Zd�ej4jjd�d�Z5Gd�d	ej6j�ZGd7�dej8j�ZGd�d
ej9j�Z
Gd�dej10j�ZGd�dej11j�ZGd�dej12j�ZGd�dej13j�ZGd�dej14j�ZGd�dej15j�ZGd�dej16j�Zy)�N)�optimized_attention_for_devicec�&��eZdZ�fd�Zdd�Z�xZS)�
CLIPAttentionc���t�|��||_|j||d||��|_|j||d||��|_|j||d||��|_|j||d||��|_y�NT��bias�dtype�device)�super�__init__�heads�Linear�q_proj�k_proj�v_proj�out_proj)�self�	embed_dimrr17r�18operations�	__class__s      ��[/Users/fredrodrigues/Library/CloudStorage/Dropbox/Code/comfyui_hardware/comfy/clip_model.pyr
zCLIPAttention.__init__s����
������19� �'�'�	�9�4�u�]c�'�d��� �'�'�	�9�4�u�]c�'�d��� �'�'�	�9�4�u�]c�'�d���"�)�)�)�Y�T�QV�_e�)�f��
�c��|j|�}|j|�}|j|�}|||||j|�}|j	|�S�N)rrrrr)r�x�mask�optimized_attention�q�k�v�outs        r�forwardzCLIPAttention.forwardsN���K�K��N���K�K��N���K�K��N��!�!�Q��4�:�:�t�<���}�}�S�!�!r�NN��__name__�20__module__�__qualname__r
r#�
__classcell__�rs@rrrs���g�"rrc�8�|tjd|z�zS)Ng�Zd;�?)�torch�sigmoid��as r�<lambda>r0s��q�5�=�=����+C�'Crc�X�tjjj|d��S)N�tanh)�approximate)r,�nn�21functional�gelur.s rr0r0s��e�h�h�.A�.A�.F�.F�q�V\�.F�.]r)�22quick_gelur6�gelu_pytorch_tanhc�$��eZdZ�fd�Zd�Z�xZS)�CLIPMLPc���t�|��|j||d||��|_t||_|j||d||��|_yr)rr
r�fc1�ACTIVATIONS�23activation�fc2)rr�intermediate_sizer>r24rrrs       �rr
zCLIPMLP.__init__sZ���
�����$�$�Y�0A��TY�bh�$�i���%�j�1����$�$�%6�	��TY�bh�$�i��rc�l�|j|�}|j|�}|j|�}|Sr)r<r>r?�rrs  rr#zCLIPMLP.forward$s/���H�H�Q�K���O�O�A����H�H�Q�K���rr%r*s@rr:r:s���j�rr:c�&��eZdZ�fd�Zdd�Z�xZS)�	CLIPLayerc����t�|��|j|||��|_t	|||||�|_|j|||��|_t||||||�|_y�N�r25r)	rr
�	LayerNorm�layer_norm1r�	self_attn�layer_norm2r:�mlp)	rrrr@�intermediate_activationr26rrrs	        �rr
zCLIPLayer.__init__+sq���
����%�/�/�	��v�/�V���&�y�%���27�S���%�/�/�	��v�/�V����9�&7�9P�RW�Y_�ak�l��rc��||j|j|�||�z
}||j|j|��z
}|Sr)rJrIrLrK)rrrrs    rr#zCLIPLayer.forward2sH��	�T�^�^�D�,�,�Q�/��7J�
K�K��	�T�X�X�d�&�&�q�)�
*�*���rr$r%r*s@rrDrD*s���m�rrDc�&��eZdZ�fd�Zdd�Z�xZS)�CLIPEncoderc	����t�28|��tjj	t|�D�	cgc]}	t
|||||||���c}	�|_ycc}	wr)rr
r,r4�29ModuleList�rangerD�layers)r�30num_layersrrr@rMr31rr�irs          �rr
zCLIPEncoder.__init__9s����
�����h�h�)�)�X]�^h�Xi�+j�Xi�ST�9�Y��GX�Zq�sx�{A�CM�,N�Xi�+j�k����+js�Ac���t|j|dud��}|�|dkrt|j�|z}d}t	|j�D]%\}}||||�}||k(s�|j�}�'||fS)NT)r�small_inputr)rr�lenrT�	enumerate�clone)rrr�intermediate_outputr�intermediaterV�ls        rr#zCLIPEncoder.forward=s���<�Q�X�X�D�X\�L\�jn�o���*�"�Q�&�&)�$�+�+�&6�9L�&L�#����d�k�k�*�D�A�q��!�T�.�/�A��'�'� �w�w�y��+��,��rr$r%r*s@rrPrP8s���k�rrPc�>��eZdZd�fd�	Zej32fd�Z�xZS)�CLIPEmbeddingsc���t�|��|j||||��|_|j||||��|_yrF)rr
�	Embedding�token_embedding�position_embedding)rr�33vocab_size�
num_positionsr34rrrs       �rr
zCLIPEmbeddings.__init__LsJ���
����)�3�3�J�	�QV�_e�3�f���",�"6�"6�}�i�W\�ek�"6�"l��rc��|j||��tjj|jj35||j��zS)N)�	out_dtyperG)rc�comfy�ops�cast_tord�weightr)r�input_tokensr36s   rr#zCLIPEmbeddings.forwardQsb���#�#�L�E�#�B�U�Y�Y�EV�EV�W[�Wn�Wn�Wu�Wu�~C�LX�L_�L_�EV�F`�`�	`r)i��MNNN�r&r'r(r
r,�float32r#r)r*s@rr`r`Ks���m�37+0�-�-�`rr`c�H��eZdZ�fd�Zddddddej38fd�Z�xZS)�CLIPTextModel_c39����|d}|d}|d}|d}|d}	|d}40|d|_t�|�	�t||41|||��|_t|||||	|||�|_|j|||�	�|_y)42N�num_hidden_layers�hidden_size�num_attention_headsr@�43hidden_act�max_position_embeddings�eos_token_id)rfr44rrrG)	ryrr
r`�45embeddingsrP�encoderrH�final_layer_norm)r�config_dictr46rrrUrrr@rMrfrs           �rr
zCLIPTextModel_.__init__Vs���� �!4�5�47��
�.�	��1�2��'�(;�<��"-�l�";��#�$=�>�
�'��7���
����(��-�W\�ek�yC�D���"�:�y�%�AR�Tk�mr�tz�}G�H��� *� 4� 4�Y�e�TZ� 4� [��rNTc���|�N|tjj|jjj48||j��z}n|j||��}d}	|��d|j|j�j|jddd|jdf�j|jdd|jd|jd�z49}	|	j|	jtj�tj|j�j �}	tj"|jd|jdftj|j�j |j|j��j%d�}50|	�|	|51z
}	n|52}	|j'||	|��\}}|j)|�}|�|r|j)|�}|�;|t+t-|jd��t+t/d�|��f}n�|tj0|jd|j�	�tj2|�jtj4|j��|j6k(j5�j9d�53�f}|||fS)NrG)r54g�?r�������rr\c��|dz55S�Nr�r.s rr0z(CLIPTextModel_.forward.<locals>.<lambda>|s��!�a�%r)r��dim)rirjrkrzrdrlr�tor56�reshape�shape�expand�masked_fillr,�bool�finfo�max�full�triu_r{r|�listrS�map�arange�round�intry�argmax)
rrm�attention_mask�embeds�57num_tokensr\�final_layer_norm_intermediater58rr�causal_maskrV�
pooled_outputs
             rr#zCLIPTextModel_.forwardds��������*�*�4�?�?�+M�+M�+T�+T�\a�jp�jw�jw�*�x�x�A�����E��:�A����%���*�*�1�7�7�3�;�;�^�=Q�=Q�RS�=T�VW�Y[�]k�]q�]q�rt�]u�<v�w�~�~�@N�@T�@T�UV�@W�YZ�\j�\p�\p�qs�\t�vD�vJ�vJ�KM�vN�O�O�D��#�#�D�G�G�E�J�J�$7�%�+�+�a�g�g�:N�:R�:R�9R�S�D��j�j�!�'�'�!�*�a�g�g�a�j�!9�E�K�K����<P�<T�<T�;T�\]�\c�\c�lm�lt�lt�u�{�{�|}�~�����K��D��D��|�|�A�D�>Q�|�R���1��!�!�!�$���=�:��%�%�a�(�A��!��d�5������#4�5�t�C��Q[�<\�7]�]�^�M��e�l�l�1�7�7�1�:�a�h�h�G�%�+�+�Vb�Jc�Jf�Jf�mr�mv�mv�@A�@H�@H�Jf�KI�MQ�M^�M^�K^�Jc�Jc�Je�Jl�Jl�qs�Jl�Jt�u�v�M��!�]�"�"rror*s@rrrrrUs9���\�$(��T�VZ�pt�UY�af�an�an�#rrrc�0��eZdZ�fd�Zd�Zd�Zd�Z�xZS)�
CLIPTextModelc���t�|��|d|_t||||�|_|d}|j||d||��|_||_y)NrtruFr)rr
rUrr�59text_modelr�text_projectionr60)rr}r61rrrrs      �rr
zCLIPTextModel.__init__�s^���
����%�&9�:���(��e�V�Z�P����
�.�	�)�0�0��I�E�Y^�gm�0�n�����62rc�B�|jjjSr�r�rzrc)rs r�get_input_embeddingsz"CLIPTextModel.get_input_embeddings�s�����)�)�9�9�9rc�:�||jj_yrr�)rrzs  r�set_input_embeddingsz"CLIPTextModel.set_input_embeddings�s��5?����"�"�2rc�l�|j|i|��}|j|d�}|d|d||dfS)N�rr)r�r�)r�args�kwargsrr"s     rr#zCLIPTextModel.forward�sF���D�O�O�T�,�V�,���"�"�1�Q�4�(���!��a��d�C��1��&�&r)r&r'r(r
r�r�r#r)r*s@rr�r��s����:�@�'rr�c�&��eZdZd�fd�	Zd�Z�xZS)�CLIPVisionEmbeddingsc		�@��t�|��||zdz}	|dk(r63d|_d}64nA|	dz}	tjjtj|||���|_d}65|j|||||66||��|_|j|	|||��|_67y)Nr��siglip_vision_modelTrrGF)�in_channels�out_channels�kernel_size�strider	r68r)rr
�class_embeddingr,r4�	Parameter�empty�Conv2d�patch_embeddingrbrd)rr�num_channels�69patch_size�70image_size�71model_typer72rr�num_patches�73patch_biasrs           �rr
zCLIPVisionEmbeddings.__init__�s����
����!�Z�/�A�5���.�.�#'�D� ��J�%��/�K�#(�8�8�#5�#5�e�k�k�)�SX�ag�6h�#i�D� ��J�)�0�0�$�"�"����� 1� 74���#-�"6�"6�{�I�UZ�ci�"6�"j��rc��|j|�jd�jdd�}|j�_t	j75tjj|j|�j|jddd�|gd��}|tjj|jj|�zS)Nr�rrr�r�)
r��flatten�	transposer�r,�catrirj�
cast_to_inputr�r�rdrl)r�pixel_valuesr�s   rr#zCLIPVisionEmbeddings.forward�s����%�%�l�3�;�;�A�>�H�H��A�N�����+��Y�Y��	�	� 7� 7��8L�8L�f� U� \� \�]i�]o�]o�pq�]r�tu�wy� z�}C� D�JK�L�F���	�	�/�/��0G�0G�0N�0N�PV�W�W�Wr)�����NNNr%r*s@rr�r��s���k�0Xrr�c�&��eZdZ�fd�Zdd�Z�xZS)�76CLIPVisionc77�^��t�|��|d}|d}|d}|d}|d}	|d}78t||d|d|d	|79|||�80�|_|81dk(rd�|_d
|_n|j
|�|_d|_t|||||	|||�|_|j
|�|_	y)Nrtrurvr@rwr�r�r�r�)r�r82rrr�c��|Srr�r.s rr0z%CLIPVision.__init__.<locals>.<lambda>�s��!rTF)83rr
r�rz�pre_layrnorm�output_layernormrHrPr{�post_layernorm)rr}r84rrrUrrr@rMr�rs           �rr
zCLIPVision.__init__�s���
���� �!4�5�85��
�.�	��1�2��'�(;�<��"-�l�";�� ��.�86�.�y�+�n�:U�Wb�co�Wp�r}�K�sL�Yc�kp�y�LV�W����.�.� +�D��$(�D�!� *� 4� 4�Y� ?�D��$)�D�!�"�:�y�%�AR�Tk�mr�tz�}G�H���(�2�2�9�=��rc���|j|�}|j|�}|j|d|��\}}|jr|j	|�}|}n|j	|dd�ddd�f�}|||fS)Nr�r)rzr�r{r�r�)rr�r�r\rrVr�s       rr#zCLIPVision.forward�s~���O�O�L�)�����a� ���|�|�A�D�>Q�|�R���1�� � ��#�#�A�&�A��M� �/�/��!�Q��'�87�;�M��!�]�"�"rr$r%r*s@rr�r��s
���>�&88#rr�c�$��eZdZ�fd�Zd�Z�xZS)�LlavaProjectorc���t�|��|j||d||��|_|j||d||��|_y)NT)r	rr89)rr
r�linear_1�linear_2)r�in_dim�out_dimr90rrrs      �rr
zLlavaProjector.__init__�sJ���
����"�)�)�&�'��V�[`�)�a��
�"�)�)�'�7��f�\a�)�b��
rc91��|jtjjj	|j|dd�dd�f���Sr�)r�r,r4r5r6r�rBs  rr#zLlavaProjector.forward�s9���}�}�U�X�X�0�0�5�5�d�m�m�A�a���e�H�6M�N�O�Orr%r*s@rr�r��s���c�92Prr�c�$��eZdZ�fd�Zd�Z�xZS)�CLIPVisionModelProjectionc���t�|��t||||�|_d|vr |j	|d|dd��|_nd�|_d|j
dd�k(rt|dd|||�|_yd|_y)	N�projection_dimruF)r	c��|Srr�r.s rr0z4CLIPVisionModelProjection.__init__.<locals>.<lambda>�s��qr�llava3�projector_typei)	rr
r��vision_modelr�visual_projection�getr��multi_modal_projector)rr}r93rrrs     �rr
z"CLIPVisionModelProjection.__init__�s����
����&�{�E�6�:�N����{�*�%/�%6�%6�{�=�7Q�S^�_o�Sp�w|�%6�%}�D�"�%0�D�"��{���'7��>�>�)7��M�8R�TX�Z_�ag�is�)t�D�&�)-�D�&rc��|j|i|��}|j|d�}d}|j�|j|d�}|d|d||fS)Nr�rr)r�r�r�)rr�r�rr"�	projecteds      rr#z!CLIPVisionModelProjection.forward�si���D���t�.�v�.���$�$�Q�q�T�*���	��%�%�1��2�2�1�Q�4�8�I��!��a��d�C��+�+rr%r*s@rr�r��s
���.�,rr�)r,�comfy.ldm.modules.attentionr�	comfy.opsrir4�Modulerr5r6r=r:rDrPr`rrr�r�r�r�r�r�rr�<module>r�s���F��"�E�H�H�O�O�"�&D��x�x�*�*�/�/�$]���94�e�h�h�o�o����������%�(�(�/�/��&`�U�X�X�_�_�`�*#�U�X�X�_�_�*#�X'�E�H�H�O�O�'�*X�5�8�8�?�?�X�@#������#�@P�U�X�X�_�_�P�,������,r