CoolFace
Modelpublic

cara-ai/ALM2Vec-PT

sourceHugging Faceapache-2.0updated 3mo agoView on Hugging Face
2likes21downloads
modeling_alm2vec.cpython-313.pyc329 linesDownload Raw Back to __pycache__
1�

2^[2jl��	�&�SrSSKrSSKJr SSKrSSKJr SSKJr SSK	J3r4 SSKJr SSK
Jr S	S5KJr SSKrSSKrSSKJr SSKrSSKJr SSKJr SSKJr SS
KJr SSKJrJ r J!r!J"r"J#r#J$r$J%r%J&r&J'r'J(r( SSK)J*r*J+r+J,r, SSK)J-r- SSK.J/r/ SSK0J1r1 SSK2J3r3 SSK4J5r5J6r6 SSK7J8r8 SSK9r9SSK:r:\'\<\&\<\<4\%\<4r=Sr>Sr?Sr@S\=S\&\<\<44SjrA0SS_SS_S S!_S"S_S#S$_S%S&_S'S_S(S)_S*S_S+S	_S,S-_S.S/_S0S1_S2S3_S4S5_S6S7_S8S3S9/_S3S9/SSS:S1S;.ES$S<S=S>.0S?S_S@SA_SBS=_SCSD_SESD_SFSG_SHSI_SJSK_S.SL_SMSK_SNSK_SOS9_SPSQ_SR/SSQSTSTSU._SVSW_SXSI_SYS_SZS[S\.E0S?S_S@SA_SBS]_SCSD_SESD_SFS^_SHS__SJS`_S.SL_SMSa_SNS`_SOS9_SPSQ_SR/SbQSTSTSU._SVSW_SXS__SYS_SZS[S\.ESc.rB"SdSe\+5rC"SfSg\R�5rE"ShSi\R�5rF"SjSk\R�5rG"SlSm\R�5rH"SnSo\R�5rI"SpSq\R�5rJ"SrSs\R�5rK"StSu\*5rL"SvSw\R�5rM\"SxSy\655rN"SzS{\*\,5rOSrPS=rQ"S|S}\*5rRg!\;a Sr:GN�f=f)~z#ALM2Vec audio-text embedding model.�N)�6checkpoint)�BytesIO)�Path)�NamedTemporaryFile)�urlparse)�urlopen�)�
ALM2VecConfig)�	dataclass)�Tensor)�scaled_dot_product_attention)7�Any�Dict�Callable�Iterable�List�Optional�Sequence�Tuple�Union�cast)�PreTrainedModel�PreTrainedConfig�GenerationMixin)�
AutoTokenizer)�Qwen2_5OmniTextConfig)�Qwen2_5OmniThinkerTextModel)�Cache)�BaseModelOutputWithPast�ModelOutput)�can_return_tuple�>z�Based on the question asked in the text query and context in the audio query, retrieve the relevant text document associated with that question.zRepresent the user's input.�x�returnc���[U[RR5(aK[	U5S:XdSUS[	U535e[[[[4[U55$X4$)N�z%Expected a sequence of length 2, got z
 with length )	�8isinstance�collections�abcr�lenrr�int�tuple�r#s �modeling_alm2vec.py�_resolve_tuple2r/5sf���!�[�_�_�-�-�.�.��1�v��{�	9�3�A�3�m�C��F�8�L�	10�{��E�#�s�(�O�U�1�X�.�.�
�6�M��attn_drop_rate��centerT�depth� �	drop_rate�	embed_dimi�f_max�@�@�f_min�11hop_length��init_values�input_channels�	mlp_ratio�@�12model_type�midashenglm_dasheng_encoder�n_fft��n_mels�@�	num_heads��	outputdim��13patch_size�i�)�patch_stride�qkv_bias�sample_rate�
target_length�14win_length�i)�in_dim�downsample_rate�out_dim�attention_dropout�15hidden_act�silu�hidden_size�init_std�{�G�z�?�initializer_range�intermediate_sizeiJ�max_position_embeddingsi��max_window_layers��qwen2_5_omni_text�num_attention_heads�num_hidden_layers�num_key_value_heads�rms_norm_eps�����ư>�rope_scaling)rH�rh�default)�
mrope_section�	rope_type�type�16rope_theta���.A�sliding_window�	use_cacheFiR)�use_sliding_window�17vocab_sizei@ii��)rsrsrs)�audio_encoder_config�audio_projector_config�text_config�lite_random_decoder_configc+��^�\rSrSrSrSS\S\S\\\\\44S\\\\\44S\S\S	\S18\S\S\	S
\19\S\S\S\S\S\	S\S\S\S\S\4*U4SjjjrSrU=r
$)�
DashengConfig�rBr7rIrKrMr>rPr4rGr?rNr=r6r1r:r8r3rQr;rOrCrEc�.>�XlX lX0lX@lXPlX`lXplX�lX�lX�l	X�l20X�lX�lX�l
X�lUUlUUlUUlUUlUUlUUl[*TU]X"S0UD6 g)N�)r7rIrKrMr>rPr4rGr?rNr=r6r1r:r8r3rQr;rOrCrE�super�__init__)�selfr7rIrKrMr>rPr4rGr?rNr=r6r1r:r8r3rQr;rOrCrE�kwargs�	__class__s                       �r.r�DashengConfig.__init__�s����2#��"��$��(��,��*���21�"��"�� �
�&��"��,���22��23����$���$���&�����24����
���"�6�"r0)r1r3r4r6r7r8r:r;r=r>r?rCrErGrIrKrMrNrOrPrQ)�rJrHrHr	i�rsrsr@TNr2r2r2r9TrDr<r"rDrF)�__name__�25__module__�__qualname__�__firstlineno__rAr+rr�float�boolrr�__static_attributes__�
__classcell__�r�s@r.rzrz�sK���.�J���24�46��!�����'+�� #������ ���-.#��.#��.#��#�u�S�#�X��.�/�	.#�26�C��s�C�x��0�1�.#��
.#��.#��.#��.#��.#��.#��e�_�.#��.#��.#��.#� �!.#�"�#.#�$�%.#�&�'.#�(�).#�*�+.#�,�-.#�.#r0rzc�^�\rSrSrSS\S\S\S\S\S\\S\4U4S	jjjr	S27\28RS\29R4SjrS
r
U=r$)�AudioPatchEmbed���30input_sizerKrM�in_chansr7�31norm_layer�flattenc�>�[TU]5 [U5Ul[U5Ul[U5UlURSUR32S-URSUR33S-4UlURSURS-UlXpl[R"UUURUR34S9UlU(aU"U5Ul
g[R"5Ul
g)Nrr	)�kernel_size�stride)r~rr/r�rKrM�	grid_size�num_patchesr��nn�Conv2d�proj�Identity�norm)	r�r�rKrMr�r7r�r�r�s	        �r.r�AudioPatchEmbed.__init__�s����	����)�*�5���)�*�5���+�L�9����O�O�A��$�"3�"3�A�"6�6��O�O�A��$�"3�"3�A�"6�6�35��� �>�>�!�,�t�~�~�a�/@�@������I�I�������$�$�	36��	�.8�J�y�)��	�R�[�[�]��	r0r#r$c���URU5nUR(a-[R"[R"USS5S5nUR	U5nU$)Nr&��rr&r	)r�r��torch�permuter��r�r#s  r.�forward�AudioPatchEmbed.forward�sJ���I�I�a�L���<�<��
�
��
�
�a��A�&�	��A�
�I�I�a�L���r0)r�r�r�r�r�rKrMr�)rFrHrHr	r�NF)r�r�r�r��_Tuple2r+rrr�rr�rr�r�r�r�s@r.r�r��s����!� � "���)-��K��K��K��	K�37�K��
K��X�&�K��K�K�:�����%�,�,��r0r�c�f^�\rSrSrSU4SjjrS\RS\R4SjrSrU=r	$)�38LayerScale�c�>�[TU]5 X0l[R"U[39R"U5-5Ulg�N)r~r�inplacer��	Parameterr��ones�gamma)r��dimr=r�r�s    �r.r�LayerScale.__init__s0���
�������\�\�+��40�41�3��"?�@��42r0r#r$c�v�UR(aURUR5$XR-$r�)r��mul_r�r�s  r.r��LayerScale.forwards&��%)�\�\�q�v�v�d�j�j�!�E�q�:�:�~�Er0)r�r�)g�h㈵��>F)43r�r�r�r�rr�rr�r�r�r�s@r.r�r�s-���A�44F����F�%�,�,�F�Fr0r�c	�^�\rSrSrSS\S\\S\\S\4U4SjjjrS\RS\R4S	jr45S46rU=r$)�47DashengMlpi48�in_features�hidden_features�out_features�dropc�$>�[TU]5 U=(d UnU=(d Un[R"X5Ul[R49"5Ul[R"X#5Ul[R"U5Ul	gr�)50r~rr��Linear�fc1�GELU�act�fc2�Dropoutr�)r�r�r�r�r�r�s     �r.r�DashengMlp.__init__sa���	����#�2�{��)�8�[���9�9�[�:����7�7�9����9�9�_�;����J�J�t�$��	r0r#r$c��URU5nURU5nURU5nURU5nURU5nU$r�)r�r�r�r�r�s  r.r��DashengMlp.forwardsH���H�H�Q�K���H�H�Q�K���I�I�a�L���H�H�Q�K���I�I�a�L���r0)r�r�r�r�)NNr2)
r�r�r�r�r+rr�rr�rr�r�r�r�s@r.r�r�51sf���*.�&*��
%��
%�"�#��
%��s�m�	
%�52�
%�
%������%�,�,��r0r�c�^�\rSrSrSS\S\S\S\S\453U4SjjjrS
S\RS	\54\R4S55jjrSrU=r
$)�DashengAttentioni#r�rGrN�	attn_drop�	proj_dropc�D>�[TU]5 X-S:XdS5eX lX-nUS-Ul[R56"XS-US9Ul[R"U5Ul[R57"X5Ul	[R"U5Ul58g)Nrz$dim should be divisible by num_headsg�r���bias)r~rrG�scaler�r��qkvr�r�r�r�)r�r�rGrNr�r��head_dimr�s       �r.r�DashengAttention.__init__$s����	������!�#�K�%K�K�#�"���#���t�^��59��9�9�S��'��9������I�.����I�I�c�'��	����I�.��r0r#�maskc��URup4nURU5RX4SURXPR-5R	SSSSS5RS5upgn[
UUUUb
USS2SSSS24OSS9nURSS5RX4U5nURU5nURU5nU$)Nr�r&rr	rL)�	attn_mask)60�shaper��reshaperGr��unbindr
�	transposer�r�)	r�r#r��B�N�C�q�k�vs	         r.r��DashengAttention.forward7s����'�'���a��H�H�Q�K�
�W�Q�1�d�n�n�a�>�>�.A�
B�
�W�Q��1�a��
#�
�V�A�Y�		��a�
)�
�
�
�04�0@�d�1�d�D�!�+�,�d�	
61��
�K�K��1��%�%�a�A�.���I�I�a�L���N�N�1����r0)r�rGr�r�r�r�)rtFr2r2r�)r�r�r�r�r+r�r�rr�rrr�r�r�r�s@r.r�r�#sp�������
/�
�/��/��	/�62�/��
/�/�&�����X�e�l�l�-C��r0r�c�^�\rSrSrSS\S\S\S\S\S\S\\4U4S	jjjrSS63\	RS\\	RS\	R4S
jjrSrU=r
$)�DashengBlockiKr�rGr?rNr�r�r=c�>�[TU]5 [R"USS9Ul[UUUUUS9UlU(a	[XS9O[R"5Ul	[R"USS9Ul64[U[X-5US9Ul
U(a[XS9Ulg[R"5Ulg)Nrf��eps)rGrNr�r�)r=)r�r�r�)r~rr��	LayerNorm�norm1r��attnr�r��ls1�norm2r�r+�mlp�ls2)	r�r�rGr?rNr�r�r=r�s	        �r.r�DashengBlock.__init__Ls����	�����\�\�#�4�0��65�$������66��	�9D�J�s�4�����	
���\�\�#�4�0��67������0��68���9D�J�s�4�	
��IK����	
�r0r#r�r$c���XRURURU5U55-nXRUR	URU555-nU$r�)r�r�r�r�r�r�)r�r#r�s   r.r��DashengBlock.forwardnsO��69
������4�:�:�a�=�$�7�8�8��
������$�*�*�Q�-�0�1�1���r0)r�r�r�r�r�r�)r@Fr2r2Nr�)r�r�r�r�r+r�r�rrr�rr�r�r�r�s@r.r�r�Ks����70����'+�71�
�72��73��	74�75�76��
77��78��e�_�79�80�J(,���<�<���u�|�|�$��81���	�r0r�c��^�\rSrSrS\4U4SjjrS\\R\R44Sjr	S\RSS4SjrS	\RS\R4S82jrSr
U=r$)�DashengFrontendix�configc�>�[TU]5 XlUR5up#UR	SUSS9 U UR	SUSS9 U g)N�spectrogram_windowF)�83persistent�melscale_fbanks)r~rr��_build_frontend_buffers�register_buffer)r�r�r�r�r�s    �r.r�DashengFrontend.__init__ys_���
������.2�.J�.J�.L�+����� ���	�	84�85	
����.��E��R�r0r$c	��[R"S5 [R"URR[R86S9n[R"URRS-S-URRURRURRURRS9R875nSSS5 X4$!,(df   WW4$=f)N�cpu��dtyper&r	)�n_freqsr:r8rErO)r��device�hann_windowr�rQ�float32�Fr�rCr:r8rErO�to)r�r�r�s   r.r��'DashengFrontend._build_frontend_buffers�s���
�\�\�%�
 �!&�!2�!2����&�&��m�m�"�� �/�/����)�)�Q�.��2��k�k�'�'��k�k�'�'��{�{�)�)� �K�K�3�3���b�����

�!�"�2�2�!�
 �"�?�2�2�s�CC9�988D89r�Nc���URR4nURRS-S-URR4nS[S[90RS[[S4S[4SjnU"S	URU5nU"S91URU5nU(dU(aLUR5upxURUS9Ul92URUS9Ul[SUS
US3SS9 gURRU:waURRUS9Ul93URRU:waURRUS9Ulgg)zKSelf-heal non-persistent audio frontend buffers if corrupted/uninitialized.r&r	�name�tensor�expected_shape.r$c��Ucg[USS5(ag[UR5U:wagUR5R	5n[94R"U5R5R5(dgUR5S:�a1UR5R5R5S:�agg)NT�is_metaFrrn)�getattrr,r��detachr�r��isfinite�all�item�numel�abs�max)rrr�ts    r.�_is_bad�8DashengFrontend.ensure_frontend_buffers.<locals>._is_bad�s����~���v�y�%�0�0���V�\�\�"�n�4���
�
��%�%�'�A��>�>�!�$�(�(�*�/�/�1�1���w�w�y�1�}��������!3�!3�!5��!;��r0r�r��r�z)[WARN] Rebuilt frontend buffers (win_bad=z	, fb_bad=�)T)�flushN)r�rQrCrE�strr�rr,r+r�r�r�r�r�printr�)	r�r��expected_win_shape�expected_fb_shaper�win_bad�fb_bad�new_win�new_fbs	         r.�ensure_frontend_buffers�'DashengFrontend.ensure_frontend_buffers�s[��"�k�k�4�4�6��!�[�[�.�.�!�3�a�7����9K�9K�L��	�#�	�u�|�|�	�U�3�PS�8�_�	�Y]�	��.��0G�0G�I[�\���*�D�,@�,@�BS�T���f�"�:�:�<�O�G�&-�j�j��j�&?�D�#�#)�9�9�F�9�#;�D� ��;�G�9�I�f�X�UV�W��
�95�&�&�-�-��7�*.�*A�*A�*D�*D�F�*D�*S��'��#�#�*�*�f�4�'+�';�';�'>�'>�f�'>�'M��$�5r0�waveformc��URUR5 [R"UR	[96R5SURURRURRURRSSURRS9	nURURR	[97R5-Rn[R"UR!S5SSSSS	9R#S5nUR	UR$5$)98Nrr&F)	r"�pad�windowrCr;rQ�power�99normalizedr3r	�100g�����|�=�x)�101multiplier�amin�
db_multiplier�top_db)r r�r�spectrogramrr�rr�r�rCr;rQr3�mTr��amplitude_to_DB�	unsqueeze�squeezer�)r�r"r.�mel_spectrogram�log_mel_spectrograms     r.r��DashengFrontend.forward�s����$�$�X�_�_�5��m�m��[�[����/���*�*��+�+�#�#��{�{�-�-��{�{�-�-����;�;�%�%�102103��'�>�>�D�,@�,@�,C�,C�E�M�M�,R�R�V�V�� �/�/��%�%�a�(�����104��'�!�*�
	�#�%�%�h�n�n�5�5r0)r�r�r�)r�r�r�r�rzrr,r�rr�r�r r�r�r�r�s@r.r�r�xsh���+�}�+�3��u�|�|�U�\�\�/I�)J�3� !N�e�l�l�!N�t�!N�F6����6����6�6r0r�c��^�\rSrSrSrS\RS\R4U4Sjjr\S\	RSS4Sj5rS	\RS\R4S105jrSr
U=r$)�
FixedAffine2di�zn106Per-channel fixed affine transform:107    y = x * scale + bias108where scale/bias are broadcast on (B, C, H, W).109r�r�c	�>�[TU]5 URSURSSSS55 URSURSSSS55 g)Nr�r	�����r�)r~rr�r�)r�r�r�r�s   �r.r�FixedAffine2d.__init__�sL���
�������W�e�m�m�A�r�1�a�&@�A����V�T�\�\�!�R��A�%>�?r0�bnr$c�$�URb
URc[S5eUR(a5URR5nURR5nO@[R"UR5n[R"UR5nURR5nURR5nU[R"XQR-5-nX4U--110nU"XgS9$)Nz4BatchNorm2d must have running stats to be converted.�r�r�)�running_mean�running_var�111ValueError�affine�weightrr�r��	ones_like�112zeros_like�sqrtr�)�clsr;r��betar>r?r�r�s        r.�from_batchnorm2d�FixedAffine2d.from_batchnorm2d�s���
�?�?�"�b�n�n�&<��S�T�T�
�9�9��I�I�$�$�&�E��7�7�>�>�#�D��O�O�B�O�O�4�E��#�#�B�O�O�4�D����-�-�/���n�n�+�+�-����113�114�;���#7�8�8���U�*�*����*�*r0r#c�8�XR-UR-$r�r=r�s  r.r��FixedAffine2d.forward�s���:�:�~��	�	�)�)r0r})r�r�r�r��__doc__r�rr�classmethodr��BatchNorm2drHr�r�r�r�s@r.r7r7�sl����@�e�l�l�@�%�,�,�@�115�+�"�.�.�+�_�+��+�$*����*�%�,�,�*�*r0r7c116�\^�\rSrSr\rSrS\4U4SjjrSrSS\	RS\\	RS\	R4S	jjrS117\	RS\
S\	R4SjrSS\	RS
\\	RS\\	R\\	R44SjjrSrU=r$)�DashengAudioTransformeri�Tr�c	�>^�[TU]T5 TRUlTRUlTRUlSUl[
T5Ul[R"TRSS9Ul[TRTR4TRTRTRSTRS9Ul[R""[$R&"STRSUR R(S5S-5Ul[R""[$R&"STRUR R(SS5S-5Ul[R."TR0S9Ul[R4"U4S	j[7TR8555Ul[R<"TRS118S9UlURA5 g)NFg{�G�z�?)�momentum)r�r7r�rKr�rMr	r[r)�pc3��># �UHZn[TRTRTRTRTR119TRTRS9v� M\ g7f))r�rGr?rNr=r�r�N)r�r7rGr?rNr=r6r1)�.0�_r�s  �r.�	<genexpr>�3DashengAudioTransformer.__init__.<locals>.<genexpr>s^����$120�)��
��$�$� �*�*� �*�*����"�.�.��%�%� �/�/�
�)�s�A"A%rfr�)!r~rrPr7r;�gradient_checkpointingr��	front_endr�rNrE�init_bnr�r>rKrM�patch_embedr�r��randnr��time_pos_embed�freq_pos_embedr�r6�pos_drop�121ModuleList�ranger4�blocksr�r��	post_init�r�r�r�s `�r.r� DashengAudioTransformer.__init__s����
���� �#�1�1����)�)��� �+�+���&+��#�(��0����~�~�f�m�m�d�C���*��
�
�v�';�';�<��&�&��*�*��(�(���,�,�
122���!�l�l��K�K��6�+�+�Q��0@�0@�0J�0J�1�0M�N�QU�U�123���!�l�l��K�K��6�+�+�T�-=�-=�-G�-G��-J�A�N�QU�U�124����125�126�V�%5�%5�6��
��m�m�$127��6�<�<�(�$128�129����L�L��!1�!1�t�<��	����r0c���[UR[R5(a?URR	5 [130R
UR5Ulgg)zC131Call this after checkpoint is loaded and before inference/export.132N)r'r[r�rN�evalr7rH)r�s r.�!replace_init_bn_with_fixed_affine�9DashengAudioTransformer.replace_init_bn_with_fixed_affine.sD���d�l�l�B�N�N�3�3��L�L����(�9�9�$�,�,�G�D�L�4r0r#r�r$c���URSnXRSS2SS2SS2SU24-nXRSS2SS2SS2SS24-n[R"[R133"USS5S5nUR
U5nURHAnUR(a%UR(aURXAU5nM9U"X5nMC URU5nU$)Nr9r&r�r�)r�r^r_r�r�r�r`rcrY�training�_gradient_checkpointing_funcr�)r�r#r�r�blocks     r.�forward_features�(DashengAudioTransformer.forward_features6s���134
�G�G�B�K��
�#�#�A�q�!�R�a�R�K�0�0��
�#�#�A�q�!�Q�J�/�/�	135�
�M�M��M�M�!�Q��"�I�
136��
�M�M�!����[�[�E��*�*�t�}�}��5�5�e��E���!�N��	!�137
�I�I�a�L���r0�lengths�138max_lengthc���[U5n[R"X!RS9nUR	U5RX25nXAR
S5:R5nU$)Nrr9)r*r��aranger��repeat�viewr1r�)r�rqrr�139batch_size�idxr�s      r.�_to_mask� DashengAudioTransformer._to_maskLsW����\�140��l�l�:�n�n�=���j�j��$�)�)�*�A���'�'��+�+�1�1�3���r0�x_lengthc���URU5nURS-nURS5n[R"US5nURU5n[R"US5nUR
U5nURSnURUSS9nUbu[U5[U5:XdS5eURS:XdS5eX RS--R5nURXFS9nURUSS9nOSnS/[U5-n/n	[XX5H%up�URX�S	9n141U	R!U1425 M' [R""U	SS9nX4$)143NrLr	)rr&r	r�r9�r�z2batchsizes of input x and x_length need to be samezLengths are of size (B,))rrrq)r�)rZrPr1r�r�r[r\r��splitr*�ndimr;�longry�zipro�append�cat)r�r#r{�target_length_in_patchesr�input_splits�scaled_lengthsr��split_masks�outputs�split_x�144split_masks            r.r��DashengAudioTransformer.forwardSsp��145
�N�N�1���#'�#5�#5��#:� �
�K�K��N���M�M�!�\�*���L�L��O���M�M�!�\�*�����Q���
�G�G�B�K���w�w�7�R�w�@�����x�=�C��F�*�
�D�
�*��=�=�A�%�A�'A�A�%�&�/�/�A�*=�>�D�D�F�N��=�=�A�=�F�D��*�*�%=�2�*�F�K��D��&�3�|�#4�4�K���#&�|�#A��G��+�+�G�+�E�G��N�N�7�#�$B�
�I�I�g�1�%���w�r0)rcr7r_rZrYr;r[r�r\r`rPr^r�)r�r�r�r�rz�config_class�supports_gradient_checkpointingrrir�rrror+ryrr�r�r�r�s@r.rPrP�s���� �L�&*�#�+�}�+�ZH�(,���<�<���u�|�|�$��146���	�,�����#��%�,�,��,0�$��<�<�$��5�<�<�(�$�147�u�|�|�X�e�l�l�3�3�	4�	$�$r0rPc�d^�\rSrSrSS\S\S\\R4U4SjjjrS	Sjr	Sr148U=r$)149�AudioProjectorSubsampleizrSrUr�c150��>�[TU]5 X0lX l[R151"[R"XR-X$S9[R"5[R"X"US95Ulg)Nr�)	r~rr�rUr��152Sequentialr�r��net)r�rSrUrTr�r�s     �r.r� AudioProjectorSubsample.__init__{sU���	���� �����=�=��I�I�f�v�v�o�w�<��G�G�I��I�I�g�e�4�153��r0c���URup4nX@R-nUS:�aUSS2SU*2SS24nUbUSS2SU*24nUc;[R"URSS[RUR154S9nUR
USURU-5nURU5nUR
USUR5nURSS9R	5nX4$)Nrr9�r�r�r})	r�r�r�r�r�r�r�r��any)r�r#r�rw�seq_lenr��num_frames_to_discards       r.r��AudioProjectorSubsample.forward�s���#$�7�7� �155�S� '�&�&� 0�� �1�$��!�,�,�,�,�a�/�0�A����A�6�!6� 6�6�6�7���<��:�:�a�g�g�c�r�l�%�*�*�Q�X�X�N�D�
�I�I���D�F�F�S�L�
156��
�H�H�Q�K���|�|���D�F�F�157���x�x�B�x��$�$�&���w�r0)r�r�rU)rRNr�)r�r�r�r�r+rr�r�rr�r�r�r�s@r.r�r�zsD���158�'+�159��160��161�162����$�163�164� �r0r�c���\rSrSr%Sr\\R\S'Sr	\\R\S'Sr165\\\S'Sr\\
\RS4\S'Sr\\
\RS4\S'S	rg)166�Qwen25OmniTextModelOutputi�N�loss�logits�past_key_values.�
hidden_states�167attentionsr})r�r�r�r�r�rr��FloatTensor�__annotations__r�r�rr�rr�r�r}r0r.r�r��sx��(,�D�(�5�$�$�168%�,�*.�F�H�U�&�&�'�.�'+�O�X�e�_�+�=A�M�8�E�%�"3�"3�S�"8�9�:�A�:>�J���u�0�0�#�5�6�7�>r0r�c�t^�\rSrSr\rSrSrSrSr	S\4U4Sjjr169\SS\\
RS\\
RS\\
RS\\\
R"S	\\
R"S170\\S\\S\\S
\\
RS\\
RS\\\44Sjj5rSrU=r$)� Qwen25OmniThinkerTextOnlyDecoderi�Tr�c��>�[TU]U5 [R"U5Ul[171R"URURSS9Ul	UR5 g)NFr�)r~rr�_from_config�modelr�r�rYrr�lm_headrdres  �r.r�)Qwen25OmniThinkerTextOnlyDecoder.__init__�sS���
���� �0�=�=�f�E��172��y�y��������173���174	
���r0�	input_ids�attention_mask�position_idsr��
inputs_embedsrp�output_attentions�output_hidden_states�cache_position�labelsr$c
��Ub6Uc3UR5RSS9RUS:HS5S-175nURUUUUUUUUU	SS9176nURn
URU
5nU177b*UR"S	UU178URRS.UD6OSn[UUURURURS9$)179Nr9r}rr	T)180r�r�r�r�r�rpr�r�r��return_dict)r�r�rr)r�r�r�r�r�r})
r��cumsum�masked_fill_r��last_hidden_stater��
loss_functionr�rrr�r�r�r�)r�r�r�r�r�r�rpr�r�r�r�r�r�r�r�r�s                r.r��(Qwen25OmniThinkerTextOnlyDecoder.forward�s���%�,�*>��#�#�%���B�����n��1�1�5���
�,0�:�:��)�%�+�'��/�!5�)��,6�,181�� �1�1�
����m�,���!�

���
����;�;�1�1�
��	
��	
�)���#�3�3�!�/�/��)�)�182�	183r0)r�r�)184NNNNNNNNNN)r�r�r�r�rr��_supports_flash_attn_2�_supports_sdpa�_supports_cache_class�_supports_static_cacherr!rr��185LongTensorrrr�r�rrr�r�r�r�r�s@r.r�r��s1���(�L�!���N� ��!���4���15�15�37�=A�59�$(�,0�/3�59�)-�6186��E�,�,�-�6187�!����.�6188��u�/�/�0�	6189�190"�$�u�'8�'8�"9�:�6191� �� 1� 1�2�
6192��D�>�6193�$�D�>�6194�'�t�n�6195�!��!1�!1�2�6196�����&�6197�198�u�/�/�	0�6199��6200r0r�c�^�\rSrSr\rS\4U4SjjrS%SjrSr\	S\201S\\2024S	j5r\	S203\
S\\
S\
4Sj5rS
\\\
S\
S204\\
S\R"S\\R&\R&44205SjrS\\
\4S\S\R&4SjrS\\\\
\\R&4S\S\R"S\\\R&\\R&44SjrSSSSSSS.S\\\
\\
4S\\\
\\R&\\\\
\\R&44S\
S206\\
S\S\\\
\R"4S\R&4Sjjr\	S\
S\207S\\S4SS4S j5rS&S\\\
S\\\\
\\R&4S\R&4S!jjr S&S\\\
S\\\\
\\R&4S\R&4S"jjr!S&S#jr"S$r#U=r$$)'�ALM2VecModeli�r�c�6>�[TU]U5 [S
0[SD6n[	U5nUR208Ul[
[S
0[SD65Ul[S
0[SD6Ul209[R"5Ul
URS[R "S[R"S95 URS[R "S[R"S95 URS[R "S	[R"S95 UR210R$R&UlS211UlS212Ul[,(aK[.R0"S5nURS[R2"U/[R4S95 OSUl[R8"UR&UR&5UlURR=5 SUlURA5 g)Nrwrurv�audio_start_tokenr	r��audio_end_token�	eos_token�Fg$I�$I�,@�logit_scaler})!r~rr�DASHENG_ARCH_CONFIGr�r�rPrz�dashengr��dasheng_downr�r��dasheng_projr�r��zerosr�r�rY�use_checkpointing�checkpoint_reentrant�USE_LOGIT_SCALE�math�logrrr�r��siglip_headri�213_tokenizerrd)r�r�rw�decoder�214init_valuer�s     �r.r�ALM2VecModel.__init__�s����
���� �+�Q�.A�-�.P�Q��2�;�?���]�]��215�.��H�/�0F�G�H�216���4�d�6I�Jb�6c�d����K�K�M���	
���0�%�+�+�a�u�z�z�2R�S����.����A�U�Z�Z�0P�Q����[�%�+�+�a�u�z�z�*J�K��:�:�,�,�8�8���!&���$)��!��?����(�+�J�� � ��u�|�|�Z�L��
�
�N�
� $�D���9�9�T�%5�%5�t�7G�7G�H������6�6�8�������r0r$Nc��Xlg)z.Attach a tokenizer for high-level encode APIs.N)r��r��	tokenizers  r.�
set_tokenizer�ALM2VecModel.set_tokenizers��#�r0c��URbUR$[R"URSS9nXlU$)NT)�trust_remote_code)r�r�from_pretrained�name_or_pathr�s  r.�_resolve_tokenizer�ALM2VecModel._resolve_tokenizer!s:���?�?�&��?�?�"�!�1�1�$�2C�2C�W[�\�	�#���r0r#c�^�Uc/$[U[[45(a[U5$U/$r�)r'�listr,r-s r.�_to_list�ALM2VecModel._to_list(s-���9��I��a�$���'�'���7�N��s�217r0�instruction�textc�2�SUS3nUcSnX#-$SU3nX#-$)Nz<|im_start|>system218z<|im_end|>219z<|im_start|>user220r})r�r��system_part�	user_parts    r.�
_build_prompt�ALM2VecModel._build_prompt0s<��,�[�M��F���<�,�I��&�&�-�T�F�3�I��&�&r0�texts�taskr�c�2�US;a[SUS35eUS:Xa[O[nU=(d UnUVs/sHopRXG5PM nnU"USSSS9n	U	S	R	U5n221U	S222R	U5RSS9nX�4$s snf)
N)�query�documentzUnsupported task=z. Use 'query' or 'document'.r�TF�pt)�padding�add_special_tokens�return_tensorsr�r�r	r})r@�QUERY_INSTRUCTION�DOC_INSTRUCTIONr�r�sum)r�r�r�r�r�r��default_instructionr��prompts�encoded�text_ids�	text_lenss            r.�_prepare_text_batch� ALM2VecModel._prepare_text_batch9s����,�,��0���6R�S�T�T�37�7�?�/���!�8�%8��EJ�K�U�T�%�%�k�8�U��K��G�T�e�\`�a���;�'�*�*�6�2���,�-�0�0��8�<�<��<�C�	��"�"��	Ls�B�path�	target_src���[U5n[U5nURS;nU(aSO223[U5nU(a[UR5OUnUR224R
5nU(ai[U5n	U	R5n225SSS5 [U=(d SSS9nURW2265 UR5 UR5 OSnU(a[W2275O228[U5nUS;Gat[R"US5n
U
R!5nU
R#5nU
R%5nU
R'U
R)55nSSS5 WS:XaC[*R,"[/W5[*R0S9R35nUS	-229S	-nO�US230:Xa>[*R,"[/W5[*R4S9R35S-nORUS:Xa>[*R,"[/W5[*R6S9R35S
-nO[9SU35eWS:�aUR;SU5R=SS9nO}[>c[AS5eU(aURBO231[U5n[>RD"U5unnURFSS:�aUR=SSS9nURIS5nUb$[URB5RKSS9 OWU:waP[>c[AS5e[>RNRQURSS5X�5RIS5nUR35$!,(df   GN�=f!,(df   GN1=f![La N�f=f!Ub4[URB5RKSS9 f![La ff=ff=f)N)�http�httpsz.audioF)�suffix�delete)z.wavz.wave�rbr	r�g`@r&g�@rLg�AzUnsupported WAV sample width: r9r}z/torchaudio is required for non-WAV audio paths.rT)r��keepdim)�232missing_okz&torchaudio is required for resampling.)*rr�schemerr�r�lowerr�readr�writer�closer�wave�open�getframerate�getnchannels�getsampwidth�233readframes�234getnframesr��235frombuffer�	bytearray�uint8r��int16�int32r@r��mean�236torchaudio�ImportErrorr�loadr�r2�unlink�OSError�237functional�resampler1)r�r�r��raw_path�parsed�
is_remote_url�238local_path�
suffix_sourcer�resp�audio_bytes�source_for_torchaudio�
path_for_wave�wf�sr�239n_channels�sample_width�raw�audio�load_targetr"s                     r.�_load_audio_path�ALM2VecModel._load_audio_pathLsT���t�9���(�#���
�
�):�:�
�*�T��T�240�241�-:��V�[�[�)�242�
��%�%�+�+�-�����"�d�"�i�i�k��#�$6��)���%�!�
"�'�'��4�!�'�'�)�!�'�'�)�$(�!�0=���,�3�z�?�
�!	��*�*��Y�Y�}�d�3�r����*�B�!#���!2�J�#%�?�?�#4�L��-�-��
�
��8�C�	4�243 �1�$�!�,�,�Y�s�^�5�;�;�O�U�U�W�E�"�U�]�e�3�E�!�Q�&�!�,�,�Y�s�^�5�;�;�O�U�U�W�Za�a�E�!�Q�&�!�,�,�Y�s�^�5�;�;�O�U�U�W�Zf�f�E�$�'E�l�^�%T�U�U���>�!�M�M�"�j�9�>�>�1�>�E�E���%�%�&W�X�X�2?�)�.�.�S��_�� *���{�;���"��>�>�!�$�q�(�'�}�}��D�}�A�H� �(�(��+��$�0���.�3�3�4�;�;�t�;�L�1�244��?��!�!�"J�K�K��)�)�2�2�5�?�?�1�3E�r�U�]�]�^_�`�E��{�{�}��i#�"�� 4�3��<�����%�0���.�3�3�4�;�;�t�;�L�������1�sh�N�N<� AN�0F245N<�>"N,�246N�247N)�$N<�,248N9�8N9�<O5�"O$�#O5�$249O1�.O5�0O1�1O5�audio_itemsc��/n/nSnUGH�nUcEUR[R"S[RS95 URS5 MLSn[	U[250[45(aURXrS9nO�[	U[R5(apUR5R5R5nUR5S:XaURSS9nO7UR5S:wa[S	5eO[S251[!U535eUR#5S:Xa&[R"S[RS9nSn	O[%UR#55n	URU5 URU	5 GM� U(dg['SU55n252[R"[)U5U253[RS9n[+U5H%up�UR#5n
U
S:�dMX�USU
24'M' UR-U5[R."U[R0US
94$)NFr	r�rT)r�r&r}z:Audio tensor must be 1D waveform or 2D [channels, length].zUnsupported audio item type: �NNc3�@# �UHoR5v� M g7fr�)r�rUrs  r.rW�4ALM2VecModel._prepare_audio_batch.<locals>.<genexpr>�s���5��A�g�g�i�i��s�r�)r�r�r�rr'rrr1rrr�r�r�rr@�	TypeErrorrlrr+rr*�	enumeraterrr�)r�r3r�r��tensor_list�	lens_list�	has_audior�wav�length�max_len�padded�i�Ls              r.�_prepare_audio_batch�!ALM2VecModel._prepare_audio_batch�s���+-��!�	��	��D��|��"�"�5�;�;�q��
�
�#F�G�� � ��#���I��$��d��,�,��+�+�D�+�F���D�%�,�,�/�/��k�k�m�)�)�+�/�/�1���7�7�9��>��(�(�q�(�/�C��W�W�Y�!�^�$�%a�b�b�$� �"?��T�254�|� L�M�M��y�y�{�a���k�k�!�5�=�=�9�����S�Y�Y�[�)�����s�#����V�$�- �.���5��5�5�����S��-�w�e�m�m�L����,�F�A��	�	��A��1�u� #�q�"�1�"�u�
�-��y�y�� �%�,�,�y��255�256�SY�"Z�Z�Zr0r�T)r�r/r�r��	normalizer�r/rFc���UR5 UR5nUb[R"U5O"[	UR55RnUR
U5nUR
U5n	[[U5[U	55n257U258S:Xa[S5e[U5S:XaS/U259-nO4[U5S:XaU260S:�aX�-nO[U5U261:wa[S5e[U	5S:XaS/U262-n	O4[U	5S:XaU263S:�aX�-n	O[U	5U264:wa[S5eURUUUUUS9up�URU	[US9up�[R"5 U"UUU
US9nSSS5 U(a6[RRR!WR#5S	S2659nW$!,(df   NM=f)zHHigh-level embedding API. Accepts raw text/audio and returns embeddings.Nrz,At least one of text/audio must be provided.r	zEtext and audio batch sizes must match (or be broadcastable length 1).)r�r�r�r�)r�r�)r�r�r/�266audio_lensr9r})rhr�r�r��next�267parametersr�rr*r@r�rD�	TARGET_SR�inference_moder�rrFr�)r�r�r/r�r�rFr�r��268text_itemsr3rwr�r��audio_tensorrH�embs                r.�encode�ALM2VecModel.encode�s���	
�	�	���+�+�-�	�)/�);����f�%��d�o�o�FW�AX�A_�A_���]�]�4�(�269��m�m�E�*����Z��#�k�*:�;�270���?��K�L�L��z�?�a����*�,�J�
��_��
!�j�1�n�#�0�J�
��_�271�
*��d�e�e��{��q� ��&�:�-�K�
��
��
"�z�A�~�%�2�K�
��
��
+��d�e�e�"�6�6����#��7�272���$(�#<�#<����$=�$273� ���
!�
!�
#��!�#�"�%�	�C�$���(�(�%�%�/�/��	�	���/�D�C��274�$�
#�s�275G�276G"r�value�277item_types.c
��Ucg[U[5(d%[SUS[U5RS35e[U5S:Xa[
SUS35e[U5HXup4[XB5(aMSRSU55n[SUSUS	US278[U5RS3	5e g)N�`z` must be a list, got �.rz)` must be a non-empty list when provided.z, c3�8# �UHoRv� M g7fr�)r�r7s  r.rW�/ALM2VecModel._check_list_arg.<locals>.<genexpr>�s���#C�279�1�J�J�280�s��[z]` must be one of (z), got )	r'r�r9rlr�r*r@r:�join)rrRrSrBr�alloweds      r.�_check_list_arg�ALM2VecModel._check_list_arg�s����=���%��&�&���D�6�/��U��0D�0D�/E�Q�G��
��u�:��?��q���&O�P�Q�Q� ��'�G�A��d�/�/��)�)�#C�281�#C�C�����v�Q�q�c�!4�W�I�>���:�.�.�/�q�2���(r0c�Z�URSU[45 URSU[[[R45 UcUc[S5eUb?Ub<[
U5[
U5:wa$[S[
U5S[
U5S35eUR"S	XSS.UD6$)282z7Encode queries. Accepts text-only, audio-only, or both.r�r/z8encode_query requires at least one of `text` or `audio`.zencode_query: `text` (len=z) and `audio` (len=z3) must have the same length when both are provided.r��r�r/r�r})r\rrr�rr@r*rP�r�r�r/r�s    r.�encode_query�ALM2VecModel.encode_querys���	
���V�T�C�6�2����W�e�c�4����-F�G��<�E�M��J��
���� 1�c�$�i�3�u�:�6M��,�S��Y�K�7J�3�u�:�,�WD�D��
��{�{�J���J�6�J�Jr0c���URSU[45 URSU[[[R45 USLUSL:Xa[S5eUR"SXSS.UD6$)z;Encode documents. Accepts exactly one of `text` or `audio`.r�r/NzRencode_document requires exactly one of `text` or `audio` (not both, not neither).r�r_r})r\rrr�rr@rPr`s    r.�encode_document�ALM2VecModel.encode_documentsr��	
���V�T�C�6�2����W�e�c�4����-F�G��D�L�e�t�m�,��+��
��{�{�M��283�M�f�M�Mr0c	��URnURS5nURRRR284nUbHUR
UU5up�URX�5up�U	RSS9nURU5nOSnURRU5n285URRURR55nURRURR55nURRURR55n
/n/n/n[U5H�nU286USUU24/nUbFUUS:�a=URU5 URUUSUU245 URU5 URU
5 [ R""USS9nURU5 UR[ R$"URS5US95 URURS5S-2875 M� ['SU55n[ R("UUUR*XWS9n[ R("UUUS9n[U5H/nUURS5nUUUUSU24'UUUUSU24'M1 URUUS9R,n[ R."[U5Vs/sH
nUUUU4PM sn5nUR1U5R3S5nU$s snf)Nrr	r}rc3�B# �UHoRS5v� M g7f)rN)�size)rUr#s  r.rW�'ALM2VecModel.forward.<locals>.<genexpr>Ls���6��A�f�f�Q�i�i��s�)r�r�)r�r�)r�rhr��embed_tokensrBr�r�r�r�r�r��cloner�r�rbr�r�r�r�rr�rYr��stackr�r2)r�r�r�r/rHr�r��embed_dtype�	audio_emb�288audio_mask�text_emb�audio_start_emb�
audio_end_emb�eos_emb�input_embeds�attention_masks�last_indicesrB�seqr@�
padded_embeds�padded_maskrCr��final_hidden�outs                          r.r��ALM2VecModel.forward's��������M�M�!����j�j�-�-�4�4�:�:����$(�L�L���%�!�I�%)�$5�$5�i�$L�!�I�#���A��.�J��)�)�)�4�I��I��:�:�*�*�8�4���*�*�1�1�$�2H�2H�2N�2N�2P�Q���289�290�/�/��0D�0D�0J�0J�0L�M�
��*�*�)�)�$�.�.�*>�*>�*@�A���������q��A��A�~��1��~�-�.�/�C��$��A���):��291�292�?�+��293�294�9�Q��*�Q�-��%7�8�9��295�296�=�)��J�J�w���)�)�C�Q�'�C�����$��"�"�5�:�:�c�h�h�q�k�&�#I�J���������a��0���6��6�6�����
�w��(�(��297�
��k�k�!�W�V�<���q��A��Q��$�$�Q�'�A�#/��?�M�!�R�a�R�%� �!0��!3�K��2�A�2����298�*�*�'�&��299��300�	�301�{�{��q��#R��A�G�A�|�A��,>�$?��#R�S�����|�,�4�4�Q�7���302��$Ss�L?)303r�r�r�r�r�rYr�r�r�r�)r$Nr5)%r�r�r�r�r304r�rr�r��staticmethodrr�r�rrr�r�r�r,rr�rrr+r1rDr�rPrrlr\rrardr�r�r�r�s@r.r�r��s
��� �L��}��B$����C��D��I�����'�3�'�h�s�m�'��'��'�#��H�S�M�"�#�305�#��c�]�
#����#�306�u�|�|�U�\�\�)�	*�#�&=�U�3��9�%5�=�#�=�%�,�,�=�~)[��(�5��d�E�L�L�)@�#A�B�C�)[��	)[�307���)[�308�x����%�x����'=�=�	>�
)[�\15�jn��%)��59�9��u�S�$�s�)�^�,�-�9���c�4����t�H�U�3�PT�V[�Vb�Vb�Kb�Ec�<d�7e�e�f�g�	9�309�9��c�]�
9��9���s�E�L�L�0�1�2�9�310���9�v��c��#��5��s��;K��PT����&%)�@D�K��t�C�y�!�K���U�3��e�l�l�#:�;�<�=�K�311312���K�,%)�@D�N��t�C�y�!�N���U�3��e�l�l�#:�;�<�=�N�313314���N� 6�6r0r�)SrLr��torch.utils.checkpointrr
�ior�pathlibr�tempfiler�urllib.parser�urllib.requestr�configuration_alm2vecr315r(�collections.abc�dataclassesrr��torch.nnr��torchaudio.functionalrrr�torch.nn.functionalr
�typingrrrrrrrrrr�transformersrrrr�;transformers.models.qwen2_5_omni.configuration_qwen2_5_omnir�6transformers.models.qwen2_5_omni.modeling_qwen2_5_omnir�transformers.cache_utilsr�transformers.modeling_outputsrr �transformers.utilsr!�copyrrr+r�rKr�r�r/r�rz�Moduler�r�r�r�r�r�r7rPr�r�r�r��HIDDEN_SIZEr�r}r0r.�<module>r�s���)��-����'�!�"�0���!���!��<�^�^�^�K�K�&���+�N�/������U�3��8�_�h�s�m�3�3164���	�g��/���w��5��c��?����#���$��	���	�S�	�317	�T��	��
�	���	�c��	�t��	�!��	�S��	�3��	���	�"��	�R�� 	�S�!�"	�318�	�319�#�,	�	�320�����9�@������S���f��	�t��	�D�	�321	�T��	�U�
�	"�5��	�R��	�)��	�r��	�R��	�q��	���	�322�323��324��.	�i�/�0	�%�1�2	�T�3�4$��7�<#��S�#��f�#�	�s�#�	�D�	#�325	�T�#�	�T�
#�	"�4�#�	�R�#�	�)�#�	�q�#�	�R�#�	�q�#�	��#�	��326#��327�#�.	�i�/#�0	�$�1#�2	�T�3#�4$��7#�Ia��H1#�$�1#�h%�b�i�i�%�PF����F������2%�r�y�y�%�P*�2�9�9�*�Z_6�b�i�i�_6�D *�B�I�I� *�Fz�o�z�z"�b�i�i�"�L�?��?��?�H328���H329�X����d�?�d��_���J��s�,J�J�J