CoolFace
Apppublic

Linhz/ViMNer

sourceHugging Faceupdated 2y agoView on Hugging Face
1likes
rootModelMultimodelNER__pycache__UMT.cpython-39.pyc
UMT.cpython-39.pyc79 linesDownload Raw Back to __pycache__
1a

2F�df�7�@s|dZddlmZmZmZmZddlZddlZddlZddl	Z	ddl3Z4ddlZddlZddl
Z
ddlZddlmZddlmZddlZddlmZddlmZddlmmZddlmZe�e�Zd	d5�Zdd�Z eejjj!e d
�Z"ddl#m$Z$ddl%m&Z&m'Z'm(Z(m)Z)m*Z*Gdd�dej+�Z,Gdd�dej+�Z-Gdd�dej+�Z.Gdd�dej+�Z/Gdd�dej+�Z0Gdd�de'�Z1dS)zPyTorch BERT model.�)�absolute_import�division�print_function�unicode_literalsN)�open)�CRF)�nn)�CrossEntropyLoss)�VariablecCs |ddt�|t�d��S)a-Implementation of the gelu activation function.6        For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):7        0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))8        Also see https://arxiv.org/abs/1606.084159    ��?��?g@)�torch�erf�math�sqrt��x�r�6E:\demo_datn\pythonProject1\Model\MultimodelNER\UMT.py�gelu*srcCs|t�|�S�N)r
�sigmoidrrrr�swish3sr)r�relur)�RobertaModel)�RobertaLayer�RobertaPreTrainedModel�
RobertaOutput�RobertaSelfOutput�RobertaIntermediatecs&eZdZ�fdd�Zddd�Z�ZS)�RobertaSelfEncodercs8tt|���t|��t��fdd�td�D��|_dS)Ncsg|]}t����qSr��copy�deepcopy��.0�_��layerrr�10<listcomp>B�z/RobertaSelfEncoder.__init__.<locals>.<listcomp>�)�superr �__init__rr�11ModuleList�ranger(��self�config��	__class__r'rr-?szRobertaSelfEncoder.__init__TcCs:g}|jD]}|||�}|r12|�|�q13|s6|�|�|Sr�r(�append)r1Z
hidden_states�attention_mask�output_all_encoded_layers�all_encoder_layers�layer_modulerrr�forwardDs141516zRobertaSelfEncoder.forward)T��__name__�17__module__�__qualname__r-r;�
__classcell__rrr3rr >sr cs&eZdZ�fdd�Zddd�Z�ZS)�RobertaCrossEncodercs8tt|���t|��t��fdd�t|�D��|_dS)Ncsg|]}t����qSrr!r$r'rrr)Sr*z0RobertaCrossEncoder.__init__.<locals>.<listcomp>)r,rAr-�RobertaCrossAttentionLayerrr.r/r()r1r2Z	layer_numr3r'rr-PszRobertaCrossEncoder.__init__TcCs<g}|jD]}||||�}|r18|�|�q19|s8|�|�|Srr5)r1�s1_hidden_states�s2_hidden_states�s2_attention_maskr8r9r:rrrr;Us2021zRobertaCrossEncoder.forward)Tr<rrr3rrAOsrAcs,eZdZ�fdd�Zdd�Zdd�Z�ZS)�RobertaCoAttentioncs�tt|���|j|jdkr2td|j|jf��|j|_t|j|j�|_|j|j|_t	�22|j|j�|_t	�23|j|j�|_t	�24|j|j�|_
t	�|j�|_dS)NrzLThe hidden size (%d) is not a multiple of the number of attention heads (%d))r,rFr-�hidden_size�num_attention_heads�25ValueError�int�attention_head_size�
all_head_sizer�Linear�query�key�value�Dropout�attention_probs_dropout_prob�dropoutr0r3rrr-as26��zRobertaCoAttention.__init__cCs6|��dd�|j|jf}|j|�}|�dddd�S)N�����r�r+�)�sizerHrK�view�permute)r1rZnew_x_shaperrr�transpose_for_scoresqs27z'RobertaCoAttention.transpose_for_scorescCs�|�|�}|�|�}|�|�}|�|�}|�|�}|�|�}	t�||�dd��}28|29t�|j	�}30|31|}32t33jdd�|34�}|�|�}t�||	�}|�
dddd���}|��dd�|jf}
|j|
�}|S)NrT�������dimrrUr+rV)rNrOrPrZr
�matmul�	transposerrrKr�SoftmaxrSrY�35contiguousrWrLrX)r1rCrDrEZmixed_query_layerZmixed_key_layerZmixed_value_layerZquery_layerZ	key_layerZvalue_layerZattention_scoresZattention_probsZ
context_layerZnew_context_layer_shaperrrr;vs 3637383940414243zRobertaCoAttention.forward)r=r>r?r-rZr;r@rrr3rrF`srFcs$eZdZ�fdd�Zdd�Z�ZS)�RobertaCrossAttentioncs&tt|���t|�|_t|�|_dSr)r,rbr-rFr1r�outputr0r3rrr-�s44zRobertaCrossAttention.__init__cCs|�|||�}|�||�}|Sr)r1rc)r1Zs1_input_tensorZs2_input_tensorrEZs1_cross_output�attention_outputrrrr;�szRobertaCrossAttention.forwardr<rrr3rrb�srbcs$eZdZ�fdd�Zdd�Z�ZS)rBcs0tt|���t|�|_t|�|_t|�|_dSr)	r,rBr-rb�	attentionr�intermediaterrcr0r3rrr-�s4546z#RobertaCrossAttentionLayer.__init__cCs(|�|||�}|�|�}|�||�}|Sr)rerfrc)r1rCrDrErdZintermediate_outputZlayer_outputrrrr;�s47z"RobertaCrossAttentionLayer.forwardr<rrr3rrB�srBcs,eZdZdZd	�fdd�	Zd48dd�Z�ZS)�UMTz]Coupled Cross-Modal Attention BERT model for token-level classification with CRF on top.49    r+rUcs�tt|��|�||_t|�|_t|�|_t|�|_t	�50|j�|_t	�
d|j�|_t	�
d|j�|_t||�|_t||�|_t||�|_t	�
|jd|j�|_t	�
|jd|�|_t	�
|j|�|_t|dd�|_t|dd�|_|��dS)N�rUT)�batch_first)r,rgr-�51num_labelsr�robertar �self_attention�self_attention_v2rrQ�hidden_dropout_probrSrMrG�vismap2text�vismap2text_v2rA�txt2img_attention�img2txt_attention�txt2txt_attention�gate�52classifier�aux_classifierr�crf�aux_crfZinit_weights)r1r2�53layer_num1�54layer_num2�55layer_num3�num_labels_�
auxnum_labelsr3rrr-�s"565758zUMT.__init__Nc	*CsB|j|||d�}	|	d}59|�|60�}61|�d��d�}|jt|���jd�}d|d}|�|62|�}|d}
|
d	}
|�|
�}t	j63|t	j|jd64�}t	�
||�}|�|65|�}|d}|d	}|�ddd��d	dd�}|�|�}|dd�dd�f}|�d��d�}|jt|���jd�}d|d}|�|||�}|d}|�|�}|�|||�}|d}|�|||�}|d}t	j||fdd
�}t	�|�|��}t	�||�} t	j|| fdd
�}!|�|!�}"d}#t	�t	�|"|#�t	�|d|#��}$|du�r&d}%|j|||��dd�}&|j|$||��dd�}'|'|%|&}(|(S|jj|$|��d�})|)SdS)N)�token_type_idsr7�last_hidden_stater+rU)�dtyperg���rTr)r��devicerh�1r\r�mean)�mask�	reduction)r�) rkrS�	unsqueeze�to�next�66parametersr�rlrvr
�tensor�float32r�r^rmrXrYrorqrprrrs�catrrt�mulru�addrx�byterw�decode)*r1�	input_idsZsegment_ids�67input_maskZadded_attention_maskZvisual_embeds_att�trans_matrix�labelsZ	auxlabels�featuresZsequence_outputZextended_txt_maskZaux_addon_sequence_encoderZaux_addon_sequence_outputZaux_bert_featsZtrans_matrix_tensorZtrans_bert_featsZmain_addon_sequence_encoderZmain_addon_sequence_outputZ
vis_embed_mapZconverted_vis_embed_mapZimg_maskZextended_img_maskZ
cross_encoderZcross_output_layerZconverted_vis_embed_map_v2Zcross_txt_encoderZcross_txt_output_layerZcross_final_txt_encoderZcross_final_txt_layerZmerge_representationZ68gate_valueZgated_converted_att_vis_embedZfinal_outputZ69bert_feats�alphaZfinal_bert_feats�betaZaux_lossZ	main_loss�lossZ	pred_tagsrrrr;�sb�70717273��74 75zUMT.forward)r+r+r+rUrU)NN)r=r>r?�__doc__r-r;r@rrr3rrg�s�rg)2r��76__future__rrrrr"�json�loggingr�os�shutil�tarfile�tempfile�sys�iorZtorchcrfrr
r�torch.nnr	�torch.nn.functional�77functional�F�torch.autogradr78�	getLoggerr=�loggerrrrZACT2FN�transformersrZ,transformers.models.roberta.modeling_robertarrrrr�Moduler rArFrbrBrgrrrr�<module>s:79	6