CoolFace
Apppublic

YuWang0103/LGGM-Text2Graph

sourceHugging Facemitupdated 2y agoView on Hugging Face
1likes
transformer_model.cpython-39.pyc117 linesDownload Raw Back to __pycache__
1a

2�'XfX,�@s�ddlZddlZddlmZddlmZddlmZddlm	Z	ddlm3ZddlmZddl
Z
ddlmZddlmZmZmZGd	d4�d5ej�ZGdd�dej�ZGd
d�dej�ZdS)�N)�Dropout)�Linear)�	LayerNorm)�6functional)�Tensor)�diffusion_utils)�Xtoy�Etoy�masked_softmaxcsLeZdZdZd
eeeeeeeeedd�7�fdd	�
Zeeed8�dd�Z�Z	S)�XEyTransformerLayera� Transformer that updates node, edge and global features9        d_x: node features10        d_e: edge features11        dz : global features12        n_head: the number of heads in the multi_head_attention13        dim_feedforward: the dimension of the feedforward network model after self-attention14        dropout: dropout probablility. 0 to disable15        layer_norm_eps: eps value in layer normalizations.16    ��皙�����?��h㈵��>N)17�dx�de�dy�n_head�dim_ffX�dim_ffE�dim_ffy�dropout�layer_norm_eps�returnc
s�|18|d�}t���t||||fi|��|_t||fi|��|_t||fi|��|_t|fd|	i|��|_t|fd|	i|��|_	t19|�|_t20|�|_t21|�|_
t||fi|��|_t||fi|��|_t|fd|	i|��|_t|fd|	i|��|_t22|�|_t23|�|_t24|�|_t||fi|��|_t||fi|��|_t|fd|	i|��|_t|fd|	i|��|_t25|�|_t26|�|_t27|�|_tj|_dS)N)�device�dtype�eps)�super�__init__�
NodeEdgeBlock�	self_attnr�linX1�linX2r�normX1�normX2r�	dropoutX1�	dropoutX2�	dropoutX3�linE1�linE2�normE1�normE2�	dropoutE1�	dropoutE2�	dropoutE3�lin_y1�lin_y2�norm_y1�norm_y2�28dropout_y1�29dropout_y2�30dropout_y3�F�relu�31activation)
�selfrrrrrrrrrrr�kw��	__class__��A/home/yu/Dropbox/Project/LGGM-hg-demo/models/transformer_model.pyrs23233343536373839404142zXEyTransformerLayer.__init__)�X�E�	node_maskc	Cs|j||||d�\}}}|�|�}|�||�}|�|�}	|�||	�}|�|�}43|�||44�}|�|�|�	|�45|����}|�|�}|�||�}|�
|�|�	|�|����}|�|�}|�||�}|�|�|�	|�|����}
|�|
�}
|�||
�}|||fS)a Pass the input through the encoder layer.46            X: (bs, n, d)47            E: (bs, n, n, d)48            y: (bs, dy)49            node_mask: (bs, n) Mask for the src keys per batch (optional)50            Output: newX, newE, new_y with the same shape.51        )rA)r r%r#r,r*r3r1r"r&r8r!r'r$r)r-r(r.r+r0r4r/r5r2)r9r?r@�yrA�newX�newE�new_yZnewX_dZnewE_dZnew_y_dZ52ff_outputXZ53ff_outputEZff_output_yr=r=r>�forward<s"	545556575859zXEyTransformerLayer.forward)rr
rrrNN)60�__name__�61__module__�__qualname__�__doc__�int�floatrrrF�
__classcell__r=r=r;r>rs	��"rcs(eZdZdZ�fdd�Zdd�Z�ZS)rzJ Self attention layer that also updates the representations on the edges. cs t���||dks*Jd|�d|����||_||_||_t||�|_||_t||�|_	t||�|_62t||�|_t||�|_t||�|_
t||�|_t||�|_t||�|_t||�|_t||�|_t||�|_t||�|_t||�|_t||�|_t�t�||�t��t�||��|_dS)Nrzdx: z -- nhead: )rrrrrrK�dfrr�q�k�v�e_add�e_mul�y_e_mul�y_e_add�y_x_mul�y_x_add�y_yr�x_yr	�e_y�x_out�e_out�nn�63Sequential�ReLU�y_out)r9rrrr�kwargsr;r=r>ras,64 zNodeEdgeBlock.__init__cCs�|j\}}}|�d�}|�d�}	|�d�}65|�|�|}|�|�|}t�||�|�|�d�|�d�|j|j	f�}|�|�d�|�d�|j|j	f�}|�d�}|�d�}||}
|
t66�|
�d��}
t�|
|	|67�d��|�|�|	|68}|�|�d�|�d�|�d�|j|j	f�}|�
|�|	|69}|�|�d�|�d�|�d�|j|j	f�}|
|d|}
|
jdd�}|�|��d��d�}|�|��d��d�}||d|}|�|�|	|70}t�||	|71�|72�d|d|j�}t|
|dd�}|�|�|}|�|�d�|�d�|j|j	f�}|�d�}||}|jdd�}|jdd�}|�|��d�}|�|��d�}||d|}|�|�|}t�||�|�|�}|�|�}|�|�}|||}|�|�}|||fS)z�73        :param X: bs, n, d        node features74        :param E: bs, n, n, d     edge features75        :param y: bs, dz           global features76        :param node_mask: bs, n77        :return: newX, newE, new_y with the same shape.78        �������r�)�	start_dim)�dim)�shape�	unsqueezerOrPrZassert_correctly_masked�reshape�sizerrN�math�sqrtrSrR�flattenrUrTr\�expandr79rQ�sumrWrVr[rXrZrYr`)r9r?r@rBrA�bs�n�_�x_mask�e_mask1�e_mask2�Q�K�Y�E1�E2rDZye1Zye2Zsoftmax_mask�attn�VZ80weighted_VZyx1Zyx2rCrZrYrEr=r=r>rF�sX818283""8485**"8687888990zNodeEdgeBlock.forward)rGrHrIrJrrFrMr=r=r;r>r_s$rc	sDeZdZdZeeeeeee��e��d��fdd�Zdd�Z	�Z91S)�GraphTransformerzi92    n_layers : int -- number of layers93    dims : dict -- contains dimensions for each feature type94    )�n_layers�95input_dims�	cond_dims�hidden_mlp_dims�hidden_dims�output_dims�	act_fn_in�96act_fn_outc			s~t���||_|d|_|d|_|d|_t�t�|d||d�|t�|d�d�|�|_	t�t�|d||d�|t�|d�d�|�|_97t�t�|d|d�|t�|d�d�|�|_t��fdd�t
|�D��|_t�t��d|d�|t�|d|d��|_t�t��d|d�|t�|d|d��|_t�t��d|d�|t�|d|d��|_dS)	Nr?r@rBrrrc98s8g|]0}t�d�d�d�d�d�dd��qS)rrrrrr)rrrrrr)r)�.0�i�r�r=r>�99<listcomp>�s��z-GraphTransformer.__init__.<locals>.<listcomp>)rrr�	out_dim_X�	out_dim_E�	out_dim_yr]r^r�mlp_in_X�mlp_in_E�mlp_in_y�100ModuleList�range�	tf_layers�	mlp_out_X�	mlp_out_E�	mlp_out_y)	r9rr�r�r�r�r�r�r�r;r�r>r�s4101102103104����105���zGraphTransformer.__init__cCs^|jd|jd}}t�|�}|�|���}|�d��d��|ddd�}|dd|j�f}|dd|j�f}	|dd|j	�f}106|�107|�}||�dd�d}tj
|�|�||�|�d��|�}|j|j|j}}}|jD]}
|
||||�\}}}q�|�|�}|�|�}|�|�}||}||	|}||108}d|t�|dd�}tj
|||d��|�S)Nrrdrb.rc)r?r@rBg�?)rh�torch�eye�type_as�boolriror�r�r�r��	transpose�utils�PlaceHolderr�r��maskr?r@rBr�r�r�r�)r9r?r@rBrArqrr�	diag_maskZX_to_outZE_to_outZy_to_outZnew_EZafter_in�layerr=r=r>rF�s*109110"111112113114zGraphTransformer.forward)rGrHrIrJrK�dictr]r_rrFrMr=r=r;r>r~�s115116�"r~)rlr��torch.nnr]Ztorch.nn.modules.dropoutrZtorch.nn.modules.linearrZtorch.nn.modules.normalizationrrr6rr��	diffusionrZ
models.layersrr	r117�Modulerrr~r=r=r=r>�<module>sOy