CoolFace
Apppublic

xdecoder/Instruct-X-Decoder

sourceHugging Faceafl-3.0updated 3y agoView on Hugging Face
163likes
xdecoder_model.cpython-38.pyc138 linesDownload Raw Back to __pycache__
1U

2E$�c�l�@s$ddlZddlmZddlmZddlZddlmZddlmZ	ddl3Zddlm
Z
dd	lmZdd4lmZmZddlmZddlmZmZdd
lmZddlmZddlmZddlmZddl m!Z!m"Z"m#Z#m$Z$m%Z%ddl&m'Z'ddl(m)Z)ddl*m+Z+e�Z,Gdd�dej-�Z.e
dd��Z/dS)�N)�Tuple)�name)�nn)�5functional�)�register_model�)�configurable)�build_backbone�Backbone)�build_xdecoder_head)�sem_seg_postprocess�bbox_postprocess)�build_language_encoder)�
vl_similarity)�
trunc_normal_)�LancasterStemmer)�Boxes�	ImageList�	Instances�BitMasks�BoxMode)�retry_if_cuda_oom)�MetadataCatalog)�prompt_engineeringcs�eZdZeeejejeee	e	ee	ee6ee	ee	e7e8e9eee10eed��fdd��Z
edd��Zedd��Zd d	d11�Zdd�Zd
d�Zifdd�Zdd�Zdd�Zdd�Zdd�Zd!dd�Zdd�Zdd�Z�ZS)"�X_Decoder_Model)�backbone�sem_seg_head�	criterion�losses�num_queries�object_mask_threshold�overlap_threshold�task_switch�phrase_prob�size_divisibility�$sem_seg_postprocess_before_inference�12pixel_mean�	pixel_std�semantic_on�panoptic_on�instance_on�test_topk_per_image�train_dataset_name�retrieval_emsemble�backbone_dim�dim_projcst���||_||_||_||_||_||_||_||_	|dkrJ|jj13}||_14||_|�dt
�|
��ddd�d�|�dt
�|��ddd�d�||_||_||_|	|_|15|_||_d|_||_|	dr�|r�t�t
�||��|_t|jdd	�|j�s|j�st�dS)16Nrr'�����rFr(�	retrievalg{�G�z�?)�std)�super�__init__rrrrr r"r!�metadatar%r&�register_buffer�torch�Tensor�viewr)r+r*r#r$r,Ztrain_class_namesr.r�	Parameter�empty�
backbone_projr�AssertionError)�selfrrrrr r!r"r6r#r$r%r&r'r(r)r*r+r,r-r.r/r0��	__class__��>/data/arXiv/demo/Demo/xdecoder/architectures/xdecoder_model.pyr5$s817zX_Decoder_Model.__init__cCs~|dd}|dd}|�dd�|�dd�|d�d	d�|d18�d	d�|d�d	d�|d�d	d�d
�}d|i}t|�}t|�}t||��||�}i}	d}19i}i}i}
i}d}d}d}d}d}|||||d|dd|ddd|d|dd�p|dd�p|dd|dd|dd|||dd|dd|dd|dddd||dd|dd|ddd�S) NZMODELZENCODERZDECODERZ	DETECTIONF�MASKTZCAPTIONZENABLEDZ20CAPTIONINGZ	RETRIEVALZ	GROUNDING)�bbox�mask�caption�21captioningr2�	groundingr#ZNUM_OBJECT_QUERIESZTESTZOBJECT_MASK_THRESHOLDZOVERLAP_THRESHOLDZSIZE_DIVISIBILITYZ'SEM_SEG_POSTPROCESSING_BEFORE_INFERENCEZPANOPTIC_ONZINSTANCE_ONZINPUTZ22PIXEL_MEANZ	PIXEL_STDZSEMANTIC_ONZDETECTIONS_PER_IMAGEZENSEMBLEZBACKBONE_DIMZDIM_PROJ)rrrrr r!r"r6r%r&r'r(r#r$r)r+r*r,r-r.r/r0)�getr23rrZoutput_shape)�cls�cfgZenc_cfgZdec_cfgr#�extrar�lang_encoderrZloss_weights�matcherr�weight_dictZ24grd_weightZtop_x_layersrr-r$Zdeep_supervisionZno_object_weightrBrBrC�from_configfsd2526�272829�303132333435363738�zX_Decoder_Model.from_configcCs|jjS�N)r'�device�r?rBrBrCrS�szX_Decoder_Model.deviceNcCsl|jrdshtd��nT|dkr&|�|�S|dkr8|�|�S|dkrJ|�|�S|dkr^|�||�S|�|�SdS)NFzNot support trianing mode.r2rHZclassification)�grounding_phrasecut�grounding_refcoco)�trainingr>�evaluate_retrieval�evaluate_captioning�evaluate_classification�evaluate_grounding�evaluate)r?�batched_inputs�moderBrBrC�forward�s394041zX_Decoder_Model.forwardcs^�fdd�|D�}�fdd�|D�}t�|�j�}|jjd}d}}}��|j�}�j||d�}|d}	|d}42�jdr�|d	nd43d�tt	|44��D�}�jdr�|dnd
d�tt	|45��D�}t46j|47|jjd|jjdfddd�}48|49jdd�}
t�j
d��r�j
jnd}~g}t|	|50||||j�D�]*\}}}}}}|�d|d�}|�d|d�}|�i��j�r�tt�||||�}|�|�}�j�r�t�j�|||�}�j�s�tt�||||�}||dd<�j�r�t�j�||�}||dd<�j�r2�jd�rt||
|||�}t�j�|||�}||dd<�jd�r,||dd<||dd<�q,|S)Ncsg|]}|d��j��qS��image��torS��.0�xrTrBrC�51<listcomp>�sz,X_Decoder_Model.evaluate.<locals>.<listcomp>csg|]}|�j�j�qSrB�r'r(rdrTrBrCrg�sr��target_queries�pred_logits�52pred_masksrE�53pred_boxescSsg|]}d�qSrRrB�re�irBrBrCrg�srG�
pred_captionscSsg|]}d�qSrRrBrnrBrBrCrg�s�����r1�bilinearF��sizer^�
align_corners�keep_sem_bgd�height�widthrZsem_seg�panoptic_seg�	instances�captions�masks)r�from_tensorsr%�tensor�shaperrr#�range�len�F�interpolate�hasattrr6rv�zip�image_sizesrJ�appendr&rr
rcr)�semantic_inferencer*�panoptic_inferencer+r�instance_inference)r?r]�images�img_bs�targets�targets_grounding�queries_grounding�features�outputsZmask_cls_results�mask_pred_resultsZbox_pred_results�caption_pred_results�54input_sizerv�processed_resultsZmask_cls_result�mask_pred_resultZbox_pred_result�caption_pred_result�input_per_image�55image_sizerwrx�rZ56panoptic_rZ57instance_rrBrTrCr\�sr((��58�59zX_Decoder_Model.evaluatecs��fdd�|D�}�fdd�|D�}t�|�j�}|jjd}d}}}��|j�}�j||d�}|ddd�df}	�jdrƈjr�|d	}60|61j\}}}
}
|62�	||d�}63t64�|65d66��	||��j}67g}t
|�D]�\}}g}g}|�i�|dD]4}�jjj�|�}|d}|�|d
�|�|�q�tj|dd�}|	|�d�g}�jd�rp�j�rp||68|�d�g7}||||d
d�}||dd<q�|S)Ncsg|]}|d��j��qSr`rbrdrTrBrCrg�sz6X_Decoder_Model.evaluate_retrieval.<locals>.<listcomp>csg|]}|�j�j�qSrBrhrdrTrBrCrg�srrirpr1r2Zres5rr{�	class_emb�image_id��dim)�image_embedsZtext_embeds�caption_idsZ	image_idsrG)rr}r%r~rrrr#r.�reshaper��adaptive_avg_pool1dr=�	enumerater��	predictorrN�get_text_token_embeddingsr8�cat�	unsqueeze)r?r]r�r�r�r�r�r�r�Zv_emb_itZ	_v_emb_it�bs�nc�_r��idx�69batch_datar�Z	t_emb_itsrG�lang_resultsZt_emb_itr�Zcaption_resultsrBrTrCrX�sD70�z"X_Decoder_Model.evaluate_retrievalcs,�fdd�|D�}�fdd�|D�}t�|�j�}|jjd}t�d�sbtjdgdg�jd��_d}}}��	|j�}d}	d	|d71kr�t�72dd�|D��}	|��j|	d���j||d
|d�}73g}t
|�D]X\}}
|�i�|74d||d75d<|76d|�d�d|d77d<||d|d78d<q�|S)Ncsg|]}|d��j��qSr`rbrdrTrBrCrg&sz7X_Decoder_Model.evaluate_captioning.<locals>.<listcomp>csg|]}|�j�j�qSrBrhrdrTrBrCrg'sr�start_tokeni���M�rS�captioning_maskr1cSsg|]}|d�qS)r�rBrdrBrBrCrg3s)r�r�Zcaptioning_infer)rj�taskrMZpred_captioningsZcaptioning_tokenZ79pred_texts�.Zcaptioning_textr�)rr}r%r~rr�r8rSr�rr��updaterr�r��split)r?r]rMr�r�r�r�r�r�r�r�r�r�r�rBrTrCrY%s(8081z#X_Decoder_Model.evaluate_captioningcs��fdd�|D�}�fdd�|D�}t�|�j�}|jjd}d}}}��|j�}�j||d�}g}	t|�D]*\}82}|	�i�|d|83df|	dd<qp|	S)	Ncsg|]}|d��j��qSr`rbrdrTrBrCrgBsz;X_Decoder_Model.evaluate_classification.<locals>.<listcomp>csg|]}|�j�j�qSrBrhrdrTrBrCrgCsrrirkr1�84pred_class)	rr}r%r~rrrr�r�)r?r]r�r�r�r�r�r�r�r�r�r�rBrTrCrZAs85z'X_Decoder_Model.evaluate_classificationcs��fdd�|D�}�fdd�|D�}t�|�j�}|jjd}d}}}��|j�}�j||d�}	|	d}86�jdr~|	dnd	d�tt	|87��D�}t88j|89|jjd90|jjdfdd
d�}91g}t|92|||j
�D�](\}
}}}|�d|d�}|�d|d�}|�i�tt�|
|||�dd�}
|dd}g}|D]�}|dk�r\�jjjj|dd
dd�n"|dk�r~�jjjj|ddd
d�t�jjjd�d����}|dd�}||jddd�d}||}|�d�dd}||
|g7}�q4t�|�|dd<q�|S)Ncsg|]}|d��j��qSr`rbrdrTrBrCrgRsz?X_Decoder_Model.evaluate_grounding_baseline.<locals>.<listcomp>csg|]}|�j�j�qSrBrhrdrTrBrCrgSsrrirlrGrpcSsg|]}d�qSrRrBrnrBrBrCrg\srqr1rrFrsrwrxr�93groundings�textsrVrIT)r�prompt�is_evalrUz{}_text_embeddings�r��keepdim�H�����z>�grounding_mask)rr}r%r~rrrr#r�r�r�r�r�r�rJr�rr
r�rN�get_text_embeddings�getattr�format�t�norm�maxr8�stack)r?r]r^r�r�r�r�r�r�r�r�r�r�r�r�r�r�rwrxZ	texts_allZ	grd_masksr��t_emb�v_embZvt_simZmax_idrBrTrC�evaluate_grounding_baselineQs`(��94��9596z+X_Decoder_Model.evaluate_grounding_baselinecs>�fdd�|D�}�fdd�|D�}t�|�j�}i}g}t|�D�]B\}}|dd}dd�|D�}�jjjj|dddd	�}	|	d97}98|	d}|99|d��}|dd�df|d
<��	|j100�}
�j|
|dd�}|d|�jd�jd�f}|d|�jd�jd�f}|	d}||jddd�d}||jddd�d}�jjjj
}t|||d�}|�d�d}|||dd�dd�fg7}qBtt|��D]<}tj||d|j101jd|j102jdfddd�d||<�q�g}t|||j�D]T\}}}|�d|d�}|�d |d�}|�i�tt�||||�}||dd!<�q�|S)"Ncsg|]}|d��j��qSr`rbrdrTrBrCrg�sz6X_Decoder_Model.evaluate_grounding.<locals>.<listcomp>csg|]}|�j�j�qSrBrhrdrTrBrCrg�sr�r�cSsg|]}|d�qS)rrBrdrBrBrCrg�srIF)r�tokenr��	token_emb�tokens�attention_maskZgrounding_tokensZgrounding_eval)rMr�rlrrrpr�r1Tr�r�)�temperaturerrRrqrrrsrwrxr�)rr}r%r�rr�rNr��boolrr~r r�Zlogit_scalerr�r�r�r�r�rr�r�rJr�rr
)r?r]r^r�rMr�r�Zbatch_per_imageZ	grd_textsZgtextr�r�Z	query_embr�r�Zpred_gmasksr�r�r�Zout_probZ103matched_idror�r�r�r�rwrxrBrTrCr[�sd   104���105�z"X_Decoder_Model.evaluate_groundingc
Cs0g}g}t|�D]\\}}|d}t�dt|�d�}||dd||d�7}||dd||d�7}qt�|�}t�|�}||d�}	|jjjj	|	dd	�}106g}t|�D]~\}}i}|107d108||d�|d<|109d||d�|d
<|110dd||d�|d<|111dd||d�|d<|�112|�q�|S)Nr{rrr��	input_idsr�)r�r�T)r�r�Zcaption_tokensr�Zcaption_projZcaption_tokenidsZcaption_mask)r��random�randintr�r8r�rr�rNr�r�)
r?r]rSr�r��cntrfr{Zrandidr�r�Z113target_vlp�target_dictrBrBrC�prepare_vlp_targets�s(114115116z#X_Decoder_Model.prepare_vlp_targetsFcCsH|rtj|dd�}ntj|dd�ddd�f}|��}t�d||�}|S)Nr1r�.zqc,qhw->chw)r��softmax�sigmoidr8�einsum)r?�mask_cls�	mask_predrvZsemsegrBrBrCr��sz"X_Decoder_Model.semantic_inferencecCs�tj|dd��d�\}}|��}|�|jj�||jk@}||}||}||}||}	|	dd�dd�f}	|�ddd�|}117|j	dd�\}}t118j||ft119j|j
d�}
g}d}|j	ddkr�|
|fS|120�d�}i}t|jd�r�|jjni}t|j	d�D]�}||��}||��k}||k����}||dk����}||k||dk@}|dkr�|dkr�|����dkr�|||jk�r�q�|�s�t|�|��k�r�|t|�|
|<q�n|d|t|�<|d7}||
|<|�|t|�t|�d	��q�|
|fSdS)121Nr1r�rrq)�dtyperSr�!thing_dataset_id_to_contiguous_idg�?)�id�isthingZcategory_id)r�r�r�r��ner�num_classesr!r:rr8�zeros�int32rS�argmaxr�r6r�r��item�values�sumr"�int�keysr�r�)r?r�r��scores�labels�keepZ122cur_scoresZcur_classesZ	cur_masksZcur_mask_clsZcur_prob_masks�h�wryZ
segments_infoZcurrent_segment_idZcur_mask_idsZstuff_memory_listr��kr�r�Z	mask_areaZ
original_arearFrBrBrCr�sR123 ��z"X_Decoder_Model.panoptic_inferencecCs�|jdd�}tj|dd�dd�dd�f}tj|jj|jd��d��	|j124d��dd�}|�dd�j|j
dd�\}}||}	||jj}||}|dk	r�||}|j�rt|jd	�r�|jjni}125t�|���}t|	�D]\}}
|
|126��k||<q�||}|	|}	||}|dk	�r||}t|�}|dk��|_|dk	�rNt|dk���|_ntt�|�d�d127��|_|���d�|j�d�� d�|j�d�� d�d}|||_!|	|_"|S)Nrqr1r�r�rrF)�sortedr��g�����ư>)#rr�r�r8�arangerr�rSr��repeatr �flatten�topkr,r*r�r6r��128zeros_liker�r�r�r�floatrlrZget_bounding_boxesrmrr�rtr�r�r�Zpred_classes)r?r�r�Zbox_predr�r�r�Zscores_per_imageZtopk_indicesZlabels_per_imager�r�ro�lab�resultZmask_scores_per_imagerBrBrCr�:s8,1291306131z"X_Decoder_Model.instance_inference)N)F)�__name__�132__module__�__qualname__r	rr�Module�dictr�r�r�r�strr5�classmethodrQ�propertyrSr_r\rXrYrZr�r[r�r�r�r��
__classcell__rBrBr@rCr#sL�A133=134135C+2]136	9rcKst|�SrR)r)rL�kwargsrBrBrC�get_segmentation_modellsr)0r��typingr�unicodedatarr8r�torch.nnrr��numpy�np�registryr�utilsr	rr137r�bodyr�modulesr
r�languagerZ
language.lossrZtimm.models.layersrZnltk.stem.lancasterrZdetectron2.structuresrrrrrZdetectron2.utils.memoryrZdetectron2.datarZ138utils.miscr�str�rrrBrBrBrC�<module>s6M