CoolFace
Apppublic

Linhz/ViMNer

sourceHugging Faceupdated 2y agoView on Hugging Face
1likes
rootModelMultimodelNERVLSP2016__pycache__dataset_roberta.cpython-39.pyc
dataset_roberta.cpython-39.pyc91 linesDownload Raw Back to __pycache__
1a

2��df�=�@s�ddlZddlZddlZe�e�ZddlmZddlm	Z	Gdd�de3�ZGdd�de4�Zdd	�Z
Gd5d�de6�ZGdd
�d
e�Zdd�Zdd�ZdS)�N)�7transforms)�Imagec@seZdZdZddd�ZdS)�SBInputExamplezBA single training/test example for simple sequence classification.NcCs(||_||_||_||_||_||_dS)a Constructs a InputExample.8 9        Args:10            guid: Unique id for the example.11            text_a: string. The untokenized text of the first sequence. For single12            sequence tasks, only this sequence must be specified.13            text_b: (Optional) string. The untokenized text of the second sequence.14            Only must be specified for sequence pair tasks.15            label: (Optional) string. The label of the example. This should be16            specified for train and dev examples, but not for test examples.17        N��guid�text_a�text_b�img_id�label�auxlabel)�selfrrrr	r18r�r
�KE:\demo_datn\pythonProject1\Model\MultimodelNER\VLSP2016\dataset_roberta.py�__init__
szSBInputExample.__init__)NN��__name__�19__module__�__qualname__�__doc__rr
r
r
rr20src@seZdZdZdd�ZdS)�SBInputFeaturesz A single set of features of datacCs.||_||_||_||_||_||_||_dS)N��	input_ids�21input_mask�added_input_mask�segment_ids�img_feat�label_id�auxlabel_id)rrrrrrrrr
r
rr&szSBInputFeatures.__init__Nrr
r
r
rr#src
CsLtd|�t|dd�}g}g}g}g}g}g}d}|D]�}	|	��}	|	�d�r`|	�d�dd}q6|	dkr�t|�dkr6|�||f�|�|�|�|�g}g}g}d}q6|	�d	�}22t|23�d24kr6|25\}}|�|�|�|�|�|d�q6t|�dk�r|�||f�|�|�|�|�tdtt|���tdtt|���|||fS)
u�26    Đọc dữ liệu từ tệp và trả về dưới dạng danh sách các cặp từ và nhãn, cùng với danh sách hình ảnh và nhãn phụ.27    uChuẩn bị dữ liệu cho �utf8)�encoding�zIMGID:�z.jpgr�	�uSố lượng mẫu: uSố lượng hình ảnh: )�print�open�strip�28startswith�split�len�append�str)
�filename�f�data�imgs�	auxlabels�sentencer29rZimgid�line�splits�wordZ	cur_labelr
r
r�30sbreadfile0sJ313233343536373839r5c@s6eZdZdZdd�Zdd�Zdd�Zedd	d40��ZdS)�
DataProcessorzEBase class for data converters for sequence classification data sets.cCs41t��dS)z7Gets a collection of `InputExample`s for the train set.N��NotImplementedError�r�data_dirr
r
r�get_train_examples�sz DataProcessor.get_train_examplescCs42t��dS)z5Gets a collection of `InputExample`s for the dev set.Nr7r9r
r
r�get_dev_examples�szDataProcessor.get_dev_examplescCs43t��dS)z*Gets the list of labels for this data set.Nr7�rr
r
r�44get_labels�szDataProcessor.get_labelsNcCst|�S)z!Reads a tab separated value file.)r5)�cls�45input_file�	quotecharr
r
r�_read_sbtsv�szDataProcessor._read_sbtsv)N)	rrrrr;r<r>�classmethodrBr
r
r
rr6�sr6c@sPeZdZdZdd�Zdd�Zdd�Zdd	�Zd46d�Zdd
�Z	dd�Z47dd�ZdS)�MNERProcessor_2016z&Processor for the CoNLL-2003 data set.cCs*|�tj�|d��\}}}|�|||d�S)�See base class.z	train.txt�train�rB�os�path�join�_create_examples�rr:r.r/r0r
r
rr;�sz%MNERProcessor_2016.get_train_examplescCs*|�tj�|d��\}}}|�|||d�S)rEzdev.txt�devrGrLr
r
rr<�sz#MNERProcessor_2016.get_dev_examplescCs*|�tj�|d��\}}}|�|||d�S)rEztest.txt�testrGrLr
r
r�get_test_examples�sz$MNERProcessor_2016.get_test_examplescCsgd�S)N)zB-ORGzB-MISCzI-PERzI-ORGzB-LOCzI-MISCzI-LOC�OzB-PER�X�<s>�</s>r
r=r
r
rr>�sXzMNERProcessor_2016.get_labelscCsgd�S)N)rP�B�IrQrRrSr
r=r
r
r�
get_auxlabels,sz MNERProcessor_2016.get_auxlabelscCs$|��}dd�t|d�D�}|dS)NcSsi|]\}}||�qSr
r
��.0�ir48r
r
r�49<dictcomp>1�z9MNERProcessor_2016.get_start_label_id.<locals>.<dictcomp>r!rR�r>�	enumerate�r�50label_list�	label_mapr
r
r�get_start_label_id/sz%MNERProcessor_2016.get_start_label_idcCs$|��}dd�t|d�D�}|dS)NcSsi|]\}}||�qSr
r
rWr
r
rrZ6r[z8MNERProcessor_2016.get_stop_label_id.<locals>.<dictcomp>r!rSr\r^r
r
r�get_stop_label_id4sz$MNERProcessor_2016.get_stop_label_idcCsfg}t|�D]T\}\}}d||f}	d�|�}51d}||}|}||}
|�t|	|52||||
d��q|S)Nz%s-%s� r)r]rJr*r)r�linesr/r0�set_type�examplesrYr1r53rrrr	rr
r
rrK9s54�z#MNERProcessor_2016._create_examplesN)rrrrr;r<rOr>rVrarbrKr
r
r
rrD�ssrDcCst�|��d�}||�}|S)N�RGB)rr%�convert)�55image_path�	transform�imager
r
r�
image_processGsrlc%Cs^dd�t|d�D�}dd�t|d�D�}g}	d}56t�t�ddg�t�|�t��t��t�dd�g�}t|�D�]�\}}
|
j�	d	�}|
j57}|
j}g}g}g}t|�D]p\}}|�|�}|�
|�||}||}tt|��D]6}|dkr�|�|�|�|�q�|�d58�|�d59�q�q�t|�|dk�rV|d|d�}|d|d�}|d|d�}g}g}g}g}|�d�|�d�|�|d�|�|d�t|�D]B\}}|�|�|�d�|�|||�|�|||��q�|�d
�|�d�|�|d
�|�|d
�|�|�}dgt|�}dgt|�d} t|�|k�r�|�d�|�d�| �d�|�d�|�d�|�d��q<t|�|k�s�J�t|�|k�s�J�t|�|k�s�J�t|�|k�s�J�t|�|k�s�J�|
j}!tj�||!�}"tj�|"��sd|"v�rt|"�zt|"|�}#Wn,|60d7}61tj�|d�}$t|$|�}#Yql0|dk�r,t�d�t�d|
j�t�dd	�dd�|D���t�dd	�dd�|D���t�dd	�dd�|D���t�dd	�dd�|D���t�dd	�dd�|D���t�dd	�dd�|D���|	�t||| ||#||d ��qltd!t|62��|	S)"NcSsi|]\}}||�qSr
r
rWr
r
rrZOr[z3convert_mm_examples_to_features.<locals>.<dictcomp>r!cSsi|]\}}||�qSr
r
rWr
r
rrZPr[r�)g63ףp=64�?g�v��/�?g�C�l���?)gZd;�O�?gy�&1��?g�������?rcrQr#rRrS�1�NaNzbackground.jpgz*** Example ***zguid: %sz65tokens: %scSsg|]}t|��qSr
�r+�rX�xr
r
r�66<listcomp>�r[z3convert_mm_examples_to_features.<locals>.<listcomp>z
input_ids: %scSsg|]}t|��qSr
rprqr
r
rrs�r[zinput_mask: %scSsg|]}t|��qSr
rprqr
r
rrs�r[zsegment_ids: %scSsg|]}t|��qSr
rprqr
r
rrs�r[z	label: %scSsg|]}t|��qSr
rprqr
r
rrs�r[zauxlabel: %scSsg|]}t|��qSr
rprqr
r
rrs�r[rz#the number of problematic samples: )r]rZCompose�ResizeZ67RandomCropZRandomHorizontalFlipZToTensorZ	Normalizerr(r68r�tokenize�extend�ranger)r*�convert_tokens_to_idsr	rHrIrJ�existsr$rl�logger�inforrr+)%rfr_�
auxlabel_list�max_seq_length�	tokenizerZ	crop_sizeZpath_imgr`Zauxlabel_map�features�countrjZex_index�exampleZtextlistZ	labellistZauxlabellist�tokens�labelsr0rYr4�tokenZlabel_1Z69auxlabel_1�mZntokensrZ	label_idsZauxlabel_idsrrr�70image_namerirkZimage_path_failr
r
r�convert_mm_examples_to_featuresMs���7172737475767778798081828384858687888990����r�)�torch�loggingrH�	getLoggerrrz�torchvisionr�PILr�objectrrr5r6rDrlr�r
r
r
r�<module>s91
b!