CoolFace
Apppublic

shekkari21/codereviewer

sourceHugging Faceupdated 1y agoView on Hugging Face
0likes
utils.cpython-39.pyc256 linesDownload Raw Back to __pycache__
1a

2��h���@s~ddlZddlZddlZddlZddlZddlZddlmZddl	m3Z4ddlmZddl
mZmZddlZejddejd�e�e�ZGd	d5�d6e�ZGdd�de�ZGd
d�de7�ZGdd�de�ZGdd�de�ZGdd�de8�ZGdd�de�ZGdd�de�ZGdd�de�ZGdd�de�Z Gdd�de�Z!Gdd �d e�Z"Gd!d"�d"e�Z#Gd#d$�d$e�Z$d*d&d'�Z%d(d)�Z&dS)+�N)�deepcopy)�Dataset)�ByteLevelBPETokenizer)�T5Tokenizer�RobertaTokenizerz6%(asctime)s - %(levelname)s - %(name)s -   %(message)sz%m/%d/%Y %H:%M:%S)�format�datefmt�levelc@s^eZdZdZddd�Zedd��Zdd�Zd	d9�Zdd�Z	d
d�Z10dd�Zdd�Zdd�Z
dS)�MyTokenizerz+11    Wrapper for ByteLevelBPETokenizer12    NcKs t||fi|��|_|��dS�N)r�	tokenizer�update_id2token)�self�vocabZmerges�kwargs�r�1/Users/akhilshekkari/Desktop/github-test/utils.py�__init__szMyTokenizer.__init__cCs*tj�|d�}tj�|d�}t||�}|S)Nz13vocab.jsonz14merges.txt)�os�path�joinr15)rZvocabpZmergespZmytokenrrr�from_pretraineds16zMyTokenizer.from_pretrainedcs"|j����fdd��D�|_dS)Ncsi|]}�||�qSrr)�.0�token�rrr�17<dictcomp>$�z/MyTokenizer.update_id2token.<locals>.<dictcomp>)r�	get_vocab�id2token�rrrrr
"s18zMyTokenizer.update_id2tokencCs&|��D]}|j�|�q|��dSr)�valuesr�add_special_tokensr
)r�dicr rrrr!&szMyTokenizer.add_special_tokenscs|j��fdd�|D�S)Ncsg|]}�|�qSrr)r�irrr�19<listcomp>-rz5MyTokenizer.convert_ids_to_tokens.<locals>.<listcomp>)r)r�idsrrr�convert_ids_to_tokens+sz!MyTokenizer.convert_ids_to_tokenscKs|�|�}d�|�S�N� )r&r)rr%r�tokensrrr�decode/s20zMyTokenizer.decodecKs"|jddd��d�}|j�|�jS)N�ascii�ignore)�errors)�encoder*rr%)r�textrrrrr.3szMyTokenizer.encodecCs21|j��Sr)rrrrrrr7szMyTokenizer.get_vocabcCst|j���Sr)�lenrrrrrr�__len__:szMyTokenizer.__len__)NN)�__name__�22__module__�__qualname__�__doc__r�staticmethodrr
r!r&r*r.rr1rrrrr23s2425r26c@seZdZdd�ZdS)�RefineFeaturescCs||_||_||_dSr)�27example_id�28source_ids�29target_ids)rr8r9r:rrrr?szRefineFeatures.__init__N�r2r3r4rrrrrr7>sr7c@sJeZdZddd�Zdd�Zedd��Zdd	�Zd30d�Zdd
�Z	dd�Z31dS)�
RefineDataset�����cs��|_�|_t�d�|��dd�t|�D�}tt|��D]}d||vr:|||d<q:|dkrl|d|�}t�d|���|�|j	��fdd�|D��|_32dS)N�Reading examples from {}cSsg|]}t�|��qSr)�json�loads�r�linerrrr$Irz*RefineDataset.__init__.<locals>.<listcomp>�idr�Tokenize examples: csg|]}|��f�qSrr�r�example��argsrrrr$Qr)rrH�logger�infor�open�ranger0�map�tokenize�feats)rr�poolrH�	file_path�	samplenum�examplesr#rrGrrEs�zRefineDataset.__init__c33Cs�|\}}}|d�d�}|d�d�}dd�|D�}dd�|D�}d�|�}d�|�}d|�dd�}d|�dd�}|d}|�|||�}||jg|�|||�7}|�|||�}	|�||	||�\}}	t|d	||	�S)34N�old�35�newcSsg|]}|dd����qS��N��striprArrrr$Wrz*RefineDataset.tokenize.<locals>.<listcomp>cSsg|]}|dd����qSrWrYrArrrr$Xr�<add>�commentrC)�splitr�replace�
encode_remove�msg_id�36pad_assertr7�37r�itemrFrrHZoldlines�newlinesr\�srcids�tgtidsrrrrNSs383940zRefineDataset.tokenizecCsD|�d�}dd�|D�}d�|�}d�|���}|�dd�}||fS)NrUcSsg|]}|dd����qSrWrYrArrrr$grz3RefineDataset.process_pred_gold.<locals>.<listcomp>r(z<add> �)r]rr^��pred�goldrrr�process_pred_goldds4142zRefineDataset.process_pred_goldcCs�|d|jd�}|jg||jg}|jt|�}||jg|7}|d|jd�}|jg||jg}|jt|�}||jg|7}t|�|jks�Jd��t|�|jks�Jd��||fS)N��Not equal length.��max_source_length�bos_id�eos_idr0�pad_id�max_target_length�rr9r:rHr�pad_lenrrrramszRefineDataset.pad_assertcCsZ|j||jdd�}t|�tkr*|dd�St|�tkrB|dd�St|�tkrR|St�dS�NT)�43max_length�44truncationr=rX�r.ro�typerrr45�NotImplementedError�rrr/rHrrrr_zszRefineDataset.encode_removecCs46t|j�Sr�r0rOrrrrr1�szRefineDataset.__len__cCs47|j|Sr�rO�rr#rrr�__getitem__�szRefineDataset.__getitem__N)r=)r2r3r4rrNr6rkrar_r1r�rrrrr<Ds4849
r<c@s eZdZdd�Zedd��ZdS)�SimpleRefineDatasetc50Cs�|\}}}|d�d�}|d�d�}dd�|D�}dd�|D�}d�|�}d�|�}|d}|�|||�}||jg|�|||�7}|�|||�}	|�||	||�\}}	t|d	||	�S)51NrTrUrVcSsg|]}|dd����qSrWrYrArrrr$�rz0SimpleRefineDataset.tokenize.<locals>.<listcomp>cSsg|]}|dd����qSrWrYrArrrr$�rr(r\rC)r]rr_r`rar7rbrrrrN�s525354zSimpleRefineDataset.tokenizecCs8|�d�}dd�|D�}d�|�}d�|���}||fS)NrUcSsg|]}|dd����qSrWrYrArrrr$�rz9SimpleRefineDataset.process_pred_gold.<locals>.<listcomp>r()r]rrhrrrrk�s555657z%SimpleRefineDataset.process_pred_goldN�r2r3r4rNr6rkrrrrr��sr�c@s eZdZdd�Zedd��ZdS)�Seq2SeqDatasetc	Csx|\}}}|d|d}}d�|���}d�|���}|�|||�}|�|||�}|�||||�\}}t|d||�S)NrTrVr(rC)rr]r_rar7)	rrcrFrrH�inputs�outputsrerfrrrrN�s58zSeq2SeqDataset.tokenizecCs$d�|���}d�|���}||fSr')rr]rhrrrrk�sz Seq2SeqDataset.process_pred_goldNr�rrrrr��s59r�c@sveZdZddd�Zdd�Zdd�Zdd	�Zd60d�Zdd
�Zdd�Z	dd�Z61dd�Zdd�Zdd�Z
dd�Zdd�ZdS)�TextDatasetr=c	sd|_�|_�|_t�t�r"d}n$t�t�r2d}nt�t�rBd}nd}|�d|d�}tj	�62|�r~t�d�
|��t�|�}nVt�d	�
|��t||��}t�d63|���|�|j��fdd�|D��}t�||�t�d
�|�|�|�|j��fdd�|D��|_dd�|jD�|_dS)Nr�mytokrg�rb�unk�.jsonl�.exps�Loading examples from {}r>rDcsg|]}|��f�qSrrrErGrrr$�rz(TextDataset.__init__.<locals>.<listcomp>�Convert examples to features...csg|]}|��f�qSrrrErGrrr$�rcSsg|]}|D]}|�qqSrr)rrO�featrrrr$�r)�cntrrH�64isinstancer65rrr^rr�existsrIrJr�torch�load�read_review_examplesrMrN�save�set_start_end_ids�convert_examples_to_featuresZfeatssrO�	rrrPrHrQrR�tokenizer_type�saveprSrrGrr�s6666768�6970�zTextDataset.__init__cCs71t|j�Srr}rrrrr1�szTextDataset.__len__cCs72|j|Srr~rrrrr��szTextDataset.__getitem__cCs&t|j�|ksJ�|jd|�|_dSrr})r�data_lenrrr�	reset_len�szTextDataset.reset_lencCs�|D]z}|j}d}t|�d}t|�D]\}}|dkr&|}q@q&tt|�ddd�D]}||}|dkrT|}qrqT||_||_qdS)NrrX����r=)�labelsr0�	enumeraterL�start_id�end_id)rrSrFr�r�r�r#�labelrrrr��szTextDataset.set_start_end_idsc
Cs�|\}}}|�||j|�|_|jd}d�dd�|jD��}|�dt|�d�}dd�|D�}dd�|D�}ttt|��}t|�t	|�}	dt|�}73}|	|j74d	kr�|75d	dkr�|	d76t||77�8}	|78d797}80q�|d818}|	d82t||�8}	q�||83|�}|j|84|�}t|�t	tt|��|j85d	k�s4Jd��t|�t|�k�rpt�
d�|dt|��}|dt|��}||_||_|�||j|�|_|S)
N�<e0>r(css|]}t|�VqdSr)�str)rrCrrr�	<genexpr>�rz'TextDataset.tokenize.<locals>.<genexpr>cSs g|]}dd�|�d�D��qS)cSs g|]}t|�dkrt|��qS�r)r0�int)r�vrrrr$�rz3TextDataset.tokenize.<locals>.<listcomp>.<listcomp>r(�r]rArrrr$�sz(TextDataset.tokenize.<locals>.<listcomp>cSsg|]}t|��qSr�r0rArrrr$�rrrlrXz(Too long inputs in TextDataset.tokenize.z)Not equal length in TextDataset.tokenize.)r_�input�special_dictrr]r��listrMr0�sumror�rIrJ�lines�msg)
rrcrFrrHZe0idr�r��lensZcurlen�left�rightr�rrrrN�s:8687�88*89zTextDataset.tokenizecCs�|\}}}t|j�dkr\g}td�D]2}t��dkrF|�|�|��q$|�|�|��q$|St��dkrt|�|�gS|�|�gS)Nr�g�?)	r0r�rL�random�append�genmsg_example�daemsg_example�encoder_example�decoder_example)rrcrF�_Zexsrrrr�s90z(TextDataset.convert_examples_to_featurescCs|\}}}|j}|j}|jg|j}gg}}	tt||��D]�\}91\}}|92|jkrh|�|j�|	�d�|dkr�|�|j�|	�|�|�	|�|	�	dgt93|��|94|jkr<|�|j�|	�d�q<t95|	�t96|�ks�Jd��t97|	�|jk�sJdt98|	��d���|d|jd�}|	d|jd�}	|j
g||jg}dg|	dg}	|jt99|�}
||jg|
7}|	dg|
7}	g}|j|j|jd�}|	D]*}|dk�r�|�d�n|�||��q�|}	t100|�|jk�s�Jd��t101|	�|jk�s�Jd��t|j||	|dd�S)	Nr�rm�Too long inputs: �.rl�rrXrlr��rz)r�r�rrrsr��zipr�r��mask_id�extendr0r�rorprq�del_id�add_id�keep_id�ReviewFeatures�idx)rrcrFrrHr�r�r:r9�input_labelsr#rBr�ruZnew_input_labelsZmap_dictrrrr� sH102103104105106107108$109zTextDataset.encoder_examplecCsN|\}}}|j}|j}dg|j}gg}}	d}110tjtt|��tt|�|j�d�}|j	|j111|jd�}tt
||��D]�\}
\}}|
|jkr�|�|j�||vr�|�||�|
|v�r|�|jd|112�d��|	�|jd|113�d��|	�|�|114dk�r|115d7}116n117|�|�|
|jkrt|�|j�qt|�||	||�\}}	t|j|||	d	d118�S)Nr�r)�kr��<e�>�crXrBr�)r�r�ror��choicesrLr0r�Z	mask_rater�r�r�r�r�r�r�r�r�r�rar�r�)rrcrFrrHr�r�r�r9r:�119SPECIAL_IDZ	mask_idxs�id_dictr#rBr�rrrr�Hs0120121$122123124125126127128zTextDataset.decoder_examplecCs|\}}}|j}|j}dg|j}gg}}	|j|j|jd�}129tt||��D]X\}\}}
||jkrn|�	|j�|
dkr�|�	|130|
�|�131|�||jkrL|�	|j�qL|	�	|j�|	�132|j
�t|�|jks�Jdt|��d���|�||	||�\}}	t|j|||	dd�S)Nr�r�r�r��genmsgr�)r�r�ror�r�r�r�r�r�r�r�r�r`r�r0rar�r�)rrcrFrrHr�r�r�r9r:r�r#rBr�rrrr�ds&133134135136137"zTextDataset.genmsg_examplecCsf|\}}}dg|j}gg}}t|j�}dd�tt|��D�}	t|	�dkrft�tt|���}138d|	|139<gg}}d}d}|t|	�k�r>|}
|
t|	�kr�|	|
s�|�||
�|
d7}
q�|
t|	�krƐq>|�|j	d|�d��|�|j	d|�d��|
t|	�k�r&|	|
�r&|�||
�|
d7}
q�|d	k�r8|d7}|
}qx|�140||||�\}}t|j|||d141d�S)Nr�cSsg|]}t��dk�qS)g�������?)r�)rr�rrrr$~rz.TextDataset.daemsg_example.<locals>.<listcomp>rTrXr�r�r��daemsgr�)
ro�cpr�rLr0r�r��choicer�r�rar�r�)rrcrFrrHr�r9r:Zmsg_ids�masksr�r#r��jrrrr�ys8142143144145146147148zTextDataset.daemsg_examplecCs�|d|jd�}|jg||jg}|jt|�}||jg|7}|d|jd�}||jg}|jt|�}||jg|7}t|�|jks�Jd��t|�|jks�Jd��||fS)NrlrXrmrnrtrrrra�szTextDataset.pad_assertcCsZ|j||jdd�}t|�tkr*|dd�St|�tkrB|dd�St|�tkrR|St�dSrvryr|rrrr_�szTextDataset.encode_removeN)r=)r2r3r4rr1r�r�r�rNr�r�r�r�r�rar_rrrrr��s149"(
r�c@seZdZddd�Zdd�ZdS)�CommentGenDatasetr=c	s�|_t�t�rd}n$t�t�r&d}nt�t�r6d}nd}|�d|d�}tj�|�rrt	�150d�|��t�
|�}nVt	�151d�|��t||��}t	�152d	|���|�|j��fd153d�|D��}t�||�t	�154d�|�|�|�|j��fd
d�|D��|_dd�|jD�|_dS)Nr�rgr�r�r�r�r�r>rDcsg|]}|��f�qSrrrErGrrr$�rz.CommentGenDataset.__init__.<locals>.<listcomp>r�csg|]}|��f�qSrrrErGrrr$�rcSsg|]}|dur|�qSrr)rr�rrrr$�r�rr�r155rrr^rrr�rIrJrr�r�r�rMrNr�r�r�rOr�rrGrr�s2156157158�159160�zCommentGenDataset.__init__cCs&|\}}}t|j�dkrdS|�|�S)Nr)r0r�r�)rrcrFrrHrrrr��s161z.CommentGenDataset.convert_examples_to_featuresN)r=�r2r3r4rr�rrrrr��s162r�c@seZdZddd�Zdd�ZdS)�CommentClsDatasetr=c	s�|_t�t�rd}n$t�t�r&d}nt�t�r6d}nd}|�d|d�}tj�|�rrt	�163d�|��t�
|�}nVt	�164d�|��t||��}t	�165d	|���|�|j��fd166d�|D��}t�||�t	�167d�|�|�|�|j��fd
d�|D��|_dS)Nr�rgr�r�r�r�r�r>rDcsg|]}|��f�qSrrrErGrrr$�rz.CommentClsDataset.__init__.<locals>.<listcomp>r�csg|]}|��f�qSrrrErGrrr$�rr�r�rrGrr�s0168169170�171172�zCommentClsDataset.__init__cCs&|\}}}|�|�}t|j|j|j�Sr)r��ClsFeaturesr8r9�y)rrcrFrrHZ173tmpfeaturerrrr��s174175z.CommentClsDataset.convert_examples_to_featuresN)r=r�rrrrr��s176r�c@seZdZddd�Zdd�ZdS)�SimpleClsDatasetr=c	s��|_t�t�rd}n$t�t�r&d}nt�t�r6d}nd}|�d|d�}tj�|�rtt	�177d�|��t�
|�|_nZt	�178d�|��t||��}t	�179d	|���|�|j��fd180d�|D��|_t�|j|�dS)Nr�rgr�r�r�z	.simpexpsr�r>rDcsg|]}|��f�qSrrrErGrrr$181rz-SimpleClsDataset.__init__.<locals>.<listcomp>)rr�r182rrr^rrr�rIrJrr�r�rOr�rMr�r�r�rrGrr�s&183184185�zSimpleClsDataset.__init__cCs:|\}}}|j�d�|_t|j�}|jd|�|_tt|j��D]J}|j|dkrhd|j||j|<q@|j|dkr@d|j||j|<q@d�|j�|_|�||j|�}t|�|jd}|dkr�|dd}	||	|	�}|d|jd�}186|j	g|187|j188g}189|jt|190�}|191|jg|7}192|j}|j
}
t||193|
�S)Nr�rXz+ rz- r(rl)r�r]Zinput_linesr0r�rLrr_rorprqrrr�r�r�)rrcrFrrHZlabels_lr#�	input_idsZexceed_lZhalfexlr9rur8r�rrrr�
s,194195z-SimpleClsDataset.convert_examples_to_featuresN)r=r�rrrrr��s196r�c@seZdZddd�Zdd�ZdS)�SimpleGenDatasetr=c197s���_t�t�rd}n$t�t�r&d}nt�t�r6d}nd}|�d|d�}tj�|�rtt	�198d�|��t�
|��_nlt	�199d�|��t|�}tt|��D]}	|	||	d	<q�t	�200d201|������fdd�|D��_t��j|�dS)
Nr�rgr�r�r�z.simpgenexpsr�r>r�rDcsg|]}��|��f��qSr)r�)rr"�rHrrrrr$>rz-SimpleGenDataset.__init__.<locals>.<listcomp>)rr�r202rrr^rrr�rIrJrr�r�rO�203read_jsonlrLr0r�)204rrrPrHrQrRr�r��datar#rr�rr's&205206207zSimpleGenDataset.__init__cs6|\}}}|d|d}}|�d�dd�}dd�|D�}dddd	���fd208d���fdd�|D�}d
d�|D�}d}	t||�D]@\}209}|210dkr�|	d|7}	q�|211dkr�|	d|7}	q�|	d|7}	q�|�||	|�}g}
|
�|j�|�||d|�}|
�|�|�||
||�\}}
dgt|�}t|d|||
dd�S)N�patchr�rUrXcSs g|]}t|���dkr|�qSr�)r0rZrArrrr$ErzASimpleGenDataset.convert_examples_to_features.<locals>.<listcomp>rrl)�-�+r(cs|�vr�|SdSdS)Nrlr)�s)�map_dicrr�fGsz8SimpleGenDataset.convert_examples_to_features.<locals>.fcsg|]}�|d��qSr�rrA)r�rrr$LrcSsg|]}|dd����qSrWrYrArrrr$Mrrgr[z<del>z<keep>r�r�r�r�)	r]r�r_r�r`r�rar0r�)rrcr"rrH�diffr��	difflinesr��inputstrr�rBr9r:r�r)r�r�rr�As.212213z-SimpleGenDataset.convert_examples_to_featuresN)r=r�rrrrr�&s214r�c@seZdZdZddd�ZdS)�
InputFeaturesz.A single training/test features for a example.NcCs||_||_||_||_dSr)r8r9r:�url)rr8r9r:r�rrrrcszInputFeatures.__init__)N)r2r3r4r5rrrrrr�`sr�c@seZdZdd�ZdS)r�cCs.||_||_||_||_|dvs$J�||_dS)N)r�rBr�r�)r8r9�
source_labelsr:rz)rr8r9r�r:rzrrrrkszReviewFeatures.__init__Nr;rrrrr�jsr�c@seZdZdd�ZdS)r�cCs||_||_||_dSr)r8r9r�)rr8r9r�rrrrtszClsFeatures.__init__Nr;rrrrr�ssr�c@s0eZdZdZdd�Zdd�Zdd�Zdd	�Zd215S)�
ReviewExamplezA single training/test example.cCsb||_||_||_||_||_||_||_g|_g|_g|_	g|_216d|_d|_|�
�|��dS)NFrg)r��oldfr�r��cmtid�max_lenr��	prevlines�217afterlinesr�r��availr��align_and_clean�postprocess)rr�r�r�r�r�r�r�rrrr|szReviewExample.__init__c218Cs|js219dSdd�|jD�}t|�}|ttt|��7}dt|�}}||jkr�|ddkrv|t||�d8}|d7}qB|d8}|t||�d8}qB|||�}|j||�|_|j||�|_|j}|j}t	t|�t|��}d}||jk�r�||k�r�|t|�k�rZ|t|d|�220��d}	|	|jk�r2�q�|j�d|d|�|j�dd�|	}|t|�k�r�|t||�221��d}	|	|jk�r��q�|j�||�|j�d�|	}|d7}q�||jk�s�Jd��t|j�t|j�k�s�Jd	��d222�
|j�|_ggg|_|_|_dS)NcSsg|]}|���qSrr�)rZ223source_strrrrr$�rz-ReviewExample.postprocess.<locals>.<listcomp>rrlrXr=r�zToo long inputs.rmr�)r�r�r0r�rMr�r�r�r��maxr]�insertr�rr�)224rr�Zinputlr�r�r�r�Zprev_after_lenr#Znewlrrrr��sN225226227zReviewExample.postprocesscCshd}t|�}d}||kr.|||vr.|d7}q|d}|dkrT|||vrT|d8}q6|||d�}|S)z7228            Remove start and end empty chars.229        z 	
rrXr�)rrB�repZtotallenr#r�rrr�remove_space_clean�s230231z ReviewExample.remove_space_cleanc
s�j�d�}�j�d�}|d}|dd�}dd�|D�}d}t�||�}|rf|��\}}}}	d�_n232d�_dSt|�dt|�}}||}233|d|��_||234d��_	|D]v}|�235d	�rވj�|dd���j
�d�q�|�236d237��r�j�|dd���j
�d�q��j�|��j
�d�q��fdd��jD��_�fd
d��j	D��_	�fdd��jD��_���j��_dd��jD��_dd��j	D��_	t�j�t�j
�k�s�Jd��ttdd�t�j�j
�D���}|gk�r�d�_dS|\�_�_
t�j��_t�j
��_
dS)NrUrrXcSsg|]}|dkr|�qS)z\ No newline at end of filerrArrrr$�rz1ReviewExample.align_and_clean.<locals>.<listcomp>z @@ -(\d+),(\d+) \+(\d+),(\d+) @@TFr�r�rlcsg|]}��|��qSr�r�rArrrr$�rcsg|]}��|��qSrr�rArrrr$�rcsg|]}��|��qSrr�rArrrr$�rcSsg|]}t|�dkr|�qSr�r�rArrrr$�rcSsg|]}t|�dkr|�qSr�r�rArrrr$�rzNot equal length in align.cSs$g|]\}}t|�dkr||f�qSr�r�)rrBr�rrrr$�s�)r�r]r��re�match�groupsr�r�r�r��238startswithr�r�r�r�r�r0r�r�)
rZ	oldflinesr��239first_line�regexZmatchres�	startlineZrangelen�startpos�endposZendlinerBZtopackrrrr��sZ240���	241zReviewExample.align_and_cleanN)r2r3r4r5rr�r�r�rrrrr�ys242,r�r=c243Csg}d}t|���}|D]�}zt�|���}Wntd�YqYn0d}d|vrZd|d<d|vrzt|d�dkrzd|d<t||d|dd|vr�|dnd	d244|vr�|d245nd	||dd�}	|	jr�|�|	�|d7}||kr�q�q|d7}||krq�qWd�n1�s0Y|S)
zRead examples from filename.r�Error during reading json data.��r�r�rXr�r�rgr�)r�r�r�r�r�r�r�N)	rKr?r@rZ�printr0r�r�r�)246�filenameZdata_numrrSr�r�rB�jsZmaxlrFrrrr�s@247248�	249&r�cCspg}t|��P}|D]:}zt�|���}Wntd�YqYn0|�|�qWd�n1sb0Y|S)Nr)rKr?r@rZrr�)rr�r�rBr	rrrr�-s250251*r�)r=N)'r�r?rr�r��logging�copyrr��torch.utils.datarZ252tokenizersr�transformersrrZnltk�basicConfig�INFO�	getLoggerr2rI�objectr253r7r<r�r�r�r�r�r�r�r�r�r�r�r�r�rrrr�<module>s>�254+G{%!0:255	256'