CoolFace
Apppublic

Yyk040316/long-context-icl

sourceHugging Faceupdated 2y agoView on Hugging Face
0likes
utils.cpython-310.pyc121 linesDownload Raw Back to __pycache__
1o

2�۔g�T�@sjddlZddlZddlmZmZddlZddlZddl	m3ZddlmZddl
mZddlmZmZmZddlmZddlZddlZddlmZe�e�Zejejd	d4�dejdejd
ede de f5dd�Z!d[dejdejde"fdd�Z#dejd
edejfdd�Z$d\d
ede"de fdd�Z%d]d^d d!�Z&d\d"e"d#e"deej'e(ee ffd$d%�Z)	d_d'e"d(ee d)ej*e d*ee"d+e"d,e"d-e+ddfd.d/�Z,d
ed0ee"deee fd1d2�Z-d
ed3e"de fd4d5�Z.d6d7�Z/d8d9�Z0d:d;�Z1d<d=�Z2d>d?�Z3d@dA�Z4dBdC�Z5dDdE�Z6dFdG�Z7dHdI�Z8dJdK�Z9dLdM�Z:dNdO�Z;d\dPdQ�Z<d`dej=dRedSe"dTe fdUdV�Z>dWdX�Z?d`dYdZ�Z@dS)a�N)�List�Tuple)�pyplot)�typing)�distributed)�PreTrainedTokenizerBase�LlamaTokenizer�LlamaTokenizerFast)�SparseRetriever)�TEXT_BETWEEN_SHOTSz%(message)s)�level�format�train_df�test_df�	tokenizer�prompt_size�returnc	Cs�|t��}t�d|���t|t�}|t|}|�d�}td|�d����tdt|�t	|����||}t6t�||��S)Nzlongest_test_prompt = g�������?z Median length of demonstration: g�?zMean length of demonstration: )
�N_TOKENS�max�_logger�info�n_tokens_in_promptr�quantile�print�sum�len�int�np�floor)	rrrrZlongest_test_prompt�n_tokens_between_shotsZshot_lengthsZprompt_length_percentileZmax_possible_shots_length�r �=/data/yyk/experiment/long-context-icl/Integrate_Code/utils.py�get_max_n_shotss78r"�text�df1�df2�comp_columncCs8|j||�||�}|j||�||�}||fS�N)�loc�isin)r$r%r&r r r!�synchronize_examples_across_dfs(sr*�dfcsr|t��fdd��|t<|t|t�d�k}t�dt|��d��|j|��}t�d|t�	����|S)Ncs9t�|�Sr')r��x�rr r!�<lambda>.s10z/filter_extremely_long_samples.<locals>.<lambda>g�G�z��?z	filtered z$ from  dataset due to extreme lengthz1longest remaining prompt according to tokenizer: )11ZPROMPTS�maprrrrrr(�copyr)r+r�maskr r.r!�filter_extremely_long_samples-sr3F�promptcCst|j||d��S)N��add_special_tokens)r�encode)rr4r6r r r!r6sr�cCsxt��tj|tj|dd�tj|dd�dd�t�d�t�|�d}t�|�d|���t�	|�d|�d|���dS)N�)�axis�*)�fmtz# shotsZAccuracy� )12�plt�figureZerrorbarr�mean�std�xlabel�xticks�ylabel�title)�results�dataset_name�n_shots�model�metricr r r!�plot_results_graph:s&1314rKrG�15output_dircs�t�|�}�fdd�|D�}t|�dkrtdt|��d���|d}t�tj�||��}dd�|�d�d	�d16�D�}|rCt	|�|�||fS)Ncs g|]}|���d��r|�qS)�_)�17startswith)�.0�r�rGr r!�18<listcomp>F� z load_results.<locals>.<listcomp>r9zFound z	 results!rcSsg|]19}|��rt|��qSr )�isdigitr�rO�dr r r!rRKs�.�����rM)20�os�listdirr�21ValueErrorr�load�path�join�splitrK)rGrL�plot�all_resultsZresults_pathrFrHr rQr!�load_resultsDs22rbT�datasetrHrF�predictions�outpathrI�plot_resultsc2324Cs�|r
t||||�t��t��rt��dk�rvt�||�t|�	d�ddd��}25ddl26}|�||27�Wd�n1s?wY|�	d�d�	d�d}t|�D�]"\}
}|
t
|�krbdSt||
�D�]\}}|j|d<||d<|d	kr�||d28<ttj�|�d|�	d�ddt|�d
t|�dddd��
}29|�|30�Wd�n1s�wYqh|d	kr�||d<ttj�|�d|�	d�ddt|�dt|�dddd��
}31|�|32�Wd�n1s�wYqh|	d	k�r8||d<ttj�|�d|�	d�ddt|�dt|�dddd��
}33|�|34�Wd�n	1�s2wYqh||d<ttj�|�d|�	d�ddt|�dt|�dddd��
}35|�|36�Wd�n	1�sowYqhqTdSdS)NrrWz-outputs.pkl�wb�/������idrHTZshuffle_numberZn_shots_z	+n_shots=z	+shuffle=�.csv�wzutf-8)�encoding�noisy_levelz
+noisy_level=Zreinforce_numberz+reinforce_number=Z37run_numberz+run=)rKr>�show�dist�is_initialized�get_rankr�save�openr_�pickle�dump�	enumerater�indexrYr]�dirname�str�to_csv)rcrHrFrdrerIrf�shuffle�noisy�	reinforce�fru�38clean_name�numZnshots�i�repr r r!�save_resultsPsT�39F��F��40F��F����r��labelscs.t�t�r�fdd�|D�S�fdd�|D�S)Ncs g|]}�j|���dd��qS)Fr5�r7�lstrip�rO�labelr.r r!rRxrSz!encode_labels.<locals>.<listcomp>cs$g|]}�jd|����dd��qS)r=Fr5r�r�r.r r!rRzs$)�41isinstancer)rr�r r.r!�
encode_labelsus42r��stop_seqcCsR|j|dd�}t|t�st|t�rt|�dksJ�|dSt|�dks%J�|dS)NFr5�r9ri)r7r�rr	r)rr�Zstop_seq_token_idr r r!�encode_stop_seq}s�r�cC�&d}t�||�}|r|�d�St|�S)Nz[aA]nswer is \(?([A-J])\)?r9)�re�search�group�
extract_final�r#�pattern�matchr r r!�
extract_again��4344r�cCs"t�d|�}|r
|�d�St|�S)Nz.*[aA]nswer:\s*\(?([A-J])\)?r9)r�r�r�r�)r#r�r r r!�extract_answer�s45r�cCs&d}t�||tj�}|r|�d�SdS)Nz(\(?\b([A-J])\b\)?(?!.*\(?\b([A-J])\b\)?)r)r�r��DOTALLr�r�r r r!r��s4647r�cCs*t�d�}|�|�}|r|�d���SdS)u 48    从字符串中匹配:49    1. "final answer is" (大小写不敏感)50    2. 任意内容(非贪婪)51    3. 直到 ". I hope" (大小写不敏感)52    53    返回从 "final answer is" 后开始,到 ". I hope" 前结束的部分。54    如果找不到匹配,返回 None。55    z,(?i)final\s*answer\s*is\s*(.*?)\.\s*i\s*hoper9N)r��compiler�r��stripr�r r r!�extract_answer_math�s56�57r�cCr�)Nz*[aA]nswer\s*:\s*(.+?)(?:\.?\s*[Aa]nswer|$)r9)r�r�r��extract_final_mathr�r r r!�extract_again_math�r�r�cCs�|�d�}|dkrdS|td�7}d}d}|t|�krC||}|dkr(|d7}n
|dkr5|d8}|dkr5n||7}|d7}|t|�ks|dkrI|SdS)Nz\boxed{rir9r8�{�}r)�findr)r#rxZbrace_count�content�charr r r!r��s$5859�60r�cCr�)Nz####\s*([\d,]+)r9)r�r�r�r�r�r r r!�extract_answer_gsm8k�s6162r�cCs0|�d�}|d}t|�dkr�|dd�}|D]|}|d7}|ddkr(||7}qz63t|�dks1J�Wn	|YS|d}|d}|dkrpt|�dkrc|dd�}|d|d|d|7}q|d|d|d7}qt|�dkr�|dd�}|d|d||7}q|d|d|7}q|}|S)Nz\fracrr9r�r��}{r��r_r)�stringZsubstrsZnew_str�substr�a�bZpost_substrr r r!�64_fix_fracs�s26566r�cCs�t|�d��dkr|S|�d�d}|�d�d}z#t|�}t|�}|d�||�ks,J�dt|�dt|�d}|WS|YS)	Nrhr�rr9z{}/{}z\frac{r�r�)rr_rr
rz)r�r�r��67new_stringr r r!�_fix_a_slash_bsr�cCsXd|vr|�d�}t|�dksJ�|dSd|vr*|�d�}t|�dks&J�|dS|S)Nz\text{ r�rz\text{r�)r��splitsr r r!�_remove_right_unitss6869r�cCsrd|vr|S|�d�}|d}|dd�D]!}|ddkr.|d}d|d|dd�}nd|}||7}q|S)Nz\sqrtrr9r�z\sqrt{r�)r_)r�r�r�r_r�Z70new_substrr r r!�	_fix_sqrt!s7172r�cCsd}d}t�|||�}|S)Nz\\frac\{([^{}]+)\}\{([^{}]+)\}z\1/\2)r��sub)r�r��replr r r!�
_replace_frac/sr�cCs�|�dd�}|�dd�}|�dd�}|�dd�}|�dd�}|�dd�}|�dd�}|�dd	�}|�d73d�}|�dd�}|�d
d�}|�dd�}|�dd�}|�dd�}|�dd�}|�dd�}|�dd�}t|�}|�dd�}|�dd�}|�dd�}|�dd�}t|�dkr�|S|ddkr�d|}t|�d��dkr�t|�d�d�dkr�|�d�d}t|�}|�dd�}t|�}|dkr�d }t|�}t|�}|��r�|d!d�d"kr�|dd!�}|S)#N�74r8z\(z\)z\,�,z\!z\\�\Ztfrac�fracZdfracz\leftz\rightz^{\circ}z^\circz\$�$z\%z .z 0.z{.z{0.rrW�0�=r�r9r=z0.5z\frac{1}{2}rXz.0)	�replacer�rr_r�r�r�r�rT)r�r r r!�
_strip_string6sPr�cCsl|dur|durtd�dS|dus|durdSzt|�}t|�}|r(t||�||kWS||kYS)NzWARNING: Both NoneTF)rr�)�str1�str2�verboseZss1Zss2r r r!�is_equiv�s7576r�rx�curr_example�77n_examplescCs�|j|d|d�}dd�|D�}t|�|kr8td|t|��d��|�|d�|t|��j|d�|��t|j|d�|�d�}|r_d	dl}	|	�	�}78|	�79|�|	�|�|	�|80�|�
|�}|S)81NF)�queryZreturn_docs�cutoffcSsg|]}t|��qSr )rrUr r r!rR��z$retrieve_context.<locals>.<listcomp>zWARNING: sampling z! examples randomly to fill windowrjr4r)r�rr�extend�sampler(r)�list�random�getstate�seedr|�setstater^)rrxr�r�Z82split_textZshuffle_seed�	retrieved�inds�dpsr��83prev_stater#r r r!�retrieve_context�s$�.84858687r�c
Cs�tdddddddddddd�}ddl}d	|�dd88��d|�dd��d
�}|j|d<ddlm}ddl}|j�|�r@|�	||��|�89|�|j|ddd�d�|�	||��|S)Nztraining-examplesZbm25r9�90whitespace�englishT)�91index_namerIZmin_dfrZstemmerZ	stopwordsZdo_lowercasingZdo_ampersand_normalizationZdo_special_chars_normalizationZdo_acronyms_normalizationZdo_punctuation_removalrZ__temp_index_file_irMiorkrj)�PathcSs|d|dd�S)Nrjr4)rjr#r )�docr r r!r/�s�z"create_retriever.<locals>.<lambda>)r]�
show_progress�callback)r92r��randintrx�pathlibr�rYr]�exists�unlinkr{�93index_file)r�srr��filenamer�rYr r r!�create_retriever�s6�
"9495�r�cs4|dkr|S|j��}�fdd�|D�}|dks|dkrX|d|d<t�||t���}�|}	|jt�|j��|�df��}96|97|j|	df<	|d|dd	|d98<||	fS|dkr�|d|d<t�|j��|�}|j|df���t��}t|�d
kr�tt���D]$}d}
	t�t	|��}|
d
7}
|�|kr�n
|
t|�kr�t99d�J�q�q�n)tt���D]"}d}
	t�|�}|
d
7}
|�|kr�n
|
t|�kr�t100d�J�q�q��|j|df<|d|dd	|d101<||fS|dk�r�|d|d<|d|d<t�|j��|�}|j|df���|j|df���t��tt���k�s%Jd��d}
	t�tt���t���}�fdd�|D���fdd�|D�}|
d
7}
t��fdd�tt���D���r\n|
dk�ret102d��q(�|j|df<||j|df<|dd|dd	dd|d�dd��d d	|d103<||fSdS)!Nrcsg|]}|�vr|�qSr r )rO�idx)�	noisy_idxr r!rR�szadd_noisy.<locals>.<listcomp>�
summarization�multilingual�solutionZnoisy_solution�problemr��104prompt_new�classificationZshuffle_solutionr9TzEWARNING: could not shuffle noisy examples after all labels were triedF�qa�answerZshuffle_answerzNoisy values contain duplicatesc�g|]}�|�qSr r �rOr�)�original_valuesr r!rR@r�cr�r r r�)�original_answersr r!rRAr�c3s �|]}�|�|kVqdSr'r r�)r��shuffled_valuesr r!�	<genexpr>Es�zadd_noisy.<locals>.<genexpr>�dz>WARNING: could not shuffle noisy examples after 100 iterationsz105Solution:106zAnswer: �(cSs|d��S)Nr�)�rstripr,r r r!r/Nszadd_noisy.<locals>.<lambda>�))
rx�tolistr�r�rr(�set�range�choicer�r�all�apply)r+�taskrnr��107residue_dfr�Zdf_idxZ
candidate_idxZ
noisy_indicesZall_noisy_indicesZresidue_values�valuesr��iter�	new_valueZshuffled_indicesZshuffled_answersr )r�r�r�r�r!�	add_noisy�s�108 ��
109�110"111�>�r)r#)F)r8)rN)r8TFFFr')A�loggingrYrrr�numpyr�pandas�pd�112matplotlibrr>�npt�torchrrp�transformersrrr	Zretrivr113r�r��	constantsr�	getLogger�__name__r�basicConfig�INFO�	DataFramerr"rzr*r3rrK�NDArray�floatrb�ndarray�boolr�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r��
DatetimeIndexr�r�rr r r r!�<module>sh114�115�	*116
&���117�"%	118119	120
121Y!