CoolFace
Apppublic

Yyk040316/long-context-icl

sourceHugging Faceupdated 2y agoView on Hugging Face
0likes
utils.cpython-310.pyc64 linesDownload Raw Back to __pycache__
1o

28ɀg!�@sddlZddlZddlmZmZddlZddlZddlZ	ddl3mZddlmZ
ddlmZddlmZmZmZddlmZddlmZmZmZe�e�Zejejd	d4�de	j de	j d
ede!de!f5dd�Z"dCde	j#dede$de!fdd�Z%dd�Z&dDde	j de	j de$fdd�Z'de	j d
ede	j fd d!�Z(dEd
ed#e$de!fd$d%�Z)dFdGd'd(�Z*dEd)e$d*e$dee
j+e,ee!ffd+d,�Z-	-dHd.e$d/ee!d0ej.e!d1ee$d2e$d3e$d4e/ddfd5d6�Z0d
ed7ee$deee!fd8d9�Z1d
ed:e$de!fd;d<�Z2d=d>�Z3d?d@�Z4dAdB�Z5dS)I�N)�List�Tuple)�pyplot)�typing)�distributed)�PreTrainedTokenizerBase�LlamaTokenizer�LlamaTokenizerFast)�SparseRetriever)�TEXT_BETWEEN_SHOTS�N_TOKENS�PROMPTSz%(message)s)�level�format�train_df�test_df�	tokenizer�prompt_size�returnc	Cs�|t��}t�d|���t|t�}|t|}|�d�}td|�d����tdt|�t	|����||}t6t�||��S)Nzlongest_test_prompt = g�������?z Median length of demonstration: g�?zMean length of demonstration: )
r�max�_logger�info�n_tokens_in_promptr�quantile�print�sum�len�int�np�floor)	rrrrZlongest_test_promptZn_tokens_between_shotsZshot_lengthsZprompt_length_percentileZmax_possible_shots_length�r �./data/yyk/experiment/long-context-icl/utils.py�get_max_n_shotss78r"�index�curr_example�9n_examplescCs�|j|d|d�}dd�|D�}t|�|kr/td|t|��d��|�|d�|t|���t|j|d�|�d�}|rVd	dl}	|	�	�}10|	�11|�|	�|�|	�|12�|�
|�}|S)13NF)�queryZreturn_docs�cutoffcSsg|]}t|��qSr )r��.0�dr r r!�14<listcomp>*sz$retrieve_context.<locals>.<listcomp>zWARNING: sampling z! examples randomly to fill window�id�promptsr)�searchrr�extend�sample�list�loc�isin�random�getstate�seed�shuffle�setstate�join)rr#r$r%Z15split_textZshuffle_seed�	retrieved�inds�dpsr4�16prev_state�textr r r!�retrieve_context$s$�17181920r?c
Cs�tdddddddddddd�}ddl}d	|�dd21��d|�dd��d
�}|j|d<ddlm}ddl}|j�|�r@|�	||��|�22|�|j|ddd�d�|�	||��|S)Nztraining-examplesZbm25��23whitespace�englishT)�24index_name�modelZmin_dfrZstemmerZ	stopwordsZdo_lowercasingZdo_ampersand_normalizationZdo_special_chars_normalizationZdo_acronyms_normalizationZdo_punctuation_removalrZ__temp_index_file_i�_io�.csvr,)�PathcSs|d|dd�S)Nr,r>)r,r>r )�docr r r!�<lambda>Ts�z"create_retriever.<locals>.<lambda>)�path�
show_progress�callback)r25r4�randintr#�pathlibrG�osrJ�exists�unlink�to_csv�26index_file)r�srr4�filenamerGrOr r r!�create_retriever;s6�
"2728�rVr>�df1�df2�comp_columncCs8|j||�||�}|j||�||�}||fS�N)r2r3)rWrXrYr r r!�synchronize_examples_across_dfs\sr[�dfcsr|t��fdd��|t<|t|t�d�k}t�dt|��d��|j|��}t�d|t�	����|S)Ncs29t�|�SrZ)r)�x�rr r!rIbs30z/filter_extremely_long_samples.<locals>.<lambda>g�G�z��?z	filtered z$ from  dataset due to extreme lengthz1longest remaining prompt according to tokenizer: )31r
�maprrrrrr2�copyr)r\r�maskr r^r!�filter_extremely_long_samplesasrbF�promptcCst|j||d��S)N��add_special_tokens)r�encode)rrcrer r r!rjsr�cCsxt��tj|tj|dd�tj|dd�dd�t�d�t�|�d}t�|�d|���t�	|�d|�d|���dS)Nr@)�axis�*)�fmtz# shotsZAccuracy� )32�plt�figureZerrorbarr�mean�std�xlabel�xticks�ylabel�title)�results�dataset_name�n_shotsrD�metricr r r!�plot_results_graphns&3334rxru�35output_dircs�t�|�}�fdd�|D�}t|�dkrtdt|��d���|d}t�tj�||��}dd�|�d�d	�d36�D�}|rCt	|�|�||fS)Ncs g|]}|���d��r|�qS)rE)�37startswith)r)�r�rur r!r+z� z load_results.<locals>.<listcomp>r@zFound z	 results!rcSsg|]38}|��rt|��qSr )�isdigitrr(r r r!r+s�.�����rE)39rO�listdirr�40ValueErrorr�loadrJr9�splitrx)rury�plot�all_resultsZresults_pathrtrvr r|r!�load_resultsxs41r�T�datasetrvrt�predictions�outpathrD�plot_resultsc42CsR|r
t||||�t��t��rt��dkr�t�||�t|�	d�ddd��}ddl43}|�||�Wd�n1s>wY|�	d�d�	d�d}	t|�D]S\}44}t||45�D]F\}}
|
j
|
d<||
d<||
d	<ttj�|�d|	�	d46�ddt|�dt|�d
d��
}|
�|�Wd�n1s�wYq]qSdSdS)Nrrz-outputs.pkl�wb�/�����r,rvZ47run_numberZn_shots_z	+n_shots=z+run=rF�w)rxrl�show�dist�is_initialized�get_rankr�save�openr��pickle�dump�	enumerater#rOrJ�dirname�strrR)r�rvrtr�r�rDr��fr��48clean_name�numZnshots�i�repr r r!�save_results�s,�49B����r��labelscs.t�t�r�fdd�|D�S�fdd�|D�S)Ncs g|]}�j|���dd��qS)Frd�rf�lstrip�r)�labelr^r r!r+�r}z!encode_labels.<locals>.<listcomp>cs$g|]}�jd|����dd��qS)rkFrdr�r�r^r r!r+�s$)�50isinstancer)rr�r r^r!�
encode_labels�s51r��stop_seqcCsR|j|dd�}t|t�st|t�rt|�dksJ�|dSt|�dks%J�|dS)NFrd�r@r�)rfr�rr	r)rr�Zstop_seq_token_idr r r!�encode_stop_seq�s�r�cCs&d}t�||�}|r|�d�St|�S)Nz[aA]nswer is \(?([A-J])\)?r@)�rer.�group�
extract_final�r>�pattern�matchr r r!�
extract_again�s5253r�cCs"t�d|�}|r
|�d�St|�S)Nz.*[aA]nswer:\s*\(?([A-J])\)?r@)r�r.r�r�)r>r�r r r!�extract_answer�s54r�cCs&d}t�||tj�}|r|�d�SdS)Nz(\(?\b([A-J])\b\)?(?!.*\(?\b([A-J])\b\)?)r)r�r.�DOTALLr�r�r r r!r��s5556r�rZ)r>)F)rg)rN)rgT)6�loggingrOrrrr��numpyr�pandas�pd�57matplotlibrrl�npt�torchrr��transformersrrr	Zretrivr58�	constantsrrr
�	getLogger�__name__r�basicConfig�INFO�	DataFramerr"�
DatetimeIndexr�r?rVr[rbrrx�NDArray�floatr��ndarray�boolr�r�r�r�r�r�r r r r!�<module>sN59�60�!	*61
&���62�"6364