CoolFace
Apppublic

Yyk040316/long-context-icl

sourceHugging Faceupdated 2y agoView on Hugging Face
0likes
experiment_manager.cpython-310.pyc163 linesDownload Raw Back to __pycache__
1o

2է�g���@sPddlZddlZddlmZmZddlmZddlmZmZddl	Z	ddl3ZddlZddl
mZddlZddlmZddlmZmZddlmZddlmZmZdd	lmZdd4lmZddlmZ ddl!m"Z"ddl#Z#ddl$m%Z%dd
l&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1e�2e3�Z4ej5ej6dd�dZ7gd�Z8Gdd�d�Z9dS)�N)�List�Dict)�Counter)�Optional�Union)�tqdm)�LLM�SamplingParams)�contextmanager)�HarmCategory�HarmBlockThreshold)� RestrictiveTokensLogitsProcessor��TEXT_BETWEEN_SHOTS)�pad_sequence)�OpenAI)�n_tokens_in_prompt�extract_answer_math�extract_answer�is_equiv�extract_answer_gsm8k�
encode_labels�encode_stop_seq�synchronize_examples_across_dfs�retrieve_context�create_retriever�	add_noisyz%(message)s)�level�format�5)�A�B�C�D�E�F�G�H�I�J�K�L�M�N�O�Pc@s�eZdZ			d@dejdejdeded	eed6ededed
ededefdd�Z	deddfdd�Z7deededefdd�ZdAdedeedeefdd�Z
defdd�Zd eefd!d"�Zd#edefd$d%�Z	&		dBd'eed(ed)ed*ed+ef8d,d-�Zded.edejefd/d0�Zed1eedefd2d3��ZdCd4d5�ZdCd6d7�ZdCd8efd9d:�Zd;eed*edefd<d=�Zd1eedefd>d?�ZdS)D�ExperimentManagerN�*�F�test_df�train_df�task�9model_name�labels�
datasets_name�random_seed�context_size�
use_retrieval�language�subjectcs�|�_|�_|�_tj�|	�|}d|vsd|vsd|vr$d�_d�_n>�jdkrFt�fdd�|dD���_t�fd	d�|d10D���_nt�fdd�|dD���_t�fdd�|d11D���_|�_	|�_12|	�_|13�_|�_
d
�_|
�_tj�|	��fdd�td�D��_d�_|�_|�_|�_d�_d�_d�_d�_d|vs�d|vs�d|vr�ddd�ddd�dgd�d�ddd�ddd�d��_n%d�jdd��jdd�d�jgd�d�d�jdd��jdd�d��_d�_dS)N�gemini�claude�gpt������gkuc3��|]	}t�j|�VqdS�N�r�	tokenizer��.0�problem��self��J/data/yyk/experiment/long-context-icl/Integrate_Code/experiment_manager.py�	<genexpr>;��z-ExperimentManager.__init__.<locals>.<genexpr>rIc3rCrDrE�rH�solutionrJrLrMrN<rOrQc3rCrDrErGrJrLrMrN>rOc3s �|]}t�j|d�VqdS)rNrErPrJrLrMrN?s��cudac� g|]}tj�t�jj���qSrL��np�random�permutation�listr4�index�rH�irJrLrM�14<listcomp>M� z.ExperimentManager.__init__.<locals>.<listcomp>�rF)�15max_tokens�stop_tokens)�Problem:�problem:�	Question:�	question:)�
summarization�multilingual�math�qa�classification�)rF�modelr5rUrV�seed�longest_test_problemZlongest_test_solution�maxr3r4�base_random_seedr:r;�devicer=�range�
random_orders�times_shuffledr<r8r6�shuffle�noisy�	reinforce�
test_in_train�	param_map�logit_processor)rKr3r4rkrFr5r6r7r8r9r:r;r<r=rLrJrM�__init__+sX1617�1819�20zExperimentManager.__init__�returncCstj�|�t�|�dSrD)rUrVrl)rKr9rLrLrM�_set_random_seedmsz"ExperimentManager._set_random_seed�windows_many_shot�n_shotscCs,|jr	|�|�}n|j||jd�}|�|�S)N)�context�restrictive_logit_preprocessor)r;�get_predicted_retrieval�
get_predictedry�calc_acc)rKr}r~�	predictedrLrLrM�get_many_shots_accps21z$ExperimentManager.get_many_shots_acc��used_idx�used_prompt_listcs|�jdk�r<gd�}|d8}|t|�8}d�tdddd��}|��D]}�|7�q"Wd�n1s3wY��|�}�d	7��j}	t�fd22d��jdD���_�jj	�23�}24|}g}g}
td
d�jd||d|d�}t|�|k�r8|t|�}t|25�t|�kr�tt
|26�t
|��}nJd����||�}|�|�t�jj|d�}t�jj|d�}�fdd�|D�}t���k|	�||�}t|�D]X}||��fdd�t|�D�}tt|��D]@}t||�}|du�r|���t�}|�d	�d�d�d��}t|||��r |�||d	||�|
�||�nq�q�Wd�n	1�s-wYt|�|ksu||
fSdS)Nrg)rarbrcrd�==��z+./Integrate_Code/initial_reinforce_math.txt�r�utf-8��encodingrc3rCrDrErPrJrLrMrN�rOz2ExperimentManager.reinforce_icl.<locals>.<genexpr>rQgffffff�?g�?�2�)�temperaturer_�top_k�n�best_of�stopFzEThe number of already_used_idx is larger than the number of train_idxrI�answerc�g|]}�d|�qS�rrLrG��initial_promptrLrMr\��z3ExperimentManager.reinforce_icl.<locals>.<listcomp>csg|]}�j|j�qSrL)�outputs�textrZ)�outputrLrMr\�r�rr�)r5�len�open�	readlines�build_many_shots_textrkrnr4Zlongest_train_solutionrY�to_listr	rX�set�sample_n_shots�extend�loc�torch�no_grad�generaterqr�lstrip�strip�
STOP_SEQUENCE�split�rstripr�append)rKr~r�r�Z
candidate_numr`�fi�lineZshotsZgenerate_modelZ	train_idxZalready_used_idx�new_prompt_list�	right_idx�
sample_paramsZadd_numZ
new_train_idxZcandidate_list�problem_list�answer_list�prompts_list�res�k�predicted_list�jr�rL)r�r�rKrM�
reinforce_iclxsd27��28293031�����)�zExperimentManager.reinforce_iclcCsdSrDrL)rKr~rLrLrMr��sz)ExperimentManager.get_predicted_retrievalrc	s�g}|�|��|jdkr1|jd�dd�����t��dkr"�g}n�fdd�tdt��d�D�}t��dkr;�g}n�fd	d�tdt��d�D�}d}t|d32d�D]�}dd�|D�}|jdkre||}	nd}	|}33|jd
kr�td|j	�d�ddd��}|�34|35d�Wd�n1s�wYn%td|j���d�ddd��}|�36|37d�Wd�n1s�wY|jdkr�|jdks�|jdkr�|j
|j}d|d<n|j
|j}|�|38|j||	�}
|�|
�|d7}qR|S)Nrhr�cSs|dS)N�num_optionsrL��xrLrLrM�<lambda>�sz1ExperimentManager.get_predicted.<locals>.<lambda>��c�g|]39}�||d��qS�r�rLrZ)�num_options_listrLrMr\��z3ExperimentManager.get_predicted.<locals>.<listcomp>rcr�r�rLrZ)r�rLrMr\�r�zProcessing groups)�desccSsg|]}t|�qSrLr)rH�qrLrLrMr\�srfz./Integrate_Code/final_prompt_�.txt�wr�r��Commonsense�Lawr_r�)�construct_final_promptr5r3�apply�tolistr�rqrr�r<�writer8�lowerrx�
get_responsesr6r�)rKrr�r�Zgrouped_num_optionsZgrouped_prompts�num�groupZencoded_task_textr�Zfinal_prompt�f�paramsr�rL)r�r�rMr��sB4041424344���454647zExperimentManager.get_predictedr�cs�tj||jjdd�}|jdkrL|jd}tj||gdd�}t�d��|j�fdd	�dd�|d48<t	�49||ddkd50�}t�dt	�
|d
����||fS|jdkr�|jd}tj||gdd�}t�d��|j�fdd	�dd�|d<t	�51||ddkd�}t�dt	�
|d
����||fS|jdkr�|jd}tj||gdd�}|jdd	�dd�|d<t	�52||ddkd�}t�dt	�
|d
����||fS|jdk�r	|jd�dd	��}tj||gdd�}|d|dk|d<t	�53||ddkd�}t�dt	�
|d
����||fS|jdk�rA|jd}tj||gdd�}|jdd	�dd�|d<t	�54||ddkd�}t�dt	�
|d
����||fS)Nr�)rY�namererQr�)�axisz0./Integrate_Code/evaluate/metrics/rouge/rouge.pycs�j|dg|dgd�dS)Nr�rQ)�predictions�55referencesZrougeL��computer�)�rouge_scorerLrMr�%sz,ExperimentManager.calc_acc.<locals>.<lambda>ZRougeL_Score�ERRORz	RougeL = �rfz../Integrate_Code/evaluate/metrics/chrf/chrf.pycs �j|dg|dgdd�dS)Nr�rQrj)r�r�Z56word_order�scorer�r�)�57chrf_scorerLrMr�3r]zchrf++z	chrf++ = rgr�cSst|d|d�S)Nr�r�)rr�rLrLrMr�=s�correctzaccuracy = rhcSs|d��S)Nr�)r�r�rLrLrMr�DsricSs|d��|d��kS)Nr�rQ)r�r�rLrLrMr�Rr�)�pd�Seriesr3rYr5�concat�evaluate�loadr�rU�mean�_logger�info�round)rKr�Ztrue_labels�58save_stater�rL)r�r�rMr�sL596061/62�6364!65�66��67zExperimentManager.calc_acc皙�����?�n_shots_to_test�n_runs�too_long_patience�context_window_size�shuffle_numc+68sjdd�tdd�D�}t�t|�|f�}t�t|�|f�}t�t|�|f�}	t�t|�t|�f�}69g}dd�t|�D�}g�g}
dd�t|�D�}dd�t|�D�}d}tt|��D�]�\}}g}t�d	|���|�|j	|�|j70dkr�|t��}|dkr�|�|��}��|��fd71d�t|�D�}t|�D]B}||}|j
j|}t|d�}|�|�}|�||�\}}|dkr�n|�||�\|	||f<}tt|��|d
<||d<|�|�q�|dkr��n�|�|�qY|jdk�r�g}|t|
�}|
}|dk�r	|�||
�}|
�|�|j
j|}|j
j|j
j�|�}tt|��D]c}||dk�r6t|d�}|�|�}n!t|||�} t||j| ||d�\}!}"|"}t|!d�}|�|�}|�||�\}}|dk�rfn|�||�\|72||f<}tt|��|d
<||d<|�|��q!|dk�r��nS|�|�qY|jdk�rd}#|#|k�r�|�|||#||#�\}$}%||#�|$�||#�|%�||#}|�|�}|�||�\}}|dk�r�n&|�||�\|||#f<}tt|%��|d
<||d<|�|�|#d7}#|#|k�s�|dk�r�n�|�|�qYd}#|#|k�r�||#}&|t|&�}|dk�r(|�||&�}||#�|�||#}|j
j|}t|d�}|jdk�r�|jdk�r�|jd���|jd�����fdd�tt���D�}'t �!|j	|#�t �"dt|��}(g}tt|'��D]}t#�$|�})|�|�|)d|(�|'|g|)|(d����qxn|�|�g}|�||�\}}|dk�r�n&|�||�\|||#f<}tt|��|d
<||d<|�|�|#d7}#|#|k�s|dk�r�n|�|�qY|j73dk�r�|	|fS|jdk�r�|74|fS|jdk�r1t%d|j�d�ddd��}*tt|��D]}|*�&||d��q
Wd�||fS1�s(wY||fS||fS)NcSsg|]}dd|�qS)rg{�G�z�?rLrZrLrLrMr\`r�zAExperimentManager.run_experiment_across_shots.<locals>.<listcomp>r�cS�g|]}g�qSrLrL�rH�_rLrLrMr\j�cSr�rLrLr�rLrLrMr\nr�cSr�rLrLr�rLrLrMr\or�Tzstarting with n = csg|]75}t��t����qSrL)rV�sampler�r�)�base_indices_shufflerLrMr\�r��promptFZprompt_example_indicesZtoken_number_of_prompt)�	noisy_idxZ76residue_dfZ77prompt_newr�rhrIrQcsg|]78}�|�|�qSrLrLrZ)r��
solution_listrLrMr\r�z"./Integrate_Code/prompt_reinforce_r�r�r�r�z79 80)'rqrU�zerosr��	enumeraterr�r�r|rortr�r�r4r�rXr��check_prompt_lengthr��strr�rurY�isin�intrr5rvr�rwr3r�rVrl�randint�copy�deepcopyr�r�)+rKr�r�r�r�r�Znoisy_ratio�81accuraciesZaccuracies_reinforceZaccuracies_shuffleZaccuracies_noisyr�Zbase_indices_per_runZbase_indices_noisyZprompt_reinforceZright_idx_reinforce�stater[r~Zpredictions_rowZadditional_shotsZ	new_shotsZshuffled_indices_listr�Zmany_shots_idx�selected�many_shots_prompts�windows_many_shots�longest_window_n_tokensZthis_predictionr��otherZnoisy_levelZselected_noisyZ
all_noisy_idxr�r�Z
new_right_idx�base_indicesZtest_prompts_listZ82insert_idxZcopied_listr�rL)r�r�r�rM�run_experiment_across_shotsYs83848586878889909192939495�969798992�100101102�1034�104��z-ExperimentManager.run_experiment_across_shotsrcs��jt�j�krd�_�fdd�td�D��_�fdd��j�jD�}|t|�kr5�jj|d|�}ntd�|jjsAJd���jd7_|jS)	NrcrSrLrTrZrJrLrMr\8r]z4ExperimentManager.sample_n_shots.<locals>.<listcomp>r^csg|]}|�vr|�qSrLrLrZ)rrLrMr\<r�z*n_shots is larger than the length of indexz#many shots samples were not unique!r�)	rsr�rrrqr4r��printrY�	is_unique)rKr~rZ	index_newZ
many_shots_dfrL)rrKrMr�4sz ExperimentManager.sample_n_shotsr105cCst�|dt|���SrD)r�joinr�)r106rLrLrMr�Jsz'ExperimentManager.build_many_shots_textcCs�g}d|vr3	tt|��D]"}|jdkr!|�||||||�}n	|�||||�}|�|�q
	|Sd|vrctt|��D]"}|jdkrQ|�||||||�}n	|�||||�}|�|�q=	|Sd|vrj	|S|ddkr�|ddkr�td|d|dd�}n/|ddkr�|ddkr�td|dd	�}n|ddkr�|ddkr�td|dd107�}ntdd�}t���;|j�	||�}	tt|	��D]$}|	|}108|109j110dj}|jdkr�|�|||�}n|�|�}|�|�q�Wd�|S1s�wY|S)Nr>rhr@r?r_r`r�r�r_r��r�r_�r�r��r�)
rqr�r5�get_responser�r	r�r�rkr�r�r��process_outputs)rKr�r6r�r�r�r[r�r�r�r�r�rLrLrMr�OsT111"�112��113114115116�117�
�
zExperimentManager.get_responsesc
	Cs�d}d|vr�|ddkr!|ddkr!tjjd|d|ddd�}n8|ddkr9|ddkr9tjjd|ddd�}n |ddkrQ|ddkrQtjjd|ddd�}ntjjddd	�}t���<	tjd118dd�t�d
�}|j||d�}z|j}	|�	|	|�}Wnt119y�d}Yn	wWd�|SWd�|S1s�wY|Sd|vr�tdd�}120|121jj
jdddd�d|d�gd�}z|jdjj}	|�	|	|�}W|St122y�d}Y|Swd|vr�	|S|ddkr�|ddkr�td|d|dd�}n3|ddk�r|ddk�rtd|dd�}n|ddk�r'|ddk�r'td|dd�}ntdd�}t���|j�|g|�d}|jdj}	|�	|	|�}Wd�|S1�sSwY|S)Nr>r_r`r�g)�candidate_count�max_output_tokens�stop_sequencesr�)rrr�)rrr�)rr�Z'AIzaSyAmTdSjoQXgyImpVjLOTQu5QyqbcPRQo8k�rest)�api_key�	transportzgemini-1.5-pro)�generation_configr�r@z�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)rzgpt-4oZ	developerzYou are a helpful assistant.)�role�content�user)rk�messagesrr?rrrr)�genai�types�GenerationConfigr�r��	configureZGenerativeModelZgenerate_contentr�r�	Exceptionr�chat�completions�create�choices�messager!r	rkr�r�)
rKZ123prompt_oner6r�Znum_options_oner�rrkr�r��clientZ124completionr�rLrLrMr�s�125126��127�4�128�4�4�129���130���131132133��zExperimentManager.get_responser�cCs�|jdkr134t|�}n&|jdkr.t|�}|dkr-|}t�|j�t�dt|�d�}t|}n|}|durK|�	��135t�}|�d�d�d�d�
�}|S|}|S)Nrgrhrr�rr�)r5rrrVrlrorrr,r�r�r�r�r�)rKr�r��predZ136option_numr�r�rLrLrMr�s"137138139��z!ExperimentManager.process_outputsrcsld�jvsd�jvsd�jvrd}d}nt�fdd�|D��}t�jt�}||�j|kr2d|fSd|fS)	Nr>r?r@rAc3rCrDrE)rH�windowrJrLrMrN�s��z8ExperimentManager.check_prompt_length.<locals>.<genexpr>FT)r6rnrrFrrm)rKrr�rZn_tokens_between_shotsrLrJrMr�s�z%ExperimentManager.check_prompt_lengthcsBd�|jdkr�|jdkr/tdddd��}|��D]}�|7�qWd�n1s)wYn�|jdkrWtd	ddd��}|��D]}�|7�q@Wd�n1sQwYn�|jd140krtdddd��}|��D]}�|7�qhWd�n1sywYn]|jdkr�td
ddd��}|��D]}�|7�q�Wd�n1s�wYn5td|j���d�ddd��}|��D]}�|7�q�Wd�n1s�wY|jdkr܈�d|j���d7�t|�dk�r�d|d�|j	d�141�}�fdd�|D���S�fdd�|D��|j	d�142�}��fdd�tt���D���S)Nr�rfzEnglish->Kurdishz+./Integrate_Code/initial_prompt_Kurdish.txtr�r�r�zEnglish->Bembaz)./Integrate_Code/initial_prompt_Bemba.txtzEnglish->Frenchz*./Integrate_Code/initial_prompt_French.txtzEnglish->Germanz*./Integrate_Code/initial_prompt_German.txtz ./Integrate_Code/initial_prompt_r�rBz{$}rr�rrIcsg|]}�t|�qSrLr)rH�question)�manyshots_examplesrLrMr\r�z<ExperimentManager.construct_final_prompt.<locals>.<listcomp>cr�r�rL)rHr�r�rLrMr\r�cs g|]}�|t�|�qSrLrrZ)r2r�rLrMr\"r])r5r<r�r�r8r��replacer=r�r3r�rq)rKr143r�r�r�rL)r�r2r�rMr��s^144145146���147148���149150���151152���153154��155z(ExperimentManager.construct_final_prompt)Nr1r2FNN)r�)r�r2r�rD)�__name__�156__module__�__qualname__r��	DataFramerrr�boolrzr|�floatr�r�r�r�r�rrX�npt�NDArrayr��staticmethodr�r�rrrr�rLrLrLrMr0*sP$�����157�B YCD���158�\1591601Er0):�loggingrV�typingrr�collectionsrrrr��numpyrUr��numpy.typingr:�pandasr�r�vllmrr	�161contextlibr162Zgoogle.generativeai.typesrr�logits_processorr
�	constantsr�google.generativeai�generativeair$Ztorch.nn.utils.rnnrr�openair�utilsrrrrrrrrrrr�	getLoggerr4r��basicConfig�INFOr�r,r0rLrLrLrM�<module>s44163