CoolFace
Apppublic

iruno/test_wprm3

sourceHugging Faceupdated 1y agoView on Hugging Face
0likes
agent.cpython-311.pyc98 linesDownload Raw Back to __pycache__
1�

2�[+h�G��D�ddlmZmZddlZddlZddlZddlZddlmZddl	m3Z4ddlmZddl
mZmZmZmZddlmZd	Zd5Zddd
�ddd
�d�ZGd�de��ZGd�de��ZGd�de��ZGd�de��ZGd�de��ZGd�de��ZdS)�)�ABC�abstractmethodN)�Client)�6ChatOpenAI���get_messages)�&JUDGE_OURS_BT_MODELING_PROMPT_TEMPLATE�3JUDGE_OURS_BT_MODELING_WO_CHECKLIST_PROMPT_TEMPLATE�1JUDGE_OURS_BT_MODELING_MULTIMODAL_PROMPT_TEMPLATE�>JUDGE_OURS_BT_MODELING_MULTIMODAL_WO_CHECKLIST_PROMPT_TEMPLATE)�image_to_base64_url��g333333�?g333333�?)�input_token_cost�output_token_costg@�7)zgpt-4o-minizgpt-4oc�.�eZdZededefd���ZdS)�Agent�inputs�returnc��dS�N�)�selfrs  �J/home/intern/junhee/webrm_demo/Web-Shepherd-Demo/agent/mini_bench/agent.py�generate_responsezAgent.generate_response!s����N)�__name__�8__module__�__qualname__r�dict�strrrrrrr s@�������
��
��
�
�
��^�
�
�
rrc�T�eZdZdefd�Zd�Zddefd�Zdedefd	�Z	ddededefd9�Z10dS)�	BaseAgent�agent_configc�<�||_|���dSr)r&�_setup)rr&s  r�__init__zBaseAgent.__init__&s��(������
�
�
�
�
rc	���|j�dd��}|rHt|jd|jd|jd|jdddd	�11��|_nEt|jd|jd|jd|jdd���|_|jd|_|jd|_|j�d
d��|_|j�dd��|_t�|jdd��}|r+d|jdvr|d|_	|d|_12dSd|_	d|_13dS)N�
use_log_probsF�14model_name�base_url�api_key�temperature�,Tr)�modelr-r.r/�timeout�logprobs�top_logprobs)r1r-r.r/r2�num_generate�
use_checklist�use_multimodal�apirr�)r&�getr�llmr/r5r6r7�MODEL_COST_MAPPINGrr)rr+�15model_costs   rr(zBaseAgent._setup*s����)�-�-�o�u�E�E�
��	�!��'��5��*�:�6��)�)�4� �-�m�<�������D�H�H�"��'��5��*�:�6��)�)�4� �-�m�<�����D�H� �,�]�;��� �-�n�=���!�.�2�2�?�E�J�J���"�/�3�3�4D�e�L�L���(�+�+�D�,=�l�,K�T�R�R�16��	)�%�4�#4�Z�#@�@�@�$.�/A�$B�D�!�%/�0C�%D�D�"�"�"�$'�D�!�%(�D�"�"�"rN�constraint_str_listc��d}d}|jdkrH|j�|��}||jddz
}||jddz
}�nt	t17��D]�}	|j�|��}||jddz
}||jddz
}|r\d}|D]}||jvr|dz
}�|t|��krnytd|dz�dt18����t|j��nnE��#t$r7}	td|	����d}tjt��Yd}	~	��d}	~	wwxYw|j
|zd	z|j|zd	zz}19|�d20|21fS|j|22fS)Nr�token_usage�
prompt_tokens�completion_tokensrz Agent has fomat issue, retry... �/zAgent returned an Error: �@B�)r/r;�invoke�response_metadata�range�	MAX_RETRY�content�len�print�	Exception�time�sleep�RETRY_SLEEPrr)r�model_inputr>�total_input_tokens�total_output_tokens�response�i�pass_constraint_num�constraint_str�e�costs           r�generate_with_retryzBaseAgent.generate_with_retryLs���������q� � ��x���{�3�3�H��(�"<�]�"K�O�"\�\���8�#=�m�#L�M`�#a�a����9�%�%�
,�
,��,�#�x���{�;�;�H�&�(�*D�]�*S�Tc�*d�d�&�'�8�+E�m�+T�Uh�+i�i�'�*��./�+�.A�9�9�N�-��1A�A�A� 3�q� 8� 3��.�#�6I�2J�2J�J�J�!�E�!�"V�Q�q�S�"V�"V�9�"V�"V�W�W�W�!�(�"2�3�3�3�3���4��!�,�,�,��9�a�9�9�:�:�:�#�H��J�{�+�+�+�+�+�+�+�+�����,����23�$�'9�9�G�C�d�F\�_r�Fr�u|�F|�|�����t�8�O��#�T�)�)s�/A0D�!2D�24E� -E�ErQ�prompt_typec�25�g}|Srr�rrQr[�messages    r�prepare_messagezBaseAgent.prepare_messageqs�����rc���d}g}|�||��}t|j��D]5}|�||��\}}	|�|��||	z
}�6||fS)Nr)r_rHr5rZ�append)26rrQr[r>�27total_cost�
response_listr^rUrTrYs28          rrzBaseAgent.generate_responseus����29��
��&�&�{�K�@�@���t�(�)�)�	�	�A�!�5�5�g�?R�S�S�N�H�d�� � ��*�*�*��$��J�J��j�(�(rr)rr r!r"r)r(�listrZr#r_rrrrr%r%%s��������T����� )� )� )�D#*�#*�D�#*�#*�#*�#*�J�4��c�����
)�
)�T�
)��
)�Z^�
)�
)�
)�
)�
)�
)rr%c�0��eZdZdef�fd�Zdefd�Z�xZS)�GroundingJudgeAgentr&c�r��t���|��|���dSr��superr)r(�rr&�	__class__s  �rr)zGroundingJudgeAgent.__init__��,���
������&�&�&����
�
�
�
�
rrQc	�f�t|d||j|jd|jd���}|S)N�judge_grounding�
text_obs_type�image_obs_type)�30input_info�inference_moder[r7�text_obs�	image_obs)r	r7r&r]s    rr_z#GroundingJudgeAgent.prepare_message�sC���"�,�#��.��&��7��'�(8�9�
31�32�33���r�rr r!r"r)r_�
__classcell__�rks@rrfrf�s_��������T�������	�4�	�	�	�	�	�	�	�	rrfc�\��eZdZdef�fd�Zdefd�Zdedefd�Zdefd	�Z	ded34e35fd�Z�xZS)�ProgressJudgeAgentr&c�r��t���|��|���dSrrhrjs  �rr)zProgressJudgeAgent.__init__�rlrrQc36��|jddkrd}|jd}d}nn|jddkrd}d}|jd}nK|jddkrd}|jd}|jd}ntd	|jd�����|jd37rd}nd}t|d||j||||���}|S)
N�38input_type�	text_onlyFro�39image_onlyTrp�40text_imagezInvalid input type: �use_in_progress�judge_progress)rqrrr[r6r7rsrtr�)r&�41ValueErrorr	r6)rrQr[r7rsrtr�r^s        rr_z"ProgressJudgeAgent.prepare_message�s����\�*�K�7�7�"�N��(��9�H��I�I�
�
�|�
,�l�
:�
:�!�N��H��)�*:�;�I�I�
�
�|�
,�l�
:�
:�!�N��(��9�H��)�*:�;�I�I��U�D�4E�l�4S�U�U�V�V�V���.�/�	$�"�O�O�#�O��"�+�#��,�)���+�	42�	43�	44���r�ori_logprob�add_logprobc��|�|Stj|��}tj|��}tj||z��Sr)�math�exp�log)rr�r��ori_prob�add_probs     rr�zProgressJudgeAgent.add_logprob�sA�������x��,�,�H��x��,�,�H��8�H�x�/�0�0�0rr3c	�X��gd�gd�gd�d�}d}g}t|��D]�\}}d|vr�d}|D]}|d||vr|}n�|r�dddd�}	d|vr|dD]A}45|D]<}||D]1}||46dvr%|�|	||47d	��|	|<�2�=�B|	D]}|	|�td48��|	|<�|�|	��n>|	D]&}||kr|d	|	|<�td49��|	|<�'|�|	��d|vrn||dz
}��t	|��dkrd
d
d
d�gSg}|D]k}
d�|
d|
d|
dfD��}t|����fd�|D��}|�|d|d|dd����l|S)N)z YesuĠYes�YesuĊYesuĠyes�yesuĊyesuĠYES�YESuĊYESuĠDone�DoneuĊDoneuĠCompleted�	CompleteduĊCompletedu	ĠCorrect�Correctu	ĊCorrect)z NouĠNo�NouĊNouĠNO�NOuĊNOuĠNot�NotuĊNotuĠNone�NoneuĊNoneuĠNope�NopeuĊNopeuĠUn�UnuĊUnuĠWrong�WronguĊWrong)z InuĠIn�InuĊInu	ĠPending�Pendingu	ĊPendinguĠPart�PartuĊPartu	ĠPartial�Partialu	ĊPartialuĠInProgress�50InProgressuĊInProgress)r��no�inrEz<answer>�tokenr4�logprobz-infz	</answer>rr9c�6�g|]}tj|����Sr)r�r�)�.0�xs  r�51<listcomp>z6ProgressJudgeAgent.get_judge_probs.<locals>.<listcomp>s ��p�p�p�������p�p�prr�r�r�c���g|]}|�z��Srr)r�r��sum_exp_logprobss  �rr�z6ProgressJudgeAgent.get_judge_probs.<locals>.<listcomp>!s��� L� L� L�!��%5�!5� L� L� Lrr�)�	enumerater��floatrarK�sum)rr3�target_judge�response_str�judge_probs_listrU�log_prob�find_judge_str�52judge_type�token_judge_dict�53token_info�	judge_str�final_judge_probs_list�judge_probs�exp_logprobs�
softmax_probsr�s                @r�get_judge_probsz"ProgressJudgeAgent.get_judge_probs�s������������%54�55��4����$�X�.�.�+	.�+	.�K�A�x��\�)�)�!%��".���J���(�L��,D�D�D�)3����E�"�B� $�"�"�(�(�$�56&��1�1�*2�>�*B�M�M�J�.:�M�M�57�1=�j�1I�!M�!M�I�(1�J�w�4G�'G�'G�GK�GW�GW�Xh�is�Xt�wA�BK�wL�HM�HM�(8��(D��!M�M�+;�M�M�J�/�58�;�C�?D�V�}�}� 0�� <��(�/�/�0@�A�A�A�A�+;�M�M�J�)�^�;�;�?G�	�?R� 0�� <� <�?D�V�}�}� 0�� <� <�(�/�/�0@�A�A�A��l�*�*����H�W�-�-�L�L��� � �A�%�%�������
�&(�"�/�
�
��p�p�k�%�6H�+�VZ�J[�]h�im�]n�5o�p�p�p��#&�|�#4�#4� � L� L� L� L�|� L� L� L�
�&�-�-�(��+�'��*�'��*�/�/�����59*�)rr[c�6�d}g}|�||��}t|j��D]�}	|j�|��}|jdd}|jdd}	|j|zdz|j|	zdzz}|jdd}60|�|j	|�61|62��d�����#t$r9}td	|����|�|j	gd���Yd}~��d}~wwxYw||fS)63Nrr@rArBrDr3rJ)rTr�zError: )
r_rHr5r;rFrGrrrarJr�rMrL)rrQr[rbrcr^rUrTrRrSr3rXs            r�generate_probsz!ProgressJudgeAgent.generate_probs)s����64��
��&�&�{�K�@�@���t�(�)�)�	�	�A�
��8�?�?�7�3�3��%-�%?�
�%N��%_�"�&.�&@��&O�Pc�&d�#�!�2�5G�G�'�Q�TX�Tj�nA�UA�DK�UK�K�65�#�5�j�A�)�L���$�$�$,�$4�'+�';�';�H�'E�'E���������
�
�
��m��m�m�$�$�$��$�$�$,�$4�')���������������
�����j�(�(s�BC�66D�/D�D)
rr r!r"r)r_r�r�rdr�r#r�rvrws@rryry�s���������T��������4�����B1�u�1�5�1�1�1�1�c*��c*�c*�c*�c*�J)�$�)�S�)�)�)�)�)�)�)�)rryc�0��eZdZdef�fd�Zdefd�Z�xZS)�ChecklistGenerationAgentr&c�r��t���|��|���dSrrhrjs  �rr)z!ChecklistGenerationAgent.__init__JrlrrQc�*�t|d|���}|S)N�checklist_generation)rqrrr[rr]s    rr_z(ChecklistGenerationAgent.prepare_messageNs&���"�1�#�67�68�69��70�rrurws@rr�r�Is_��������T��������4��������rr�c��eZdZddededefd�Zdedeefd�Zd	ed71eeezfd�Z	dd72eeezdefd
�Z73deezfd�Zdd74eeezdefd�ZdS)�ClassifierRewardAgentF�urlr6r7c�0�||_||_||_dSr)r�r6r7)rr�r6r7s    rr)zClassifierRewardAgent.__init__Xs�����*���,����r�prompt�75image_listc��g}|�d��d}|�d��d}d|d�dt|d��d�d|d�g}|S)Nz<IMAGE_PLACEHOLDER>rr�text)�typer��image)r�r�)�splitr)rr�r��multimodal_message�text_prompt_prefix�text_prompt_suffixs      r�_process_multimodal_messagez1ClassifierRewardAgent._process_multimodal_message]sx����#�\�\�*?�@�@��C��#�\�\�*?�@�@��C���%7�8�8��':�:�a�=�'I�'I�J�J��%7�8�8�	76��"�!r�user_prompt_templaterQc���|jr0|djdi|��}|�||d��}n|djdi|��}|djdi|��}d|d�d|d�g}|S)N�userr��	assistant)�rolerJr)r7�formatr�)rr�rQ�tmp_user_prompt�user_prompt�assistant_prompt�querys       r�_make_queryz!ClassifierRewardAgent._make_queryis�����	�A�2�6�:�A������O� �;�;�O�[�Ye�Mf�g�g�K�K�=�.�v�6�=������K�D�/��<�C�77�78��79�80����4�4� �-=�>�>�81���r�batchc�0����jr�jrt�nt�n�jrt�nt82��jr*|r��fd�|D��}nF���|��g}n.|r��fd�|D��gd�}n���|��gd�}|S)Nc�<��g|]}���|����Sr�r��r��inputrr�s  ��rr�z9ClassifierRewardAgent.prepare_message.<locals>.<listcomp>�s*���b�b�b�U�4�+�+�,@�%�H�H�b�b�brc�<��g|]}���|����Srr�r�s  ��rr�z9ClassifierRewardAgent.prepare_message.<locals>.<listcomp>�s+���e�e�e�PU�d�.�.�/C�U�K�K�e�e�er)r��promptts)r��prompts)r6r7rr83r
rr�)rrQr�r^r�s`   @rr_z%ClassifierRewardAgent.prepare_message|s�������		[��"�
N�'X�$�$�'M�$�$��"�
[�'e�$�$�'Z�$���	��
P�b�b�b�b�b�Va�b�b�b����+�+�,@�+�N�N�O����	
�e�e�e�e�e�Yd�e�e�e� "�����"�-�-�.B�K�P�P�!����84�rr^c�,�ddi}	|jrtj|jd|id���}n/tj|j|t	j|��d���}|���|���}d|vrtd	|����gSd85|jvr|dSd|jvr|dStd
|j�����#tj	j86$rtd|j����gcYStj	j$r&}td|j�d|����gcYd}~Sd}~wtj$rtd|j����gcYSt$r&}td|�d|j����gcYd}~Sd}~wwxYw)NzContent-Typezapplication/json�messagesiX)�jsonr2r0)�headers�datar2�rewardsz0Error: 'rewards' key not found in API response: �87get_reward�pooling�rewardz
Invalid URL: zError: Request timed out to zError during request to z: z+Error: Failed to decode JSON response from zError: Missing key z in response from )r7�requests�postr�r��dumps�raise_for_statusrLr��88exceptions�Timeout�RequestException�JSONDecodeError�KeyError)rr^r�rT�
response_jsonrXs      r�get_rm_scroez"ClassifierRewardAgent.get_rm_scroe�s��!�#5�6��+	��"�
�#�=��H�$�g�.�������$�=��H�#���G�,�,��	����
�%�%�'�'�'�$�M�M�O�O�M��
�-�-��X��X�X�Y�Y�Y��	��t�x�'�'�$�Y�/�/��d�h�&�&�$�X�.�.�!�!;���!;�!;�<�<�<���"�*�	�	�	��;���;�;�<�<�<��I�I�I��"�3�	�	�	��<�T�X�<�<��<�<�=�=�=��I�I�I�I�I�I������#�	�	�	��J���J�J�K�K�K��I�I�I��	�	�	�
�H��H�H�d�h�H�H�
I�
I�
I��Y�Y�Y�Y�Y�Y�����	���sN�BC�C�,C�=C�-F�F�D7�1F�7+F�$	F�-F�F�Fc��|r|�|d���}n|�|��}|�|��}|dfS)NT)r�r)r_r�)rrQr�r^r�s     rrz'ClassifierRewardAgent.generate_response�sT���	8��*�*�;�d�*�C�C�G�G��*�*�;�7�7�G��#�#�G�,�,����z�rN)FF)F)
rr r!r#�boolr)rdr�r"r�r_r�rrrrr�r�Ws������-�-�C�-��-�d�-�-�-�-�89 90"�#�91"�4��9�92"�93"�94"�95"����4�$�t�*�CT�����&��4�$�t�*�+<��T�����>.�D�4�K�.�.�.�.�b��T�D��J�->��t������rr�)�abcrrrNr�r�r��	langsmithr�langchain_openairr�r	�prompts.judge_promptr96rrr
�prompts.image_utilsrrIrPr<rr%rfryr�r�rrr�<module>r�s$��#�#�#�#�#�#�#�#�����������������������'�'�'�'�'�'�!�!�!�!�!�!�������������5�4�4�4�4�4�
�	���!� ���97 ����	�	��
�
�
�
�
�C�
�
�
�98])�])�])�])�])��])�])�])�@�����)����"p)�p)�p)�p)�p)��p)�p)�p)�f�����y����|�|�|�|�|�E�|�|�|�|�|r
iruno/test_wprm3 · CoolFace