tawondazz/Mobile-Agent

1o

28�|f��@s�dd
lZdd
l
Z
ddlmZ
ddlmZ
ddlmZ
ddlm	Z	m3Z4mZmZ
ddl
mZmZmZmZmZ
ddlmZ
dd
lZdd
lZdZd	Zd5d�Zdd
�Zej�d�
sYe�d�

ej�d�
sde�d�

dd�Zd
S)�N)
�BytesIO)
�Image)
�inference_chat)�get_action_prompt�get_reflect_prompt�get_memory_prompt�get_process_prompt)�init_action_chat�init_reflect_chat�init_memory_chat�add_response�add_response_two_image)
�MultiModalConversationzhttp://47.88.8.18:8088/api/askz�eyJ0eXAiOiJqd3QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6ImxjbDE5Mzc5OC0xOTM3OTgiLCJwYXNzd29yZCI6IjkyMTIwMyIsImV4cCI6MTk5NDkxNTQwNX0.QvFuzu035i5ApMuf1w75vKjEiQ0Dj0QPlfJmuZtfM30c
Cs8|�d
�
r|�
d�
d}t�|�
}
t|
�
}t�|�
}|S)Nz6data:image�
,�����)�7startswith�split�base64�	b64decoderr�open)Z
base64_stringZ8image_dataZimage_stream�	pil_image�r�Ae:\Desktop\Mobile-Agent-v2-demo-local\MobileAgent\local_server.py�
base64_to_pils9

10

11

rcCsld
t_
d|}dd|i
d|
i
gd�g
}tjd|d�}z|d	d12ddd
dd}W|S


d}Y|S)Nz#sk-f5573cde82d84ebe8620ace6f5e301adzfile://�user�image�text)�role�contentzqwen-vl-plus)�model�messages�output�choicesr
�messagerzThis is an icon.)�	dashscope�api_keyr�call)r�queryr �responserrr�
process_images 

����"
�
r)�13screenshot�tempc
,
sp|d
}
|
dkrr|d�
|d}g}|D]}|d}|d}t|�
}d|}|�
|d�
|�|�

qi}tj���+���
fd	d14�t|�
D�
}	tj�|	�
D]}15|	|16}|17��}|||d<qLWd�
n1sgw



Y
d|i
}
|
S|
d
kr�|d}|d}|d}|d}|d}|d}t	||||||�}t18�}td||�}t|dt
t�}d
|i
}
|
S|
dk�
rI|d}t|�
}d}|�
|d�
|d}|d}|d}|d}|d}|d}|d}|d}|d}|d}|d}|d}|d }|d!} t||||||||||||�}!t�}"td|!|"|�}"t|"d"t
t�}#|#d#k�
r&d#dd$�}
|
Std%|#|"�}"d}$|�
rBt| �
}%td|%|"�}"t|"d"t
t�}$|#|$d$�}
|
S|
d&k�
r�|d}t|�
}d}|�
|d�
|d'}&t|&�
}&d(}'|&�
|'d�
|d}|d)}(|d}|d}|d}|d}|d}|d}t||(||||||�})t�}*td|)|*|'|g�}*t|*d"t
t�}+d&|+i
}
|
Sd*d+i
}
|
S),N�task�captionr'�images�19image_name�20image_fileztemp/�PNGc
s i|]\}
}��t
|�
�|
�qSr)�submitr))�.0�
ir��executorr'rr�21<dictcomp>Js z&mobile_agent_infer.<locals>.<dictcomp>�
�icon_map�planning�instruction�thought_history�summary_history�action_history�completed_requirements�add_inforzgpt-4-turbo�decision�screenshot_filezscreenshot/screenshot_local.png�perception_infos�width�height�summary�action�22error_flag�memory�
memory_switch�insightzgpt-4ozNo token)rArIZ	assistant�23reflection�last_screenshot_filez$screenshot/last_screenshot_local.png�last_perception_infos�errorzGThe task must be in "caption", "planning", "decision" and "reflection".)r�save�append�24concurrent�futures�ThreadPoolExecutor�	enumerate�as_completed�resultrrrr�API_url�tokenrr	rrr25r
),�	json_datar,r.Zlocal_imagesrr/r0�26image_pathr9rS�futurer4r(r!r;r<r=r>r?r@Zprompt_planningZ
chat_planning�output_planningrBrCrDrErFrGrHrIrJrKZ
prompt_actionZchat_action�
output_action�
output_memoryZ
prompt_memoryrMZlast_image_pathrNZprompt_reflectZchat_reflectZoutput_reflectrr5r�mobile_agent_infer:s�
















��






27
�


28


















�

29
30






31
32















�



r`)�osr�ior�PILrZMobileAgent.apirZMobileAgent.prompt_no_inputrrrrZMobileAgent.chatr	r33rrr
r$rrRrXrYrr)�path�exists�mkdirr`rrrr�<module>
s&







	34

35