Pamudu13/gemma-3-chat
0
1import asyncio2import time3import platform4import json5import os6from typing import List, Optional, Tuple7from functools import wraps8 9# ================== 核心修复：延迟导入 GUI 库 ==================10_pag = None11_pp = None12GUI_AVAILABLE = False13 14def _lazy_pag():15 global _pag, GUI_AVAILABLE16 if _pag is not None:17 return _pag18 import pyautogui19 _lazy_pag().FAILSAFE = True20 _lazy_pag().PAUSE = 0.0521 _pag = pyautogui22 GUI_AVAILABLE = True23 return _pag24 25def _lazy_pp():26 global _pp27 if _pp is not None:28 return _pp29 import pyperclip30 _pp = pyperclip31 return _pp32 33def _check_gui():34 if GUI_AVAILABLE:35 return True36 if _pag is not None:37 return True38 try:39 _lazy_pag()40 _lazy_pp()41 return True42 except (KeyError, ImportError, Exception) as e:43 print(f"⚠️ [Warning] 桌面鼠标键盘工具已禁用 (缺少 DISPLAY): {e}")44 return False45 46def require_gui(func):47 @wraps(func)48 async def wrapper(*args, **kwargs):49 if not _check_gui():50 return "执行失败：当前系统运行在无头环境(如Docker)中，没有物理显示器，无法执行鼠标和键盘操作。"51 return await func(*args, **kwargs)52 return wrapper53# ==============================================================54 55 56CURRENT_SCREEN_REGION = None57 58def set_screen_region(region: Optional[Tuple[int, int, int, int]]):59 """设置当前激活的屏幕映射区域"""60 global CURRENT_SCREEN_REGION61 CURRENT_SCREEN_REGION = region62 63def _percent_to_pixel(x_percent: float, y_percent: float) -> Tuple[int, int]:64 """内部辅助函数：将千分比 (0 到 1000) 转换为当前屏幕或指定区域的实际像素坐标。"""65 x_percent = max(0, min(1000, float(x_percent)))66 y_percent = max(0, min(1000, float(y_percent)))67 68 # 如果指定了局部屏幕区域，则基于局部区域计算坐标69 if CURRENT_SCREEN_REGION is not None:70 rx, ry, rw, rh = CURRENT_SCREEN_REGION71 px = rx + int(rw * (x_percent / 1000))72 py = ry + int(rh * (y_percent / 1000))73 74 # 确保不超出该区域的边界75 px = min(px, rx + rw - 1)76 py = min(py, ry + rh - 1)77 return px, py78 79 # 否则默认映射全屏坐标80 width, height = _lazy_pag().size()81 px = min(int(width * (x_percent / 1000)), width - 1)82 py = min(int(height * (y_percent / 1000)), height - 1)83 84 return px, py85 86 87@require_gui88async def mouse_move(x: float, y: float, duration: float = 0.5) -> str:89 """移动鼠标到屏幕千分比位置"""90 if x < 0 or x > 1000 or y < 0 or y > 1000:91 return "千分比坐标超出范围，请输入 0 到 1000 之间的值。"92 93 px, py = _percent_to_pixel(x, y)94 95 def _move():96 _lazy_pag().moveTo(px, py, duration=duration, tween=_lazy_pag().easeInOutQuad)97 time.sleep(0.02)98 99 await asyncio.to_thread(_move)100 return f"鼠标已成功移动到屏幕位置 ({x}‰, {y}‰)。 [LAST_ACTION: MOVE({x},{y})]"101 102 103@require_gui104async def mouse_click(button: str = "left", clicks: int = 1, x: Optional[float] = None, y: Optional[float] = None) -> str:105 """点击鼠标（支持千分比坐标）"""106 if x is not None and y is not None:107 if x < 0 or x > 1000 or y < 0 or y > 1000: 108 return "千分比坐标超出范围，请输入 0 到 1000 之间的值。"109 110 def _click_at():111 px, py = _percent_to_pixel(x, y)112 _lazy_pag().moveTo(px, py, duration=0.2)113 time.sleep(0.2) 114 _lazy_pag().click(x=px, y=py, clicks=clicks, button=button, interval=0.1)115 116 await asyncio.to_thread(_click_at)117 # 根据点击次数打上不同的标签118 tag = f"CLICK({x},{y})" if clicks == 1 else f"DOUBLE_CLICK({x},{y})"119 return f"鼠标已移动到 ({x}‰, {y}‰) 并使用 {button} 键点击了 {clicks} 次。 [LAST_ACTION: {tag}]"120 else:121 # 如果没有传入坐标（原地点击），我们无法在图片上准确标出位置，所以不带坐标标签122 await asyncio.to_thread(_lazy_pag().click, clicks=clicks, button=button, interval=0.1)123 return f"鼠标在当前位置使用 {button} 键点击了 {clicks} 次。[LAST_ACTION: CLICK_CURRENT]"124 125 126@require_gui127async def mouse_double_click(button: str = "left", x: Optional[float] = None, y: Optional[float] = None) -> str:128 """双击鼠标"""129 if x is not None and y is not None:130 if x < 0 or x > 1000 or y < 0 or y > 1000: 131 return "千分比坐标超出范围，请输入 0 到 1000 之间的值。"132 133 def _double_click():134 px, py = _percent_to_pixel(x, y)135 _lazy_pag().moveTo(px, py, duration=0.2)136 time.sleep(0.2)137 _lazy_pag().click(x=px, y=py, clicks=2, button=button, interval=0.1)138 139 await asyncio.to_thread(_double_click)140 return f"鼠标已移动到 ({x}‰, {y}‰) 并使用 {button} 键双击。 [LAST_ACTION: DOUBLE_CLICK({x},{y})]"141 else:142 await asyncio.to_thread(_lazy_pag().click, clicks=2, button=button, interval=0.1)143 return f"鼠标在当前位置使用 {button} 键双击。 [LAST_ACTION: CLICK_CURRENT]"144 145 146@require_gui147async def mouse_drag(x1: float, y1: float, x2: float, y2: float, duration: float = 1.0, button: str = "left") -> str:148 """从起始位置 (x1, y1) 拖拽到终点位置 (x2, y2)"""149 try:150 coords = {"x1": x1, "y1": y1, "x2": x2, "y2": y2}151 for name, val in coords.items():152 if val < 0 or val > 1000:153 return f"错误：{name} 坐标 ({val}) 超出范围，请输入 0 到 1000 之间的值。"154 155 px1, py1 = _percent_to_pixel(x1, y1)156 px2, py2 = _percent_to_pixel(x2, y2)157 158 def _drag():159 _lazy_pag().moveTo(px1, py1, duration=0.2)160 time.sleep(0.2) 161 _lazy_pag().dragTo(x=px2, y=py2, duration=duration, button=button, tween=_lazy_pag().easeInOutQuad)162 time.sleep(0.1)163 164 await asyncio.to_thread(_drag)165 return f"已成功将鼠标从 ({x1}‰, {y1}‰) 拖拽到 ({x2}‰, {y2}‰)。[LAST_ACTION: DRAG({x1},{y1},{x2},{y2})]"166 except Exception as e:167 return f"拖拽失败：{e}"168 169 170@require_gui171async def mouse_scroll(clicks: int) -> str:172 """滚动鼠标"""173 def _scroll():174 chunk_size = 10 if abs(clicks) > 10 else abs(clicks)175 direction = 1 if clicks > 0 else -1176 remaining = abs(clicks)177 178 while remaining > 0:179 current_chunk = min(chunk_size, remaining)180 _lazy_pag().scroll(current_chunk * direction)181 remaining -= current_chunk182 if remaining > 0:183 time.sleep(0.01)184 185 await asyncio.to_thread(_scroll)186 direction = "向上" if clicks > 0 else "向下"187 # 滚动无法标点，仅返回状态188 return f"鼠标滚轮已{direction}滚动了 {abs(clicks)} 个单位。[LAST_ACTION: SCROLL]"189 190 191@require_gui192async def mouse_hold(button: str, duration: float) -> str:193 """长按鼠标按键"""194 if duration > 30: duration = 30195 196 def _hold_logic():197 try:198 _lazy_pag().mouseDown(button=button)199 time.sleep(duration)200 finally:201 _lazy_pag().mouseUp(button=button)202 203 await asyncio.to_thread(_hold_logic)204 return f"已成功按住鼠标 {button} 键持续 {duration} 秒。[LAST_ACTION: HOLD]"205 206 207 208@require_gui209async def copy_to_input_box(text: str) -> str:210 """输入文本 (优化版：解决偶发性只输入字符 'v' 的 Bug)"""211 def _type_text():212 old_clipboard = ""213 try:214 old_clipboard = _lazy_pp().paste()215 except Exception:216 pass217 218 sys_os = platform.system()219 220 try:221 _lazy_pp().copy("")222 _lazy_pp().copy(text)223 wait_time = 0.2 if sys_os == "Windows" else 0.15224 time.sleep(wait_time)225 226 for i in range(3):227 if _lazy_pp().paste() == text: break228 time.sleep(0.1)229 _lazy_pp().copy(text)230 231 modifier = 'command' if sys_os == "Darwin" else 'ctrl'232 233 # 🌟 修复核心：显式按下修饰键并等待，确保操作系统队列 100% 确认 Ctrl/Cmd 处于被按住状态 🌟234 _lazy_pag().keyDown(modifier)235 time.sleep(0.05) # 50 毫秒的系统缓冲延迟，彻底阻断输入法或系统抢跑236 _lazy_pag().press('v')237 time.sleep(0.05) # 释放前的短暂等待238 _lazy_pag().keyUp(modifier)239 240 time.sleep(0.15)241 finally:242 time.sleep(0.05)243 for _ in range(2):244 try:245 if old_clipboard: _lazy_pp().copy(old_clipboard)246 break247 except Exception:248 time.sleep(0.05)249 250 await asyncio.to_thread(_type_text)251 return f"已复制文本到输入框：'{text}'"252 253@require_gui254async def keyboard_press(key: str, presses: int = 1) -> str:255 """按下单个按键多次"""256 def _press_logic():257 _lazy_pag().press(key, presses=presses, interval=0.05)258 259 await asyncio.to_thread(_press_logic)260 return f"已按下键盘按键 '{key}' {presses} 次。"261 262 263@require_gui264async def keyboard_sequence(keys: List[str]) -> str:265 """按顺序按下多个不同的按键，中间间隔 0.5 秒"""266 if not keys:267 return "错误：未提供按键列表。"268 269 def _sequence_logic():270 for i, key in enumerate(keys):271 _lazy_pag().press(key)272 # 如果不是最后一个按键，则等待 0.5 秒273 if i < len(keys) - 1:274 time.sleep(0.5)275 276 await asyncio.to_thread(_sequence_logic)277 return f"已按顺序执行按键序列：{', '.join(keys)}，按键间隔 0.5 秒。"278 279@require_gui280async def keyboard_hotkey(keys: List[str]) -> str:281 """按下组合快捷键"""282 if not keys: return "错误：未提供按键组合"283 284 def _hotkey():285 if len(keys) == 1:286 _lazy_pag().press(keys[0])287 else:288 modifier = keys[0]289 rest_keys = keys[1:]290 with _lazy_pag().hold(modifier):291 for k in rest_keys:292 _lazy_pag().press(k)293 time.sleep(0.02)294 295 await asyncio.to_thread(_hotkey)296 return f"已触发组合键：{' + '.join(keys)}。"297 298 299@require_gui300async def keyboard_hold(keys: List[str], duration: float) -> str:301 """长按按键"""302 if duration > 30: duration = 30303 304 def _hold_logic():305 start_time = time.time()306 try:307 for key in keys:308 _lazy_pag().keyDown(key)309 time.sleep(0.02)310 311 elapsed = 0312 while elapsed < duration:313 sleep_time = min(0.1, duration - elapsed)314 time.sleep(sleep_time)315 elapsed = time.time() - start_time316 except Exception as e:317 print(f"按住按键时出错: {e}")318 finally:319 for key in reversed(keys):320 try:321 _lazy_pag().keyUp(key)322 time.sleep(0.02)323 except Exception:324 pass325 326 await asyncio.to_thread(_hold_logic)327 return f"已成功长按组合键 {keys} 持续 {duration} 秒。"328 329 330@require_gui331async def logical_click(id: int) -> str:332 """通过 UI 树节点 ID 执行无障碍逻辑点击（支持窗口被遮挡及熄屏/锁屏后台操作）"""333 # 动态引入 UI 树缓存查询方法334 from py.ui_tree_helper import get_cached_element335 336 cached = get_cached_element(id)337 if not cached:338 return f"错误：未找到 ID 为 {id} 的有效 UI 元素。页面可能已刷新，请重新获取截图后再试。"339 340 system, handle = cached341 342 try:343 if system == "Windows":344 def _win_click():345 # 尝试一：标准 Invoke 动作 (对应大多数 Button 按钮)346 try:347 pattern = handle.GetInvokePattern()348 if pattern:349 pattern.Invoke()350 return True351 except Exception:352 pass353 354 # 尝试二：Toggle 动作 (对应复选框 Checkbox/单选框 Radio)355 try:356 pattern = handle.GetTogglePattern()357 if pattern:358 pattern.Toggle()359 return True360 except Exception:361 pass362 363 # 尝试三：SelectionItem 动作 (对应列表项/页签 Tab)364 try:365 pattern = handle.GetSelectionItemPattern()366 if pattern:367 pattern.Select()368 return True369 except Exception:370 pass371 372 # 尝试四：模拟无障碍点击 (不移动物理鼠标)373 try:374 handle.Click(simulateMove=True)375 return True376 except Exception:377 pass378 379 raise Exception("当前 Windows UIA 节点不支持任何已知的无障碍点击动作。")380 381 await asyncio.to_thread(_win_click)382 return f"已成功通过 Windows UIA 模式对节点 ID {id} 执行后台逻辑点击。[LAST_ACTION: LOGICAL_CLICK({id})]"383 384 elif system == "Darwin":385 import ApplicationServices as AX386 387 def _mac_click():388 # 尝试一：AXPress (macOS 标准按钮按下动作)389 err = AX.AXUIElementPerformAction(handle, "AXPress")390 if err == 0:391 return True392 393 # 尝试二：AXPick (菜单弹出项选择动作)394 err = AX.AXUIElementPerformAction(handle, "AXPick")395 if err == 0:396 return True397 398 # 尝试三：AXShowMenu (触发右键/下拉菜单动作)399 err = AX.AXUIElementPerformAction(handle, "AXShowMenu")400 if err == 0:401 return True402 403 raise Exception(f"AXUIElementPerformAction 返回无障碍错误码: {err}")404 405 await asyncio.to_thread(_mac_click)406 return f"已成功通过 macOS AXPress 模式对节点 ID {id} 执行后台逻辑点击。[LAST_ACTION: LOGICAL_CLICK({id})]"407 408 elif system == "Linux":409 import pyatspi410 411 def _linux_click():412 action = handle.queryAction()413 if action and action.nActions > 0:414 # 默认执行该节点的第一个关联行为（通常为点击/激活）415 action.doAction(0)416 return True417 raise Exception("当前 Linux AT-SPI 节点不具备动作接口。")418 419 await asyncio.to_thread(_linux_click)420 return f"已成功通过 Linux AT-SPI 模式对节点 ID {id} 执行后台逻辑点击。[LAST_ACTION: LOGICAL_CLICK({id})]"421 422 else:423 return f"未知的操作系统类型 {system}。"424 425 except Exception as e:426 # 当无障碍接口调用遇到应用不配合等死角时，提示 AI 退化执行物理鼠标点击427 return f"逻辑点击 ID {id} 失败（原因: {str(e)}）。建议立刻使用原物理工具 mouse_click 传入该节点的 center 坐标进行兜底点击。"428 429 430@require_gui431async def logical_type(id: int, text: str) -> str:432 """通过无障碍节点 ID 在后台输入文本（无需物理移动鼠标或使用剪贴板，支持锁屏和后台输入）"""433 from py.ui_tree_helper import get_cached_element434 cached = get_cached_element(id)435 if not cached:436 return f"错误：未找到 ID 为 {id} 的有效输入框。页面可能已刷新，请重新截图。"437 438 system, handle = cached439 try:440 if system == "Windows":441 def _win_type():442 # 尝试一：UIA ValuePattern (最标准的输入框赋值方法)443 try:444 pattern = handle.GetValuePattern()445 if pattern:446 pattern.SetValue(text)447 return True448 except Exception:449 pass450 # 尝试二：LegacyIAccessiblePattern 赋值451 try:452 pattern = handle.GetLegacyIAccessiblePattern()453 if pattern:454 pattern.SetValue(text)455 return True456 except Exception:457 pass458 raise Exception("该组件不支持 Windows UIA Value 赋值模式。")459 460 await asyncio.to_thread(_win_type)461 return f"已成功通过 Windows UIA 后台向输入框 ID {id} 输入文本：'{text}'"462 463 elif system == "Darwin":464 import ApplicationServices as AX465 466 def _mac_type():467 # macOS 底层魔法：直接通过系统无障碍接口重写该节点的 AXValue 属性468 err = AX.AXUIElementSetAttributeValue(handle, "AXValue", text)469 if err == 0:470 return True471 raise Exception(f"macOS AXValue 写入失败，无障碍错误码: {err}")472 473 await asyncio.to_thread(_mac_type)474 return f"已成功通过 macOS AXValue 后台向输入框 ID {id} 输入文本：'{text}'"475 476 else:477 return f"暂时不支持该系统平台后台逻辑输入。"478 except Exception as e:479 # 退化机制：如果逻辑输入失败，提示 AI 采用物理点击该输入框 + 粘贴的传统方式480 return f"后台逻辑输入失败（原因：{str(e)}）。请尝试先点击目标输入框，再调用 copy_to_input_box 粘贴输入。"481 482# 注意：wait 不需要 GUI，所以【不要】加 @require_gui483async def wait(seconds: float) -> str:484 """等待一段时间，让页面或程序加载"""485 seconds = min(max(0, seconds), 60)486 await asyncio.sleep(seconds)487 return f"已等待 {seconds} 秒。"488 489async def screenshot() -> str:490 """获取截图"""491 await asyncio.sleep(0.3)492 return "[Getting screenshot]"493 494# ================= 对应的 OpenAI 工具 Schema 定义 =================495 496mouse_move_tool = {497 "type": "function",498 "function": {499 "name": "mouse_move",500 "description": "将鼠标移动到屏幕上的指定位置。坐标使用千分比表示（0到1000）。(0,0)是屏幕左上角，(1000,1000)是右下角，(500,500)是屏幕正中心。",501 "parameters": {502 "type": "object",503 "properties": {504 "x": {"type": "number", "description": "目标水平坐标(X轴)，范围 0 到 1000 的千分比。例如 500 表示宽度正中间","maximum": 1000, "minimum": 0},505 "y": {"type": "number", "description": "目标垂直坐标(Y轴)，范围 0 到 1000 的千分比。例如 500 表示高度正中间","maximum": 1000, "minimum": 0},506 "duration": {"type": "number", "description": "移动耗时（秒），默认为0.5秒。为了拟真，建议不要设为0", "default": 0.5}507 },508 "required": ["x", "y"]509 }510 }511}512 513mouse_click_tool = {514 "type": "function",515 "function": {516 "name": "mouse_click",517 "description": "点击鼠标。如果传入千分比坐标，则会先移动到该位置再点击；如果不传坐标则在当前位置点击。",518 "parameters": {519 "type": "object",520 "properties": {521 "button": {"type": "string", "enum": ["left", "right", "middle"], "description": "点击的按键，左键/右键/中键"},522 "clicks": {"type": "integer", "description": "点击次数。1为单击，2为双击，当你需要打开链接或文件时，建议使用双击。如果单击某个图标没有任何反应，也要优先考虑双击。", "default": 1},523 "x": {"type": "number", "description": "点击前的目标水平坐标（0 到 1000 的千分比），可选","maximum": 1000, "minimum": 0},524 "y": {"type": "number", "description": "点击前的目标垂直坐标（0 到 1000 的千分比），可选","maximum": 1000, "minimum": 0}525 },526 "required": ["button"]527 }528 }529}530 531mouse_double_click_tool = {532 "type": "function",533 "function": {534 "name": "mouse_double_click",535 "description": "双击鼠标以快速打开链接、文件、应用等。如果传入千分比坐标，则会先移动到该位置再点击；如果不传坐标则在当前位置点击。",536 "parameters": {537 "type": "object",538 "properties": {539 "button": {"type": "string", "enum": ["left", "right", "middle"], "description": "点击的按键，左键/右键/中键"},540 "x": {"type": "number", "description": "点击前的目标水平坐标（0 到 1000 的千分比），可选","maximum": 1000, "minimum": 0},541 "y": {"type": "number", "description": "点击前的目标垂直坐标（0 到 1000 的千分比），可选","maximum": 1000, "minimum": 0}542 },543 "required": ["button"]544 }545 }546}547 548mouse_drag_tool = {549 "type": "function",550 "function": {551 "name": "mouse_drag",552 "description": "按下鼠标按键从起始坐标拖动到终点坐标。常用于拖动窗口、滑块、移动文件或框选一段区域。",553 "parameters": {554 "type": "object",555 "properties": {556 "x1": {"type": "number", "description": "起始点水平坐标 (0-1000)","maximum": 1000, "minimum": 0},557 "y1": {"type": "number", "description": "起始点垂直坐标 (0-1000)","maximum": 1000, "minimum": 0},558 "x2": {"type": "number", "description": "终点水平坐标 (0-1000)","maximum": 1000, "minimum": 0},559 "y2": {"type": "number", "description": "终点垂直坐标 (0-1000)","maximum": 1000, "minimum": 0},560 "duration": {"type": "number", "description": "拖拽过程耗时（秒），默认为 1.0 秒", "default": 1.0},561 "button": {"type": "string", "enum": ["left", "right"], "description": "按住哪个键拖拽，默认左键", "default": "left"}562 },563 "required": ["x1", "y1", "x2", "y2"]564 }565 }566}567 568mouse_hold_tool = {569 "type": "function",570 "function": {571 "name": "mouse_hold",572 "description": "长按鼠标某个按键一段时间。适用于游戏中的蓄力、持续开火或某些 UI 的长按菜单。",573 "parameters": {574 "type": "object",575 "properties": {576 "button": {577 "type": "string", 578 "enum": ["left", "right", "middle"],579 "description": "要按住的鼠标按键。"580 },581 "duration": {582 "type": "number", 583 "description": "按住的时长（秒）。"584 }585 },586 "required": ["button", "duration"]587 }588 }589}590 591 592mouse_scroll_tool = {593 "type": "function",594 "function": {595 "name": "mouse_scroll",596 "description": "滚动鼠标滚轮以浏览网页或文档。正数表示向上滚动，负数表示向下滚动。",597 "parameters": {598 "type": "object",599 "properties": {600 "clicks": {"type": "integer", "description": "滚动单位。大于0为向上滚，小于0为向下滚。如 500 或 -500。一般网页滚动一次可以尝试 300 到 800 的数值。"}601 },602 "required": ["clicks"]603 }604 }605}606 607keyboard_type_tool = {608 "type": "function",609 "function": {610 "name": "copy_to_input_box",611 "description": "在当前焦点输入框中复制你给的一段文本。支持输入中文和英文字符。注意：调用前请确保已经点击了正确的输入框使之获得了焦点！这个输入只是复制粘贴，与键盘控制无关，不是真的按键交互",612 "parameters": {613 "type": "object",614 "properties": {615 "text": {"type": "string", "description": "需要输入的具体文本内容"}616 },617 "required": ["text"]618 }619 }620}621 622keyboard_press_tool = {623 "type": "function",624 "function": {625 "name": "keyboard_press",626 "description": "按下单个按键。适用于需要连续按下同一个键的情况，例如删除多个字符或连续下移。",627 "parameters": {628 "type": "object",629 "properties": {630 "key": {631 "type": "string", 632 "description": "按键名称，例如: 'enter', 'backspace', 'tab', 'down', 'esc'。"633 },634 "presses": {635 "type": "integer", 636 "description": "按下该按键的次数，默认为 1。", 637 "default": 1638 }639 },640 "required": ["key"]641 }642 }643}644 645keyboard_sequence_tool = {646 "type": "function",647 "function": {648 "name": "keyboard_sequence",649 "description": "按顺序按下多个不同的按键。程序会在每个按键之间自动停顿 0.5 秒。适用于流程化的按键操作，例如 '先按 Tab 切换焦点，再按 Enter 确认'。",650 "parameters": {651 "type": "object",652 "properties": {653 "keys": {654 "type": "array",655 "items": {"type": "string"},656 "description": "按键名称的列表。例如 ['tab', 'enter'] 或 ['up', 'up', 'space']。"657 }658 },659 "required": ["keys"]660 }661 }662}663 664keyboard_hotkey_tool = {665 "type": "function",666 "function": {667 "name": "keyboard_hotkey",668 "description": "按下键盘组合快捷键。例如复制是['ctrl', 'c']，切换窗口是['alt', 'tab']。如果是mac系统请使用'command'代替'ctrl'。",669 "parameters": {670 "type": "object",671 "properties": {672 "keys": {673 "type": "array",674 "items": {"type": "string"},675 "description": "快捷键组合数组，必须按照按下的先后顺序排列。例如: ['ctrl', 'shift', 'esc']"676 }677 },678 "required": ["keys"]679 }680 }681}682 683keyboard_hold_tool = {684 "type": "function",685 "function": {686 "name": "keyboard_hold",687 "description": "长按键盘上的一个或多个按键一段时间。这对于控制游戏角色移动或执行需要按住的操作非常有用。",688 "parameters": {689 "type": "object",690 "properties": {691 "keys": {692 "type": "array",693 "items": {"type": "string"},694 "description": "需要按住的按键列表。例如 ['w'] 或 ['w', 'shift']。"695 },696 "duration": {697 "type": "number", 698 "description": "按住的时长（秒）。"699 }700 },701 "required": ["keys", "duration"]702 }703 }704}705 706 707wait_tool = {708 "type": "function",709 "function": {710 "name": "wait",711 "description": "让操作暂停并等待一段时间。在点击了加载页面的链接、启动软件、或者输入内容后，必须调用此工具等待 UI 刷新完成，否则下一步操作可能会因为找不到目标而失败。",712 "parameters": {713 "type": "object",714 "properties": {715 "seconds": {"type": "number", "description": "需要等待的秒数，如 1, 2.5, 5等。如果网速慢或程序加载慢，请适当延长。"}716 },717 "required": ["seconds"]718 }719 }720}721screenshot_tool = {722 "type": "function",723 "function": {724 "name": "screenshot",725 "description": "截取带有千分比辅助网格的当前桌面的图像",726 "parameters": {727 "type": "object",728 "properties": {},729 "required": []730 }731 }732}733 734# 逻辑点击的工具配置声明735logical_click_tool = {736 "type": "function",737 "function": {738 "name": "logical_click",739 "description": "通过当前网页/窗口 UI 树的节点 ID 在后台执行逻辑点击（无障碍点击），不需要物理移动鼠标，支持窗口遮挡和熄屏操作。如果你能拿到有效的节点 ID，请优先使用此工具代替物理鼠标点击。",740 "parameters": {741 "type": "object",742 "properties": {743 "id": {744 "type": "integer", 745 "description": "要点击的 UI 元素的 ID（对应当前 UI 树 JSON 中提供的 id 字段）。"746 }747 },748 "required": ["id"]749 }750 }751}752 753 754logical_type_tool = {755 "type": "function",756 "function": {757 "name": "logical_type",758 "description": "通过当前网页/窗口 UI 树的节点 ID 在后台向输入框直接输入文本（无障碍输入），不需要物理移动鼠标，支持窗口遮挡和熄屏操作。如果你能拿到有效的输入框节点 ID，请优先使用此工具代替 copy_to_input_box 输入文字。",759 "parameters": {760 "type": "object",761 "properties": {762 "id": {763 "type": "integer", 764 "description": "要输入文本的输入框或文本域元素的 ID（对应当前 UI 树 JSON 中提供的 id 字段）。"765 },766 "text": {767 "type": "string",768 "description": "需要输入的具体文本内容。"769 }770 },771 "required": ["id", "text"]772 }773 }774}775 776# 导出所有工具到列表，方便主程序统一挂载777computer_use_tools = [778 wait_tool779 780]781 782desktopVision_use_tools = [783 screenshot_tool784]785 786mouse_use_tools = [787 mouse_move_tool,788 mouse_click_tool,789 mouse_double_click_tool,790 mouse_drag_tool,791 mouse_scroll_tool,792 mouse_hold_tool,793 logical_click_tool,794]795 796keyboard_use_tools = [797 keyboard_type_tool,798 keyboard_press_tool,799 keyboard_sequence_tool,800 keyboard_hotkey_tool,801 keyboard_hold_tool,802 logical_type_tool,803]