HyperCluster/Fara-BrowserUse
5
1import math
2
3from typing import Union, Tuple
4
5from .qwen_helpers.base_tool import BaseTool
6from .qwen_helpers.fncall_prompt import NousFnCallPrompt
7from .qwen_helpers.schema import (
8 ContentItem,
9 Message,
10)
11
12IMAGE_FACTOR = 28
13MIN_PIXELS = 4 * 28 * 28
14MAX_PIXELS = 16384 * 28 * 28
15MAX_RATIO = 200
16
17
18# @register_tool("computer_use")
19class FaraComputerUse(BaseTool):
20 name = "computer_use"
21
22 @property
23 def description(self):
24 return f"""
25Use a mouse and keyboard to interact with a computer, and take screenshots.
26* This is an interface to a desktop GUI. You do not have access to a terminal or applications menu. You must click on desktop icons to start applications.
27* Some applications may take time to start or process actions, so you may need to wait and take successive screenshots to see the results of your actions. E.g. if you click on Firefox and a window doesn't open, try wait and taking another screenshot.
28* The screen's resolution is {self.display_width_px}x{self.display_height_px}.
29* Whenever you intend to move the cursor to click on an element like an icon, you should consult a screenshot to determine the coordinates of the element before moving the cursor.
30* If you tried clicking on a program or link but it failed to load, even after waiting, try adjusting your cursor position so that the tip of the cursor visually falls on the element that you want to click.
31* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges unless asked.
32* When a separate scrollable container prominently overlays the webpage, if you want to scroll within it, you typically need to mouse_move() over it first and then scroll().
33* If a popup window appears that you want to close, if left_click() on the 'X' or close button doesn't work, try key(keys=['Escape']) to close it.
34* On some search bars, when you type(), you may need to press_enter=False and instead separately call left_click() on the search button to submit the search query. This is especially true of search bars that have auto-suggest popups for e.g. locations
35* For calendar widgets, you usually need to left_click() on arrows to move between months and left_click() on dates to select them; type() is not typically used to input dates there.
36""".strip()
37
38 parameters = {
39 "properties": {
40 "action": {
41 "description": """
42The action to perform. The available actions are:
43* `key`: Performs key down presses on the arguments passed in order, then performs key releases in reverse order. Includes "Enter", "Alt", "Shift", "Tab", "Control", "Backspace", "Delete", "Escape", "ArrowUp", "ArrowDown", "ArrowLeft", "ArrowRight", "PageDown", "PageUp", "Shift", etc.
44* `type`: Type a string of text on the keyboard.
45* `mouse_move`: Move the cursor to a specified (x, y) pixel coordinate on the screen.
46* `left_click`: Click the left mouse button.
47* `scroll`: Performs a scroll of the mouse scroll wheel.
48* `visit_url`: Visit a specified URL.
49* `web_search`: Perform a web search with a specified query.
50* `history_back`: Go back to the previous page in the browser history.
51* `pause_and_memorize_fact`: Pause and memorize a fact for future reference.
52* `wait`: Wait specified seconds for the change to happen.
53* `terminate`: Terminate the current task and report its completion status.
54""".strip(),
55 "enum": [
56 "key",
57 "type",
58 "mouse_move",
59 "left_click",
60 "scroll",
61 "visit_url",
62 "web_search",
63 "history_back",
64 "pause_and_memorize_fact",
65 "wait",
66 "terminate",
67 ],
68 "type": "string",
69 },
70 "keys": {
71 "description": "Required only by `action=key`.",
72 "type": "array",
73 },
74 "text": {
75 "description": "Required only by `action=type`.",
76 "type": "string",
77 },
78 "press_enter": {
79 "description": "Whether to press the Enter key after typing. Required only by `action=type`.",
80 "type": "boolean",
81 },
82 "delete_existing_text": {
83 "description": "Whether to delete existing text before typing. Required only by `action=type`.",
84 "type": "boolean",
85 },
86 "coordinate": {
87 "description": "(x, y): The x (pixels from the left edge) and y (pixels from the top edge) coordinates to move the mouse to. Required only by `action=left_click`, `action=mouse_move`, and `action=type`.",
88 "type": "array",
89 },
90 "pixels": {
91 "description": "The amount of scrolling to perform. Positive values scroll up, negative values scroll down. Required only by `action=scroll`.",
92 "type": "number",
93 },
94 "url": {
95 "description": "The URL to visit. Required only by `action=visit_url`.",
96 "type": "string",
97 },
98 "query": {
99 "description": "The query to search for. Required only by `action=web_search`.",
100 "type": "string",
101 },
102 "fact": {
103 "description": "The fact to remember for the future. Required only by `action=pause_and_memorize_fact`.",
104 "type": "string",
105 },
106 "time": {
107 "description": "The seconds to wait. Required only by `action=wait`.",
108 "type": "number",
109 },
110 "status": {
111 "description": "The status of the task. Required only by `action=terminate`.",
112 "type": "string",
113 "enum": ["success", "failure"],
114 },
115 },
116 "required": ["action"],
117 "type": "object",
118 }
119
120 def __init__(self, cfg=None):
121 self.display_width_px = cfg["display_width_px"]
122 self.display_height_px = cfg["display_height_px"]
123 include_input_text_key_args = cfg.pop("include_input_text_key_args", False)
124 if not include_input_text_key_args:
125 self.parameters["properties"].pop("press_enter", None)
126 self.parameters["properties"].pop("delete_existing_text", None)
127 super().__init__(cfg)
128
129 def call(self, params: Union[str, dict], **kwargs):
130 params = self._verify_json_format_args(params)
131 action = params["action"]
132 if action == "key":
133 return self._key(params["text"])
134 elif action == "click":
135 return self._click(coordinate=params["coordinate"])
136 elif action == "long_press":
137 return self._long_press(
138 coordinate=params["coordinate"], time=params["time"]
139 )
140 elif action == "swipe":
141 return self._swipe(
142 coordinate=params["coordinate"], coordinate2=params["coordinate2"]
143 )
144 elif action == "type":
145 return self._type(params["text"])
146 elif action == "system_button":
147 return self._system_button(params["button"])
148 elif action == "open":
149 return self._open(params["text"])
150 elif action == "wait":
151 return self._wait(params["time"])
152 elif action == "terminate":
153 return self._terminate(params["status"])
154 else:
155 raise ValueError(f"Unknown action: {action}")
156
157 def _key(self, text: str):
158 raise NotImplementedError()
159
160 def _click(self, coordinate: Tuple[int, int]):
161 raise NotImplementedError()
162
163 def _long_press(self, coordinate: Tuple[int, int], time: int):
164 raise NotImplementedError()
165
166 def _swipe(self, coordinate: Tuple[int, int], coordinate2: Tuple[int, int]):
167 raise NotImplementedError()
168
169 def _type(self, text: str):
170 raise NotImplementedError()
171
172 def _system_button(self, button: str):
173 raise NotImplementedError()
174
175 def _open(self, text: str):
176 raise NotImplementedError()
177
178 def _wait(self, time: int):
179 raise NotImplementedError()
180
181 def _terminate(self, status: str):
182 raise NotImplementedError()
183
184
185def round_by_factor(number: int, factor: int) -> int:
186 """Returns the closest integer to 'number' that is divisible by 'factor'."""
187 return round(number / factor) * factor
188
189
190def ceil_by_factor(number: int, factor: int) -> int:
191 """Returns the smallest integer greater than or equal to 'number' that is divisible by 'factor'."""
192 return math.ceil(number / factor) * factor
193
194
195def floor_by_factor(number: int, factor: int) -> int:
196 """Returns the largest integer less than or equal to 'number' that is divisible by 'factor'."""
197 return math.floor(number / factor) * factor
198
199
200def smart_resize(
201 height: int,
202 width: int,
203 factor: int = IMAGE_FACTOR,
204 min_pixels: int = MIN_PIXELS,
205 max_pixels: int = MAX_PIXELS,
206) -> tuple[int, int]:
207 """
208 Rescales the image so that the following conditions are met:
209
210 1. Both dimensions (height and width) are divisible by 'factor'.
211
212 2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
213
214 3. The aspect ratio of the image is maintained as closely as possible.
215 """
216 if max(height, width) / min(height, width) > MAX_RATIO:
217 raise ValueError(
218 f"absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(height, width) / min(height, width)}"
219 )
220 h_bar = max(factor, round_by_factor(height, factor))
221 w_bar = max(factor, round_by_factor(width, factor))
222 if h_bar * w_bar > max_pixels:
223 beta = math.sqrt((height * width) / max_pixels)
224 h_bar = floor_by_factor(height / beta, factor)
225 w_bar = floor_by_factor(width / beta, factor)
226 elif h_bar * w_bar < min_pixels:
227 beta = math.sqrt(min_pixels / (height * width))
228 h_bar = ceil_by_factor(height * beta, factor)
229 w_bar = ceil_by_factor(width * beta, factor)
230 return h_bar, w_bar
231
232
233def get_computer_use_system_prompt(
234 image,
235 processor_im_cfg,
236 include_input_text_key_args=False,
237 fn_call_template="default",
238):
239 patch_size = processor_im_cfg["patch_size"]
240 merge_size = processor_im_cfg["merge_size"]
241 min_pixels = processor_im_cfg["min_pixels"]
242 max_pixels = processor_im_cfg["max_pixels"]
243
244 resized_height, resized_width = smart_resize(
245 image.height,
246 image.width,
247 factor=patch_size * merge_size,
248 min_pixels=min_pixels,
249 max_pixels=max_pixels,
250 )
251
252 computer_use = FaraComputerUse(
253 cfg={
254 "display_width_px": resized_width,
255 "display_height_px": resized_height,
256 "include_input_text_key_args": include_input_text_key_args,
257 }
258 )
259
260 conversation = NousFnCallPrompt(
261 template_name=fn_call_template
262 ).preprocess_fncall_messages(
263 messages=[
264 Message(
265 role="system",
266 content=[ContentItem(text="You are a helpful assistant.")],
267 ),
268 ],
269 functions=[computer_use.function],
270 lang=None,
271 )
272
273 return {
274 "conversation": [msg.model_dump() for msg in conversation],
275 "im_size": (resized_width, resized_height),
276 }
277