openenv/browsergym_env
0
1"""BrowserGym Environment implementation for OpenEnv.2 3This module wraps the BrowserGym framework to provide a compatible interface4with OpenEnv's Environment ABC. BrowserGym includes multiple benchmarks:5- MiniWoB++: Training environment with 100+ simple web tasks6- WebArena: Realistic evaluation with 812 complex tasks7- VisualWebArena: Visual web navigation tasks8- WorkArena: Enterprise task automation9"""10 11import importlib12import logging13from typing import Any, Dict, Optional14from uuid import uuid415 16import gymnasium as gym17from browsergym_env.models import (18 BrowserGymAction,19 BrowserGymObservation,20 BrowserGymState,21)22from openenv.core.env_server.interfaces import Environment23 24logger = logging.getLogger(__name__)25 26 27def _get_axtree_txt(obs: Dict[str, Any]) -> str:28 """Extract accessibility tree text from BrowserGym observation.29 30 BrowserGym returns raw `axtree_object` which needs to be converted to text31 using the `flatten_axtree_to_str` utility function.32 """33 # If already processed as text, return directly34 if "axtree_txt" in obs and obs["axtree_txt"]:35 return obs["axtree_txt"]36 37 # Try to convert from raw axtree_object38 if "axtree_object" in obs and obs["axtree_object"]:39 try:40 from browsergym.utils.obs import flatten_axtree_to_str41 42 return flatten_axtree_to_str(obs["axtree_object"])43 except ImportError:44 logger.warning(45 "browsergym.utils.obs not available, cannot convert axtree_object to text"46 )47 except Exception as e:48 logger.warning(f"Failed to convert axtree_object to text: {e}")49 50 return ""51 52 53def _get_pruned_html(obs: Dict[str, Any]) -> str:54 """Extract pruned HTML from BrowserGym observation.55 56 BrowserGym returns raw `dom_object` which needs to be converted to text57 and then pruned using the `flatten_dom_to_str` and `prune_html` utilities.58 """59 # If already processed as pruned_html, return directly60 if "pruned_html" in obs and obs["pruned_html"]:61 return obs["pruned_html"]62 63 # Try to convert from raw dom_object64 if "dom_object" in obs and obs["dom_object"]:65 try:66 from browsergym.utils.obs import flatten_dom_to_str, prune_html67 68 dom_str = flatten_dom_to_str(obs["dom_object"])69 return prune_html(dom_str)70 except ImportError:71 logger.warning(72 "browsergym.utils.obs not available, cannot convert dom_object to pruned_html"73 )74 except Exception as e:75 logger.warning(f"Failed to convert dom_object to pruned_html: {e}")76 77 return ""78 79 80_MINIWOB_LOAD_HELP = (81 "MiniWoB tasks require the MiniWoB HTML bundle to be served over HTTP. "82 "The official BrowserGym Docker image handles this automatically by "83 "serving the bundle on port 8888. For custom or non-Docker deployments, "84 "clone the MiniWoB++ repository, start a static server inside "85 "`miniwob-plusplus/miniwob/html` (e.g. `python -m http.server 8888`), and "86 "set the MINIWOB_URL environment variable to the served base URL such as "87 "`http://localhost:8888/miniwob/`."88)89 90 91class BrowserGymEnvironment(Environment):92 """BrowserGym environment wrapper for OpenEnv.93 94 This environment wraps BrowserGym's Gymnasium-compatible environments to95 provide unified access to multiple web navigation benchmarks.96 """97 98 SUPPORTS_CONCURRENT_SESSIONS = True99 REQUIRES_SINGLE_THREAD_EXECUTOR = True100 101 def __init__(102 self,103 benchmark: str = "miniwob",104 task_name: Optional[str] = None,105 headless: bool = True,106 viewport_width: int = 1280,107 viewport_height: int = 720,108 timeout: float = 10000.0,109 **gym_kwargs: Any,110 ):111 """Initialize the BrowserGym environment.112 113 Args:114 benchmark: Benchmark to use ('miniwob', 'webarena', 'visualwebarena', etc.)115 task_name: Specific task within the benchmark (e.g., 'click-test', 'click-button')116 If None, will use first available task117 headless: Whether to run browser in headless mode118 viewport_width: Browser viewport width119 viewport_height: Browser viewport height120 timeout: Action timeout in milliseconds121 **gym_kwargs: Additional arguments passed to gym.make()122 """123 super().__init__()124 125 self.benchmark = benchmark126 self.task_name = task_name127 self.headless = headless128 self.viewport_width = viewport_width129 self.viewport_height = viewport_height130 self.timeout = timeout131 self.gym_kwargs = dict(gym_kwargs)132 133 # Build environment ID134 if task_name:135 self.env_id = f"browsergym/{benchmark}.{task_name}"136 else:137 self.env_id = f"browsergym/{benchmark}"138 139 # force import the benchmark module140 benchmark_modules = {141 "miniwob": "browsergym.miniwob",142 "webarena": "browsergym.webarena",143 "visualwebarena": "browsergym.visualwebarena",144 "workarena": "browsergym.workarena",145 }146 module_path = benchmark_modules.get(benchmark)147 try:148 if module_path:149 importlib.import_module(module_path)150 else:151 importlib.import_module("browsergym")152 except ModuleNotFoundError as import_error:153 message = (154 "Failed to import BrowserGym benchmark "155 f"'{benchmark}': {import_error}\n"156 "Install the matching browsergym package "157 f"(e.g., browsergym-{benchmark})."158 )159 raise ValueError(message) from import_error160 161 # Create the BrowserGym environment162 try:163 self.gym_env = gym.make(164 self.env_id,165 headless=headless,166 viewport={"width": viewport_width, "height": viewport_height},167 timeout=timeout,168 **self.gym_kwargs,169 )170 except Exception as e: # noqa: BLE001 - gym.make171 message = (172 "Failed to create BrowserGym environment "173 f"'{self.env_id}': {e}\n"174 "Make sure the benchmark package is installed "175 f"(e.g., pip install browsergym-{benchmark})."176 )177 raise ValueError(message) from e178 179 # State tracking180 self._state = BrowserGymState(181 episode_id=str(uuid4()),182 step_count=0,183 benchmark=benchmark,184 task_name=task_name or "",185 )186 187 self._last_obs: Optional[Dict[str, Any]] = None188 self._last_info: Optional[Dict[str, Any]] = None189 190 def reset(191 self,192 seed: Optional[int] = None,193 task_name: Optional[str] = None,194 ) -> BrowserGymObservation:195 """Reset the environment with a specific task.196 197 Args:198 seed: Random seed for reproducibility199 task_name: Override task name for this episode200 201 Returns:202 Initial observation for the task203 """204 # Generate new episode ID205 self._state = BrowserGymState(206 episode_id=str(uuid4()),207 step_count=0,208 benchmark=self.benchmark,209 task_name=task_name or self.task_name or "",210 )211 212 # Reset options213 reset_options = {}214 if seed is not None:215 reset_options["seed"] = seed216 217 # Reset the gym environment218 try:219 obs, info = self.gym_env.reset(**reset_options)220 except AttributeError as err:221 if "context" in str(err) and hasattr(self.gym_env, "close"):222 # BrowserGym can leave partially initialized state after a223 # failed reset. Close the hanging resources and try once more.224 self.gym_env.close()225 obs, info = self.gym_env.reset(**reset_options)226 else:227 raise228 except Exception as err: # noqa: BLE001 - browsergym229 message = str(err)230 if self.benchmark == "miniwob" and "core is not defined" in message:231 raise ValueError(_MINIWOB_LOAD_HELP) from err232 raise233 234 self._last_obs = obs235 self._last_info = info236 237 # Extract observation details238 return self._create_observation(obs, info, done=False, reward=0.0)239 240 def step(self, action: BrowserGymAction) -> BrowserGymObservation:241 """Execute an action in the environment.242 243 Args:244 action: The action to execute245 246 Returns:247 Observation after executing the action248 """249 self._state.step_count += 1250 251 # Execute action in gym environment252 try:253 obs, reward, terminated, truncated, info = self.gym_env.step(254 action.action_str255 )256 257 self._last_obs = obs258 self._last_info = info259 260 # Update state261 done = terminated or truncated262 self._state.cum_reward += float(reward)263 264 # Extract goal from info if available265 if "goal" in info:266 self._state.goal = str(info["goal"])267 268 return self._create_observation(obs, info, done=done, reward=float(reward))269 270 except Exception as e:271 # Handle action execution errors272 error_msg = str(e)273 return BrowserGymObservation(274 text=self._last_obs.get("text", "") if self._last_obs else "",275 url=self._last_obs.get("url", "") if self._last_obs else "",276 goal=self._state.goal,277 error=error_msg,278 last_action_error=True,279 done=False,280 reward=0.0,281 )282 283 def _create_observation(284 self,285 obs: Dict[str, Any],286 info: Dict[str, Any],287 done: bool,288 reward: float,289 ) -> BrowserGymObservation:290 """Convert BrowserGym observation to OpenEnv format.291 292 Args:293 obs: BrowserGym observation dict294 info: BrowserGym info dict295 done: Whether episode is done296 reward: Reward for the step297 298 Returns:299 BrowserGymObservation300 """301 # Generate text representations from raw BrowserGym objects302 # BrowserGym returns axtree_object and dom_object which need conversion303 axtree_txt = _get_axtree_txt(obs) if isinstance(obs, dict) else ""304 pruned_html = _get_pruned_html(obs) if isinstance(obs, dict) else ""305 306 # Extract text observation - prefer axtree_txt, fallback to pruned_html307 text = axtree_txt or pruned_html308 if not text and isinstance(obs, str):309 text = obs310 311 # Extract URL from obs (BrowserGym stores it there)312 url = ""313 if isinstance(obs, dict):314 url = obs.get("url", "")315 316 # Extract goal/instruction from goal_object or legacy goal field317 goal = ""318 if isinstance(obs, dict):319 # New format: goal_object is a list of messages320 goal_object = obs.get("goal_object", [])321 if goal_object:322 # Extract text content from goal messages323 goal_texts = []324 for msg in goal_object:325 if isinstance(msg, dict):326 content = msg.get("content", "")327 if isinstance(content, str):328 goal_texts.append(content)329 elif isinstance(content, list):330 for item in content:331 if (332 isinstance(item, dict)333 and item.get("type") == "text"334 ):335 goal_texts.append(item.get("text", ""))336 goal = " ".join(goal_texts)337 # Fallback to legacy goal field338 if not goal:339 goal = obs.get("goal", "")340 341 # Update state342 self._state.current_url = url343 self._state.goal = goal344 345 # Extract additional observation modalities346 screenshot = obs.get("screenshot") if isinstance(obs, dict) else None347 348 # Extract last_action_error from obs (BrowserGym includes this)349 last_action_error = False350 if isinstance(obs, dict):351 last_action_error = bool(obs.get("last_action_error"))352 353 # Store full BrowserGym observation and info in metadata354 # This preserves timestamps, additional fields, and any future extensions355 # Note: We exclude large objects (dom_object, axtree_object) to reduce payload size356 browsergym_metadata = {}357 if isinstance(obs, dict):358 # Include useful fields but exclude large raw objects359 browsergym_metadata["browsergym_obs"] = {360 k: v361 for k, v in obs.items()362 if k not in ("dom_object", "axtree_object", "screenshot")363 }364 browsergym_metadata["browsergym_info"] = info365 366 return BrowserGymObservation(367 text=text,368 url=url,369 screenshot=screenshot,370 goal=goal,371 axtree_txt=axtree_txt,372 pruned_html=pruned_html,373 error="",374 last_action_error=last_action_error,375 done=done,376 reward=reward,377 metadata=browsergym_metadata,378 )379 380 @property381 def state(self) -> BrowserGymState:382 """Get the current environment state."""383 return self._state384 385 def close(self) -> None:386 """Clean up environment resources."""387 if hasattr(self, "gym_env"):388 try:389 self.gym_env.close()390 except Exception as exc: # noqa: BLE001 - browsergym/playwright cleanup391 logger.warning("BrowserGym cleanup failed: %s", exc)392