arena: assistantbench import quarantined -- HF datasets machinery was corrupting playwright process-wide

The module-scope import of browsergym.assistantbench pulls in HuggingFace datasets'
multiprocessing at interpreter start; 248 of 252 v16 MiniWoB episodes died to the
resulting 'no running event loop' cascade before the import moved inside env setup,
loaded only when a dotted task name actually needs it. v16 relaunched clean.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WsbS5x2rYsMDxP2kW3qqmQ
This commit is contained in:
ciregenz
2026-08-11 02:12:03 -07:00
co-authored by Claude Fable 5
parent 3ea7b93428
commit c41c78dd0d
2 changed files with 7 additions and 2 deletions
+2 -1
View File
@@ -20,7 +20,6 @@ from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import browsergym.assistantbench # noqa: F401 registers the live-web research envs
import browsergym.miniwob # noqa: F401 registers the 125 envs
import gymnasium as gym
@@ -96,6 +95,8 @@ def reap_leftover_browsers() -> None:
def make_env(task: str, seed: int, max_steps: int):
patch_launch_with_cdp_port()
if "." in task:
import browsergym.assistantbench # noqa: F401 lazy: HF datasets machinery breaks playwright
env_id = f"browsergym/{task}" if "." in task else f"browsergym/miniwob.{task}"
env = gym.make(env_id, headless=True, max_episode_steps=max_steps)
obs, _ = env.reset(seed=seed)
+5 -1
View File
@@ -19,7 +19,6 @@ from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parent))
import browsergym.assistantbench # noqa: F401 registers the 215 live-web research envs
import browsergym.miniwob # noqa: F401 importing is what registers the 125 envs
import gymnasium as gym
@@ -93,6 +92,11 @@ def run_episode(arm: str, task: str, seed: int, rec: Recorder, args: argparse.Na
strict=False, multiaction=True)
# A task name containing '.' is a full BrowserGym suffix (assistantbench.validation.3);
# bare names stay MiniWoB. One grader per suite, none of them ours.
if "." in task:
# Lazy: importing assistantbench pulls HF datasets' multiprocessing machinery in,
# which corrupts sync-playwright's event loop for the WHOLE process -- 248 of 252
# v16 MiniWoB episodes died to it before this moved out of module scope.
import browsergym.assistantbench # noqa: F401
env_id = f"browsergym/{task}" if "." in task else f"browsergym/miniwob.{task}"
holder.append(gym.make(env_id, headless=not args.headed,
max_episode_steps=args.max_steps, wait_for_user_message=False,