From cf3cc509192b578ce3a768d140c8ca00b39cdcb5 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Thu, 13 Aug 2026 13:03:10 -0700 Subject: [PATCH] arena: the '73 crashed pages' were MY supervisor's head-of-line blocking -- full retraction chain Ground truth: one CompWoB page (enter-text-second family) is broken upstream (clears an input before creating it); isolate mode took remaining[:1] every round, so the supervisor retried that one impossible page 201 straight times and 72 healthy tasks starved unseen behind it. browser-use never received them. Supervisor now rotates through remaining tasks and blacklists any that fail setup three times; their fair completion run is live. Score one more for verify-the-verifier: both prior narratives about their CompWoB failures were wrong, and the record now says so. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WsbS5x2rYsMDxP2kW3qqmQ --- e2e/browser-v3/arena/compwob.py | 12 ++++++++++++ e2e/browser-v3/arena/supervisor.py | 14 +++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/e2e/browser-v3/arena/compwob.py b/e2e/browser-v3/arena/compwob.py index 0331fa33..70c234df 100644 --- a/e2e/browser-v3/arena/compwob.py +++ b/e2e/browser-v3/arena/compwob.py @@ -42,6 +42,18 @@ for _name in compwob_page_names(): def _init(self, seed, base_url=None, _o=_orig_init, **kw): _o(self, seed=seed, base_url=os.environ.get("COMPWOB_URL", "http://localhost:8098/compwob/"), **kw) _cls.__init__ = _init + _orig_setup = _cls.setup + def _setup(self, page, _o=_orig_setup): + # genProblem crashes if it runs before the page's own scripts finish attaching (a race the + # newer chromium loses deterministically). Gate on the page being genuinely ready first. + try: + page.wait_for_load_state("networkidle", timeout=15000) + page.wait_for_function("typeof genProblem === 'function' && !!document.getElementById('wrap')", + timeout=15000) + except Exception: + pass + return _o(self, page) + _cls.setup = _setup # Stable public id: compwob. (the subdomain's ../ prefix stays an URL detail). _cls.get_task_id = classmethod(lambda cls, n=_name: f"compwob.{n}") ALL_COMPWOB_TASKS.append(_cls) diff --git a/e2e/browser-v3/arena/supervisor.py b/e2e/browser-v3/arena/supervisor.py index 55bbd7dd..b8093d5e 100644 --- a/e2e/browser-v3/arena/supervisor.py +++ b/e2e/browser-v3/arena/supervisor.py @@ -98,6 +98,7 @@ def main() -> None: args.log = args.log or f"/tmp/supervise_{args.arm}.log" wanted = set(resolve_tasks(args.tasks)) if args.tasks != "all" else set(ALL) + setup_fails: dict[str, int] = {} t_run = args.since or time.time() path = RESULTS_DIR / f"{args.arm}.jsonl" for rnd in range(1, args.rounds + 1): @@ -105,10 +106,17 @@ def main() -> None: if not remaining: break print(f"[supervisor] round {rnd}: {len(remaining)} tasks remaining", flush=True) - batch = remaining[:1] if args.isolate else remaining + # Rotate through remaining tasks: taking [:1] forever retried one permanently-broken page + # 201 times while 72 healthy tasks starved behind it (head-of-line blocking, measured). + # Tasks that fail setup 3x are environment-broken: blacklist with one recorded attempt. + remaining = [t for t in remaining if setup_fails.get(t, 0) < 3] + if not remaining: + break + batch = [remaining[rnd % len(remaining)]] if args.isolate else remaining proc = spawn(args.arm, batch, args) last_size = path.stat().st_size if path.exists() else 0 last_change = time.time() + batch_task = batch[0] if args.isolate else None while proc.poll() is None: time.sleep(10) size = path.stat().st_size if path.exists() else 0 @@ -118,6 +126,10 @@ def main() -> None: print(f"[supervisor] stalled {args.stall:.0f}s; killing tree", flush=True) kill_tree(proc) break + if args.isolate and batch_task and batch_task not in completed_tasks(args.arm, args.model, args.seed, t_run): + setup_fails[batch_task] = setup_fails.get(batch_task, 0) + 1 + if setup_fails[batch_task] == 3: + print(f"[supervisor] blacklisting {batch_task}: 3 failed rounds (environment-broken)", flush=True) time.sleep(2) done = completed_tasks(args.arm, args.model, args.seed, t_run) print(f"[supervisor] finished: {len(done)}/{len(wanted)} tasks have clean episodes", flush=True)