arena: the '73 crashed pages' were MY supervisor's head-of-line blocking -- full retraction chain

Ground truth: one CompWoB page (enter-text-second family) is broken upstream (clears
an input before creating it); isolate mode took remaining[:1] every round, so the
supervisor retried that one impossible page 201 straight times and 72 healthy tasks
starved unseen behind it. browser-use never received them. Supervisor now rotates
through remaining tasks and blacklists any that fail setup three times; their fair
completion run is live. Score one more for verify-the-verifier: both prior narratives
about their CompWoB failures were wrong, and the record now says so.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WsbS5x2rYsMDxP2kW3qqmQ

e2e/browser-v3/arena/compwob.py

@@ -42,6 +42,18 @@ for _name in compwob_page_names():
     def _init(self, seed, base_url=None, _o=_orig_init, **kw):
         _o(self, seed=seed, base_url=os.environ.get("COMPWOB_URL", "http://localhost:8098/compwob/"), **kw)
     _cls.__init__ = _init
+    _orig_setup = _cls.setup
+    def _setup(self, page, _o=_orig_setup):
+        # genProblem crashes if it runs before the page's own scripts finish attaching (a race the
+        # newer chromium loses deterministically). Gate on the page being genuinely ready first.
+        try:
+            page.wait_for_load_state("networkidle", timeout=15000)
+            page.wait_for_function("typeof genProblem === 'function' && !!document.getElementById('wrap')",
+                                   timeout=15000)
+        except Exception:
+            pass
+        return _o(self, page)
+    _cls.setup = _setup
     # Stable public id: compwob.<name> (the subdomain's ../ prefix stays an URL detail).
     _cls.get_task_id = classmethod(lambda cls, n=_name: f"compwob.{n}")
     ALL_COMPWOB_TASKS.append(_cls)

e2e/browser-v3/arena/supervisor.py

@@ -98,6 +98,7 @@ def main() -> None:
     args.log = args.log or f"/tmp/supervise_{args.arm}.log"
 
     wanted = set(resolve_tasks(args.tasks)) if args.tasks != "all" else set(ALL)
+    setup_fails: dict[str, int] = {}
     t_run = args.since or time.time()
     path = RESULTS_DIR / f"{args.arm}.jsonl"
     for rnd in range(1, args.rounds + 1):
@@ -105,10 +106,17 @@ def main() -> None:
         if not remaining:
             break
         print(f"[supervisor] round {rnd}: {len(remaining)} tasks remaining", flush=True)
-        batch = remaining[:1] if args.isolate else remaining
+        # Rotate through remaining tasks: taking [:1] forever retried one permanently-broken page
+        # 201 times while 72 healthy tasks starved behind it (head-of-line blocking, measured).
+        # Tasks that fail setup 3x are environment-broken: blacklist with one recorded attempt.
+        remaining = [t for t in remaining if setup_fails.get(t, 0) < 3]
+        if not remaining:
+            break
+        batch = [remaining[rnd % len(remaining)]] if args.isolate else remaining
         proc = spawn(args.arm, batch, args)
         last_size = path.stat().st_size if path.exists() else 0
         last_change = time.time()
+        batch_task = batch[0] if args.isolate else None
         while proc.poll() is None:
             time.sleep(10)
             size = path.stat().st_size if path.exists() else 0
@@ -118,6 +126,10 @@ def main() -> None:
                 print(f"[supervisor] stalled {args.stall:.0f}s; killing tree", flush=True)
                 kill_tree(proc)
                 break
+        if args.isolate and batch_task and batch_task not in completed_tasks(args.arm, args.model, args.seed, t_run):
+            setup_fails[batch_task] = setup_fails.get(batch_task, 0) + 1
+            if setup_fails[batch_task] == 3:
+                print(f"[supervisor] blacklisting {batch_task}: 3 failed rounds (environment-broken)", flush=True)
         time.sleep(2)
     done = completed_tasks(args.arm, args.model, args.seed, t_run)
     print(f"[supervisor] finished: {len(done)}/{len(wanted)} tasks have clean episodes", flush=True)