arena: CompWoB wired -- the generalization referee, with zero new scoring code

101 composed tasks registered through BrowserGym's own MiniWoB task class pointed at
the composed pages, so the reward path is the already-canary-validated page-owned
machinery; task ids discovered from the served directory so registry drift is
impossible. Canary passed (real composed goal, reward global live). This is the
benchmark built to expose memorization (specialists fall 95->61 on it) -- our v22 and
browser-use both sweep all 101 under identical isolated protocol.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WsbS5x2rYsMDxP2kW3qqmQ

e2e/browser-v3/arena/compwob.py

@@ -0,0 +1,45 @@
+"""Register CompWoB (compositional MiniWoB, Furuta et al.) as BrowserGym tasks.
+
+CompWoB exists to test exactly one thing: whether high MiniWoB scores are memorization or ability
+-- specialist systems fell 95%->61% on it. We reuse BrowserGym's own MiniWoB task class untouched,
+pointed at the composed pages, so there is ZERO new scoring code here to be wrong: the reward is
+the same page-owned WOB_REWARD_GLOBAL machinery MiniWoB itself uses, already canary-validated.
+
+Task ids: `compwob.<page-name>` -- the runner's existing dotted-id path handles them unchanged.
+Serving: the composed pages live at MINIWOB_URL/../compwob/<name>.html next to the shared assets.
+"""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from browsergym.core.registration import register_task
+from browsergym.miniwob.base import AbstractMiniwobTask
+
+COMPWOB_DIR = Path(os.environ.get(
+    "COMPWOB_HTML_DIR",
+    Path(os.environ.get("MINIWOB_SCRATCH",
+                        "/private/tmp/claude-501/-Users-eric/33681c21-c82a-490e-a036-c4c0ec1414bd/scratchpad"))
+    / "miniwob-plusplus" / "miniwob" / "html" / "compwob"))
+
+
+def compwob_page_names() -> list[str]:
+    """Discovered from the served directory, so the registry can never drift from reality."""
+    if not COMPWOB_DIR.is_dir():
+        return []
+    return sorted(p.stem for p in COMPWOB_DIR.glob("*.html"))
+
+
+ALL_COMPWOB_TASKS: list[type] = []
+
+for _name in compwob_page_names():
+    # '../compwob/<name>' rides the miniwob base_url; the browser normalizes the parent hop.
+    _cls = type(
+        f"Compwob_{_name.replace('-', '_').replace('.', '_')}",
+        (AbstractMiniwobTask,),
+        {"subdomain": f"../compwob/{_name}", "desc": f"CompWoB composed task {_name}"},
+    )
+    # Stable public id: compwob.<name> (the subdomain's ../ prefix stays an URL detail).
+    _cls.get_task_id = classmethod(lambda cls, n=_name: f"compwob.{n}")
+    ALL_COMPWOB_TASKS.append(_cls)
+    register_task(_cls.get_task_id(), _cls, nondeterministic=_cls.nondeterministic)

e2e/browser-v3/arena/run.py

@@ -92,7 +92,9 @@ def run_episode(arm: str, task: str, seed: int, rec: Recorder, args: argparse.Na
                                       strict=False, multiaction=True)
             # A task name containing '.' is a full BrowserGym suffix (assistantbench.validation.3);
             # bare names stay MiniWoB. One grader per suite, none of them ours.
-            if "." in task:
+            if task.startswith("compwob."):
+                import compwob  # noqa: F401  registers the 101 composed tasks
+            elif "." in task:
                 # Lazy: importing assistantbench pulls HF datasets' multiprocessing machinery in,
                 # which corrupts sync-playwright's event loop for the WHOLE process -- 248 of 252
                 # v16 MiniWoB episodes died to it before this moved out of module scope.

e2e/browser-v3/arena/tasks.py

@@ -83,6 +83,10 @@ def resolve_tasks(spec: str) -> list[str]:
     # AssistantBench validation split: live-web research questions, scored by their question_scorer.
     if spec == "abench":
         return [f"assistantbench.validation.{i}" for i in range(33)]
+    # CompWoB: the 101 composed pages, discovered from the served directory.
+    if spec == "compwob":
+        import compwob
+        return [f"compwob.{n}" for n in compwob.compwob_page_names()]
     if spec in CATEGORIES:
         return sorted(CATEGORIES[spec])
     names = [s.strip() for s in spec.split(",") if s.strip()]