From 228d65c9e9de9fc04343a8be94d8a82a31bb3bdf Mon Sep 17 00:00:00 2001 From: ciregenz Date: Wed, 12 Aug 2026 19:08:11 -0700 Subject: [PATCH] arena: CompWoB wired -- the generalization referee, with zero new scoring code 101 composed tasks registered through BrowserGym's own MiniWoB task class pointed at the composed pages, so the reward path is the already-canary-validated page-owned machinery; task ids discovered from the served directory so registry drift is impossible. Canary passed (real composed goal, reward global live). This is the benchmark built to expose memorization (specialists fall 95->61 on it) -- our v22 and browser-use both sweep all 101 under identical isolated protocol. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WsbS5x2rYsMDxP2kW3qqmQ --- e2e/browser-v3/arena/compwob.py | 45 +++++++++++++++++++++++++++++++++ e2e/browser-v3/arena/run.py | 4 ++- e2e/browser-v3/arena/tasks.py | 4 +++ 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 e2e/browser-v3/arena/compwob.py diff --git a/e2e/browser-v3/arena/compwob.py b/e2e/browser-v3/arena/compwob.py new file mode 100644 index 00000000..951a31a1 --- /dev/null +++ b/e2e/browser-v3/arena/compwob.py @@ -0,0 +1,45 @@ +"""Register CompWoB (compositional MiniWoB, Furuta et al.) as BrowserGym tasks. + +CompWoB exists to test exactly one thing: whether high MiniWoB scores are memorization or ability +-- specialist systems fell 95%->61% on it. We reuse BrowserGym's own MiniWoB task class untouched, +pointed at the composed pages, so there is ZERO new scoring code here to be wrong: the reward is +the same page-owned WOB_REWARD_GLOBAL machinery MiniWoB itself uses, already canary-validated. + +Task ids: `compwob.` -- the runner's existing dotted-id path handles them unchanged. +Serving: the composed pages live at MINIWOB_URL/../compwob/.html next to the shared assets. +""" +from __future__ import annotations + +import os +from pathlib import Path + +from browsergym.core.registration import register_task +from browsergym.miniwob.base import AbstractMiniwobTask + +COMPWOB_DIR = Path(os.environ.get( + "COMPWOB_HTML_DIR", + Path(os.environ.get("MINIWOB_SCRATCH", + "/private/tmp/claude-501/-Users-eric/33681c21-c82a-490e-a036-c4c0ec1414bd/scratchpad")) + / "miniwob-plusplus" / "miniwob" / "html" / "compwob")) + + +def compwob_page_names() -> list[str]: + """Discovered from the served directory, so the registry can never drift from reality.""" + if not COMPWOB_DIR.is_dir(): + return [] + return sorted(p.stem for p in COMPWOB_DIR.glob("*.html")) + + +ALL_COMPWOB_TASKS: list[type] = [] + +for _name in compwob_page_names(): + # '../compwob/' rides the miniwob base_url; the browser normalizes the parent hop. + _cls = type( + f"Compwob_{_name.replace('-', '_').replace('.', '_')}", + (AbstractMiniwobTask,), + {"subdomain": f"../compwob/{_name}", "desc": f"CompWoB composed task {_name}"}, + ) + # Stable public id: compwob. (the subdomain's ../ prefix stays an URL detail). + _cls.get_task_id = classmethod(lambda cls, n=_name: f"compwob.{n}") + ALL_COMPWOB_TASKS.append(_cls) + register_task(_cls.get_task_id(), _cls, nondeterministic=_cls.nondeterministic) diff --git a/e2e/browser-v3/arena/run.py b/e2e/browser-v3/arena/run.py index df113efa..68a1699d 100644 --- a/e2e/browser-v3/arena/run.py +++ b/e2e/browser-v3/arena/run.py @@ -92,7 +92,9 @@ def run_episode(arm: str, task: str, seed: int, rec: Recorder, args: argparse.Na strict=False, multiaction=True) # A task name containing '.' is a full BrowserGym suffix (assistantbench.validation.3); # bare names stay MiniWoB. One grader per suite, none of them ours. - if "." in task: + if task.startswith("compwob."): + import compwob # noqa: F401 registers the 101 composed tasks + elif "." in task: # Lazy: importing assistantbench pulls HF datasets' multiprocessing machinery in, # which corrupts sync-playwright's event loop for the WHOLE process -- 248 of 252 # v16 MiniWoB episodes died to it before this moved out of module scope. diff --git a/e2e/browser-v3/arena/tasks.py b/e2e/browser-v3/arena/tasks.py index 80531db6..e3309d9c 100644 --- a/e2e/browser-v3/arena/tasks.py +++ b/e2e/browser-v3/arena/tasks.py @@ -83,6 +83,10 @@ def resolve_tasks(spec: str) -> list[str]: # AssistantBench validation split: live-web research questions, scored by their question_scorer. if spec == "abench": return [f"assistantbench.validation.{i}" for i in range(33)] + # CompWoB: the 101 composed pages, discovered from the served directory. + if spec == "compwob": + import compwob + return [f"compwob.{n}" for n in compwob.compwob_page_names()] if spec in CATEGORIES: return sorted(CATEGORIES[spec]) names = [s.strip() for s in spec.split(",") if s.strip()]