diff --git a/e2e/browser-v3/arena/compwob.py b/e2e/browser-v3/arena/compwob.py new file mode 100644 index 00000000..951a31a1 --- /dev/null +++ b/e2e/browser-v3/arena/compwob.py @@ -0,0 +1,45 @@ +"""Register CompWoB (compositional MiniWoB, Furuta et al.) as BrowserGym tasks. + +CompWoB exists to test exactly one thing: whether high MiniWoB scores are memorization or ability +-- specialist systems fell 95%->61% on it. We reuse BrowserGym's own MiniWoB task class untouched, +pointed at the composed pages, so there is ZERO new scoring code here to be wrong: the reward is +the same page-owned WOB_REWARD_GLOBAL machinery MiniWoB itself uses, already canary-validated. + +Task ids: `compwob.` -- the runner's existing dotted-id path handles them unchanged. +Serving: the composed pages live at MINIWOB_URL/../compwob/.html next to the shared assets. +""" +from __future__ import annotations + +import os +from pathlib import Path + +from browsergym.core.registration import register_task +from browsergym.miniwob.base import AbstractMiniwobTask + +COMPWOB_DIR = Path(os.environ.get( + "COMPWOB_HTML_DIR", + Path(os.environ.get("MINIWOB_SCRATCH", + "/private/tmp/claude-501/-Users-eric/33681c21-c82a-490e-a036-c4c0ec1414bd/scratchpad")) + / "miniwob-plusplus" / "miniwob" / "html" / "compwob")) + + +def compwob_page_names() -> list[str]: + """Discovered from the served directory, so the registry can never drift from reality.""" + if not COMPWOB_DIR.is_dir(): + return [] + return sorted(p.stem for p in COMPWOB_DIR.glob("*.html")) + + +ALL_COMPWOB_TASKS: list[type] = [] + +for _name in compwob_page_names(): + # '../compwob/' rides the miniwob base_url; the browser normalizes the parent hop. + _cls = type( + f"Compwob_{_name.replace('-', '_').replace('.', '_')}", + (AbstractMiniwobTask,), + {"subdomain": f"../compwob/{_name}", "desc": f"CompWoB composed task {_name}"}, + ) + # Stable public id: compwob. (the subdomain's ../ prefix stays an URL detail). + _cls.get_task_id = classmethod(lambda cls, n=_name: f"compwob.{n}") + ALL_COMPWOB_TASKS.append(_cls) + register_task(_cls.get_task_id(), _cls, nondeterministic=_cls.nondeterministic) diff --git a/e2e/browser-v3/arena/run.py b/e2e/browser-v3/arena/run.py index df113efa..68a1699d 100644 --- a/e2e/browser-v3/arena/run.py +++ b/e2e/browser-v3/arena/run.py @@ -92,7 +92,9 @@ def run_episode(arm: str, task: str, seed: int, rec: Recorder, args: argparse.Na strict=False, multiaction=True) # A task name containing '.' is a full BrowserGym suffix (assistantbench.validation.3); # bare names stay MiniWoB. One grader per suite, none of them ours. - if "." in task: + if task.startswith("compwob."): + import compwob # noqa: F401 registers the 101 composed tasks + elif "." in task: # Lazy: importing assistantbench pulls HF datasets' multiprocessing machinery in, # which corrupts sync-playwright's event loop for the WHOLE process -- 248 of 252 # v16 MiniWoB episodes died to it before this moved out of module scope. diff --git a/e2e/browser-v3/arena/tasks.py b/e2e/browser-v3/arena/tasks.py index 80531db6..e3309d9c 100644 --- a/e2e/browser-v3/arena/tasks.py +++ b/e2e/browser-v3/arena/tasks.py @@ -83,6 +83,10 @@ def resolve_tasks(spec: str) -> list[str]: # AssistantBench validation split: live-web research questions, scored by their question_scorer. if spec == "abench": return [f"assistantbench.validation.{i}" for i in range(33)] + # CompWoB: the 101 composed pages, discovered from the served directory. + if spec == "compwob": + import compwob + return [f"compwob.{n}" for n in compwob.compwob_page_names()] if spec in CATEGORIES: return sorted(CATEGORIES[spec]) names = [s.strip() for s in spec.split(",") if s.strip()]