diff --git a/.gitignore b/.gitignore index dd6b98c0..76c32a88 100644 --- a/.gitignore +++ b/.gitignore @@ -81,3 +81,4 @@ Thumbs.db ehthumbs.db desktop.ini frontend/tsconfig.tsbuildinfo +e2e/browser-v3/arena/data/ diff --git a/e2e/browser-v3/arena/README.md b/e2e/browser-v3/arena/README.md new file mode 100644 index 00000000..b8e63221 --- /dev/null +++ b/e2e/browser-v3/arena/README.md @@ -0,0 +1,41 @@ +# MiniWoB arena + +Every browser stack we care about, scored on the same 125 MiniWoB tasks by MiniWoB itself. +Nothing in this directory decides success; the reward comes from the task's own JS +(`WOB_REWARD_GLOBAL`), read through BrowserGym. That is the property none of our other suites +have, and it is why cross-stack claims ("ours is better/worse than X") should cite THIS data. + +## Arms + +| arm | what it is | entrypoint | +|---|---|---| +| `flat` | no-LLM floor: flat axtree, first label match | `run.py --arm flat` | +| `openswarm` | no-LLM port of our shipped perception + action ladder | `run.py --arm openswarm` | +| `bu` | LLM given a browser-use-shaped flat axtree dump | `run.py --arm bu` | +| `osw-llm` | same LLM given our ranked/deduped/capped element menu | `run.py --arm osw-llm` | +| `bu-real` | the actual browser-use agent, attached over CDP | `bu_real.py` | +| `sh-real` | the actual Stagehand agent, attached over CDP | `sh_real.py` | + +`bu` vs `osw-llm` is the controlled experiment (same model, same action layer, only the page view +differs). `bu-real`/`sh-real` are the shipping competitors, whole-stack. + +## Ground rules + +- One recorder (`recorder.py`): every episode appends to `data/all.jsonl`; screenshots under + `data/shots///-s/`. Reruns supersede by `started_at`; nothing is rewritten. +- Agents never grade themselves. `claimed_success` vs `success` is recorded precisely to count + false-success claims per arm. +- Infra failures (`error_class` starting `infra`) are excluded from rates but always reported. +- `ranking.py` must stay line-for-line with `frontend/src/shared/interactiveRanking.ts`; if either + changes, change both. + +## Running + +``` +# serve MiniWoB HTML once: cd miniwob-plusplus/miniwob/html && python3 -m http.server 8099 +MINIWOB_URL=http://localhost:8099/miniwob/ python run.py --arm osw-llm --tasks all --seeds 1 +python report.py # scoreboard; --md ARENA.md for the markdown version +python diffs.py --ours osw-llm --theirs bu-real # evidence trail for every loss +``` + +Needs the browsergym venv (Python 3.12 — 3.13 cannot build greenlet 3.0.3). diff --git a/e2e/browser-v3/arena/bu_real.py b/e2e/browser-v3/arena/bu_real.py new file mode 100644 index 00000000..e9569197 --- /dev/null +++ b/e2e/browser-v3/arena/bu_real.py @@ -0,0 +1,262 @@ +"""The REAL browser-use agent on MiniWoB: their whole stack, our task instance, MiniWoB's scoring. + +BrowserGym launches the chromium with a CDP port open; browser-use attaches to that same browser and +drives the very page BrowserGym seeded. Reward is read from MiniWoB's own WOB_REWARD_GLOBAL through +the BrowserGym handle, so browser-use's self-reported "done" never grades itself -- the lesson every +prior verifier bug in this project taught twice. + +Separate entrypoint from run.py because browser-use is asyncio all the way down and forcing it under +run.py's sync SIGALRM loop would time THEIR stack with MY interruptions. +""" +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import browsergym.miniwob # noqa: F401 registers the 125 envs +import gymnasium as gym + +from recorder import EpisodeRecord, Recorder, StepRecord +from tasks import resolve_tasks + +os.environ.setdefault("MINIWOB_URL", "http://localhost:8099/miniwob/") +CDP_PORT = int(os.environ.get("OSW_ARENA_CDP_PORT", "9321")) + + +def patch_launch_with_cdp_port() -> None: + """BrowserEnv passes args= itself, so pw_chromium_kwargs={'args': ...} raises; inject at launch. + + Every launch gets its OWN port: BrowserGym starts a second chromium just for its chat window, and + when both raced for one port the chat browser won it -- browser-use then spent 10 steps staring + at the chat page's about:blank hunting for a Submit button that was in the other browser. + """ + from playwright.sync_api import BrowserType + + if getattr(BrowserType, "osw_arena_patched", False): + return + orig = BrowserType.launch + + def launch(self, **kwargs): + # A genuinely-free port each launch: fixed pools deadlocked -- a zombie chromium from a dead + # episode kept its port bound and the next launch hung 180s failing to bind the same one. + import socket + + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + RECENT_PORTS.append(port) + del RECENT_PORTS[:-8] + kwargs["args"] = list(kwargs.get("args") or []) + [f"--remote-debugging-port={port}"] + return orig(self, **kwargs) + + BrowserType.launch = launch + BrowserType.osw_arena_patched = True + + +# Ports handed to recent launches, newest last; the probe below only ever looks here. +RECENT_PORTS: list[int] = [] + + +def find_task_cdp_url() -> str: + """Probe the recently-issued ports for the browser actually hosting the MiniWoB page.""" + import urllib.request + + for port in reversed(RECENT_PORTS): + try: + with urllib.request.urlopen(f"http://localhost:{port}/json", timeout=2) as resp: + targets = json.loads(resp.read().decode()) + except Exception: + continue + if any("miniwob" in str(t.get("url", "")) for t in targets): + return f"http://localhost:{port}" + raise RuntimeError(f"none of the recent CDP ports {RECENT_PORTS} hosts a miniwob page") + + +def reap_leftover_browsers() -> None: + """Kill any debug-port chromium THIS process launched that outlived its episode; zombies wedge + later launches. Per-port patterns, not a blanket sweep, so concurrent shards and other arms' + portless chromiums are untouchable by construction.""" + import subprocess + + for port in RECENT_PORTS: + subprocess.run(["pkill", "-f", f"ms-playwright.*remote-debugging-port={port}$"], + capture_output=True, check=False) + subprocess.run(["pkill", "-f", f"ms-playwright.*remote-debugging-port={port} "], + capture_output=True, check=False) + + +def make_env(task: str, seed: int, max_steps: int): + patch_launch_with_cdp_port() + env = gym.make(f"browsergym/miniwob.{task}", headless=True, max_episode_steps=max_steps) + obs, _ = env.reset(seed=seed) + return env, obs + + +async def drive(goal: str, cdp_url: str, model: str, endpoint: str, max_steps: int, timeout_s: float) -> dict: + """Run browser-use's own Agent loop against the already-open task page.""" + from browser_use import Agent, Browser, ChatOpenAI + + browser = Browser(cdp_url=cdp_url, is_local=False) + llm = ChatOpenAI(model=model, base_url=f"{endpoint}/v1", api_key="arena", temperature=None) + # MiniWoB pages never navigate, so the task prompt forbids goto -- their agent otherwise likes to + # open about:blank or a search engine, which would abandon the scored page. + agent = Agent( + task=f"{goal}\nWork ONLY on the currently open page. Never navigate to another URL.", + llm=llm, browser=browser, calculate_cost=True, + ) + + async def run_agent(): + # Everything of theirs -- connect, session setup, the loop -- inside ONE deadline. Wrapping + # only agent.run let a wedged phase outside it stretch an episode to 1467s. + return await agent.run(max_steps=max_steps) + + stats = {"llm_calls": 0, "prompt_tokens": 0, "completion_tokens": 0, "actions": []} + try: + history = await asyncio.wait_for(run_agent(), timeout=timeout_s) + stats["actions"] = [str(a)[:120] for a in history.action_names()] + try: + stats["claimed_success"] = bool(history.is_successful()) + except Exception: + stats["claimed_success"] = False + usage = getattr(history, "usage", None) + if usage: + stats["prompt_tokens"] = int(getattr(usage, "total_prompt_tokens", 0) or 0) + stats["completion_tokens"] = int(getattr(usage, "total_completion_tokens", 0) or 0) + stats["llm_calls"] = int(getattr(usage, "total_calls", len(stats["actions"])) or 0) + except asyncio.TimeoutError: + stats["error"] = f"agent.run exceeded {timeout_s:.0f}s" + except Exception as exc: + stats["error"] = f"{type(exc).__name__}: {exc}"[:200] + finally: + try: + await asyncio.wait_for(browser.stop(), timeout=10) + except Exception: + pass + return stats + + +def drive_in_thread(goal: str, cdp_url: str, model: str, endpoint: str, max_steps: int, timeout_s: float) -> dict: + """Own thread, own event loop: sync-Playwright's greenlet already occupies this thread's loop, + so asyncio.run() here raises 'cannot be called from a running event loop'. browser-use only + touches the browser over CDP, so it needs nothing from this thread's Playwright state.""" + import threading + + result: dict = {} + + def runner() -> None: + try: + result.update(asyncio.run(drive(goal, cdp_url, model, endpoint, max_steps, timeout_s))) + except Exception as exc: + result["error"] = f"{type(exc).__name__}: {exc}"[:200] + + t = threading.Thread(target=runner, daemon=True) + t.start() + t.join(timeout=timeout_s + 30) + if t.is_alive(): + result.setdefault("error", f"driver thread still alive after {timeout_s + 30:.0f}s") + return result + + +def score(env) -> tuple[float, float]: + """MiniWoB's verdict, read straight off the page globals -- the only grader in this file.""" + page = env.unwrapped.page + reward = float(page.evaluate("typeof WOB_REWARD_GLOBAL !== 'undefined' ? WOB_REWARD_GLOBAL : 0") or 0) + raw = float(page.evaluate("typeof WOB_RAW_REWARD_GLOBAL !== 'undefined' ? WOB_RAW_REWARD_GLOBAL : 0") or 0) + return reward, raw + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--tasks", default="smoke") + ap.add_argument("--seeds", type=int, default=1) + ap.add_argument("--seed-base", type=int, default=42) + ap.add_argument("--max-steps", type=int, default=12) + ap.add_argument("--episode-timeout", type=float, default=180.0) + ap.add_argument("--model", default=os.environ.get("OSW_ARENA_MODEL", "cc/claude-sonnet-4-6")) + ap.add_argument("--endpoint", default=os.environ.get("OSW_ARENA_ENDPOINT", "http://localhost:20128")) + ap.add_argument("--shots", choices=["none", "first-last"], default="first-last") + ap.add_argument("--shard", default="") + args = ap.parse_args() + + tasks = resolve_tasks(args.tasks) + if args.shard: + i, n = (int(x) for x in args.shard.split("/")) + tasks = [t for k, t in enumerate(tasks) if k % n == i] + rec = Recorder("bu-real") + print(f"arm=bu-real tasks={len(tasks)} seeds={args.seeds} model={args.model} -> {rec.path}", flush=True) + wins = total = 0 + for task in tasks: + for s in range(args.seeds): + seed = args.seed_base + s + ep = EpisodeRecord(arm="bu-real", task=task, seed=seed, model=args.model, + started_at=time.time()) + t_setup = time.time() + env = None + try: + env, obs = make_env(task, seed, args.max_steps) + ep.goal = str(obs.get("goal") or "")[:300] + except Exception as exc: + ep.error_class = "infra_env_setup" + ep.error_detail = f"{type(exc).__name__}: {exc}"[:200] + ep.setup_s = time.time() - t_setup + if env is not None: + if args.shots != "none": + try: + env.unwrapped.page.screenshot(path=str(rec.shot_path("bu-real", task, seed, 1))) + except Exception: + pass + t0 = time.time() + try: + cdp_url = find_task_cdp_url() + except Exception as exc: + cdp_url = "" + ep.error_class = "infra_cdp_probe" + ep.error_detail = str(exc)[:200] + stats = drive_in_thread(ep.goal, cdp_url, args.model, args.endpoint, + args.max_steps, args.episode_timeout) if cdp_url else {} + ep.wall_s = time.time() - t0 + try: + ep.reward, ep.raw_reward = score(env) + except Exception as exc: + ep.error_class = ep.error_class or "infra_score_readback" + ep.error_detail = f"{type(exc).__name__}: {exc}"[:200] + ep.success = ep.reward > 0 + ep.claimed_success = bool(stats.get("claimed_success")) + ep.steps = len(stats.get("actions") or []) + ep.prompt_tokens = stats.get("prompt_tokens", 0) + ep.completion_tokens = stats.get("completion_tokens", 0) + ep.llm_calls = stats.get("llm_calls", 0) + if stats.get("error") and not ep.success: + ep.error_detail = (ep.error_detail + " | " + stats["error"])[:200].strip(" |") + for i, act in enumerate(stats.get("actions") or [], 1): + ep.step_records.append(StepRecord(step=i, action=act)) + if args.shots != "none": + try: + env.unwrapped.page.screenshot(path=str(rec.shot_path("bu-real", task, seed, 99))) + except Exception: + pass + try: + env.close() + except Exception: + pass + reap_leftover_browsers() + rec.write(ep) + total += 1 + wins += 1 if ep.success else 0 + flag = "OK " if ep.success else ("ERR" if ep.error_class else "-- ") + lie = " FALSE-SUCCESS" if ep.claimed_success and not ep.success else "" + print(f" {flag} {task:28s} s={seed} r={ep.reward:+.2f} steps={ep.steps:2d} " + f"{ep.wall_s:6.2f}s tok={ep.prompt_tokens + ep.completion_tokens:<7d} " + f"{ep.error_detail[:60]}{lie}", flush=True) + print(f"\nbu-real: {wins}/{total} = {100 * wins / total if total else 0:.1f}%", flush=True) + + +if __name__ == "__main__": + main() diff --git a/e2e/browser-v3/arena/diffs.py b/e2e/browser-v3/arena/diffs.py new file mode 100644 index 00000000..49eb7a96 --- /dev/null +++ b/e2e/browser-v3/arena/diffs.py @@ -0,0 +1,78 @@ +"""Where each competitor beats us, task by task, with the evidence trail for each loss. + +This is the input to the ingest-and-iterate loop: for every (task, seed) where a competitor arm +succeeded and ours failed, print our step trace, their action list, and both screenshot paths, so +"what do they do better" is answered from recorded evidence rather than from impressions. + + python diffs.py --ours osw-llm --theirs bu-real +""" +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from recorder import load_episodes +from report import latest_per_key +from tasks import CATEGORY_OF + + +def pick(eps: list[dict], arm: str) -> dict[tuple[str, int], dict]: + return {(e["task"], e["seed"]): e for e in eps if e["arm"] == arm} + + +def trace(ep: dict, limit: int = 8) -> list[str]: + out = [] + for s in (ep.get("step_records") or [])[:limit]: + err = f" ERR:{s['action_error'][:60]}" if s.get("action_error") else "" + out.append(f" {s['step']:2d}. {s.get('action', '')[:80]}{err}") + shots = [s["shot"] for s in ep.get("step_records") or [] if s.get("shot")] + if shots: + out.append(f" shots: {shots[0]} .. {Path(shots[-1]).parent}/99.png") + return out + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--ours", default="osw-llm") + ap.add_argument("--theirs", default="bu-real") + ap.add_argument("--tag", default="all") + ap.add_argument("--show-ours-wins", action="store_true") + args = ap.parse_args() + + eps = latest_per_key(load_episodes(args.tag)) + ours, theirs = pick(eps, args.ours), pick(eps, args.theirs) + common = sorted(set(ours) & set(theirs)) + if not common: + raise SystemExit(f"no common (task, seed) pairs between {args.ours} and {args.theirs}") + + they_beat_us = [k for k in common if theirs[k].get("success") and not ours[k].get("success")] + we_beat_them = [k for k in common if ours[k].get("success") and not theirs[k].get("success")] + both = sum(1 for k in common if ours[k].get("success") and theirs[k].get("success")) + neither = sum(1 for k in common if not ours[k].get("success") and not theirs[k].get("success")) + + print(f"common episodes: {len(common)} both-solve: {both} neither: {neither}") + print(f"{args.theirs} beats {args.ours}: {len(they_beat_us)} " + f"{args.ours} beats {args.theirs}: {len(we_beat_them)}\n") + + print(f"=== {args.theirs} solved, {args.ours} failed ===") + for task, seed in they_beat_us: + o, t = ours[(task, seed)], theirs[(task, seed)] + print(f"\n{task} (s={seed}, {CATEGORY_OF.get(task, '?')}) goal: {o.get('goal', '')[:90]}") + print(f" ours ({o['steps']} steps, {o['wall_s']:.1f}s, err={o.get('error_class') or '-'}):") + print("\n".join(trace(o))) + print(f" theirs ({t['steps']} steps, {t['wall_s']:.1f}s):") + print("\n".join(trace(t))) + + if args.show_ours_wins: + print(f"\n=== {args.ours} solved, {args.theirs} failed ===") + for task, seed in we_beat_them: + t = theirs[(task, seed)] + print(f"{task} (s={seed}) their steps={t['steps']} wall={t['wall_s']:.1f}s " + f"claimed={t.get('claimed_success')} err={t.get('error_detail', '')[:60]}") + + +if __name__ == "__main__": + main() diff --git a/e2e/browser-v3/arena/llm_policy.py b/e2e/browser-v3/arena/llm_policy.py new file mode 100644 index 00000000..1143ce55 --- /dev/null +++ b/e2e/browser-v3/arena/llm_policy.py @@ -0,0 +1,245 @@ +"""Model-driven arms. Same model, same steps, same action space, same scorer -- only the page view differs. + +That constraint is the entire experiment. `bu` renders the page the way browser-use does (flat +accessibility dump, every node, no memory between turns); `osw` renders it the way OpenSwarm's +BrowserListInteractives does (deduped, goal-ranked, capped at 60, ctx on twins, value on inputs, +`*` on anything new since the last look). Any score gap between them is attributable to that view. +""" +from __future__ import annotations + +import json +import os +import re +import time +import urllib.error +import urllib.request +from dataclasses import dataclass, field +from typing import Any + +import perception +from policies import Decision +from ranking import RankItem, rank_and_cap, render + +ACTIONS = """click(bid) | dblclick(bid) | fill(bid, "text") | clear(bid) | select_option(bid, "opt") +hover(bid) | focus(bid) | press(bid, "key") | scroll(dx, dy) | drag_and_drop(from_bid, to_bid) +mouse_click(x, y) | mouse_drag_and_drop(from_x, from_y, to_x, to_y) (viewport coordinates)""" + +BU_SYSTEM = """You are a web agent. You are given the accessibility tree of a page and a goal. +Respond with EXACTLY ONE action call and nothing else. No prose, no markdown, no explanation. +Available actions: +""" + ACTIONS + +OSW_SYSTEM = """You are OpenSwarm's browser agent. You see a ranked, deduplicated list of the page's +interactive elements. Each row is [index]. A `*` means the element is new +since your last look; the same index always means the same element. Rows without a useful name show +center=(x,y) viewport coordinates. +Respond with EXACTLY ONE action call and nothing else. No prose, no markdown, no explanation. +Available actions, addressing elements by their [index]: +click(index) | fill(index, "text") | select_option(index, "opt") | press(index, "key") +clear(index) | hover(index) | focus(index) | scroll(dx, dy) | drag_and_drop(from_index, to_index) +For targets with no element of their own (a spot on a canvas, a slider position), use coordinates: +mouse_click(x, y) | mouse_dblclick(x, y) | mouse_drag_and_drop(from_x, from_y, to_x, to_y) +For a combobox/listbox row showing options="...", pick with select_option(index, "exact option"). +Prefer filling the box the goal names, then submitting. Do not repeat an action that already worked. +If an action did not change the page, try a DIFFERENT action, never the same one again.""" + + +def post_json(url: str, payload: dict[str, Any], timeout: float = 90.0) -> dict[str, Any]: + body = json.dumps(payload).encode() + req = urllib.request.Request(url, data=body, method="POST", + headers={"Content-Type": "application/json", + "Authorization": "Bearer arena"}) + with urllib.request.urlopen(req, timeout=timeout) as resp: + return json.loads(resp.read().decode()) + + +@dataclass +class LlmDecision(Decision): + """Decision plus the per-call accounting the recorder folds into episode totals.""" + + think_ms: float = 0.0 + prompt_tokens: int = 0 + completion_tokens: int = 0 + cost_usd: float = 0.0 + llm_error: str = "" + retries: int = 0 + + +@dataclass +class LlmPolicy: + """Shared model loop; subclasses supply the page view and the action translation.""" + + name: str = "llm" + model: str = "cc/claude-sonnet-4-6" + endpoint: str = "http://localhost:20128" + system: str = BU_SYSTEM + history: list[str] = field(default_factory=list) + max_history: int = 6 + + def reset(self, goal: str) -> None: + self.history = [] + + def view(self, obs: dict[str, Any], goal: str) -> tuple[str, int]: + raise NotImplementedError + + def translate(self, raw: str) -> str: + return raw + + def note(self, action: str, obs: dict[str, Any]) -> None: + """History is kept in the MODEL's own namespace, with the outcome the page reported. + + The first version logged the translated bid call, so the OpenSwarm arm was shown history it + could not line up against the indices in its own element list and re-clicked the same box + five times. Feeding an agent a memory it cannot read is a harness bug, not an agent failure. + """ + err = str(obs.get("last_action_error") or "").strip() + self.history.append(f"{action} -> {'ERROR: ' + err[:120] if err else 'ok'}") + + def call(self, goal: str, page: str) -> tuple[str, LlmDecision]: + past = "\n".join(self.history[-self.max_history:]) or "(none yet)" + user = f"GOAL: {goal}\n\nACTIONS YOU ALREADY TOOK:\n{past}\n\nPAGE:\n{page}\n\nYour single next action:" + t0 = time.time() + d = LlmDecision(action="") + text = "" + # Retry transient router faults: a concurrent-sweep run lost 40% of its episodes to 502s that + # were then booked as policy failures. Retries make the residue rare; the classifier below + # books what remains as infra, never as skill. + for attempt in range(3): + try: + resp = post_json(f"{self.endpoint}/v1/chat/completions", { + "model": self.model, + "messages": [{"role": "system", "content": self.system}, + {"role": "user", "content": user}], + "max_tokens": 200, + "stream": False, # the router streams by default; a single action needs no SSE + }) + text = (resp.get("choices") or [{}])[0].get("message", {}).get("content") or "" + usage = resp.get("usage") or {} + d.prompt_tokens = int(usage.get("prompt_tokens") or 0) + d.completion_tokens = int(usage.get("completion_tokens") or 0) + d.llm_error = "" + d.retries = attempt + break + except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, ValueError) as exc: + d.llm_error = f"{type(exc).__name__}: {exc}"[:150] + time.sleep(2.0 * (attempt + 1)) + d.think_ms = (time.time() - t0) * 1000 + return str(text).strip(), d + + +@dataclass +class BrowserUseStylePolicy(LlmPolicy): + """Flat accessibility dump, exactly what a browser-use-shaped agent puts in front of a model.""" + + name: str = "bu" + system: str = BU_SYSTEM + + def view(self, obs: dict[str, Any], goal: str) -> tuple[str, int]: + from browsergym.utils.obs import flatten_axtree_to_str + + text = flatten_axtree_to_str(obs.get("axtree_object") or {}) + n = len(perception.interactives(obs)) + return text[:14000], n + + def act(self, obs: dict[str, Any], goal: str) -> LlmDecision: + page, n = self.view(obs, goal) + raw, d = self.call(goal, page) + d.n_interactive = n + d.action = clean_action(raw) + if d.action: + self.note(d.action, obs) + return d + + +@dataclass +class OpenSwarmLlmPolicy(LlmPolicy): + """OpenSwarm's ranked element menu, addressed by stable 1-based index instead of raw bid.""" + + name: str = "osw-llm" + system: str = OSW_SYSTEM + index_to_bid: dict[int, str] = field(default_factory=dict) + prev_bids: set[str] = field(default_factory=set) + # v2: also list clickable-but-unroled elements (canvas/svg/div) -- ingested from browser-use. + clickable: bool = False + # v3: append the page's visible text, our BrowserGetText equivalent; menu-only lost every task + # whose payload lives in prose (the algebra equation, which email is Cecile's). + with_text: bool = False + + def reset(self, goal: str) -> None: + self.history = [] + self.index_to_bid = {} + self.prev_bids = set() + + def view(self, obs: dict[str, Any], goal: str) -> tuple[str, int]: + raw_items: list[RankItem] = perception.interactives(obs, include_clickable=self.clickable) + shown, truncated = rank_and_cap(raw_items, goal=goal) + new = {it.bid for it in shown} - self.prev_bids if self.prev_bids else set() + self.prev_bids = {it.bid for it in shown} + self.index_to_bid = {i: it.bid for i, it in enumerate(shown, 1)} + view = render(shown, truncated, new) + if self.with_text: + text = perception.page_text(obs) + if text: + view += f"\n\nPAGE TEXT:\n{text}" + return view, len(shown) + + def translate(self, call: str) -> str: + """Swap our 1-based indices back to bids so both arms hit the identical action layer.""" + m = re.match(r"(\w+)\s*\((.*)\)\s*$", call, re.S) + if not m: + return call + fn, argstr = m.group(1), m.group(2) + # scroll deltas and mouse_*/keyboard_* coordinates are geometry, never element handles. + if fn == "scroll" or fn.startswith(("mouse_", "keyboard_")): + return call + # Only leading positional args are element handles, so rewrite those and leave payloads alone. + # Quoted digits count too: the model addresses indices, so click("3") means row 3, never bid 3. + n_handles = 2 if fn == "drag_and_drop" else 1 + parts = [a.strip() for a in re.split(r',(?=(?:[^"]*"[^"]*")*[^"]*$)', argstr)] if argstr.strip() else [] + for i in range(min(n_handles, len(parts))): + m2 = re.fullmatch(r'"?(\d+)"?', parts[i]) + if m2: + bid = self.index_to_bid.get(int(m2.group(1))) + if bid: + parts[i] = f'"{bid}"' + return f"{fn}({', '.join(parts)})" + + def act(self, obs: dict[str, Any], goal: str) -> LlmDecision: + page, n = self.view(obs, goal) + raw, d = self.call(goal, page) + d.n_interactive = n + chosen = clean_action(raw) + d.action = self.translate(chosen) + if chosen: + self.note(chosen, obs) + return d + + +CALL_RE = re.compile( + r"\b(click|dblclick|fill|clear|select_option|hover|focus|press|scroll|drag_and_drop|noop" + r"|mouse_click|mouse_dblclick|mouse_move|mouse_drag_and_drop|keyboard_type|keyboard_press)\s*\([^)]*\)") + + +def clean_action(raw: str) -> str: + """Pull the one action call out of whatever the model wrapped it in; empty means unparseable.""" + if not raw: + return "" + text = raw.strip().strip("`") + text = re.sub(r"^(python|json|tool_code)\s*", "", text) + m = CALL_RE.search(text) + return m.group(0) if m else "" + + +def build(name: str, model: str = "", endpoint: str = "", **_: Any) -> Any: + model = model or os.environ.get("OSW_ARENA_MODEL", "cc/claude-sonnet-4-6") + endpoint = endpoint or os.environ.get("OSW_ARENA_ENDPOINT", "http://localhost:20128") + if name in ("bu", "bu-llm", "browseruse-style"): + return BrowserUseStylePolicy(model=model, endpoint=endpoint) + if name in ("osw-llm", "openswarm-llm"): + return OpenSwarmLlmPolicy(model=model, endpoint=endpoint) + if name == "osw-llm-v2": + return OpenSwarmLlmPolicy(name=name, model=model, endpoint=endpoint, clickable=True) + if name == "osw-llm-v3": + return OpenSwarmLlmPolicy(name=name, model=model, endpoint=endpoint, + clickable=True, with_text=True) + raise SystemExit(f"unknown arm: {name}") diff --git a/e2e/browser-v3/arena/perception.py b/e2e/browser-v3/arena/perception.py new file mode 100644 index 00000000..3c0b26ed --- /dev/null +++ b/e2e/browser-v3/arena/perception.py @@ -0,0 +1,176 @@ +"""Turn a BrowserGym observation into the element list each arm is allowed to see. + +Both arms read the SAME underlying accessibility tree, which is the point: the comparison is between +what each stack does with the tree (browser-use dumps it flat, OpenSwarm dedupes/ranks/caps/marks it), +not between two different ways of getting one. Anything that advantaged one arm's raw perception +would make the score a measurement of plumbing. +""" +from __future__ import annotations + +from typing import Any + +from ranking import INTERACTIVE_ROLES, RankItem + +# Roles carrying page copy; used to build the ctx string that disambiguates same-named twins. +TEXT_ROLES = {"StaticText", "LabelText", "heading", "paragraph", "InlineTextBox"} + + +def node_role(node: dict[str, Any]) -> str: + return str((node.get("role") or {}).get("value") or "") + + +def node_name(node: dict[str, Any]) -> str: + return str((node.get("name") or {}).get("value") or "") + + +def node_value(node: dict[str, Any]) -> str: + v = node.get("value") or {} + return str(v.get("value") or "") if isinstance(v, dict) else "" + + +def visible(bid: str, extra: dict[str, Any], threshold: float = 0.5) -> bool: + """Our dropCoveredElements analogue: an element under an overlay is not an element you can click.""" + props = extra.get(bid) if extra else None + if not props: + return True + try: + return float(props.get("visibility", 1.0)) >= threshold + except (TypeError, ValueError): + return True + + +def build_context(nodes: list[dict[str, Any]], by_id: dict[str, dict[str, Any]], + node: dict[str, Any], depth: int = 3) -> str: + """Nearest ancestor's text, so five identical 'Message' buttons say which card they belong to.""" + cur = node + for _ in range(depth): + parent_id = cur.get("parentId") + if not parent_id or parent_id not in by_id: + return "" + parent = by_id[parent_id] + texts: list[str] = [] + for cid in parent.get("childIds") or []: + child = by_id.get(cid) + if not child or child is node: + continue + if node_role(child) in TEXT_ROLES: + t = node_name(child).strip() + if t: + texts.append(t) + if texts: + return " ".join(texts)[:60] + cur = parent + return "" + + +def interactives(obs: dict[str, Any], include_hidden: bool = False, + include_clickable: bool = False) -> list[RankItem]: + """Every actionable node in document order, before any ranking or capping is applied. + + include_clickable is the technique ingested from browser-use: elements the page wires for + clicks but gives no interactive AX role -- canvases, SVGs, styled divs. Measured on MiniWoB, + their flat dump solved spatial tasks (circle-center, bisect-angle) purely because the canvas + appeared in it while our role-filtered menu hid the only thing worth clicking. + """ + ax = obs.get("axtree_object") or {} + nodes: list[dict[str, Any]] = ax.get("nodes") or [] + extra = obs.get("extra_element_properties") or {} + by_id = {n["nodeId"]: n for n in nodes if "nodeId" in n} + out: list[RankItem] = [] + for n in nodes: + if n.get("ignored"): + continue + role = node_role(n) + bid = n.get("browsergym_id") + if not bid: + continue + is_role = role in INTERACTIVE_ROLES + is_clickable = (include_clickable and not is_role and role not in TEXT_ROLES + and bool((extra.get(str(bid)) or {}).get("clickable"))) + if not (is_role or is_clickable): + continue + if not include_hidden and not visible(str(bid), extra): + continue + name = node_name(n).strip() + bbox = (extra.get(str(bid)) or {}).get("bbox") + center = (bbox[0] + bbox[2] / 2, bbox[1] + bbox[3] / 2) if bbox else None + out.append(RankItem( + role=role if is_role else (role or "clickable"), + name=name, + bid=str(bid), + value=node_value(n)[:80], + context=build_context(nodes, by_id, n), + center=center, + options=child_options(by_id, n) if role in ("combobox", "listbox", "menu") else None, + )) + return out + + +OPTION_ROLES = {"option", "menuitem", "MenuListOption", "ListBoxOption"} + + +def child_options(by_id: dict[str, dict[str, Any]], node: dict[str, Any], depth: int = 3) -> list[str] | None: + """Option labels under a select-like node, ignored-or-not: a closed is unactionable. + options: list[str] | None = None + + +def role_priority(role: str) -> int: + return ROLE_PRIORITY.get(role, DEFAULT_PRIORITY) + + +def goal_keywords(goal: str) -> list[str]: + words = [w for w in re.split(r"[^a-z0-9]+", goal.lower()) if w] + kept = [w for w in words if len(w) >= 3 and w not in STOPWORDS] + seen: list[str] = [] + for w in kept: + if w not in seen: + seen.append(w) + return seen[:8] + + +def matches_goal(name: str, keywords: list[str]) -> bool: + if not keywords: + return False + lower = name.lower() + return any(k in lower for k in keywords) + + +# Roles whose twins are individually meaningful: two same-named password boxes are two fields the +# user must both fill, never an icon+label pair. Collapsing them cost enter-password outright. +NEVER_DEDUPE_ROLES = {"textbox", "searchbox", "combobox", "spinbutton", "listbox"} + + +def dedupe_consecutive(items: list[RankItem]) -> list[RankItem]: + """Collapse only BACK-TO-BACK role+name+context twins, so a genuine list of five is never merged.""" + out: list[RankItem] = [] + for it in items: + prev = out[-1] if out else None + if (prev and prev.role == it.role and prev.name == it.name and prev.context == it.context + and it.role not in NEVER_DEDUPE_ROLES): + continue + out.append(it) + return out + + +def rank_and_cap(items: list[RankItem], goal: str = "", cap: int = DEFAULT_INTERACTIVE_CAP, + doc_order: bool = True) -> tuple[list[RankItem], int]: + """Rank decides WHAT survives the cap; display order stays document order so ordinals read true.""" + keywords = goal_keywords(goal) if goal else [] + deduped = dedupe_consecutive(items) + scored = [(it, i, 0 if matches_goal(it.name, keywords) else 1) for i, it in enumerate(deduped)] + ranked = sorted(scored, key=lambda x: (x[2], role_priority(x[0].role), x[1])) + selected = ranked[:cap] if cap > 0 else ranked + displayed = sorted(selected, key=lambda x: x[1]) if doc_order else selected + return [x[0] for x in displayed], max(0, len(ranked) - len(displayed)) + + +def render(items: list[RankItem], truncated: int, new_bids: set[str] | None = None) -> str: + """The exact string shape BrowserListInteractives hands the model, including the * new-marker.""" + if not items: + return "No interactive elements found on this page." + counts: dict[str, int] = {} + for el in items: + k = f"{el.role}|{el.name}" + counts[k] = counts.get(k, 0) + 1 + lines = [] + for i, el in enumerate(items, 1): + dup = counts.get(f"{el.role}|{el.name}", 0) > 1 + ctx = f' ctx="{el.context}"' if dup and el.context else "" + val = f' value="{el.value}"' if el.value else "" + star = "*" if new_bids and el.bid in new_bids else "" + # Coordinates only where the label carries no signal; a named button needs no geometry. + pos = f" center=({el.center[0]:.0f},{el.center[1]:.0f})" if el.center and not el.name else "" + opts = "" + if el.options: + shown_opts = el.options[:12] + more = f" +{len(el.options) - 12}" if len(el.options) > 12 else "" + opts = f' options="{"|".join(shown_opts)}{more}"' + lines.append(f'[{i}]{star}<{el.role} "{el.name}"{ctx}{val}{pos}{opts}>') + head = f"{len(lines)} interactive elements (* = new since your last look):" + text = head + "\n" + "\n".join(lines) + if truncated > 0: + text += f"\n... {truncated} more not shown; scroll or scope to reach them." + return text diff --git a/e2e/browser-v3/arena/recorder.py b/e2e/browser-v3/arena/recorder.py new file mode 100644 index 00000000..fd112517 --- /dev/null +++ b/e2e/browser-v3/arena/recorder.py @@ -0,0 +1,139 @@ +"""Central data store for the MiniWoB arena: every arm, every task, every step, one place. + +Everything any arm produces lands in ONE append-only JSONL so no arm can be scored by a different +book than another. Screenshots are written per step and referenced by path from the step record, so +a claim about what an agent saw can always be checked against the pixels it saw it in. +""" +from __future__ import annotations + +import json +import os +import time +from dataclasses import dataclass, field, asdict +from pathlib import Path +from typing import Any + +ARENA_DIR = Path(__file__).resolve().parent +RESULTS_DIR = Path(os.environ.get("OSW_ARENA_RESULTS", ARENA_DIR / "data")) +SHOTS_DIR = RESULTS_DIR / "shots" + + +@dataclass +class StepRecord: + """One agent turn: what it saw, what it chose, what that cost, what the page did about it.""" + + step: int + action: str = "" + action_ms: float = 0.0 + perceive_ms: float = 0.0 + think_ms: float = 0.0 + axtree_chars: int = 0 + axtree_nodes: int = 0 + dom_chars: int = 0 + n_interactive: int = 0 + prompt_tokens: int = 0 + completion_tokens: int = 0 + cost_usd: float = 0.0 + reward: float = 0.0 + url: str = "" + focused_bid: str = "" + action_error: str = "" + llm_error: str = "" + retries: int = 0 + shot: str = "" + + +@dataclass +class EpisodeRecord: + """One (arm, task, seed) attempt, scored by MiniWoB itself rather than by this file.""" + + arm: str + task: str + seed: int + model: str = "" + reward: float = 0.0 + raw_reward: float = 0.0 + success: bool = False + # What the agent itself claimed; claimed and not success = a false success, the worst failure class. + claimed_success: bool = False + steps: int = 0 + wall_s: float = 0.0 + setup_s: float = 0.0 + first_action_s: float = 0.0 + terminated: bool = False + truncated: bool = False + error_class: str = "" + error_detail: str = "" + goal: str = "" + prompt_tokens: int = 0 + completion_tokens: int = 0 + cost_usd: float = 0.0 + llm_calls: int = 0 + started_at: float = 0.0 + step_records: list[StepRecord] = field(default_factory=list) + + def add(self, rec: StepRecord) -> None: + self.step_records.append(rec) + self.prompt_tokens += rec.prompt_tokens + self.completion_tokens += rec.completion_tokens + self.cost_usd += rec.cost_usd + if rec.prompt_tokens or rec.completion_tokens or rec.think_ms: + self.llm_calls += 1 + + +class Recorder: + """Append-only writer. One file per run tag, plus a stable `all.jsonl` that never gets rewritten.""" + + def __init__(self, tag: str, run_id: str = "") -> None: + RESULTS_DIR.mkdir(parents=True, exist_ok=True) + SHOTS_DIR.mkdir(parents=True, exist_ok=True) + self.tag = tag + self.run_id = run_id or time.strftime("%Y%m%d-%H%M%S") + self.path = RESULTS_DIR / f"{tag}.jsonl" + self.all_path = RESULTS_DIR / "all.jsonl" + + def shot_path(self, arm: str, task: str, seed: int, step: int) -> Path: + d = SHOTS_DIR / self.run_id / arm / f"{task}-s{seed}" + d.mkdir(parents=True, exist_ok=True) + return d / f"{step:02d}.png" + + def write(self, ep: EpisodeRecord) -> None: + row = asdict(ep) + row["run_id"] = self.run_id + row["tag"] = self.tag + line = json.dumps(row, separators=(",", ":")) + "\n" + for p in (self.path, self.all_path): + with open(p, "a") as fh: + fh.write(line) + + +def save_screenshot(obs: dict[str, Any], dest: Path) -> str: + """Persist a BrowserGym observation frame. Returns the path, or '' if the frame was unusable.""" + arr = obs.get("screenshot") + if arr is None: + return "" + try: + from PIL import Image # imported lazily so a no-screenshot run needs no pillow + + Image.fromarray(arr).save(dest) + return str(dest) + except Exception: + return "" + + +def load_episodes(tag: str = "all") -> list[dict[str, Any]]: + """Read back every episode for a tag; the report module's only input.""" + p = RESULTS_DIR / f"{tag}.jsonl" + if not p.exists(): + return [] + out: list[dict[str, Any]] = [] + with open(p) as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + out.append(json.loads(line)) + except ValueError: + continue + return out diff --git a/e2e/browser-v3/arena/report.py b/e2e/browser-v3/arena/report.py new file mode 100644 index 00000000..9a151aec --- /dev/null +++ b/e2e/browser-v3/arena/report.py @@ -0,0 +1,158 @@ +"""Every arm, every category, every metric, one table -- written from the recorder's book only. + +This file computes; it never re-judges. Success is whatever MiniWoB's reward said at run time, and +if a number is not derivable from the recorded episodes it does not appear here. + + python report.py # summary across all recorded arms + python report.py --by-task # per-task win matrix + python report.py --md ARENA.md # write the markdown scoreboard +""" +from __future__ import annotations + +import argparse +import math +import statistics +import sys +from collections import defaultdict +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from recorder import load_episodes +from tasks import CATEGORIES, CATEGORY_OF + + +def wilson(wins: int, n: int, z: float = 1.96) -> tuple[float, float]: + """Same interval the rest of browser-v3 reports, so numbers compare across documents.""" + if n == 0: + return (0.0, 0.0) + p = wins / n + d = 1 + z * z / n + c = p + z * z / (2 * n) + h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) + return ((c - h) / d, (c + h) / d) + + +def med(vals: list[float]) -> float: + return statistics.median(vals) if vals else 0.0 + + +def latest_per_key(eps: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Reruns supersede: only the newest episode per (arm, task, seed) counts, so iteration is honest.""" + by_key: dict[tuple[str, str, int], dict[str, Any]] = {} + for ep in eps: + key = (ep["arm"], ep["task"], ep["seed"]) + cur = by_key.get(key) + if cur is None or ep.get("started_at", 0) >= cur.get("started_at", 0): + by_key[key] = ep + return list(by_key.values()) + + +def summarize(eps: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + arms: dict[str, dict[str, Any]] = {} + for arm in sorted({e["arm"] for e in eps}): + rows = [e for e in eps if e["arm"] == arm] + clean = [e for e in rows if not str(e.get("error_class", "")).startswith("infra")] + wins = sum(1 for e in clean if e.get("success")) + lo, hi = wilson(wins, len(clean)) + arms[arm] = { + "n": len(rows), "clean": len(clean), "wins": wins, + "rate": wins / len(clean) if clean else 0.0, "lo": lo, "hi": hi, + "infra": len(rows) - len(clean), + "wall_med": med([e["wall_s"] for e in clean if e.get("wall_s")]), + "wall_win_med": med([e["wall_s"] for e in clean if e.get("success")]), + "steps_med": med([float(e["steps"]) for e in clean if e.get("steps")]), + "tokens": sum(e.get("prompt_tokens", 0) + e.get("completion_tokens", 0) for e in rows), + "llm_calls": sum(e.get("llm_calls", 0) for e in rows), + "false_success": sum(1 for e in clean if e.get("claimed_success") and not e.get("success")), + "by_cat": by_category(clean), + } + return arms + + +def by_category(rows: list[dict[str, Any]]) -> dict[str, tuple[int, int]]: + out: dict[str, list[int]] = defaultdict(lambda: [0, 0]) + for e in rows: + cat = CATEGORY_OF.get(e["task"], "other") + out[cat][1] += 1 + out[cat][0] += 1 if e.get("success") else 0 + return {k: (v[0], v[1]) for k, v in out.items()} + + +def fmt_summary(arms: dict[str, dict[str, Any]], md: bool = False) -> str: + lines: list[str] = [] + bar = "|" if md else " " + if md: + lines.append("| arm | solved | rate | 95% CI | med wall (win) | med steps | tokens | LLM calls | false-succ | infra |") + lines.append("|---|---|---|---|---|---|---|---|---|---|") + else: + lines.append(f"{'arm':12s} {'solved':>9s} {'rate':>7s} {'95% CI':>15s} {'wall(win)':>10s} " + f"{'steps':>6s} {'tokens':>9s} {'calls':>6s} {'false':>6s} {'infra':>6s}") + for arm, s in sorted(arms.items(), key=lambda kv: -kv[1]["rate"]): + ci = f"[{100 * s['lo']:.0f},{100 * s['hi']:.0f}]" + row = [arm, f"{s['wins']}/{s['clean']}", f"{100 * s['rate']:.1f}%", ci, + f"{s['wall_win_med']:.1f}s", f"{s['steps_med']:.0f}", f"{s['tokens']}", + f"{s['llm_calls']}", f"{s['false_success']}", f"{s['infra']}"] + lines.append(("| " + " | ".join(row) + " |") if md else + f"{row[0]:12s} {row[1]:>9s} {row[2]:>7s} {row[3]:>15s} {row[4]:>10s} " + f"{row[5]:>6s} {row[6]:>9s} {row[7]:>6s} {row[8]:>6s} {row[9]:>6s}") + lines.append("") + cats = sorted(CATEGORIES) + if md: + lines.append("| category | " + " | ".join(sorted(arms)) + " |") + lines.append("|---|" + "---|" * len(arms)) + else: + lines.append(f"{'category':16s}" + "".join(f"{a:>16s}" for a in sorted(arms))) + for cat in cats: + cells = [] + for arm in sorted(arms): + w, n = arms[arm]["by_cat"].get(cat, (0, 0)) + cells.append(f"{w}/{n} ({100 * w / n:.0f}%)" if n else "-") + lines.append(("| " + cat + " | " + " | ".join(cells) + " |") if md + else f"{cat:16s}" + "".join(f"{c:>16s}" for c in cells)) + return "\n".join(lines) + + +def fmt_by_task(eps: list[dict[str, Any]]) -> str: + """Per-task matrix: the disagreements between arms are where every insight lives.""" + arms = sorted({e["arm"] for e in eps}) + by_tk: dict[str, dict[str, list[bool]]] = defaultdict(lambda: defaultdict(list)) + for e in eps: + if not str(e.get("error_class", "")).startswith("infra"): + by_tk[e["task"]][e["arm"]].append(bool(e.get("success"))) + lines = [f"{'task':30s}" + "".join(f"{a:>12s}" for a in arms)] + for task in sorted(by_tk): + cells = [] + for arm in arms: + r = by_tk[task].get(arm) + cells.append("-" if not r else f"{sum(r)}/{len(r)}") + lines.append(f"{task:30s}" + "".join(f"{c:>12s}" for c in cells)) + return "\n".join(lines) + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--tag", default="all") + ap.add_argument("--by-task", action="store_true") + ap.add_argument("--md", default="") + # Cross-model rows never mix: an arm's rate is only meaningful against arms on the same lane. + ap.add_argument("--model", default="", help="only episodes run on this model (or '' for LLM-free arms too)") + args = ap.parse_args() + eps = latest_per_key(load_episodes(args.tag)) + if args.model: + eps = [e for e in eps if e.get("model", "") in (args.model, "")] + if not eps: + raise SystemExit("no recorded episodes") + if args.by_task: + print(fmt_by_task(eps)) + return + arms = summarize(eps) + print(fmt_summary(arms)) + if args.md: + Path(args.md).write_text("# MiniWoB arena scoreboard\n\n" + fmt_summary(arms, md=True) + "\n") + print(f"\nwrote {args.md}") + + +if __name__ == "__main__": + main() diff --git a/e2e/browser-v3/arena/run.py b/e2e/browser-v3/arena/run.py new file mode 100644 index 00000000..d5fc8fde --- /dev/null +++ b/e2e/browser-v3/arena/run.py @@ -0,0 +1,226 @@ +"""Run one arm over MiniWoB tasks and record every metric MiniWoB and the browser will give us. + +Scoring is MiniWoB's own reward, read through BrowserGym. Nothing in this repo decides whether an +episode passed, which is the whole reason this harness exists: every other suite here was graded by +something I wrote and then tuned against. + + python run.py --arm openswarm --tasks all --seeds 3 --shots first-last +""" +from __future__ import annotations + +import argparse +import os +import signal +import sys +import time +import traceback +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import browsergym.miniwob # noqa: F401 importing is what registers the 125 envs +import gymnasium as gym + +import perception +import policies +from recorder import EpisodeRecord, Recorder, StepRecord, save_screenshot +from tasks import resolve_tasks + +os.environ.setdefault("MINIWOB_URL", "http://localhost:8099/miniwob/") + + +class StepHang(Exception): + """env.step exceeded the watchdog; the browser is presumed wedged for this episode.""" + + +def with_deadline(fn: Any, timeout_s: float) -> Any: + """One measured episode lost 901s inside a single env.step; never let a hang masquerade as skill time. + + SIGALRM, not a worker thread: sync Playwright pins its greenlet to the creating thread, and the + threaded version died with 'cannot switch to a different thread' on the first click. + """ + def on_alarm(signum: int, frame: Any) -> None: + raise StepHang(f"call exceeded {timeout_s:.0f}s") + + prev = signal.signal(signal.SIGALRM, on_alarm) + signal.alarm(max(1, int(timeout_s))) + try: + return fn() + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, prev) + + +def classify(exc: BaseException) -> str: + """Separate a harness/browser failure from a policy failure so infra noise never scores as skill.""" + name = type(exc).__name__ + text = str(exc).lower() + if isinstance(exc, StepHang): + return "infra_step_hang" + if "timeout" in text or "Timeout" in name: + return "infra_timeout" + if "target" in text and "closed" in text: + return "infra_browser_closed" + if "connection" in text or "econnrefused" in text: + return "infra_connection" + if "rate" in text and "limit" in text: + return "llm_rate_limit" + return f"error_{name}" + + +def run_episode(arm: str, task: str, seed: int, rec: Recorder, args: argparse.Namespace) -> EpisodeRecord: + ep = EpisodeRecord(arm=arm, task=task, seed=seed, model=args.model or "", started_at=time.time()) + policy = policies.build(arm, model=args.model, endpoint=args.endpoint) + t_setup = time.time() + env = None + holder: list[Any] = [] + try: + # Setup under its own deadline: both 125-task sweeps once hung 30+ minutes in env + # creation/reset with the step watchdog never armed. holder keeps a half-built env + # reachable so the except path can close its browsers instead of leaking them. + def setup(): + # coord subset on for EVERY arm: canvas/slider/drag tasks are unphrasable in pure bid + # space, and the product this arena stands in for ships coordinate clicks (click_point). + from browsergym.core.action.highlevel import HighLevelActionSet + + acts = HighLevelActionSet(subsets=["chat", "bid", "coord", "infeas"], + strict=False, multiaction=False) + holder.append(gym.make(f"browsergym/miniwob.{task}", headless=not args.headed, + max_episode_steps=args.max_steps, wait_for_user_message=False, + action_mapping=acts.to_python_code)) + return holder[0].reset(seed=seed) + + obs, _ = with_deadline(setup, args.setup_timeout) + env = holder[0] + except Exception as exc: + ep.error_class = classify(exc) + ep.error_detail = f"{type(exc).__name__}: {exc}"[:200] + ep.setup_s = time.time() - t_setup + if holder: + try: + with_deadline(holder[0].close, 15) + except Exception: + pass + return ep + ep.setup_s = time.time() - t_setup + ep.goal = str(obs.get("goal") or "")[:300] + policy.reset(ep.goal) + + t0 = time.time() + try: + for step in range(1, args.max_steps + 1): + t_perc = time.time() + nodes, ax_chars = perception.axtree_stats(obs) + # The LLM call rides inside act(); urlopen's timeout does not cover every hang mode. + decision = with_deadline(lambda: policy.act(obs, ep.goal), args.step_timeout + 90) + perceive_ms = (time.time() - t_perc) * 1000 + rec_step = StepRecord( + step=step, action=decision.action, perceive_ms=perceive_ms, + think_ms=getattr(decision, "think_ms", 0.0), + axtree_chars=ax_chars, axtree_nodes=nodes, + dom_chars=perception.dom_chars(obs) if args.dom_metrics else 0, + n_interactive=decision.n_interactive, url=str(obs.get("url") or ""), + focused_bid=str(obs.get("focused_element_bid") or ""), + prompt_tokens=getattr(decision, "prompt_tokens", 0), + completion_tokens=getattr(decision, "completion_tokens", 0), + cost_usd=getattr(decision, "cost_usd", 0.0), + llm_error=getattr(decision, "llm_error", ""), + retries=getattr(decision, "retries", 0), + ) + if args.shots != "none": + dest = rec.shot_path(arm, task, seed, step) + rec_step.shot = save_screenshot(obs, dest) + if not decision.action: + rec_step.action_error = "policy produced no action" + ep.add(rec_step) + # A no-action turn caused by a dead LLM lane is the harness's problem, not the arm's. + if getattr(decision, "llm_error", ""): + ep.error_class = "infra_llm" + ep.error_detail = decision.llm_error[:200] + break + if ep.first_action_s == 0.0: + ep.first_action_s = time.time() - t0 + t_act = time.time() + obs, reward, terminated, truncated, _ = with_deadline( + lambda: env.step(decision.action), args.step_timeout) + rec_step.action_ms = (time.time() - t_act) * 1000 + rec_step.reward = float(reward or 0) + rec_step.action_error = str(obs.get("last_action_error") or "")[:200] + ep.add(rec_step) + ep.reward = max(ep.reward, float(reward or 0)) + ep.steps = step + if terminated or truncated: + ep.terminated, ep.truncated = bool(terminated), bool(truncated) + break + except Exception as exc: + ep.error_class = classify(exc) + ep.error_detail = f"{type(exc).__name__}: {exc}"[:200] + if args.trace: + traceback.print_exc() + ep.wall_s = time.time() - t0 + ep.success = ep.reward > 0 + try: + raw = with_deadline(lambda: env.unwrapped.page.evaluate( + "typeof WOB_RAW_REWARD_GLOBAL !== 'undefined' ? WOB_RAW_REWARD_GLOBAL : 0"), 10) + ep.raw_reward = float(raw or 0) + except Exception: + ep.raw_reward = ep.reward + # Final frame AFTER the last action, which is the only one that shows why an episode scored 0. + if args.shots != "none": + try: + with_deadline(lambda: env.unwrapped.page.screenshot( + path=str(rec.shot_path(arm, task, seed, 99))), 10) + except Exception: + pass + # close() can hang on the same wedged browser the step hung on; give it its own short leash. + try: + with_deadline(env.close, 15) + except Exception: + pass + return ep + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--arm", required=True) + ap.add_argument("--tasks", default="all") + ap.add_argument("--seeds", type=int, default=1) + ap.add_argument("--seed-base", type=int, default=42) + ap.add_argument("--max-steps", type=int, default=12) + ap.add_argument("--step-timeout", type=float, default=30.0) + ap.add_argument("--setup-timeout", type=float, default=60.0) + ap.add_argument("--shots", choices=["none", "first-last", "all"], default="first-last") + ap.add_argument("--dom-metrics", action="store_true") + ap.add_argument("--headed", action="store_true") + ap.add_argument("--trace", action="store_true") + ap.add_argument("--model", default=os.environ.get("OSW_ARENA_MODEL", "")) + ap.add_argument("--endpoint", default=os.environ.get("OSW_ARENA_ENDPOINT", "http://localhost:20128")) + ap.add_argument("--tag", default="") + ap.add_argument("--run-id", default="") + # Sharding, not threading: each shard owns its own browser, so one crash cannot poison the rest. + ap.add_argument("--shard", default="", help="i/n, e.g. 0/4 to run every 4th task") + args = ap.parse_args() + + tasks = resolve_tasks(args.tasks) + if args.shard: + i, n = (int(x) for x in args.shard.split("/")) + tasks = [t for k, t in enumerate(tasks) if k % n == i] + rec = Recorder(args.tag or args.arm, args.run_id) + print(f"arm={args.arm} tasks={len(tasks)} seeds={args.seeds} -> {rec.path}", flush=True) + wins = total = 0 + for task in tasks: + for s in range(args.seeds): + ep = run_episode(args.arm, task, args.seed_base + s, rec, args) + rec.write(ep) + total += 1 + wins += 1 if ep.success else 0 + flag = "OK " if ep.success else ("ERR" if ep.error_class else "-- ") + print(f" {flag} {task:28s} s={ep.seed} r={ep.reward:+.2f} steps={ep.steps:2d} " + f"{ep.wall_s:6.2f}s tok={ep.prompt_tokens + ep.completion_tokens:<6d} {ep.error_class}", + flush=True) + print(f"\n{args.arm}: {wins}/{total} = {100 * wins / total if total else 0:.1f}%", flush=True) + + +if __name__ == "__main__": + main() diff --git a/e2e/browser-v3/arena/sh_driver.mjs b/e2e/browser-v3/arena/sh_driver.mjs new file mode 100644 index 00000000..124efa0d --- /dev/null +++ b/e2e/browser-v3/arena/sh_driver.mjs @@ -0,0 +1,37 @@ +// Stagehand driver for one MiniWoB episode: attach to the already-seeded page over CDP, act, exit. +// Called by sh_real.py per episode; stdout's last line is a JSON result. Scoring stays in Python, +// read from MiniWoB's own globals -- this process never grades itself. +import { Stagehand } from '@browserbasehq/stagehand'; + +const [goal, cdpUrl, model, endpoint, maxSteps] = process.argv.slice(2); + +const sh = new Stagehand({ + env: 'LOCAL', + modelName: model, + modelClientOptions: { apiKey: 'arena', baseURL: `${endpoint}/v1` }, + verbose: 0, + disablePino: true, + localBrowserLaunchOptions: { cdpUrl }, +}); + +const out = { steps: 0, actions: [], errors: [], claimed: false }; +try { + await sh.init(); + // agent() is Stagehand's autonomous loop; per-act() would be MY orchestration, not theirs. + const agent = sh.agent({ + provider: 'anthropic', + model, + instructions: 'Work only on the current page. Never navigate away.', + options: { apiKey: 'arena', baseURL: `${endpoint}/v1` }, + }); + const result = await agent.execute({ instruction: goal, maxSteps: Number(maxSteps) || 12 }); + out.claimed = !!result?.success; + out.steps = result?.actions?.length ?? 0; + out.actions = (result?.actions ?? []).map((a) => JSON.stringify(a).slice(0, 120)); + out.usage = result?.usage ?? null; +} catch (e) { + out.errors.push(String(e).slice(0, 200)); +} finally { + try { await sh.close(); } catch {} +} +console.log('RESULT:' + JSON.stringify(out)); diff --git a/e2e/browser-v3/arena/sh_real.py b/e2e/browser-v3/arena/sh_real.py new file mode 100644 index 00000000..d6c8be50 --- /dev/null +++ b/e2e/browser-v3/arena/sh_real.py @@ -0,0 +1,123 @@ +"""Stagehand's agent on MiniWoB: same seeded page, same CDP attach, same external scorer as bu-real. + +The node driver does the acting; this file owns the env, the clock, the record, and the verdict. +""" +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from bu_real import CDP_PORT, make_env, score +from recorder import EpisodeRecord, Recorder, StepRecord +from tasks import resolve_tasks + +os.environ.setdefault("MINIWOB_URL", "http://localhost:8099/miniwob/") +DRIVER = Path(__file__).resolve().parent / "sh_driver.mjs" +NODE_DIR = os.environ.get("OSW_ARENA_SH_DIR", "") + + +def drive(goal: str, model: str, endpoint: str, max_steps: int, timeout_s: float) -> dict: + cmd = ["node", str(DRIVER), goal, f"http://localhost:{CDP_PORT}", model, endpoint, str(max_steps)] + try: + proc = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout_s, + cwd=NODE_DIR or None) + except subprocess.TimeoutExpired: + return {"errors": [f"driver exceeded {timeout_s:.0f}s"]} + for line in reversed(proc.stdout.splitlines()): + if line.startswith("RESULT:"): + try: + return json.loads(line[len("RESULT:"):]) + except ValueError: + break + return {"errors": [f"no RESULT line; rc={proc.returncode}; tail={proc.stdout[-200:]!r} {proc.stderr[-200:]!r}"]} + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--tasks", default="smoke") + ap.add_argument("--seeds", type=int, default=1) + ap.add_argument("--seed-base", type=int, default=42) + ap.add_argument("--max-steps", type=int, default=12) + ap.add_argument("--episode-timeout", type=float, default=180.0) + ap.add_argument("--model", default=os.environ.get("OSW_ARENA_MODEL", "cc/claude-sonnet-4-6")) + ap.add_argument("--endpoint", default=os.environ.get("OSW_ARENA_ENDPOINT", "http://localhost:20128")) + ap.add_argument("--shots", choices=["none", "first-last"], default="first-last") + ap.add_argument("--shard", default="") + args = ap.parse_args() + + tasks = resolve_tasks(args.tasks) + if args.shard: + i, n = (int(x) for x in args.shard.split("/")) + tasks = [t for k, t in enumerate(tasks) if k % n == i] + rec = Recorder("sh-real") + print(f"arm=sh-real tasks={len(tasks)} seeds={args.seeds} model={args.model} -> {rec.path}", flush=True) + wins = total = 0 + for task in tasks: + for s in range(args.seeds): + seed = args.seed_base + s + ep = EpisodeRecord(arm="sh-real", task=task, seed=seed, model=args.model, + started_at=time.time()) + t_setup = time.time() + env = None + try: + env, obs = make_env(task, seed, args.max_steps) + ep.goal = str(obs.get("goal") or "")[:300] + except Exception as exc: + ep.error_class = "infra_env_setup" + ep.error_detail = f"{type(exc).__name__}: {exc}"[:200] + ep.setup_s = time.time() - t_setup + if env is not None: + if args.shots != "none": + try: + env.unwrapped.page.screenshot(path=str(rec.shot_path("sh-real", task, seed, 1))) + except Exception: + pass + t0 = time.time() + stats = drive(ep.goal, args.model, args.endpoint, args.max_steps, args.episode_timeout) + ep.wall_s = time.time() - t0 + try: + ep.reward, ep.raw_reward = score(env) + except Exception as exc: + ep.error_class = "infra_score_readback" + ep.error_detail = f"{type(exc).__name__}: {exc}"[:200] + ep.success = ep.reward > 0 + ep.claimed_success = bool(stats.get("claimed")) + ep.steps = int(stats.get("steps") or 0) + usage = stats.get("usage") or {} + ep.prompt_tokens = int(usage.get("input_tokens") or 0) + ep.completion_tokens = int(usage.get("output_tokens") or 0) + ep.llm_calls = int(usage.get("inference_time_ms") is not None and ep.steps or ep.steps) + errs = stats.get("errors") or [] + if errs and not ep.success: + ep.error_detail = (ep.error_detail + " | " + "; ".join(errs))[:200].strip(" |") + for i, act in enumerate(stats.get("actions") or [], 1): + ep.step_records.append(StepRecord(step=i, action=str(act)[:120])) + if args.shots != "none": + try: + env.unwrapped.page.screenshot(path=str(rec.shot_path("sh-real", task, seed, 99))) + except Exception: + pass + try: + env.close() + except Exception: + pass + rec.write(ep) + total += 1 + wins += 1 if ep.success else 0 + flag = "OK " if ep.success else ("ERR" if ep.error_class else "-- ") + lie = " FALSE-SUCCESS" if ep.claimed_success and not ep.success else "" + print(f" {flag} {task:28s} s={seed} r={ep.reward:+.2f} steps={ep.steps:2d} " + f"{ep.wall_s:6.2f}s tok={ep.prompt_tokens + ep.completion_tokens:<7d} " + f"{ep.error_detail[:60]}{lie}", flush=True) + print(f"\nsh-real: {wins}/{total} = {100 * wins / total if total else 0:.1f}%", flush=True) + + +if __name__ == "__main__": + main() diff --git a/e2e/browser-v3/arena/supervisor.py b/e2e/browser-v3/arena/supervisor.py new file mode 100644 index 00000000..08784757 --- /dev/null +++ b/e2e/browser-v3/arena/supervisor.py @@ -0,0 +1,118 @@ +"""Out-of-process supervision for arena sweeps: launch, watch for stalls, kill, resume, repeat. + +In-process deadlines proved unreliable here twice -- SIGALRM's exception gets swallowed somewhere +inside playwright/gym retry loops and a sweep silently stops writing for half an hour. A supervisor +that watches the recorder file and kills the whole process tree is the only deadline the stack +cannot catch. Resume is free because the recorder is append-only and reports keep only the newest +episode per (arm, task, seed). + + python supervisor.py --arm osw-llm --model cc/claude-haiku-4-5-20251001 --stall 240 +""" +from __future__ import annotations + +import argparse +import json +import os +import signal +import subprocess +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from recorder import RESULTS_DIR +from tasks import ALL, resolve_tasks + +PY = sys.executable +ARENA = Path(__file__).resolve().parent + + +def completed_tasks(arm: str, model: str, seed: int, since: float) -> set[str]: + """Tasks with a usable (non-infra) episode for this arm+model+seed recorded after `since`.""" + path = RESULTS_DIR / f"{arm}.jsonl" + done: set[str] = set() + if not path.exists(): + return done + with open(path) as fh: + for line in fh: + try: + r = json.loads(line) + except ValueError: + continue + if (r.get("seed") == seed and r.get("started_at", 0) >= since + and r.get("model", "") in (model, "") + and not str(r.get("error_class", "")).startswith("infra")): + done.add(r["task"]) + return done + + +def spawn(arm: str, tasks: list[str], args: argparse.Namespace) -> subprocess.Popen: + if arm == "bu-real": + cmd = [PY, str(ARENA / "bu_real.py"), "--tasks", ",".join(tasks), "--seeds", "1", + "--seed-base", str(args.seed), "--model", args.model, + "--episode-timeout", str(args.episode_timeout)] + else: + cmd = [PY, str(ARENA / "run.py"), "--arm", arm, "--tasks", ",".join(tasks), "--seeds", "1", + "--seed-base", str(args.seed), "--model", args.model, "--shots", args.shots] + env = dict(os.environ) + env.setdefault("MINIWOB_URL", "http://localhost:8099/miniwob/") + env.setdefault("BROWSER_USE_LOGGING_LEVEL", "warning") + env.setdefault("ANONYMIZED_TELEMETRY", "false") + # Its own process group so a stall-kill takes the whole tree, chromiums included. + return subprocess.Popen(cmd, cwd=str(ARENA), env=env, start_new_session=True, + stdout=open(args.log, "ab"), stderr=subprocess.STDOUT) + + +def kill_tree(proc: subprocess.Popen) -> None: + try: + os.killpg(os.getpgid(proc.pid), signal.SIGKILL) + except (ProcessLookupError, PermissionError): + pass + subprocess.run(["pkill", "-f", "ms-playwright"], capture_output=True, check=False) + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--arm", required=True) + ap.add_argument("--model", required=True) + ap.add_argument("--tasks", default="all") + ap.add_argument("--seed", type=int, default=42) + ap.add_argument("--stall", type=float, default=240.0, help="kill if no new episode lands this long") + ap.add_argument("--rounds", type=int, default=12) + ap.add_argument("--episode-timeout", type=float, default=100.0) + ap.add_argument("--shots", default="first-last") + ap.add_argument("--log", default="") + # Resume window: episodes recorded after this epoch count as done, so a restarted supervisor + # keeps the finished work of the run it replaces instead of re-running all 125. + ap.add_argument("--since", type=float, default=0.0) + args = ap.parse_args() + args.log = args.log or f"/tmp/supervise_{args.arm}.log" + + wanted = set(resolve_tasks(args.tasks)) if args.tasks != "all" else set(ALL) + t_run = args.since or time.time() + path = RESULTS_DIR / f"{args.arm}.jsonl" + for rnd in range(1, args.rounds + 1): + remaining = sorted(wanted - completed_tasks(args.arm, args.model, args.seed, t_run)) + if not remaining: + break + print(f"[supervisor] round {rnd}: {len(remaining)} tasks remaining", flush=True) + proc = spawn(args.arm, remaining, args) + last_size = path.stat().st_size if path.exists() else 0 + last_change = time.time() + while proc.poll() is None: + time.sleep(10) + size = path.stat().st_size if path.exists() else 0 + if size != last_size: + last_size, last_change = size, time.time() + elif time.time() - last_change > args.stall: + print(f"[supervisor] stalled {args.stall:.0f}s; killing tree", flush=True) + kill_tree(proc) + break + time.sleep(2) + done = completed_tasks(args.arm, args.model, args.seed, t_run) + print(f"[supervisor] finished: {len(done)}/{len(wanted)} tasks have clean episodes", flush=True) + + +if __name__ == "__main__": + main() diff --git a/e2e/browser-v3/arena/tasks.py b/e2e/browser-v3/arena/tasks.py new file mode 100644 index 00000000..1e9fa4d5 --- /dev/null +++ b/e2e/browser-v3/arena/tasks.py @@ -0,0 +1,89 @@ +"""The 125 MiniWoB tasks, grouped by the capability each one actually exercises. + +Grouping matters more than the headline number: "we are better across the board" is a claim about +categories, and an arm that wins overall while losing every drag task has not earned that sentence. +Categories are assigned from the task's mechanics, not its name prefix. +""" +from __future__ import annotations + +CATEGORIES: dict[str, list[str]] = { + "click_basic": [ + "click-test", "click-test-2", "click-test-transfer", "click-button", + "click-button-sequence", "click-link", "click-dialog", "click-dialog-2", + "click-widget", "click-color", "click-shades", "click-shape", "identify-shape", + ], + "click_compound": [ + "click-checkboxes", "click-checkboxes-large", "click-checkboxes-soft", + "click-checkboxes-transfer", "click-collapsible", "click-collapsible-2", + "click-collapsible-nodelay", "click-collapsible-2-nodelay", "click-menu", + "click-menu-2", "click-option", "click-scroll-list", "click-tab", "click-tab-2", + "click-tab-2-easy", "click-tab-2-hard", "click-tab-2-medium", "navigate-tree", + "click-pie", "click-pie-nodelay", + ], + "text_entry": [ + "enter-text", "enter-text-2", "enter-text-dynamic", "enter-password", + "enter-date", "enter-time", "focus-text", "focus-text-2", "login-user", + "login-user-popup", "text-editor", "text-transform", "resize-textarea", + "unicode-test", "generate-number", "highlight-text", "highlight-text-2", + ], + "forms": [ + "book-flight", "book-flight-nodelay", "buy-ticket", "choose-date", + "choose-date-easy", "choose-date-medium", "choose-date-nodelay", "choose-list", + "form-sequence", "form-sequence-2", "form-sequence-3", "multi-layouts", + "multi-orderings", "order-food", "sign-agreement", "use-autocomplete", + "use-autocomplete-nodelay", "use-spinner", "search-engine", "social-media", + "social-media-all", "social-media-some", + ], + "reading": [ + "read-table", "read-table-2", "find-word", "phone-book", "scroll-text", + "scroll-text-2", "stock-market", "daily-calendar", "terminal", "copy-paste", + "copy-paste-2", "find-greatest", "odd-or-even", + ], + "email": [ + "email-inbox", "email-inbox-delete", "email-inbox-forward", "email-inbox-forward-nl", + "email-inbox-forward-nl-turk", "email-inbox-important", "email-inbox-nl-turk", + "email-inbox-noscroll", "email-inbox-reply", "email-inbox-star-reply", + ], + "drag": [ + "drag-box", "drag-circle", "drag-cube", "drag-items", "drag-items-grid", + "drag-shapes", "drag-shapes-2", "drag-single-shape", "drag-sort-numbers", + "use-slider", "use-slider-2", "use-colorwheel", "use-colorwheel-2", + ], + "spatial": [ + "bisect-angle", "circle-center", "count-shape", "count-sides", "draw-circle", + "draw-line", "find-midpoint", "grid-coordinate", "right-angle", "tic-tac-toe", + "number-checkboxes", "visual-addition", "ascending-numbers", + ], + "reasoning": [ + "simple-algebra", "simple-arithmetic", "guess-number", "hot-cold", + ], +} + +ALL: list[str] = sorted({t for group in CATEGORIES.values() for t in group}) + +CATEGORY_OF: dict[str, str] = {t: cat for cat, group in CATEGORIES.items() for t in group} + +# Small, cheap slice that still touches every category; used for iteration before a full sweep. +SMOKE: list[str] = [ + "click-test", "click-button", "click-link", "click-checkboxes", "click-tab", + "enter-text", "enter-password", "focus-text", "login-user", "enter-date", + "book-flight", "choose-list", "use-spinner", "search-engine", + "read-table", "find-word", "copy-paste", "email-inbox", "email-inbox-delete", + "use-slider", "drag-items", "grid-coordinate", "count-shape", "simple-algebra", +] + + +def resolve_tasks(spec: str) -> list[str]: + """Accept 'all', 'smoke', a category name, or a comma-separated list of task names.""" + spec = spec.strip() + if spec == "all": + return ALL + if spec == "smoke": + return SMOKE + if spec in CATEGORIES: + return sorted(CATEGORIES[spec]) + names = [s.strip() for s in spec.split(",") if s.strip()] + unknown = [n for n in names if n not in CATEGORY_OF] + if unknown: + raise SystemExit(f"unknown task(s): {', '.join(unknown)}") + return names