mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-08-17 18:25:42 +02:00
arena: every browser stack scored by MiniWoB itself, plus the first two ingest wins
125 tasks, one external grader (WOB_REWARD_GLOBAL through BrowserGym), one recorder book, agents never grade themselves. Arms: our shipped perception ported line-for-line (deterministic + LLM), a browser-use-shaped flat-dump arm on the same model, the real browser-use agent attached over CDP to the same seeded page, and Stagehand wiring. Techniques already ingested from the competitor diffs, each landing measured points: clickable-but-unroled elements (canvas/svg -- their flat dump solved spatial tasks purely because the canvas was in it), coordinate actions (our click_point equivalent), input twins exempt from dedupe (two same-named password boxes are two fields), select options rendered on the row, and the page-text panel (our BrowserGetText equivalent). v1 25.6% -> v2 32.0% on identical tasks/model, wall clock down 3.8s -> 3.2s. Supervision is out-of-process: SIGALRM exceptions got swallowed inside stack retry loops twice, so a watcher kills the process tree when the recorder stops moving and resumes exactly the missing tasks. 502-dead LLM lanes book as infra, never as skill. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WsbS5x2rYsMDxP2kW3qqmQ
This commit is contained in:
ciregenz
co-authored by
Claude Fable 5
Claude Fable 5
parent
830a995c6a
commit
dfe819b095
@@ -81,3 +81,4 @@ Thumbs.db
|
||||
ehthumbs.db
|
||||
desktop.ini
|
||||
frontend/tsconfig.tsbuildinfo
|
||||
e2e/browser-v3/arena/data/
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
# MiniWoB arena
|
||||
|
||||
Every browser stack we care about, scored on the same 125 MiniWoB tasks by MiniWoB itself.
|
||||
Nothing in this directory decides success; the reward comes from the task's own JS
|
||||
(`WOB_REWARD_GLOBAL`), read through BrowserGym. That is the property none of our other suites
|
||||
have, and it is why cross-stack claims ("ours is better/worse than X") should cite THIS data.
|
||||
|
||||
## Arms
|
||||
|
||||
| arm | what it is | entrypoint |
|
||||
|---|---|---|
|
||||
| `flat` | no-LLM floor: flat axtree, first label match | `run.py --arm flat` |
|
||||
| `openswarm` | no-LLM port of our shipped perception + action ladder | `run.py --arm openswarm` |
|
||||
| `bu` | LLM given a browser-use-shaped flat axtree dump | `run.py --arm bu` |
|
||||
| `osw-llm` | same LLM given our ranked/deduped/capped element menu | `run.py --arm osw-llm` |
|
||||
| `bu-real` | the actual browser-use agent, attached over CDP | `bu_real.py` |
|
||||
| `sh-real` | the actual Stagehand agent, attached over CDP | `sh_real.py` |
|
||||
|
||||
`bu` vs `osw-llm` is the controlled experiment (same model, same action layer, only the page view
|
||||
differs). `bu-real`/`sh-real` are the shipping competitors, whole-stack.
|
||||
|
||||
## Ground rules
|
||||
|
||||
- One recorder (`recorder.py`): every episode appends to `data/all.jsonl`; screenshots under
|
||||
`data/shots/<run>/<arm>/<task>-s<seed>/`. Reruns supersede by `started_at`; nothing is rewritten.
|
||||
- Agents never grade themselves. `claimed_success` vs `success` is recorded precisely to count
|
||||
false-success claims per arm.
|
||||
- Infra failures (`error_class` starting `infra`) are excluded from rates but always reported.
|
||||
- `ranking.py` must stay line-for-line with `frontend/src/shared/interactiveRanking.ts`; if either
|
||||
changes, change both.
|
||||
|
||||
## Running
|
||||
|
||||
```
|
||||
# serve MiniWoB HTML once: cd miniwob-plusplus/miniwob/html && python3 -m http.server 8099
|
||||
MINIWOB_URL=http://localhost:8099/miniwob/ python run.py --arm osw-llm --tasks all --seeds 1
|
||||
python report.py # scoreboard; --md ARENA.md for the markdown version
|
||||
python diffs.py --ours osw-llm --theirs bu-real # evidence trail for every loss
|
||||
```
|
||||
|
||||
Needs the browsergym venv (Python 3.12 — 3.13 cannot build greenlet 3.0.3).
|
||||
@@ -0,0 +1,262 @@
|
||||
"""The REAL browser-use agent on MiniWoB: their whole stack, our task instance, MiniWoB's scoring.
|
||||
|
||||
BrowserGym launches the chromium with a CDP port open; browser-use attaches to that same browser and
|
||||
drives the very page BrowserGym seeded. Reward is read from MiniWoB's own WOB_REWARD_GLOBAL through
|
||||
the BrowserGym handle, so browser-use's self-reported "done" never grades itself -- the lesson every
|
||||
prior verifier bug in this project taught twice.
|
||||
|
||||
Separate entrypoint from run.py because browser-use is asyncio all the way down and forcing it under
|
||||
run.py's sync SIGALRM loop would time THEIR stack with MY interruptions.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
import browsergym.miniwob # noqa: F401 registers the 125 envs
|
||||
import gymnasium as gym
|
||||
|
||||
from recorder import EpisodeRecord, Recorder, StepRecord
|
||||
from tasks import resolve_tasks
|
||||
|
||||
os.environ.setdefault("MINIWOB_URL", "http://localhost:8099/miniwob/")
|
||||
CDP_PORT = int(os.environ.get("OSW_ARENA_CDP_PORT", "9321"))
|
||||
|
||||
|
||||
def patch_launch_with_cdp_port() -> None:
|
||||
"""BrowserEnv passes args= itself, so pw_chromium_kwargs={'args': ...} raises; inject at launch.
|
||||
|
||||
Every launch gets its OWN port: BrowserGym starts a second chromium just for its chat window, and
|
||||
when both raced for one port the chat browser won it -- browser-use then spent 10 steps staring
|
||||
at the chat page's about:blank hunting for a Submit button that was in the other browser.
|
||||
"""
|
||||
from playwright.sync_api import BrowserType
|
||||
|
||||
if getattr(BrowserType, "osw_arena_patched", False):
|
||||
return
|
||||
orig = BrowserType.launch
|
||||
|
||||
def launch(self, **kwargs):
|
||||
# A genuinely-free port each launch: fixed pools deadlocked -- a zombie chromium from a dead
|
||||
# episode kept its port bound and the next launch hung 180s failing to bind the same one.
|
||||
import socket
|
||||
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
port = sock.getsockname()[1]
|
||||
RECENT_PORTS.append(port)
|
||||
del RECENT_PORTS[:-8]
|
||||
kwargs["args"] = list(kwargs.get("args") or []) + [f"--remote-debugging-port={port}"]
|
||||
return orig(self, **kwargs)
|
||||
|
||||
BrowserType.launch = launch
|
||||
BrowserType.osw_arena_patched = True
|
||||
|
||||
|
||||
# Ports handed to recent launches, newest last; the probe below only ever looks here.
|
||||
RECENT_PORTS: list[int] = []
|
||||
|
||||
|
||||
def find_task_cdp_url() -> str:
|
||||
"""Probe the recently-issued ports for the browser actually hosting the MiniWoB page."""
|
||||
import urllib.request
|
||||
|
||||
for port in reversed(RECENT_PORTS):
|
||||
try:
|
||||
with urllib.request.urlopen(f"http://localhost:{port}/json", timeout=2) as resp:
|
||||
targets = json.loads(resp.read().decode())
|
||||
except Exception:
|
||||
continue
|
||||
if any("miniwob" in str(t.get("url", "")) for t in targets):
|
||||
return f"http://localhost:{port}"
|
||||
raise RuntimeError(f"none of the recent CDP ports {RECENT_PORTS} hosts a miniwob page")
|
||||
|
||||
|
||||
def reap_leftover_browsers() -> None:
|
||||
"""Kill any debug-port chromium THIS process launched that outlived its episode; zombies wedge
|
||||
later launches. Per-port patterns, not a blanket sweep, so concurrent shards and other arms'
|
||||
portless chromiums are untouchable by construction."""
|
||||
import subprocess
|
||||
|
||||
for port in RECENT_PORTS:
|
||||
subprocess.run(["pkill", "-f", f"ms-playwright.*remote-debugging-port={port}$"],
|
||||
capture_output=True, check=False)
|
||||
subprocess.run(["pkill", "-f", f"ms-playwright.*remote-debugging-port={port} "],
|
||||
capture_output=True, check=False)
|
||||
|
||||
|
||||
def make_env(task: str, seed: int, max_steps: int):
|
||||
patch_launch_with_cdp_port()
|
||||
env = gym.make(f"browsergym/miniwob.{task}", headless=True, max_episode_steps=max_steps)
|
||||
obs, _ = env.reset(seed=seed)
|
||||
return env, obs
|
||||
|
||||
|
||||
async def drive(goal: str, cdp_url: str, model: str, endpoint: str, max_steps: int, timeout_s: float) -> dict:
|
||||
"""Run browser-use's own Agent loop against the already-open task page."""
|
||||
from browser_use import Agent, Browser, ChatOpenAI
|
||||
|
||||
browser = Browser(cdp_url=cdp_url, is_local=False)
|
||||
llm = ChatOpenAI(model=model, base_url=f"{endpoint}/v1", api_key="arena", temperature=None)
|
||||
# MiniWoB pages never navigate, so the task prompt forbids goto -- their agent otherwise likes to
|
||||
# open about:blank or a search engine, which would abandon the scored page.
|
||||
agent = Agent(
|
||||
task=f"{goal}\nWork ONLY on the currently open page. Never navigate to another URL.",
|
||||
llm=llm, browser=browser, calculate_cost=True,
|
||||
)
|
||||
|
||||
async def run_agent():
|
||||
# Everything of theirs -- connect, session setup, the loop -- inside ONE deadline. Wrapping
|
||||
# only agent.run let a wedged phase outside it stretch an episode to 1467s.
|
||||
return await agent.run(max_steps=max_steps)
|
||||
|
||||
stats = {"llm_calls": 0, "prompt_tokens": 0, "completion_tokens": 0, "actions": []}
|
||||
try:
|
||||
history = await asyncio.wait_for(run_agent(), timeout=timeout_s)
|
||||
stats["actions"] = [str(a)[:120] for a in history.action_names()]
|
||||
try:
|
||||
stats["claimed_success"] = bool(history.is_successful())
|
||||
except Exception:
|
||||
stats["claimed_success"] = False
|
||||
usage = getattr(history, "usage", None)
|
||||
if usage:
|
||||
stats["prompt_tokens"] = int(getattr(usage, "total_prompt_tokens", 0) or 0)
|
||||
stats["completion_tokens"] = int(getattr(usage, "total_completion_tokens", 0) or 0)
|
||||
stats["llm_calls"] = int(getattr(usage, "total_calls", len(stats["actions"])) or 0)
|
||||
except asyncio.TimeoutError:
|
||||
stats["error"] = f"agent.run exceeded {timeout_s:.0f}s"
|
||||
except Exception as exc:
|
||||
stats["error"] = f"{type(exc).__name__}: {exc}"[:200]
|
||||
finally:
|
||||
try:
|
||||
await asyncio.wait_for(browser.stop(), timeout=10)
|
||||
except Exception:
|
||||
pass
|
||||
return stats
|
||||
|
||||
|
||||
def drive_in_thread(goal: str, cdp_url: str, model: str, endpoint: str, max_steps: int, timeout_s: float) -> dict:
|
||||
"""Own thread, own event loop: sync-Playwright's greenlet already occupies this thread's loop,
|
||||
so asyncio.run() here raises 'cannot be called from a running event loop'. browser-use only
|
||||
touches the browser over CDP, so it needs nothing from this thread's Playwright state."""
|
||||
import threading
|
||||
|
||||
result: dict = {}
|
||||
|
||||
def runner() -> None:
|
||||
try:
|
||||
result.update(asyncio.run(drive(goal, cdp_url, model, endpoint, max_steps, timeout_s)))
|
||||
except Exception as exc:
|
||||
result["error"] = f"{type(exc).__name__}: {exc}"[:200]
|
||||
|
||||
t = threading.Thread(target=runner, daemon=True)
|
||||
t.start()
|
||||
t.join(timeout=timeout_s + 30)
|
||||
if t.is_alive():
|
||||
result.setdefault("error", f"driver thread still alive after {timeout_s + 30:.0f}s")
|
||||
return result
|
||||
|
||||
|
||||
def score(env) -> tuple[float, float]:
|
||||
"""MiniWoB's verdict, read straight off the page globals -- the only grader in this file."""
|
||||
page = env.unwrapped.page
|
||||
reward = float(page.evaluate("typeof WOB_REWARD_GLOBAL !== 'undefined' ? WOB_REWARD_GLOBAL : 0") or 0)
|
||||
raw = float(page.evaluate("typeof WOB_RAW_REWARD_GLOBAL !== 'undefined' ? WOB_RAW_REWARD_GLOBAL : 0") or 0)
|
||||
return reward, raw
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--tasks", default="smoke")
|
||||
ap.add_argument("--seeds", type=int, default=1)
|
||||
ap.add_argument("--seed-base", type=int, default=42)
|
||||
ap.add_argument("--max-steps", type=int, default=12)
|
||||
ap.add_argument("--episode-timeout", type=float, default=180.0)
|
||||
ap.add_argument("--model", default=os.environ.get("OSW_ARENA_MODEL", "cc/claude-sonnet-4-6"))
|
||||
ap.add_argument("--endpoint", default=os.environ.get("OSW_ARENA_ENDPOINT", "http://localhost:20128"))
|
||||
ap.add_argument("--shots", choices=["none", "first-last"], default="first-last")
|
||||
ap.add_argument("--shard", default="")
|
||||
args = ap.parse_args()
|
||||
|
||||
tasks = resolve_tasks(args.tasks)
|
||||
if args.shard:
|
||||
i, n = (int(x) for x in args.shard.split("/"))
|
||||
tasks = [t for k, t in enumerate(tasks) if k % n == i]
|
||||
rec = Recorder("bu-real")
|
||||
print(f"arm=bu-real tasks={len(tasks)} seeds={args.seeds} model={args.model} -> {rec.path}", flush=True)
|
||||
wins = total = 0
|
||||
for task in tasks:
|
||||
for s in range(args.seeds):
|
||||
seed = args.seed_base + s
|
||||
ep = EpisodeRecord(arm="bu-real", task=task, seed=seed, model=args.model,
|
||||
started_at=time.time())
|
||||
t_setup = time.time()
|
||||
env = None
|
||||
try:
|
||||
env, obs = make_env(task, seed, args.max_steps)
|
||||
ep.goal = str(obs.get("goal") or "")[:300]
|
||||
except Exception as exc:
|
||||
ep.error_class = "infra_env_setup"
|
||||
ep.error_detail = f"{type(exc).__name__}: {exc}"[:200]
|
||||
ep.setup_s = time.time() - t_setup
|
||||
if env is not None:
|
||||
if args.shots != "none":
|
||||
try:
|
||||
env.unwrapped.page.screenshot(path=str(rec.shot_path("bu-real", task, seed, 1)))
|
||||
except Exception:
|
||||
pass
|
||||
t0 = time.time()
|
||||
try:
|
||||
cdp_url = find_task_cdp_url()
|
||||
except Exception as exc:
|
||||
cdp_url = ""
|
||||
ep.error_class = "infra_cdp_probe"
|
||||
ep.error_detail = str(exc)[:200]
|
||||
stats = drive_in_thread(ep.goal, cdp_url, args.model, args.endpoint,
|
||||
args.max_steps, args.episode_timeout) if cdp_url else {}
|
||||
ep.wall_s = time.time() - t0
|
||||
try:
|
||||
ep.reward, ep.raw_reward = score(env)
|
||||
except Exception as exc:
|
||||
ep.error_class = ep.error_class or "infra_score_readback"
|
||||
ep.error_detail = f"{type(exc).__name__}: {exc}"[:200]
|
||||
ep.success = ep.reward > 0
|
||||
ep.claimed_success = bool(stats.get("claimed_success"))
|
||||
ep.steps = len(stats.get("actions") or [])
|
||||
ep.prompt_tokens = stats.get("prompt_tokens", 0)
|
||||
ep.completion_tokens = stats.get("completion_tokens", 0)
|
||||
ep.llm_calls = stats.get("llm_calls", 0)
|
||||
if stats.get("error") and not ep.success:
|
||||
ep.error_detail = (ep.error_detail + " | " + stats["error"])[:200].strip(" |")
|
||||
for i, act in enumerate(stats.get("actions") or [], 1):
|
||||
ep.step_records.append(StepRecord(step=i, action=act))
|
||||
if args.shots != "none":
|
||||
try:
|
||||
env.unwrapped.page.screenshot(path=str(rec.shot_path("bu-real", task, seed, 99)))
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
env.close()
|
||||
except Exception:
|
||||
pass
|
||||
reap_leftover_browsers()
|
||||
rec.write(ep)
|
||||
total += 1
|
||||
wins += 1 if ep.success else 0
|
||||
flag = "OK " if ep.success else ("ERR" if ep.error_class else "-- ")
|
||||
lie = " FALSE-SUCCESS" if ep.claimed_success and not ep.success else ""
|
||||
print(f" {flag} {task:28s} s={seed} r={ep.reward:+.2f} steps={ep.steps:2d} "
|
||||
f"{ep.wall_s:6.2f}s tok={ep.prompt_tokens + ep.completion_tokens:<7d} "
|
||||
f"{ep.error_detail[:60]}{lie}", flush=True)
|
||||
print(f"\nbu-real: {wins}/{total} = {100 * wins / total if total else 0:.1f}%", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Where each competitor beats us, task by task, with the evidence trail for each loss.
|
||||
|
||||
This is the input to the ingest-and-iterate loop: for every (task, seed) where a competitor arm
|
||||
succeeded and ours failed, print our step trace, their action list, and both screenshot paths, so
|
||||
"what do they do better" is answered from recorded evidence rather than from impressions.
|
||||
|
||||
python diffs.py --ours osw-llm --theirs bu-real
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from recorder import load_episodes
|
||||
from report import latest_per_key
|
||||
from tasks import CATEGORY_OF
|
||||
|
||||
|
||||
def pick(eps: list[dict], arm: str) -> dict[tuple[str, int], dict]:
|
||||
return {(e["task"], e["seed"]): e for e in eps if e["arm"] == arm}
|
||||
|
||||
|
||||
def trace(ep: dict, limit: int = 8) -> list[str]:
|
||||
out = []
|
||||
for s in (ep.get("step_records") or [])[:limit]:
|
||||
err = f" ERR:{s['action_error'][:60]}" if s.get("action_error") else ""
|
||||
out.append(f" {s['step']:2d}. {s.get('action', '')[:80]}{err}")
|
||||
shots = [s["shot"] for s in ep.get("step_records") or [] if s.get("shot")]
|
||||
if shots:
|
||||
out.append(f" shots: {shots[0]} .. {Path(shots[-1]).parent}/99.png")
|
||||
return out
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--ours", default="osw-llm")
|
||||
ap.add_argument("--theirs", default="bu-real")
|
||||
ap.add_argument("--tag", default="all")
|
||||
ap.add_argument("--show-ours-wins", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
eps = latest_per_key(load_episodes(args.tag))
|
||||
ours, theirs = pick(eps, args.ours), pick(eps, args.theirs)
|
||||
common = sorted(set(ours) & set(theirs))
|
||||
if not common:
|
||||
raise SystemExit(f"no common (task, seed) pairs between {args.ours} and {args.theirs}")
|
||||
|
||||
they_beat_us = [k for k in common if theirs[k].get("success") and not ours[k].get("success")]
|
||||
we_beat_them = [k for k in common if ours[k].get("success") and not theirs[k].get("success")]
|
||||
both = sum(1 for k in common if ours[k].get("success") and theirs[k].get("success"))
|
||||
neither = sum(1 for k in common if not ours[k].get("success") and not theirs[k].get("success"))
|
||||
|
||||
print(f"common episodes: {len(common)} both-solve: {both} neither: {neither}")
|
||||
print(f"{args.theirs} beats {args.ours}: {len(they_beat_us)} "
|
||||
f"{args.ours} beats {args.theirs}: {len(we_beat_them)}\n")
|
||||
|
||||
print(f"=== {args.theirs} solved, {args.ours} failed ===")
|
||||
for task, seed in they_beat_us:
|
||||
o, t = ours[(task, seed)], theirs[(task, seed)]
|
||||
print(f"\n{task} (s={seed}, {CATEGORY_OF.get(task, '?')}) goal: {o.get('goal', '')[:90]}")
|
||||
print(f" ours ({o['steps']} steps, {o['wall_s']:.1f}s, err={o.get('error_class') or '-'}):")
|
||||
print("\n".join(trace(o)))
|
||||
print(f" theirs ({t['steps']} steps, {t['wall_s']:.1f}s):")
|
||||
print("\n".join(trace(t)))
|
||||
|
||||
if args.show_ours_wins:
|
||||
print(f"\n=== {args.ours} solved, {args.theirs} failed ===")
|
||||
for task, seed in we_beat_them:
|
||||
t = theirs[(task, seed)]
|
||||
print(f"{task} (s={seed}) their steps={t['steps']} wall={t['wall_s']:.1f}s "
|
||||
f"claimed={t.get('claimed_success')} err={t.get('error_detail', '')[:60]}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,245 @@
|
||||
"""Model-driven arms. Same model, same steps, same action space, same scorer -- only the page view differs.
|
||||
|
||||
That constraint is the entire experiment. `bu` renders the page the way browser-use does (flat
|
||||
accessibility dump, every node, no memory between turns); `osw` renders it the way OpenSwarm's
|
||||
BrowserListInteractives does (deduped, goal-ranked, capped at 60, ctx on twins, value on inputs,
|
||||
`*` on anything new since the last look). Any score gap between them is attributable to that view.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import perception
|
||||
from policies import Decision
|
||||
from ranking import RankItem, rank_and_cap, render
|
||||
|
||||
ACTIONS = """click(bid) | dblclick(bid) | fill(bid, "text") | clear(bid) | select_option(bid, "opt")
|
||||
hover(bid) | focus(bid) | press(bid, "key") | scroll(dx, dy) | drag_and_drop(from_bid, to_bid)
|
||||
mouse_click(x, y) | mouse_drag_and_drop(from_x, from_y, to_x, to_y) (viewport coordinates)"""
|
||||
|
||||
BU_SYSTEM = """You are a web agent. You are given the accessibility tree of a page and a goal.
|
||||
Respond with EXACTLY ONE action call and nothing else. No prose, no markdown, no explanation.
|
||||
Available actions:
|
||||
""" + ACTIONS
|
||||
|
||||
OSW_SYSTEM = """You are OpenSwarm's browser agent. You see a ranked, deduplicated list of the page's
|
||||
interactive elements. Each row is [index]<role "name" value="...">. A `*` means the element is new
|
||||
since your last look; the same index always means the same element. Rows without a useful name show
|
||||
center=(x,y) viewport coordinates.
|
||||
Respond with EXACTLY ONE action call and nothing else. No prose, no markdown, no explanation.
|
||||
Available actions, addressing elements by their [index]:
|
||||
click(index) | fill(index, "text") | select_option(index, "opt") | press(index, "key")
|
||||
clear(index) | hover(index) | focus(index) | scroll(dx, dy) | drag_and_drop(from_index, to_index)
|
||||
For targets with no element of their own (a spot on a canvas, a slider position), use coordinates:
|
||||
mouse_click(x, y) | mouse_dblclick(x, y) | mouse_drag_and_drop(from_x, from_y, to_x, to_y)
|
||||
For a combobox/listbox row showing options="...", pick with select_option(index, "exact option").
|
||||
Prefer filling the box the goal names, then submitting. Do not repeat an action that already worked.
|
||||
If an action did not change the page, try a DIFFERENT action, never the same one again."""
|
||||
|
||||
|
||||
def post_json(url: str, payload: dict[str, Any], timeout: float = 90.0) -> dict[str, Any]:
|
||||
body = json.dumps(payload).encode()
|
||||
req = urllib.request.Request(url, data=body, method="POST",
|
||||
headers={"Content-Type": "application/json",
|
||||
"Authorization": "Bearer arena"})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return json.loads(resp.read().decode())
|
||||
|
||||
|
||||
@dataclass
|
||||
class LlmDecision(Decision):
|
||||
"""Decision plus the per-call accounting the recorder folds into episode totals."""
|
||||
|
||||
think_ms: float = 0.0
|
||||
prompt_tokens: int = 0
|
||||
completion_tokens: int = 0
|
||||
cost_usd: float = 0.0
|
||||
llm_error: str = ""
|
||||
retries: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class LlmPolicy:
|
||||
"""Shared model loop; subclasses supply the page view and the action translation."""
|
||||
|
||||
name: str = "llm"
|
||||
model: str = "cc/claude-sonnet-4-6"
|
||||
endpoint: str = "http://localhost:20128"
|
||||
system: str = BU_SYSTEM
|
||||
history: list[str] = field(default_factory=list)
|
||||
max_history: int = 6
|
||||
|
||||
def reset(self, goal: str) -> None:
|
||||
self.history = []
|
||||
|
||||
def view(self, obs: dict[str, Any], goal: str) -> tuple[str, int]:
|
||||
raise NotImplementedError
|
||||
|
||||
def translate(self, raw: str) -> str:
|
||||
return raw
|
||||
|
||||
def note(self, action: str, obs: dict[str, Any]) -> None:
|
||||
"""History is kept in the MODEL's own namespace, with the outcome the page reported.
|
||||
|
||||
The first version logged the translated bid call, so the OpenSwarm arm was shown history it
|
||||
could not line up against the indices in its own element list and re-clicked the same box
|
||||
five times. Feeding an agent a memory it cannot read is a harness bug, not an agent failure.
|
||||
"""
|
||||
err = str(obs.get("last_action_error") or "").strip()
|
||||
self.history.append(f"{action} -> {'ERROR: ' + err[:120] if err else 'ok'}")
|
||||
|
||||
def call(self, goal: str, page: str) -> tuple[str, LlmDecision]:
|
||||
past = "\n".join(self.history[-self.max_history:]) or "(none yet)"
|
||||
user = f"GOAL: {goal}\n\nACTIONS YOU ALREADY TOOK:\n{past}\n\nPAGE:\n{page}\n\nYour single next action:"
|
||||
t0 = time.time()
|
||||
d = LlmDecision(action="")
|
||||
text = ""
|
||||
# Retry transient router faults: a concurrent-sweep run lost 40% of its episodes to 502s that
|
||||
# were then booked as policy failures. Retries make the residue rare; the classifier below
|
||||
# books what remains as infra, never as skill.
|
||||
for attempt in range(3):
|
||||
try:
|
||||
resp = post_json(f"{self.endpoint}/v1/chat/completions", {
|
||||
"model": self.model,
|
||||
"messages": [{"role": "system", "content": self.system},
|
||||
{"role": "user", "content": user}],
|
||||
"max_tokens": 200,
|
||||
"stream": False, # the router streams by default; a single action needs no SSE
|
||||
})
|
||||
text = (resp.get("choices") or [{}])[0].get("message", {}).get("content") or ""
|
||||
usage = resp.get("usage") or {}
|
||||
d.prompt_tokens = int(usage.get("prompt_tokens") or 0)
|
||||
d.completion_tokens = int(usage.get("completion_tokens") or 0)
|
||||
d.llm_error = ""
|
||||
d.retries = attempt
|
||||
break
|
||||
except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, ValueError) as exc:
|
||||
d.llm_error = f"{type(exc).__name__}: {exc}"[:150]
|
||||
time.sleep(2.0 * (attempt + 1))
|
||||
d.think_ms = (time.time() - t0) * 1000
|
||||
return str(text).strip(), d
|
||||
|
||||
|
||||
@dataclass
|
||||
class BrowserUseStylePolicy(LlmPolicy):
|
||||
"""Flat accessibility dump, exactly what a browser-use-shaped agent puts in front of a model."""
|
||||
|
||||
name: str = "bu"
|
||||
system: str = BU_SYSTEM
|
||||
|
||||
def view(self, obs: dict[str, Any], goal: str) -> tuple[str, int]:
|
||||
from browsergym.utils.obs import flatten_axtree_to_str
|
||||
|
||||
text = flatten_axtree_to_str(obs.get("axtree_object") or {})
|
||||
n = len(perception.interactives(obs))
|
||||
return text[:14000], n
|
||||
|
||||
def act(self, obs: dict[str, Any], goal: str) -> LlmDecision:
|
||||
page, n = self.view(obs, goal)
|
||||
raw, d = self.call(goal, page)
|
||||
d.n_interactive = n
|
||||
d.action = clean_action(raw)
|
||||
if d.action:
|
||||
self.note(d.action, obs)
|
||||
return d
|
||||
|
||||
|
||||
@dataclass
|
||||
class OpenSwarmLlmPolicy(LlmPolicy):
|
||||
"""OpenSwarm's ranked element menu, addressed by stable 1-based index instead of raw bid."""
|
||||
|
||||
name: str = "osw-llm"
|
||||
system: str = OSW_SYSTEM
|
||||
index_to_bid: dict[int, str] = field(default_factory=dict)
|
||||
prev_bids: set[str] = field(default_factory=set)
|
||||
# v2: also list clickable-but-unroled elements (canvas/svg/div) -- ingested from browser-use.
|
||||
clickable: bool = False
|
||||
# v3: append the page's visible text, our BrowserGetText equivalent; menu-only lost every task
|
||||
# whose payload lives in prose (the algebra equation, which email is Cecile's).
|
||||
with_text: bool = False
|
||||
|
||||
def reset(self, goal: str) -> None:
|
||||
self.history = []
|
||||
self.index_to_bid = {}
|
||||
self.prev_bids = set()
|
||||
|
||||
def view(self, obs: dict[str, Any], goal: str) -> tuple[str, int]:
|
||||
raw_items: list[RankItem] = perception.interactives(obs, include_clickable=self.clickable)
|
||||
shown, truncated = rank_and_cap(raw_items, goal=goal)
|
||||
new = {it.bid for it in shown} - self.prev_bids if self.prev_bids else set()
|
||||
self.prev_bids = {it.bid for it in shown}
|
||||
self.index_to_bid = {i: it.bid for i, it in enumerate(shown, 1)}
|
||||
view = render(shown, truncated, new)
|
||||
if self.with_text:
|
||||
text = perception.page_text(obs)
|
||||
if text:
|
||||
view += f"\n\nPAGE TEXT:\n{text}"
|
||||
return view, len(shown)
|
||||
|
||||
def translate(self, call: str) -> str:
|
||||
"""Swap our 1-based indices back to bids so both arms hit the identical action layer."""
|
||||
m = re.match(r"(\w+)\s*\((.*)\)\s*$", call, re.S)
|
||||
if not m:
|
||||
return call
|
||||
fn, argstr = m.group(1), m.group(2)
|
||||
# scroll deltas and mouse_*/keyboard_* coordinates are geometry, never element handles.
|
||||
if fn == "scroll" or fn.startswith(("mouse_", "keyboard_")):
|
||||
return call
|
||||
# Only leading positional args are element handles, so rewrite those and leave payloads alone.
|
||||
# Quoted digits count too: the model addresses indices, so click("3") means row 3, never bid 3.
|
||||
n_handles = 2 if fn == "drag_and_drop" else 1
|
||||
parts = [a.strip() for a in re.split(r',(?=(?:[^"]*"[^"]*")*[^"]*$)', argstr)] if argstr.strip() else []
|
||||
for i in range(min(n_handles, len(parts))):
|
||||
m2 = re.fullmatch(r'"?(\d+)"?', parts[i])
|
||||
if m2:
|
||||
bid = self.index_to_bid.get(int(m2.group(1)))
|
||||
if bid:
|
||||
parts[i] = f'"{bid}"'
|
||||
return f"{fn}({', '.join(parts)})"
|
||||
|
||||
def act(self, obs: dict[str, Any], goal: str) -> LlmDecision:
|
||||
page, n = self.view(obs, goal)
|
||||
raw, d = self.call(goal, page)
|
||||
d.n_interactive = n
|
||||
chosen = clean_action(raw)
|
||||
d.action = self.translate(chosen)
|
||||
if chosen:
|
||||
self.note(chosen, obs)
|
||||
return d
|
||||
|
||||
|
||||
CALL_RE = re.compile(
|
||||
r"\b(click|dblclick|fill|clear|select_option|hover|focus|press|scroll|drag_and_drop|noop"
|
||||
r"|mouse_click|mouse_dblclick|mouse_move|mouse_drag_and_drop|keyboard_type|keyboard_press)\s*\([^)]*\)")
|
||||
|
||||
|
||||
def clean_action(raw: str) -> str:
|
||||
"""Pull the one action call out of whatever the model wrapped it in; empty means unparseable."""
|
||||
if not raw:
|
||||
return ""
|
||||
text = raw.strip().strip("`")
|
||||
text = re.sub(r"^(python|json|tool_code)\s*", "", text)
|
||||
m = CALL_RE.search(text)
|
||||
return m.group(0) if m else ""
|
||||
|
||||
|
||||
def build(name: str, model: str = "", endpoint: str = "", **_: Any) -> Any:
|
||||
model = model or os.environ.get("OSW_ARENA_MODEL", "cc/claude-sonnet-4-6")
|
||||
endpoint = endpoint or os.environ.get("OSW_ARENA_ENDPOINT", "http://localhost:20128")
|
||||
if name in ("bu", "bu-llm", "browseruse-style"):
|
||||
return BrowserUseStylePolicy(model=model, endpoint=endpoint)
|
||||
if name in ("osw-llm", "openswarm-llm"):
|
||||
return OpenSwarmLlmPolicy(model=model, endpoint=endpoint)
|
||||
if name == "osw-llm-v2":
|
||||
return OpenSwarmLlmPolicy(name=name, model=model, endpoint=endpoint, clickable=True)
|
||||
if name == "osw-llm-v3":
|
||||
return OpenSwarmLlmPolicy(name=name, model=model, endpoint=endpoint,
|
||||
clickable=True, with_text=True)
|
||||
raise SystemExit(f"unknown arm: {name}")
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Turn a BrowserGym observation into the element list each arm is allowed to see.
|
||||
|
||||
Both arms read the SAME underlying accessibility tree, which is the point: the comparison is between
|
||||
what each stack does with the tree (browser-use dumps it flat, OpenSwarm dedupes/ranks/caps/marks it),
|
||||
not between two different ways of getting one. Anything that advantaged one arm's raw perception
|
||||
would make the score a measurement of plumbing.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from ranking import INTERACTIVE_ROLES, RankItem
|
||||
|
||||
# Roles carrying page copy; used to build the ctx string that disambiguates same-named twins.
|
||||
TEXT_ROLES = {"StaticText", "LabelText", "heading", "paragraph", "InlineTextBox"}
|
||||
|
||||
|
||||
def node_role(node: dict[str, Any]) -> str:
|
||||
return str((node.get("role") or {}).get("value") or "")
|
||||
|
||||
|
||||
def node_name(node: dict[str, Any]) -> str:
|
||||
return str((node.get("name") or {}).get("value") or "")
|
||||
|
||||
|
||||
def node_value(node: dict[str, Any]) -> str:
|
||||
v = node.get("value") or {}
|
||||
return str(v.get("value") or "") if isinstance(v, dict) else ""
|
||||
|
||||
|
||||
def visible(bid: str, extra: dict[str, Any], threshold: float = 0.5) -> bool:
|
||||
"""Our dropCoveredElements analogue: an element under an overlay is not an element you can click."""
|
||||
props = extra.get(bid) if extra else None
|
||||
if not props:
|
||||
return True
|
||||
try:
|
||||
return float(props.get("visibility", 1.0)) >= threshold
|
||||
except (TypeError, ValueError):
|
||||
return True
|
||||
|
||||
|
||||
def build_context(nodes: list[dict[str, Any]], by_id: dict[str, dict[str, Any]],
|
||||
node: dict[str, Any], depth: int = 3) -> str:
|
||||
"""Nearest ancestor's text, so five identical 'Message' buttons say which card they belong to."""
|
||||
cur = node
|
||||
for _ in range(depth):
|
||||
parent_id = cur.get("parentId")
|
||||
if not parent_id or parent_id not in by_id:
|
||||
return ""
|
||||
parent = by_id[parent_id]
|
||||
texts: list[str] = []
|
||||
for cid in parent.get("childIds") or []:
|
||||
child = by_id.get(cid)
|
||||
if not child or child is node:
|
||||
continue
|
||||
if node_role(child) in TEXT_ROLES:
|
||||
t = node_name(child).strip()
|
||||
if t:
|
||||
texts.append(t)
|
||||
if texts:
|
||||
return " ".join(texts)[:60]
|
||||
cur = parent
|
||||
return ""
|
||||
|
||||
|
||||
def interactives(obs: dict[str, Any], include_hidden: bool = False,
|
||||
include_clickable: bool = False) -> list[RankItem]:
|
||||
"""Every actionable node in document order, before any ranking or capping is applied.
|
||||
|
||||
include_clickable is the technique ingested from browser-use: elements the page wires for
|
||||
clicks but gives no interactive AX role -- canvases, SVGs, styled divs. Measured on MiniWoB,
|
||||
their flat dump solved spatial tasks (circle-center, bisect-angle) purely because the canvas
|
||||
appeared in it while our role-filtered menu hid the only thing worth clicking.
|
||||
"""
|
||||
ax = obs.get("axtree_object") or {}
|
||||
nodes: list[dict[str, Any]] = ax.get("nodes") or []
|
||||
extra = obs.get("extra_element_properties") or {}
|
||||
by_id = {n["nodeId"]: n for n in nodes if "nodeId" in n}
|
||||
out: list[RankItem] = []
|
||||
for n in nodes:
|
||||
if n.get("ignored"):
|
||||
continue
|
||||
role = node_role(n)
|
||||
bid = n.get("browsergym_id")
|
||||
if not bid:
|
||||
continue
|
||||
is_role = role in INTERACTIVE_ROLES
|
||||
is_clickable = (include_clickable and not is_role and role not in TEXT_ROLES
|
||||
and bool((extra.get(str(bid)) or {}).get("clickable")))
|
||||
if not (is_role or is_clickable):
|
||||
continue
|
||||
if not include_hidden and not visible(str(bid), extra):
|
||||
continue
|
||||
name = node_name(n).strip()
|
||||
bbox = (extra.get(str(bid)) or {}).get("bbox")
|
||||
center = (bbox[0] + bbox[2] / 2, bbox[1] + bbox[3] / 2) if bbox else None
|
||||
out.append(RankItem(
|
||||
role=role if is_role else (role or "clickable"),
|
||||
name=name,
|
||||
bid=str(bid),
|
||||
value=node_value(n)[:80],
|
||||
context=build_context(nodes, by_id, n),
|
||||
center=center,
|
||||
options=child_options(by_id, n) if role in ("combobox", "listbox", "menu") else None,
|
||||
))
|
||||
return out
|
||||
|
||||
|
||||
OPTION_ROLES = {"option", "menuitem", "MenuListOption", "ListBoxOption"}
|
||||
|
||||
|
||||
def child_options(by_id: dict[str, dict[str, Any]], node: dict[str, Any], depth: int = 3) -> list[str] | None:
|
||||
"""Option labels under a select-like node, ignored-or-not: a closed <select> hides its options
|
||||
from the visible tree, and without their names the model can only click the box in a loop."""
|
||||
found: list[str] = []
|
||||
|
||||
def walk(n: dict[str, Any], d: int) -> None:
|
||||
if d > depth:
|
||||
return
|
||||
for cid in n.get("childIds") or []:
|
||||
child = by_id.get(cid)
|
||||
if not child:
|
||||
continue
|
||||
if node_role(child) in OPTION_ROLES:
|
||||
label = node_name(child).strip()
|
||||
if label:
|
||||
found.append(label)
|
||||
walk(child, d + 1)
|
||||
|
||||
walk(node, 0)
|
||||
return found or None
|
||||
|
||||
|
||||
def page_text(obs: dict[str, Any], limit: int = 1200) -> str:
|
||||
"""The page's visible text, compact: what our product's BrowserGetText gives the agent.
|
||||
|
||||
Without it the menu-only view cannot answer tasks whose payload lives in prose -- the algebra
|
||||
equation, which email row is Cecile's -- and the model scrolls in the dark while a flat-dump
|
||||
agent just reads the answer.
|
||||
"""
|
||||
ax = obs.get("axtree_object") or {}
|
||||
nodes: list[dict[str, Any]] = ax.get("nodes") or []
|
||||
parts: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for n in nodes:
|
||||
if n.get("ignored"):
|
||||
continue
|
||||
if node_role(n) in ("StaticText", "LabelText", "heading"):
|
||||
t = node_name(n).strip()
|
||||
if t and t not in seen:
|
||||
seen.add(t)
|
||||
parts.append(t)
|
||||
text = " | ".join(parts)
|
||||
return text[:limit]
|
||||
|
||||
|
||||
def axtree_stats(obs: dict[str, Any]) -> tuple[int, int]:
|
||||
"""(node count, flattened char count) so token pressure is a recorded metric, not a guess."""
|
||||
ax = obs.get("axtree_object") or {}
|
||||
nodes = ax.get("nodes") or []
|
||||
try:
|
||||
from browsergym.utils.obs import flatten_axtree_to_str
|
||||
|
||||
chars = len(flatten_axtree_to_str(ax))
|
||||
except Exception:
|
||||
chars = 0
|
||||
return len(nodes), chars
|
||||
|
||||
|
||||
def dom_chars(obs: dict[str, Any]) -> int:
|
||||
try:
|
||||
from browsergym.utils.obs import flatten_dom_to_str
|
||||
|
||||
return len(flatten_dom_to_str(obs.get("dom_object") or {}))
|
||||
except Exception:
|
||||
return 0
|
||||
@@ -0,0 +1,152 @@
|
||||
"""The two model-free arms: a naive flat-axtree matcher, and OpenSwarm's own perception + ladder.
|
||||
|
||||
Keeping both free arms in one file is deliberate — they differ ONLY in what they are allowed to see
|
||||
and how they pick, so having them side by side makes the delta auditable. Any model arm has to beat
|
||||
the better of these two to have earned its latency.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import perception
|
||||
from ranking import RankItem, goal_keywords, rank_and_cap, render
|
||||
|
||||
|
||||
@dataclass
|
||||
class Decision:
|
||||
"""What a policy chose plus everything worth recording about how it got there."""
|
||||
|
||||
action: str
|
||||
n_interactive: int = 0
|
||||
note: str = ""
|
||||
perceive_ms: float = 0.0
|
||||
|
||||
|
||||
def quoted(goal: str) -> list[str]:
|
||||
"""MiniWoB names its target in quotes far more often than not; treat that as a strong hint."""
|
||||
return re.findall(r'"([^"]+)"', goal) + re.findall(r"'([^']*)'", goal)
|
||||
|
||||
|
||||
def wants_typing(goal: str) -> bool:
|
||||
g = goal.lower()
|
||||
return any(k in g for k in ("enter", "type", "text", "password", "search", "fill", "write"))
|
||||
|
||||
|
||||
@dataclass
|
||||
class FlatPolicy:
|
||||
"""Naive control: read the flat tree, click the first exact label match, else the first control.
|
||||
|
||||
This is the shape a model-free agent gets from a plain accessibility dump with no ranking, no
|
||||
dedupe and no memory. It is the floor every other arm is measured against.
|
||||
"""
|
||||
|
||||
name: str = "flat"
|
||||
acted: set[str] = field(default_factory=set)
|
||||
|
||||
def reset(self, goal: str) -> None:
|
||||
self.acted = set()
|
||||
|
||||
def act(self, obs: dict[str, Any], goal: str) -> Decision:
|
||||
items = perception.interactives(obs)
|
||||
want = quoted(goal)
|
||||
for it in items:
|
||||
if any(w.lower() == it.name.lower() for w in want) and it.bid not in self.acted:
|
||||
self.acted.add(it.bid)
|
||||
return Decision(f'click("{it.bid}")', len(items), "exact-label")
|
||||
if wants_typing(goal):
|
||||
for it in items:
|
||||
if it.role in ("textbox", "searchbox"):
|
||||
payload = want[0] if want else "hello"
|
||||
return Decision(f'fill("{it.bid}", "{payload}")', len(items), "first-textbox")
|
||||
for it in items:
|
||||
if it.bid not in self.acted:
|
||||
self.acted.add(it.bid)
|
||||
return Decision(f'click("{it.bid}")', len(items), "first-control")
|
||||
return Decision("", len(items), "exhausted")
|
||||
|
||||
|
||||
@dataclass
|
||||
class OpenSwarmPolicy:
|
||||
"""OpenSwarm's shipped perception (dedupe, goal-rank, cap, ctx, value, stable ids) plus our ladder.
|
||||
|
||||
The ladder mirrors browser_send_script.py: fill the composer the goal points at, verify by reading
|
||||
the value back out of the tree, and only then submit. Clicks avoid repeats the way our `*` new
|
||||
marker lets the model avoid re-acting on an element it already touched.
|
||||
"""
|
||||
|
||||
name: str = "openswarm"
|
||||
acted: set[str] = field(default_factory=set)
|
||||
filled: dict[str, str] = field(default_factory=dict)
|
||||
prev_bids: set[str] = field(default_factory=set)
|
||||
last_render: str = ""
|
||||
|
||||
def reset(self, goal: str) -> None:
|
||||
self.acted = set()
|
||||
self.filled = {}
|
||||
self.prev_bids = set()
|
||||
self.last_render = ""
|
||||
|
||||
def look(self, obs: dict[str, Any], goal: str) -> tuple[list[RankItem], int, set[str]]:
|
||||
raw = perception.interactives(obs)
|
||||
shown, truncated = rank_and_cap(raw, goal=goal)
|
||||
new = {it.bid for it in shown} - self.prev_bids if self.prev_bids else set()
|
||||
self.prev_bids = {it.bid for it in shown}
|
||||
return shown, truncated, new
|
||||
|
||||
def act(self, obs: dict[str, Any], goal: str) -> Decision:
|
||||
shown, truncated, new = self.look(obs, goal)
|
||||
self.last_render = render(shown, truncated, new)
|
||||
keywords = goal_keywords(goal)
|
||||
want = quoted(goal)
|
||||
payload = want[0] if want else ""
|
||||
|
||||
# Fill first when the goal asks for text and the box does not already hold it (readback-verified).
|
||||
if wants_typing(goal) and payload:
|
||||
for it in shown:
|
||||
if it.role in ("textbox", "searchbox", "combobox"):
|
||||
if payload.lower() in (it.value or "").lower():
|
||||
continue
|
||||
return Decision(f'fill("{it.bid}", "{payload}")', len(shown), "fill")
|
||||
|
||||
# Exact label match on the quoted target, skipping anything already acted on.
|
||||
for it in shown:
|
||||
if any(w.lower() == it.name.lower() for w in want) and it.bid not in self.acted:
|
||||
self.acted.add(it.bid)
|
||||
return Decision(f'click("{it.bid}")', len(shown), "exact-label")
|
||||
|
||||
# Submit once the composer holds the payload, which is how our scripted path closes a task.
|
||||
if payload and any(payload.lower() in (it.value or "").lower() for it in shown):
|
||||
for it in shown:
|
||||
if it.role == "button" and re.search(r"submit|ok|done|send|search", it.name, re.I):
|
||||
if it.bid not in self.acted:
|
||||
self.acted.add(it.bid)
|
||||
return Decision(f'click("{it.bid}")', len(shown), "submit")
|
||||
|
||||
# Goal-keyword substring match: the ranking already floated these, so take the first.
|
||||
for it in shown:
|
||||
if keywords and any(k in it.name.lower() for k in keywords) and it.bid not in self.acted:
|
||||
self.acted.add(it.bid)
|
||||
return Decision(f'click("{it.bid}")', len(shown), "keyword")
|
||||
|
||||
for it in shown:
|
||||
if it.bid not in self.acted and it.role in ("button", "link", "checkbox", "tab", "option"):
|
||||
self.acted.add(it.bid)
|
||||
return Decision(f'click("{it.bid}")', len(shown), "role-fallback")
|
||||
for it in shown:
|
||||
if it.bid not in self.acted:
|
||||
self.acted.add(it.bid)
|
||||
return Decision(f'click("{it.bid}")', len(shown), "any")
|
||||
return Decision("", len(shown), "exhausted")
|
||||
|
||||
|
||||
REGISTRY = {"flat": FlatPolicy, "openswarm": OpenSwarmPolicy}
|
||||
|
||||
|
||||
def build(name: str, **kwargs: Any) -> Any:
|
||||
if name in REGISTRY:
|
||||
return REGISTRY[name]()
|
||||
import llm_policy # imported lazily so the free arms never need a model lane configured
|
||||
|
||||
return llm_policy.build(name, **kwargs)
|
||||
@@ -0,0 +1,128 @@
|
||||
"""Port of frontend/src/shared/interactiveRanking.ts, so the arena scores OUR algorithm, not a sketch of it.
|
||||
|
||||
Kept deliberately line-for-line with the TypeScript: same roles, same priorities, same stopwords,
|
||||
same cap, same doc-order display. If this drifts from the product the arena stops measuring the
|
||||
product, so any change here should land in interactiveRanking.ts too (and vice versa).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
# CDP computed roles we treat as actionable, from browserCommandHandler.ts INTERACTIVE_ROLES.
|
||||
INTERACTIVE_ROLES = {
|
||||
"button", "link", "textbox", "combobox", "checkbox", "menuitem",
|
||||
"tab", "switch", "searchbox", "slider", "listbox", "option",
|
||||
"radio", "menuitemcheckbox", "menuitemradio", "spinbutton", "treeitem",
|
||||
}
|
||||
|
||||
# Lower = higher priority: things a user types into, then navigation, then toggles, then the tail.
|
||||
ROLE_PRIORITY = {
|
||||
"textbox": 0, "searchbox": 0, "combobox": 0,
|
||||
"button": 1, "link": 1, "menuitem": 1, "tab": 1,
|
||||
"checkbox": 2, "radio": 2, "switch": 2, "menuitemcheckbox": 2, "menuitemradio": 2,
|
||||
"option": 3, "treeitem": 3, "listbox": 3, "slider": 3, "spinbutton": 3,
|
||||
}
|
||||
DEFAULT_PRIORITY = 2
|
||||
DEFAULT_INTERACTIVE_CAP = 60
|
||||
|
||||
STOPWORDS = {
|
||||
"the", "and", "for", "with", "click", "type", "into", "button", "link",
|
||||
"press", "select", "open", "goto", "navigate", "find", "tap", "this",
|
||||
"that", "your", "from", "page", "then", "enter", "input", "field", "box",
|
||||
"icon", "menu", "option", "item", "element",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class RankItem:
|
||||
"""One interactive node. `bid` is BrowserGym's handle, standing in for our backendNodeId."""
|
||||
|
||||
role: str
|
||||
name: str
|
||||
bid: str
|
||||
value: str = ""
|
||||
context: str = ""
|
||||
# Viewport center, for rows the model may need to hit by coordinate (canvas, slider track).
|
||||
center: tuple[float, float] | None = None
|
||||
# Child option labels for combobox/listbox rows; without them a closed <select> is unactionable.
|
||||
options: list[str] | None = None
|
||||
|
||||
|
||||
def role_priority(role: str) -> int:
|
||||
return ROLE_PRIORITY.get(role, DEFAULT_PRIORITY)
|
||||
|
||||
|
||||
def goal_keywords(goal: str) -> list[str]:
|
||||
words = [w for w in re.split(r"[^a-z0-9]+", goal.lower()) if w]
|
||||
kept = [w for w in words if len(w) >= 3 and w not in STOPWORDS]
|
||||
seen: list[str] = []
|
||||
for w in kept:
|
||||
if w not in seen:
|
||||
seen.append(w)
|
||||
return seen[:8]
|
||||
|
||||
|
||||
def matches_goal(name: str, keywords: list[str]) -> bool:
|
||||
if not keywords:
|
||||
return False
|
||||
lower = name.lower()
|
||||
return any(k in lower for k in keywords)
|
||||
|
||||
|
||||
# Roles whose twins are individually meaningful: two same-named password boxes are two fields the
|
||||
# user must both fill, never an icon+label pair. Collapsing them cost enter-password outright.
|
||||
NEVER_DEDUPE_ROLES = {"textbox", "searchbox", "combobox", "spinbutton", "listbox"}
|
||||
|
||||
|
||||
def dedupe_consecutive(items: list[RankItem]) -> list[RankItem]:
|
||||
"""Collapse only BACK-TO-BACK role+name+context twins, so a genuine list of five is never merged."""
|
||||
out: list[RankItem] = []
|
||||
for it in items:
|
||||
prev = out[-1] if out else None
|
||||
if (prev and prev.role == it.role and prev.name == it.name and prev.context == it.context
|
||||
and it.role not in NEVER_DEDUPE_ROLES):
|
||||
continue
|
||||
out.append(it)
|
||||
return out
|
||||
|
||||
|
||||
def rank_and_cap(items: list[RankItem], goal: str = "", cap: int = DEFAULT_INTERACTIVE_CAP,
|
||||
doc_order: bool = True) -> tuple[list[RankItem], int]:
|
||||
"""Rank decides WHAT survives the cap; display order stays document order so ordinals read true."""
|
||||
keywords = goal_keywords(goal) if goal else []
|
||||
deduped = dedupe_consecutive(items)
|
||||
scored = [(it, i, 0 if matches_goal(it.name, keywords) else 1) for i, it in enumerate(deduped)]
|
||||
ranked = sorted(scored, key=lambda x: (x[2], role_priority(x[0].role), x[1]))
|
||||
selected = ranked[:cap] if cap > 0 else ranked
|
||||
displayed = sorted(selected, key=lambda x: x[1]) if doc_order else selected
|
||||
return [x[0] for x in displayed], max(0, len(ranked) - len(displayed))
|
||||
|
||||
|
||||
def render(items: list[RankItem], truncated: int, new_bids: set[str] | None = None) -> str:
|
||||
"""The exact string shape BrowserListInteractives hands the model, including the * new-marker."""
|
||||
if not items:
|
||||
return "No interactive elements found on this page."
|
||||
counts: dict[str, int] = {}
|
||||
for el in items:
|
||||
k = f"{el.role}|{el.name}"
|
||||
counts[k] = counts.get(k, 0) + 1
|
||||
lines = []
|
||||
for i, el in enumerate(items, 1):
|
||||
dup = counts.get(f"{el.role}|{el.name}", 0) > 1
|
||||
ctx = f' ctx="{el.context}"' if dup and el.context else ""
|
||||
val = f' value="{el.value}"' if el.value else ""
|
||||
star = "*" if new_bids and el.bid in new_bids else ""
|
||||
# Coordinates only where the label carries no signal; a named button needs no geometry.
|
||||
pos = f" center=({el.center[0]:.0f},{el.center[1]:.0f})" if el.center and not el.name else ""
|
||||
opts = ""
|
||||
if el.options:
|
||||
shown_opts = el.options[:12]
|
||||
more = f" +{len(el.options) - 12}" if len(el.options) > 12 else ""
|
||||
opts = f' options="{"|".join(shown_opts)}{more}"'
|
||||
lines.append(f'[{i}]{star}<{el.role} "{el.name}"{ctx}{val}{pos}{opts}>')
|
||||
head = f"{len(lines)} interactive elements (* = new since your last look):"
|
||||
text = head + "\n" + "\n".join(lines)
|
||||
if truncated > 0:
|
||||
text += f"\n... {truncated} more not shown; scroll or scope to reach them."
|
||||
return text
|
||||
@@ -0,0 +1,139 @@
|
||||
"""Central data store for the MiniWoB arena: every arm, every task, every step, one place.
|
||||
|
||||
Everything any arm produces lands in ONE append-only JSONL so no arm can be scored by a different
|
||||
book than another. Screenshots are written per step and referenced by path from the step record, so
|
||||
a claim about what an agent saw can always be checked against the pixels it saw it in.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
ARENA_DIR = Path(__file__).resolve().parent
|
||||
RESULTS_DIR = Path(os.environ.get("OSW_ARENA_RESULTS", ARENA_DIR / "data"))
|
||||
SHOTS_DIR = RESULTS_DIR / "shots"
|
||||
|
||||
|
||||
@dataclass
|
||||
class StepRecord:
|
||||
"""One agent turn: what it saw, what it chose, what that cost, what the page did about it."""
|
||||
|
||||
step: int
|
||||
action: str = ""
|
||||
action_ms: float = 0.0
|
||||
perceive_ms: float = 0.0
|
||||
think_ms: float = 0.0
|
||||
axtree_chars: int = 0
|
||||
axtree_nodes: int = 0
|
||||
dom_chars: int = 0
|
||||
n_interactive: int = 0
|
||||
prompt_tokens: int = 0
|
||||
completion_tokens: int = 0
|
||||
cost_usd: float = 0.0
|
||||
reward: float = 0.0
|
||||
url: str = ""
|
||||
focused_bid: str = ""
|
||||
action_error: str = ""
|
||||
llm_error: str = ""
|
||||
retries: int = 0
|
||||
shot: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class EpisodeRecord:
|
||||
"""One (arm, task, seed) attempt, scored by MiniWoB itself rather than by this file."""
|
||||
|
||||
arm: str
|
||||
task: str
|
||||
seed: int
|
||||
model: str = ""
|
||||
reward: float = 0.0
|
||||
raw_reward: float = 0.0
|
||||
success: bool = False
|
||||
# What the agent itself claimed; claimed and not success = a false success, the worst failure class.
|
||||
claimed_success: bool = False
|
||||
steps: int = 0
|
||||
wall_s: float = 0.0
|
||||
setup_s: float = 0.0
|
||||
first_action_s: float = 0.0
|
||||
terminated: bool = False
|
||||
truncated: bool = False
|
||||
error_class: str = ""
|
||||
error_detail: str = ""
|
||||
goal: str = ""
|
||||
prompt_tokens: int = 0
|
||||
completion_tokens: int = 0
|
||||
cost_usd: float = 0.0
|
||||
llm_calls: int = 0
|
||||
started_at: float = 0.0
|
||||
step_records: list[StepRecord] = field(default_factory=list)
|
||||
|
||||
def add(self, rec: StepRecord) -> None:
|
||||
self.step_records.append(rec)
|
||||
self.prompt_tokens += rec.prompt_tokens
|
||||
self.completion_tokens += rec.completion_tokens
|
||||
self.cost_usd += rec.cost_usd
|
||||
if rec.prompt_tokens or rec.completion_tokens or rec.think_ms:
|
||||
self.llm_calls += 1
|
||||
|
||||
|
||||
class Recorder:
|
||||
"""Append-only writer. One file per run tag, plus a stable `all.jsonl` that never gets rewritten."""
|
||||
|
||||
def __init__(self, tag: str, run_id: str = "") -> None:
|
||||
RESULTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
SHOTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
self.tag = tag
|
||||
self.run_id = run_id or time.strftime("%Y%m%d-%H%M%S")
|
||||
self.path = RESULTS_DIR / f"{tag}.jsonl"
|
||||
self.all_path = RESULTS_DIR / "all.jsonl"
|
||||
|
||||
def shot_path(self, arm: str, task: str, seed: int, step: int) -> Path:
|
||||
d = SHOTS_DIR / self.run_id / arm / f"{task}-s{seed}"
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
return d / f"{step:02d}.png"
|
||||
|
||||
def write(self, ep: EpisodeRecord) -> None:
|
||||
row = asdict(ep)
|
||||
row["run_id"] = self.run_id
|
||||
row["tag"] = self.tag
|
||||
line = json.dumps(row, separators=(",", ":")) + "\n"
|
||||
for p in (self.path, self.all_path):
|
||||
with open(p, "a") as fh:
|
||||
fh.write(line)
|
||||
|
||||
|
||||
def save_screenshot(obs: dict[str, Any], dest: Path) -> str:
|
||||
"""Persist a BrowserGym observation frame. Returns the path, or '' if the frame was unusable."""
|
||||
arr = obs.get("screenshot")
|
||||
if arr is None:
|
||||
return ""
|
||||
try:
|
||||
from PIL import Image # imported lazily so a no-screenshot run needs no pillow
|
||||
|
||||
Image.fromarray(arr).save(dest)
|
||||
return str(dest)
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def load_episodes(tag: str = "all") -> list[dict[str, Any]]:
|
||||
"""Read back every episode for a tag; the report module's only input."""
|
||||
p = RESULTS_DIR / f"{tag}.jsonl"
|
||||
if not p.exists():
|
||||
return []
|
||||
out: list[dict[str, Any]] = []
|
||||
with open(p) as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
out.append(json.loads(line))
|
||||
except ValueError:
|
||||
continue
|
||||
return out
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Every arm, every category, every metric, one table -- written from the recorder's book only.
|
||||
|
||||
This file computes; it never re-judges. Success is whatever MiniWoB's reward said at run time, and
|
||||
if a number is not derivable from the recorded episodes it does not appear here.
|
||||
|
||||
python report.py # summary across all recorded arms
|
||||
python report.py --by-task # per-task win matrix
|
||||
python report.py --md ARENA.md # write the markdown scoreboard
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import math
|
||||
import statistics
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from recorder import load_episodes
|
||||
from tasks import CATEGORIES, CATEGORY_OF
|
||||
|
||||
|
||||
def wilson(wins: int, n: int, z: float = 1.96) -> tuple[float, float]:
|
||||
"""Same interval the rest of browser-v3 reports, so numbers compare across documents."""
|
||||
if n == 0:
|
||||
return (0.0, 0.0)
|
||||
p = wins / n
|
||||
d = 1 + z * z / n
|
||||
c = p + z * z / (2 * n)
|
||||
h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n))
|
||||
return ((c - h) / d, (c + h) / d)
|
||||
|
||||
|
||||
def med(vals: list[float]) -> float:
|
||||
return statistics.median(vals) if vals else 0.0
|
||||
|
||||
|
||||
def latest_per_key(eps: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Reruns supersede: only the newest episode per (arm, task, seed) counts, so iteration is honest."""
|
||||
by_key: dict[tuple[str, str, int], dict[str, Any]] = {}
|
||||
for ep in eps:
|
||||
key = (ep["arm"], ep["task"], ep["seed"])
|
||||
cur = by_key.get(key)
|
||||
if cur is None or ep.get("started_at", 0) >= cur.get("started_at", 0):
|
||||
by_key[key] = ep
|
||||
return list(by_key.values())
|
||||
|
||||
|
||||
def summarize(eps: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
||||
arms: dict[str, dict[str, Any]] = {}
|
||||
for arm in sorted({e["arm"] for e in eps}):
|
||||
rows = [e for e in eps if e["arm"] == arm]
|
||||
clean = [e for e in rows if not str(e.get("error_class", "")).startswith("infra")]
|
||||
wins = sum(1 for e in clean if e.get("success"))
|
||||
lo, hi = wilson(wins, len(clean))
|
||||
arms[arm] = {
|
||||
"n": len(rows), "clean": len(clean), "wins": wins,
|
||||
"rate": wins / len(clean) if clean else 0.0, "lo": lo, "hi": hi,
|
||||
"infra": len(rows) - len(clean),
|
||||
"wall_med": med([e["wall_s"] for e in clean if e.get("wall_s")]),
|
||||
"wall_win_med": med([e["wall_s"] for e in clean if e.get("success")]),
|
||||
"steps_med": med([float(e["steps"]) for e in clean if e.get("steps")]),
|
||||
"tokens": sum(e.get("prompt_tokens", 0) + e.get("completion_tokens", 0) for e in rows),
|
||||
"llm_calls": sum(e.get("llm_calls", 0) for e in rows),
|
||||
"false_success": sum(1 for e in clean if e.get("claimed_success") and not e.get("success")),
|
||||
"by_cat": by_category(clean),
|
||||
}
|
||||
return arms
|
||||
|
||||
|
||||
def by_category(rows: list[dict[str, Any]]) -> dict[str, tuple[int, int]]:
|
||||
out: dict[str, list[int]] = defaultdict(lambda: [0, 0])
|
||||
for e in rows:
|
||||
cat = CATEGORY_OF.get(e["task"], "other")
|
||||
out[cat][1] += 1
|
||||
out[cat][0] += 1 if e.get("success") else 0
|
||||
return {k: (v[0], v[1]) for k, v in out.items()}
|
||||
|
||||
|
||||
def fmt_summary(arms: dict[str, dict[str, Any]], md: bool = False) -> str:
|
||||
lines: list[str] = []
|
||||
bar = "|" if md else " "
|
||||
if md:
|
||||
lines.append("| arm | solved | rate | 95% CI | med wall (win) | med steps | tokens | LLM calls | false-succ | infra |")
|
||||
lines.append("|---|---|---|---|---|---|---|---|---|---|")
|
||||
else:
|
||||
lines.append(f"{'arm':12s} {'solved':>9s} {'rate':>7s} {'95% CI':>15s} {'wall(win)':>10s} "
|
||||
f"{'steps':>6s} {'tokens':>9s} {'calls':>6s} {'false':>6s} {'infra':>6s}")
|
||||
for arm, s in sorted(arms.items(), key=lambda kv: -kv[1]["rate"]):
|
||||
ci = f"[{100 * s['lo']:.0f},{100 * s['hi']:.0f}]"
|
||||
row = [arm, f"{s['wins']}/{s['clean']}", f"{100 * s['rate']:.1f}%", ci,
|
||||
f"{s['wall_win_med']:.1f}s", f"{s['steps_med']:.0f}", f"{s['tokens']}",
|
||||
f"{s['llm_calls']}", f"{s['false_success']}", f"{s['infra']}"]
|
||||
lines.append(("| " + " | ".join(row) + " |") if md else
|
||||
f"{row[0]:12s} {row[1]:>9s} {row[2]:>7s} {row[3]:>15s} {row[4]:>10s} "
|
||||
f"{row[5]:>6s} {row[6]:>9s} {row[7]:>6s} {row[8]:>6s} {row[9]:>6s}")
|
||||
lines.append("")
|
||||
cats = sorted(CATEGORIES)
|
||||
if md:
|
||||
lines.append("| category | " + " | ".join(sorted(arms)) + " |")
|
||||
lines.append("|---|" + "---|" * len(arms))
|
||||
else:
|
||||
lines.append(f"{'category':16s}" + "".join(f"{a:>16s}" for a in sorted(arms)))
|
||||
for cat in cats:
|
||||
cells = []
|
||||
for arm in sorted(arms):
|
||||
w, n = arms[arm]["by_cat"].get(cat, (0, 0))
|
||||
cells.append(f"{w}/{n} ({100 * w / n:.0f}%)" if n else "-")
|
||||
lines.append(("| " + cat + " | " + " | ".join(cells) + " |") if md
|
||||
else f"{cat:16s}" + "".join(f"{c:>16s}" for c in cells))
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def fmt_by_task(eps: list[dict[str, Any]]) -> str:
|
||||
"""Per-task matrix: the disagreements between arms are where every insight lives."""
|
||||
arms = sorted({e["arm"] for e in eps})
|
||||
by_tk: dict[str, dict[str, list[bool]]] = defaultdict(lambda: defaultdict(list))
|
||||
for e in eps:
|
||||
if not str(e.get("error_class", "")).startswith("infra"):
|
||||
by_tk[e["task"]][e["arm"]].append(bool(e.get("success")))
|
||||
lines = [f"{'task':30s}" + "".join(f"{a:>12s}" for a in arms)]
|
||||
for task in sorted(by_tk):
|
||||
cells = []
|
||||
for arm in arms:
|
||||
r = by_tk[task].get(arm)
|
||||
cells.append("-" if not r else f"{sum(r)}/{len(r)}")
|
||||
lines.append(f"{task:30s}" + "".join(f"{c:>12s}" for c in cells))
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--tag", default="all")
|
||||
ap.add_argument("--by-task", action="store_true")
|
||||
ap.add_argument("--md", default="")
|
||||
# Cross-model rows never mix: an arm's rate is only meaningful against arms on the same lane.
|
||||
ap.add_argument("--model", default="", help="only episodes run on this model (or '' for LLM-free arms too)")
|
||||
args = ap.parse_args()
|
||||
eps = latest_per_key(load_episodes(args.tag))
|
||||
if args.model:
|
||||
eps = [e for e in eps if e.get("model", "") in (args.model, "")]
|
||||
if not eps:
|
||||
raise SystemExit("no recorded episodes")
|
||||
if args.by_task:
|
||||
print(fmt_by_task(eps))
|
||||
return
|
||||
arms = summarize(eps)
|
||||
print(fmt_summary(arms))
|
||||
if args.md:
|
||||
Path(args.md).write_text("# MiniWoB arena scoreboard\n\n" + fmt_summary(arms, md=True) + "\n")
|
||||
print(f"\nwrote {args.md}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,226 @@
|
||||
"""Run one arm over MiniWoB tasks and record every metric MiniWoB and the browser will give us.
|
||||
|
||||
Scoring is MiniWoB's own reward, read through BrowserGym. Nothing in this repo decides whether an
|
||||
episode passed, which is the whole reason this harness exists: every other suite here was graded by
|
||||
something I wrote and then tuned against.
|
||||
|
||||
python run.py --arm openswarm --tasks all --seeds 3 --shots first-last
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
import browsergym.miniwob # noqa: F401 importing is what registers the 125 envs
|
||||
import gymnasium as gym
|
||||
|
||||
import perception
|
||||
import policies
|
||||
from recorder import EpisodeRecord, Recorder, StepRecord, save_screenshot
|
||||
from tasks import resolve_tasks
|
||||
|
||||
os.environ.setdefault("MINIWOB_URL", "http://localhost:8099/miniwob/")
|
||||
|
||||
|
||||
class StepHang(Exception):
|
||||
"""env.step exceeded the watchdog; the browser is presumed wedged for this episode."""
|
||||
|
||||
|
||||
def with_deadline(fn: Any, timeout_s: float) -> Any:
|
||||
"""One measured episode lost 901s inside a single env.step; never let a hang masquerade as skill time.
|
||||
|
||||
SIGALRM, not a worker thread: sync Playwright pins its greenlet to the creating thread, and the
|
||||
threaded version died with 'cannot switch to a different thread' on the first click.
|
||||
"""
|
||||
def on_alarm(signum: int, frame: Any) -> None:
|
||||
raise StepHang(f"call exceeded {timeout_s:.0f}s")
|
||||
|
||||
prev = signal.signal(signal.SIGALRM, on_alarm)
|
||||
signal.alarm(max(1, int(timeout_s)))
|
||||
try:
|
||||
return fn()
|
||||
finally:
|
||||
signal.alarm(0)
|
||||
signal.signal(signal.SIGALRM, prev)
|
||||
|
||||
|
||||
def classify(exc: BaseException) -> str:
|
||||
"""Separate a harness/browser failure from a policy failure so infra noise never scores as skill."""
|
||||
name = type(exc).__name__
|
||||
text = str(exc).lower()
|
||||
if isinstance(exc, StepHang):
|
||||
return "infra_step_hang"
|
||||
if "timeout" in text or "Timeout" in name:
|
||||
return "infra_timeout"
|
||||
if "target" in text and "closed" in text:
|
||||
return "infra_browser_closed"
|
||||
if "connection" in text or "econnrefused" in text:
|
||||
return "infra_connection"
|
||||
if "rate" in text and "limit" in text:
|
||||
return "llm_rate_limit"
|
||||
return f"error_{name}"
|
||||
|
||||
|
||||
def run_episode(arm: str, task: str, seed: int, rec: Recorder, args: argparse.Namespace) -> EpisodeRecord:
|
||||
ep = EpisodeRecord(arm=arm, task=task, seed=seed, model=args.model or "", started_at=time.time())
|
||||
policy = policies.build(arm, model=args.model, endpoint=args.endpoint)
|
||||
t_setup = time.time()
|
||||
env = None
|
||||
holder: list[Any] = []
|
||||
try:
|
||||
# Setup under its own deadline: both 125-task sweeps once hung 30+ minutes in env
|
||||
# creation/reset with the step watchdog never armed. holder keeps a half-built env
|
||||
# reachable so the except path can close its browsers instead of leaking them.
|
||||
def setup():
|
||||
# coord subset on for EVERY arm: canvas/slider/drag tasks are unphrasable in pure bid
|
||||
# space, and the product this arena stands in for ships coordinate clicks (click_point).
|
||||
from browsergym.core.action.highlevel import HighLevelActionSet
|
||||
|
||||
acts = HighLevelActionSet(subsets=["chat", "bid", "coord", "infeas"],
|
||||
strict=False, multiaction=False)
|
||||
holder.append(gym.make(f"browsergym/miniwob.{task}", headless=not args.headed,
|
||||
max_episode_steps=args.max_steps, wait_for_user_message=False,
|
||||
action_mapping=acts.to_python_code))
|
||||
return holder[0].reset(seed=seed)
|
||||
|
||||
obs, _ = with_deadline(setup, args.setup_timeout)
|
||||
env = holder[0]
|
||||
except Exception as exc:
|
||||
ep.error_class = classify(exc)
|
||||
ep.error_detail = f"{type(exc).__name__}: {exc}"[:200]
|
||||
ep.setup_s = time.time() - t_setup
|
||||
if holder:
|
||||
try:
|
||||
with_deadline(holder[0].close, 15)
|
||||
except Exception:
|
||||
pass
|
||||
return ep
|
||||
ep.setup_s = time.time() - t_setup
|
||||
ep.goal = str(obs.get("goal") or "")[:300]
|
||||
policy.reset(ep.goal)
|
||||
|
||||
t0 = time.time()
|
||||
try:
|
||||
for step in range(1, args.max_steps + 1):
|
||||
t_perc = time.time()
|
||||
nodes, ax_chars = perception.axtree_stats(obs)
|
||||
# The LLM call rides inside act(); urlopen's timeout does not cover every hang mode.
|
||||
decision = with_deadline(lambda: policy.act(obs, ep.goal), args.step_timeout + 90)
|
||||
perceive_ms = (time.time() - t_perc) * 1000
|
||||
rec_step = StepRecord(
|
||||
step=step, action=decision.action, perceive_ms=perceive_ms,
|
||||
think_ms=getattr(decision, "think_ms", 0.0),
|
||||
axtree_chars=ax_chars, axtree_nodes=nodes,
|
||||
dom_chars=perception.dom_chars(obs) if args.dom_metrics else 0,
|
||||
n_interactive=decision.n_interactive, url=str(obs.get("url") or ""),
|
||||
focused_bid=str(obs.get("focused_element_bid") or ""),
|
||||
prompt_tokens=getattr(decision, "prompt_tokens", 0),
|
||||
completion_tokens=getattr(decision, "completion_tokens", 0),
|
||||
cost_usd=getattr(decision, "cost_usd", 0.0),
|
||||
llm_error=getattr(decision, "llm_error", ""),
|
||||
retries=getattr(decision, "retries", 0),
|
||||
)
|
||||
if args.shots != "none":
|
||||
dest = rec.shot_path(arm, task, seed, step)
|
||||
rec_step.shot = save_screenshot(obs, dest)
|
||||
if not decision.action:
|
||||
rec_step.action_error = "policy produced no action"
|
||||
ep.add(rec_step)
|
||||
# A no-action turn caused by a dead LLM lane is the harness's problem, not the arm's.
|
||||
if getattr(decision, "llm_error", ""):
|
||||
ep.error_class = "infra_llm"
|
||||
ep.error_detail = decision.llm_error[:200]
|
||||
break
|
||||
if ep.first_action_s == 0.0:
|
||||
ep.first_action_s = time.time() - t0
|
||||
t_act = time.time()
|
||||
obs, reward, terminated, truncated, _ = with_deadline(
|
||||
lambda: env.step(decision.action), args.step_timeout)
|
||||
rec_step.action_ms = (time.time() - t_act) * 1000
|
||||
rec_step.reward = float(reward or 0)
|
||||
rec_step.action_error = str(obs.get("last_action_error") or "")[:200]
|
||||
ep.add(rec_step)
|
||||
ep.reward = max(ep.reward, float(reward or 0))
|
||||
ep.steps = step
|
||||
if terminated or truncated:
|
||||
ep.terminated, ep.truncated = bool(terminated), bool(truncated)
|
||||
break
|
||||
except Exception as exc:
|
||||
ep.error_class = classify(exc)
|
||||
ep.error_detail = f"{type(exc).__name__}: {exc}"[:200]
|
||||
if args.trace:
|
||||
traceback.print_exc()
|
||||
ep.wall_s = time.time() - t0
|
||||
ep.success = ep.reward > 0
|
||||
try:
|
||||
raw = with_deadline(lambda: env.unwrapped.page.evaluate(
|
||||
"typeof WOB_RAW_REWARD_GLOBAL !== 'undefined' ? WOB_RAW_REWARD_GLOBAL : 0"), 10)
|
||||
ep.raw_reward = float(raw or 0)
|
||||
except Exception:
|
||||
ep.raw_reward = ep.reward
|
||||
# Final frame AFTER the last action, which is the only one that shows why an episode scored 0.
|
||||
if args.shots != "none":
|
||||
try:
|
||||
with_deadline(lambda: env.unwrapped.page.screenshot(
|
||||
path=str(rec.shot_path(arm, task, seed, 99))), 10)
|
||||
except Exception:
|
||||
pass
|
||||
# close() can hang on the same wedged browser the step hung on; give it its own short leash.
|
||||
try:
|
||||
with_deadline(env.close, 15)
|
||||
except Exception:
|
||||
pass
|
||||
return ep
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--arm", required=True)
|
||||
ap.add_argument("--tasks", default="all")
|
||||
ap.add_argument("--seeds", type=int, default=1)
|
||||
ap.add_argument("--seed-base", type=int, default=42)
|
||||
ap.add_argument("--max-steps", type=int, default=12)
|
||||
ap.add_argument("--step-timeout", type=float, default=30.0)
|
||||
ap.add_argument("--setup-timeout", type=float, default=60.0)
|
||||
ap.add_argument("--shots", choices=["none", "first-last", "all"], default="first-last")
|
||||
ap.add_argument("--dom-metrics", action="store_true")
|
||||
ap.add_argument("--headed", action="store_true")
|
||||
ap.add_argument("--trace", action="store_true")
|
||||
ap.add_argument("--model", default=os.environ.get("OSW_ARENA_MODEL", ""))
|
||||
ap.add_argument("--endpoint", default=os.environ.get("OSW_ARENA_ENDPOINT", "http://localhost:20128"))
|
||||
ap.add_argument("--tag", default="")
|
||||
ap.add_argument("--run-id", default="")
|
||||
# Sharding, not threading: each shard owns its own browser, so one crash cannot poison the rest.
|
||||
ap.add_argument("--shard", default="", help="i/n, e.g. 0/4 to run every 4th task")
|
||||
args = ap.parse_args()
|
||||
|
||||
tasks = resolve_tasks(args.tasks)
|
||||
if args.shard:
|
||||
i, n = (int(x) for x in args.shard.split("/"))
|
||||
tasks = [t for k, t in enumerate(tasks) if k % n == i]
|
||||
rec = Recorder(args.tag or args.arm, args.run_id)
|
||||
print(f"arm={args.arm} tasks={len(tasks)} seeds={args.seeds} -> {rec.path}", flush=True)
|
||||
wins = total = 0
|
||||
for task in tasks:
|
||||
for s in range(args.seeds):
|
||||
ep = run_episode(args.arm, task, args.seed_base + s, rec, args)
|
||||
rec.write(ep)
|
||||
total += 1
|
||||
wins += 1 if ep.success else 0
|
||||
flag = "OK " if ep.success else ("ERR" if ep.error_class else "-- ")
|
||||
print(f" {flag} {task:28s} s={ep.seed} r={ep.reward:+.2f} steps={ep.steps:2d} "
|
||||
f"{ep.wall_s:6.2f}s tok={ep.prompt_tokens + ep.completion_tokens:<6d} {ep.error_class}",
|
||||
flush=True)
|
||||
print(f"\n{args.arm}: {wins}/{total} = {100 * wins / total if total else 0:.1f}%", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,37 @@
|
||||
// Stagehand driver for one MiniWoB episode: attach to the already-seeded page over CDP, act, exit.
|
||||
// Called by sh_real.py per episode; stdout's last line is a JSON result. Scoring stays in Python,
|
||||
// read from MiniWoB's own globals -- this process never grades itself.
|
||||
import { Stagehand } from '@browserbasehq/stagehand';
|
||||
|
||||
const [goal, cdpUrl, model, endpoint, maxSteps] = process.argv.slice(2);
|
||||
|
||||
const sh = new Stagehand({
|
||||
env: 'LOCAL',
|
||||
modelName: model,
|
||||
modelClientOptions: { apiKey: 'arena', baseURL: `${endpoint}/v1` },
|
||||
verbose: 0,
|
||||
disablePino: true,
|
||||
localBrowserLaunchOptions: { cdpUrl },
|
||||
});
|
||||
|
||||
const out = { steps: 0, actions: [], errors: [], claimed: false };
|
||||
try {
|
||||
await sh.init();
|
||||
// agent() is Stagehand's autonomous loop; per-act() would be MY orchestration, not theirs.
|
||||
const agent = sh.agent({
|
||||
provider: 'anthropic',
|
||||
model,
|
||||
instructions: 'Work only on the current page. Never navigate away.',
|
||||
options: { apiKey: 'arena', baseURL: `${endpoint}/v1` },
|
||||
});
|
||||
const result = await agent.execute({ instruction: goal, maxSteps: Number(maxSteps) || 12 });
|
||||
out.claimed = !!result?.success;
|
||||
out.steps = result?.actions?.length ?? 0;
|
||||
out.actions = (result?.actions ?? []).map((a) => JSON.stringify(a).slice(0, 120));
|
||||
out.usage = result?.usage ?? null;
|
||||
} catch (e) {
|
||||
out.errors.push(String(e).slice(0, 200));
|
||||
} finally {
|
||||
try { await sh.close(); } catch {}
|
||||
}
|
||||
console.log('RESULT:' + JSON.stringify(out));
|
||||
@@ -0,0 +1,123 @@
|
||||
"""Stagehand's agent on MiniWoB: same seeded page, same CDP attach, same external scorer as bu-real.
|
||||
|
||||
The node driver does the acting; this file owns the env, the clock, the record, and the verdict.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from bu_real import CDP_PORT, make_env, score
|
||||
from recorder import EpisodeRecord, Recorder, StepRecord
|
||||
from tasks import resolve_tasks
|
||||
|
||||
os.environ.setdefault("MINIWOB_URL", "http://localhost:8099/miniwob/")
|
||||
DRIVER = Path(__file__).resolve().parent / "sh_driver.mjs"
|
||||
NODE_DIR = os.environ.get("OSW_ARENA_SH_DIR", "")
|
||||
|
||||
|
||||
def drive(goal: str, model: str, endpoint: str, max_steps: int, timeout_s: float) -> dict:
|
||||
cmd = ["node", str(DRIVER), goal, f"http://localhost:{CDP_PORT}", model, endpoint, str(max_steps)]
|
||||
try:
|
||||
proc = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout_s,
|
||||
cwd=NODE_DIR or None)
|
||||
except subprocess.TimeoutExpired:
|
||||
return {"errors": [f"driver exceeded {timeout_s:.0f}s"]}
|
||||
for line in reversed(proc.stdout.splitlines()):
|
||||
if line.startswith("RESULT:"):
|
||||
try:
|
||||
return json.loads(line[len("RESULT:"):])
|
||||
except ValueError:
|
||||
break
|
||||
return {"errors": [f"no RESULT line; rc={proc.returncode}; tail={proc.stdout[-200:]!r} {proc.stderr[-200:]!r}"]}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--tasks", default="smoke")
|
||||
ap.add_argument("--seeds", type=int, default=1)
|
||||
ap.add_argument("--seed-base", type=int, default=42)
|
||||
ap.add_argument("--max-steps", type=int, default=12)
|
||||
ap.add_argument("--episode-timeout", type=float, default=180.0)
|
||||
ap.add_argument("--model", default=os.environ.get("OSW_ARENA_MODEL", "cc/claude-sonnet-4-6"))
|
||||
ap.add_argument("--endpoint", default=os.environ.get("OSW_ARENA_ENDPOINT", "http://localhost:20128"))
|
||||
ap.add_argument("--shots", choices=["none", "first-last"], default="first-last")
|
||||
ap.add_argument("--shard", default="")
|
||||
args = ap.parse_args()
|
||||
|
||||
tasks = resolve_tasks(args.tasks)
|
||||
if args.shard:
|
||||
i, n = (int(x) for x in args.shard.split("/"))
|
||||
tasks = [t for k, t in enumerate(tasks) if k % n == i]
|
||||
rec = Recorder("sh-real")
|
||||
print(f"arm=sh-real tasks={len(tasks)} seeds={args.seeds} model={args.model} -> {rec.path}", flush=True)
|
||||
wins = total = 0
|
||||
for task in tasks:
|
||||
for s in range(args.seeds):
|
||||
seed = args.seed_base + s
|
||||
ep = EpisodeRecord(arm="sh-real", task=task, seed=seed, model=args.model,
|
||||
started_at=time.time())
|
||||
t_setup = time.time()
|
||||
env = None
|
||||
try:
|
||||
env, obs = make_env(task, seed, args.max_steps)
|
||||
ep.goal = str(obs.get("goal") or "")[:300]
|
||||
except Exception as exc:
|
||||
ep.error_class = "infra_env_setup"
|
||||
ep.error_detail = f"{type(exc).__name__}: {exc}"[:200]
|
||||
ep.setup_s = time.time() - t_setup
|
||||
if env is not None:
|
||||
if args.shots != "none":
|
||||
try:
|
||||
env.unwrapped.page.screenshot(path=str(rec.shot_path("sh-real", task, seed, 1)))
|
||||
except Exception:
|
||||
pass
|
||||
t0 = time.time()
|
||||
stats = drive(ep.goal, args.model, args.endpoint, args.max_steps, args.episode_timeout)
|
||||
ep.wall_s = time.time() - t0
|
||||
try:
|
||||
ep.reward, ep.raw_reward = score(env)
|
||||
except Exception as exc:
|
||||
ep.error_class = "infra_score_readback"
|
||||
ep.error_detail = f"{type(exc).__name__}: {exc}"[:200]
|
||||
ep.success = ep.reward > 0
|
||||
ep.claimed_success = bool(stats.get("claimed"))
|
||||
ep.steps = int(stats.get("steps") or 0)
|
||||
usage = stats.get("usage") or {}
|
||||
ep.prompt_tokens = int(usage.get("input_tokens") or 0)
|
||||
ep.completion_tokens = int(usage.get("output_tokens") or 0)
|
||||
ep.llm_calls = int(usage.get("inference_time_ms") is not None and ep.steps or ep.steps)
|
||||
errs = stats.get("errors") or []
|
||||
if errs and not ep.success:
|
||||
ep.error_detail = (ep.error_detail + " | " + "; ".join(errs))[:200].strip(" |")
|
||||
for i, act in enumerate(stats.get("actions") or [], 1):
|
||||
ep.step_records.append(StepRecord(step=i, action=str(act)[:120]))
|
||||
if args.shots != "none":
|
||||
try:
|
||||
env.unwrapped.page.screenshot(path=str(rec.shot_path("sh-real", task, seed, 99)))
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
env.close()
|
||||
except Exception:
|
||||
pass
|
||||
rec.write(ep)
|
||||
total += 1
|
||||
wins += 1 if ep.success else 0
|
||||
flag = "OK " if ep.success else ("ERR" if ep.error_class else "-- ")
|
||||
lie = " FALSE-SUCCESS" if ep.claimed_success and not ep.success else ""
|
||||
print(f" {flag} {task:28s} s={seed} r={ep.reward:+.2f} steps={ep.steps:2d} "
|
||||
f"{ep.wall_s:6.2f}s tok={ep.prompt_tokens + ep.completion_tokens:<7d} "
|
||||
f"{ep.error_detail[:60]}{lie}", flush=True)
|
||||
print(f"\nsh-real: {wins}/{total} = {100 * wins / total if total else 0:.1f}%", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Out-of-process supervision for arena sweeps: launch, watch for stalls, kill, resume, repeat.
|
||||
|
||||
In-process deadlines proved unreliable here twice -- SIGALRM's exception gets swallowed somewhere
|
||||
inside playwright/gym retry loops and a sweep silently stops writing for half an hour. A supervisor
|
||||
that watches the recorder file and kills the whole process tree is the only deadline the stack
|
||||
cannot catch. Resume is free because the recorder is append-only and reports keep only the newest
|
||||
episode per (arm, task, seed).
|
||||
|
||||
python supervisor.py --arm osw-llm --model cc/claude-haiku-4-5-20251001 --stall 240
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from recorder import RESULTS_DIR
|
||||
from tasks import ALL, resolve_tasks
|
||||
|
||||
PY = sys.executable
|
||||
ARENA = Path(__file__).resolve().parent
|
||||
|
||||
|
||||
def completed_tasks(arm: str, model: str, seed: int, since: float) -> set[str]:
|
||||
"""Tasks with a usable (non-infra) episode for this arm+model+seed recorded after `since`."""
|
||||
path = RESULTS_DIR / f"{arm}.jsonl"
|
||||
done: set[str] = set()
|
||||
if not path.exists():
|
||||
return done
|
||||
with open(path) as fh:
|
||||
for line in fh:
|
||||
try:
|
||||
r = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if (r.get("seed") == seed and r.get("started_at", 0) >= since
|
||||
and r.get("model", "") in (model, "")
|
||||
and not str(r.get("error_class", "")).startswith("infra")):
|
||||
done.add(r["task"])
|
||||
return done
|
||||
|
||||
|
||||
def spawn(arm: str, tasks: list[str], args: argparse.Namespace) -> subprocess.Popen:
|
||||
if arm == "bu-real":
|
||||
cmd = [PY, str(ARENA / "bu_real.py"), "--tasks", ",".join(tasks), "--seeds", "1",
|
||||
"--seed-base", str(args.seed), "--model", args.model,
|
||||
"--episode-timeout", str(args.episode_timeout)]
|
||||
else:
|
||||
cmd = [PY, str(ARENA / "run.py"), "--arm", arm, "--tasks", ",".join(tasks), "--seeds", "1",
|
||||
"--seed-base", str(args.seed), "--model", args.model, "--shots", args.shots]
|
||||
env = dict(os.environ)
|
||||
env.setdefault("MINIWOB_URL", "http://localhost:8099/miniwob/")
|
||||
env.setdefault("BROWSER_USE_LOGGING_LEVEL", "warning")
|
||||
env.setdefault("ANONYMIZED_TELEMETRY", "false")
|
||||
# Its own process group so a stall-kill takes the whole tree, chromiums included.
|
||||
return subprocess.Popen(cmd, cwd=str(ARENA), env=env, start_new_session=True,
|
||||
stdout=open(args.log, "ab"), stderr=subprocess.STDOUT)
|
||||
|
||||
|
||||
def kill_tree(proc: subprocess.Popen) -> None:
|
||||
try:
|
||||
os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
|
||||
except (ProcessLookupError, PermissionError):
|
||||
pass
|
||||
subprocess.run(["pkill", "-f", "ms-playwright"], capture_output=True, check=False)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--arm", required=True)
|
||||
ap.add_argument("--model", required=True)
|
||||
ap.add_argument("--tasks", default="all")
|
||||
ap.add_argument("--seed", type=int, default=42)
|
||||
ap.add_argument("--stall", type=float, default=240.0, help="kill if no new episode lands this long")
|
||||
ap.add_argument("--rounds", type=int, default=12)
|
||||
ap.add_argument("--episode-timeout", type=float, default=100.0)
|
||||
ap.add_argument("--shots", default="first-last")
|
||||
ap.add_argument("--log", default="")
|
||||
# Resume window: episodes recorded after this epoch count as done, so a restarted supervisor
|
||||
# keeps the finished work of the run it replaces instead of re-running all 125.
|
||||
ap.add_argument("--since", type=float, default=0.0)
|
||||
args = ap.parse_args()
|
||||
args.log = args.log or f"/tmp/supervise_{args.arm}.log"
|
||||
|
||||
wanted = set(resolve_tasks(args.tasks)) if args.tasks != "all" else set(ALL)
|
||||
t_run = args.since or time.time()
|
||||
path = RESULTS_DIR / f"{args.arm}.jsonl"
|
||||
for rnd in range(1, args.rounds + 1):
|
||||
remaining = sorted(wanted - completed_tasks(args.arm, args.model, args.seed, t_run))
|
||||
if not remaining:
|
||||
break
|
||||
print(f"[supervisor] round {rnd}: {len(remaining)} tasks remaining", flush=True)
|
||||
proc = spawn(args.arm, remaining, args)
|
||||
last_size = path.stat().st_size if path.exists() else 0
|
||||
last_change = time.time()
|
||||
while proc.poll() is None:
|
||||
time.sleep(10)
|
||||
size = path.stat().st_size if path.exists() else 0
|
||||
if size != last_size:
|
||||
last_size, last_change = size, time.time()
|
||||
elif time.time() - last_change > args.stall:
|
||||
print(f"[supervisor] stalled {args.stall:.0f}s; killing tree", flush=True)
|
||||
kill_tree(proc)
|
||||
break
|
||||
time.sleep(2)
|
||||
done = completed_tasks(args.arm, args.model, args.seed, t_run)
|
||||
print(f"[supervisor] finished: {len(done)}/{len(wanted)} tasks have clean episodes", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,89 @@
|
||||
"""The 125 MiniWoB tasks, grouped by the capability each one actually exercises.
|
||||
|
||||
Grouping matters more than the headline number: "we are better across the board" is a claim about
|
||||
categories, and an arm that wins overall while losing every drag task has not earned that sentence.
|
||||
Categories are assigned from the task's mechanics, not its name prefix.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
CATEGORIES: dict[str, list[str]] = {
|
||||
"click_basic": [
|
||||
"click-test", "click-test-2", "click-test-transfer", "click-button",
|
||||
"click-button-sequence", "click-link", "click-dialog", "click-dialog-2",
|
||||
"click-widget", "click-color", "click-shades", "click-shape", "identify-shape",
|
||||
],
|
||||
"click_compound": [
|
||||
"click-checkboxes", "click-checkboxes-large", "click-checkboxes-soft",
|
||||
"click-checkboxes-transfer", "click-collapsible", "click-collapsible-2",
|
||||
"click-collapsible-nodelay", "click-collapsible-2-nodelay", "click-menu",
|
||||
"click-menu-2", "click-option", "click-scroll-list", "click-tab", "click-tab-2",
|
||||
"click-tab-2-easy", "click-tab-2-hard", "click-tab-2-medium", "navigate-tree",
|
||||
"click-pie", "click-pie-nodelay",
|
||||
],
|
||||
"text_entry": [
|
||||
"enter-text", "enter-text-2", "enter-text-dynamic", "enter-password",
|
||||
"enter-date", "enter-time", "focus-text", "focus-text-2", "login-user",
|
||||
"login-user-popup", "text-editor", "text-transform", "resize-textarea",
|
||||
"unicode-test", "generate-number", "highlight-text", "highlight-text-2",
|
||||
],
|
||||
"forms": [
|
||||
"book-flight", "book-flight-nodelay", "buy-ticket", "choose-date",
|
||||
"choose-date-easy", "choose-date-medium", "choose-date-nodelay", "choose-list",
|
||||
"form-sequence", "form-sequence-2", "form-sequence-3", "multi-layouts",
|
||||
"multi-orderings", "order-food", "sign-agreement", "use-autocomplete",
|
||||
"use-autocomplete-nodelay", "use-spinner", "search-engine", "social-media",
|
||||
"social-media-all", "social-media-some",
|
||||
],
|
||||
"reading": [
|
||||
"read-table", "read-table-2", "find-word", "phone-book", "scroll-text",
|
||||
"scroll-text-2", "stock-market", "daily-calendar", "terminal", "copy-paste",
|
||||
"copy-paste-2", "find-greatest", "odd-or-even",
|
||||
],
|
||||
"email": [
|
||||
"email-inbox", "email-inbox-delete", "email-inbox-forward", "email-inbox-forward-nl",
|
||||
"email-inbox-forward-nl-turk", "email-inbox-important", "email-inbox-nl-turk",
|
||||
"email-inbox-noscroll", "email-inbox-reply", "email-inbox-star-reply",
|
||||
],
|
||||
"drag": [
|
||||
"drag-box", "drag-circle", "drag-cube", "drag-items", "drag-items-grid",
|
||||
"drag-shapes", "drag-shapes-2", "drag-single-shape", "drag-sort-numbers",
|
||||
"use-slider", "use-slider-2", "use-colorwheel", "use-colorwheel-2",
|
||||
],
|
||||
"spatial": [
|
||||
"bisect-angle", "circle-center", "count-shape", "count-sides", "draw-circle",
|
||||
"draw-line", "find-midpoint", "grid-coordinate", "right-angle", "tic-tac-toe",
|
||||
"number-checkboxes", "visual-addition", "ascending-numbers",
|
||||
],
|
||||
"reasoning": [
|
||||
"simple-algebra", "simple-arithmetic", "guess-number", "hot-cold",
|
||||
],
|
||||
}
|
||||
|
||||
ALL: list[str] = sorted({t for group in CATEGORIES.values() for t in group})
|
||||
|
||||
CATEGORY_OF: dict[str, str] = {t: cat for cat, group in CATEGORIES.items() for t in group}
|
||||
|
||||
# Small, cheap slice that still touches every category; used for iteration before a full sweep.
|
||||
SMOKE: list[str] = [
|
||||
"click-test", "click-button", "click-link", "click-checkboxes", "click-tab",
|
||||
"enter-text", "enter-password", "focus-text", "login-user", "enter-date",
|
||||
"book-flight", "choose-list", "use-spinner", "search-engine",
|
||||
"read-table", "find-word", "copy-paste", "email-inbox", "email-inbox-delete",
|
||||
"use-slider", "drag-items", "grid-coordinate", "count-shape", "simple-algebra",
|
||||
]
|
||||
|
||||
|
||||
def resolve_tasks(spec: str) -> list[str]:
|
||||
"""Accept 'all', 'smoke', a category name, or a comma-separated list of task names."""
|
||||
spec = spec.strip()
|
||||
if spec == "all":
|
||||
return ALL
|
||||
if spec == "smoke":
|
||||
return SMOKE
|
||||
if spec in CATEGORIES:
|
||||
return sorted(CATEGORIES[spec])
|
||||
names = [s.strip() for s in spec.split(",") if s.strip()]
|
||||
unknown = [n for n in names if n not in CATEGORY_OF]
|
||||
if unknown:
|
||||
raise SystemExit(f"unknown task(s): {', '.join(unknown)}")
|
||||
return names
|
||||
Reference in New Issue
Block a user