diff --git a/e2e/browser-v3/arena/bu_real.py b/e2e/browser-v3/arena/bu_real.py index 0e60991d..1673f487 100644 --- a/e2e/browser-v3/arena/bu_real.py +++ b/e2e/browser-v3/arena/bu_real.py @@ -75,7 +75,8 @@ def find_task_cdp_url() -> str: targets = json.loads(resp.read().decode()) except Exception: continue - if any("miniwob" in str(t.get("url", "")) for t in targets): + if any(str(t.get("url", "")).startswith("http") and "devtools" not in str(t.get("url", "")) + for t in targets if t.get("type") == "page"): return f"http://localhost:{port}" raise RuntimeError(f"none of the recent CDP ports {RECENT_PORTS} hosts a miniwob page") diff --git a/e2e/browser-v3/arena/data/MANIFEST.md b/e2e/browser-v3/arena/data/MANIFEST.md new file mode 100644 index 00000000..a70f527e --- /dev/null +++ b/e2e/browser-v3/arena/data/MANIFEST.md @@ -0,0 +1,19 @@ +# Arena data store — everything, one place + +This directory is the single source of truth for every measurement in the arena. Nothing here is +ever rewritten; reruns append and reports keep the newest episode per (arm, model, task, seed). + +| path | contents | +|---|---| +| `all.jsonl` | every episode ever run, all arms, all models, all benchmarks — full metadata (goal text, reward raw+discounted, claim-vs-truth, wall/setup/first-action seconds, steps, tokens, LLM calls, error class) plus per-step records (action, think/act/perceive ms, tokens, vision flag, retries, page URL, error verdict, screenshot path) | +| `.jsonl` | the same rows, sliced per arm for fast reads | +| `shots///-s/NN.png` | what the agent saw at each step; `99.png` = final frame | +| `logs/` | raw stdout of every sweep and supervisor round, including browser-use's own internal agent logs (its plans, evals, memory lines) | +| `MANIFEST.md` | this file | + +Benchmarks recorded: MiniWoB (125 tasks; benchmark's own reward), AssistantBench validation +(33 live-web questions; official question_scorer accuracy). Models: haiku-4-5, sonnet-4-6, +sonnet-5, opus-5, plus early sonnet-4-6 partials. Stacks: ours (v1–v16) and real browser-use. + +Query tools live one directory up: `report.py` (scoreboards), `diffs.py` (task-level evidence +trails between any two arms). diff --git a/e2e/browser-v3/arena/run.py b/e2e/browser-v3/arena/run.py index e77240d2..3acef197 100644 --- a/e2e/browser-v3/arena/run.py +++ b/e2e/browser-v3/arena/run.py @@ -59,6 +59,8 @@ def classify(exc: BaseException) -> str: text = str(exc).lower() if isinstance(exc, StepHang): return "infra_step_hang" + if "no running event loop" in text or "has been closed" in text: + return "infra_playwright_state" if "timeout" in text or "Timeout" in name: return "infra_timeout" if "target" in text and "closed" in text: @@ -110,7 +112,7 @@ def run_episode(arm: str, task: str, seed: int, rec: Recorder, args: argparse.Na pass return ep ep.setup_s = time.time() - t_setup - ep.goal = str(obs.get("goal") or "")[:300] + ep.goal = str(obs.get("goal") or "")[:600] policy.reset(ep.goal) t0 = time.time() diff --git a/e2e/browser-v3/arena/supervisor.py b/e2e/browser-v3/arena/supervisor.py index d74fa6e2..5aa0f9f0 100644 --- a/e2e/browser-v3/arena/supervisor.py +++ b/e2e/browser-v3/arena/supervisor.py @@ -55,7 +55,8 @@ def spawn(arm: str, tasks: list[str], args: argparse.Namespace) -> subprocess.Po else: cmd = [PY, str(ARENA / "run.py"), "--arm", arm, "--tasks", ",".join(tasks), "--seeds", "1", "--seed-base", str(args.seed), "--model", args.model, "--shots", args.shots, - "--max-steps", str(args.max_steps)] + "--max-steps", str(args.max_steps), "--step-timeout", str(args.step_timeout), + "--setup-timeout", str(args.setup_timeout)] env = dict(os.environ) env.setdefault("MINIWOB_URL", "http://localhost:8099/miniwob/") env.setdefault("BROWSER_USE_LOGGING_LEVEL", "warning") @@ -83,6 +84,8 @@ def main() -> None: ap.add_argument("--rounds", type=int, default=12) ap.add_argument("--episode-timeout", type=float, default=100.0) ap.add_argument("--max-steps", type=int, default=12) + ap.add_argument("--step-timeout", type=float, default=30.0) + ap.add_argument("--setup-timeout", type=float, default=60.0) ap.add_argument("--shots", default="first-last") ap.add_argument("--log", default="") # Resume window: episodes recorded after this epoch count as done, so a restarted supervisor