arena: central data store manifest -- one directory holds every episode, screenshot and raw log

data/ now also carries logs/ (raw sweep + supervisor stdout, including browser-use's
own internal plan/eval lines) next to the episode book and screenshots; MANIFEST.md
documents the full schema. Goal capture widened to 600 chars for the research suites.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WsbS5x2rYsMDxP2kW3qqmQ

e2e/browser-v3/arena/bu_real.py

@@ -75,7 +75,8 @@ def find_task_cdp_url() -> str:
                 targets = json.loads(resp.read().decode())
         except Exception:
             continue
-        if any("miniwob" in str(t.get("url", "")) for t in targets):
+        if any(str(t.get("url", "")).startswith("http") and "devtools" not in str(t.get("url", ""))
+               for t in targets if t.get("type") == "page"):
             return f"http://localhost:{port}"
     raise RuntimeError(f"none of the recent CDP ports {RECENT_PORTS} hosts a miniwob page")
 

e2e/browser-v3/arena/data/MANIFEST.md

@@ -0,0 +1,19 @@
+# Arena data store — everything, one place
+
+This directory is the single source of truth for every measurement in the arena. Nothing here is
+ever rewritten; reruns append and reports keep the newest episode per (arm, model, task, seed).
+
+| path | contents |
+|---|---|
+| `all.jsonl` | every episode ever run, all arms, all models, all benchmarks — full metadata (goal text, reward raw+discounted, claim-vs-truth, wall/setup/first-action seconds, steps, tokens, LLM calls, error class) plus per-step records (action, think/act/perceive ms, tokens, vision flag, retries, page URL, error verdict, screenshot path) |
+| `<arm>.jsonl` | the same rows, sliced per arm for fast reads |
+| `shots/<run>/<arm>/<task>-s<seed>/NN.png` | what the agent saw at each step; `99.png` = final frame |
+| `logs/` | raw stdout of every sweep and supervisor round, including browser-use's own internal agent logs (its plans, evals, memory lines) |
+| `MANIFEST.md` | this file |
+
+Benchmarks recorded: MiniWoB (125 tasks; benchmark's own reward), AssistantBench validation
+(33 live-web questions; official question_scorer accuracy). Models: haiku-4-5, sonnet-4-6,
+sonnet-5, opus-5, plus early sonnet-4-6 partials. Stacks: ours (v1–v16) and real browser-use.
+
+Query tools live one directory up: `report.py` (scoreboards), `diffs.py` (task-level evidence
+trails between any two arms).

e2e/browser-v3/arena/run.py

@@ -59,6 +59,8 @@ def classify(exc: BaseException) -> str:
     text = str(exc).lower()
     if isinstance(exc, StepHang):
         return "infra_step_hang"
+    if "no running event loop" in text or "has been closed" in text:
+        return "infra_playwright_state"
     if "timeout" in text or "Timeout" in name:
         return "infra_timeout"
     if "target" in text and "closed" in text:
@@ -110,7 +112,7 @@ def run_episode(arm: str, task: str, seed: int, rec: Recorder, args: argparse.Na
                 pass
         return ep
     ep.setup_s = time.time() - t_setup
-    ep.goal = str(obs.get("goal") or "")[:300]
+    ep.goal = str(obs.get("goal") or "")[:600]
     policy.reset(ep.goal)
 
     t0 = time.time()

e2e/browser-v3/arena/supervisor.py

@@ -55,7 +55,8 @@ def spawn(arm: str, tasks: list[str], args: argparse.Namespace) -> subprocess.Po
     else:
         cmd = [PY, str(ARENA / "run.py"), "--arm", arm, "--tasks", ",".join(tasks), "--seeds", "1",
                "--seed-base", str(args.seed), "--model", args.model, "--shots", args.shots,
-               "--max-steps", str(args.max_steps)]
+               "--max-steps", str(args.max_steps), "--step-timeout", str(args.step_timeout),
+               "--setup-timeout", str(args.setup_timeout)]
     env = dict(os.environ)
     env.setdefault("MINIWOB_URL", "http://localhost:8099/miniwob/")
     env.setdefault("BROWSER_USE_LOGGING_LEVEL", "warning")
@@ -83,6 +84,8 @@ def main() -> None:
     ap.add_argument("--rounds", type=int, default=12)
     ap.add_argument("--episode-timeout", type=float, default=100.0)
     ap.add_argument("--max-steps", type=int, default=12)
+    ap.add_argument("--step-timeout", type=float, default=30.0)
+    ap.add_argument("--setup-timeout", type=float, default=60.0)
     ap.add_argument("--shots", default="first-last")
     ap.add_argument("--log", default="")
     # Resume window: episodes recorded after this epoch count as done, so a restarted supervisor