diff --git a/e2e/browser-v3/arena/ARENA.md b/e2e/browser-v3/arena/ARENA.md index 185bc23a..a866e335 100644 --- a/e2e/browser-v3/arena/ARENA.md +++ b/e2e/browser-v3/arena/ARENA.md @@ -261,6 +261,16 @@ Ungated (cost is ~1 line) — the pilot's 12 controls decide if that is a tax. P 8 targets (incl >=1 five-plus-part), controls 12/12. This is the second half of the browser-use loop diff (step verdicts); the first half (plan state) is dead. +VERDICT (same day): **fail — targets 1/8 (first-ever pilot target win, but a 3-part; pre-reg +required >=2 incl a >=5-part), controls 10/12 (both login-popup variants lost).** Both halves +of the browser-use loop diff are now tested and neither transfers into our loop. Escalation per +method: stop guessing mechanisms; trace-level diff of their WIN vs our LOSS on the same 6-part +task. First finding from our side: the loss is 8 confident, error-free clicks scoring 0 — and +step records did not store WHAT was clicked. Instrumentation fixed (StepRecord.target: resolved +accessible names per action); labeled diagnostic episode next. Their side shows a self-authored +running done-list in the memory field, persisted verbatim — possibly the real carrier, but no +v29 until the labeled trace says where ours actually diverges. + ## Positioning vs public generic-harness baselines (user-supplied 2026 survey) The comparable class is generic agents, NOT MiniWoB-specialized systems (HTML-T5++ 95.2 trained diff --git a/e2e/browser-v3/arena/config_files b/e2e/browser-v3/arena/config_files new file mode 120000 index 00000000..72a8e531 --- /dev/null +++ b/e2e/browser-v3/arena/config_files @@ -0,0 +1 @@ +/Users/eric/.cache/arena/webchorearena/BrowserGym/config_files \ No newline at end of file diff --git a/e2e/browser-v3/arena/recorder.py b/e2e/browser-v3/arena/recorder.py index 3ef161b5..58b1e9e5 100644 --- a/e2e/browser-v3/arena/recorder.py +++ b/e2e/browser-v3/arena/recorder.py @@ -24,6 +24,9 @@ class StepRecord: step: int action: str = "" + # Accessible name(s) of the element(s) the action targeted, resolved at act time. Added after + # a 6-part CompWoB loss read as 8 anonymous clicks -- a trace nobody can diagnose is data lost. + target: str = "" action_ms: float = 0.0 perceive_ms: float = 0.0 think_ms: float = 0.0 diff --git a/e2e/browser-v3/arena/run.py b/e2e/browser-v3/arena/run.py index e1c117b9..a21c4c9b 100644 --- a/e2e/browser-v3/arena/run.py +++ b/e2e/browser-v3/arena/run.py @@ -10,6 +10,7 @@ from __future__ import annotations import argparse import os +import re import signal import sys import time @@ -172,8 +173,14 @@ def run_episode(arm: str, task: str, seed: int, rec: Recorder, args: argparse.Na # The LLM call rides inside act(); urlopen's timeout does not cover every hang mode. decision = with_deadline(lambda: policy.act(obs, ep.goal), args.step_timeout + 90) perceive_ms = (time.time() - t_perc) * 1000 + tgt = "" + i2b, rnames = getattr(policy, "index_to_bid", None), getattr(policy, "row_names", None) + if i2b and rnames: + b2n = {b: rnames.get(i, "") for i, b in i2b.items()} + hit = [n for n in (b2n.get(m) for m in re.findall(r'"([^"]+)"', decision.action)) if n] + tgt = " | ".join(hit)[:160] rec_step = StepRecord( - step=step, action=decision.action, perceive_ms=perceive_ms, + step=step, action=decision.action, target=tgt, perceive_ms=perceive_ms, think_ms=getattr(decision, "think_ms", 0.0), axtree_chars=ax_chars, axtree_nodes=nodes, dom_chars=perception.dom_chars(obs) if args.dom_metrics else 0,