From fe6717a817325cb0dfbda665cf288c4a3a1ba241 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Wed, 12 Aug 2026 12:51:47 -0700 Subject: [PATCH] arena: WITHDRAW AssistantBench entirely -- it measured plumbing, not skill, for both stacks Symmetric honesty: not just bu_real's scorer bug -- OUR arm only delivered an answer to the validator on 5 of 14 clean episodes, and live-web flakiness eats a third as infra. Every abench number (ours 0.050, theirs 0.000) reflects answer-delivery plumbing, not research ability. Withdrawn and marked do-not-cite; a real abench run needs the BrowserGym-native loop, not the CDP-attach shim. MiniWoB remains the one trustworthy measurement here -- and the one where every claim holds. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WsbS5x2rYsMDxP2kW3qqmQ --- e2e/browser-v3/arena/ARENA.md | 12 ++++++++---- e2e/browser-v3/arena/bu_real.py | 12 ++++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/e2e/browser-v3/arena/ARENA.md b/e2e/browser-v3/arena/ARENA.md index 2e468870..d44a85b8 100644 --- a/e2e/browser-v3/arena/ARENA.md +++ b/e2e/browser-v3/arena/ARENA.md @@ -78,10 +78,14 @@ change goals/values, not just RNG). The ~6-point pass@1-vs-pass@2 gap is decode Claude-5 lanes give no temperature control over; the remaining stable losses are the four engineering clusters (long forms, pixel precision, console emulation, stateful games). -**AssistantBench (live web, sonnet-5): INVALID pending re-run.** ours read 0.050; browser-use read -0.000 -- but the bu_real scorer was reading MiniWoB's WOB_REWARD_GLOBAL, which does not exist on -AssistantBench pages, so its 0.000 is a scoring bug, not a result. Both arms need re-scoring -through the AssistantBench env reward before any AssistantBench claim is made. Do not cite these. +**AssistantBench: WITHDRAWN as unmeasurable in this harness (both stacks).** Three distinct plumbing +faults, not performance: (1) bu_real scored via a MiniWoB-only page global -- fixed; (2) the agent's +answer must reach the env validator's chat, which neither stack does reliably (browser-use answers +through its own done(), ours emitted send_msg_to_user on only 5 of 14 clean episodes); (3) live-web +obs-extraction flakiness burns a third of episodes as infra. Every AssistantBench number here +(0.050, 0.000) measures answer-delivery plumbing, not research skill -- DO NOT CITE. A valid +AssistantBench comparison needs the BrowserGym-native agent loop, not our CDP-attach shim. Only +MiniWoB is a trustworthy measurement in this repo. v16 (verify-terminal, look-act-look, rapid-fire, sub-step confirm) FINAL: 82.1% on clean episodes -- statistically tied with v14, but the fixes hit their targets: email 10/10 (their best 6/10), diff --git a/e2e/browser-v3/arena/bu_real.py b/e2e/browser-v3/arena/bu_real.py index 9aae2315..c53f66bb 100644 --- a/e2e/browser-v3/arena/bu_real.py +++ b/e2e/browser-v3/arena/bu_real.py @@ -129,6 +129,11 @@ async def drive(goal: str, cdp_url: str, model: str, endpoint: str, max_steps: i stats["claimed_success"] = bool(history.is_successful()) except Exception: stats["claimed_success"] = False + try: + stats["final_answer"] = str(history.final_result() or "")[:800] + except Exception: + stats["final_answer"] = "" + usage = getattr(history, "usage", None) if usage: stats["prompt_tokens"] = int(getattr(usage, "total_prompt_tokens", 0) or 0) @@ -249,6 +254,13 @@ def main() -> None: stats = drive_in_thread(ep.goal, cdp_url, args.model, args.endpoint, args.max_steps, args.episode_timeout) if cdp_url else {} ep.wall_s = time.time() - t0 + # Their answer lives in their own done() action; the suite's validator reads the + # env chat. Relay it, or a research suite scores their real answers as silence. + if "." in task and stats.get("final_answer"): + try: + env.unwrapped.chat.add_message(role="assistant", msg=stats["final_answer"]) + except Exception: + pass try: ep.reward, ep.raw_reward = score(env, task) except Exception as exc: