From ce418d4df6515034b57607e1e1d3971b7a6935d1 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Wed, 12 Aug 2026 15:47:05 -0700 Subject: [PATCH] arena: v21 -- the retry that could not truncate v20's strict retry reused the full verbose system prompt, so 7 episodes truncated on the retry exactly as on the original. The retry now swaps to a bare action-only prompt for that one call and restores state after. Also stated plainly in ARENA.md terms: our 0 false claims is structural (the arm never claims; the referee speaks), distinct from their volunteered wrong claims. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WsbS5x2rYsMDxP2kW3qqmQ --- e2e/browser-v3/arena/bu_real.py | 2 +- e2e/browser-v3/arena/llm_policy.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/e2e/browser-v3/arena/bu_real.py b/e2e/browser-v3/arena/bu_real.py index c53f66bb..6a2dfd0c 100644 --- a/e2e/browser-v3/arena/bu_real.py +++ b/e2e/browser-v3/arena/bu_real.py @@ -98,7 +98,7 @@ def make_env(task: str, seed: int, max_steps: int): if "." in task: import browsergym.assistantbench # noqa: F401 lazy: HF datasets machinery breaks playwright env_id = f"browsergym/{task}" if "." in task else f"browsergym/miniwob.{task}" - env = gym.make(env_id, headless=True, max_episode_steps=max_steps) + env = gym.make(env_id, headless=os.environ.get("OSW_ARENA_HEADED") != "1", max_episode_steps=max_steps) obs, _ = env.reset(seed=seed) return env, obs diff --git a/e2e/browser-v3/arena/llm_policy.py b/e2e/browser-v3/arena/llm_policy.py index c9d799f7..6c9bee60 100644 --- a/e2e/browser-v3/arena/llm_policy.py +++ b/e2e/browser-v3/arena/llm_policy.py @@ -704,6 +704,12 @@ def build(name: str, model: str = "", endpoint: str = "", **_: Any) -> Any: scripted_drag=True, auto_complete=True, som=False, native_pickers=True, verify_terminal=True, post_mouse_vision=True, multi_cap=6, fill_verify=True, **v17) + if name == "osw-llm-v21": # v20 + bare-prompt retry (the 7 residual truncation deaths) + v21 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16, max_tokens=800) + return OpenSwarmLlmPolicy(name=name, multi=True, vision="progressive", fastpath=True, + scripted_drag=True, auto_complete=True, som=False, + native_pickers=True, verify_terminal=True, post_mouse_vision=True, + multi_cap=6, fill_verify=True, dispatch=True, offscreen=True, **v21) if name == "osw-llm-v20": # v19 + strict-retry on unparseable replies + 800-token headroom v20 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16, max_tokens=800) return OpenSwarmLlmPolicy(name=name, multi=True, vision="progressive", fastpath=True,