From 3972f543fc178f5fec585cd6566c071efe87bfcb Mon Sep 17 00:00:00 2001 From: ciregenz Date: Sun, 16 Aug 2026 12:57:38 -0700 Subject: [PATCH] arena: v42 terminal answer protocol (bare final answer on string-match goals) -- generic output hygiene, offline-verified Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WsbS5x2rYsMDxP2kW3qqmQ --- e2e/browser-v3/arena/llm_policy.py | 12 ++++++++++++ e2e/browser-v3/arena/run.py | 20 ++++++++++++-------- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/e2e/browser-v3/arena/llm_policy.py b/e2e/browser-v3/arena/llm_policy.py index 7ecb328b..4e98cb69 100644 --- a/e2e/browser-v3/arena/llm_policy.py +++ b/e2e/browser-v3/arena/llm_policy.py @@ -192,6 +192,8 @@ class LlmPolicy: table_md: bool = False # v41: scripted freehand-circle geometry (run.py-side ring path). Feature-gated on circle goals. draw_circle: bool = False + # v42: terminal answer protocol (run.py-side). Force a bare final answer on string-match goals. + answer_protocol: bool = False # v39: mutation-diff action feedback -- run.py appends the page-text delta after each action # (Agent-E's MutationObserver, approximated text-side). Autocomplete popups, error banners, # and new rows become explicit feedback instead of something the model must notice unaided. @@ -1028,6 +1030,16 @@ def build(name: str, model: str = "", endpoint: str = "", **_: Any) -> Any: local_ctx=True, blocker_probe=True, suppress_wrappers=True, force_unblock=True, native_js_fallback=True, table_md=True, mutation_diff=True, **c) + if name == "osw-llm-v42": # v41 + terminal answer protocol (string-match benchmarks) + v42 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30 + + OSW_SYSTEM_V36, max_tokens=800) + return OpenSwarmLlmPolicy(name=name, multi=True, vision="progressive", fastpath=True, + scripted_drag=True, auto_complete=True, som=False, + native_pickers=True, verify_terminal=True, post_mouse_vision=True, + multi_cap=6, fill_verify=True, dispatch=True, offscreen=True, + local_ctx=True, blocker_probe=True, suppress_wrappers=True, + force_unblock=True, native_js_fallback=True, escape_token=True, + table_md=True, draw_circle=True, answer_protocol=True, **v42) if name == "osw-llm-v41": # v40 champion + scripted draw-circle geometry primitive v41 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30 + OSW_SYSTEM_V36, max_tokens=800) diff --git a/e2e/browser-v3/arena/run.py b/e2e/browser-v3/arena/run.py index 309ad2e8..d84c3c2c 100644 --- a/e2e/browser-v3/arena/run.py +++ b/e2e/browser-v3/arena/run.py @@ -191,14 +191,18 @@ def run_episode(arm: str, task: str, seed: int, rec: Recorder, args: argparse.Na try: for step in range(1, args.max_steps + 1): t_perc = time.time() - # Q&A goals are scored on the CHAT ANSWER; an episode that pages forever and never - # answers scores 0 no matter what it learned (measured: 1/17 chore episodes sent one). - # On the last budgeted step, tell the policy the budget is up so it answers NOW. - if (step == args.max_steps and hasattr(policy, "history") - and re.search(r"^(calculate|how many|what|which|find|count|tell me|list|give me)|\?\s*$", - ep.goal.strip(), re.I)): - policy.history.append("(FINAL STEP: budget exhausted -- if the goal asks for " - "information, reply ONLY send_msg_to_user(\"\") now)") + # Terminal answer protocol (gated by policy.answer_protocol): string-match benchmarks + # grade ONLY the last chat message and auto-fail on none; exact-match dies on wrapper + # text. On the last budgeted step force a BARE answer -- no markdown, no hedging. + if (getattr(policy, "answer_protocol", False) and step == args.max_steps + and hasattr(policy, "history") + and re.search(r"^(calculate|how many|what|which|find|count|tell me|list|give me" + r"|name|identify|when|who|where)\b|\?\s*$", ep.goal.strip(), re.I)): + policy.history.append( + "(FINAL STEP -- budget is up. Reply with EXACTLY ONE action: " + "send_msg_to_user(\"\") where is the BARE value only -- a " + "number, a name, or a comma-separated list. NO markdown, NO 'the answer is', " + "NO units unless the goal asked. If genuinely unknown, answer your best guess.)") nodes, ax_chars = perception.axtree_stats(obs) # The LLM call rides inside act(); urlopen's timeout does not cover every hang mode. decision = with_deadline(lambda: policy.act(obs, ep.goal), args.step_timeout + 90)