From 3eac8e8bcd353b58a0dcaafe2357d72756481c71 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Fri, 14 Aug 2026 02:59:17 -0700 Subject: [PATCH] arena: v27 verdict (fail, harmful; plan-state scaffolds 0-for-2) + v28 per-step self-eval (pre-registered) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WsbS5x2rYsMDxP2kW3qqmQ --- e2e/browser-v3/arena/ARENA.md | 20 ++++++++++++++++++++ e2e/browser-v3/arena/llm_policy.py | 27 +++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/e2e/browser-v3/arena/ARENA.md b/e2e/browser-v3/arena/ARENA.md index 93ef5357..185bc23a 100644 --- a/e2e/browser-v3/arena/ARENA.md +++ b/e2e/browser-v3/arena/ARENA.md @@ -241,6 +241,26 @@ nothing. Prediction: >=2 wins on the 8 pilot targets (incl. >=1 of the five >=5- 12/12 unharmed. browser-use's edge here is an actively-updated plan + step verdicts; this is the plan-state half. Same pilot lists as v26 (v26_pilot.json). +VERDICT (2026-08-14): **fail, and harmful — targets 0/8, controls 6/11 (baseline 12/12).** The +first launch also surfaced a reply-format collision (CLAUSE marker vs action-first replies → +empty actions), fixed before the counted run; the counted run still failed both prongs. The +'work ONLY on this clause' constraint evidently fights the fastpath/multi-action machinery that +wins the short tasks. Plan-state scaffolds are now 0-for-2 (passive v24, active v27): the model +does not need to be TOLD where it is. Goal integrity also verified intact end-to-end (309-char +8-part goal arrives whole). Remaining live hypothesis: the model never JUDGES whether its last +action achieved its intent — browser-use's reply schema forces a per-step self-evaluation that +rides in memory. That is v28. + +## PRE-REGISTERED (2026-08-14, before any v28 episode): per-step self-eval line + +Mechanism: `eval_line` (v28) — the reply format gains one trailing line, 'EVAL: '; the line is parsed and +rides in history, so each turn opens with the model's own verdict on its last step. No extra +LLM calls, action-first reply untouched (EVAL trails the action like the CLAUSE marker fix). +Ungated (cost is ~1 line) — the pilot's 12 controls decide if that is a tax. Prediction: >=2 of +8 targets (incl >=1 five-plus-part), controls 12/12. This is the second half of the +browser-use loop diff (step verdicts); the first half (plan state) is dead. + ## Positioning vs public generic-harness baselines (user-supplied 2026 survey) The comparable class is generic agents, NOT MiniWoB-specialized systems (HTML-T5++ 95.2 trained diff --git a/e2e/browser-v3/arena/llm_policy.py b/e2e/browser-v3/arena/llm_policy.py index 0e9ce15d..d4f50976 100644 --- a/e2e/browser-v3/arena/llm_policy.py +++ b/e2e/browser-v3/arena/llm_policy.py @@ -135,6 +135,20 @@ class LlmPolicy: return (f"\nINSTRUCTION CLAUSES (complete IN ORDER; you last reported clause {self.cur_clause}):\n" f"{rows}\nBegin your PLAN line with 'CLAUSE :' stating the clause you are working on.") + # v28: per-step self-evaluation. The reply gains one trailing line -- 'EVAL: ' -- which rides in history, so every turn + # opens with the model's own judgment of whether its last step worked. browser-use's schema + # forces the same thing (evaluation_previous_goal); ours trails the action so the + # action-first/truncation-proof reply shape is untouched. No extra LLM calls. + eval_line: bool = False + _last_eval: str = "" + + def eval_block(self) -> str: + if not self.eval_line: + return "" + return ("\nAfter your action line(s), end with one line 'EVAL: '.") + # v27: ACTIVE sub-goal ledger. v24's static checklist (all clauses, full text, every turn) # taxed attention and moved nothing; v26 proved plans execute cleanly against fresh pages. # What dies mid-chain is knowing WHICH clause is live. So: done clauses collapse to ticks, @@ -188,6 +202,8 @@ class LlmPolicy: extra = self.clause_block() if hasattr(self, "clause_block") else "" if hasattr(self, "history_block"): extra += self.history_block() + if getattr(self, "eval_line", False): + extra += self.eval_block() user = f"GOAL: {goal}{extra}\n\nACTIONS YOU ALREADY TOOK:\n{past}\n\nPAGE:\n{page}\n\nYour single next action:" content: Any = user if image_b64: @@ -665,6 +681,10 @@ class OpenSwarmLlmPolicy(LlmPolicy): redo = [c for c in clean_actions(raw2, limit=self.multi_cap) if c] if redo: chosen_list = redo + if self.eval_line: + m_ev = re.search(r"EVAL:\s*(.+)", raw or "") + if m_ev: + self.history.append(f"(your step-eval: {m_ev.group(1).strip()[:200]})") m_cl = re.search(r"CLAUSE\s+(\d+)", raw or "") if m_cl and (self.checklist or self.ledger): self.cur_clause = max(self.cur_clause, int(m_cl.group(1))) @@ -853,6 +873,13 @@ def build(name: str, model: str = "", endpoint: str = "", **_: Any) -> Any: scripted_drag=True, auto_complete=True, som=False, native_pickers=True, verify_terminal=True, post_mouse_vision=True, multi_cap=6, fill_verify=True, **v17) + if name == "osw-llm-v28": # v22 + per-step self-eval line riding in history (eval-memory half) + v28 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16, max_tokens=800) + return OpenSwarmLlmPolicy(name=name, multi=True, vision="progressive", fastpath=True, + scripted_drag=True, auto_complete=True, som=False, + native_pickers=True, verify_terminal=True, post_mouse_vision=True, + multi_cap=6, fill_verify=True, dispatch=True, offscreen=True, + eval_line=True, **v28) if name == "osw-llm-v27": # v22 + active sub-goal ledger (focused current-clause anchor, gated >=3 clauses) v27 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16, max_tokens=800) return OpenSwarmLlmPolicy(name=name, multi=True, vision="progressive", fastpath=True,