diff --git a/e2e/browser-v3/arena/ARENA.md b/e2e/browser-v3/arena/ARENA.md index 4dbe843c..6047071d 100644 --- a/e2e/browser-v3/arena/ARENA.md +++ b/e2e/browser-v3/arena/ARENA.md @@ -215,6 +215,18 @@ given a fresh page. Updated diagnosis: the model loses track of which composed s active (goal-side, not page-side). Mechanism kept (it is correct-by-construction and free), but the cluster needs sub-goal tracking, not execution hygiene. +## PRE-REGISTERED (2026-08-13, before any v27 episode): active sub-goal ledger pilot + +Follow-up to the v26 disconfirmation; also killed a rival explanation first: the 8-part page +shows 34/34 interactives, 0 truncated — visibility is NOT the constraint. Mechanism: `ledger` +(v27) — the v24 clause split, rendered ACTIVELY: done clauses collapse to ticks, only the +current clause carries full text plus an imperative anchor, upcoming ones are 40-char stubs; +advancement is model-declared ('CLAUSE n') on page evidence. Differs from failed v24 exactly in +focus (one live clause) vs. passive full-list display. Gate: >=3 clauses; short tasks see +nothing. Prediction: >=2 wins on the 8 pilot targets (incl. >=1 of the five >=5-part), controls +12/12 unharmed. browser-use's edge here is an actively-updated plan + step verdicts; this is +the plan-state half. Same pilot lists as v26 (v26_pilot.json). + ## Positioning vs public generic-harness baselines (user-supplied 2026 survey) The comparable class is generic agents, NOT MiniWoB-specialized systems (HTML-T5++ 95.2 trained diff --git a/e2e/browser-v3/arena/llm_policy.py b/e2e/browser-v3/arena/llm_policy.py index 194d701e..e6e4e12c 100644 --- a/e2e/browser-v3/arena/llm_policy.py +++ b/e2e/browser-v3/arena/llm_policy.py @@ -127,12 +127,32 @@ class LlmPolicy: cur_clause: int = 1 def clause_block(self) -> str: + if self.ledger and len(self.clauses) >= 3: + return self.ledger_block() if not (self.checklist and self.clauses): return "" rows = "\n".join(f" {i}) {c}" for i, c in enumerate(self.clauses, 1)) return (f"\nINSTRUCTION CLAUSES (complete IN ORDER; you last reported clause {self.cur_clause}):\n" f"{rows}\nBegin your PLAN line with 'CLAUSE :' stating the clause you are working on.") + # v27: ACTIVE sub-goal ledger. v24's static checklist (all clauses, full text, every turn) + # taxed attention and moved nothing; v26 proved plans execute cleanly against fresh pages. + # What dies mid-chain is knowing WHICH clause is live. So: done clauses collapse to ticks, + # the current clause alone gets full text and an imperative anchor, upcoming ones are stubs. + # Advancement is model-declared ('CLAUSE n') only after the page shows the current one done. + ledger: bool = False + + def ledger_block(self) -> str: + k = min(self.cur_clause, len(self.clauses)) + done = " ".join(f"✓{i}" for i in range(1, k)) + up = "; ".join(f"{i}) {c[:40]}" for i, c in enumerate(self.clauses[k:], k + 1)) + return ("\nGOAL PROGRESS (strict order):" + + (f"\n done: {done}" if done else "") + + f"\n → CURRENT clause {k}: {self.clauses[k - 1]} <- work ONLY on this now" + + (f"\n after: {up}" if up else "") + + f"\nWhen the page shows clause {k} is complete, start your reply with 'CLAUSE {k + 1}' " + "to advance. Never work past the current clause; never re-do a ticked one.") + # v23: echo the page's REACTION into memory. Feedback tasks (hot/cold, too-high/too-low, # score counters) answer every action in page text; a history of bare actions hides the only # signal that matters. General mechanism: the text delta rides along, whatever it says. @@ -246,7 +266,7 @@ class OpenSwarmLlmPolicy(LlmPolicy): def reset(self, goal: str) -> None: self.history = [] - if self.checklist: + if self.checklist or self.ledger: parts = re.split(r",\s+(?:and\s+)?(?:then\s+)?|\s+then\s+|\s+and then\s+|\.\s+", goal) self.clauses = [p.strip().rstrip(".") for p in parts if len(p.strip()) > 3][:12] self.cur_clause = 1 @@ -645,7 +665,7 @@ class OpenSwarmLlmPolicy(LlmPolicy): if redo: chosen_list = redo m_cl = re.search(r"CLAUSE\s+(\d+)", raw or "") - if m_cl and self.checklist: + if m_cl and (self.checklist or self.ledger): self.cur_clause = max(self.cur_clause, int(m_cl.group(1))) if self.serial_multi and len(chosen_list) > 1: chosen_list = self.queue_rest(chosen_list) @@ -832,6 +852,13 @@ def build(name: str, model: str = "", endpoint: str = "", **_: Any) -> Any: scripted_drag=True, auto_complete=True, som=False, native_pickers=True, verify_terminal=True, post_mouse_vision=True, multi_cap=6, fill_verify=True, **v17) + if name == "osw-llm-v27": # v22 + active sub-goal ledger (focused current-clause anchor, gated >=3 clauses) + v27 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16, max_tokens=800) + return OpenSwarmLlmPolicy(name=name, multi=True, vision="progressive", fastpath=True, + scripted_drag=True, auto_complete=True, som=False, + native_pickers=True, verify_terminal=True, post_mouse_vision=True, + multi_cap=6, fill_verify=True, dispatch=True, offscreen=True, + ledger=True, **v27) if name == "osw-llm-v26": # v22 + serialized multi-action (queued steps re-resolve targets per-obs) v26 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16, max_tokens=800) return OpenSwarmLlmPolicy(name=name, multi=True, vision="progressive", fastpath=True,