diff --git a/e2e/browser-v3/arena/llm_policy.py b/e2e/browser-v3/arena/llm_policy.py index 8e1ba612..41fdd9fe 100644 --- a/e2e/browser-v3/arena/llm_policy.py +++ b/e2e/browser-v3/arena/llm_policy.py @@ -196,6 +196,8 @@ class LlmPolicy: pick_result: bool = False # v42: terminal answer protocol (run.py-side). Force a bare final answer on string-match goals. answer_protocol: bool = False + # v45: read-your-writes doctrine (system rung, gated on state-change goals). Re-read + retry. + read_your_writes: bool = False # v43: answer-schema conformance gate (run.py-side). Validate send_msg against the task's own # provided JSON schema and bounce non-conforming answers -- generic instruction-following. schema_gate: bool = False @@ -276,6 +278,10 @@ class LlmPolicy: extra += self.eval_block() if getattr(self, "note_pad", False) and self.notes_gated(goal): extra += self.note_block() + if getattr(self, "read_your_writes", False) and re.search( + r"\b(post|comment|upvote|down ?vote|vote|create|add|edit|submit|reply|open an issue|change|update|delete|remove)\b", goal, re.I): + extra += ("\nSTATE-CHANGE TASK: after your write action, RE-READ and CONFIRM the change " + "is visible before finishing; if not visible, repeat the write.") user = f"GOAL: {goal}{extra}\n\nACTIONS YOU ALREADY TOOK:\n{past}\n\nPAGE:\n{page}\n\nYour single next action:" content: Any = user if image_b64: @@ -863,6 +869,13 @@ If the goal specifies a required response format or JSON schema (e.g. a FinalAge fields like task_type and status), your FINAL send_msg_to_user MUST be exactly that JSON object with all required fields filled -- not prose, not a bare value. Match the schema literally.""" +OSW_SYSTEM_V45 = """ +If the task CHANGES state (post, comment, vote, create, edit, submit, upload), a single click is +not enough -- the write often does not register on the first try. After the action, RE-READ: look +at the page (or re-open where the change should appear) and CONFIRM the change is actually visible +(the comment shows, the vote is highlighted, the record exists). If it is NOT visible, do the write +AGAIN. Only finish once you have SEEN the change persist.""" + OSW_SYSTEM_V36 = """ If NO row in the list matches the goal's named target, reply exactly no_match() (a first-class choice) instead of clicking a similar-but-wrong row -- a wrong click on a named target is often @@ -1101,6 +1114,16 @@ def build(name: str, model: str = "", endpoint: str = "", **_: Any) -> Any: force_unblock=True, native_js_fallback=True, escape_token=True, table_md=True, draw_circle=True, answer_protocol=True, schema_gate=False, **v43) # DISABLED: gate falsely bounced valid JSON + if name == "osw-llm-v45": # v42 + read-your-writes (state-change tasks: re-read + retry) + v45 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30 + + OSW_SYSTEM_V36 + OSW_SYSTEM_V45, max_tokens=900) + return OpenSwarmLlmPolicy(name=name, multi=True, vision="progressive", fastpath=True, + scripted_drag=True, auto_complete=True, som=False, + native_pickers=True, verify_terminal=True, post_mouse_vision=True, + multi_cap=6, fill_verify=True, dispatch=True, offscreen=True, + local_ctx=True, blocker_probe=True, suppress_wrappers=True, + force_unblock=True, native_js_fallback=True, escape_token=True, + table_md=True, answer_protocol=True, read_your_writes=True, **v45) if name == "osw-llm-v42": # v41 + terminal answer protocol (string-match benchmarks) v42 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30 + OSW_SYSTEM_V36, max_tokens=800)