diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index 7f63a314..861e68bb 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -58,6 +58,13 @@ from backend.apps.tools_lib.tools_lib import load_builtin_permissions logger = logging.getLogger(__name__) +# Mutating actions that can carry an `expect` (the change they should cause) and be +# confirmed after running. Reads/waits aren't here, there's nothing to confirm. +_CONFIRM_TOOLS = { + "BrowserClick", "BrowserClickIndex", "BrowserClickByName", + "BrowserType", "BrowserNavigate", "BrowserPressKey", "BrowserBatch", +} + async def execute_browser_tool( tool_name: str, tool_input: dict, browser_id: str, tab_id: str = "", @@ -979,12 +986,13 @@ async def run_browser_agent( tool_input = tu.input if tu.name == "BrowserListInteractives" and current_next_goal: tool_input = {**tu.input, "goal": current_next_goal} + + async def _wait_exec(tool, params, bid, tid): + return await _cancellable(execute_browser_tool(tool, params, bid, tid)) + if tu.name == "BrowserWait": - # Smart wait: return as soon as the page's network settles - # instead of sleeping the full fixed duration (the audit's - # 42%-of-time hog). Caps at the requested ms; never premature. - async def _wait_exec(tool, params, bid, tid): - return await _cancellable(execute_browser_tool(tool, params, bid, tid)) + # Smart wait: return as soon as the page is ready (target or DOM + # settle), not on a blind timer (the audit's 42%-of-time hog). result = await browser_wait.smart_wait( _wait_exec, browser_id, tab_id, tu.input.get("milliseconds"), until=(tu.input.get("until") or ""), @@ -998,6 +1006,27 @@ async def run_browser_agent( break elapsed_ms = int((time.time() - start) * 1000) + # Act-and-confirm: if the agent declared the change it expects, VERIFY + # it actually happened, success is observed, never assumed. A hit returns + # fast (act + confirm in one turn); a miss is a clear "may not have worked" + # (and a wedge surfaces as a clean not-confirmed, not a blind 20s timeout), + # so the agent never claims a success it didn't see or re-fires blindly. + _expect = (str(tu.input.get("expect") or "").strip() + if isinstance(tu.input, dict) else "") + if _expect and "error" not in result and tu.name in _CONFIRM_TOOLS: + _conf = await browser_wait.smart_wait(_wait_exec, browser_id, tab_id, 3500, until=_expect) + if isinstance(_conf, dict): + result["confirmed"] = bool(_conf.get("found")) + if _conf.get("found"): + result["text"] = f"{result.get('text') or ''}\nConfirmed: '{_expect}' is now present." + else: + result["text"] = ( + f"{result.get('text') or ''}\nNOT confirmed: '{_expect}' did not appear within " + f"{_conf.get('waited_ms')}ms, so the action may not have worked. Check the page " + "before assuming success, and never re-fire an irreversible action " + "(Send/Submit/Pay/Post) without first verifying the previous one did not go through." + ) + action_log.append({ "tool": tu.name, "input": tu.input, diff --git a/backend/apps/agents/browser/browser_schema.py b/backend/apps/agents/browser/browser_schema.py index b4f08763..e1d318b2 100644 --- a/backend/apps/agents/browser/browser_schema.py +++ b/backend/apps/agents/browser/browser_schema.py @@ -12,6 +12,20 @@ MODEL_MAP = { "haiku": "claude-haiku-4-5-20251001", } +# The change an action should cause, declared by the agent and CONFIRMED after the +# action runs (success is observed, never assumed). A hit returns fast; a miss tells +# the agent it may not have worked instead of letting it claim a false success. +_EXPECT_DESC = { + "type": "string", + "description": ( + "Optional but recommended: the specific change this action should cause, a " + "button label, text, or element you expect to see afterward (e.g. 'Write a " + "message', the recipient's name in the thread). It's confirmed right after, so " + "you learn whether it actually worked. REQUIRED for anything you can't undo " + "(Send/Submit/Pay/Post): set it to proof the action landed." + ), +} + BROWSER_TOOLS_SCHEMA = [ { "name": "ReportProgress", @@ -108,6 +122,7 @@ BROWSER_TOOLS_SCHEMA = [ "type": "object", "properties": { "selector": {"type": "string", "description": "CSS selector of the element to click."}, + "expect": _EXPECT_DESC, }, "required": ["selector"], }, @@ -210,6 +225,7 @@ BROWSER_TOOLS_SCHEMA = [ "type": "integer", "description": "The numeric index from BrowserListInteractives (1-based).", }, + "expect": _EXPECT_DESC, }, "required": ["index"], }, @@ -528,6 +544,17 @@ SYSTEM_PROMPT = ( "(BrowserScreenshot, BrowserGetText, BrowserGetConsole, BrowserGetElements, BrowserWait) do not " "require ReportProgress.\n\n" + "## Act and confirm: trust only what you observe\n" + "Success is OBSERVED, never assumed. On any action that changes the page (click, " + "type, navigate), add `expect`: the change it should cause (a label, text, or the " + "element you expect to see). It's confirmed right after, a hit comes back fast and " + "you move on; a 'NOT confirmed' means it may not have worked, so check the page " + "instead of pressing on. For anything you CANNOT undo (Send, Submit, Pay, Post): " + "first make sure the goal isn't already done (e.g. your message isn't already the " + "last one in the thread), pass `expect` set to proof it landed, and NEVER fire it a " + "second time unless you have verified the first did NOT go through. This is how you " + "avoid both ghost-successes and double-sends.\n\n" + "## Loop awareness\n" "If you see a tool result containing 'LOOP DETECTED' or '⚠️', it means you " "have called the same tool with the same parameters and gotten the same " diff --git a/backend/apps/agents/browser/browser_wait.py b/backend/apps/agents/browser/browser_wait.py index 8288c50c..10264f20 100644 --- a/backend/apps/agents/browser/browser_wait.py +++ b/backend/apps/agents/browser/browser_wait.py @@ -162,5 +162,5 @@ async def smart_wait(execute_fn, browser_id, tab_id, max_ms, *, until="", text += " The page or tab appears unresponsive." if last_url: text += f" Current URL: {last_url}" - return {"text": text, "url": last_url, "settled": settled, "hung": hung, + return {"text": text, "url": last_url, "settled": settled, "found": found, "hung": hung, "waited_ms": waited, **({"error": "page unresponsive"} if hung else {})} diff --git a/backend/tests/test_browser_agent_loop.py b/backend/tests/test_browser_agent_loop.py index 5de95709..4be14b20 100644 --- a/backend/tests/test_browser_agent_loop.py +++ b/backend/tests/test_browser_agent_loop.py @@ -93,7 +93,10 @@ def _install(monkeypatch, primary, aux): # smart-wait probes via evaluate; report 'settled' so BrowserWait returns # fast in tests instead of riding the full cap. if action == "evaluate" and "getEntriesByType('resource')" in str(params.get("expression", "")): - return {"text": '{"ready": true, "quiet": 9999}', "url": DOC_URL} + expr = str(params.get("expression", "")) + # a confirm/target probe embeds a non-empty `const spec="..."`; report it found + found = "const spec=" in expr and 'const spec=""' not in expr + return {"text": json.dumps({"ready": True, "quiet": 9999, "elems": 100, "found": found}), "url": DOC_URL} if action == "list_interactives": return {"text": '1 interactive elements:\n[1]