diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index 1bbbc803..8ce5d3b7 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -1602,6 +1602,65 @@ async def run_browser_agent( continue # Intra-run batch replay: run a learned mechanical flow for many inputs at machine speed, verify every step, gate sends, never ghost. Reads/searches loop freely; irreversible steps refuse. + if tu.name == "BrowserActVerified": + from backend.apps.agents.browser import browser_verified_step + from backend.apps.agents.browser.browser_prestage import P_BLOCKED_CLICK_RE + p_steps_in = tu.input.get("steps") or [] + p_step_lines: list[str] = [] + p_all_ok = True + if not p_steps_in: + p_va_text = "No steps given; nothing to do." + else: + for p_si, p_raw in enumerate(p_steps_in[:4], start=1): + p_tgt = str((p_raw or {}).get("target") or "") + # the solo-send rule holds here in CODE: an irreversible-smelling target is refused, exactly like a batch + if P_BLOCKED_CLICK_RE.search(p_tgt): + p_step_lines.append(f"{p_si}. REFUSED: {p_tgt!r} looks irreversible; do it as a SOLO click with an `expect` proof.") + p_all_ok = False + break + p_vstep = browser_verified_step.VerifiedStep( + kind=str(p_raw.get("action") or "click"), target=p_tgt, + role=str(p_raw.get("role") or ""), text=str(p_raw.get("text") or ""), + expect=str(p_raw.get("expect") or "")) + p_st = time.time() + p_vr = await p_cancellable(browser_verified_step.run_verified_step( + p_vstep, browser_id, tab_id, execute_browser_tool)) + if p_vr is None: + p_step_lines.append(f"{p_si}. cancelled"); p_all_ok = False; break + p_el = int((time.time() - p_st) * 1000) + action_log.append({ + "tool": "BrowserActVerified", "input": p_raw, "ok": p_vr["ok"], + "result_summary": (f"{p_vstep.kind} {p_tgt!r} verified" if p_vr["ok"] + else str(p_vr["note"]))[:200], + "elapsed_ms": p_el, + }) + browser_metrics.record_tool( + session_id, browser_id, turn, "BrowserActVerified", p_el, ok=p_vr["ok"], + error="" if p_vr["ok"] else str(p_vr["note"]), is_loop=False, + stagnation_streak=0, result_len=0) + if p_vr["ok"]: + p_step_lines.append(f"{p_si}. {p_vstep.kind} {p_tgt!r}: OK (verified)") + else: + p_step_lines.append(f"{p_si}. {p_vstep.kind} {p_tgt!r}: FAILED ({p_vr['note']}); remaining steps skipped") + p_all_ok = False + break + p_va_text = ("All steps verified:\n" if p_all_ok else "Stopped early:\n") + "\n".join(p_step_lines) + # fold the post-plan page state in so the model's next turn already sees the result + p_va_state = await post_action_state( + "BrowserBatch", {}, {"ok": True}, browser_id, tab_id, + wait_exec=execute_browser_tool, goal=current_next_goal or "", + seen_lines=attached_state_seen) + if p_va_state: + p_va_text += p_va_state + fresh_state_pending = True + tool_results.append({"type": "tool_result", "tool_use_id": tu.id, "content": [{"type": "text", "text": p_va_text}]}) + result_msg = Message(role="tool_result", content={"text": p_va_text, "tool_name": tu.name, "elapsed_ms": 0}) + session.messages.append(result_msg) + await ws_manager.send_to_session(session_id, "agent:message", { + "session_id": session_id, "message": result_msg.model_dump(mode="json"), + }) + continue + if tu.name == "BrowserRepeatFlow": steps_tmpl = tu.input.get("steps") or [] values = [str(v) for v in (tu.input.get("values") or [])] diff --git a/backend/apps/agents/browser/browser_loop.py b/backend/apps/agents/browser/browser_loop.py index 0a536f29..141237cd 100644 --- a/backend/apps/agents/browser/browser_loop.py +++ b/backend/apps/agents/browser/browser_loop.py @@ -219,7 +219,7 @@ def stagnation_exhausted(streak: int) -> bool: # State-changing tools: a task that needed to DO something must land one of these. P_PRODUCTIVE_TOOLS = { "BrowserClick", "BrowserClickIndex", "BrowserType", "BrowserNavigate", - "BrowserPressKey", "BrowserScroll", "BrowserBatch", + "BrowserPressKey", "BrowserScroll", "BrowserBatch", "BrowserActVerified", } # Read/extract tools: a look-only task's evidence is that a read returned content. P_READ_TOOLS = { @@ -263,7 +263,7 @@ def recoverable_tool_error(err: str) -> bool: # Actions that DIRTY the page so replay-from-here is no longer equivalent to a clean dispatch. Navigation and reads don't dirty anything (they just get us to the page), so the deferred replay re-check is allowed after only those. P_REPLAY_DIRTYING_TOOLS = { "BrowserType", "BrowserClick", "BrowserClickIndex", - "BrowserPressKey", "BrowserScroll", "BrowserBatch", + "BrowserPressKey", "BrowserScroll", "BrowserBatch", "BrowserActVerified", } diff --git a/backend/apps/agents/browser/browser_schema.py b/backend/apps/agents/browser/browser_schema.py index a57ecc43..0e62bc8b 100644 --- a/backend/apps/agents/browser/browser_schema.py +++ b/backend/apps/agents/browser/browser_schema.py @@ -387,6 +387,47 @@ BROWSER_TOOLS_SCHEMA = [ "required": ["index"], }, }, + { + "name": "BrowserActVerified", + "description": ( + "Run a short SEQUENCE of dependent UI steps (2-4) in one call, where each " + "step must take effect before the next: open a menu then pick an item, " + "fill a field then the next one, expand a section then click inside it. " + "Each step names its target ELEMENT BY NAME (resolved fresh against the " + "live page at act time, so a stale index can't bite) and is VERIFIED in " + "code (did the expected change actually happen), with one automatic " + "re-aim on a miss. Steps:\n" + "- { action: 'click', target: '', role?: 'button'|'link'|..., " + "expect?: 'appeared:'|'gone:'|'url_changed'|'changed' }\n" + "- { action: 'fill', target: '', text: '' } " + "(auto-verifies the text committed)\n" + "Execution stops at the first step that can't be verified and you get " + "per-step results plus what went wrong. NEVER put an irreversible action " + "(send/submit/post/pay/delete/confirm) here; those stay SOLO clicks with " + "an `expect` proof, as always." + ), + "input_schema": { + "type": "object", + "properties": { + "steps": { + "type": "array", + "maxItems": 4, + "items": { + "type": "object", + "properties": { + "action": {"type": "string", "enum": ["click", "fill"]}, + "target": {"type": "string"}, + "role": {"type": "string"}, + "text": {"type": "string"}, + "expect": {"type": "string"}, + }, + "required": ["action", "target"], + }, + }, + }, + "required": ["steps"], + }, + }, { "name": "BrowserBatch", "description": ( @@ -1119,5 +1160,6 @@ ACTION_TOOLS_REQUIRING_REPORT = { "BrowserClickIndex", # Phase 3 "BrowserClickPoint", # app mode: tap a canvas/game at a screen point "BrowserBatch", # Phase 4 + "BrowserActVerified", # verified-step sequence (mutates state like a batch) "AppInvoke", # app mode: invoking an app action mutates state } diff --git a/backend/apps/agents/browser/browser_verified_action.py b/backend/apps/agents/browser/browser_verified_action.py index 2eae16b3..276f9694 100644 --- a/backend/apps/agents/browser/browser_verified_action.py +++ b/backend/apps/agents/browser/browser_verified_action.py @@ -18,12 +18,47 @@ Expectations (kind, or "kind:arg"): """ import re -from typing import Tuple +from typing import List, Optional, Tuple # Match payload_in_textbox: long values truncate in the list, so compare on a prefix. P_VALUE_PREFIX_LEN = 24 P_TEXTBOX_LINE = "]<*>?< ""...>, the format every list uses. +P_ROW_RE = re.compile(r'\[(\d+)\]\*?<\s*([a-z]+)\s+"([^"]*)"', re.I) +P_NAME_PREFIX_LEN = 40 # long card-blob names mutate their suffix between visits + + +def parse_rows(state_text: str) -> List[Tuple[int, str, str]]: + """(index, role, name) for each interactive row. Site-agnostic: it reads the + universal list shape, not any particular page's elements.""" + return [(int(m.group(1)), m.group(2).lower(), m.group(3)) + for m in P_ROW_RE.finditer(state_text or "")] + + +def resolve_target(state_text: str, name: str, role: str = "") -> Optional[Tuple[int, str, str]]: + """Resolve a semantic target against the LIVE list, late, the moment before acting, + so a stale index can't bite. Strictest UNAMBIGUOUS tier wins: exact (role,name) -> + exact name -> name-prefix. Two matches at a tier = ambiguous = None (hand back + rather than click the wrong thing). Mirrors the renderer's click-by-name tiers.""" + want = (name or "").strip().lower() + if not want: + return None + wrole = (role or "").strip().lower() + rows = parse_rows(state_text) + + def uniq(cands: List[Tuple[int, str, str]]) -> Optional[Tuple[int, str, str]]: + return cands[0] if len(cands) == 1 else None + + hit = uniq([r for r in rows if r[2].strip().lower() == want and (not wrole or r[1] == wrole)]) + if hit: + return hit + hit = uniq([r for r in rows if r[2].strip().lower() == want]) + if hit: + return hit + pre = want[:P_NAME_PREFIX_LEN] + return uniq([r for r in rows if r[2].strip().lower().startswith(pre) and (not wrole or r[1] == wrole)]) + def parse_expectation(expect: str) -> Tuple[str, str]: """(kind, arg) from 'kind' or 'kind:arg'. Unknown kinds are returned as-is and diff --git a/backend/apps/agents/browser/browser_verified_step.py b/backend/apps/agents/browser/browser_verified_step.py new file mode 100644 index 00000000..af3f7a82 --- /dev/null +++ b/backend/apps/agents/browser/browser_verified_step.py @@ -0,0 +1,101 @@ +"""One verified action, the executor's unit of work: resolve the target LATE against +the live page, act, verify the SPECIFIC expected effect, and re-aim on a miss, all in +code, no LLM turn. This generalizes the send-script's proven fill->verify->send->verify +from one LinkedIn flow to any site: the target is a semantic name, the effect is a +generic expectation, and neither knows about any particular page. + +The one safety invariant, same bar as the send-script: an IRREVERSIBLE step (send / +submit / pay) is NEVER re-fired. If it acted but the effect can't be verified, it +returns an honest "acted, unverified, do NOT repeat" note instead of retrying, so a +receipt we couldn't read can never become a double-send. +""" + +import asyncio +import logging +from typing import Awaitable, Callable, Optional, Tuple + +from pydantic import BaseModel, ConfigDict + +from backend.apps.agents.browser import browser_verified_action as va + +logger = logging.getLogger(__name__) + +ToolRunner = Callable[[str, dict, str, str], Awaitable[dict]] + + +class VerifiedStep(BaseModel): + model_config = ConfigDict(validate_assignment=True) + kind: str # "click" | "fill" + target: str # semantic element name to resolve against the live list + role: str = "" # optional role hint ("button", "link", "textbox") to disambiguate + text: str = "" # for a fill + expect: str = "" # generic expectation; defaults to filled: / changed + irreversible: bool = False # send/submit/pay: acted-but-unverified NEVER re-fires + + +async def p_fresh(execute_tool: ToolRunner, browser_id: str, tab_id: str) -> Tuple[str, str]: + try: + r = await asyncio.wait_for( + execute_tool("BrowserListInteractives", {}, browser_id, tab_id), timeout=6.0) + except Exception: + return "", "" + if not isinstance(r, dict) or "error" in r: + return "", "" + return str(r.get("text") or ""), str(r.get("url") or "") + + +async def p_act(step: VerifiedStep, index: Optional[int], + browser_id: str, tab_id: str, execute_tool: ToolRunner) -> dict: + if step.kind == "fill": + return await execute_tool( + "BrowserClickIndex", {"index": index, "text": step.text}, browser_id, tab_id) + if index is not None: + return await execute_tool("BrowserClickIndex", {"index": index}, browser_id, tab_id) + # a click whose index didn't resolve falls to by-name (full-DOM search, past the list cap) + return await execute_tool( + "BrowserClickByName", {"name": step.target, "role": step.role}, browser_id, tab_id) + + +def p_default_expect(step: VerifiedStep) -> str: + if step.expect: + return step.expect + return f"filled:{step.text}" if step.kind == "fill" else "changed" + + +async def run_verified_step( + step: VerifiedStep, browser_id: str, tab_id: str, execute_tool: ToolRunner, + settle_s: float = 0.8, max_reaim: int = 1, +) -> dict: + """{ok, verified, acted, note}. ok == the expected effect was observed. A reversible + step that doesn't verify is re-aimed (re-resolve + re-act) up to max_reaim times; an + irreversible one is never re-fired once it has acted.""" + expect = p_default_expect(step) + note = "" + for attempt in range(max_reaim + 1): + before, before_url = await p_fresh(execute_tool, browser_id, tab_id) + tgt = va.resolve_target(before, step.target, step.role) + index = tgt[0] if tgt else None + if step.kind == "fill" and index is None: + return {"ok": False, "verified": False, "acted": False, + "note": f"could not resolve a field named {step.target!r} to fill"} + r = await p_act(step, index, browser_id, tab_id, execute_tool) + acted = isinstance(r, dict) and "error" not in r + if not acted: + note = f"action errored: {r.get('error') if isinstance(r, dict) else r}" + if step.irreversible: + # an errored irreversible action provably did NOT happen; safe to stop, never retry blindly + return {"ok": False, "verified": False, "acted": False, "note": note} + continue # reversible: re-aim + await asyncio.sleep(settle_s) + after, after_url = await p_fresh(execute_tool, browser_id, tab_id) + if va.expectation_met(expect, before, after, before_url, after_url): + logger.info(f"[verified-step] {step.kind} {step.target!r} -> {expect} OK (attempt {attempt + 1})") + return {"ok": True, "verified": True, "acted": True, "note": ""} + if step.irreversible: + # acted, effect unverifiable: the send-script's honesty rule, never a blind repeat + return {"ok": False, "verified": False, "acted": True, + "note": (f"an irreversible {step.target!r} action already RAN but its effect is " + "unverified; verify on the page, do NOT repeat it unless verifiably absent")} + note = f"expected {expect!r} not observed after {step.kind} {step.target!r}" + logger.info(f"[verified-step] {step.kind} {step.target!r} unverified: {note}") + return {"ok": False, "verified": False, "acted": True, "note": note} diff --git a/backend/tests/test_browser_agent_loop.py b/backend/tests/test_browser_agent_loop.py index 100102ce..5a3ac123 100644 --- a/backend/tests/test_browser_agent_loop.py +++ b/backend/tests/test_browser_agent_loop.py @@ -1574,3 +1574,29 @@ def test_loop_tier_pin_flag_overrides_model_failsafe(monkeypatch): primary.turn = 0 asyncio.run(BA.run_browser_agent(task="t", browser_id="b3", model="opus", initial_url=None)) assert primary.calls[-1]["model"] == "primary-x" + + +def test_act_verified_refuses_irreversible_and_runs_reversible(monkeypatch): + # BrowserActVerified: an irreversible-smelling target is REFUSED in code (the + # solo-send rule holds), and a reversible step actually executes through the + # verified path (resolve-late -> click_index) with an honest per-step verdict. + BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear() + primary = FakeLLM([ + Resp([p_rp("send it via the plan tool"), + p_tu("BrowserActVerified", steps=[{"action": "click", "target": "Send message"}])]), + Resp([p_rp("ok, do a reversible step"), + p_tu("BrowserActVerified", steps=[{"action": "click", "target": "Search", "role": "button"}])]), + Resp([Blk("text", "done exploring")], stop_reason="end_turn"), + ]) + aux = FakeAux() + sent = p_install(monkeypatch, primary, aux) + + asyncio.run(BA.run_browser_agent(task="use the search", browser_id="b1", model="sonnet")) + + all_msgs = json.dumps([c["messages"] for c in primary.calls]) + # 1) the irreversible target never executed; the model got the refusal + guidance + assert "REFUSED" in all_msgs and "SOLO click" in all_msgs + # 2) the reversible step resolved "Search" against the live list and clicked index 1 + assert any(c["action"] == "click_index" and c["params"].get("index") == 1 for c in sent) + # 3) honest verdict fed back (static fake page = no observable change; never a fake OK) + assert "FAILED" in all_msgs or "OK (verified)" in all_msgs diff --git a/backend/tests/test_browser_verified_action.py b/backend/tests/test_browser_verified_action.py index 2cbcb922..d1417a58 100644 --- a/backend/tests/test_browser_verified_action.py +++ b/backend/tests/test_browser_verified_action.py @@ -43,3 +43,30 @@ def test_generic_verifier_agrees_with_the_proven_inline_check(): def test_unknown_expectation_fails_safe(): assert not va.expectation_met("teleported:X", EMPTY, FILLED) # typo/unknown = not met + + +PROFILE = '[22]*\n[50]*\n[51]