From e0c4b5859fa0b9300e2e51c7d1ae5779ad20ac7e Mon Sep 17 00:00:00 2001 From: SirKentut <81878031+SirKentut@users.noreply.github.com> Date: Thu, 18 Jun 2026 02:33:18 -0700 Subject: [PATCH] [pierre] feat: add BrowserClickPoint and human-style canvas/game control for app agent --- backend/apps/agents/browser/browser_agent.py | 82 ++++++++++++++++++- backend/apps/agents/browser/browser_schema.py | 69 ++++++++++++++-- 2 files changed, 141 insertions(+), 10 deletions(-) diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index 70318128..10fe71bd 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -219,6 +219,71 @@ def _persist_app_controls(browser_id: str, describe_value: object) -> None: logger.debug("[app-agent] failed to persist controls cache", exc_info=True) +# Single-tool names -> the sub-action type they map to, so one summarizer covers +# both BrowserPressKey({key}) and a batch's {"type":"press_key","params":{key}}. +_SINGLE_ACTION_TYPE = { + "BrowserClick": "click", "BrowserClickIndex": "click_index", + "BrowserClickByName": "click_name", "BrowserType": "type", + "BrowserPressKey": "press_key", "BrowserScroll": "scroll", + "BrowserNavigate": "navigate", "BrowserClickPoint": "click_point", +} + + +def _summ_step(stype: str, params: dict) -> str: + """Compact human label for one action step: the actual key/selector/text, not + just the verb. This is what lets the [backend] pane show 'key:ArrowRight x5' + instead of an opaque 'BrowserBatch'.""" + p = params or {} + if stype == "press_key": + return f"key:{p.get('key', '?')}" + if stype == "click": + return f"click({p.get('selector', '?')})" + if stype == "click_index": + return f"click#{p.get('index', '?')}" + if stype == "click_point": + return f"tap({p.get('xPercent', '?')}%,{p.get('yPercent', '?')}%)" + if stype == "click_name": + return f"clickName({p.get('name', '?')})" + if stype == "type": + return f"type({p.get('selector', '')}={str(p.get('text', ''))[:30]!r})" + if stype == "wait": + return f"wait({p.get('milliseconds') or p.get('until') or ''})" + if stype == "scroll": + return f"scroll({p.get('direction', 'down')})" + if stype == "navigate": + return f"nav({str(p.get('url', ''))[:60]})" + if stype == "list_interactives": + return "list" + return stype or "?" + + +def _collapse_steps(items: list[str]) -> str: + """'ArrowRight, ArrowRight, ArrowRight' -> 'key:ArrowRight x3' so a 5-key + burst reads as one token instead of scrolling the pane.""" + runs: list[list] = [] + for it in items: + if runs and runs[-1][0] == it: + runs[-1][1] += 1 + else: + runs.append([it, 1]) + return ", ".join(s if n == 1 else f"{s} x{n}" for s, n in runs) + + +def _summarize_action(tool_name: str, tool_input: dict) -> str: + """One-line summary of what an action tool is about to do, or "" for pure + reads (screenshot/list/describe/getstate) that need no action log.""" + ti = tool_input or {} + if tool_name == "BrowserBatch": + steps = [_summ_step((a or {}).get("type", ""), (a or {}).get("params")) + for a in (ti.get("actions") or [])] + return _collapse_steps(steps) or "(empty batch)" + if tool_name == "AppInvoke": + args = ti.get("args") + return f"{ti.get('name', '?')}" + (f"({json.dumps(args)[:60]})" if args else "") + stype = _SINGLE_ACTION_TYPE.get(tool_name) + return _summ_step(stype, ti) if stype else "" + + async def execute_browser_tool( tool_name: str, tool_input: dict, browser_id: str, tab_id: str = "", ) -> dict: @@ -227,6 +292,13 @@ async def execute_browser_tool( # browser-agent runs stay quiet. Greppable prefix; remove when done. _trace = browser_id.startswith("app:") or tool_name in APP_BRIDGE_TOOLS + # One greppable line naming the actual buttons/keys/selectors this call drives, + # so a run reads as "key:ArrowRight x5" rather than an opaque tool name. Fires + # for action tools only (reads stay quiet) and ungated so web runs get it too. + _action = _summarize_action(tool_name, tool_input) + if _action: + logger.info(f"[browser-action] {tool_name}: {_action} -> {browser_id}") + # App bridge tools translate to a single BrowserEvaluate against the app's # window.OPENSWARM_APP, so they need no frontend command-handler changes. if tool_name in APP_BRIDGE_TOOLS: @@ -745,10 +817,12 @@ async def run_browser_agent( "action_log": [], "final_screenshot": None, } app_front_load = ( - f"\n\n[{_msg}. AppDescribe/AppInvoke will not work. Fall back to " - "driving the UI directly (BrowserListInteractives, " - "BrowserClickIndex, BrowserBatch, BrowserScreenshot). If you " - "cannot operate it, say so in Done with success=false.]" + f"\n\n[{_msg}. Operate it like a person instead: see with " + "BrowserScreenshot, then play with BrowserPressKey (keys like " + "w/a/s/d, arrows, Space, Enter) and BrowserClickPoint (tap a screen " + "point). For a normal HTML app use BrowserListInteractives + " + "BrowserClickIndex. Only give up (Done success=false) after you have " + "actually tried pressing keys and nothing responds.]" ) # Front-load perception (browser) or the app's controls (app mode) into the diff --git a/backend/apps/agents/browser/browser_schema.py b/backend/apps/agents/browser/browser_schema.py index fee0fca5..67ea3025 100644 --- a/backend/apps/agents/browser/browser_schema.py +++ b/backend/apps/agents/browser/browser_schema.py @@ -394,6 +394,8 @@ BROWSER_TOOLS_SCHEMA = [ "Sub-action types and their params:\n" "- click_index: { index: int }\n" "- press_key: { key: str }\n" + "- click_point: { xPercent: number, yPercent: number, hold_ms?: int } " + "(tap a screen point; for canvas apps/games)\n" "- type: { selector: str, text: str }\n" "- click: { selector: str }\n" "- scroll: { direction?: 'up'|'down', amount?: int }\n" @@ -418,7 +420,7 @@ BROWSER_TOOLS_SCHEMA = [ "properties": { "type": { "type": "string", - "enum": ["click_index", "press_key", "type", "wait", "scroll", "navigate", "click", "list_interactives"], + "enum": ["click_index", "press_key", "click_point", "type", "wait", "scroll", "navigate", "click", "list_interactives"], }, "params": {"type": "object"}, }, @@ -454,6 +456,43 @@ BROWSER_TOOLS_SCHEMA = [ "required": ["key"], }, }, + { + "name": "BrowserClickPoint", + "description": ( + "Tap/click at a point on the screen using a real native mouse event, " + "WITHOUT needing a DOM element. This is the way to operate a " + "app or game (the kind with no clickable HTML elements): you click a " + "spot the way a person does. Give the position as a PERCENT of the view " + "(xPercent/yPercent, 0-100, with 0,0 = top-left and 50,50 = center), " + "read off the screenshot. Optional hold_ms presses and holds (e.g. a " + "charge-up or a platformer jump). For element-based pages prefer " + "BrowserClickIndex; use this when there is nothing in the element list " + "to click." + ), + "input_schema": { + "type": "object", + "properties": { + "xPercent": { + "type": "number", + "description": "Horizontal position as a percent of view width (0=left, 100=right).", + }, + "yPercent": { + "type": "number", + "description": "Vertical position as a percent of view height (0=top, 100=bottom).", + }, + "hold_ms": { + "type": "number", + "description": "Optional. Milliseconds to hold the button down before releasing (default 0 = a tap). Max 5000.", + }, + "button": { + "type": "string", + "enum": ["left", "right", "middle"], + "description": "Mouse button; defaults to left.", + }, + }, + "required": ["xPercent", "yPercent"], + }, + }, { "name": "BrowserWait", "description": ( @@ -642,7 +681,7 @@ BROWSER_TOOLS_SCHEMA = [ # tools are not offered to it at all; acting means a BrowserBatch array, and # the one deliberate solo path is BrowserClickIndex (irreversible step with # expect, or a text-box fill). Executors and replay still support everything. -_SOLO_MUTATORS_HIDDEN = {"BrowserNavigate", "BrowserClick", "BrowserType", "BrowserScroll", "BrowserPressKey"} +_SOLO_MUTATORS_HIDDEN = {"BrowserNavigate", "BrowserClick", "BrowserType", "BrowserScroll", "BrowserPressKey", "BrowserClickPoint"} MODEL_VISIBLE_TOOLS = [t for t in BROWSER_TOOLS_SCHEMA if t["name"] not in _SOLO_MUTATORS_HIDDEN] ACTION_MAP = { @@ -659,6 +698,7 @@ ACTION_MAP = { "BrowserPressKey": "press_key", "BrowserListInteractives": "list_interactives", "BrowserClickIndex": "click_index", + "BrowserClickPoint": "click_point", "BrowserBatch": "batch", "BrowserDetectWebMCP": "detect_webmcp", "BrowserListRoutes": "list_routes", @@ -739,6 +779,9 @@ _APP_FALLBACK_TOOL_NAMES = [ "ReportProgress", "Done", "BrowserScreenshot", "BrowserGetText", "BrowserListInteractives", "BrowserClickIndex", "BrowserBatch", + # Human-style native input for apps with no clickable DOM (canvas games): + # press real keys and tap real screen points the way a person plays. + "BrowserPressKey", "BrowserClickPoint", ] _app_fallback_tools = [t for t in BROWSER_TOOLS_SCHEMA if t["name"] in _APP_FALLBACK_TOOL_NAMES] # ReportProgress + Done lead, then the bridge tools, then UI fallback. @@ -1004,10 +1047,23 @@ APP_SYSTEM_PROMPT = ( "so call AppDescribe ONCE to refresh them. As long as __rev is unchanged, " "trust the controls you already have and do not re-describe.\n\n" - "## If there is no bridge\n" - "If AppDescribe (or AppGetState) returns null, this app doesn't expose the " - "bridge. Fall back to driving the UI directly: BrowserListInteractives to find " - "controls, BrowserClickIndex / BrowserBatch to act, BrowserScreenshot to see. " + "## If there is no bridge: operate it like a person\n" + "If AppDescribe (or AppGetState) returns null, this app exposes no bridge, so " + "drive it directly the way a human would, by looking at the screen and using " + "the keyboard and mouse:\n" + "- SEE with BrowserScreenshot (your main sense here); read on-screen text/score " + "from it. BrowserGetText helps for text-heavy apps.\n" + "- For a normal HTML app (buttons, inputs, forms): BrowserListInteractives to " + "find controls, then BrowserClickIndex / BrowserBatch to act.\n" + "- For a CANVAS app or GAME (no clickable elements in the list): play it with " + "native input. BrowserPressKey for keyboard (e.g. 'Space' to flap, " + "'ArrowLeft'/'ArrowRight' to move, 'Enter' to start) and BrowserClickPoint to " + "tap a spot, giving xPercent/yPercent read off the screenshot (50,50 = center). " + "These are REAL OS-level events, identical to you pressing a key or clicking, so " + "the game responds exactly as it does for a person. Take a screenshot to " + "confirm what changed, then act again.\n" + "- Need fast repeated input (rapid flaps/taps)? Put several BrowserPressKey or " + "BrowserClickPoint steps in one BrowserBatch so they fire in a single turn.\n" "If you truly cannot operate it, say so plainly in Done with success=false.\n\n" "## ReportProgress before acting\n" @@ -1036,6 +1092,7 @@ _ACTION_TOOLS_REQUIRING_REPORT = { "BrowserScroll", "BrowserEvaluate", "BrowserClickIndex", # Phase 3 + "BrowserClickPoint", # app mode: tap a canvas/game at a screen point "BrowserBatch", # Phase 4 "AppInvoke", # app mode: invoking an app action mutates state }