WEBDIENSTE/openswarm
1
0
Fork 0
mirror of https://github.com/openswarm-ai/openswarm.git synced 2026-09-27 20:14:49 +02:00
Code Issues Packages Projects Releases Wiki Activity

[pierre] feat: add BrowserClickPoint and human-style canvas/game control for app agent

Browse Source
This commit is contained in:
SirKentut
2026-06-18 02:33:18 -07:00
parent b745c54c3e
commit e0c4b5859f
2 changed files with 141 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/apps/agents/browser/browser_agent.py
+78 -4
View File
@@ -219,6 +219,71 @@ def _persist_app_controls(browser_id: str, describe_value: object) -> None:
logger.debug("[app-agent] failed to persist controls cache", exc_info=True)
# Single-tool names -> the sub-action type they map to, so one summarizer covers
# both BrowserPressKey({key}) and a batch's {"type":"press_key","params":{key}}.
_SINGLE_ACTION_TYPE = {
"BrowserClick": "click", "BrowserClickIndex": "click_index",
"BrowserClickByName": "click_name", "BrowserType": "type",
"BrowserPressKey": "press_key", "BrowserScroll": "scroll",
"BrowserNavigate": "navigate", "BrowserClickPoint": "click_point",
}
def _summ_step(stype: str, params: dict) -> str:
"""Compact human label for one action step: the actual key/selector/text, not
just the verb. This is what lets the [backend] pane show 'key:ArrowRight x5'
instead of an opaque 'BrowserBatch'."""
p = params or {}
if stype == "press_key":
return f"key:{p.get('key', '?')}"
if stype == "click":
return f"click({p.get('selector', '?')})"
if stype == "click_index":
return f"click#{p.get('index', '?')}"
if stype == "click_point":
return f"tap({p.get('xPercent', '?')}%,{p.get('yPercent', '?')}%)"
if stype == "click_name":
return f"clickName({p.get('name', '?')})"
if stype == "type":
return f"type({p.get('selector', '')}={str(p.get('text', ''))[:30]!r})"
if stype == "wait":
return f"wait({p.get('milliseconds') or p.get('until') or ''})"
if stype == "scroll":
return f"scroll({p.get('direction', 'down')})"
if stype == "navigate":
return f"nav({str(p.get('url', ''))[:60]})"
if stype == "list_interactives":
return "list"
return stype or "?"
def _collapse_steps(items: list[str]) -> str:
"""'ArrowRight, ArrowRight, ArrowRight' -> 'key:ArrowRight x3' so a 5-key
burst reads as one token instead of scrolling the pane."""
runs: list[list] = []
for it in items:
if runs and runs[-1][0] == it:
runs[-1][1] += 1
else:
runs.append([it, 1])
return ", ".join(s if n == 1 else f"{s} x{n}" for s, n in runs)
def _summarize_action(tool_name: str, tool_input: dict) -> str:
"""One-line summary of what an action tool is about to do, or "" for pure
reads (screenshot/list/describe/getstate) that need no action log."""
ti = tool_input or {}
if tool_name == "BrowserBatch":
steps = [_summ_step((a or {}).get("type", ""), (a or {}).get("params"))
for a in (ti.get("actions") or [])]
return _collapse_steps(steps) or "(empty batch)"
if tool_name == "AppInvoke":
args = ti.get("args")
return f"{ti.get('name', '?')}" + (f"({json.dumps(args)[:60]})" if args else "")
stype = _SINGLE_ACTION_TYPE.get(tool_name)
return _summ_step(stype, ti) if stype else ""
async def execute_browser_tool(
tool_name: str, tool_input: dict, browser_id: str, tab_id: str = "",
) -> dict:
@@ -227,6 +292,13 @@ async def execute_browser_tool(
# browser-agent runs stay quiet. Greppable prefix; remove when done.
_trace = browser_id.startswith("app:") or tool_name in APP_BRIDGE_TOOLS
# One greppable line naming the actual buttons/keys/selectors this call drives,
# so a run reads as "key:ArrowRight x5" rather than an opaque tool name. Fires
# for action tools only (reads stay quiet) and ungated so web runs get it too.
_action = _summarize_action(tool_name, tool_input)
if _action:
logger.info(f"[browser-action] {tool_name}: {_action} -> {browser_id}")
# App bridge tools translate to a single BrowserEvaluate against the app's
# window.OPENSWARM_APP, so they need no frontend command-handler changes.
if tool_name in APP_BRIDGE_TOOLS:
@@ -745,10 +817,12 @@ async def run_browser_agent(
"action_log": [], "final_screenshot": None,
}
app_front_load = (
f"\n\n[{_msg}. AppDescribe/AppInvoke will not work. Fall back to "
"driving the UI directly (BrowserListInteractives, "
"BrowserClickIndex, BrowserBatch, BrowserScreenshot). If you "
"cannot operate it, say so in Done with success=false.]"
f"\n\n[{_msg}. Operate it like a person instead: see with "
"BrowserScreenshot, then play with BrowserPressKey (keys like "
"w/a/s/d, arrows, Space, Enter) and BrowserClickPoint (tap a screen "
"point). For a normal HTML app use BrowserListInteractives + "
"BrowserClickIndex. Only give up (Done success=false) after you have "
"actually tried pressing keys and nothing responds.]"
)
# Front-load perception (browser) or the app's controls (app mode) into the
backend/apps/agents/browser/browser_schema.py
+63 -6
View File
@@ -394,6 +394,8 @@ BROWSER_TOOLS_SCHEMA = [
"Sub-action types and their params:\n"
"- click_index: { index: int }\n"
"- press_key: { key: str }\n"
"- click_point: { xPercent: number, yPercent: number, hold_ms?: int } "
"(tap a screen point; for canvas apps/games)\n"
"- type: { selector: str, text: str }\n"
"- click: { selector: str }\n"
"- scroll: { direction?: 'up'|'down', amount?: int }\n"
@@ -418,7 +420,7 @@ BROWSER_TOOLS_SCHEMA = [
"properties": {
"type": {
"type": "string",
"enum": ["click_index", "press_key", "type", "wait", "scroll", "navigate", "click", "list_interactives"],
"enum": ["click_index", "press_key", "click_point", "type", "wait", "scroll", "navigate", "click", "list_interactives"],
},
"params": {"type": "object"},
},
@@ -454,6 +456,43 @@ BROWSER_TOOLS_SCHEMA = [
"required": ["key"],
},
},
{
"name": "BrowserClickPoint",
"description": (
"Tap/click at a point on the screen using a real native mouse event, "
"WITHOUT needing a DOM element. This is the way to operate a <canvas> "
"app or game (the kind with no clickable HTML elements): you click a "
"spot the way a person does. Give the position as a PERCENT of the view "
"(xPercent/yPercent, 0-100, with 0,0 = top-left and 50,50 = center), "
"read off the screenshot. Optional hold_ms presses and holds (e.g. a "
"charge-up or a platformer jump). For element-based pages prefer "
"BrowserClickIndex; use this when there is nothing in the element list "
"to click."
),
"input_schema": {
"type": "object",
"properties": {
"xPercent": {
"type": "number",
"description": "Horizontal position as a percent of view width (0=left, 100=right).",
},
"yPercent": {
"type": "number",
"description": "Vertical position as a percent of view height (0=top, 100=bottom).",
},
"hold_ms": {
"type": "number",
"description": "Optional. Milliseconds to hold the button down before releasing (default 0 = a tap). Max 5000.",
},
"button": {
"type": "string",
"enum": ["left", "right", "middle"],
"description": "Mouse button; defaults to left.",
},
},
"required": ["xPercent", "yPercent"],
},
},
{
"name": "BrowserWait",
"description": (
@@ -642,7 +681,7 @@ BROWSER_TOOLS_SCHEMA = [
# tools are not offered to it at all; acting means a BrowserBatch array, and
# the one deliberate solo path is BrowserClickIndex (irreversible step with
# expect, or a text-box fill). Executors and replay still support everything.
_SOLO_MUTATORS_HIDDEN = {"BrowserNavigate", "BrowserClick", "BrowserType", "BrowserScroll", "BrowserPressKey"}
_SOLO_MUTATORS_HIDDEN = {"BrowserNavigate", "BrowserClick", "BrowserType", "BrowserScroll", "BrowserPressKey", "BrowserClickPoint"}
MODEL_VISIBLE_TOOLS = [t for t in BROWSER_TOOLS_SCHEMA if t["name"] not in _SOLO_MUTATORS_HIDDEN]
ACTION_MAP = {
@@ -659,6 +698,7 @@ ACTION_MAP = {
"BrowserPressKey": "press_key",
"BrowserListInteractives": "list_interactives",
"BrowserClickIndex": "click_index",
"BrowserClickPoint": "click_point",
"BrowserBatch": "batch",
"BrowserDetectWebMCP": "detect_webmcp",
"BrowserListRoutes": "list_routes",
@@ -739,6 +779,9 @@ _APP_FALLBACK_TOOL_NAMES = [
"ReportProgress", "Done",
"BrowserScreenshot", "BrowserGetText",
"BrowserListInteractives", "BrowserClickIndex", "BrowserBatch",
# Human-style native input for apps with no clickable DOM (canvas games):
# press real keys and tap real screen points the way a person plays.
"BrowserPressKey", "BrowserClickPoint",
]
_app_fallback_tools = [t for t in BROWSER_TOOLS_SCHEMA if t["name"] in _APP_FALLBACK_TOOL_NAMES]
# ReportProgress + Done lead, then the bridge tools, then UI fallback.
@@ -1004,10 +1047,23 @@ APP_SYSTEM_PROMPT = (
"so call AppDescribe ONCE to refresh them. As long as __rev is unchanged, "
"trust the controls you already have and do not re-describe.\n\n"
"## If there is no bridge\n"
"If AppDescribe (or AppGetState) returns null, this app doesn't expose the "
"bridge. Fall back to driving the UI directly: BrowserListInteractives to find "
"controls, BrowserClickIndex / BrowserBatch to act, BrowserScreenshot to see. "
"## If there is no bridge: operate it like a person\n"
"If AppDescribe (or AppGetState) returns null, this app exposes no bridge, so "
"drive it directly the way a human would, by looking at the screen and using "
"the keyboard and mouse:\n"
"- SEE with BrowserScreenshot (your main sense here); read on-screen text/score "
"from it. BrowserGetText helps for text-heavy apps.\n"
"- For a normal HTML app (buttons, inputs, forms): BrowserListInteractives to "
"find controls, then BrowserClickIndex / BrowserBatch to act.\n"
"- For a CANVAS app or GAME (no clickable elements in the list): play it with "
"native input. BrowserPressKey for keyboard (e.g. 'Space' to flap, "
"'ArrowLeft'/'ArrowRight' to move, 'Enter' to start) and BrowserClickPoint to "
"tap a spot, giving xPercent/yPercent read off the screenshot (50,50 = center). "
"These are REAL OS-level events, identical to you pressing a key or clicking, so "
"the game responds exactly as it does for a person. Take a screenshot to "
"confirm what changed, then act again.\n"
"- Need fast repeated input (rapid flaps/taps)? Put several BrowserPressKey or "
"BrowserClickPoint steps in one BrowserBatch so they fire in a single turn.\n"
"If you truly cannot operate it, say so plainly in Done with success=false.\n\n"
"## ReportProgress before acting\n"
@@ -1036,6 +1092,7 @@ _ACTION_TOOLS_REQUIRING_REPORT = {
"BrowserScroll",
"BrowserEvaluate",
"BrowserClickIndex", # Phase 3
"BrowserClickPoint", # app mode: tap a canvas/game at a screen point
"BrowserBatch", # Phase 4
"AppInvoke", # app mode: invoking an app action mutates state
}