From b498ed30973ed35e8809402731a9dc489ba30b8f Mon Sep 17 00:00:00 2001 From: SirKentut <81878031+SirKentut@users.noreply.github.com> Date: Sat, 13 Jun 2026 01:01:11 -0700 Subject: [PATCH 01/16] implemented app-describe for agent use: exposing the command tree of an app --- backend/apps/agents/agent_manager.py | 4 +- backend/apps/agents/browser/browser_agent.py | 133 ++++++++++++---- backend/apps/agents/browser/browser_schema.py | 125 +++++++++++++++ .../apps/agents/browser_agent_mcp_server.py | 89 ++++++++--- .../agents/manager/prompt/prompt_context.py | 13 +- backend/apps/outputs/app_builder_skill.md | 55 +++++++ backend/main.py | 8 + backend/tests/test_app_agent.py | 148 ++++++++++++++++++ frontend/package-lock.json | 19 +-- .../Dashboard/cards/DashboardViewCard.tsx | 1 + frontend/src/app/pages/Views/ViewPreview.tsx | 24 +++ frontend/src/shared/browserCommandHandler.ts | 20 ++- frontend/src/shared/browserRegistry.ts | 5 + 13 files changed, 569 insertions(+), 75 deletions(-) create mode 100644 backend/tests/test_app_agent.py diff --git a/backend/apps/agents/agent_manager.py b/backend/apps/agents/agent_manager.py index 6abe7937..ed350768 100644 --- a/backend/apps/agents/agent_manager.py +++ b/backend/apps/agents/agent_manager.py @@ -1142,7 +1142,7 @@ class AgentManager: # dispatch layer (see _build_mcp_servers docstring). mcp_servers = await self._build_mcp_servers(session.allowed_tools, session.active_mcps) - _browser_delegation_tools = ["CreateBrowserAgent", "BrowserAgent", "BrowserAgents"] + _browser_delegation_tools = ["CreateBrowserAgent", "BrowserAgent", "BrowserAgents", "AppAgent"] _browser_all_denied = all( _builtin_perms.get(t, "always_allow") == "deny" for t in _browser_delegation_tools @@ -1165,6 +1165,8 @@ class AgentManager: "OPENSWARM_AGENT_MODEL": session.model, "OPENSWARM_DASHBOARD_ID": session.dashboard_id or "", "OPENSWARM_PRE_SELECTED_BROWSER_IDS": ",".join(pre_selected_bids), + # Apps the user selected this turn; AppAgent may only target these. + "OPENSWARM_SELECTED_APP_IDS": ",".join(selected_app_output_ids or []), "OPENSWARM_PARENT_SESSION_ID": session.id, }, "type": "stdio", diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index fafb1a9a..21851de3 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -71,6 +71,9 @@ from backend.apps.agents.browser import browser_schema from backend.apps.agents.browser.browser_schema import ( _ACTION_TOOLS_REQUIRING_REPORT, ACTION_MAP, + APP_BRIDGE_TOOLS, + APP_SYSTEM_PROMPT, + APP_VISIBLE_TOOLS, BROWSER_TOOLS_SCHEMA, MAX_TURNS, MODEL_MAP, @@ -90,19 +93,64 @@ _CONFIRM_TOOLS = { } +def _app_bridge_expression(tool_name: str, tool_input: dict) -> str: + """JS for an app bridge tool. Each expression returns a JSON STRING (so it + round-trips as text) and never throws; bridge errors come back as JSON.""" + if tool_name == "AppDescribe": + call = "window.OPENSWARM_APP.describe()" + elif tool_name == "AppGetState": + call = "window.OPENSWARM_APP.getState()" + else: # AppInvoke + name = json.dumps(tool_input.get("name", "")) + args = json.dumps(tool_input.get("args") or {}) + call = f"window.OPENSWARM_APP.invoke({name}, {args})" + return ( + "(function(){try{" + "var A=window.OPENSWARM_APP;" + "if(!A||typeof A.describe!=='function'){return JSON.stringify(null);}" + f"var r={call};" + "return JSON.stringify(r===undefined?null:r);" + "}catch(e){return JSON.stringify({__error__:String((e&&e.message)||e)});}})()" + ) + + async def execute_browser_tool( tool_name: str, tool_input: dict, browser_id: str, tab_id: str = "", ) -> dict: """Execute a browser tool via ws_manager directly (no MCP/HTTP round-trip).""" + # [app-agent] step trace: only for app targets / bridge tools so normal + # browser-agent runs stay quiet. Greppable prefix; remove when done. + _trace = browser_id.startswith("app:") or tool_name in APP_BRIDGE_TOOLS + + # App bridge tools translate to a single BrowserEvaluate against the app's + # window.OPENSWARM_APP, so they need no frontend command-handler changes. + if tool_name in APP_BRIDGE_TOOLS: + action = "evaluate" + expr = _app_bridge_expression(tool_name, tool_input) + params = {"expression": expr} + request_id = uuid4().hex + if _trace: + logger.info(f"[app-agent] DISPATCH {tool_name} -> {browser_id} (req {request_id[:8]}) js={expr[:120]}") + result = await ws_manager.send_browser_command( + request_id, action, browser_id, params, tab_id=tab_id, + ) + if _trace: + logger.info(f"[app-agent] RESULT {tool_name} <- {browser_id}: {json.dumps(result)[:300]}") + return result + action = ACTION_MAP.get(tool_name) if not action: return {"error": f"Unknown browser tool: {tool_name}"} params = {k: v for k, v in tool_input.items()} request_id = uuid4().hex + if _trace: + logger.info(f"[app-agent] DISPATCH {tool_name}/{action} -> {browser_id} (req {request_id[:8]})") result = await ws_manager.send_browser_command( request_id, action, browser_id, params, tab_id=tab_id, ) + if _trace: + logger.info(f"[app-agent] RESULT {tool_name} <- {browser_id}: {json.dumps(result)[:300]}") return result @@ -352,27 +400,36 @@ async def run_browser_agent( pre_selected: bool = False, initial_url: str | None = None, parent_session_id: str | None = None, + app_mode: bool = False, ) -> dict: """Run a browser sub-agent loop for a single browser card. Creates a visible AgentSession, streams progress via WebSocket, and returns the full action log + summary + final screenshot. + + When app_mode is set, browser_id points at an OpenSwarm-built app's webview + (registered as "app:"). The agent drives it through the app's + native bridge (window.OPENSWARM_APP) instead of web perception: no initial + navigate, no AX-tree front-load, and a lean app toolset + prompt. """ from backend.apps.agents.agent_manager import agent_manager _browser_perms = load_builtin_permissions() + if app_mode: + logger.info(f"[app-agent] START loop: browser_id={browser_id} task={task[:140]!r}") + session_id = uuid4().hex cancel_event = asyncio.Event() session = AgentSession( id=session_id, - name=f"Browser Agent", + name="App Agent" if app_mode else "Browser Agent", model=model, mode="browser-agent", status="running", dashboard_id=dashboard_id, browser_id=browser_id, - system_prompt=SYSTEM_PROMPT, + system_prompt=APP_SYSTEM_PROMPT if app_mode else SYSTEM_PROMPT, parent_session_id=parent_session_id, ) session._cancel_event = cancel_event @@ -431,16 +488,20 @@ async def run_browser_agent( current_url = "" preloaded_reads: list[dict] = [] # real front-loaded reads, seeded into action_log _resumed = bool(browser_history._browser_history.get(browser_id)) - if initial_url: - nav_result = await execute_browser_tool( - "BrowserNavigate", {"url": initial_url}, browser_id, tab_id, - ) - logger.info(f"Browser agent {session_id}: navigated to {initial_url}: {nav_result.get('text', nav_result.get('error', ''))}") - preloaded_perception, current_url, preloaded_reads = await _perceive(initial_url) - elif not _resumed: - # Fresh task on an existing card: perceive the current page to learn its - # host (for replay) and front-load turn 1 (this path used to start cold). - preloaded_perception, current_url, preloaded_reads = await _perceive("") + # App mode skips this whole block: the app is already loaded and its DOM is + # uninformative (often a bare ), so the agent perceives via the bridge + # (AppDescribe) on turn 1 instead of navigating or front-loading the AX tree. + if not app_mode: + if initial_url: + nav_result = await execute_browser_tool( + "BrowserNavigate", {"url": initial_url}, browser_id, tab_id, + ) + logger.info(f"Browser agent {session_id}: navigated to {initial_url}: {nav_result.get('text', nav_result.get('error', ''))}") + preloaded_perception, current_url, preloaded_reads = await _perceive(initial_url) + elif not _resumed: + # Fresh task on an existing card: perceive the current page to learn its + # host (for replay) and front-load turn 1 (this path used to start cold). + preloaded_perception, current_url, preloaded_reads = await _perceive("") from backend.apps.settings.settings import load_settings from backend.apps.settings.credentials import get_anthropic_client_for_model @@ -594,8 +655,8 @@ async def run_browser_agent( # Advisory per-domain hints: seed the system prompt with what a prior agent # learned about this domain (if we know the domain at start), and keep the # store fresh from each ReportProgress. Re-verify, never blindly trust. - start_domain = _extract_domain(initial_url) if initial_url else None - run_system_prompt = SYSTEM_PROMPT + start_domain = None if app_mode else (_extract_domain(initial_url) if initial_url else None) + run_system_prompt = APP_SYSTEM_PROMPT if app_mode else SYSTEM_PROMPT if start_domain: prior_note = browser_history.get_domain_note(start_domain) if prior_note: @@ -611,20 +672,23 @@ async def run_browser_agent( # from past successful runs) so the model skips re-discovery. Advisory text, # re-verified by the agent, never auto-run. Keyed by full host like skills. pb_seeded = False # whether tier-2 strategy was injected, for measuring its effect - _pb_host = browser_skills.host_of(initial_url or current_url or "") - if _pb_host: - _pb_block = browser_playbook.format_for_prompt(_pb_host) - if _pb_block: - run_system_prompt = run_system_prompt + _pb_block - pb_seeded = True - # Tier-3 memory: the cross-site priors learned on EVERY other site, injected on - # every run (host-agnostic) so a brand-new site isn't fully cold. Advisory, capped. - try: - _meta_block = browser_meta_playbook.format_for_prompt() - if _meta_block: - run_system_prompt = run_system_prompt + _meta_block - except Exception: - pass + # Playbooks are web-only (keyed by host); app mode has no host, the bridge is + # the whole strategy, so skip both tiers. + if not app_mode: + _pb_host = browser_skills.host_of(initial_url or current_url or "") + if _pb_host: + _pb_block = browser_playbook.format_for_prompt(_pb_host) + if _pb_block: + run_system_prompt = run_system_prompt + _pb_block + pb_seeded = True + # Tier-3 memory: the cross-site priors learned on EVERY other site, injected + # on every run (host-agnostic) so a brand-new site isn't fully cold. + try: + _meta_block = browser_meta_playbook.format_for_prompt() + if _meta_block: + run_system_prompt = run_system_prompt + _meta_block + except Exception: + pass # Prompt-caching shapes built once: system as a single cached text block, # and the last tool carrying the cache_control marker (Anthropic keys on the @@ -633,7 +697,7 @@ async def run_browser_agent( "type": "text", "text": run_system_prompt, "cache_control": {"type": "ephemeral"}, }] - _cached_tools = [dict(t) for t in browser_schema.MODEL_VISIBLE_TOOLS] + _cached_tools = [dict(t) for t in (APP_VISIBLE_TOOLS if app_mode else browser_schema.MODEL_VISIBLE_TOOLS)] if _cached_tools: _cached_tools[-1] = {**_cached_tools[-1], "cache_control": {"type": "ephemeral"}} @@ -2277,6 +2341,9 @@ async def run_browser_agents( browser_id = task_def.get("browser_id", "") task_text = task_def.get("task", "") url = task_def.get("url", "") + # App mode: browser_id is a pre-registered app webview ("app:"); + # never create/reuse a browser card, never navigate (the app is loaded). + app_mode = bool(task_def.get("app_mode")) # advisory deep entry (from the fast-path brief): a NEW card opens on it # directly (no google detour); a REUSED card is never moved by it, so a # warm card's deeper page state always wins @@ -2306,7 +2373,7 @@ async def run_browser_agents( pass else: await asyncio.sleep(2.0) - elif browser_id: + elif browser_id and not app_mode: _active_agent_cards.add(browser_id) is_pre_selected = browser_id in pre_selected @@ -2318,11 +2385,13 @@ async def run_browser_agents( model=model, dashboard_id=dashboard_id, pre_selected=is_pre_selected, - initial_url=_nav_url if _nav_url and browser_id not in pre_selected else None, + initial_url=None if app_mode else (_nav_url if _nav_url and browser_id not in pre_selected else None), parent_session_id=parent_session_id, + app_mode=app_mode, ) finally: - _active_agent_cards.discard(browser_id) + if not app_mode: + _active_agent_cards.discard(browser_id) results = await asyncio.gather(*[_run_one(t) for t in tasks], return_exceptions=True) diff --git a/backend/apps/agents/browser/browser_schema.py b/backend/apps/agents/browser/browser_schema.py index afc14fac..546d127b 100644 --- a/backend/apps/agents/browser/browser_schema.py +++ b/backend/apps/agents/browser/browser_schema.py @@ -668,6 +668,80 @@ ACTION_MAP = { "BrowserClickByName": "click_by_name", } +# --- App agent: driving an OpenSwarm-built app via its native bridge --------- +# Apps expose window.OPENSWARM_APP = { describe(), getState(), invoke(name,args) }. +# The agent reads structure/state and acts through that bridge in single +# executeJavaScript round-trips, no screenshots or accessibility tree. These +# three tools are translated to BrowserEvaluate in execute_browser_tool, so they +# need no frontend command-handler changes. +APP_TOOLS_SCHEMA = [ + { + "name": "AppDescribe", + "description": ( + "Read the app's CURRENT list of actions you can take, straight from the " + "app itself (window.OPENSWARM_APP.describe()). Returns an array of " + "{name, args, description}. The app's controls are DYNAMIC, they appear " + "and disappear as state changes, so call this again after any AppInvoke " + "that could add or remove actions; never assume the list is stable. " + "Returns null if the app does not expose the bridge (then fall back to " + "BrowserListInteractives/BrowserScreenshot)." + ), + "input_schema": {"type": "object", "properties": {}, "required": []}, + }, + { + "name": "AppGetState", + "description": ( + "Read a small JSON snapshot of the app's current state " + "(window.OPENSWARM_APP.getState()). Use it to check what's on screen " + "and to verify an action landed. Returns null if the bridge is absent." + ), + "input_schema": {"type": "object", "properties": {}, "required": []}, + }, + { + "name": "AppInvoke", + "description": ( + "Perform one app action by name with arguments " + "(window.OPENSWARM_APP.invoke(name, args)). The name MUST be one that " + "AppDescribe just returned; never invent or modify actions, only invoke " + "the ones the app exposes. Returns the action's result, or an " + "{__error__} if it threw. After invoking, re-AppDescribe if the action " + "may have changed which controls exist." + ), + "input_schema": { + "type": "object", + "properties": { + "name": {"type": "string", "description": "Action name from AppDescribe."}, + "args": { + "type": "object", + "description": "Arguments object for the action; {} if none.", + }, + }, + "required": ["name"], + }, + }, +] + +# Bridge tool names -> the JS the app exposes. execute_browser_tool maps these to +# a BrowserEvaluate. Each expression returns a JSON string (so it round-trips as +# text) and never throws (errors are returned as JSON). +APP_BRIDGE_TOOLS = {"AppDescribe", "AppGetState", "AppInvoke"} + +# Lean toolset for app mode: bridge tools first, then a UI-driving fallback for +# apps that don't expose the bridge. Built by name-selecting the shared defs so +# their schemas stay in one place. +_APP_FALLBACK_TOOL_NAMES = [ + "ReportProgress", "Done", + "BrowserScreenshot", "BrowserGetText", + "BrowserListInteractives", "BrowserClickIndex", "BrowserBatch", +] +_app_fallback_tools = [t for t in BROWSER_TOOLS_SCHEMA if t["name"] in _APP_FALLBACK_TOOL_NAMES] +# ReportProgress + Done lead, then the bridge tools, then UI fallback. +APP_VISIBLE_TOOLS = ( + [t for t in _app_fallback_tools if t["name"] in ("ReportProgress", "Done")] + + APP_TOOLS_SCHEMA + + [t for t in _app_fallback_tools if t["name"] not in ("ReportProgress", "Done")] +) + SYSTEM_PROMPT = ( "You are a website-agnostic browser automation agent. You can operate on ANY " "website the user is signed into; social media, dating apps, email, productivity " @@ -892,6 +966,56 @@ SYSTEM_PROMPT = ( MAX_TURNS = 40 +# App mode: drive an OpenSwarm-built app through its native bridge. This is the +# global "how to operate an app" guidance (decision: one global doc, not per-app; +# per-app specifics come live from AppDescribe). Deliberately short, the bridge +# does the heavy lifting and future models need less hand-holding. +APP_SYSTEM_PROMPT = ( + "You operate an OpenSwarm-built app (a small web app the user created, e.g. a " + "graphing tool or a form). The app is ALREADY open in front of you; do not " + "navigate anywhere.\n\n" + + "## How you see and act: the app's own bridge (this is the fast path)\n" + "The app exposes a native bridge, window.OPENSWARM_APP, with three calls you " + "reach through tools:\n" + "- AppDescribe -> the CURRENT list of actions {name, args, description}.\n" + "- AppGetState -> a small JSON snapshot of what's on screen.\n" + "- AppInvoke(name, args) -> perform one action.\n" + "Always start with AppDescribe to learn the real action names and arg shapes, " + "then AppInvoke them. This reads the app's true structure directly, so you do " + "NOT need screenshots, the DOM, or the accessibility tree.\n" + "Only ever call actions that AppDescribe actually returned. You operate the app, " + "you do NOT change it: never invent action names, and never try to add, remove, " + "or redefine the app's available actions or edit its code. If what the user wants " + "isn't reachable through the exposed actions, say so in Done.\n\n" + + "## Controls are DYNAMIC\n" + "Actions and state change as you interact (the app adds and removes controls). " + "Never cache the action list: after any AppInvoke that could change what's " + "available, call AppDescribe again before relying on it. Verify outcomes with " + "AppGetState rather than assuming.\n\n" + + "## If there is no bridge\n" + "If AppDescribe (or AppGetState) returns null, this app doesn't expose the " + "bridge. Fall back to driving the UI directly: BrowserListInteractives to find " + "controls, BrowserClickIndex / BrowserBatch to act, BrowserScreenshot to see. " + "If you truly cannot operate it, say so plainly in Done with success=false.\n\n" + + "## ReportProgress before acting\n" + "Before any AppInvoke (or UI action), call ReportProgress in the SAME turn with " + "a telegraphic next_goal and working_memory. AppDescribe and AppGetState are " + "reads and do not require it.\n\n" + + "## Speed\n" + "Fewer model turns is the #1 driver. Once AppDescribe tells you the actions, " + "fire the AppInvokes you need; don't re-describe between every step unless the " + "action list could have changed. Keep your notes a few words each.\n\n" + + "When finished, end by calling Done; put a plain one or two sentence reply in " + "its message (what you did and the proof, e.g. what's now graphed), zero " + "interface words. Set success=false if you couldn't finish." +) + # Tools that count as "action tools"; calling any of these in a turn requires # the model to also call ReportProgress in the same turn (after the first # turn). Read-only tools and meta tools are exempt. @@ -904,4 +1028,5 @@ _ACTION_TOOLS_REQUIRING_REPORT = { "BrowserEvaluate", "BrowserClickIndex", # Phase 3 "BrowserBatch", # Phase 4 + "AppInvoke", # app mode: invoking an app action mutates state } diff --git a/backend/apps/agents/browser_agent_mcp_server.py b/backend/apps/agents/browser_agent_mcp_server.py index a978fbf4..25a10606 100644 --- a/backend/apps/agents/browser_agent_mcp_server.py +++ b/backend/apps/agents/browser_agent_mcp_server.py @@ -22,6 +22,8 @@ MODEL = os.environ.get("OPENSWARM_AGENT_MODEL", "sonnet") DASHBOARD_ID = os.environ.get("OPENSWARM_DASHBOARD_ID", "") PRE_SELECTED_BROWSER_IDS = os.environ.get("OPENSWARM_PRE_SELECTED_BROWSER_IDS", "") PARENT_SESSION_ID = os.environ.get("OPENSWARM_PARENT_SESSION_ID", "") +# Apps the user selected on the dashboard; AppAgent may only target these (anti-hallucination). +SELECTED_APP_IDS = [a.strip() for a in os.environ.get("OPENSWARM_SELECTED_APP_IDS", "").split(",") if a.strip()] TOOLS = [ { @@ -112,6 +114,38 @@ TOOLS = [ "required": ["tasks"], }, }, + { + "name": "AppAgent", + "description": ( + "Operate one of the user's OpenSwarm-built apps (a small web app they " + "created, e.g. a graphing or form app) that is open on the dashboard. A " + "dedicated app agent reads the app's own actions and state through its " + "native bridge and performs the task (no screenshots or DOM scraping), " + "then returns a summary plus a final screenshot. Use this for the apps " + "listed in the selected-app context, not for websites (use BrowserAgent " + "for those)." + ), + "inputSchema": { + "type": "object", + "properties": { + "output_id": { + "type": "string", + "description": ( + "The id of the selected app to operate (from the selected-app " + "context block)." + ), + }, + "task": { + "type": "string", + "description": ( + "What to do in the app. Be specific (e.g. 'graph y=x^2 and " + "y=sin(x)')." + ), + }, + }, + "required": ["output_id", "task"], + }, + }, ] @@ -245,37 +279,54 @@ def format_batch_results(results: list[dict]) -> dict: return {"content": all_content} +def _text_error(message: str) -> dict: + return {"content": [{"type": "text", "text": message}], "isError": True} + + +def _run_single_task(task_def: dict) -> dict: + """Dispatch one task to the backend and format its single result.""" + result = call_backend([task_def]) + if "error" in result: + return _text_error(f"Error: {result['error']}") + results = result.get("results", [result]) + if results: + return format_result(results[0]) + return _text_error("No result returned.") + + def handle_tool_call(tool_name: str, arguments: dict) -> dict: if tool_name == "CreateBrowserAgent": - task_def = { + return _run_single_task({ "task": arguments.get("task", ""), "browser_id": "", "url": arguments.get("url", ""), - } - result = call_backend([task_def]) - if "error" in result: - return {"content": [{"type": "text", "text": f"Error: {result['error']}"}], "isError": True} - results = result.get("results", [result]) - if results: - return format_result(results[0]) - return {"content": [{"type": "text", "text": "No result returned."}], "isError": True} + }) elif tool_name == "BrowserAgent": browser_id = arguments.get("browser_id", "") if not browser_id: - return {"content": [{"type": "text", "text": "Error: browser_id is required"}], "isError": True} - task_def = { + return _text_error("Error: browser_id is required") + return _run_single_task({ "task": arguments.get("task", ""), "browser_id": browser_id, "url": "", - } - result = call_backend([task_def]) - if "error" in result: - return {"content": [{"type": "text", "text": f"Error: {result['error']}"}], "isError": True} - results = result.get("results", [result]) - if results: - return format_result(results[0]) - return {"content": [{"type": "text", "text": "No result returned."}], "isError": True} + }) + + elif tool_name == "AppAgent": + output_id = arguments.get("output_id", "") + if not output_id: + return _text_error("Error: output_id is required") + # Only drive apps the user actually selected (anti-hallucination), when we + # know the selection. Empty list = unknown, so don't block. + if SELECTED_APP_IDS and output_id not in SELECTED_APP_IDS: + valid = ", ".join(SELECTED_APP_IDS) or "(none)" + return _text_error(f"Error: '{output_id}' is not a selected app. Selected apps: {valid}") + return _run_single_task({ + "task": arguments.get("task", ""), + "browser_id": f"app:{output_id}", + "url": "", + "app_mode": True, + }) elif tool_name == "BrowserAgents": tasks = arguments.get("tasks", []) diff --git a/backend/apps/agents/manager/prompt/prompt_context.py b/backend/apps/agents/manager/prompt/prompt_context.py index 116ffec3..3c12f2be 100644 --- a/backend/apps/agents/manager/prompt/prompt_context.py +++ b/backend/apps/agents/manager/prompt/prompt_context.py @@ -209,6 +209,7 @@ def _build_selected_app_context(selected_app_output_ids: list[str] | None) -> st name = output.name or "Untitled App" lines = [ f'- App: "{name}"', + f" App id (for AppAgent): {output_id}", f" Workspace path: {path}", f" Entry point: {os.path.join(path, 'index.html')}", ] @@ -224,10 +225,14 @@ def _build_selected_app_context(selected_app_output_ids: list[str] | None) -> st return None return ( "\n" - "The user selected these App cards on the dashboard for you to edit. " - "They are existing web apps; edit the files in place at the paths below " - "and the dashboard preview live-reloads on save. Do not scaffold a new " - "project or write files anywhere else.\n\n" + "The user selected these App cards on the dashboard. You can do two things " + "with them:\n" + "- EDIT them: change the files in place at the paths below and the dashboard " + "preview live-reloads on save. Do not scaffold a new project or write files " + "elsewhere.\n" + "- OPERATE them: to actually use a running app (e.g. 'graph y=x^2', 'fill in " + "the form'), call AppAgent(output_id, task) with the App id below. A " + "dedicated agent drives the live app through its own actions, no editing.\n\n" + "\n\n".join(entries) + "\n" ) diff --git a/backend/apps/outputs/app_builder_skill.md b/backend/apps/outputs/app_builder_skill.md index 9b15cdf8..4fb08c9a 100644 --- a/backend/apps/outputs/app_builder_skill.md +++ b/backend/apps/outputs/app_builder_skill.md @@ -325,6 +325,61 @@ export const JOBS_LIST = '/api/jobs/list'; --- +## Make the app agent-operable — the `OPENSWARM_APP` bridge + +An agent can drive this app on the user's behalf (e.g. "graph y=x^2 on my +Desmos app"). It does NOT do that by clicking pixels or scraping the DOM — +that's slow and an app's DOM is often a bare ``. Instead, expose a +tiny bridge on `window` and the agent reads/acts through it in one fast +`executeJavaScript` call. **Always add this bridge to every app you build.** + +Set `window.OPENSWARM_APP` with three functions: + +- `describe()` → the **current** list of actions, `[{ name, args?, description? }]`. + Recompute it live on every call; controls are dynamic, so return only what's + actually available right now. +- `getState()` → a **small** JSON snapshot of the app's relevant state (used to + verify an action landed). Keep it compact — this is the latency budget. +- `invoke(name, args)` → perform the named action and return a result (or throw + a string the agent will read). + +Keep `args` shapes simple (strings, numbers, booleans, small objects). The agent +only ever calls actions that `describe()` returned; it never edits your code. + +```tsx +// Register once the app's core object exists (e.g. after the calculator mounts). +// `calc` here is the app's own API (Desmos example); use whatever yours exposes. +function registerAgentBridge(calc: any) { + (window as any).OPENSWARM_APP = { + describe() { + const actions = [ + { name: 'addExpr', args: { latex: 'string' }, description: 'Add a graph expression, e.g. y=x^2' }, + { name: 'clear', description: 'Remove all expressions' }, + ]; + // Dynamic: only offer removeExpr when something is on the graph. + if (calc.getExpressions().length > 0) { + actions.push({ name: 'removeExpr', args: { id: 'string' }, description: 'Remove one expression by id' }); + } + return actions; + }, + getState() { + return { expressions: calc.getExpressions().map((e: any) => ({ id: e.id, latex: e.latex })) }; + }, + invoke(name: string, args: any = {}) { + if (name === 'addExpr') { const id = String(Date.now()); calc.setExpression({ id, latex: args.latex }); return { id }; } + if (name === 'removeExpr') { calc.removeExpression({ id: args.id }); return { ok: true }; } + if (name === 'clear') { calc.setBlank(); return { ok: true }; } + throw `Unknown action: ${name}`; + }, + }; +} +``` + +If you skip the bridge the agent falls back to slow UI-driving, so apps meant to +be agent-operated should always register it. + +--- + ## Debugging — use `swarm_debug`, not `print()` The backend has `swarm_debug` pre-installed. It's a colored frame-aware diff --git a/backend/main.py b/backend/main.py index d51267a8..d21c9d31 100644 --- a/backend/main.py +++ b/backend/main.py @@ -529,6 +529,14 @@ async def browser_agent_run(request: Request): if not tasks: return JSONResponse({"error": "tasks array is required"}, status_code=400) + # [app-agent] step trace: log app-mode dispatches arriving at the route. + for _t in tasks: + if _t.get("app_mode") or str(_t.get("browser_id", "")).startswith("app:"): + logger.info( + f"[app-agent] ROUTE /run: browser_id={_t.get('browser_id')!r} " + f"app_mode={_t.get('app_mode')} task={str(_t.get('task',''))[:120]!r}" + ) + results = await run_browser_agents( tasks=tasks, model=model, diff --git a/backend/tests/test_app_agent.py b/backend/tests/test_app_agent.py new file mode 100644 index 00000000..eb0d734b --- /dev/null +++ b/backend/tests/test_app_agent.py @@ -0,0 +1,148 @@ +"""App agent: driving an OpenSwarm-built app through its window.OPENSWARM_APP bridge. + +Pins the wiring that makes an app drivable without touching a real webview or LLM: + - the three App bridge tools translate to a single BrowserEvaluate against the + app's bridge, + - run_browser_agents forwards app_mode + the "app:" target without creating + or navigating a browser card, + - the AppAgent delegation tool builds the right task and gates on the user's + selected apps, + - the orchestrator's selected-app context advertises AppAgent. +""" + +import asyncio +import json + +from backend.apps.agents.browser import browser_agent as BA + + +# --- bridge expression ------------------------------------------------------- +def test_app_bridge_expression_describe_and_state(): + desc = BA._app_bridge_expression("AppDescribe", {}) + assert "window.OPENSWARM_APP.describe()" in desc + assert "JSON.stringify" in desc # round-trips as text + state = BA._app_bridge_expression("AppGetState", {}) + assert "window.OPENSWARM_APP.getState()" in state + + +def test_app_bridge_expression_invoke_serializes_args(): + expr = BA._app_bridge_expression("AppInvoke", {"name": "addExpr", "args": {"latex": "y=x^2"}}) + # name + args are JSON-encoded into the call + assert 'window.OPENSWARM_APP.invoke("addExpr", {"latex": "y=x^2"})' in expr + # missing bridge is handled inside the expression, never throws + assert "typeof A.describe!=='function'" in expr + + +def test_app_bridge_expression_invoke_defaults_args_to_empty_object(): + expr = BA._app_bridge_expression("AppInvoke", {"name": "clear"}) + assert 'window.OPENSWARM_APP.invoke("clear", {})' in expr + + +# --- execute_browser_tool routing ------------------------------------------- +def test_execute_browser_tool_app_bridge_routes_to_evaluate(monkeypatch): + captured = {} + + async def _send(request_id, action, browser_id, params, tab_id=""): + captured.update(action=action, browser_id=browser_id, params=params) + return {"text": json.dumps([{"name": "addExpr"}])} + + monkeypatch.setattr(BA.ws_manager, "send_browser_command", _send, raising=False) + out = asyncio.run(BA.execute_browser_tool( + "AppInvoke", {"name": "addExpr", "args": {"latex": "y=x^2"}}, "app:abc", + )) + assert captured["action"] == "evaluate" + assert captured["browser_id"] == "app:abc" + assert "window.OPENSWARM_APP.invoke" in captured["params"]["expression"] + assert out["text"] # result passes straight through + + +# --- run_browser_agents app_mode wiring ------------------------------------- +def test_run_browser_agents_app_mode_forwards_flag_no_card(monkeypatch): + recorded = {} + + async def _fake_run_browser_agent(**kwargs): + recorded.update(kwargs) + return {"summary": "done", "action_log": [], "final_screenshot": None} + + async def _boom(*a, **k): + raise AssertionError("app mode must not create a browser card") + + monkeypatch.setattr(BA, "run_browser_agent", _fake_run_browser_agent, raising=True) + monkeypatch.setattr(BA, "_create_browser_card", _boom, raising=True) + # a connected dashboard so dispatch isn't refused + monkeypatch.setattr(BA.ws_manager, "global_connections", [object()], raising=False) + + results = asyncio.run(BA.run_browser_agents( + tasks=[{"task": "graph y=x^2", "browser_id": "app:abc", "app_mode": True}], + model="sonnet", + dashboard_id="dash-1", + )) + + assert results and results[0]["summary"] == "done" + assert recorded["app_mode"] is True + assert recorded["browser_id"] == "app:abc" + assert recorded["initial_url"] is None # app already loaded; never navigate + + +# --- AppAgent delegation tool ----------------------------------------------- +def _load_mcp_server(monkeypatch, selected): + import backend.apps.agents.browser_agent_mcp_server as srv + monkeypatch.setattr(srv, "SELECTED_APP_IDS", list(selected), raising=False) + return srv + + +def test_app_agent_tool_builds_app_task(monkeypatch): + srv = _load_mcp_server(monkeypatch, ["abc"]) + captured = {} + + def _call_backend(tasks): + captured["tasks"] = tasks + return {"results": [{"summary": "graphed it", "action_log": []}]} + + monkeypatch.setattr(srv, "call_backend", _call_backend, raising=True) + res = srv.handle_tool_call("AppAgent", {"output_id": "abc", "task": "graph y=x^2"}) + + assert not res.get("isError") + task_def = captured["tasks"][0] + assert task_def["browser_id"] == "app:abc" + assert task_def["app_mode"] is True + assert task_def["task"] == "graph y=x^2" + + +def test_app_agent_tool_rejects_unselected_app(monkeypatch): + srv = _load_mcp_server(monkeypatch, ["abc"]) + + def _call_backend(tasks): + raise AssertionError("must not dispatch an unselected app") + + monkeypatch.setattr(srv, "call_backend", _call_backend, raising=True) + res = srv.handle_tool_call("AppAgent", {"output_id": "zzz", "task": "graph"}) + assert res.get("isError") + assert "not a selected app" in res["content"][0]["text"] + + +def test_app_agent_tool_requires_output_id(monkeypatch): + srv = _load_mcp_server(monkeypatch, []) + res = srv.handle_tool_call("AppAgent", {"task": "graph"}) + assert res.get("isError") + assert "output_id is required" in res["content"][0]["text"] + + +# --- orchestrator context advertises AppAgent -------------------------------- +def test_selected_app_context_advertises_app_agent(monkeypatch): + import os + from backend.apps.agents.manager.prompt import prompt_context as pc + import backend.apps.outputs.workspace_io as wio + + class _Out: + workspace_id = "ws-1" + name = "Grapher" + + monkeypatch.setattr(wio, "load_output", lambda oid: _Out(), raising=True) + monkeypatch.setattr(os.path, "isdir", lambda p: True, raising=True) + monkeypatch.setattr(os.path, "isfile", lambda p: False, raising=True) + + ctx = pc._build_selected_app_context(["abc"]) + assert ctx is not None + assert "App id (for AppAgent): abc" in ctx + assert "AppAgent(output_id, task)" in ctx diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 8e97f035..92d511d7 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -84,7 +84,6 @@ "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -1965,7 +1964,6 @@ "resolved": "https://registry.npmjs.org/@emotion/react/-/react-11.14.0.tgz", "integrity": "sha512-O000MLDBDdk/EohJPFUqvnp4qnHeYkVP5B0xEG0D/L7cOKP9kefu2DXn8dj74cQfsEzUqh+sr1RzFqiL1o+PpA==", "license": "MIT", - "peer": true, "dependencies": { "@babel/runtime": "^7.18.3", "@emotion/babel-plugin": "^11.13.5", @@ -2009,7 +2007,6 @@ "resolved": "https://registry.npmjs.org/@emotion/styled/-/styled-11.14.1.tgz", "integrity": "sha512-qEEJt42DuToa3gurlH4Qqc1kVpNq8wO8cJtDzU46TjlzWjDlsVyevtYCRijVq3SrHsROS+gVQ8Fnea108GnKzw==", "license": "MIT", - "peer": true, "dependencies": { "@babel/runtime": "^7.18.3", "@emotion/babel-plugin": "^11.13.5", @@ -2245,7 +2242,6 @@ "resolved": "https://registry.npmjs.org/@mui/material/-/material-7.3.10.tgz", "integrity": "sha512-cHvGOk2ZEfbQt3LnGe0ZKd/ETs9gsUpkW66DCO+GSjMZhpdKU4XsuIr7zJ/B/2XaN8ihxuzHfYAR4zPtCN4RYg==", "license": "MIT", - "peer": true, "dependencies": { "@babel/runtime": "^7.28.6", "@mui/core-downloads-tracker": "^7.3.10", @@ -3378,7 +3374,6 @@ "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.28.tgz", "integrity": "sha512-z9VXpC7MWrhfWipitjNdgCauoMLRdIILQsAEV+ZesIzBq/oUlxk0m3ApZuMFCXdnS4U7KrI+l3WRUEGQ8K1QKw==", "license": "MIT", - "peer": true, "dependencies": { "@types/prop-types": "*", "csstype": "^3.2.2" @@ -3775,7 +3770,6 @@ "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -3815,7 +3809,6 @@ "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -4144,7 +4137,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.10.12", "caniuse-lite": "^1.0.30001782", @@ -8192,7 +8184,6 @@ "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -8249,7 +8240,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -8468,7 +8458,6 @@ "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", "license": "MIT", - "peer": true, "dependencies": { "loose-envify": "^1.1.0" }, @@ -8481,7 +8470,6 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", "license": "MIT", - "peer": true, "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" @@ -8528,7 +8516,6 @@ "resolved": "https://registry.npmjs.org/react-redux/-/react-redux-9.2.0.tgz", "integrity": "sha512-ROY9fvHhwOD9ySfrF0wmvu//bKCQ6AeZZq1nJNtbDC+kk5DuSuNX/n6YWYF/SYy7bSba4D4FSz8DJeKY/S/r+g==", "license": "MIT", - "peer": true, "dependencies": { "@types/use-sync-external-store": "^0.0.6", "use-sync-external-store": "^1.4.0" @@ -8667,8 +8654,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/redux/-/redux-5.0.1.tgz", "integrity": "sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/redux-thunk": { "version": "3.1.0", @@ -8980,7 +8966,6 @@ "integrity": "sha512-kgW13M54DUB7IsIRM5LvJkNlpH+WhMpooUcaWGFARkF1Tc82v9mIWkCbCYf+MBvpIUBSeSOTilpZjEPr2VYE6Q==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "chokidar": "^4.0.0", "immutable": "^5.1.5", @@ -10084,7 +10069,6 @@ "integrity": "sha512-wGN3qcrBQIFmQ/c0AiOAQBvrZ5lmY8vbbMv4Mxfgzqd/B6+9pXtLo73WuS1dSGXM5QYY3hZnIbvx+K1xxe6FyA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/eslint-scope": "^3.7.7", "@types/estree": "^1.0.8", @@ -10133,7 +10117,6 @@ "integrity": "sha512-pIDJHIEI9LR0yxHXQ+Qh95k2EvXpWzZ5l+d+jIo+RdSm9MiHfzazIxwwni/p7+x4eJZuvG1AJwgC4TNQ7NRgsg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@discoveryjs/json-ext": "^0.5.0", "@webpack-cli/configtest": "^2.1.1", diff --git a/frontend/src/app/pages/Dashboard/cards/DashboardViewCard.tsx b/frontend/src/app/pages/Dashboard/cards/DashboardViewCard.tsx index 034b4341..07889307 100644 --- a/frontend/src/app/pages/Dashboard/cards/DashboardViewCard.tsx +++ b/frontend/src/app/pages/Dashboard/cards/DashboardViewCard.tsx @@ -587,6 +587,7 @@ const DashboardOutputPreview: React.FC<{ frontendCode={output.files?.['index.html'] ?? ''} inputData={inputData} backendResult={backendResult} + agentBrowserId={`app:${output.id}`} /> ); }; diff --git a/frontend/src/app/pages/Views/ViewPreview.tsx b/frontend/src/app/pages/Views/ViewPreview.tsx index 59e31b6b..ae6801c0 100644 --- a/frontend/src/app/pages/Views/ViewPreview.tsx +++ b/frontend/src/app/pages/Views/ViewPreview.tsx @@ -6,6 +6,7 @@ import { useElementSelection } from '@/app/components/editor/ElementSelectionCon import { useIframeElementSelector } from './useIframeElementSelector'; import { getAuthToken, ensureAuthToken } from '@/shared/config'; import { useClaudeTokens } from '@/shared/styles/ThemeContext'; +import { registerWebview, unregisterWebview, setActiveTab, type BrowserWebview } from '@/shared/browserRegistry'; // In Electron use to escape iframe restrictions (popups, mic/camera, WebAuthn, cookied fetch); outside Electron fall back to iframe. const isElectron = navigator.userAgent.includes('Electron'); @@ -55,8 +56,13 @@ interface Props { onConsoleMessage?: (level: string, text: string) => void; /** Fires once the embedded app has actually painted, so cold-start placeholders don't unmount during the vite-ready to first-paint gap. */ onContentLoad?: () => void; + /** When set (webview mode only), registers this preview's webview in browserRegistry under this id so an app agent can drive it via the browser command pipeline. */ + agentBrowserId?: string; } +// Single tab per app preview; the browser command pipeline keys on browserId:tabId. +const APP_TAB_ID = 'main'; + function buildSrcdoc( frontendCode: string, inputData: Record, @@ -92,6 +98,7 @@ const ViewPreview = forwardRef(({ style, onConsoleMessage, onContentLoad, + agentBrowserId, }, ref) => { const iframeRef = useRef(null); const webviewRef = useRef(null); @@ -280,6 +287,23 @@ const ViewPreview = forwardRef(({ }; }, [useWebview, handleNavigationLoad]); + // Register the live webview so an app agent can drive it through the same + // browser command pipeline that drives browser cards. Webview-only: the iframe + // fallback has no executeJavaScript channel from the host. + useEffect(() => { + if (!useWebview || !agentBrowserId) return; + const wv = webviewRef.current as BrowserWebview | null; + if (!wv) return; + registerWebview(agentBrowserId, APP_TAB_ID, wv); + setActiveTab(agentBrowserId, APP_TAB_ID); + return () => { + try { unregisterWebview(agentBrowserId, APP_TAB_ID); } catch (_e) {} + }; + // Keyed on useWebview (flips true when the webview mounts), not iframeSrc: + // the webview element has a stable key and survives src/data changes, so + // re-registering on every data update would just thrash the registry. + }, [useWebview, agentBrowserId]); + const hasContent = !!(serveUrl || frontendCode?.trim()); if (!hasContent) { diff --git a/frontend/src/shared/browserCommandHandler.ts b/frontend/src/shared/browserCommandHandler.ts index 45e0d379..74436048 100644 --- a/frontend/src/shared/browserCommandHandler.ts +++ b/frontend/src/shared/browserCommandHandler.ts @@ -1,4 +1,4 @@ -import { getWebview, type BrowserWebview } from './browserRegistry'; +import { getWebview, registeredKeys, type BrowserWebview } from './browserRegistry'; import { store } from './state/store'; import { resumeBrowserCard } from './state/dashboardLayoutSlice'; import { dashboardWs } from './ws/WebSocketManager'; @@ -1330,8 +1330,13 @@ async function handleReplayRoute(wv: BrowserWebview, params: Record async function handleEvaluate(wv: BrowserWebview, params: Record): Promise> { const expression = params.expression as string; if (!expression) return { error: 'expression parameter is required' }; + // [app-agent] step trace: surface what the app's bridge actually returned. + const _bridgeCall = expression.includes('OPENSWARM_APP'); try { const result = await wv.executeJavaScript(expression); + if (_bridgeCall) { + console.log(`[app-agent] BRIDGE eval -> ${typeof result === 'string' ? result : JSON.stringify(result)}`); + } const text = typeof result === 'string' ? result : JSON.stringify(result, null, 2); // evaluate is the agent's main read path; sample routes here too (XHRs have // fired by now) so the backend can surface the fast network tier once. @@ -1397,14 +1402,27 @@ async function runBrowserCommand( request_id: string, action: string, browser_id: string, tab_id: string | undefined, params: Record, ) { + // [app-agent] step trace: only for app targets so browser-agent runs stay quiet. + const _appTrace = String(browser_id || '').startsWith('app:'); + if (_appTrace) { + console.log(`[app-agent] CMD recv: action=${action} browser_id=${browser_id} tab_id=${tab_id ?? ''} registered=[${registeredKeys().join(', ')}]`); + } const wv = await awaitWebview(browser_id, tab_id || undefined); if (!wv) { + if (_appTrace) { + console.warn(`[app-agent] LOOKUP MISS: no webview for '${browser_id}' (registered keys: [${registeredKeys().join(', ')}]) -> the app card isn't mounted on the active dashboard`); + } dashboardWs.send('browser:result', { request_id, error: `Browser card '${browser_id}'${tab_id ? ` tab '${tab_id}'` : ''} not found or not an Electron webview`, }); return; } + if (_appTrace) { + let _url = ''; + try { _url = wv.getURL(); } catch (_e) {} + console.log(`[app-agent] LOOKUP HIT: webview for '${browser_id}' found, url=${_url} loading=${(() => { try { return wv.isLoading(); } catch { return '?'; } })()}`); + } const detail = params.url || params.selector || params.expression || undefined; setActivity(browser_id, { action: action as BrowserAction, detail }); diff --git a/frontend/src/shared/browserRegistry.ts b/frontend/src/shared/browserRegistry.ts index 8a03e481..536e32d5 100644 --- a/frontend/src/shared/browserRegistry.ts +++ b/frontend/src/shared/browserRegistry.ts @@ -51,6 +51,11 @@ export function setActiveTab(browserId: string, tabId: string): void { activeTabMap.set(browserId, tabId); } +// [app-agent] diagnostic: list currently-registered keys ("browserId:tabId"). +export function registeredKeys(): string[] { + return Array.from(registry.keys()); +} + export function getWebview(browserId: string, tabId?: string): BrowserWebview | undefined { const resolvedTabId = tabId || activeTabMap.get(browserId); if (!resolvedTabId) return undefined; From eb6278ff5c9a720959d34871f806f31d4f053824 Mon Sep 17 00:00:00 2001 From: SirKentut <81878031+SirKentut@users.noreply.github.com> Date: Sun, 14 Jun 2026 02:35:19 -0700 Subject: [PATCH 02/16] fix(webapp-template): self-heal half-installed vite warm cache An npm install killed mid-warm leaves node_modules/ populated but no .bin/, so _ensure_warm_cache (which only checked the dir exists) trusted the half-tree forever and every app symlinked to it died with `vite: command not found`. Gate the cache on .bin/vite existing, and harden the app's frontend/run.sh skip-install guard the same way. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../apps/outputs/view_builder_templates.py | 24 ++++++++++++++----- .../outputs/webapp_template/frontend/run.sh | 9 +++++-- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/backend/apps/outputs/view_builder_templates.py b/backend/apps/outputs/view_builder_templates.py index 3cc089f7..4803f607 100644 --- a/backend/apps/outputs/view_builder_templates.py +++ b/backend/apps/outputs/view_builder_templates.py @@ -273,19 +273,31 @@ def _warm_cache_dir() -> str: return os.path.join(base, _warm_cache_digest()) +def _warm_cache_is_complete(cache_modules: str) -> bool: + """A populated node_modules/ dir is not proof of a *finished* install. + npm links package bins (node_modules/.bin/*) in the final phase, so an + install killed partway (e.g. Electron quit mid-warm) leaves the package + trees on disk but no .bin/. The old `os.path.isdir(node_modules)` check + then trusted that half-tree forever, every app symlinked to it, and + `npm run dev` died with `vite: command not found`. Require the one bin + every webapp-template app actually launches with so a partial cache is + treated as not-ready and repopulated instead of cached as good.""" + return os.path.exists(os.path.join(cache_modules, ".bin", "vite")) + + def _ensure_warm_cache() -> str | None: - """Populate the warm-cache node_modules if missing. Returns the - absolute path to the populated `node_modules` directory, or None on - failure. Thread-safe; concurrent callers block on a single install - instead of racing. Idempotent and fast after the first call.""" + """Populate the warm-cache node_modules if missing or incomplete. + Returns the absolute path to the populated `node_modules` directory, or + None on failure. Thread-safe; concurrent callers block on a single + install instead of racing. Idempotent and fast after the first call.""" cache_dir = _warm_cache_dir() cache_modules = os.path.join(cache_dir, "node_modules") - if os.path.isdir(cache_modules): + if _warm_cache_is_complete(cache_modules): return cache_modules with _warm_cache_lock: - if os.path.isdir(cache_modules): + if _warm_cache_is_complete(cache_modules): return cache_modules # Fast path: pre-built archive shipped inside the release. The # build script generates this so users hitting OpenSwarm for the diff --git a/backend/apps/outputs/webapp_template/frontend/run.sh b/backend/apps/outputs/webapp_template/frontend/run.sh index 1ea9dfe8..8c4e611c 100755 --- a/backend/apps/outputs/webapp_template/frontend/run.sh +++ b/backend/apps/outputs/webapp_template/frontend/run.sh @@ -43,8 +43,13 @@ fi # to vite. Only run npm install when node_modules is genuinely missing # or empty — e.g. a workspace seeded before the warm-cache existed, or # the user's cache was cleared. -if [ -d node_modules ] && [ -n "$(ls -A node_modules 2>/dev/null)" ]; then - echo "Dependencies already present — skipping install." +# A non-empty node_modules is NOT proof of a finished install. +# non-empty -> skip" check then trusted that, and `npm run dev` died with +# `vite: command not found`. Gate on the bin we actually launch with so a +# broken/partial tree self-heals via install instead of being skipped. +#So thats why i explicitly have "/.bin/vite" +if [ -e node_modules/.bin/vite ]; then + echo "Dependencies already present - skipping install." else echo "Installing dependencies..." "$NPM" install --prefer-offline --no-audit --no-fund From f90975f00640b90fa04bee5538e51f532d2e1ccc Mon Sep 17 00:00:00 2001 From: SirKentut <81878031+SirKentut@users.noreply.github.com> Date: Sun, 14 Jun 2026 07:49:00 -0700 Subject: [PATCH 03/16] feat(webapp-template): ship agent bridge (window.OPENSWARM_APP) as a hook, not a doc Add agentBridge.ts and import it FIRST in index.tsx so the bridge is installed on the window before any app code runs. The visitor / door / doorbell analogy: - The visitor is the agent. It shows up wanting to operate the app: read the rules, see the controls, take an action, check what happened. - The door is window.OPENSWARM_APP. A single, always-present surface the visitor can knock on: describe() to learn the app, getState() to read a snapshot, invoke(name, args) to act. - The doorbell is register({ rules, controls, getState, invoke }). The app presses it on mount to say "I'm home, here's how to talk to me." Until it does, describe()/getState() answer { __ready: false } so the visitor knows the app is still booting and waits, instead of concluding nobody lives here. Why a hook and not just an .md: A markdown instruction ("please expose your controls on window.X") is a request the app can forget, half-implement, or drift from, and nothing fails when it does. The bridge ships WITH the template and is imported before app code, so the door exists from first paint whether or not the author thought about agents. The app's only job is to press the doorbell once; it never wires the plumbing, so it cannot get the plumbing wrong. The contract is executable, not aspirational, which is the difference between every generated app being agent-operable by default and hoping each one remembered to be. --- .../frontend/src/agentBridge.ts | 99 +++++++++++++++++++ .../webapp_template/frontend/src/index.tsx | 4 + 2 files changed, 103 insertions(+) create mode 100644 backend/apps/outputs/webapp_template/frontend/src/agentBridge.ts diff --git a/backend/apps/outputs/webapp_template/frontend/src/agentBridge.ts b/backend/apps/outputs/webapp_template/frontend/src/agentBridge.ts new file mode 100644 index 00000000..4d29ad05 --- /dev/null +++ b/backend/apps/outputs/webapp_template/frontend/src/agentBridge.ts @@ -0,0 +1,99 @@ +// window.OPENSWARM_APP - the agent bridge, shipped with the template so it +// EXISTS from first paint, before any app-specific code runs (index.tsx imports +// this first). An app makes itself agent-operable by calling +// window.OPENSWARM_APP.register({ rules, controls, getState, invoke }) on mount; +// it never has to wire up the plumbing, so it cannot forget it. Until the app +// registers, describe()/getState() report { __ready: false } so the agent knows +// the app is still booting (and waits) instead of declaring it bridge-less. + +export type AgentControl = { + name: string; + args?: Record; + description?: string; + keys?: string; // optional key hint, e.g. "Space = flap", "WASD to move" +}; + +export type AgentRegistration = { + rules?: string; // what the app is and its objective, plain prose + controls: AgentControl[] | (() => AgentControl[]); // a function for dynamic controls + getState?: () => unknown; // small JSON snapshot, used to verify an action landed + invoke: (name: string, args?: Record) => unknown; +}; + +type Bridge = { + __openswarm: true; + __ready: boolean; + __rev: number; + register: (api: AgentRegistration) => void; + refresh: () => void; // bump __rev after dynamic controls change so the agent re-reads + describe: () => unknown; + getState: () => unknown; + invoke: (name: string, args?: Record) => unknown; +}; + +declare global { + interface Window { + OPENSWARM_APP?: Bridge; + } +} + +let registration: AgentRegistration | null = null; + +function resolveControls(): AgentControl[] { + if (!registration) return []; + const c = registration.controls; + try { + return typeof c === 'function' ? c() || [] : c || []; + } catch { + return []; + } +} + +const bridge: Bridge = { + __openswarm: true, + __ready: false, + __rev: 0, + register(api: AgentRegistration) { + registration = api; + bridge.__ready = true; + bridge.__rev += 1; + }, + refresh() { + bridge.__rev += 1; + }, + describe() { + if (!bridge.__ready || !registration) { + return { __ready: false, __rev: bridge.__rev }; + } + return { + rules: registration.rules || '', + controls: resolveControls(), + __rev: bridge.__rev, + }; + }, + getState() { + if (!bridge.__ready || !registration) { + return { __ready: false, __rev: bridge.__rev }; + } + let state: unknown = {}; + try { + state = registration.getState ? registration.getState() : {}; + } catch (e) { + return { __error__: String((e as Error)?.message || e), __rev: bridge.__rev }; + } + // Carry __rev alongside the app's own state so the agent can detect a + // controls change with a single getState, without re-describing every turn. + if (state && typeof state === 'object' && !Array.isArray(state)) { + return { ...(state as Record), __rev: bridge.__rev }; + } + return { value: state, __rev: bridge.__rev }; + }, + invoke(name: string, args?: Record) { + if (!bridge.__ready || !registration) { + throw 'OPENSWARM_APP not registered yet'; + } + return registration.invoke(name, args || {}); + }, +}; + +window.OPENSWARM_APP = bridge; diff --git a/backend/apps/outputs/webapp_template/frontend/src/index.tsx b/backend/apps/outputs/webapp_template/frontend/src/index.tsx index 6c7c991f..3925ad74 100644 --- a/backend/apps/outputs/webapp_template/frontend/src/index.tsx +++ b/backend/apps/outputs/webapp_template/frontend/src/index.tsx @@ -1,3 +1,7 @@ +// Install window.OPENSWARM_APP BEFORE anything else so the agent bridge exists +// from first paint, even while React + the app are still mounting. The app fills +// it in by calling window.OPENSWARM_APP.register(...) on mount. +import './agentBridge'; import React from 'react'; import { createRoot } from 'react-dom/client'; import Main from './app/Main'; From a35bfaaf7dd7f86dee5d0421ef2e378cc2a22189 Mon Sep 17 00:00:00 2001 From: SirKentut <81878031+SirKentut@users.noreply.github.com> Date: Mon, 15 Jun 2026 15:06:10 -0700 Subject: [PATCH 04/16] [pierre] feat/app-agent: drive web apps through the OPENSWARM_APP bridge Browser agent gains app-control logic + schema, app-builder skill doc, run.sh agent-bridge presence check, and app-agent tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/apps/agents/browser/browser_agent.py | 193 +++++++++++++++++- backend/apps/agents/browser/browser_schema.py | 45 ++-- backend/apps/outputs/app_builder_skill.md | 68 +++--- .../outputs/webapp_template/frontend/run.sh | 26 +++ backend/apps/outputs/workspace_io.py | 1 + backend/tests/test_app_agent.py | 95 +++++++++ electron/package-lock.json | 8 +- 7 files changed, 378 insertions(+), 58 deletions(-) diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index 21851de3..93c043eb 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -9,6 +9,7 @@ Sub-agents appear as visible AgentSession cards on the dashboard. import asyncio import json import logging +import os import re import time from datetime import datetime @@ -114,6 +115,110 @@ def _app_bridge_expression(tool_name: str, tool_input: dict) -> str: ) +# App-bridge readiness. The template ships window.OPENSWARM_APP from first paint +# but in a "not ready" state until the app calls register(...). On the agent's +# first turn the app may still be mounting (Vite cold-boot is 10-30s), so the +# reads poll briefly for the bridge to come up instead of declaring it absent. +_BRIDGE_READY_WAIT_MS = 8000 +_BRIDGE_POLL_INTERVAL_MS = 400 + + +def _parse_bridge_result(result: dict) -> object: + """Decode the JSON string an app-bridge evaluate returns (it always returns + JSON text and never throws). Returns the decoded value, or None when it is + undecodable or errored at the transport level.""" + if not isinstance(result, dict) or "error" in result: + return None + raw = result.get("text") + if raw is None: + return None + try: + return json.loads(raw) + except (TypeError, ValueError): + return None + + +def _bridge_ready(value: object) -> bool: + """True when a decoded describe()/getState() value means a registered bridge. + Legacy apps return a plain array (ready); the template stub returns + {'__ready': False} until register() runs; None means no bridge present yet.""" + if isinstance(value, list): + return True + if isinstance(value, dict): + return value.get("__ready") is not False and "__error__" not in value + return value is not None + + +def _app_output_id(browser_id: str) -> str | None: + return browser_id[4:] if browser_id.startswith("app:") else None + + +def _app_workspace_dir(browser_id: str) -> str | None: + """Resolve the on-disk workspace folder for an `app:` target.""" + oid = _app_output_id(browser_id) + if not oid: + return None + try: + from backend.apps.outputs.workspace_io import load_output + from backend.config.paths import OUTPUTS_WORKSPACE_DIR + out = load_output(oid) + if not out or not getattr(out, "workspace_id", None): + return None + return os.path.join(OUTPUTS_WORKSPACE_DIR, out.workspace_id) + except Exception: + return None + + +def _render_app_controls(describe_value: object) -> tuple[str, str] | None: + """From a decoded describe() value build (rules_md, controls_md). Returns + None when the value is not a ready, usable describe.""" + if isinstance(describe_value, list): + rules, controls = "", describe_value + elif _bridge_ready(describe_value) and isinstance(describe_value, dict): + rules = str(describe_value.get("rules") or "") + controls = describe_value.get("controls") or [] + else: + return None + lines = ["# Controls", ""] + for c in controls: + if not isinstance(c, dict): + continue + row = f"- `{c.get('name', '')}`" + if c.get("args"): + row += f" args={json.dumps(c['args'])}" + if c.get("keys"): + row += f" [{c['keys']}]" + if c.get("description"): + row += f": {c['description']}" + lines.append(row) + controls_md = "\n".join(lines) + "\n" + rules_md = (rules.strip() + "\n") if rules.strip() else "" + return rules_md, controls_md + + +def _persist_app_controls(browser_id: str, describe_value: object) -> None: + """Cache rules.md + controls.md into the app workspace so the agent reads them + up front and only re-describes when controls change. Best-effort; written + under .openswarm/ so they stay out of the app's own file tree.""" + rendered = _render_app_controls(describe_value) + if not rendered: + return + folder = _app_workspace_dir(browser_id) + if not folder or not os.path.isdir(folder): + return + rules_md, controls_md = rendered + try: + cache_dir = os.path.join(folder, ".openswarm") + os.makedirs(cache_dir, exist_ok=True) + with open(os.path.join(cache_dir, "controls.md"), "w", encoding="utf-8") as f: + f.write(controls_md) + if rules_md: + with open(os.path.join(cache_dir, "rules.md"), "w", encoding="utf-8") as f: + f.write(rules_md) + except Exception: + logger.debug("[app-agent] failed to persist controls cache", exc_info=True) + + async def execute_browser_tool( tool_name: str, tool_input: dict, browser_id: str, tab_id: str = "", ) -> dict: @@ -128,14 +233,28 @@ async def execute_browser_tool( action = "evaluate" expr = _app_bridge_expression(tool_name, tool_input) params = {"expression": expr} - request_id = uuid4().hex - if _trace: - logger.info(f"[app-agent] DISPATCH {tool_name} -> {browser_id} (req {request_id[:8]}) js={expr[:120]}") - result = await ws_manager.send_browser_command( - request_id, action, browser_id, params, tab_id=tab_id, - ) - if _trace: - logger.info(f"[app-agent] RESULT {tool_name} <- {browser_id}: {json.dumps(result)[:300]}") + + async def _eval_once() -> dict: + rid = uuid4().hex + if _trace: + logger.info(f"[app-agent] DISPATCH {tool_name} -> {browser_id} (req {rid[:8]}) js={expr[:120]}") + r = await ws_manager.send_browser_command(rid, action, browser_id, params, tab_id=tab_id) + if _trace: + logger.info(f"[app-agent] RESULT {tool_name} <- {browser_id}: {json.dumps(r)[:300]}") + return r + + result = await _eval_once() + # Reads poll for the bridge to come up (app still mounting on turn 1). + # AppInvoke does not wait: its action either exists right now or it does + # not, and a missing action should surface immediately. + if tool_name in ("AppDescribe", "AppGetState"): + waited = 0 + while waited < _BRIDGE_READY_WAIT_MS and not _bridge_ready(_parse_bridge_result(result)): + await asyncio.sleep(_BRIDGE_POLL_INTERVAL_MS / 1000) + waited += _BRIDGE_POLL_INTERVAL_MS + result = await _eval_once() + if tool_name == "AppDescribe": + _persist_app_controls(browser_id, _parse_bridge_result(result)) return result action = ACTION_MAP.get(tool_name) @@ -572,10 +691,66 @@ async def run_browser_agent( ) clear_browser_history(browser_id) prior_messages = [] + # App mode: read the bridge's rules + controls ONCE up front and front-load + # them, so the agent knows the app's purpose and every control before its + # first action (no screenshot fumbling) and need not call AppDescribe again + # until controls change. This is also the runtime bridge gate: if the bridge + # never comes up, fail loudly into the logs + the agent's first message (and, + # under OPENSWARM_REQUIRE_BRIDGE=1, end the run rather than UI-fumble). + app_front_load = "" + if app_mode and not prior_messages: + try: + _dv = _parse_bridge_result(await execute_browser_tool("AppDescribe", {}, browser_id, tab_id)) + except Exception: + _dv = None + logger.debug("[app-agent] startup AppDescribe failed", exc_info=True) + _rendered = _render_app_controls(_dv) + if _rendered: + _rules_md, _controls_md = _rendered + _rev = _dv.get("__rev") if isinstance(_dv, dict) else None + _block = [ + "\n\n[The app's bridge is live; its rules and controls were read " + "for you. Act directly; do NOT call AppDescribe again unless " + "AppGetState reports a changed __rev.]" + ] + if _rules_md.strip(): + _block.append("App rules / objective:\n" + _rules_md.strip()) + _block.append(_controls_md.strip()) + if _rev is not None: + _block.append(f"(controls __rev: {_rev})") + app_front_load = "\n\n".join(_block) + else: + _oid = _app_output_id(browser_id) or browser_id + _msg = ( + f"BRIDGE MISSING: window.OPENSWARM_APP not registered - " + f"app '{_oid}' is not agent-operable" + ) + logger.error(f"[app-agent] {_msg}") + if os.environ.get("OPENSWARM_REQUIRE_BRIDGE") == "1": + session.status = "completed" + agent_manager._sync_session_close(session) + await ws_manager.send_to_session(session_id, "agent:status", { + "session_id": session_id, "status": "completed", + "session": session.model_dump(mode="json"), + }) + return { + "session_id": session_id, "browser_id": browser_id, + "summary": f"This app is not agent-operable: {_msg}.", + "done": True, "success": False, + "action_log": [], "final_screenshot": None, + } + app_front_load = ( + f"\n\n[{_msg}. AppDescribe/AppInvoke will not work. Fall back to " + "driving the UI directly (BrowserListInteractives, " + "BrowserClickIndex, BrowserBatch, BrowserScreenshot). If you " + "cannot operate it, say so in Done with success=false.]" + ) + # Front-load the prefetched perception into the first user turn so the model # can act immediately (only when this is a fresh conversation; a resumed one # already knows the page). The visible task text stays clean. - first_user_content = task + preloaded_perception if (preloaded_perception and not prior_messages) else task + _front = preloaded_perception or app_front_load + first_user_content = task + _front if (_front and not prior_messages) else task messages: list[dict] = list(prior_messages) + [{"role": "user", "content": first_user_content}] # Seed with the front-loaded reads: they really ran and returned content, so a # read task the agent answers straight from them is NOT a "did nothing" ghost. diff --git a/backend/apps/agents/browser/browser_schema.py b/backend/apps/agents/browser/browser_schema.py index 546d127b..fee0fca5 100644 --- a/backend/apps/agents/browser/browser_schema.py +++ b/backend/apps/agents/browser/browser_schema.py @@ -678,13 +678,16 @@ APP_TOOLS_SCHEMA = [ { "name": "AppDescribe", "description": ( - "Read the app's CURRENT list of actions you can take, straight from the " - "app itself (window.OPENSWARM_APP.describe()). Returns an array of " - "{name, args, description}. The app's controls are DYNAMIC, they appear " - "and disappear as state changes, so call this again after any AppInvoke " - "that could add or remove actions; never assume the list is stable. " - "Returns null if the app does not expose the bridge (then fall back to " - "BrowserListInteractives/BrowserScreenshot)." + "Read the app's rules and CURRENT list of actions, straight from the " + "app itself (window.OPENSWARM_APP.describe()). Returns " + "{rules, controls, __rev}: rules is what the app is and its objective, " + "controls is an array of {name, args, description, keys}, and __rev is " + "a revision number. (Older apps may return a bare array of controls.) " + "These are ALREADY front-loaded into your first message, so you rarely " + "need to call this. The app's controls are DYNAMIC: call this again " + "ONLY when AppGetState reports a changed __rev (e.g. after an AppInvoke " + "added or removed actions). Returns null if the app exposes no bridge " + "(then fall back to BrowserListInteractives/BrowserScreenshot)." ), "input_schema": {"type": "object", "properties": {}, "required": []}, }, @@ -693,7 +696,10 @@ APP_TOOLS_SCHEMA = [ "description": ( "Read a small JSON snapshot of the app's current state " "(window.OPENSWARM_APP.getState()). Use it to check what's on screen " - "and to verify an action landed. Returns null if the bridge is absent." + "and to verify an action landed. The snapshot includes __rev, the " + "controls revision: if it differs from the __rev you were given, the " + "controls changed, so call AppDescribe to refresh them. Returns null " + "if the bridge is absent." ), "input_schema": {"type": "object", "properties": {}, "required": []}, }, @@ -978,22 +984,25 @@ APP_SYSTEM_PROMPT = ( "## How you see and act: the app's own bridge (this is the fast path)\n" "The app exposes a native bridge, window.OPENSWARM_APP, with three calls you " "reach through tools:\n" - "- AppDescribe -> the CURRENT list of actions {name, args, description}.\n" - "- AppGetState -> a small JSON snapshot of what's on screen.\n" + "- AppDescribe -> {rules, controls, __rev}: the app's objective and its " + "current actions {name, args, description, keys}.\n" + "- AppGetState -> a small JSON snapshot of what's on screen (includes __rev).\n" "- AppInvoke(name, args) -> perform one action.\n" - "Always start with AppDescribe to learn the real action names and arg shapes, " - "then AppInvoke them. This reads the app's true structure directly, so you do " - "NOT need screenshots, the DOM, or the accessibility tree.\n" + "The app's rules and controls have ALREADY been read for you and placed in " + "your first message, so you can start invoking actions immediately; you do " + "NOT need screenshots, the DOM, or the accessibility tree, and you usually do " + "NOT need to call AppDescribe at all.\n" "Only ever call actions that AppDescribe actually returned. You operate the app, " "you do NOT change it: never invent action names, and never try to add, remove, " "or redefine the app's available actions or edit its code. If what the user wants " "isn't reachable through the exposed actions, say so in Done.\n\n" - "## Controls are DYNAMIC\n" - "Actions and state change as you interact (the app adds and removes controls). " - "Never cache the action list: after any AppInvoke that could change what's " - "available, call AppDescribe again before relying on it. Verify outcomes with " - "AppGetState rather than assuming.\n\n" + "## Controls are DYNAMIC, but you only re-read on a __rev change\n" + "Actions can change as you interact (the app adds and removes controls). The " + "front-loaded controls came with a __rev number. Use AppGetState to verify " + "outcomes; if its __rev differs from the one you have, the controls changed, " + "so call AppDescribe ONCE to refresh them. As long as __rev is unchanged, " + "trust the controls you already have and do not re-describe.\n\n" "## If there is no bridge\n" "If AppDescribe (or AppGetState) returns null, this app doesn't expose the " diff --git a/backend/apps/outputs/app_builder_skill.md b/backend/apps/outputs/app_builder_skill.md index 4fb08c9a..8c3b86bf 100644 --- a/backend/apps/outputs/app_builder_skill.md +++ b/backend/apps/outputs/app_builder_skill.md @@ -325,58 +325,76 @@ export const JOBS_LIST = '/api/jobs/list'; --- -## Make the app agent-operable — the `OPENSWARM_APP` bridge +## Make the app agent-operable: the `OPENSWARM_APP` bridge An agent can drive this app on the user's behalf (e.g. "graph y=x^2 on my -Desmos app"). It does NOT do that by clicking pixels or scraping the DOM — -that's slow and an app's DOM is often a bare ``. Instead, expose a -tiny bridge on `window` and the agent reads/acts through it in one fast -`executeJavaScript` call. **Always add this bridge to every app you build.** +Desmos app"). It does NOT do that by clicking pixels or scraping the DOM (slow, +and an app's DOM is often a bare ``). Instead it reads and acts through +`window.OPENSWARM_APP`, a bridge the template already ships for you +(`src/agentBridge.ts`, installed before your app mounts). -Set `window.OPENSWARM_APP` with three functions: +**You do not wire up the bridge; you `register()` into it.** Call +`window.OPENSWARM_APP.register({ rules, controls, getState, invoke })` once your +app's core object exists (e.g. in a mount `useEffect`). This is REQUIRED for +every app: the runtime verifies it and the agent's first action fails loudly +with `BRIDGE MISSING` if you forget. Pass: -- `describe()` → the **current** list of actions, `[{ name, args?, description? }]`. - Recompute it live on every call; controls are dynamic, so return only what's - actually available right now. -- `getState()` → a **small** JSON snapshot of the app's relevant state (used to - verify an action landed). Keep it compact — this is the latency budget. -- `invoke(name, args)` → perform the named action and return a result (or throw +- `rules` (string) - what the app is and its objective, in plain prose. This is + what the agent reads to understand the app (e.g. "Flappy Bird. Keep the bird + airborne through the pipe gaps; the game ends on a collision."). +- `controls` - an array of `{ name, args?, description?, keys? }`, OR a function + returning that array when controls are dynamic. `keys` is an optional + human-style hint (e.g. `"Space = flap"`). Return only what's available now. +- `getState()` - a **small** JSON snapshot used to verify an action landed. Keep + it compact; this is the latency budget. +- `invoke(name, args)` - perform the named action and return a result (or throw a string the agent will read). Keep `args` shapes simple (strings, numbers, booleans, small objects). The agent -only ever calls actions that `describe()` returned; it never edits your code. +only ever calls actions that `controls` listed; it never edits your code. When +dynamic controls change, call `window.OPENSWARM_APP.refresh()` so the agent +knows to re-read them (it bumps the `__rev` the agent watches). ```tsx -// Register once the app's core object exists (e.g. after the calculator mounts). // `calc` here is the app's own API (Desmos example); use whatever yours exposes. function registerAgentBridge(calc: any) { - (window as any).OPENSWARM_APP = { - describe() { - const actions = [ + window.OPENSWARM_APP!.register({ + rules: 'A graphing calculator. Plot and remove expressions like y=x^2 or y=sin(x).', + controls() { + const controls = [ { name: 'addExpr', args: { latex: 'string' }, description: 'Add a graph expression, e.g. y=x^2' }, { name: 'clear', description: 'Remove all expressions' }, ]; // Dynamic: only offer removeExpr when something is on the graph. if (calc.getExpressions().length > 0) { - actions.push({ name: 'removeExpr', args: { id: 'string' }, description: 'Remove one expression by id' }); + controls.push({ name: 'removeExpr', args: { id: 'string' }, description: 'Remove one expression by id' }); } - return actions; + return controls; }, getState() { return { expressions: calc.getExpressions().map((e: any) => ({ id: e.id, latex: e.latex })) }; }, invoke(name: string, args: any = {}) { - if (name === 'addExpr') { const id = String(Date.now()); calc.setExpression({ id, latex: args.latex }); return { id }; } - if (name === 'removeExpr') { calc.removeExpression({ id: args.id }); return { ok: true }; } - if (name === 'clear') { calc.setBlank(); return { ok: true }; } + if (name === 'addExpr') { const id = String(Date.now()); calc.setExpression({ id, latex: args.latex }); window.OPENSWARM_APP!.refresh(); return { id }; } + if (name === 'removeExpr') { calc.removeExpression({ id: args.id }); window.OPENSWARM_APP!.refresh(); return { ok: true }; } + if (name === 'clear') { calc.setBlank(); window.OPENSWARM_APP!.refresh(); return { ok: true }; } throw `Unknown action: ${name}`; }, - }; + }); } ``` -If you skip the bridge the agent falls back to slow UI-driving, so apps meant to -be agent-operated should always register it. +### Real-time games need a high-level action, not per-frame controls + +An agent acts in discrete tool calls separated by network + model latency +(hundreds of ms to seconds). It physically CANNOT hit frame-timing, so exposing +only `invoke('flap')` makes a reflex game like Flappy Bird understandable but +unwinnable: by the time the agent decides to flap, the bird has already fallen. +For anything real-time, expose a **high-level action** the app executes on its +own tick loop, e.g. `invoke('autopilot', { on: true })` that runs the optimal +input internally, or `invoke('setDifficulty', ...)`. Let the agent set intent; +let the app handle the milliseconds. Non-real-time apps (tools, forms, a +Spotify-style player) don't need this: their actions are already at agent cadence. --- diff --git a/backend/apps/outputs/webapp_template/frontend/run.sh b/backend/apps/outputs/webapp_template/frontend/run.sh index 8c4e611c..4906bddf 100755 --- a/backend/apps/outputs/webapp_template/frontend/run.sh +++ b/backend/apps/outputs/webapp_template/frontend/run.sh @@ -18,6 +18,32 @@ FRONTEND_DIR_ABSPATH="$(dirname "$RUN_FRONTEND_ABSPATH")" cd "$FRONTEND_DIR_ABSPATH" +# --- Agent bridge check ------------------------------------------------------- +# Every OpenSwarm app must register window.OPENSWARM_APP so an agent can drive +# it. The plumbing ships in src/agentBridge.ts; what an app can still forget is +# to CALL register(...) with its own actions. Scan the app source for that call. +# Missing -> warn loudly but DON'T block (work-in-progress apps must still load). +# The real gate is the runtime check in the app agent; OPENSWARM_REQUIRE_BRIDGE=1 +# turns this into a hard failure for anyone who wants strict mode. +check_agent_bridge() { + local src_dir="$FRONTEND_DIR_ABSPATH/src" + [ -d "$src_dir" ] || return 0 + if grep -rEl --include='*.ts' --include='*.tsx' \ + 'OPENSWARM_APP\.register|window\.OPENSWARM_APP[[:space:]]*=' \ + "$src_dir" 2>/dev/null | grep -qv 'agentBridge\.'; then + return 0 + fi + echo "" + printf '\033[31m❌ BRIDGE MISSING: window.OPENSWARM_APP not registered - this app is not agent-operable.\033[0m\n' + printf '\033[31m Call window.OPENSWARM_APP.register({ rules, controls, getState, invoke }) when your app mounts (see SKILL.md).\033[0m\n' + echo "" + if [[ "${OPENSWARM_REQUIRE_BRIDGE:-}" == "1" ]]; then + echo "OPENSWARM_REQUIRE_BRIDGE=1 set; refusing to start without the agent bridge." + exit 1 + fi +} +check_agent_bridge + # Put the bundled Node on PATH so `npm`, `node`, and the vite child # processes all resolve even on a machine with no system Node. The # packaged Electron shell exports OPENSWARM_NODE_PATH (e.g. diff --git a/backend/apps/outputs/workspace_io.py b/backend/apps/outputs/workspace_io.py index ccb75b28..f4df157d 100644 --- a/backend/apps/outputs/workspace_io.py +++ b/backend/apps/outputs/workspace_io.py @@ -68,6 +68,7 @@ _WALK_SKIP_DIRS = frozenset({ ".pytest_cache", ".mypy_cache", ".ruff_cache", + ".openswarm", }) # Cap per-file response size at 256 KB. Hand-written source rarely diff --git a/backend/tests/test_app_agent.py b/backend/tests/test_app_agent.py index eb0d734b..1438eb4a 100644 --- a/backend/tests/test_app_agent.py +++ b/backend/tests/test_app_agent.py @@ -146,3 +146,98 @@ def test_selected_app_context_advertises_app_agent(monkeypatch): assert ctx is not None assert "App id (for AppAgent): abc" in ctx assert "AppAgent(output_id, task)" in ctx + + +# --- bridge readiness + parsing --------------------------------------------- +def test_parse_bridge_result_decodes_json_text(): + assert BA._parse_bridge_result({"text": json.dumps([{"name": "x"}])}) == [{"name": "x"}] + assert BA._parse_bridge_result({"text": "null"}) is None + assert BA._parse_bridge_result({"error": "boom"}) is None + assert BA._parse_bridge_result({"text": "not json"}) is None + assert BA._parse_bridge_result({}) is None + + +def test_bridge_ready_distinguishes_stub_from_registered(): + assert BA._bridge_ready([{"name": "x"}]) is True # legacy array + assert BA._bridge_ready({"controls": [], "__rev": 1}) is True + assert BA._bridge_ready({"__ready": False, "__rev": 0}) is False # template stub + assert BA._bridge_ready({"__error__": "threw"}) is False + assert BA._bridge_ready(None) is False + + +def test_render_app_controls_array_and_object_forms(): + # Legacy array form: no rules, controls rendered. + rules_md, controls_md = BA._render_app_controls([{"name": "clear", "description": "wipe"}]) + assert rules_md == "" + assert "- `clear`: wipe" in controls_md + + # New object form: rules + keys + args rendered. + rules_md, controls_md = BA._render_app_controls({ + "rules": "Flappy Bird. Keep the bird airborne.", + "controls": [{"name": "flap", "keys": "Space = flap", "args": {"force": "number"}, "description": "Flap once"}], + "__rev": 3, + }) + assert "Flappy Bird" in rules_md + assert "- `flap`" in controls_md + assert "[Space = flap]" in controls_md + assert '"force"' in controls_md + + # Not-ready / absent bridge yields nothing to render. + assert BA._render_app_controls({"__ready": False}) is None + assert BA._render_app_controls(None) is None + + +# --- AppDescribe waits for a still-booting bridge ---------------------------- +def test_app_describe_polls_until_bridge_ready(monkeypatch): + calls = {"n": 0} + ready = {"rules": "r", "controls": [{"name": "x"}], "__rev": 1} + + async def _send(request_id, action, browser_id, params, tab_id=""): + calls["n"] += 1 + if calls["n"] < 3: + return {"text": json.dumps({"__ready": False, "__rev": 0})} + return {"text": json.dumps(ready)} + + async def _no_sleep(_s): + return None + + monkeypatch.setattr(BA.ws_manager, "send_browser_command", _send, raising=False) + monkeypatch.setattr(BA.asyncio, "sleep", _no_sleep, raising=True) + monkeypatch.setattr(BA, "_persist_app_controls", lambda *a, **k: None, raising=True) + + out = asyncio.run(BA.execute_browser_tool("AppDescribe", {}, "app:abc")) + assert calls["n"] == 3 # polled twice, succeeded on the third + assert BA._parse_bridge_result(out) == ready + + +def test_app_invoke_does_not_poll(monkeypatch): + calls = {"n": 0} + + async def _send(request_id, action, browser_id, params, tab_id=""): + calls["n"] += 1 + return {"text": json.dumps({"__ready": False})} # would loop forever if AppInvoke waited + + async def _no_sleep(_s): + return None + + monkeypatch.setattr(BA.ws_manager, "send_browser_command", _send, raising=False) + monkeypatch.setattr(BA.asyncio, "sleep", _no_sleep, raising=True) + + asyncio.run(BA.execute_browser_tool("AppInvoke", {"name": "flap"}, "app:abc")) + assert calls["n"] == 1 # single shot, no readiness wait + + +def test_app_describe_persists_controls_cache(monkeypatch, tmp_path): + ready = {"rules": "Keep the bird airborne.", "controls": [{"name": "flap", "keys": "Space"}], "__rev": 1} + + async def _send(request_id, action, browser_id, params, tab_id=""): + return {"text": json.dumps(ready)} + + monkeypatch.setattr(BA.ws_manager, "send_browser_command", _send, raising=False) + monkeypatch.setattr(BA, "_app_workspace_dir", lambda bid: str(tmp_path), raising=True) + + asyncio.run(BA.execute_browser_tool("AppDescribe", {}, "app:abc")) + controls = (tmp_path / ".openswarm" / "controls.md").read_text() + rules = (tmp_path / ".openswarm" / "rules.md").read_text() + assert "- `flap`" in controls and "[Space]" in controls + assert "Keep the bird airborne." in rules diff --git a/electron/package-lock.json b/electron/package-lock.json index 96e0bad9..0f70e4af 100644 --- a/electron/package-lock.json +++ b/electron/package-lock.json @@ -1,12 +1,12 @@ { "name": "openswarm", - "version": "1.2.77", + "version": "1.2.79", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "openswarm", - "version": "1.2.77", + "version": "1.2.79", "hasInstallScript": true, "dependencies": { "electron-updater": "6.8.3", @@ -567,7 +567,6 @@ "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -1435,7 +1434,6 @@ "integrity": "sha512-glMJgnTreo8CFINujtAhCgN96QAqApDMZ8Vl1r8f0QT8QprvC1UCltV4CcWj20YoIyLZx6IUskaJZ0NV8fokcg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "app-builder-lib": "26.8.1", "builder-util": "26.8.1", @@ -1584,7 +1582,6 @@ "integrity": "sha512-o288fIdgPLHA76eDrFADHPoo7VyGkDCYbLV1GzndaMSAVBoZrGvM9m2IehdcVMzdAZJ2eV9bgyissQXHv5tGzA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "app-builder-lib": "26.8.1", "builder-util": "26.8.1", @@ -2888,7 +2885,6 @@ "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, From 8cf868784fa2796bd2864befb13b9ae2bd2cc4e2 Mon Sep 17 00:00:00 2001 From: SirKentut <81878031+SirKentut@users.noreply.github.com> Date: Tue, 16 Jun 2026 00:52:00 -0700 Subject: [PATCH 05/16] [pierre] fix/app-agent: re-attach app controls on every task Run AppDescribe and front-load the app's controls each task in app mode, not just on fresh conversations. The bridge can appear between runs and a resumed history may carry a stale screenshot-it strategy; re-reading the controls re-points the agent at the bridge instead of inheriting old fumbling. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/apps/agents/browser/browser_agent.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index 93c043eb..70318128 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -697,8 +697,13 @@ async def run_browser_agent( # until controls change. This is also the runtime bridge gate: if the bridge # never comes up, fail loudly into the logs + the agent's first message (and, # under OPENSWARM_REQUIRE_BRIDGE=1, end the run rather than UI-fumble). + # NOTE: app mode runs this on EVERY task, even a resumed conversation. The + # bridge can appear between runs (e.g. the app was just made agent-operable), + # and a resumed history may carry a stale "no bridge, screenshot it" strategy; + # re-reading + re-attaching the controls each task re-points the agent at the + # bridge instead of letting it inherit the old fumbling. app_front_load = "" - if app_mode and not prior_messages: + if app_mode: try: _dv = _parse_bridge_result(await execute_browser_tool("AppDescribe", {}, browser_id, tab_id)) except Exception: @@ -746,11 +751,12 @@ async def run_browser_agent( "cannot operate it, say so in Done with success=false.]" ) - # Front-load the prefetched perception into the first user turn so the model - # can act immediately (only when this is a fresh conversation; a resumed one - # already knows the page). The visible task text stays clean. + # Front-load perception (browser) or the app's controls (app mode) into the + # new task's user turn so the model can act immediately. Browser-mode + # perception is only set on a fresh conversation; app-mode controls attach on + # every task (see note above). The visible task text stays clean. _front = preloaded_perception or app_front_load - first_user_content = task + _front if (_front and not prior_messages) else task + first_user_content = task + _front if _front else task messages: list[dict] = list(prior_messages) + [{"role": "user", "content": first_user_content}] # Seed with the front-loaded reads: they really ran and returned content, so a # read task the agent answers straight from them is NOT a "did nothing" ghost. From b745c54c3e640b7b4648a03ee166b6f2bd584259 Mon Sep 17 00:00:00 2001 From: SirKentut <81878031+SirKentut@users.noreply.github.com> Date: Thu, 18 Jun 2026 02:24:13 -0700 Subject: [PATCH 06/16] [pierre] chore: add watch-moves.sh for human-readable App Agent action log --- backend/watch-moves.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 backend/watch-moves.sh diff --git a/backend/watch-moves.sh b/backend/watch-moves.sh new file mode 100644 index 00000000..ad3d7589 --- /dev/null +++ b/backend/watch-moves.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# watch-moves.sh: human-readable, moves-only view of the App Agent. +# Shows just the actions it takes (clicks, keypresses, waits), one per line, +# plus task-start and bridge-missing milestones. Hides all the dispatch/result/ +# electron echo noise. Usage: bash backend/watch-moves.sh [logfile] +LOG="${1:-/tmp/openswarm.log}" +tail -f "$LOG" | awk ' + { gsub(/\033\[[0-9;]*m/, "") } # strip color codes + match($0, /[0-9][0-9]:[0-9][0-9]:[0-9][0-9]/) { t = substr($0, RSTART, 8) } + /\[app-agent\] START loop/ { print ""; print "=== " t " TASK START ==="; next } + /BRIDGE MISSING/ { print t " !! bridge missing: app is NOT agent-operable, driving UI blind"; next } + /\[browser-action\]/ { + sub(/.*\[browser-action\] [A-Za-z]+: /, "") # drop everything up to the action + sub(/ *-> .*/, "") # drop the trailing browser_id + print t " > " $0 + next + } +' From e0c4b5859fa0b9300e2e51c7d1ae5779ad20ac7e Mon Sep 17 00:00:00 2001 From: SirKentut <81878031+SirKentut@users.noreply.github.com> Date: Thu, 18 Jun 2026 02:33:18 -0700 Subject: [PATCH 07/16] [pierre] feat: add BrowserClickPoint and human-style canvas/game control for app agent --- backend/apps/agents/browser/browser_agent.py | 82 ++++++++++++++++++- backend/apps/agents/browser/browser_schema.py | 69 ++++++++++++++-- 2 files changed, 141 insertions(+), 10 deletions(-) diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index 70318128..10fe71bd 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -219,6 +219,71 @@ def _persist_app_controls(browser_id: str, describe_value: object) -> None: logger.debug("[app-agent] failed to persist controls cache", exc_info=True) +# Single-tool names -> the sub-action type they map to, so one summarizer covers +# both BrowserPressKey({key}) and a batch's {"type":"press_key","params":{key}}. +_SINGLE_ACTION_TYPE = { + "BrowserClick": "click", "BrowserClickIndex": "click_index", + "BrowserClickByName": "click_name", "BrowserType": "type", + "BrowserPressKey": "press_key", "BrowserScroll": "scroll", + "BrowserNavigate": "navigate", "BrowserClickPoint": "click_point", +} + + +def _summ_step(stype: str, params: dict) -> str: + """Compact human label for one action step: the actual key/selector/text, not + just the verb. This is what lets the [backend] pane show 'key:ArrowRight x5' + instead of an opaque 'BrowserBatch'.""" + p = params or {} + if stype == "press_key": + return f"key:{p.get('key', '?')}" + if stype == "click": + return f"click({p.get('selector', '?')})" + if stype == "click_index": + return f"click#{p.get('index', '?')}" + if stype == "click_point": + return f"tap({p.get('xPercent', '?')}%,{p.get('yPercent', '?')}%)" + if stype == "click_name": + return f"clickName({p.get('name', '?')})" + if stype == "type": + return f"type({p.get('selector', '')}={str(p.get('text', ''))[:30]!r})" + if stype == "wait": + return f"wait({p.get('milliseconds') or p.get('until') or ''})" + if stype == "scroll": + return f"scroll({p.get('direction', 'down')})" + if stype == "navigate": + return f"nav({str(p.get('url', ''))[:60]})" + if stype == "list_interactives": + return "list" + return stype or "?" + + +def _collapse_steps(items: list[str]) -> str: + """'ArrowRight, ArrowRight, ArrowRight' -> 'key:ArrowRight x3' so a 5-key + burst reads as one token instead of scrolling the pane.""" + runs: list[list] = [] + for it in items: + if runs and runs[-1][0] == it: + runs[-1][1] += 1 + else: + runs.append([it, 1]) + return ", ".join(s if n == 1 else f"{s} x{n}" for s, n in runs) + + +def _summarize_action(tool_name: str, tool_input: dict) -> str: + """One-line summary of what an action tool is about to do, or "" for pure + reads (screenshot/list/describe/getstate) that need no action log.""" + ti = tool_input or {} + if tool_name == "BrowserBatch": + steps = [_summ_step((a or {}).get("type", ""), (a or {}).get("params")) + for a in (ti.get("actions") or [])] + return _collapse_steps(steps) or "(empty batch)" + if tool_name == "AppInvoke": + args = ti.get("args") + return f"{ti.get('name', '?')}" + (f"({json.dumps(args)[:60]})" if args else "") + stype = _SINGLE_ACTION_TYPE.get(tool_name) + return _summ_step(stype, ti) if stype else "" + + async def execute_browser_tool( tool_name: str, tool_input: dict, browser_id: str, tab_id: str = "", ) -> dict: @@ -227,6 +292,13 @@ async def execute_browser_tool( # browser-agent runs stay quiet. Greppable prefix; remove when done. _trace = browser_id.startswith("app:") or tool_name in APP_BRIDGE_TOOLS + # One greppable line naming the actual buttons/keys/selectors this call drives, + # so a run reads as "key:ArrowRight x5" rather than an opaque tool name. Fires + # for action tools only (reads stay quiet) and ungated so web runs get it too. + _action = _summarize_action(tool_name, tool_input) + if _action: + logger.info(f"[browser-action] {tool_name}: {_action} -> {browser_id}") + # App bridge tools translate to a single BrowserEvaluate against the app's # window.OPENSWARM_APP, so they need no frontend command-handler changes. if tool_name in APP_BRIDGE_TOOLS: @@ -745,10 +817,12 @@ async def run_browser_agent( "action_log": [], "final_screenshot": None, } app_front_load = ( - f"\n\n[{_msg}. AppDescribe/AppInvoke will not work. Fall back to " - "driving the UI directly (BrowserListInteractives, " - "BrowserClickIndex, BrowserBatch, BrowserScreenshot). If you " - "cannot operate it, say so in Done with success=false.]" + f"\n\n[{_msg}. Operate it like a person instead: see with " + "BrowserScreenshot, then play with BrowserPressKey (keys like " + "w/a/s/d, arrows, Space, Enter) and BrowserClickPoint (tap a screen " + "point). For a normal HTML app use BrowserListInteractives + " + "BrowserClickIndex. Only give up (Done success=false) after you have " + "actually tried pressing keys and nothing responds.]" ) # Front-load perception (browser) or the app's controls (app mode) into the diff --git a/backend/apps/agents/browser/browser_schema.py b/backend/apps/agents/browser/browser_schema.py index fee0fca5..67ea3025 100644 --- a/backend/apps/agents/browser/browser_schema.py +++ b/backend/apps/agents/browser/browser_schema.py @@ -394,6 +394,8 @@ BROWSER_TOOLS_SCHEMA = [ "Sub-action types and their params:\n" "- click_index: { index: int }\n" "- press_key: { key: str }\n" + "- click_point: { xPercent: number, yPercent: number, hold_ms?: int } " + "(tap a screen point; for canvas apps/games)\n" "- type: { selector: str, text: str }\n" "- click: { selector: str }\n" "- scroll: { direction?: 'up'|'down', amount?: int }\n" @@ -418,7 +420,7 @@ BROWSER_TOOLS_SCHEMA = [ "properties": { "type": { "type": "string", - "enum": ["click_index", "press_key", "type", "wait", "scroll", "navigate", "click", "list_interactives"], + "enum": ["click_index", "press_key", "click_point", "type", "wait", "scroll", "navigate", "click", "list_interactives"], }, "params": {"type": "object"}, }, @@ -454,6 +456,43 @@ BROWSER_TOOLS_SCHEMA = [ "required": ["key"], }, }, + { + "name": "BrowserClickPoint", + "description": ( + "Tap/click at a point on the screen using a real native mouse event, " + "WITHOUT needing a DOM element. This is the way to operate a " + "app or game (the kind with no clickable HTML elements): you click a " + "spot the way a person does. Give the position as a PERCENT of the view " + "(xPercent/yPercent, 0-100, with 0,0 = top-left and 50,50 = center), " + "read off the screenshot. Optional hold_ms presses and holds (e.g. a " + "charge-up or a platformer jump). For element-based pages prefer " + "BrowserClickIndex; use this when there is nothing in the element list " + "to click." + ), + "input_schema": { + "type": "object", + "properties": { + "xPercent": { + "type": "number", + "description": "Horizontal position as a percent of view width (0=left, 100=right).", + }, + "yPercent": { + "type": "number", + "description": "Vertical position as a percent of view height (0=top, 100=bottom).", + }, + "hold_ms": { + "type": "number", + "description": "Optional. Milliseconds to hold the button down before releasing (default 0 = a tap). Max 5000.", + }, + "button": { + "type": "string", + "enum": ["left", "right", "middle"], + "description": "Mouse button; defaults to left.", + }, + }, + "required": ["xPercent", "yPercent"], + }, + }, { "name": "BrowserWait", "description": ( @@ -642,7 +681,7 @@ BROWSER_TOOLS_SCHEMA = [ # tools are not offered to it at all; acting means a BrowserBatch array, and # the one deliberate solo path is BrowserClickIndex (irreversible step with # expect, or a text-box fill). Executors and replay still support everything. -_SOLO_MUTATORS_HIDDEN = {"BrowserNavigate", "BrowserClick", "BrowserType", "BrowserScroll", "BrowserPressKey"} +_SOLO_MUTATORS_HIDDEN = {"BrowserNavigate", "BrowserClick", "BrowserType", "BrowserScroll", "BrowserPressKey", "BrowserClickPoint"} MODEL_VISIBLE_TOOLS = [t for t in BROWSER_TOOLS_SCHEMA if t["name"] not in _SOLO_MUTATORS_HIDDEN] ACTION_MAP = { @@ -659,6 +698,7 @@ ACTION_MAP = { "BrowserPressKey": "press_key", "BrowserListInteractives": "list_interactives", "BrowserClickIndex": "click_index", + "BrowserClickPoint": "click_point", "BrowserBatch": "batch", "BrowserDetectWebMCP": "detect_webmcp", "BrowserListRoutes": "list_routes", @@ -739,6 +779,9 @@ _APP_FALLBACK_TOOL_NAMES = [ "ReportProgress", "Done", "BrowserScreenshot", "BrowserGetText", "BrowserListInteractives", "BrowserClickIndex", "BrowserBatch", + # Human-style native input for apps with no clickable DOM (canvas games): + # press real keys and tap real screen points the way a person plays. + "BrowserPressKey", "BrowserClickPoint", ] _app_fallback_tools = [t for t in BROWSER_TOOLS_SCHEMA if t["name"] in _APP_FALLBACK_TOOL_NAMES] # ReportProgress + Done lead, then the bridge tools, then UI fallback. @@ -1004,10 +1047,23 @@ APP_SYSTEM_PROMPT = ( "so call AppDescribe ONCE to refresh them. As long as __rev is unchanged, " "trust the controls you already have and do not re-describe.\n\n" - "## If there is no bridge\n" - "If AppDescribe (or AppGetState) returns null, this app doesn't expose the " - "bridge. Fall back to driving the UI directly: BrowserListInteractives to find " - "controls, BrowserClickIndex / BrowserBatch to act, BrowserScreenshot to see. " + "## If there is no bridge: operate it like a person\n" + "If AppDescribe (or AppGetState) returns null, this app exposes no bridge, so " + "drive it directly the way a human would, by looking at the screen and using " + "the keyboard and mouse:\n" + "- SEE with BrowserScreenshot (your main sense here); read on-screen text/score " + "from it. BrowserGetText helps for text-heavy apps.\n" + "- For a normal HTML app (buttons, inputs, forms): BrowserListInteractives to " + "find controls, then BrowserClickIndex / BrowserBatch to act.\n" + "- For a CANVAS app or GAME (no clickable elements in the list): play it with " + "native input. BrowserPressKey for keyboard (e.g. 'Space' to flap, " + "'ArrowLeft'/'ArrowRight' to move, 'Enter' to start) and BrowserClickPoint to " + "tap a spot, giving xPercent/yPercent read off the screenshot (50,50 = center). " + "These are REAL OS-level events, identical to you pressing a key or clicking, so " + "the game responds exactly as it does for a person. Take a screenshot to " + "confirm what changed, then act again.\n" + "- Need fast repeated input (rapid flaps/taps)? Put several BrowserPressKey or " + "BrowserClickPoint steps in one BrowserBatch so they fire in a single turn.\n" "If you truly cannot operate it, say so plainly in Done with success=false.\n\n" "## ReportProgress before acting\n" @@ -1036,6 +1092,7 @@ _ACTION_TOOLS_REQUIRING_REPORT = { "BrowserScroll", "BrowserEvaluate", "BrowserClickIndex", # Phase 3 + "BrowserClickPoint", # app mode: tap a canvas/game at a screen point "BrowserBatch", # Phase 4 "AppInvoke", # app mode: invoking an app action mutates state } From edd018cb4ce795efb276643a7ba99fed917c47b0 Mon Sep 17 00:00:00 2001 From: SirKentut <81878031+SirKentut@users.noreply.github.com> Date: Thu, 18 Jun 2026 02:35:25 -0700 Subject: [PATCH 08/16] [pierre] feat: handle click_point via native CDP mouse event in browser command handler --- frontend/src/shared/browserCommandHandler.ts | 55 +++++++++++++++++++- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/frontend/src/shared/browserCommandHandler.ts b/frontend/src/shared/browserCommandHandler.ts index 74436048..4b76ed90 100644 --- a/frontend/src/shared/browserCommandHandler.ts +++ b/frontend/src/shared/browserCommandHandler.ts @@ -8,7 +8,7 @@ import { shouldStopWaiting, SETTLE_POLL_MS, settleProbeJs } from './browserSettl let initialized = false; -export type BrowserAction = 'screenshot' | 'get_text' | 'get_console' | 'navigate' | 'click' | 'type' | 'evaluate' | 'get_elements' | 'scroll' | 'wait' | 'press_key' | 'list_interactives' | 'click_index' | 'batch' | 'detect_webmcp' | 'list_routes' | 'replay_route' | 'click_by_name'; +export type BrowserAction = 'screenshot' | 'get_text' | 'get_console' | 'navigate' | 'click' | 'type' | 'evaluate' | 'get_elements' | 'scroll' | 'wait' | 'press_key' | 'list_interactives' | 'click_index' | 'click_point' | 'batch' | 'detect_webmcp' | 'list_routes' | 'replay_route' | 'click_by_name'; export interface BrowserActivity { action: BrowserAction; @@ -68,6 +68,7 @@ const ACTION_LABELS: Record = { press_key: 'Pressing key...', list_interactives: 'Reading page structure...', click_index: 'Clicking element...', + click_point: 'Tapping screen...', click_by_name: 'Clicking element...', batch: 'Running batch...', }; @@ -349,6 +350,45 @@ async function handlePressKey(wv: BrowserWebview, params: Record): return { text: `Pressed ${rawKey}` }; } +// Click at a viewport coordinate (percent of the view's width/height) with a +// real, trusted CDP mouse event, NO DOM element required. This is what lets the +// app agent operate a bare game the way a person taps the screen: the +// AX-tree click paths (click_index/click_by_name) can't target a canvas because +// it exposes no nodes, but a coordinate dispatch lands anywhere. Optional +// hold_ms presses and holds (platformers, charge-up mechanics). +async function handleClickPoint(wv: BrowserWebview, params: Record): Promise> { + const xPercent = Number(params.xPercent); + const yPercent = Number(params.yPercent); + if (!Number.isFinite(xPercent) || !Number.isFinite(yPercent)) { + return { error: 'xPercent and yPercent are required (0-100, percent of the view).' }; + } + const cx = Math.max(0, Math.min(100, xPercent)); + const cy = Math.max(0, Math.min(100, yPercent)); + const button = params.button === 'right' ? 'right' : params.button === 'middle' ? 'middle' : 'left'; + const holdMs = Math.max(0, Math.min(Number(params.hold_ms) || 0, 5000)); + // Read the guest's own viewport so coords are correct under zoom/DPR, not the + // host element's box. One cheap round-trip; falls back to the element box. + let vw = wv.clientWidth, vh = wv.clientHeight; + try { + const d = await wv.executeJavaScript('({w: window.innerWidth, h: window.innerHeight})'); + if (d && d.w > 0 && d.h > 0) { vw = d.w; vh = d.h; } + } catch { /* use the element box as a fallback */ } + const x = (cx / 100) * vw; + const y = (cy / 100) * vh; + try { + await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x, y }); + await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button, clickCount: 1 }); + if (holdMs > 0) await new Promise((r) => setTimeout(r, holdMs)); + await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x, y, button, clickCount: 1 }); + } catch (err: any) { + return { error: `Click point failed: ${err?.message || String(err)}` }; + } + return { + text: `Clicked at (${Math.round(x)}, ${Math.round(y)})${holdMs ? ` held ${holdMs}ms` : ''}.`, + clickX: cx, clickY: cy, url: wv.getURL(), + }; +} + // CDP Accessibility.getFullAXTree sees computed roles/names even on hostile sites with unlabeled DOMs. const INTERACTIVE_ROLES = new Set([ 'button', 'link', 'textbox', 'combobox', 'checkbox', 'menuitem', @@ -961,11 +1001,12 @@ const MAX_BATCH_ACTIONS = 5; type SubActionType = | 'click_index' | 'press_key' | 'type' | 'wait' - | 'scroll' | 'navigate' | 'click' | 'list_interactives'; + | 'scroll' | 'navigate' | 'click' | 'click_point' | 'list_interactives'; const BATCH_DISPATCH: Record) => Promise>> = { click_index: handleClickIndex, press_key: handlePressKey, + click_point: handleClickPoint, type: handleType, wait: handleWait, scroll: handleScroll, @@ -1483,6 +1524,16 @@ async function runBrowserCommand( }); } break; + case 'click_point': + result = await handleClickPoint(wv, params); + if (result.clickX != null && result.clickY != null) { + setActivity(browser_id, { + action: 'click_point', + detail, + coords: { xPercent: result.clickX, yPercent: result.clickY }, + }); + } + break; case 'batch': result = await handleBatch(wv, params); break; From a45bcf7f01e26549efae588b3301d489b393d50a Mon Sep 17 00:00:00 2001 From: SirKentut <81878031+SirKentut@users.noreply.github.com> Date: Thu, 18 Jun 2026 02:36:32 -0700 Subject: [PATCH 09/16] [pierre] docs: note OPENSWARM_APP bridge works in lightweight app builder mode --- backend/apps/outputs/app_builder_skill.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/backend/apps/outputs/app_builder_skill.md b/backend/apps/outputs/app_builder_skill.md index 8c3b86bf..66a5e8ec 100644 --- a/backend/apps/outputs/app_builder_skill.md +++ b/backend/apps/outputs/app_builder_skill.md @@ -56,6 +56,13 @@ fast. 3. Leave `frontend/package.json`, `frontend/vite.config.ts`, `run.sh`, `.env`, `meta.json` alone — vite still needs them. 4. Don't run `bash backend_init.sh` — lightweight mode has no backend. +5. Agent control still works without the template: `window.OPENSWARM_APP` is + injected by the app shell, so even here you can make the app agent-operable + by calling `window.OPENSWARM_APP.register({ rules, controls, getState, + invoke })` from your inline `