diff --git a/backend/apps/agents/agent_manager.py b/backend/apps/agents/agent_manager.py index 6abe7937..ed350768 100644 --- a/backend/apps/agents/agent_manager.py +++ b/backend/apps/agents/agent_manager.py @@ -1142,7 +1142,7 @@ class AgentManager: # dispatch layer (see _build_mcp_servers docstring). mcp_servers = await self._build_mcp_servers(session.allowed_tools, session.active_mcps) - _browser_delegation_tools = ["CreateBrowserAgent", "BrowserAgent", "BrowserAgents"] + _browser_delegation_tools = ["CreateBrowserAgent", "BrowserAgent", "BrowserAgents", "AppAgent"] _browser_all_denied = all( _builtin_perms.get(t, "always_allow") == "deny" for t in _browser_delegation_tools @@ -1165,6 +1165,8 @@ class AgentManager: "OPENSWARM_AGENT_MODEL": session.model, "OPENSWARM_DASHBOARD_ID": session.dashboard_id or "", "OPENSWARM_PRE_SELECTED_BROWSER_IDS": ",".join(pre_selected_bids), + # Apps the user selected this turn; AppAgent may only target these. + "OPENSWARM_SELECTED_APP_IDS": ",".join(selected_app_output_ids or []), "OPENSWARM_PARENT_SESSION_ID": session.id, }, "type": "stdio", diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index fafb1a9a..21851de3 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -71,6 +71,9 @@ from backend.apps.agents.browser import browser_schema from backend.apps.agents.browser.browser_schema import ( _ACTION_TOOLS_REQUIRING_REPORT, ACTION_MAP, + APP_BRIDGE_TOOLS, + APP_SYSTEM_PROMPT, + APP_VISIBLE_TOOLS, BROWSER_TOOLS_SCHEMA, MAX_TURNS, MODEL_MAP, @@ -90,19 +93,64 @@ _CONFIRM_TOOLS = { } +def _app_bridge_expression(tool_name: str, tool_input: dict) -> str: + """JS for an app bridge tool. Each expression returns a JSON STRING (so it + round-trips as text) and never throws; bridge errors come back as JSON.""" + if tool_name == "AppDescribe": + call = "window.OPENSWARM_APP.describe()" + elif tool_name == "AppGetState": + call = "window.OPENSWARM_APP.getState()" + else: # AppInvoke + name = json.dumps(tool_input.get("name", "")) + args = json.dumps(tool_input.get("args") or {}) + call = f"window.OPENSWARM_APP.invoke({name}, {args})" + return ( + "(function(){try{" + "var A=window.OPENSWARM_APP;" + "if(!A||typeof A.describe!=='function'){return JSON.stringify(null);}" + f"var r={call};" + "return JSON.stringify(r===undefined?null:r);" + "}catch(e){return JSON.stringify({__error__:String((e&&e.message)||e)});}})()" + ) + + async def execute_browser_tool( tool_name: str, tool_input: dict, browser_id: str, tab_id: str = "", ) -> dict: """Execute a browser tool via ws_manager directly (no MCP/HTTP round-trip).""" + # [app-agent] step trace: only for app targets / bridge tools so normal + # browser-agent runs stay quiet. Greppable prefix; remove when done. + _trace = browser_id.startswith("app:") or tool_name in APP_BRIDGE_TOOLS + + # App bridge tools translate to a single BrowserEvaluate against the app's + # window.OPENSWARM_APP, so they need no frontend command-handler changes. + if tool_name in APP_BRIDGE_TOOLS: + action = "evaluate" + expr = _app_bridge_expression(tool_name, tool_input) + params = {"expression": expr} + request_id = uuid4().hex + if _trace: + logger.info(f"[app-agent] DISPATCH {tool_name} -> {browser_id} (req {request_id[:8]}) js={expr[:120]}") + result = await ws_manager.send_browser_command( + request_id, action, browser_id, params, tab_id=tab_id, + ) + if _trace: + logger.info(f"[app-agent] RESULT {tool_name} <- {browser_id}: {json.dumps(result)[:300]}") + return result + action = ACTION_MAP.get(tool_name) if not action: return {"error": f"Unknown browser tool: {tool_name}"} params = {k: v for k, v in tool_input.items()} request_id = uuid4().hex + if _trace: + logger.info(f"[app-agent] DISPATCH {tool_name}/{action} -> {browser_id} (req {request_id[:8]})") result = await ws_manager.send_browser_command( request_id, action, browser_id, params, tab_id=tab_id, ) + if _trace: + logger.info(f"[app-agent] RESULT {tool_name} <- {browser_id}: {json.dumps(result)[:300]}") return result @@ -352,27 +400,36 @@ async def run_browser_agent( pre_selected: bool = False, initial_url: str | None = None, parent_session_id: str | None = None, + app_mode: bool = False, ) -> dict: """Run a browser sub-agent loop for a single browser card. Creates a visible AgentSession, streams progress via WebSocket, and returns the full action log + summary + final screenshot. + + When app_mode is set, browser_id points at an OpenSwarm-built app's webview + (registered as "app:"). The agent drives it through the app's + native bridge (window.OPENSWARM_APP) instead of web perception: no initial + navigate, no AX-tree front-load, and a lean app toolset + prompt. """ from backend.apps.agents.agent_manager import agent_manager _browser_perms = load_builtin_permissions() + if app_mode: + logger.info(f"[app-agent] START loop: browser_id={browser_id} task={task[:140]!r}") + session_id = uuid4().hex cancel_event = asyncio.Event() session = AgentSession( id=session_id, - name=f"Browser Agent", + name="App Agent" if app_mode else "Browser Agent", model=model, mode="browser-agent", status="running", dashboard_id=dashboard_id, browser_id=browser_id, - system_prompt=SYSTEM_PROMPT, + system_prompt=APP_SYSTEM_PROMPT if app_mode else SYSTEM_PROMPT, parent_session_id=parent_session_id, ) session._cancel_event = cancel_event @@ -431,16 +488,20 @@ async def run_browser_agent( current_url = "" preloaded_reads: list[dict] = [] # real front-loaded reads, seeded into action_log _resumed = bool(browser_history._browser_history.get(browser_id)) - if initial_url: - nav_result = await execute_browser_tool( - "BrowserNavigate", {"url": initial_url}, browser_id, tab_id, - ) - logger.info(f"Browser agent {session_id}: navigated to {initial_url}: {nav_result.get('text', nav_result.get('error', ''))}") - preloaded_perception, current_url, preloaded_reads = await _perceive(initial_url) - elif not _resumed: - # Fresh task on an existing card: perceive the current page to learn its - # host (for replay) and front-load turn 1 (this path used to start cold). - preloaded_perception, current_url, preloaded_reads = await _perceive("") + # App mode skips this whole block: the app is already loaded and its DOM is + # uninformative (often a bare ), so the agent perceives via the bridge + # (AppDescribe) on turn 1 instead of navigating or front-loading the AX tree. + if not app_mode: + if initial_url: + nav_result = await execute_browser_tool( + "BrowserNavigate", {"url": initial_url}, browser_id, tab_id, + ) + logger.info(f"Browser agent {session_id}: navigated to {initial_url}: {nav_result.get('text', nav_result.get('error', ''))}") + preloaded_perception, current_url, preloaded_reads = await _perceive(initial_url) + elif not _resumed: + # Fresh task on an existing card: perceive the current page to learn its + # host (for replay) and front-load turn 1 (this path used to start cold). + preloaded_perception, current_url, preloaded_reads = await _perceive("") from backend.apps.settings.settings import load_settings from backend.apps.settings.credentials import get_anthropic_client_for_model @@ -594,8 +655,8 @@ async def run_browser_agent( # Advisory per-domain hints: seed the system prompt with what a prior agent # learned about this domain (if we know the domain at start), and keep the # store fresh from each ReportProgress. Re-verify, never blindly trust. - start_domain = _extract_domain(initial_url) if initial_url else None - run_system_prompt = SYSTEM_PROMPT + start_domain = None if app_mode else (_extract_domain(initial_url) if initial_url else None) + run_system_prompt = APP_SYSTEM_PROMPT if app_mode else SYSTEM_PROMPT if start_domain: prior_note = browser_history.get_domain_note(start_domain) if prior_note: @@ -611,20 +672,23 @@ async def run_browser_agent( # from past successful runs) so the model skips re-discovery. Advisory text, # re-verified by the agent, never auto-run. Keyed by full host like skills. pb_seeded = False # whether tier-2 strategy was injected, for measuring its effect - _pb_host = browser_skills.host_of(initial_url or current_url or "") - if _pb_host: - _pb_block = browser_playbook.format_for_prompt(_pb_host) - if _pb_block: - run_system_prompt = run_system_prompt + _pb_block - pb_seeded = True - # Tier-3 memory: the cross-site priors learned on EVERY other site, injected on - # every run (host-agnostic) so a brand-new site isn't fully cold. Advisory, capped. - try: - _meta_block = browser_meta_playbook.format_for_prompt() - if _meta_block: - run_system_prompt = run_system_prompt + _meta_block - except Exception: - pass + # Playbooks are web-only (keyed by host); app mode has no host, the bridge is + # the whole strategy, so skip both tiers. + if not app_mode: + _pb_host = browser_skills.host_of(initial_url or current_url or "") + if _pb_host: + _pb_block = browser_playbook.format_for_prompt(_pb_host) + if _pb_block: + run_system_prompt = run_system_prompt + _pb_block + pb_seeded = True + # Tier-3 memory: the cross-site priors learned on EVERY other site, injected + # on every run (host-agnostic) so a brand-new site isn't fully cold. + try: + _meta_block = browser_meta_playbook.format_for_prompt() + if _meta_block: + run_system_prompt = run_system_prompt + _meta_block + except Exception: + pass # Prompt-caching shapes built once: system as a single cached text block, # and the last tool carrying the cache_control marker (Anthropic keys on the @@ -633,7 +697,7 @@ async def run_browser_agent( "type": "text", "text": run_system_prompt, "cache_control": {"type": "ephemeral"}, }] - _cached_tools = [dict(t) for t in browser_schema.MODEL_VISIBLE_TOOLS] + _cached_tools = [dict(t) for t in (APP_VISIBLE_TOOLS if app_mode else browser_schema.MODEL_VISIBLE_TOOLS)] if _cached_tools: _cached_tools[-1] = {**_cached_tools[-1], "cache_control": {"type": "ephemeral"}} @@ -2277,6 +2341,9 @@ async def run_browser_agents( browser_id = task_def.get("browser_id", "") task_text = task_def.get("task", "") url = task_def.get("url", "") + # App mode: browser_id is a pre-registered app webview ("app:"); + # never create/reuse a browser card, never navigate (the app is loaded). + app_mode = bool(task_def.get("app_mode")) # advisory deep entry (from the fast-path brief): a NEW card opens on it # directly (no google detour); a REUSED card is never moved by it, so a # warm card's deeper page state always wins @@ -2306,7 +2373,7 @@ async def run_browser_agents( pass else: await asyncio.sleep(2.0) - elif browser_id: + elif browser_id and not app_mode: _active_agent_cards.add(browser_id) is_pre_selected = browser_id in pre_selected @@ -2318,11 +2385,13 @@ async def run_browser_agents( model=model, dashboard_id=dashboard_id, pre_selected=is_pre_selected, - initial_url=_nav_url if _nav_url and browser_id not in pre_selected else None, + initial_url=None if app_mode else (_nav_url if _nav_url and browser_id not in pre_selected else None), parent_session_id=parent_session_id, + app_mode=app_mode, ) finally: - _active_agent_cards.discard(browser_id) + if not app_mode: + _active_agent_cards.discard(browser_id) results = await asyncio.gather(*[_run_one(t) for t in tasks], return_exceptions=True) diff --git a/backend/apps/agents/browser/browser_schema.py b/backend/apps/agents/browser/browser_schema.py index afc14fac..546d127b 100644 --- a/backend/apps/agents/browser/browser_schema.py +++ b/backend/apps/agents/browser/browser_schema.py @@ -668,6 +668,80 @@ ACTION_MAP = { "BrowserClickByName": "click_by_name", } +# --- App agent: driving an OpenSwarm-built app via its native bridge --------- +# Apps expose window.OPENSWARM_APP = { describe(), getState(), invoke(name,args) }. +# The agent reads structure/state and acts through that bridge in single +# executeJavaScript round-trips, no screenshots or accessibility tree. These +# three tools are translated to BrowserEvaluate in execute_browser_tool, so they +# need no frontend command-handler changes. +APP_TOOLS_SCHEMA = [ + { + "name": "AppDescribe", + "description": ( + "Read the app's CURRENT list of actions you can take, straight from the " + "app itself (window.OPENSWARM_APP.describe()). Returns an array of " + "{name, args, description}. The app's controls are DYNAMIC, they appear " + "and disappear as state changes, so call this again after any AppInvoke " + "that could add or remove actions; never assume the list is stable. " + "Returns null if the app does not expose the bridge (then fall back to " + "BrowserListInteractives/BrowserScreenshot)." + ), + "input_schema": {"type": "object", "properties": {}, "required": []}, + }, + { + "name": "AppGetState", + "description": ( + "Read a small JSON snapshot of the app's current state " + "(window.OPENSWARM_APP.getState()). Use it to check what's on screen " + "and to verify an action landed. Returns null if the bridge is absent." + ), + "input_schema": {"type": "object", "properties": {}, "required": []}, + }, + { + "name": "AppInvoke", + "description": ( + "Perform one app action by name with arguments " + "(window.OPENSWARM_APP.invoke(name, args)). The name MUST be one that " + "AppDescribe just returned; never invent or modify actions, only invoke " + "the ones the app exposes. Returns the action's result, or an " + "{__error__} if it threw. After invoking, re-AppDescribe if the action " + "may have changed which controls exist." + ), + "input_schema": { + "type": "object", + "properties": { + "name": {"type": "string", "description": "Action name from AppDescribe."}, + "args": { + "type": "object", + "description": "Arguments object for the action; {} if none.", + }, + }, + "required": ["name"], + }, + }, +] + +# Bridge tool names -> the JS the app exposes. execute_browser_tool maps these to +# a BrowserEvaluate. Each expression returns a JSON string (so it round-trips as +# text) and never throws (errors are returned as JSON). +APP_BRIDGE_TOOLS = {"AppDescribe", "AppGetState", "AppInvoke"} + +# Lean toolset for app mode: bridge tools first, then a UI-driving fallback for +# apps that don't expose the bridge. Built by name-selecting the shared defs so +# their schemas stay in one place. +_APP_FALLBACK_TOOL_NAMES = [ + "ReportProgress", "Done", + "BrowserScreenshot", "BrowserGetText", + "BrowserListInteractives", "BrowserClickIndex", "BrowserBatch", +] +_app_fallback_tools = [t for t in BROWSER_TOOLS_SCHEMA if t["name"] in _APP_FALLBACK_TOOL_NAMES] +# ReportProgress + Done lead, then the bridge tools, then UI fallback. +APP_VISIBLE_TOOLS = ( + [t for t in _app_fallback_tools if t["name"] in ("ReportProgress", "Done")] + + APP_TOOLS_SCHEMA + + [t for t in _app_fallback_tools if t["name"] not in ("ReportProgress", "Done")] +) + SYSTEM_PROMPT = ( "You are a website-agnostic browser automation agent. You can operate on ANY " "website the user is signed into; social media, dating apps, email, productivity " @@ -892,6 +966,56 @@ SYSTEM_PROMPT = ( MAX_TURNS = 40 +# App mode: drive an OpenSwarm-built app through its native bridge. This is the +# global "how to operate an app" guidance (decision: one global doc, not per-app; +# per-app specifics come live from AppDescribe). Deliberately short, the bridge +# does the heavy lifting and future models need less hand-holding. +APP_SYSTEM_PROMPT = ( + "You operate an OpenSwarm-built app (a small web app the user created, e.g. a " + "graphing tool or a form). The app is ALREADY open in front of you; do not " + "navigate anywhere.\n\n" + + "## How you see and act: the app's own bridge (this is the fast path)\n" + "The app exposes a native bridge, window.OPENSWARM_APP, with three calls you " + "reach through tools:\n" + "- AppDescribe -> the CURRENT list of actions {name, args, description}.\n" + "- AppGetState -> a small JSON snapshot of what's on screen.\n" + "- AppInvoke(name, args) -> perform one action.\n" + "Always start with AppDescribe to learn the real action names and arg shapes, " + "then AppInvoke them. This reads the app's true structure directly, so you do " + "NOT need screenshots, the DOM, or the accessibility tree.\n" + "Only ever call actions that AppDescribe actually returned. You operate the app, " + "you do NOT change it: never invent action names, and never try to add, remove, " + "or redefine the app's available actions or edit its code. If what the user wants " + "isn't reachable through the exposed actions, say so in Done.\n\n" + + "## Controls are DYNAMIC\n" + "Actions and state change as you interact (the app adds and removes controls). " + "Never cache the action list: after any AppInvoke that could change what's " + "available, call AppDescribe again before relying on it. Verify outcomes with " + "AppGetState rather than assuming.\n\n" + + "## If there is no bridge\n" + "If AppDescribe (or AppGetState) returns null, this app doesn't expose the " + "bridge. Fall back to driving the UI directly: BrowserListInteractives to find " + "controls, BrowserClickIndex / BrowserBatch to act, BrowserScreenshot to see. " + "If you truly cannot operate it, say so plainly in Done with success=false.\n\n" + + "## ReportProgress before acting\n" + "Before any AppInvoke (or UI action), call ReportProgress in the SAME turn with " + "a telegraphic next_goal and working_memory. AppDescribe and AppGetState are " + "reads and do not require it.\n\n" + + "## Speed\n" + "Fewer model turns is the #1 driver. Once AppDescribe tells you the actions, " + "fire the AppInvokes you need; don't re-describe between every step unless the " + "action list could have changed. Keep your notes a few words each.\n\n" + + "When finished, end by calling Done; put a plain one or two sentence reply in " + "its message (what you did and the proof, e.g. what's now graphed), zero " + "interface words. Set success=false if you couldn't finish." +) + # Tools that count as "action tools"; calling any of these in a turn requires # the model to also call ReportProgress in the same turn (after the first # turn). Read-only tools and meta tools are exempt. @@ -904,4 +1028,5 @@ _ACTION_TOOLS_REQUIRING_REPORT = { "BrowserEvaluate", "BrowserClickIndex", # Phase 3 "BrowserBatch", # Phase 4 + "AppInvoke", # app mode: invoking an app action mutates state } diff --git a/backend/apps/agents/browser_agent_mcp_server.py b/backend/apps/agents/browser_agent_mcp_server.py index a978fbf4..25a10606 100644 --- a/backend/apps/agents/browser_agent_mcp_server.py +++ b/backend/apps/agents/browser_agent_mcp_server.py @@ -22,6 +22,8 @@ MODEL = os.environ.get("OPENSWARM_AGENT_MODEL", "sonnet") DASHBOARD_ID = os.environ.get("OPENSWARM_DASHBOARD_ID", "") PRE_SELECTED_BROWSER_IDS = os.environ.get("OPENSWARM_PRE_SELECTED_BROWSER_IDS", "") PARENT_SESSION_ID = os.environ.get("OPENSWARM_PARENT_SESSION_ID", "") +# Apps the user selected on the dashboard; AppAgent may only target these (anti-hallucination). +SELECTED_APP_IDS = [a.strip() for a in os.environ.get("OPENSWARM_SELECTED_APP_IDS", "").split(",") if a.strip()] TOOLS = [ { @@ -112,6 +114,38 @@ TOOLS = [ "required": ["tasks"], }, }, + { + "name": "AppAgent", + "description": ( + "Operate one of the user's OpenSwarm-built apps (a small web app they " + "created, e.g. a graphing or form app) that is open on the dashboard. A " + "dedicated app agent reads the app's own actions and state through its " + "native bridge and performs the task (no screenshots or DOM scraping), " + "then returns a summary plus a final screenshot. Use this for the apps " + "listed in the selected-app context, not for websites (use BrowserAgent " + "for those)." + ), + "inputSchema": { + "type": "object", + "properties": { + "output_id": { + "type": "string", + "description": ( + "The id of the selected app to operate (from the selected-app " + "context block)." + ), + }, + "task": { + "type": "string", + "description": ( + "What to do in the app. Be specific (e.g. 'graph y=x^2 and " + "y=sin(x)')." + ), + }, + }, + "required": ["output_id", "task"], + }, + }, ] @@ -245,37 +279,54 @@ def format_batch_results(results: list[dict]) -> dict: return {"content": all_content} +def _text_error(message: str) -> dict: + return {"content": [{"type": "text", "text": message}], "isError": True} + + +def _run_single_task(task_def: dict) -> dict: + """Dispatch one task to the backend and format its single result.""" + result = call_backend([task_def]) + if "error" in result: + return _text_error(f"Error: {result['error']}") + results = result.get("results", [result]) + if results: + return format_result(results[0]) + return _text_error("No result returned.") + + def handle_tool_call(tool_name: str, arguments: dict) -> dict: if tool_name == "CreateBrowserAgent": - task_def = { + return _run_single_task({ "task": arguments.get("task", ""), "browser_id": "", "url": arguments.get("url", ""), - } - result = call_backend([task_def]) - if "error" in result: - return {"content": [{"type": "text", "text": f"Error: {result['error']}"}], "isError": True} - results = result.get("results", [result]) - if results: - return format_result(results[0]) - return {"content": [{"type": "text", "text": "No result returned."}], "isError": True} + }) elif tool_name == "BrowserAgent": browser_id = arguments.get("browser_id", "") if not browser_id: - return {"content": [{"type": "text", "text": "Error: browser_id is required"}], "isError": True} - task_def = { + return _text_error("Error: browser_id is required") + return _run_single_task({ "task": arguments.get("task", ""), "browser_id": browser_id, "url": "", - } - result = call_backend([task_def]) - if "error" in result: - return {"content": [{"type": "text", "text": f"Error: {result['error']}"}], "isError": True} - results = result.get("results", [result]) - if results: - return format_result(results[0]) - return {"content": [{"type": "text", "text": "No result returned."}], "isError": True} + }) + + elif tool_name == "AppAgent": + output_id = arguments.get("output_id", "") + if not output_id: + return _text_error("Error: output_id is required") + # Only drive apps the user actually selected (anti-hallucination), when we + # know the selection. Empty list = unknown, so don't block. + if SELECTED_APP_IDS and output_id not in SELECTED_APP_IDS: + valid = ", ".join(SELECTED_APP_IDS) or "(none)" + return _text_error(f"Error: '{output_id}' is not a selected app. Selected apps: {valid}") + return _run_single_task({ + "task": arguments.get("task", ""), + "browser_id": f"app:{output_id}", + "url": "", + "app_mode": True, + }) elif tool_name == "BrowserAgents": tasks = arguments.get("tasks", []) diff --git a/backend/apps/agents/manager/prompt/prompt_context.py b/backend/apps/agents/manager/prompt/prompt_context.py index 116ffec3..3c12f2be 100644 --- a/backend/apps/agents/manager/prompt/prompt_context.py +++ b/backend/apps/agents/manager/prompt/prompt_context.py @@ -209,6 +209,7 @@ def _build_selected_app_context(selected_app_output_ids: list[str] | None) -> st name = output.name or "Untitled App" lines = [ f'- App: "{name}"', + f" App id (for AppAgent): {output_id}", f" Workspace path: {path}", f" Entry point: {os.path.join(path, 'index.html')}", ] @@ -224,10 +225,14 @@ def _build_selected_app_context(selected_app_output_ids: list[str] | None) -> st return None return ( "\n" - "The user selected these App cards on the dashboard for you to edit. " - "They are existing web apps; edit the files in place at the paths below " - "and the dashboard preview live-reloads on save. Do not scaffold a new " - "project or write files anywhere else.\n\n" + "The user selected these App cards on the dashboard. You can do two things " + "with them:\n" + "- EDIT them: change the files in place at the paths below and the dashboard " + "preview live-reloads on save. Do not scaffold a new project or write files " + "elsewhere.\n" + "- OPERATE them: to actually use a running app (e.g. 'graph y=x^2', 'fill in " + "the form'), call AppAgent(output_id, task) with the App id below. A " + "dedicated agent drives the live app through its own actions, no editing.\n\n" + "\n\n".join(entries) + "\n" ) diff --git a/backend/apps/outputs/app_builder_skill.md b/backend/apps/outputs/app_builder_skill.md index 9b15cdf8..4fb08c9a 100644 --- a/backend/apps/outputs/app_builder_skill.md +++ b/backend/apps/outputs/app_builder_skill.md @@ -325,6 +325,61 @@ export const JOBS_LIST = '/api/jobs/list'; --- +## Make the app agent-operable — the `OPENSWARM_APP` bridge + +An agent can drive this app on the user's behalf (e.g. "graph y=x^2 on my +Desmos app"). It does NOT do that by clicking pixels or scraping the DOM — +that's slow and an app's DOM is often a bare ``. Instead, expose a +tiny bridge on `window` and the agent reads/acts through it in one fast +`executeJavaScript` call. **Always add this bridge to every app you build.** + +Set `window.OPENSWARM_APP` with three functions: + +- `describe()` → the **current** list of actions, `[{ name, args?, description? }]`. + Recompute it live on every call; controls are dynamic, so return only what's + actually available right now. +- `getState()` → a **small** JSON snapshot of the app's relevant state (used to + verify an action landed). Keep it compact — this is the latency budget. +- `invoke(name, args)` → perform the named action and return a result (or throw + a string the agent will read). + +Keep `args` shapes simple (strings, numbers, booleans, small objects). The agent +only ever calls actions that `describe()` returned; it never edits your code. + +```tsx +// Register once the app's core object exists (e.g. after the calculator mounts). +// `calc` here is the app's own API (Desmos example); use whatever yours exposes. +function registerAgentBridge(calc: any) { + (window as any).OPENSWARM_APP = { + describe() { + const actions = [ + { name: 'addExpr', args: { latex: 'string' }, description: 'Add a graph expression, e.g. y=x^2' }, + { name: 'clear', description: 'Remove all expressions' }, + ]; + // Dynamic: only offer removeExpr when something is on the graph. + if (calc.getExpressions().length > 0) { + actions.push({ name: 'removeExpr', args: { id: 'string' }, description: 'Remove one expression by id' }); + } + return actions; + }, + getState() { + return { expressions: calc.getExpressions().map((e: any) => ({ id: e.id, latex: e.latex })) }; + }, + invoke(name: string, args: any = {}) { + if (name === 'addExpr') { const id = String(Date.now()); calc.setExpression({ id, latex: args.latex }); return { id }; } + if (name === 'removeExpr') { calc.removeExpression({ id: args.id }); return { ok: true }; } + if (name === 'clear') { calc.setBlank(); return { ok: true }; } + throw `Unknown action: ${name}`; + }, + }; +} +``` + +If you skip the bridge the agent falls back to slow UI-driving, so apps meant to +be agent-operated should always register it. + +--- + ## Debugging — use `swarm_debug`, not `print()` The backend has `swarm_debug` pre-installed. It's a colored frame-aware diff --git a/backend/main.py b/backend/main.py index d51267a8..d21c9d31 100644 --- a/backend/main.py +++ b/backend/main.py @@ -529,6 +529,14 @@ async def browser_agent_run(request: Request): if not tasks: return JSONResponse({"error": "tasks array is required"}, status_code=400) + # [app-agent] step trace: log app-mode dispatches arriving at the route. + for _t in tasks: + if _t.get("app_mode") or str(_t.get("browser_id", "")).startswith("app:"): + logger.info( + f"[app-agent] ROUTE /run: browser_id={_t.get('browser_id')!r} " + f"app_mode={_t.get('app_mode')} task={str(_t.get('task',''))[:120]!r}" + ) + results = await run_browser_agents( tasks=tasks, model=model, diff --git a/backend/tests/test_app_agent.py b/backend/tests/test_app_agent.py new file mode 100644 index 00000000..eb0d734b --- /dev/null +++ b/backend/tests/test_app_agent.py @@ -0,0 +1,148 @@ +"""App agent: driving an OpenSwarm-built app through its window.OPENSWARM_APP bridge. + +Pins the wiring that makes an app drivable without touching a real webview or LLM: + - the three App bridge tools translate to a single BrowserEvaluate against the + app's bridge, + - run_browser_agents forwards app_mode + the "app:" target without creating + or navigating a browser card, + - the AppAgent delegation tool builds the right task and gates on the user's + selected apps, + - the orchestrator's selected-app context advertises AppAgent. +""" + +import asyncio +import json + +from backend.apps.agents.browser import browser_agent as BA + + +# --- bridge expression ------------------------------------------------------- +def test_app_bridge_expression_describe_and_state(): + desc = BA._app_bridge_expression("AppDescribe", {}) + assert "window.OPENSWARM_APP.describe()" in desc + assert "JSON.stringify" in desc # round-trips as text + state = BA._app_bridge_expression("AppGetState", {}) + assert "window.OPENSWARM_APP.getState()" in state + + +def test_app_bridge_expression_invoke_serializes_args(): + expr = BA._app_bridge_expression("AppInvoke", {"name": "addExpr", "args": {"latex": "y=x^2"}}) + # name + args are JSON-encoded into the call + assert 'window.OPENSWARM_APP.invoke("addExpr", {"latex": "y=x^2"})' in expr + # missing bridge is handled inside the expression, never throws + assert "typeof A.describe!=='function'" in expr + + +def test_app_bridge_expression_invoke_defaults_args_to_empty_object(): + expr = BA._app_bridge_expression("AppInvoke", {"name": "clear"}) + assert 'window.OPENSWARM_APP.invoke("clear", {})' in expr + + +# --- execute_browser_tool routing ------------------------------------------- +def test_execute_browser_tool_app_bridge_routes_to_evaluate(monkeypatch): + captured = {} + + async def _send(request_id, action, browser_id, params, tab_id=""): + captured.update(action=action, browser_id=browser_id, params=params) + return {"text": json.dumps([{"name": "addExpr"}])} + + monkeypatch.setattr(BA.ws_manager, "send_browser_command", _send, raising=False) + out = asyncio.run(BA.execute_browser_tool( + "AppInvoke", {"name": "addExpr", "args": {"latex": "y=x^2"}}, "app:abc", + )) + assert captured["action"] == "evaluate" + assert captured["browser_id"] == "app:abc" + assert "window.OPENSWARM_APP.invoke" in captured["params"]["expression"] + assert out["text"] # result passes straight through + + +# --- run_browser_agents app_mode wiring ------------------------------------- +def test_run_browser_agents_app_mode_forwards_flag_no_card(monkeypatch): + recorded = {} + + async def _fake_run_browser_agent(**kwargs): + recorded.update(kwargs) + return {"summary": "done", "action_log": [], "final_screenshot": None} + + async def _boom(*a, **k): + raise AssertionError("app mode must not create a browser card") + + monkeypatch.setattr(BA, "run_browser_agent", _fake_run_browser_agent, raising=True) + monkeypatch.setattr(BA, "_create_browser_card", _boom, raising=True) + # a connected dashboard so dispatch isn't refused + monkeypatch.setattr(BA.ws_manager, "global_connections", [object()], raising=False) + + results = asyncio.run(BA.run_browser_agents( + tasks=[{"task": "graph y=x^2", "browser_id": "app:abc", "app_mode": True}], + model="sonnet", + dashboard_id="dash-1", + )) + + assert results and results[0]["summary"] == "done" + assert recorded["app_mode"] is True + assert recorded["browser_id"] == "app:abc" + assert recorded["initial_url"] is None # app already loaded; never navigate + + +# --- AppAgent delegation tool ----------------------------------------------- +def _load_mcp_server(monkeypatch, selected): + import backend.apps.agents.browser_agent_mcp_server as srv + monkeypatch.setattr(srv, "SELECTED_APP_IDS", list(selected), raising=False) + return srv + + +def test_app_agent_tool_builds_app_task(monkeypatch): + srv = _load_mcp_server(monkeypatch, ["abc"]) + captured = {} + + def _call_backend(tasks): + captured["tasks"] = tasks + return {"results": [{"summary": "graphed it", "action_log": []}]} + + monkeypatch.setattr(srv, "call_backend", _call_backend, raising=True) + res = srv.handle_tool_call("AppAgent", {"output_id": "abc", "task": "graph y=x^2"}) + + assert not res.get("isError") + task_def = captured["tasks"][0] + assert task_def["browser_id"] == "app:abc" + assert task_def["app_mode"] is True + assert task_def["task"] == "graph y=x^2" + + +def test_app_agent_tool_rejects_unselected_app(monkeypatch): + srv = _load_mcp_server(monkeypatch, ["abc"]) + + def _call_backend(tasks): + raise AssertionError("must not dispatch an unselected app") + + monkeypatch.setattr(srv, "call_backend", _call_backend, raising=True) + res = srv.handle_tool_call("AppAgent", {"output_id": "zzz", "task": "graph"}) + assert res.get("isError") + assert "not a selected app" in res["content"][0]["text"] + + +def test_app_agent_tool_requires_output_id(monkeypatch): + srv = _load_mcp_server(monkeypatch, []) + res = srv.handle_tool_call("AppAgent", {"task": "graph"}) + assert res.get("isError") + assert "output_id is required" in res["content"][0]["text"] + + +# --- orchestrator context advertises AppAgent -------------------------------- +def test_selected_app_context_advertises_app_agent(monkeypatch): + import os + from backend.apps.agents.manager.prompt import prompt_context as pc + import backend.apps.outputs.workspace_io as wio + + class _Out: + workspace_id = "ws-1" + name = "Grapher" + + monkeypatch.setattr(wio, "load_output", lambda oid: _Out(), raising=True) + monkeypatch.setattr(os.path, "isdir", lambda p: True, raising=True) + monkeypatch.setattr(os.path, "isfile", lambda p: False, raising=True) + + ctx = pc._build_selected_app_context(["abc"]) + assert ctx is not None + assert "App id (for AppAgent): abc" in ctx + assert "AppAgent(output_id, task)" in ctx diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 8e97f035..92d511d7 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -84,7 +84,6 @@ "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -1965,7 +1964,6 @@ "resolved": "https://registry.npmjs.org/@emotion/react/-/react-11.14.0.tgz", "integrity": "sha512-O000MLDBDdk/EohJPFUqvnp4qnHeYkVP5B0xEG0D/L7cOKP9kefu2DXn8dj74cQfsEzUqh+sr1RzFqiL1o+PpA==", "license": "MIT", - "peer": true, "dependencies": { "@babel/runtime": "^7.18.3", "@emotion/babel-plugin": "^11.13.5", @@ -2009,7 +2007,6 @@ "resolved": "https://registry.npmjs.org/@emotion/styled/-/styled-11.14.1.tgz", "integrity": "sha512-qEEJt42DuToa3gurlH4Qqc1kVpNq8wO8cJtDzU46TjlzWjDlsVyevtYCRijVq3SrHsROS+gVQ8Fnea108GnKzw==", "license": "MIT", - "peer": true, "dependencies": { "@babel/runtime": "^7.18.3", "@emotion/babel-plugin": "^11.13.5", @@ -2245,7 +2242,6 @@ "resolved": "https://registry.npmjs.org/@mui/material/-/material-7.3.10.tgz", "integrity": "sha512-cHvGOk2ZEfbQt3LnGe0ZKd/ETs9gsUpkW66DCO+GSjMZhpdKU4XsuIr7zJ/B/2XaN8ihxuzHfYAR4zPtCN4RYg==", "license": "MIT", - "peer": true, "dependencies": { "@babel/runtime": "^7.28.6", "@mui/core-downloads-tracker": "^7.3.10", @@ -3378,7 +3374,6 @@ "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.28.tgz", "integrity": "sha512-z9VXpC7MWrhfWipitjNdgCauoMLRdIILQsAEV+ZesIzBq/oUlxk0m3ApZuMFCXdnS4U7KrI+l3WRUEGQ8K1QKw==", "license": "MIT", - "peer": true, "dependencies": { "@types/prop-types": "*", "csstype": "^3.2.2" @@ -3775,7 +3770,6 @@ "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -3815,7 +3809,6 @@ "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -4144,7 +4137,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.10.12", "caniuse-lite": "^1.0.30001782", @@ -8192,7 +8184,6 @@ "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -8249,7 +8240,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -8468,7 +8458,6 @@ "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", "license": "MIT", - "peer": true, "dependencies": { "loose-envify": "^1.1.0" }, @@ -8481,7 +8470,6 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", "license": "MIT", - "peer": true, "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" @@ -8528,7 +8516,6 @@ "resolved": "https://registry.npmjs.org/react-redux/-/react-redux-9.2.0.tgz", "integrity": "sha512-ROY9fvHhwOD9ySfrF0wmvu//bKCQ6AeZZq1nJNtbDC+kk5DuSuNX/n6YWYF/SYy7bSba4D4FSz8DJeKY/S/r+g==", "license": "MIT", - "peer": true, "dependencies": { "@types/use-sync-external-store": "^0.0.6", "use-sync-external-store": "^1.4.0" @@ -8667,8 +8654,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/redux/-/redux-5.0.1.tgz", "integrity": "sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/redux-thunk": { "version": "3.1.0", @@ -8980,7 +8966,6 @@ "integrity": "sha512-kgW13M54DUB7IsIRM5LvJkNlpH+WhMpooUcaWGFARkF1Tc82v9mIWkCbCYf+MBvpIUBSeSOTilpZjEPr2VYE6Q==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "chokidar": "^4.0.0", "immutable": "^5.1.5", @@ -10084,7 +10069,6 @@ "integrity": "sha512-wGN3qcrBQIFmQ/c0AiOAQBvrZ5lmY8vbbMv4Mxfgzqd/B6+9pXtLo73WuS1dSGXM5QYY3hZnIbvx+K1xxe6FyA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/eslint-scope": "^3.7.7", "@types/estree": "^1.0.8", @@ -10133,7 +10117,6 @@ "integrity": "sha512-pIDJHIEI9LR0yxHXQ+Qh95k2EvXpWzZ5l+d+jIo+RdSm9MiHfzazIxwwni/p7+x4eJZuvG1AJwgC4TNQ7NRgsg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@discoveryjs/json-ext": "^0.5.0", "@webpack-cli/configtest": "^2.1.1", diff --git a/frontend/src/app/pages/Dashboard/cards/DashboardViewCard.tsx b/frontend/src/app/pages/Dashboard/cards/DashboardViewCard.tsx index 034b4341..07889307 100644 --- a/frontend/src/app/pages/Dashboard/cards/DashboardViewCard.tsx +++ b/frontend/src/app/pages/Dashboard/cards/DashboardViewCard.tsx @@ -587,6 +587,7 @@ const DashboardOutputPreview: React.FC<{ frontendCode={output.files?.['index.html'] ?? ''} inputData={inputData} backendResult={backendResult} + agentBrowserId={`app:${output.id}`} /> ); }; diff --git a/frontend/src/app/pages/Views/ViewPreview.tsx b/frontend/src/app/pages/Views/ViewPreview.tsx index 59e31b6b..ae6801c0 100644 --- a/frontend/src/app/pages/Views/ViewPreview.tsx +++ b/frontend/src/app/pages/Views/ViewPreview.tsx @@ -6,6 +6,7 @@ import { useElementSelection } from '@/app/components/editor/ElementSelectionCon import { useIframeElementSelector } from './useIframeElementSelector'; import { getAuthToken, ensureAuthToken } from '@/shared/config'; import { useClaudeTokens } from '@/shared/styles/ThemeContext'; +import { registerWebview, unregisterWebview, setActiveTab, type BrowserWebview } from '@/shared/browserRegistry'; // In Electron use to escape iframe restrictions (popups, mic/camera, WebAuthn, cookied fetch); outside Electron fall back to iframe. const isElectron = navigator.userAgent.includes('Electron'); @@ -55,8 +56,13 @@ interface Props { onConsoleMessage?: (level: string, text: string) => void; /** Fires once the embedded app has actually painted, so cold-start placeholders don't unmount during the vite-ready to first-paint gap. */ onContentLoad?: () => void; + /** When set (webview mode only), registers this preview's webview in browserRegistry under this id so an app agent can drive it via the browser command pipeline. */ + agentBrowserId?: string; } +// Single tab per app preview; the browser command pipeline keys on browserId:tabId. +const APP_TAB_ID = 'main'; + function buildSrcdoc( frontendCode: string, inputData: Record, @@ -92,6 +98,7 @@ const ViewPreview = forwardRef(({ style, onConsoleMessage, onContentLoad, + agentBrowserId, }, ref) => { const iframeRef = useRef(null); const webviewRef = useRef(null); @@ -280,6 +287,23 @@ const ViewPreview = forwardRef(({ }; }, [useWebview, handleNavigationLoad]); + // Register the live webview so an app agent can drive it through the same + // browser command pipeline that drives browser cards. Webview-only: the iframe + // fallback has no executeJavaScript channel from the host. + useEffect(() => { + if (!useWebview || !agentBrowserId) return; + const wv = webviewRef.current as BrowserWebview | null; + if (!wv) return; + registerWebview(agentBrowserId, APP_TAB_ID, wv); + setActiveTab(agentBrowserId, APP_TAB_ID); + return () => { + try { unregisterWebview(agentBrowserId, APP_TAB_ID); } catch (_e) {} + }; + // Keyed on useWebview (flips true when the webview mounts), not iframeSrc: + // the webview element has a stable key and survives src/data changes, so + // re-registering on every data update would just thrash the registry. + }, [useWebview, agentBrowserId]); + const hasContent = !!(serveUrl || frontendCode?.trim()); if (!hasContent) { diff --git a/frontend/src/shared/browserCommandHandler.ts b/frontend/src/shared/browserCommandHandler.ts index 45e0d379..74436048 100644 --- a/frontend/src/shared/browserCommandHandler.ts +++ b/frontend/src/shared/browserCommandHandler.ts @@ -1,4 +1,4 @@ -import { getWebview, type BrowserWebview } from './browserRegistry'; +import { getWebview, registeredKeys, type BrowserWebview } from './browserRegistry'; import { store } from './state/store'; import { resumeBrowserCard } from './state/dashboardLayoutSlice'; import { dashboardWs } from './ws/WebSocketManager'; @@ -1330,8 +1330,13 @@ async function handleReplayRoute(wv: BrowserWebview, params: Record async function handleEvaluate(wv: BrowserWebview, params: Record): Promise> { const expression = params.expression as string; if (!expression) return { error: 'expression parameter is required' }; + // [app-agent] step trace: surface what the app's bridge actually returned. + const _bridgeCall = expression.includes('OPENSWARM_APP'); try { const result = await wv.executeJavaScript(expression); + if (_bridgeCall) { + console.log(`[app-agent] BRIDGE eval -> ${typeof result === 'string' ? result : JSON.stringify(result)}`); + } const text = typeof result === 'string' ? result : JSON.stringify(result, null, 2); // evaluate is the agent's main read path; sample routes here too (XHRs have // fired by now) so the backend can surface the fast network tier once. @@ -1397,14 +1402,27 @@ async function runBrowserCommand( request_id: string, action: string, browser_id: string, tab_id: string | undefined, params: Record, ) { + // [app-agent] step trace: only for app targets so browser-agent runs stay quiet. + const _appTrace = String(browser_id || '').startsWith('app:'); + if (_appTrace) { + console.log(`[app-agent] CMD recv: action=${action} browser_id=${browser_id} tab_id=${tab_id ?? ''} registered=[${registeredKeys().join(', ')}]`); + } const wv = await awaitWebview(browser_id, tab_id || undefined); if (!wv) { + if (_appTrace) { + console.warn(`[app-agent] LOOKUP MISS: no webview for '${browser_id}' (registered keys: [${registeredKeys().join(', ')}]) -> the app card isn't mounted on the active dashboard`); + } dashboardWs.send('browser:result', { request_id, error: `Browser card '${browser_id}'${tab_id ? ` tab '${tab_id}'` : ''} not found or not an Electron webview`, }); return; } + if (_appTrace) { + let _url = ''; + try { _url = wv.getURL(); } catch (_e) {} + console.log(`[app-agent] LOOKUP HIT: webview for '${browser_id}' found, url=${_url} loading=${(() => { try { return wv.isLoading(); } catch { return '?'; } })()}`); + } const detail = params.url || params.selector || params.expression || undefined; setActivity(browser_id, { action: action as BrowserAction, detail }); diff --git a/frontend/src/shared/browserRegistry.ts b/frontend/src/shared/browserRegistry.ts index 8a03e481..536e32d5 100644 --- a/frontend/src/shared/browserRegistry.ts +++ b/frontend/src/shared/browserRegistry.ts @@ -51,6 +51,11 @@ export function setActiveTab(browserId: string, tabId: string): void { activeTabMap.set(browserId, tabId); } +// [app-agent] diagnostic: list currently-registered keys ("browserId:tabId"). +export function registeredKeys(): string[] { + return Array.from(registry.keys()); +} + export function getWebview(browserId: string, tabId?: string): BrowserWebview | undefined { const resolvedTabId = tabId || activeTabMap.get(browserId); if (!resolvedTabId) return undefined;