implemented app-describe for agent use: exposing the command tree of an app

2026-09-13 13:17:40 +02:00 · 2026-06-13 01:01:11 -07:00
parent a680df8817
commit b498ed3097
13 changed files with 569 additions and 75 deletions
@@ -1142,7 +1142,7 @@ class AgentManager:
            # dispatch layer (see _build_mcp_servers docstring).
            mcp_servers = await self._build_mcp_servers(session.allowed_tools, session.active_mcps)

-            _browser_delegation_tools = ["CreateBrowserAgent", "BrowserAgent", "BrowserAgents"]
+            _browser_delegation_tools = ["CreateBrowserAgent", "BrowserAgent", "BrowserAgents", "AppAgent"]
            _browser_all_denied = all(
                _builtin_perms.get(t, "always_allow") == "deny"
                for t in _browser_delegation_tools
@@ -1165,6 +1165,8 @@ class AgentManager:
                        "OPENSWARM_AGENT_MODEL": session.model,
                        "OPENSWARM_DASHBOARD_ID": session.dashboard_id or "",
                        "OPENSWARM_PRE_SELECTED_BROWSER_IDS": ",".join(pre_selected_bids),
+                        # Apps the user selected this turn; AppAgent may only target these.
+                        "OPENSWARM_SELECTED_APP_IDS": ",".join(selected_app_output_ids or []),
                        "OPENSWARM_PARENT_SESSION_ID": session.id,
                    },
                    "type": "stdio",
@@ -71,6 +71,9 @@ from backend.apps.agents.browser import browser_schema
 from backend.apps.agents.browser.browser_schema import (
    _ACTION_TOOLS_REQUIRING_REPORT,
    ACTION_MAP,
+    APP_BRIDGE_TOOLS,
+    APP_SYSTEM_PROMPT,
+    APP_VISIBLE_TOOLS,
    BROWSER_TOOLS_SCHEMA,
    MAX_TURNS,
    MODEL_MAP,
@@ -90,19 +93,64 @@ _CONFIRM_TOOLS = {
 }


+def _app_bridge_expression(tool_name: str, tool_input: dict) -> str:
+    """JS for an app bridge tool. Each expression returns a JSON STRING (so it
+    round-trips as text) and never throws; bridge errors come back as JSON."""
+    if tool_name == "AppDescribe":
+        call = "window.OPENSWARM_APP.describe()"
+    elif tool_name == "AppGetState":
+        call = "window.OPENSWARM_APP.getState()"
+    else:  # AppInvoke
+        name = json.dumps(tool_input.get("name", ""))
+        args = json.dumps(tool_input.get("args") or {})
+        call = f"window.OPENSWARM_APP.invoke({name}, {args})"
+    return (
+        "(function(){try{"
+        "var A=window.OPENSWARM_APP;"
+        "if(!A||typeof A.describe!=='function'){return JSON.stringify(null);}"
+        f"var r={call};"
+        "return JSON.stringify(r===undefined?null:r);"
+        "}catch(e){return JSON.stringify({__error__:String((e&&e.message)||e)});}})()"
+    )
+
+
 async def execute_browser_tool(
    tool_name: str, tool_input: dict, browser_id: str, tab_id: str = "",
 ) -> dict:
    """Execute a browser tool via ws_manager directly (no MCP/HTTP round-trip)."""
+    # [app-agent] step trace: only for app targets / bridge tools so normal
+    # browser-agent runs stay quiet. Greppable prefix; remove when done.
+    _trace = browser_id.startswith("app:") or tool_name in APP_BRIDGE_TOOLS
+
+    # App bridge tools translate to a single BrowserEvaluate against the app's
+    # window.OPENSWARM_APP, so they need no frontend command-handler changes.
+    if tool_name in APP_BRIDGE_TOOLS:
+        action = "evaluate"
+        expr = _app_bridge_expression(tool_name, tool_input)
+        params = {"expression": expr}
+        request_id = uuid4().hex
+        if _trace:
+            logger.info(f"[app-agent] DISPATCH {tool_name} -> {browser_id} (req {request_id[:8]}) js={expr[:120]}")
+        result = await ws_manager.send_browser_command(
+            request_id, action, browser_id, params, tab_id=tab_id,
+        )
+        if _trace:
+            logger.info(f"[app-agent] RESULT   {tool_name} <- {browser_id}: {json.dumps(result)[:300]}")
+        return result
+
    action = ACTION_MAP.get(tool_name)
    if not action:
        return {"error": f"Unknown browser tool: {tool_name}"}

    params = {k: v for k, v in tool_input.items()}
    request_id = uuid4().hex
+    if _trace:
+        logger.info(f"[app-agent] DISPATCH {tool_name}/{action} -> {browser_id} (req {request_id[:8]})")
    result = await ws_manager.send_browser_command(
        request_id, action, browser_id, params, tab_id=tab_id,
    )
+    if _trace:
+        logger.info(f"[app-agent] RESULT   {tool_name} <- {browser_id}: {json.dumps(result)[:300]}")
    return result


@@ -352,27 +400,36 @@ async def run_browser_agent(
    pre_selected: bool = False,
    initial_url: str | None = None,
    parent_session_id: str | None = None,
+    app_mode: bool = False,
 ) -> dict:
    """Run a browser sub-agent loop for a single browser card.

    Creates a visible AgentSession, streams progress via WebSocket,
    and returns the full action log + summary + final screenshot.
+
+    When app_mode is set, browser_id points at an OpenSwarm-built app's webview
+    (registered as "app:<output_id>"). The agent drives it through the app's
+    native bridge (window.OPENSWARM_APP) instead of web perception: no initial
+    navigate, no AX-tree front-load, and a lean app toolset + prompt.
    """
    from backend.apps.agents.agent_manager import agent_manager

    _browser_perms = load_builtin_permissions()

+    if app_mode:
+        logger.info(f"[app-agent] START loop: browser_id={browser_id} task={task[:140]!r}")
+
    session_id = uuid4().hex
    cancel_event = asyncio.Event()
    session = AgentSession(
        id=session_id,
-        name=f"Browser Agent",
+        name="App Agent" if app_mode else "Browser Agent",
        model=model,
        mode="browser-agent",
        status="running",
        dashboard_id=dashboard_id,
        browser_id=browser_id,
-        system_prompt=SYSTEM_PROMPT,
+        system_prompt=APP_SYSTEM_PROMPT if app_mode else SYSTEM_PROMPT,
        parent_session_id=parent_session_id,
    )
    session._cancel_event = cancel_event
@@ -431,16 +488,20 @@ async def run_browser_agent(
    current_url = ""
    preloaded_reads: list[dict] = []  # real front-loaded reads, seeded into action_log
    _resumed = bool(browser_history._browser_history.get(browser_id))
-    if initial_url:
-        nav_result = await execute_browser_tool(
-            "BrowserNavigate", {"url": initial_url}, browser_id, tab_id,
-        )
-        logger.info(f"Browser agent {session_id}: navigated to {initial_url}: {nav_result.get('text', nav_result.get('error', ''))}")
-        preloaded_perception, current_url, preloaded_reads = await _perceive(initial_url)
-    elif not _resumed:
-        # Fresh task on an existing card: perceive the current page to learn its
-        # host (for replay) and front-load turn 1 (this path used to start cold).
-        preloaded_perception, current_url, preloaded_reads = await _perceive("")
+    # App mode skips this whole block: the app is already loaded and its DOM is
+    # uninformative (often a bare <canvas>), so the agent perceives via the bridge
+    # (AppDescribe) on turn 1 instead of navigating or front-loading the AX tree.
+    if not app_mode:
+        if initial_url:
+            nav_result = await execute_browser_tool(
+                "BrowserNavigate", {"url": initial_url}, browser_id, tab_id,
+            )
+            logger.info(f"Browser agent {session_id}: navigated to {initial_url}: {nav_result.get('text', nav_result.get('error', ''))}")
+            preloaded_perception, current_url, preloaded_reads = await _perceive(initial_url)
+        elif not _resumed:
+            # Fresh task on an existing card: perceive the current page to learn its
+            # host (for replay) and front-load turn 1 (this path used to start cold).
+            preloaded_perception, current_url, preloaded_reads = await _perceive("")

    from backend.apps.settings.settings import load_settings
    from backend.apps.settings.credentials import get_anthropic_client_for_model
@@ -594,8 +655,8 @@ async def run_browser_agent(
    # Advisory per-domain hints: seed the system prompt with what a prior agent
    # learned about this domain (if we know the domain at start), and keep the
    # store fresh from each ReportProgress. Re-verify, never blindly trust.
-    start_domain = _extract_domain(initial_url) if initial_url else None
-    run_system_prompt = SYSTEM_PROMPT
+    start_domain = None if app_mode else (_extract_domain(initial_url) if initial_url else None)
+    run_system_prompt = APP_SYSTEM_PROMPT if app_mode else SYSTEM_PROMPT
    if start_domain:
        prior_note = browser_history.get_domain_note(start_domain)
        if prior_note:
@@ -611,20 +672,23 @@ async def run_browser_agent(
    # from past successful runs) so the model skips re-discovery. Advisory text,
    # re-verified by the agent, never auto-run. Keyed by full host like skills.
    pb_seeded = False  # whether tier-2 strategy was injected, for measuring its effect
-    _pb_host = browser_skills.host_of(initial_url or current_url or "")
-    if _pb_host:
-        _pb_block = browser_playbook.format_for_prompt(_pb_host)
-        if _pb_block:
-            run_system_prompt = run_system_prompt + _pb_block
-            pb_seeded = True
-    # Tier-3 memory: the cross-site priors learned on EVERY other site, injected on
-    # every run (host-agnostic) so a brand-new site isn't fully cold. Advisory, capped.
-    try:
-        _meta_block = browser_meta_playbook.format_for_prompt()
-        if _meta_block:
-            run_system_prompt = run_system_prompt + _meta_block
-    except Exception:
-        pass
+    # Playbooks are web-only (keyed by host); app mode has no host, the bridge is
+    # the whole strategy, so skip both tiers.
+    if not app_mode:
+        _pb_host = browser_skills.host_of(initial_url or current_url or "")
+        if _pb_host:
+            _pb_block = browser_playbook.format_for_prompt(_pb_host)
+            if _pb_block:
+                run_system_prompt = run_system_prompt + _pb_block
+                pb_seeded = True
+        # Tier-3 memory: the cross-site priors learned on EVERY other site, injected
+        # on every run (host-agnostic) so a brand-new site isn't fully cold.
+        try:
+            _meta_block = browser_meta_playbook.format_for_prompt()
+            if _meta_block:
+                run_system_prompt = run_system_prompt + _meta_block
+        except Exception:
+            pass

    # Prompt-caching shapes built once: system as a single cached text block,
    # and the last tool carrying the cache_control marker (Anthropic keys on the
@@ -633,7 +697,7 @@ async def run_browser_agent(
        "type": "text", "text": run_system_prompt,
        "cache_control": {"type": "ephemeral"},
    }]
-    _cached_tools = [dict(t) for t in browser_schema.MODEL_VISIBLE_TOOLS]
+    _cached_tools = [dict(t) for t in (APP_VISIBLE_TOOLS if app_mode else browser_schema.MODEL_VISIBLE_TOOLS)]
    if _cached_tools:
        _cached_tools[-1] = {**_cached_tools[-1], "cache_control": {"type": "ephemeral"}}

@@ -2277,6 +2341,9 @@ async def run_browser_agents(
        browser_id = task_def.get("browser_id", "")
        task_text = task_def.get("task", "")
        url = task_def.get("url", "")
+        # App mode: browser_id is a pre-registered app webview ("app:<output_id>");
+        # never create/reuse a browser card, never navigate (the app is loaded).
+        app_mode = bool(task_def.get("app_mode"))
        # advisory deep entry (from the fast-path brief): a NEW card opens on it
        # directly (no google detour); a REUSED card is never moved by it, so a
        # warm card's deeper page state always wins
@@ -2306,7 +2373,7 @@ async def run_browser_agents(
                        pass
            else:
                await asyncio.sleep(2.0)
-        elif browser_id:
+        elif browser_id and not app_mode:
            _active_agent_cards.add(browser_id)

        is_pre_selected = browser_id in pre_selected
@@ -2318,11 +2385,13 @@ async def run_browser_agents(
                model=model,
                dashboard_id=dashboard_id,
                pre_selected=is_pre_selected,
-                initial_url=_nav_url if _nav_url and browser_id not in pre_selected else None,
+                initial_url=None if app_mode else (_nav_url if _nav_url and browser_id not in pre_selected else None),
                parent_session_id=parent_session_id,
+                app_mode=app_mode,
            )
        finally:
-            _active_agent_cards.discard(browser_id)
+            if not app_mode:
+                _active_agent_cards.discard(browser_id)

    results = await asyncio.gather(*[_run_one(t) for t in tasks], return_exceptions=True)

@@ -668,6 +668,80 @@ ACTION_MAP = {
    "BrowserClickByName": "click_by_name",
 }

+# --- App agent: driving an OpenSwarm-built app via its native bridge ---------
+# Apps expose window.OPENSWARM_APP = { describe(), getState(), invoke(name,args) }.
+# The agent reads structure/state and acts through that bridge in single
+# executeJavaScript round-trips, no screenshots or accessibility tree. These
+# three tools are translated to BrowserEvaluate in execute_browser_tool, so they
+# need no frontend command-handler changes.
+APP_TOOLS_SCHEMA = [
+    {
+        "name": "AppDescribe",
+        "description": (
+            "Read the app's CURRENT list of actions you can take, straight from the "
+            "app itself (window.OPENSWARM_APP.describe()). Returns an array of "
+            "{name, args, description}. The app's controls are DYNAMIC, they appear "
+            "and disappear as state changes, so call this again after any AppInvoke "
+            "that could add or remove actions; never assume the list is stable. "
+            "Returns null if the app does not expose the bridge (then fall back to "
+            "BrowserListInteractives/BrowserScreenshot)."
+        ),
+        "input_schema": {"type": "object", "properties": {}, "required": []},
+    },
+    {
+        "name": "AppGetState",
+        "description": (
+            "Read a small JSON snapshot of the app's current state "
+            "(window.OPENSWARM_APP.getState()). Use it to check what's on screen "
+            "and to verify an action landed. Returns null if the bridge is absent."
+        ),
+        "input_schema": {"type": "object", "properties": {}, "required": []},
+    },
+    {
+        "name": "AppInvoke",
+        "description": (
+            "Perform one app action by name with arguments "
+            "(window.OPENSWARM_APP.invoke(name, args)). The name MUST be one that "
+            "AppDescribe just returned; never invent or modify actions, only invoke "
+            "the ones the app exposes. Returns the action's result, or an "
+            "{__error__} if it threw. After invoking, re-AppDescribe if the action "
+            "may have changed which controls exist."
+        ),
+        "input_schema": {
+            "type": "object",
+            "properties": {
+                "name": {"type": "string", "description": "Action name from AppDescribe."},
+                "args": {
+                    "type": "object",
+                    "description": "Arguments object for the action; {} if none.",
+                },
+            },
+            "required": ["name"],
+        },
+    },
+]
+
+# Bridge tool names -> the JS the app exposes. execute_browser_tool maps these to
+# a BrowserEvaluate. Each expression returns a JSON string (so it round-trips as
+# text) and never throws (errors are returned as JSON).
+APP_BRIDGE_TOOLS = {"AppDescribe", "AppGetState", "AppInvoke"}
+
+# Lean toolset for app mode: bridge tools first, then a UI-driving fallback for
+# apps that don't expose the bridge. Built by name-selecting the shared defs so
+# their schemas stay in one place.
+_APP_FALLBACK_TOOL_NAMES = [
+    "ReportProgress", "Done",
+    "BrowserScreenshot", "BrowserGetText",
+    "BrowserListInteractives", "BrowserClickIndex", "BrowserBatch",
+]
+_app_fallback_tools = [t for t in BROWSER_TOOLS_SCHEMA if t["name"] in _APP_FALLBACK_TOOL_NAMES]
+# ReportProgress + Done lead, then the bridge tools, then UI fallback.
+APP_VISIBLE_TOOLS = (
+    [t for t in _app_fallback_tools if t["name"] in ("ReportProgress", "Done")]
+    + APP_TOOLS_SCHEMA
+    + [t for t in _app_fallback_tools if t["name"] not in ("ReportProgress", "Done")]
+)
+
 SYSTEM_PROMPT = (
    "You are a website-agnostic browser automation agent. You can operate on ANY "
    "website the user is signed into; social media, dating apps, email, productivity "
@@ -892,6 +966,56 @@ SYSTEM_PROMPT = (

 MAX_TURNS = 40

+# App mode: drive an OpenSwarm-built app through its native bridge. This is the
+# global "how to operate an app" guidance (decision: one global doc, not per-app;
+# per-app specifics come live from AppDescribe). Deliberately short, the bridge
+# does the heavy lifting and future models need less hand-holding.
+APP_SYSTEM_PROMPT = (
+    "You operate an OpenSwarm-built app (a small web app the user created, e.g. a "
+    "graphing tool or a form). The app is ALREADY open in front of you; do not "
+    "navigate anywhere.\n\n"
+
+    "## How you see and act: the app's own bridge (this is the fast path)\n"
+    "The app exposes a native bridge, window.OPENSWARM_APP, with three calls you "
+    "reach through tools:\n"
+    "- AppDescribe -> the CURRENT list of actions {name, args, description}.\n"
+    "- AppGetState -> a small JSON snapshot of what's on screen.\n"
+    "- AppInvoke(name, args) -> perform one action.\n"
+    "Always start with AppDescribe to learn the real action names and arg shapes, "
+    "then AppInvoke them. This reads the app's true structure directly, so you do "
+    "NOT need screenshots, the DOM, or the accessibility tree.\n"
+    "Only ever call actions that AppDescribe actually returned. You operate the app, "
+    "you do NOT change it: never invent action names, and never try to add, remove, "
+    "or redefine the app's available actions or edit its code. If what the user wants "
+    "isn't reachable through the exposed actions, say so in Done.\n\n"
+
+    "## Controls are DYNAMIC\n"
+    "Actions and state change as you interact (the app adds and removes controls). "
+    "Never cache the action list: after any AppInvoke that could change what's "
+    "available, call AppDescribe again before relying on it. Verify outcomes with "
+    "AppGetState rather than assuming.\n\n"
+
+    "## If there is no bridge\n"
+    "If AppDescribe (or AppGetState) returns null, this app doesn't expose the "
+    "bridge. Fall back to driving the UI directly: BrowserListInteractives to find "
+    "controls, BrowserClickIndex / BrowserBatch to act, BrowserScreenshot to see. "
+    "If you truly cannot operate it, say so plainly in Done with success=false.\n\n"
+
+    "## ReportProgress before acting\n"
+    "Before any AppInvoke (or UI action), call ReportProgress in the SAME turn with "
+    "a telegraphic next_goal and working_memory. AppDescribe and AppGetState are "
+    "reads and do not require it.\n\n"
+
+    "## Speed\n"
+    "Fewer model turns is the #1 driver. Once AppDescribe tells you the actions, "
+    "fire the AppInvokes you need; don't re-describe between every step unless the "
+    "action list could have changed. Keep your notes a few words each.\n\n"
+
+    "When finished, end by calling Done; put a plain one or two sentence reply in "
+    "its message (what you did and the proof, e.g. what's now graphed), zero "
+    "interface words. Set success=false if you couldn't finish."
+)
+
 # Tools that count as "action tools"; calling any of these in a turn requires
 # the model to also call ReportProgress in the same turn (after the first
 # turn). Read-only tools and meta tools are exempt.
@@ -904,4 +1028,5 @@ _ACTION_TOOLS_REQUIRING_REPORT = {
    "BrowserEvaluate",
    "BrowserClickIndex",  # Phase 3
    "BrowserBatch",  # Phase 4
+    "AppInvoke",  # app mode: invoking an app action mutates state
 }
@@ -22,6 +22,8 @@ MODEL = os.environ.get("OPENSWARM_AGENT_MODEL", "sonnet")
 DASHBOARD_ID = os.environ.get("OPENSWARM_DASHBOARD_ID", "")
 PRE_SELECTED_BROWSER_IDS = os.environ.get("OPENSWARM_PRE_SELECTED_BROWSER_IDS", "")
 PARENT_SESSION_ID = os.environ.get("OPENSWARM_PARENT_SESSION_ID", "")
+# Apps the user selected on the dashboard; AppAgent may only target these (anti-hallucination).
+SELECTED_APP_IDS = [a.strip() for a in os.environ.get("OPENSWARM_SELECTED_APP_IDS", "").split(",") if a.strip()]

 TOOLS = [
    {
@@ -112,6 +114,38 @@ TOOLS = [
            "required": ["tasks"],
        },
    },
+    {
+        "name": "AppAgent",
+        "description": (
+            "Operate one of the user's OpenSwarm-built apps (a small web app they "
+            "created, e.g. a graphing or form app) that is open on the dashboard. A "
+            "dedicated app agent reads the app's own actions and state through its "
+            "native bridge and performs the task (no screenshots or DOM scraping), "
+            "then returns a summary plus a final screenshot. Use this for the apps "
+            "listed in the selected-app context, not for websites (use BrowserAgent "
+            "for those)."
+        ),
+        "inputSchema": {
+            "type": "object",
+            "properties": {
+                "output_id": {
+                    "type": "string",
+                    "description": (
+                        "The id of the selected app to operate (from the selected-app "
+                        "context block)."
+                    ),
+                },
+                "task": {
+                    "type": "string",
+                    "description": (
+                        "What to do in the app. Be specific (e.g. 'graph y=x^2 and "
+                        "y=sin(x)')."
+                    ),
+                },
+            },
+            "required": ["output_id", "task"],
+        },
+    },
 ]


@@ -245,37 +279,54 @@ def format_batch_results(results: list[dict]) -> dict:
    return {"content": all_content}


+def _text_error(message: str) -> dict:
+    return {"content": [{"type": "text", "text": message}], "isError": True}
+
+
+def _run_single_task(task_def: dict) -> dict:
+    """Dispatch one task to the backend and format its single result."""
+    result = call_backend([task_def])
+    if "error" in result:
+        return _text_error(f"Error: {result['error']}")
+    results = result.get("results", [result])
+    if results:
+        return format_result(results[0])
+    return _text_error("No result returned.")
+
+
 def handle_tool_call(tool_name: str, arguments: dict) -> dict:
    if tool_name == "CreateBrowserAgent":
-        task_def = {
+        return _run_single_task({
            "task": arguments.get("task", ""),
            "browser_id": "",
            "url": arguments.get("url", ""),
-        }
-        result = call_backend([task_def])
-        if "error" in result:
-            return {"content": [{"type": "text", "text": f"Error: {result['error']}"}], "isError": True}
-        results = result.get("results", [result])
-        if results:
-            return format_result(results[0])
-        return {"content": [{"type": "text", "text": "No result returned."}], "isError": True}
+        })

    elif tool_name == "BrowserAgent":
        browser_id = arguments.get("browser_id", "")
        if not browser_id:
-            return {"content": [{"type": "text", "text": "Error: browser_id is required"}], "isError": True}
-        task_def = {
+            return _text_error("Error: browser_id is required")
+        return _run_single_task({
            "task": arguments.get("task", ""),
            "browser_id": browser_id,
            "url": "",
-        }
-        result = call_backend([task_def])
-        if "error" in result:
-            return {"content": [{"type": "text", "text": f"Error: {result['error']}"}], "isError": True}
-        results = result.get("results", [result])
-        if results:
-            return format_result(results[0])
-        return {"content": [{"type": "text", "text": "No result returned."}], "isError": True}
+        })
+
+    elif tool_name == "AppAgent":
+        output_id = arguments.get("output_id", "")
+        if not output_id:
+            return _text_error("Error: output_id is required")
+        # Only drive apps the user actually selected (anti-hallucination), when we
+        # know the selection. Empty list = unknown, so don't block.
+        if SELECTED_APP_IDS and output_id not in SELECTED_APP_IDS:
+            valid = ", ".join(SELECTED_APP_IDS) or "(none)"
+            return _text_error(f"Error: '{output_id}' is not a selected app. Selected apps: {valid}")
+        return _run_single_task({
+            "task": arguments.get("task", ""),
+            "browser_id": f"app:{output_id}",
+            "url": "",
+            "app_mode": True,
+        })

    elif tool_name == "BrowserAgents":
        tasks = arguments.get("tasks", [])
@@ -209,6 +209,7 @@ def _build_selected_app_context(selected_app_output_ids: list[str] | None) -> st
        name = output.name or "Untitled App"
        lines = [
            f'- App: "{name}"',
+            f"  App id (for AppAgent): {output_id}",
            f"  Workspace path: {path}",
            f"  Entry point: {os.path.join(path, 'index.html')}",
        ]
@@ -224,10 +225,14 @@ def _build_selected_app_context(selected_app_output_ids: list[str] | None) -> st
        return None
    return (
        "<selected_app_context>\n"
-        "The user selected these App cards on the dashboard for you to edit. "
-        "They are existing web apps; edit the files in place at the paths below "
-        "and the dashboard preview live-reloads on save. Do not scaffold a new "
-        "project or write files anywhere else.\n\n"
+        "The user selected these App cards on the dashboard. You can do two things "
+        "with them:\n"
+        "- EDIT them: change the files in place at the paths below and the dashboard "
+        "preview live-reloads on save. Do not scaffold a new project or write files "
+        "elsewhere.\n"
+        "- OPERATE them: to actually use a running app (e.g. 'graph y=x^2', 'fill in "
+        "the form'), call AppAgent(output_id, task) with the App id below. A "
+        "dedicated agent drives the live app through its own actions, no editing.\n\n"
        + "\n\n".join(entries)
        + "\n</selected_app_context>"
    )
@@ -325,6 +325,61 @@ export const JOBS_LIST = '/api/jobs/list';

 ---

+## Make the app agent-operable — the `OPENSWARM_APP` bridge
+
+An agent can drive this app on the user's behalf (e.g. "graph y=x^2 on my
+Desmos app"). It does NOT do that by clicking pixels or scraping the DOM —
+that's slow and an app's DOM is often a bare `<canvas>`. Instead, expose a
+tiny bridge on `window` and the agent reads/acts through it in one fast
+`executeJavaScript` call. **Always add this bridge to every app you build.**
+
+Set `window.OPENSWARM_APP` with three functions:
+
+- `describe()` → the **current** list of actions, `[{ name, args?, description? }]`.
+  Recompute it live on every call; controls are dynamic, so return only what's
+  actually available right now.
+- `getState()` → a **small** JSON snapshot of the app's relevant state (used to
+  verify an action landed). Keep it compact — this is the latency budget.
+- `invoke(name, args)` → perform the named action and return a result (or throw
+  a string the agent will read).
+
+Keep `args` shapes simple (strings, numbers, booleans, small objects). The agent
+only ever calls actions that `describe()` returned; it never edits your code.
+
+```tsx
+// Register once the app's core object exists (e.g. after the calculator mounts).
+// `calc` here is the app's own API (Desmos example); use whatever yours exposes.
+function registerAgentBridge(calc: any) {
+  (window as any).OPENSWARM_APP = {
+    describe() {
+      const actions = [
+        { name: 'addExpr', args: { latex: 'string' }, description: 'Add a graph expression, e.g. y=x^2' },
+        { name: 'clear', description: 'Remove all expressions' },
+      ];
+      // Dynamic: only offer removeExpr when something is on the graph.
+      if (calc.getExpressions().length > 0) {
+        actions.push({ name: 'removeExpr', args: { id: 'string' }, description: 'Remove one expression by id' });
+      }
+      return actions;
+    },
+    getState() {
+      return { expressions: calc.getExpressions().map((e: any) => ({ id: e.id, latex: e.latex })) };
+    },
+    invoke(name: string, args: any = {}) {
+      if (name === 'addExpr') { const id = String(Date.now()); calc.setExpression({ id, latex: args.latex }); return { id }; }
+      if (name === 'removeExpr') { calc.removeExpression({ id: args.id }); return { ok: true }; }
+      if (name === 'clear') { calc.setBlank(); return { ok: true }; }
+      throw `Unknown action: ${name}`;
+    },
+  };
+}
+```
+
+If you skip the bridge the agent falls back to slow UI-driving, so apps meant to
+be agent-operated should always register it.
+
+---
+
 ## Debugging — use `swarm_debug`, not `print()`

 The backend has `swarm_debug` pre-installed. It's a colored frame-aware
@@ -529,6 +529,14 @@ async def browser_agent_run(request: Request):
    if not tasks:
        return JSONResponse({"error": "tasks array is required"}, status_code=400)

+    # [app-agent] step trace: log app-mode dispatches arriving at the route.
+    for _t in tasks:
+        if _t.get("app_mode") or str(_t.get("browser_id", "")).startswith("app:"):
+            logger.info(
+                f"[app-agent] ROUTE /run: browser_id={_t.get('browser_id')!r} "
+                f"app_mode={_t.get('app_mode')} task={str(_t.get('task',''))[:120]!r}"
+            )
+
    results = await run_browser_agents(
        tasks=tasks,
        model=model,
@@ -0,0 +1,148 @@
+"""App agent: driving an OpenSwarm-built app through its window.OPENSWARM_APP bridge.
+
+Pins the wiring that makes an app drivable without touching a real webview or LLM:
+  - the three App bridge tools translate to a single BrowserEvaluate against the
+    app's bridge,
+  - run_browser_agents forwards app_mode + the "app:<id>" target without creating
+    or navigating a browser card,
+  - the AppAgent delegation tool builds the right task and gates on the user's
+    selected apps,
+  - the orchestrator's selected-app context advertises AppAgent.
+"""
+
+import asyncio
+import json
+
+from backend.apps.agents.browser import browser_agent as BA
+
+
+# --- bridge expression -------------------------------------------------------
+def test_app_bridge_expression_describe_and_state():
+    desc = BA._app_bridge_expression("AppDescribe", {})
+    assert "window.OPENSWARM_APP.describe()" in desc
+    assert "JSON.stringify" in desc  # round-trips as text
+    state = BA._app_bridge_expression("AppGetState", {})
+    assert "window.OPENSWARM_APP.getState()" in state
+
+
+def test_app_bridge_expression_invoke_serializes_args():
+    expr = BA._app_bridge_expression("AppInvoke", {"name": "addExpr", "args": {"latex": "y=x^2"}})
+    # name + args are JSON-encoded into the call
+    assert 'window.OPENSWARM_APP.invoke("addExpr", {"latex": "y=x^2"})' in expr
+    # missing bridge is handled inside the expression, never throws
+    assert "typeof A.describe!=='function'" in expr
+
+
+def test_app_bridge_expression_invoke_defaults_args_to_empty_object():
+    expr = BA._app_bridge_expression("AppInvoke", {"name": "clear"})
+    assert 'window.OPENSWARM_APP.invoke("clear", {})' in expr
+
+
+# --- execute_browser_tool routing -------------------------------------------
+def test_execute_browser_tool_app_bridge_routes_to_evaluate(monkeypatch):
+    captured = {}
+
+    async def _send(request_id, action, browser_id, params, tab_id=""):
+        captured.update(action=action, browser_id=browser_id, params=params)
+        return {"text": json.dumps([{"name": "addExpr"}])}
+
+    monkeypatch.setattr(BA.ws_manager, "send_browser_command", _send, raising=False)
+    out = asyncio.run(BA.execute_browser_tool(
+        "AppInvoke", {"name": "addExpr", "args": {"latex": "y=x^2"}}, "app:abc",
+    ))
+    assert captured["action"] == "evaluate"
+    assert captured["browser_id"] == "app:abc"
+    assert "window.OPENSWARM_APP.invoke" in captured["params"]["expression"]
+    assert out["text"]  # result passes straight through
+
+
+# --- run_browser_agents app_mode wiring -------------------------------------
+def test_run_browser_agents_app_mode_forwards_flag_no_card(monkeypatch):
+    recorded = {}
+
+    async def _fake_run_browser_agent(**kwargs):
+        recorded.update(kwargs)
+        return {"summary": "done", "action_log": [], "final_screenshot": None}
+
+    async def _boom(*a, **k):
+        raise AssertionError("app mode must not create a browser card")
+
+    monkeypatch.setattr(BA, "run_browser_agent", _fake_run_browser_agent, raising=True)
+    monkeypatch.setattr(BA, "_create_browser_card", _boom, raising=True)
+    # a connected dashboard so dispatch isn't refused
+    monkeypatch.setattr(BA.ws_manager, "global_connections", [object()], raising=False)
+
+    results = asyncio.run(BA.run_browser_agents(
+        tasks=[{"task": "graph y=x^2", "browser_id": "app:abc", "app_mode": True}],
+        model="sonnet",
+        dashboard_id="dash-1",
+    ))
+
+    assert results and results[0]["summary"] == "done"
+    assert recorded["app_mode"] is True
+    assert recorded["browser_id"] == "app:abc"
+    assert recorded["initial_url"] is None  # app already loaded; never navigate
+
+
+# --- AppAgent delegation tool -----------------------------------------------
+def _load_mcp_server(monkeypatch, selected):
+    import backend.apps.agents.browser_agent_mcp_server as srv
+    monkeypatch.setattr(srv, "SELECTED_APP_IDS", list(selected), raising=False)
+    return srv
+
+
+def test_app_agent_tool_builds_app_task(monkeypatch):
+    srv = _load_mcp_server(monkeypatch, ["abc"])
+    captured = {}
+
+    def _call_backend(tasks):
+        captured["tasks"] = tasks
+        return {"results": [{"summary": "graphed it", "action_log": []}]}
+
+    monkeypatch.setattr(srv, "call_backend", _call_backend, raising=True)
+    res = srv.handle_tool_call("AppAgent", {"output_id": "abc", "task": "graph y=x^2"})
+
+    assert not res.get("isError")
+    task_def = captured["tasks"][0]
+    assert task_def["browser_id"] == "app:abc"
+    assert task_def["app_mode"] is True
+    assert task_def["task"] == "graph y=x^2"
+
+
+def test_app_agent_tool_rejects_unselected_app(monkeypatch):
+    srv = _load_mcp_server(monkeypatch, ["abc"])
+
+    def _call_backend(tasks):
+        raise AssertionError("must not dispatch an unselected app")
+
+    monkeypatch.setattr(srv, "call_backend", _call_backend, raising=True)
+    res = srv.handle_tool_call("AppAgent", {"output_id": "zzz", "task": "graph"})
+    assert res.get("isError")
+    assert "not a selected app" in res["content"][0]["text"]
+
+
+def test_app_agent_tool_requires_output_id(monkeypatch):
+    srv = _load_mcp_server(monkeypatch, [])
+    res = srv.handle_tool_call("AppAgent", {"task": "graph"})
+    assert res.get("isError")
+    assert "output_id is required" in res["content"][0]["text"]
+
+
+# --- orchestrator context advertises AppAgent --------------------------------
+def test_selected_app_context_advertises_app_agent(monkeypatch):
+    import os
+    from backend.apps.agents.manager.prompt import prompt_context as pc
+    import backend.apps.outputs.workspace_io as wio
+
+    class _Out:
+        workspace_id = "ws-1"
+        name = "Grapher"
+
+    monkeypatch.setattr(wio, "load_output", lambda oid: _Out(), raising=True)
+    monkeypatch.setattr(os.path, "isdir", lambda p: True, raising=True)
+    monkeypatch.setattr(os.path, "isfile", lambda p: False, raising=True)
+
+    ctx = pc._build_selected_app_context(["abc"])
+    assert ctx is not None
+    assert "App id (for AppAgent): abc" in ctx
+    assert "AppAgent(output_id, task)" in ctx
@@ -84,7 +84,6 @@
      "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==",
      "dev": true,
      "license": "MIT",
-      "peer": true,
      "dependencies": {
        "@babel/code-frame": "^7.29.0",
        "@babel/generator": "^7.29.0",
@@ -1965,7 +1964,6 @@
      "resolved": "https://registry.npmjs.org/@emotion/react/-/react-11.14.0.tgz",
      "integrity": "sha512-O000MLDBDdk/EohJPFUqvnp4qnHeYkVP5B0xEG0D/L7cOKP9kefu2DXn8dj74cQfsEzUqh+sr1RzFqiL1o+PpA==",
      "license": "MIT",
-      "peer": true,
      "dependencies": {
        "@babel/runtime": "^7.18.3",
        "@emotion/babel-plugin": "^11.13.5",
@@ -2009,7 +2007,6 @@
      "resolved": "https://registry.npmjs.org/@emotion/styled/-/styled-11.14.1.tgz",
      "integrity": "sha512-qEEJt42DuToa3gurlH4Qqc1kVpNq8wO8cJtDzU46TjlzWjDlsVyevtYCRijVq3SrHsROS+gVQ8Fnea108GnKzw==",
      "license": "MIT",
-      "peer": true,
      "dependencies": {
        "@babel/runtime": "^7.18.3",
        "@emotion/babel-plugin": "^11.13.5",
@@ -2245,7 +2242,6 @@
      "resolved": "https://registry.npmjs.org/@mui/material/-/material-7.3.10.tgz",
      "integrity": "sha512-cHvGOk2ZEfbQt3LnGe0ZKd/ETs9gsUpkW66DCO+GSjMZhpdKU4XsuIr7zJ/B/2XaN8ihxuzHfYAR4zPtCN4RYg==",
      "license": "MIT",
-      "peer": true,
      "dependencies": {
        "@babel/runtime": "^7.28.6",
        "@mui/core-downloads-tracker": "^7.3.10",
@@ -3378,7 +3374,6 @@
      "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.28.tgz",
      "integrity": "sha512-z9VXpC7MWrhfWipitjNdgCauoMLRdIILQsAEV+ZesIzBq/oUlxk0m3ApZuMFCXdnS4U7KrI+l3WRUEGQ8K1QKw==",
      "license": "MIT",
-      "peer": true,
      "dependencies": {
        "@types/prop-types": "*",
        "csstype": "^3.2.2"
@@ -3775,7 +3770,6 @@
      "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==",
      "dev": true,
      "license": "MIT",
-      "peer": true,
      "bin": {
        "acorn": "bin/acorn"
      },
@@ -3815,7 +3809,6 @@
      "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
      "dev": true,
      "license": "MIT",
-      "peer": true,
      "dependencies": {
        "fast-deep-equal": "^3.1.3",
        "fast-uri": "^3.0.1",
@@ -4144,7 +4137,6 @@
        }
      ],
      "license": "MIT",
-      "peer": true,
      "dependencies": {
        "baseline-browser-mapping": "^2.10.12",
        "caniuse-lite": "^1.0.30001782",
@@ -8192,7 +8184,6 @@
      "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==",
      "dev": true,
      "license": "MIT",
-      "peer": true,
      "engines": {
        "node": ">=12"
      },
@@ -8249,7 +8240,6 @@
        }
      ],
      "license": "MIT",
-      "peer": true,
      "dependencies": {
        "nanoid": "^3.3.11",
        "picocolors": "^1.1.1",
@@ -8468,7 +8458,6 @@
      "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz",
      "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==",
      "license": "MIT",
-      "peer": true,
      "dependencies": {
        "loose-envify": "^1.1.0"
      },
@@ -8481,7 +8470,6 @@
      "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz",
      "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==",
      "license": "MIT",
-      "peer": true,
      "dependencies": {
        "loose-envify": "^1.1.0",
        "scheduler": "^0.23.2"
@@ -8528,7 +8516,6 @@
      "resolved": "https://registry.npmjs.org/react-redux/-/react-redux-9.2.0.tgz",
      "integrity": "sha512-ROY9fvHhwOD9ySfrF0wmvu//bKCQ6AeZZq1nJNtbDC+kk5DuSuNX/n6YWYF/SYy7bSba4D4FSz8DJeKY/S/r+g==",
      "license": "MIT",
-      "peer": true,
      "dependencies": {
        "@types/use-sync-external-store": "^0.0.6",
        "use-sync-external-store": "^1.4.0"
@@ -8667,8 +8654,7 @@
      "version": "5.0.1",
      "resolved": "https://registry.npmjs.org/redux/-/redux-5.0.1.tgz",
      "integrity": "sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w==",
-      "license": "MIT",
-      "peer": true
+      "license": "MIT"
    },
    "node_modules/redux-thunk": {
      "version": "3.1.0",
@@ -8980,7 +8966,6 @@
      "integrity": "sha512-kgW13M54DUB7IsIRM5LvJkNlpH+WhMpooUcaWGFARkF1Tc82v9mIWkCbCYf+MBvpIUBSeSOTilpZjEPr2VYE6Q==",
      "dev": true,
      "license": "MIT",
-      "peer": true,
      "dependencies": {
        "chokidar": "^4.0.0",
        "immutable": "^5.1.5",
@@ -10084,7 +10069,6 @@
      "integrity": "sha512-wGN3qcrBQIFmQ/c0AiOAQBvrZ5lmY8vbbMv4Mxfgzqd/B6+9pXtLo73WuS1dSGXM5QYY3hZnIbvx+K1xxe6FyA==",
      "dev": true,
      "license": "MIT",
-      "peer": true,
      "dependencies": {
        "@types/eslint-scope": "^3.7.7",
        "@types/estree": "^1.0.8",
@@ -10133,7 +10117,6 @@
      "integrity": "sha512-pIDJHIEI9LR0yxHXQ+Qh95k2EvXpWzZ5l+d+jIo+RdSm9MiHfzazIxwwni/p7+x4eJZuvG1AJwgC4TNQ7NRgsg==",
      "dev": true,
      "license": "MIT",
-      "peer": true,
      "dependencies": {
        "@discoveryjs/json-ext": "^0.5.0",
        "@webpack-cli/configtest": "^2.1.1",
@@ -587,6 +587,7 @@ const DashboardOutputPreview: React.FC<{
      frontendCode={output.files?.['index.html'] ?? ''}
      inputData={inputData}
      backendResult={backendResult}
+      agentBrowserId={`app:${output.id}`}
    />
  );
 };
@@ -6,6 +6,7 @@ import { useElementSelection } from '@/app/components/editor/ElementSelectionCon
 import { useIframeElementSelector } from './useIframeElementSelector';
 import { getAuthToken, ensureAuthToken } from '@/shared/config';
 import { useClaudeTokens } from '@/shared/styles/ThemeContext';
+import { registerWebview, unregisterWebview, setActiveTab, type BrowserWebview } from '@/shared/browserRegistry';

 // In Electron use <webview> to escape iframe restrictions (popups, mic/camera, WebAuthn, cookied fetch); outside Electron fall back to iframe.
 const isElectron = navigator.userAgent.includes('Electron');
@@ -55,8 +56,13 @@ interface Props {
  onConsoleMessage?: (level: string, text: string) => void;
  /** Fires once the embedded app has actually painted, so cold-start placeholders don't unmount during the vite-ready to first-paint gap. */
  onContentLoad?: () => void;
+  /** When set (webview mode only), registers this preview's webview in browserRegistry under this id so an app agent can drive it via the browser command pipeline. */
+  agentBrowserId?: string;
 }

+// Single tab per app preview; the browser command pipeline keys on browserId:tabId.
+const APP_TAB_ID = 'main';
+
 function buildSrcdoc(
  frontendCode: string,
  inputData: Record<string, any>,
@@ -92,6 +98,7 @@ const ViewPreview = forwardRef<ViewPreviewHandle, Props>(({
  style,
  onConsoleMessage,
  onContentLoad,
+  agentBrowserId,
 }, ref) => {
  const iframeRef = useRef<HTMLIFrameElement>(null);
  const webviewRef = useRef<any>(null);
@@ -280,6 +287,23 @@ const ViewPreview = forwardRef<ViewPreviewHandle, Props>(({
    };
  }, [useWebview, handleNavigationLoad]);

+  // Register the live webview so an app agent can drive it through the same
+  // browser command pipeline that drives browser cards. Webview-only: the iframe
+  // fallback has no executeJavaScript channel from the host.
+  useEffect(() => {
+    if (!useWebview || !agentBrowserId) return;
+    const wv = webviewRef.current as BrowserWebview | null;
+    if (!wv) return;
+    registerWebview(agentBrowserId, APP_TAB_ID, wv);
+    setActiveTab(agentBrowserId, APP_TAB_ID);
+    return () => {
+      try { unregisterWebview(agentBrowserId, APP_TAB_ID); } catch (_e) {}
+    };
+    // Keyed on useWebview (flips true when the webview mounts), not iframeSrc:
+    // the webview element has a stable key and survives src/data changes, so
+    // re-registering on every data update would just thrash the registry.
+  }, [useWebview, agentBrowserId]);
+
  const hasContent = !!(serveUrl || frontendCode?.trim());

  if (!hasContent) {
@@ -1,4 +1,4 @@
-import { getWebview, type BrowserWebview } from './browserRegistry';
+import { getWebview, registeredKeys, type BrowserWebview } from './browserRegistry';
 import { store } from './state/store';
 import { resumeBrowserCard } from './state/dashboardLayoutSlice';
 import { dashboardWs } from './ws/WebSocketManager';
@@ -1330,8 +1330,13 @@ async function handleReplayRoute(wv: BrowserWebview, params: Record<string, any>
 async function handleEvaluate(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
  const expression = params.expression as string;
  if (!expression) return { error: 'expression parameter is required' };
+  // [app-agent] step trace: surface what the app's bridge actually returned.
+  const _bridgeCall = expression.includes('OPENSWARM_APP');
  try {
    const result = await wv.executeJavaScript(expression);
+    if (_bridgeCall) {
+      console.log(`[app-agent] BRIDGE eval -> ${typeof result === 'string' ? result : JSON.stringify(result)}`);
+    }
    const text = typeof result === 'string' ? result : JSON.stringify(result, null, 2);
    // evaluate is the agent's main read path; sample routes here too (XHRs have
    // fired by now) so the backend can surface the fast network tier once.
@@ -1397,14 +1402,27 @@ async function runBrowserCommand(
  request_id: string, action: string, browser_id: string, tab_id: string | undefined,
  params: Record<string, any>,
 ) {
+  // [app-agent] step trace: only for app targets so browser-agent runs stay quiet.
+  const _appTrace = String(browser_id || '').startsWith('app:');
+  if (_appTrace) {
+    console.log(`[app-agent] CMD recv: action=${action} browser_id=${browser_id} tab_id=${tab_id ?? ''} registered=[${registeredKeys().join(', ')}]`);
+  }
  const wv = await awaitWebview(browser_id, tab_id || undefined);
  if (!wv) {
+    if (_appTrace) {
+      console.warn(`[app-agent] LOOKUP MISS: no webview for '${browser_id}' (registered keys: [${registeredKeys().join(', ')}]) -> the app card isn't mounted on the active dashboard`);
+    }
    dashboardWs.send('browser:result', {
      request_id,
      error: `Browser card '${browser_id}'${tab_id ? ` tab '${tab_id}'` : ''} not found or not an Electron webview`,
    });
    return;
  }
+  if (_appTrace) {
+    let _url = '';
+    try { _url = wv.getURL(); } catch (_e) {}
+    console.log(`[app-agent] LOOKUP HIT: webview for '${browser_id}' found, url=${_url} loading=${(() => { try { return wv.isLoading(); } catch { return '?'; } })()}`);
+  }

  const detail = params.url || params.selector || params.expression || undefined;
  setActivity(browser_id, { action: action as BrowserAction, detail });
@@ -51,6 +51,11 @@ export function setActiveTab(browserId: string, tabId: string): void {
  activeTabMap.set(browserId, tabId);
 }

+// [app-agent] diagnostic: list currently-registered keys ("browserId:tabId").
+export function registeredKeys(): string[] {
+  return Array.from(registry.keys());
+}
+
 export function getWebview(browserId: string, tabId?: string): BrowserWebview | undefined {
  const resolvedTabId = tabId || activeTabMap.get(browserId);
  if (!resolvedTabId) return undefined;