WEBDIENSTE/openswarm
1
0
Fork 0
mirror of https://github.com/openswarm-ai/openswarm.git synced 2026-09-12 12:47:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

[aidan] chore/merge: resolve conflicts merging eric/dev

Browse Source
This commit is contained in:
abccodes
2026-06-23 04:41:43 -07:00
parent c284a8967f 7474291f97
commit 95f082a27f
177 changed files with 11837 additions and 856 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/apps/agents/9router_gpt5_patch.js
+30 -9
View File
@@ -39,24 +39,45 @@ const _http = require('http');
} catch (_) {}
})();
// 9Router's /callback page is a client-side relay (postMessage/BroadcastChannel/
// localStorage) that fails when the OAuth flow runs in the user's system browser:
// no opener, different cookie jar. 302 to the backend so the exchange happens
// server-side. Idempotent via _completed_oauth (backend/apps/oauth_state.py) so
// a racing renderer-driven exchange in popup mode dedups.
(function patchOauthCallbackRedirect() {
// Claude OAuth completion. Anthropic only whitelists localhost:20128/callback as the
// redirect, so Claude's callback HAS to land here on 9Router (unlike Gemini, which goes
// straight to the backend, and Codex, which has its own :1455 listener). We previously
// 302'd the user's browser across ports to the backend, but a cross-port plain-http
// localhost redirect silently fails in browsers that HTTPS-upgrade or block it, which
// hung "Connecting…" for some users (browser-dependent, Claude-only). Fix: run the code
// exchange server-to-server (9Router -> backend, same machine, no browser in the loop)
// and hand the browser a static close-page. The browser only ever talks to :20128.
// Idempotent via the backend's _pending_oauth.pop + _completed_oauth.
(function patchOauthCallbackExchange() {
try {
const http = require('http');
const origEmit = http.Server.prototype.emit;
const closePage =
'<!doctype html><meta charset="utf-8"><body style="font-family:-apple-system,system-ui;' +
'text-align:center;color:#888;padding-top:80px;background:#1a1a1a">' +
'You can close this tab, and any other Claude login tab still open.</body>';
http.Server.prototype.emit = function patchedEmit(event, req, res) {
if (event === 'request' && req && res) {
try {
const url = req.url || '';
if (url.startsWith('/callback?')) {
const backendPort = process.env.OPENSWARM_PORT || '8324';
const target = 'http://localhost:' + backendPort + '/api/subscriptions/callback' + url.slice('/callback'.length);
res.writeHead(302, { Location: target });
res.end();
const path = '/api/subscriptions/callback' + url.slice('/callback'.length);
let done = false;
const finish = () => {
if (done) return;
done = true;
try { res.writeHead(200, { 'Content-Type': 'text/html' }); res.end(closePage); } catch (_) {}
};
try {
const proxyReq = http.request(
{ host: '127.0.0.1', port: backendPort, path: path, method: 'GET' },
(proxyRes) => { proxyRes.resume(); proxyRes.on('end', finish); }
);
proxyReq.on('error', finish);
proxyReq.setTimeout(5000, () => { try { proxyReq.destroy(); } catch (_) {} finish(); });
proxyReq.end();
} catch (_) { finish(); }
return true;
}
} catch (_) {}
backend/apps/agents/agent_manager.py
+488 -72
View File
@@ -16,6 +16,7 @@ from backend.apps.agents.core.ws_manager import ws_manager
from backend.apps.settings.settings import load_settings
from backend.apps.tools_lib.tools_lib import (
_load_all as load_all_tools,
_save as save_tool,
_sanitize_server_name,
derive_mcp_config,
load_builtin_permissions,
@@ -23,6 +24,8 @@ from backend.apps.tools_lib.tools_lib import (
refresh_airtable_token,
refresh_google_token,
refresh_hubspot_token,
resolve_policy_slot,
save_builtin_permissions,
save_trusted_sensitive_paths,
)
from backend.config.paths import SESSIONS_DIR
@@ -36,6 +39,8 @@ from backend.apps.agents.core.error_classify import (
p_is_transient_capacity_error,
p_is_unknown_model_error,
p_extract_reset_hint,
parse_retry_after,
redact_for_telemetry,
)
from backend.apps.agents.manager.session.session_store import (
_delete_session_file,
@@ -62,12 +67,15 @@ from backend.apps.agents.manager.session.history_compaction import (
from backend.apps.agents.manager.prompt.prompt_context import (
_build_browser_context,
_build_selected_app_context,
_build_selected_settings_context,
_build_connected_tools_context,
_build_mcp_registry_summary,
_compose_system_prompt,
_resolve_attached_skills,
_resolve_forced_tools,
_resolve_mode,
TOOLSEARCH_LOOP_THRESHOLD,
toolsearch_loop_redirect,
)
from backend.apps.agents.manager.prompt.attachments import (
_build_dir_tree,
@@ -134,6 +142,11 @@ def get_workflow_step_usage(session_id: str) -> dict[str, dict[str, bool]]:
return mem.step_usage if mem is not None else {}
p_VIEW_BUILDER_RENDER_MAX_RETRIES = 2
p_view_builder_render_retry_counts: dict[str, int] = {}
p_view_builder_dirty_sessions: set[str] = set()
def _apply_context_window(session, settings=None) -> None:
"""Set session.context_window from the registry for its (provider, model).
@@ -187,7 +200,11 @@ class AgentManager:
def __init__(self):
self.sessions: dict[str, AgentSession] = {}
self.tasks: dict[str, asyncio.Task] = {}
# Live mirror of the in-flight streamed assistant text per session, so a
# stop can persist the partial reply instantly instead of waiting out the
# multi-second SDK teardown the cancel handler sits behind.
self._live_partial: dict[str, dict] = {}
def _resolve_mode(self, mode_id: str) -> tuple[list[str], str | None, str | None]:
return _resolve_mode(mode_id, get_all_tool_names)
@@ -242,8 +259,9 @@ class AgentManager:
continue
if tool.auth_type == "oauth2" and tool.auth_status == "connected":
if tool.name.lower() == "discord":
# Discord uses a shared bot token from .env, not user OAuth tokens.
if tool.name.lower() in ("discord", "github"):
# Discord uses a shared bot token; GitHub OAuth-app tokens don't
# expire and carry no refresh_token. Nothing to refresh either way.
refreshed = True
elif tool.name.lower() == "airtable":
refreshed = await refresh_airtable_token(tool)
@@ -264,6 +282,29 @@ class AgentManager:
logger.info(f"[MCP-DEBUG] Final mcp_servers: {list(mcp_servers.keys())}")
return mcp_servers
def _gated_mcp_server_names(self, allowed_tools: list[str], active_mcps: list[str] | None) -> list[str]:
"""Names of installed MCP servers withheld from the SDK because they're
not activated yet, exactly the servers the model sees in the
<mcp_servers> block but can't reach via ToolSearch. The only way in is
MCPActivate; used to steer a model looping on ToolSearch to the gate."""
active_set = set(active_mcps or [])
names: list[str] = []
try:
for tool in load_all_tools():
if not (tool.mcp_config and tool.enabled and tool.auth_status in ("configured", "connected")):
continue
tool_ref = f"mcp:{tool.name}"
if tool_ref not in allowed_tools and allowed_tools != get_all_tool_names():
continue
if _is_fully_denied(tool):
continue
server_name = _sanitize_server_name(tool.name)
if server_name not in active_set:
names.append(server_name)
except Exception:
logger.exception("gated MCP server enumeration failed")
return names
def _build_connected_tools_context(self, allowed_tools: list[str]) -> str | None:
return _build_connected_tools_context(allowed_tools, get_all_tool_names)
@@ -471,7 +512,7 @@ class AgentManager:
def _resolve_context_paths(self, context_paths: list | None) -> str:
return _resolve_context_paths(context_paths)
async def _run_agent_loop(self, session_id: str, prompt: str, images: list | None = None, context_paths: list | None = None, forced_tools: list[str] | None = None, attached_skills: list | None = None, fork_session: bool = False, selected_browser_ids: list[str] | None = None, selected_app_output_ids: list[str] | None = None):
async def _run_agent_loop(self, session_id: str, prompt: str, images: list | None = None, context_paths: list | None = None, forced_tools: list[str] | None = None, attached_skills: list | None = None, fork_session: bool = False, selected_browser_ids: list[str] | None = None, selected_app_output_ids: list[str] | None = None, selected_setting_ids: list[str] | None = None):
"""Run the Claude Agent SDK query loop for a session."""
session = self.sessions.get(session_id)
if not session:
@@ -758,30 +799,39 @@ class AgentManager:
return policy, None
def _get_effective_policy(tool_name: str) -> str:
"""Return 'always_allow', 'deny', or 'ask' for any tool."""
if tool_name in _builtin_perms:
return _builtin_perms[tool_name]
import re as _re
bm = _re.match(r"mcp__openswarm-browser-agent__(.+)", tool_name)
if bm:
return _builtin_perms.get(bm.group(1), _default_for(bm.group(1)))
im = _re.match(r"mcp__openswarm-invoke-agent__(.+)", tool_name)
if im:
return _builtin_perms.get(im.group(1), _default_for(im.group(1)))
m = _re.match(r"mcp__([^_]+(?:-[^_]+)*)__(.+)", tool_name)
if m:
server_slug, mcp_tool_name = m.group(1), m.group(2)
for t in load_all_tools():
if not t.mcp_config or not t.enabled:
continue
if _sanitize_server_name(t.name) == server_slug:
return t.tool_permissions.get(mcp_tool_name, "ask")
"""Return 'always_allow', 'deny', or 'ask' for any tool. Keyed through
the shared resolver so the read slot matches the write slot exactly."""
tools = load_all_tools()
slot = resolve_policy_slot(tool_name, tools)
if slot.store == "builtin":
return _builtin_perms.get(slot.key, _default_for(slot.key))
if slot.key is not None:
for t in tools:
if t.id == slot.key:
return t.tool_permissions.get(slot.action, "ask")
return _default_for(tool_name)
def _set_tool_policy(tool_name: str, policy: str) -> None:
"""Inverse of _get_effective_policy: persist `policy` into the SAME slot
the gate reads, AND update the live in-memory snapshot, so an 'Always
approve' takes effect for this running agent, not only after a restart.
(The old code wrote the raw tool name to the file and never touched the
captured _builtin_perms, so it behaved like a one-time accept.)"""
tools = load_all_tools()
slot = resolve_policy_slot(tool_name, tools)
if slot.store == "builtin":
_builtin_perms[slot.key] = policy
perms = load_builtin_permissions()
perms[slot.key] = policy
save_builtin_permissions(perms)
return
if slot.key is not None:
for t in tools:
if t.id == slot.key:
t.tool_permissions[slot.action] = policy
save_tool(t)
return
async def _request_user_approval(
tool_name: str,
tool_input,
@@ -839,6 +889,15 @@ class AgentManager:
except Exception:
logger.exception("Failed to persist trusted sensitive path")
# "Always approve" button: persist the tool's policy so it stops
# prompting. The guards above (sensitive/catastrophic) re-fire even
# on always_allow, so this can't disarm an rm -rf or a key-path write.
if decision.get("behavior") == "allow" and decision.get("set_always_allow"):
try:
_set_tool_policy(tool_name, "always_allow")
except Exception:
logger.exception("Failed to persist always-allow for %s", tool_name)
approval_latency_ms = int((datetime.now() - approval_req.created_at).total_seconds() * 1000)
try:
# Append to the session's approval log so a reload
@@ -945,11 +1004,84 @@ class AgentManager:
)
tool_start_times: dict[str, float] = {}
# Counts ToolSearch calls in a row (no other tool between them). A run
# of these with empty results is the "looping on ToolSearch" wedge.
_ts_loop = {"n": 0}
# One mid-run connect offer per session: a stuck agent fires the loop-breaker repeatedly,
# but the user should see the "connect this MCP" card once, not on every retry.
_mcp_offer_sent = {"done": False}
async def pre_tool_hook(input_data, tool_use_id, context):
tool_name = input_data.get("tool_name", "")
hook_event = input_data.get("hook_event_name", "PreToolUse")
# ToolSearch loop-breaker. Gated MCP servers are withheld from the
# SDK until MCPActivate, so the CLI's native ToolSearch can never
# find them; small models thrash (empty ToolSearch, retry) for
# minutes until the user pauses. Let the first couple through, then
# redirect to the gate. Any non-ToolSearch call is real progress, so
# the counter resets. Gated-server lookup is deferred behind the
# threshold so the common (non-looping) path stays free.
if tool_name == "ToolSearch":
_ts_loop["n"] += 1
if _ts_loop["n"] >= TOOLSEARCH_LOOP_THRESHOLD:
_gated = self._gated_mcp_server_names(session.allowed_tools, session.active_mcps)
_reason = toolsearch_loop_redirect(_ts_loop["n"], _gated)
if _reason:
logger.info(f"[MCP-DEBUG] ToolSearch loop-breaker fired for {session_id} (n={_ts_loop['n']})")
# 2B-MCP: also surface a one-click connect offer to the USER for the vetted
# gated servers the agent keeps reaching for. Suggest-only: this just shows a
# card on the same channel the preflight uses; activation still requires
# MCPActivate + the dispatch gate, so it opens no side channel. Once per run,
# fail-open (an offer hiccup must never block the agent).
if not _mcp_offer_sent["done"]:
try:
from backend.apps.agents.core.mcp_preflight import offer_for_gated_server
_s = load_settings()
_offers = [o for o in (offer_for_gated_server(n, _s) for n in _gated) if o]
if _offers:
_mcp_offer_sent["done"] = True
await ws_manager.send_to_session(session_id, "agent:mcp_suggestions", {
"session_id": session_id,
"suggestions": _offers,
"is_vague": False,
})
except Exception:
logger.debug("mid-run MCP connect offer skipped", exc_info=True)
return {
"hookSpecificOutput": {
"hookEventName": hook_event,
"permissionDecision": "deny",
"permissionDecisionReason": _reason,
}
}
else:
_ts_loop["n"] = 0
# MCPSearch is the agent saying "I need an integration I don't have" (e.g. "no email
# connected"). Don't make the user read a wall of options: fire the same curated connect
# card the launch preflight uses, keyed to their original request. Non-blocking (the search
# proceeds) and once per run; covers the common path the ToolSearch-loop branch misses
# because a capable model does one MCPSearch instead of thrashing. Suggest-only as ever.
if (tool_name.endswith("MCPSearch") or tool_name.endswith("MCPList")) and not _mcp_offer_sent["done"]:
_mcp_offer_sent["done"] = True
async def _offer_from_prompt():
try:
from backend.apps.agents.core.mcp_preflight import run_preflight
result = await run_preflight(prompt, task_id=session_id, require_vague=False)
offers = result.get("suggestions", [])
if offers:
await ws_manager.send_to_session(session_id, "agent:mcp_suggestions", {
"session_id": session_id,
"suggestions": offers,
"is_vague": False,
})
except Exception:
logger.debug("MCPSearch-triggered connect offer skipped", exc_info=True)
asyncio.create_task(_offer_from_prompt())
if tool_name and tool_name != "AskUserQuestion":
tool_input = input_data.get("tool_input", {})
if _is_claude_schedule_skill(tool_name, tool_input):
@@ -1072,26 +1204,38 @@ class AgentManager:
except Exception:
content = str(raw_response)
# When the agent writes/edits a file inside a live App
# Builder workspace, surface any build-server errors
# (vite/babel/tsc/uvicorn) that landed in the runtime's
# stderr in the moments after the write. Without this the
# agent walks away from broken JSX, the iframe shows a red
# overlay, and the user has to copy-paste the error back.
# ~400ms gives vite's file watcher + babel parse enough
# time to react; the post_tool_hook runs once per tool so
# the added latency is acceptable for the win.
hook_tool_name_for_errors = input_data.get("tool_name", "")
if hook_tool_name_for_errors in ("Write", "Edit", "MultiEdit"):
tool_in = input_data.get("tool_input") or {}
file_path = tool_in.get("file_path") or tool_in.get("path") or ""
wrote_files = hook_tool_name_for_errors in ("Write", "Edit", "MultiEdit")
tool_in = input_data.get("tool_input") or {}
file_path = tool_in.get("file_path") or tool_in.get("path") or ""
wrote_frontend_file = wrote_files and "/frontend/" in file_path
installed_pkg = False
if hook_tool_name_for_errors == "Bash":
bash_in = input_data.get("tool_input") or {}
cmd = (bash_in.get("command") or "").lower()
installed_pkg = any(s in cmd for s in (
"npm install", "npm i ", "npm uninstall", "npm ci",
"pnpm add", "pnpm install", "pnpm remove",
"yarn add", "yarn install", "yarn remove",
))
if session.mode == "view-builder" and (wrote_frontend_file or installed_pkg):
p_view_builder_dirty_sessions.add(session.id)
try:
from backend.apps.outputs.runtime import (
manager as outputs_runtime_manager,
)
outputs_runtime_manager.reset_render_state_for_workspace(session.id)
except Exception:
pass
elif wrote_files:
if file_path:
try:
await asyncio.sleep(0.4)
from backend.apps.outputs.runtime import (
manager as _outputs_runtime_manager,
manager as outputs_runtime_manager,
)
errs = _outputs_runtime_manager.drain_errors_for_path(file_path)
errs = outputs_runtime_manager.drain_errors_for_path(file_path)
except Exception:
errs = []
if errs:
@@ -1131,6 +1275,9 @@ class AgentManager:
usage = raw_response.get("usage", {})
if isinstance(usage, dict):
sub_tokens["input"] = usage.get("input_tokens", 0) + usage.get("cache_creation_input_tokens", 0) + usage.get("cache_read_input_tokens", 0)
# Pill-only lane: NEW (uncached) input, excludes the cached
# static prefix so the bubble shows what this turn added.
sub_tokens["input_fresh"] = usage.get("input_tokens", 0)
sub_tokens["output"] = usage.get("output_tokens", 0)
if raw_response.get("total_cost_usd"):
sub_cost = raw_response["total_cost_usd"]
@@ -1333,6 +1480,12 @@ class AgentManager:
if app_ctx:
composed_prompt = f"{composed_prompt}\n\n{app_ctx}" if composed_prompt else app_ctx
# The user can point the agent at specific Settings rows. Targeting
# aid only; the settings tools are always on regardless.
settings_ctx = _build_selected_settings_context(selected_setting_ids)
if settings_ctx:
composed_prompt = f"{composed_prompt}\n\n{settings_ctx}" if composed_prompt else settings_ctx
# Per-turn estimate of framework overhead (subtracted from displayed
# input). Conservative on purpose so honest over-shows beat lies.
# 16K Claude Code preset, 12K base+deferred tools, ~3K/MCP (real
@@ -1451,6 +1604,27 @@ class AgentManager:
"type": "stdio",
}
# Always-on settings-meta server: SettingsRead / SettingsWrite let the
# agent read and edit its own OpenSwarm Settings autonomously. The
# backend (/api/settings-meta) enforces the only two guardrails: it
# can't disconnect the credential powering this run, and reads come
# back with secrets redacted. No activation gate, Settings is the
# agent's own house, not a third-party MCP.
settings_meta_server_path = os.path.join(
os.path.dirname(__file__), "settings_meta_server.py"
)
from backend.auth import get_auth_token as _get_auth_token4
mcp_servers["openswarm-settings-meta"] = {
"command": sys.executable,
"args": [settings_meta_server_path],
"env": {
"OPENSWARM_PORT": os.environ.get("OPENSWARM_PORT", "8324"),
"OPENSWARM_AUTH_TOKEN": _get_auth_token4(),
"OPENSWARM_PARENT_SESSION_ID": session.id,
},
"type": "stdio",
}
# The CLI's built-in WebSearch/WebFetch wraps Anthropic's
# web_search_20250305. For non-Claude primaries the CLI
@@ -1695,6 +1869,59 @@ class AgentManager:
if len(_stderr_buffer) > 500:
del _stderr_buffer[:250]
async def stop_hook(input_data, tool_use_id, context):
"""End-of-turn render gate for App Builder sessions. Reads the
browser-reported render-state of the preview; if the app fails
to render, blocks with the error so the agent fixes it, up to
MAX_RETRIES then lets the stop through."""
if session.mode != "view-builder":
return {}
if session.id not in p_view_builder_dirty_sessions:
return {}
from backend.apps.outputs.runtime import (
manager as outputs_runtime_manager,
)
if outputs_runtime_manager.get(session.id) is None:
return {}
state, error_text = outputs_runtime_manager.get_render_state_for_workspace(session.id)
waited = 0.0
while state is None and waited < 5.0:
await asyncio.sleep(0.25)
waited += 0.25
state, error_text = outputs_runtime_manager.get_render_state_for_workspace(session.id)
if state != "error":
p_view_builder_render_retry_counts.pop(session.id, None)
p_view_builder_dirty_sessions.discard(session.id)
return {}
attempts = p_view_builder_render_retry_counts.get(session.id, 0)
if attempts >= p_VIEW_BUILDER_RENDER_MAX_RETRIES:
logger.warning(
"view-builder preview still failing after %s attempts for session %s; allowing stop",
attempts, session.id,
)
p_view_builder_render_retry_counts.pop(session.id, None)
p_view_builder_dirty_sessions.discard(session.id)
return {}
p_view_builder_render_retry_counts[session.id] = attempts + 1
logger.info(
"view-builder render block (attempt %s/%s) for session %s",
attempts + 1, p_VIEW_BUILDER_RENDER_MAX_RETRIES, session.id,
)
trimmed = error_text[-3000:] if len(error_text) > 3000 else error_text
return {
"decision": "block",
"reason": (
f"The preview failed to render (attempt {attempts + 1}/"
f"{p_VIEW_BUILDER_RENDER_MAX_RETRIES}):\n\n"
f"{trimmed}\n\n"
"Fix this so the app renders before finishing; the user "
"currently sees an error instead of the app."
),
}
options_kwargs = {
"model": resolved_model,
# 64 MB ceiling on the SDK <-> CLI JSON-RPC channel. The
@@ -1710,6 +1937,7 @@ class AgentManager:
"hooks": {
"PreToolUse": [HookMatcher(matcher=None, hooks=[pre_tool_hook])],
"PostToolUse": [HookMatcher(matcher=None, hooks=[post_tool_hook])],
"Stop": [HookMatcher(matcher=None, hooks=[stop_hook])],
},
"allowed_tools": effective_allowed,
"disallowed_tools": effective_disallowed,
@@ -2041,6 +2269,15 @@ class AgentManager:
options_kwargs["thinking"] = {"type": "disabled"}
elif level in ("low", "medium", "high"):
options_kwargs["effort"] = level
elif api_type in ("openai", "codex"):
# GPT-5 family + Codex take reasoning_effort; 9Router carries
# the Anthropic-shaped `effort` across to it, so the slider
# works for OpenAI too, not just Claude. Every OpenAI/Codex
# model we expose is reasoning-capable (registry has no
# non-reasoning ones), so no per-model gate. No "disabled"
# form on these, so "off" just omits the param.
if level in ("low", "medium", "high"):
options_kwargs["effort"] = level
except Exception as e:
logger.debug(f"thinking_level param injection skipped: {e}")
@@ -2171,6 +2408,11 @@ class AgentManager:
stream_text_msg_id = None
stream_tool_msg_ids_ordered = []
stream_block_index_map = {}
# Mirror of the streamed assistant text. The SDK envelope that
# normally commits a reply never lands when a turn is stopped
# mid-stream, so without this the text the user just watched
# appear would evaporate. Cleared the instant a block commits.
_stream_text_accum = ""
# Per-turn aggregate trackers for the consolidated thinking
# message. We accumulate across every AssistantMessage in the
# turn (think → tool → think → tool → answer) and stream
@@ -2381,10 +2623,13 @@ class AgentManager:
# baseline to get THIS TURN'S delta. Without subtracting,
# the second turn's pill would show turn-1 work added
# to turn-2 work, the third would show all three, etc.
# Pill uses the FRESH lane (uncached input only). session.tokens
# ["input"] stays full for the context-fullness bar + cost; the
# bubble shows the NEW tokens this turn, not the cached re-reads.
_cum_in = 0
_cum_out = 0
if isinstance(session.tokens, dict):
_cum_in = int(session.tokens.get("input", 0) or 0)
_cum_in = int(session.tokens.get("input_fresh", 0) or 0)
_cum_out = int(session.tokens.get("output", 0) or 0)
_cum_children_in = 0
_cum_children_out = 0
@@ -2395,7 +2640,7 @@ class AgentManager:
_ct = getattr(_child, "tokens", None)
if not isinstance(_ct, dict):
continue
_cum_children_in += int(_ct.get("input", 0) or 0)
_cum_children_in += int(_ct.get("input_fresh", 0) or 0)
_cum_children_out += int(_ct.get("output", 0) or 0)
except Exception:
pass
@@ -2413,18 +2658,14 @@ class AgentManager:
_children_in = _cum_children_in
_children_out = _cum_children_out
# Fresh input + output = the NEW tokens this turn. The old
# framework-overhead subtraction is gone on purpose: it was an
# estimate to strip the cached static prefix out of the full
# input number, and the fresh lane already excludes that prefix
# exactly, so subtracting it again would double-discount to ~0.
_turn_total_tokens: int | None = (
_parent_in + _parent_out + _children_in + _children_out
)
# Strip framework overhead so bubble shows what the user
# actually controls. Floor at output so over-estimates can't
# render absurdly small.
if _turn_total_tokens and session.framework_overhead_tokens > 0:
_adjusted = _turn_total_tokens - session.framework_overhead_tokens
_floor = _parent_out + _children_out
if _adjusted < _floor:
_adjusted = _floor
_turn_total_tokens = _adjusted
if not _turn_total_tokens or _turn_total_tokens <= 0:
_turn_total_tokens = None
consolidated = Message(
@@ -2469,6 +2710,7 @@ class AgentManager:
async def _run_streaming_turn():
nonlocal stream_text_msg_id, stream_tool_msg_ids_ordered, stream_block_index_map
nonlocal _stream_text_accum
nonlocal _turn_number, _first_event, _current_turn_emitted
# Per-turn thinking aggregation trackers (added for the
# "Thought for Ns · M tokens" persisted label). Without
@@ -2500,8 +2742,10 @@ class AgentManager:
# Snapshot cumulative tokens at turn start;
# subtracted at emit time for per-turn deltas.
try:
# Baselines track the SAME fresh lane the pill reads,
# so the per-turn delta is fresh-minus-fresh.
if isinstance(session.tokens, dict):
_turn_baseline_session_in = int(session.tokens.get("input", 0) or 0)
_turn_baseline_session_in = int(session.tokens.get("input_fresh", 0) or 0)
_turn_baseline_session_out = int(session.tokens.get("output", 0) or 0)
_ch_in = 0
_ch_out = 0
@@ -2511,7 +2755,7 @@ class AgentManager:
_ct = getattr(_child, "tokens", None)
if not isinstance(_ct, dict):
continue
_ch_in += int(_ct.get("input", 0) or 0)
_ch_in += int(_ct.get("input_fresh", 0) or 0)
_ch_out += int(_ct.get("output", 0) or 0)
_turn_baseline_children_in = _ch_in
_turn_baseline_children_out = _ch_out
@@ -2635,6 +2879,12 @@ class AgentManager:
if msg_id and delta_type == "text_delta":
_text_chunk = delta.get("text", "")
_turn_assistant_text_chars += len(_text_chunk)
_stream_text_accum += _text_chunk
self._live_partial[session_id] = {
"msg_id": stream_text_msg_id,
"text": _stream_text_accum,
"branch_id": session.active_branch_id,
}
await ws_manager.send_to_session(session_id, "agent:stream_delta", {
"session_id": session_id,
"message_id": msg_id,
@@ -2840,7 +3090,9 @@ class AgentManager:
content=_asst_text,
branch_id=session.active_branch_id,
)
session.messages.append(asst_msg)
self._upsert_message(session, asst_msg)
_stream_text_accum = ""
self._live_partial.pop(session_id, None)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id,
"message": asst_msg.model_dump(mode="json"),
@@ -2849,7 +3101,7 @@ class AgentManager:
for i, tu in enumerate(tool_uses):
msg_id = stream_tool_msg_ids_ordered[i] if i < len(stream_tool_msg_ids_ordered) else uuid4().hex
tool_msg = Message(id=msg_id, role="tool_call", content=tu, branch_id=session.active_branch_id)
session.messages.append(tool_msg)
self._upsert_message(session, tool_msg)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id,
"message": tool_msg.model_dump(mode="json"),
@@ -2903,6 +3155,9 @@ class AgentManager:
_pre_out = int(_pre_usage.get("output_tokens", 0) or 0)
if _pre_total_in > 0:
session.tokens["input"] = _pre_total_in
# Pill reads the fresh lane: uncached input only,
# so re-read/cached context doesn't inflate it.
session.tokens["input_fresh"] = _pre_in
if _pre_out > 0:
session.tokens["output"] = _pre_out
except Exception:
@@ -2970,6 +3225,7 @@ class AgentManager:
cache_read = usage.get("cache_read_input_tokens", 0) or 0
total_input = inp + cache_create + cache_read
session.tokens["input"] = total_input
session.tokens["input_fresh"] = inp
session.tokens["output"] = out
cost = getattr(message, "total_cost_usd", None)
@@ -3113,6 +3369,8 @@ class AgentManager:
"message_id": stream_text_msg_id,
})
stream_text_msg_id = None
_stream_text_accum = ""
self._live_partial.pop(session_id, None)
for _tool_msg_id in stream_tool_msg_ids_ordered:
await ws_manager.send_to_session(session_id, "agent:stream_end", {
"session_id": session_id,
@@ -3155,7 +3413,24 @@ class AgentManager:
except Exception:
logger.exception("auto-continuation dispatch failed")
except asyncio.CancelledError:
session.status = "stopped"
# Only act if we're still the session's live task. A user stop pops
# this task (stop_agent already finalized status + partial), and a
# follow-up message may have started a newer turn; either way this
# dying task must NOT clobber the live status or pop the new turn's
# in-flight partial mirror.
if self.tasks.get(session_id) is asyncio.current_task():
session.status = "stopped"
# A cancelled turn desyncs the CLI's resume transcript from
# session.messages (the SDK never recorded the interrupted
# turn), so force the next turn to rebuild history from
# session.messages, else resume/follow-ups replay a transcript
# with no trace of the stopped reply ("nothing to continue").
session.needs_fresh_session = True
# Persist whatever streamed before the cancel (edit / branch
# switch paths; the user-stop path already did this in stop_agent).
await self._commit_partial_now(session)
stream_text_msg_id = None
_stream_text_accum = ""
except Exception as e:
logger.exception(f"Agent {session_id} error: {e}")
session.status = "error"
@@ -3232,10 +3507,29 @@ class AgentManager:
"framework_overhead_tokens": session.framework_overhead_tokens,
"active_mcps_count": len(session.active_mcps),
"messages_count": len(session.messages),
"error_preview": (str(e) or "")[:500],
"error_preview": redact_for_telemetry(str(e), limit=500),
})
except Exception:
logger.debug("submit_diagnostic for context_overflow failed", exc_info=True)
elif p_is_transient_capacity_error(e, extra_text=_stderr_tail):
# A genuine throttle (429/overload/capacity) that already burned
# the whole silent-backoff budget (the only way one reaches here).
# It's a limit, not a failure, so don't append a system-message
# card; emit a transient signal for the muted pill and mark the
# turn completed so it doesn't read as an error.
session.status = "completed"
if stream_text_msg_id:
try:
await ws_manager.send_to_session(session_id, "agent:stream_end", {
"session_id": session_id,
"message_id": stream_text_msg_id,
})
except Exception:
pass
await ws_manager.send_to_session(session_id, "agent:rate_limited", {
"session_id": session_id,
"retry_after_s": parse_retry_after(e, _stderr_tail),
})
elif p_is_free_trial_exhausted(e, extra_text=_stderr_tail):
# Free runs spent. Flip back to own_key and show a friendly
# "connect a model" upsell instead of a raw 402.
@@ -3362,7 +3656,8 @@ class AgentManager:
"model": session.model,
"provider": session.provider,
"connection_mode": getattr(load_settings(), "connection_mode", "own_key"),
"error_preview": (str(e) or "")[:400],
"error_preview": redact_for_telemetry(str(e), limit=400),
"stderr_tail": redact_for_telemetry(_stderr_tail),
})
except Exception:
logger.debug("submit_diagnostic model_error failed", exc_info=True)
@@ -3407,7 +3702,8 @@ class AgentManager:
"model": session.model,
"provider": session.provider,
"connection_mode": getattr(load_settings(), "connection_mode", "own_key"),
"error_preview": (str(e) or "")[:400],
"error_preview": redact_for_telemetry(str(e), limit=400),
"stderr_tail": redact_for_telemetry(_stderr_tail),
})
except Exception:
logger.debug("submit_diagnostic model_error failed", exc_info=True)
@@ -3430,7 +3726,15 @@ class AgentManager:
"message": error_msg.model_dump(mode="json"),
})
finally:
if session_id in self.sessions:
# Only the session's live task finalizes. A stopped task (popped by
# stop_agent, which already finalized status + saved) or one
# superseded by a newer turn must not pop the new turn's partial
# mirror, broadcast a stale terminal status, or overwrite the
# snapshot the live turn is writing.
_is_live_task = self.tasks.get(session_id) is asyncio.current_task()
if _is_live_task:
self._live_partial.pop(session_id, None)
if session_id in self.sessions and _is_live_task:
# For canvas-launched App Builder sessions, the workspace
# folder IS the session_id (see launch_agent), so meta.json
# lives at outputs_workspace/<session_id>/meta.json. Read it
@@ -3616,6 +3920,7 @@ class AgentManager:
hidden: bool = False,
selected_browser_ids: list[str] | None = None,
selected_app_output_ids: list[str] | None = None,
selected_setting_ids: list[str] | None = None,
client_message_id: str | None = None,
prepend_context: str | None = None,
):
@@ -3752,7 +4057,7 @@ class AgentManager:
if fast_verdict != "no":
task = asyncio.create_task(self._run_browser_fast_path(session_id, model_prompt, selected_browser_ids, fast_brief, fast_verdict))
else:
task = asyncio.create_task(self._run_agent_loop(session_id, model_prompt, images=images, context_paths=context_paths, forced_tools=forced_tools, attached_skills=attached_skills, selected_browser_ids=selected_browser_ids, selected_app_output_ids=selected_app_output_ids))
task = asyncio.create_task(self._run_agent_loop(session_id, model_prompt, images=images, context_paths=context_paths, forced_tools=forced_tools, attached_skills=attached_skills, selected_browser_ids=selected_browser_ids, selected_app_output_ids=selected_app_output_ids, selected_setting_ids=selected_setting_ids))
self.tasks[session_id] = task
async def _run_browser_fast_path(self, session_id: str, prompt: str, selected_browser_ids: list[str] | None, brief: str = "", verdict: str = "act"):
@@ -3909,21 +4214,88 @@ class AgentManager:
session.pending_approvals = []
session.status = "stopped"
session.needs_fresh_session = True
if not session.closed_at:
session.closed_at = datetime.now()
# Persist the partial reply NOW, before tearing down the SDK. The
# cancel handler also does this, but it sits behind the generator's
# teardown, which can take several seconds; doing it here means the
# streamed text stays put the instant Stop is pressed instead of
# blinking out and reappearing once teardown finishes.
await self._commit_partial_now(session)
await ws_manager.send_to_session(session_id, "agent:status", {
"session_id": session_id,
"status": "stopped",
"session": session.model_dump(mode="json"),
})
# Snapshot now: the cancelled task's finally skips the save (it's no
# longer the live task once we pop it below), so persist the partial
# here or it'd live only in memory until the next turn / shutdown.
try:
_save_session(session_id, session.model_dump(mode="json"))
except Exception:
pass
task = self.tasks.get(session_id)
# Drop the task from the registry immediately so a follow-up message
# isn't rejected as "still running" while the cancelled task slowly
# tears down (that window was eating user messages). Drain it in the
# background; we've already captured the partial above.
task = self.tasks.pop(session_id, None)
if task and not task.done():
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
asyncio.create_task(self._drain_task(task))
async def _commit_partial_now(self, session) -> bool:
"""Persist the in-flight streamed assistant text as a real message and
push it to the client, idempotently. Lets a stop show the partial
instantly instead of waiting out the SDK teardown the cancel handler
sits behind. Returns True if it committed something."""
live = self._live_partial.pop(session.id, None)
if not live:
return False
text = live.get("text") or ""
msg_id = live.get("msg_id")
if not msg_id or not text.strip():
return False
if any(getattr(m, "id", None) == msg_id for m in session.messages):
return False
partial = Message(
id=msg_id,
role="assistant",
content=text,
branch_id=live.get("branch_id") or session.active_branch_id,
)
self._upsert_message(session, partial)
try:
await ws_manager.send_to_session(session.id, "agent:message", {
"session_id": session.id,
"message": partial.model_dump(mode="json"),
})
await ws_manager.send_to_session(session.id, "agent:stream_end", {
"session_id": session.id,
"message_id": msg_id,
})
except Exception:
pass
return True
async def _drain_task(self, task) -> None:
"""Await a cancelled task's (possibly slow) teardown off the hot path."""
try:
await task
except (asyncio.CancelledError, Exception):
pass
def _upsert_message(self, session, msg) -> None:
"""Append msg, or replace it in place if its id is already present.
Makes a duplicate-id row unrepresentable when a stream commit races a
stop's early partial commit (both carry the same stream message id).
Same pattern the consolidated-thinking pill already uses inline."""
for i, existing in enumerate(session.messages):
if getattr(existing, "id", None) == msg.id:
session.messages[i] = msg
return
session.messages.append(msg)
def handle_approval(self, request_id: str, decision: dict):
"""Resolve a pending HITL approval."""
@@ -4412,9 +4784,19 @@ class AgentManager:
"dashboard_id": session.dashboard_id,
})
self._purge_session_memory(session_id)
logger.info(f"Session {session_id} closed and persisted")
def _purge_session_memory(self, session_id: str) -> None:
"""Drop a session from EVERY in-memory structure keyed by its id, so a
close or delete can't strand stale per-session state that lives until
the process dies. One chokepoint on purpose: a new per-session cache
wires its eviction in HERE and both removal paths get it for free."""
self.sessions.pop(session_id, None)
self.tasks.pop(session_id, None)
logger.info(f"Session {session_id} closed and persisted")
self._live_partial.pop(session_id, None)
p_view_builder_render_retry_counts.pop(session_id, None)
p_view_builder_dirty_sessions.discard(session_id)
async def delete_session(self, session_id: str) -> None:
"""Permanently delete a session: remove from memory and JSON file.
@@ -4434,8 +4816,7 @@ class AgentManager:
except asyncio.CancelledError:
pass
self.sessions.pop(session_id, None)
self.tasks.pop(session_id, None)
self._purge_session_memory(session_id)
_delete_session_file(session_id)
logger.info(f"Session {session_id} permanently deleted")
@@ -4774,9 +5155,44 @@ class AgentManager:
}
def get_all_sessions(self, dashboard_id: str | None = None) -> list[AgentSession]:
if dashboard_id:
return [s for s in self.sessions.values() if s.dashboard_id == dashboard_id]
return list(self.sessions.values())
if not dashboard_id:
return list(self.sessions.values())
# Memory first, then promote on-disk sessions for this dashboard, but
# ONLY ones the dashboard's layout still has a card for. A session keeps
# its dashboard_id when its card is deleted, so promoting by tag alone
# resurrected deleted chats on every reopen; the layout's cards are the
# real source of truth for what's on the board. Imported sessions ARE in
# the layout, so they still surface, and this bounds the disk read to
# once per session per run, like resume_session.
result = [s for s in self.sessions.values() if s.dashboard_id == dashboard_id]
seen = {s.id for s in result}
card_ids = self._dashboard_card_ids(dashboard_id)
for sid, data in _load_all_session_data():
if sid in seen or sid not in card_ids:
continue
if data.get("dashboard_id") != dashboard_id:
continue
try:
sess = AgentSession(**data)
except Exception:
logger.warning(f"get_all_sessions: skipping unloadable session {sid}", exc_info=True)
continue
_apply_context_window(sess)
self.sessions[sid] = sess
result.append(sess)
return result
def _dashboard_card_ids(self, dashboard_id: str) -> set[str]:
"""Session ids the dashboard's layout currently has agent cards for.
Read straight off disk (no dashboards-module import, avoids a cycle)."""
try:
import os
import backend.config.paths as _paths
from backend.config.json_store import read_json_or_none
d = read_json_or_none(os.path.join(_paths.DASHBOARDS_DIR, f"{dashboard_id}.json")) or {}
return set((d.get("layout", {}).get("cards") or {}).keys())
except Exception:
return set()
def get_session(self, session_id: str) -> Optional[AgentSession]:
return self.sessions.get(session_id)
backend/apps/agents/agents.py
+2
View File
@@ -112,6 +112,7 @@ async def send_message(session_id: str, body: dict):
hidden=body.get("hidden", False),
selected_browser_ids=body.get("selected_browser_ids"),
selected_app_output_ids=body.get("selected_app_output_ids"),
selected_setting_ids=body.get("selected_setting_ids"),
client_message_id=body.get("client_message_id"),
)
return {"ok": True}
@@ -128,6 +129,7 @@ async def handle_approval(response: ApprovalResponse):
"message": response.message,
"updated_input": response.updated_input,
"trust_pattern": response.trust_pattern,
"set_always_allow": response.set_always_allow,
})
return {"ok": True}
backend/apps/agents/browser/browser_agent.py
+27
View File
@@ -941,6 +941,7 @@ async def run_browser_agent(
done_called = False
done_message = ""
done_success = True
done_keep_open = False
# Completion detection: once an irreversible SEND has confirmed, the goal is
# met. The model otherwise stalls re-verifying what the confirm already proved
# (measured: send done at turn ~11, then ~12 wasted perception turns). We drive
@@ -1013,6 +1014,9 @@ async def run_browser_agent(
_in = response.usage.input_tokens or 0
out_tokens_total += _out
session.tokens["input"] = session.tokens.get("input", 0) + _in
# Already-uncached here (cache tracked separately below), so the
# fresh lane that feeds the parent's pill mirrors it 1:1.
session.tokens["input_fresh"] = session.tokens.get("input_fresh", 0) + _in
session.tokens["output"] = session.tokens.get("output", 0) + _out
_cr = getattr(response.usage, "cache_read_input_tokens", 0) or 0
_cw = getattr(response.usage, "cache_creation_input_tokens", 0) or 0
@@ -1427,6 +1431,7 @@ async def run_browser_agent(
done_called = True
done_message = (tu.input.get("message") or "").strip()
done_success = tu.input.get("success", True) is not False
done_keep_open = tu.input.get("keep_open", False) is True
tool_results.append({
"type": "tool_result", "tool_use_id": tu.id,
"content": [{"type": "text", "text": "ok"}],
@@ -2122,6 +2127,28 @@ async def run_browser_agent(
})
except Exception as e:
logger.debug(f"[browser-playbook] distill skipped: {e}")
# The model asked to leave the browser open because the deliverable lives
# on the page (a video playing, a page to read). Pin the card so the
# auto-close on parent finish skips it. Only on honest success: never pin
# a broken or ghost run open. The keep broadcast lands before the parent
# reaches terminal state (it awaits this run), so the frontend has the
# flag set before any close path runs.
if honest and done_keep_open and dashboard_id:
try:
from backend.apps.dashboards.dashboards import _load, _save
dashboard = _load(dashboard_id)
card = dashboard.layout.browser_cards.get(browser_id)
if card is not None:
card.keep_open = True
dashboard.updated_at = datetime.now()
_save(dashboard)
await ws_manager.broadcast_global("dashboard:browser_card_keep", {
"dashboard_id": dashboard_id,
"browser_id": browser_id,
})
except Exception as e:
logger.warning(f"[browser-agent {session_id}] keep_open persist failed: {e}")
agent_manager._sync_session_close(session)
await ws_manager.send_to_session(session_id, "agent:status", {
"session_id": session_id,
backend/apps/agents/browser/browser_schema.py
+16 -3
View File
@@ -117,6 +117,17 @@ BROWSER_TOOLS_SCHEMA = [
"(login wall, missing info, something blocked you). Default true."
),
},
"keep_open": {
"type": "boolean",
"description": (
"Set true ONLY when the result IS the open page and the user will keep "
"using it right now: a video or audio playing, a page you opened for them "
"to read or watch, a download you started, or a place left ready for them "
"to take over. The browser then stays put instead of closing. Leave false "
"(default) for info tasks where you just look something up and report the "
"answer back, since there's nothing left to keep on screen."
),
},
},
"required": ["message"],
},
@@ -885,9 +896,11 @@ SYSTEM_PROMPT = (
"tool, never by typing a sentence. Put your reply to the user in Done's `message`, "
"written like a normal chat reply: what got done plus the human proof (the name, the "
"time, what's now on screen), in one or two plain sentences with zero interface words. "
"Set `success` false if you couldn't finish. For irreversible actions, only report "
"success with real proof you actually observed (the name and where/when you saw it), "
"just phrased for a person, not for a machine."
"Set `success` false if you couldn't finish. Set `keep_open` true when the result is the "
"open page itself and the user keeps using it now (a video playing, a page opened to "
"read, a download started), so the browser stays instead of closing. For irreversible "
"actions, only report success with real proof you actually observed (the name and "
"where/when you saw it), just phrased for a person, not for a machine."
)
MAX_TURNS = 40
backend/apps/agents/core/error_classify.py
+78
View File
@@ -1,5 +1,30 @@
import re
# Secret shapes that must never ride along when we ship a stderr tail or an
# error string to telemetry. own_key mode means the subprocess stderr can echo
# the user's OWN provider key, so this scrub is the wall between a diagnostic
# and a key leak; over-redacting is fine, leaking is not.
_TELEMETRY_SECRET_PATTERNS = (
re.compile(r"sk-ant-[A-Za-z0-9_\-]{12,}"),
re.compile(r"sk-[A-Za-z0-9_\-]{16,}"),
re.compile(r"AIza[A-Za-z0-9_\-]{20,}"),
re.compile(r"gh[pousr]_[A-Za-z0-9]{20,}"),
re.compile(r"(?i)bearer\s+[A-Za-z0-9._\-]{12,}"),
re.compile(r"(?i)\b(?:api[_-]?key|access[_-]?token|refresh[_-]?token|secret|password|authorization)\b[\"']?\s*[:=]\s*[\"']?[A-Za-z0-9._\-]{6,}"),
)
def redact_for_telemetry(text: str, *, limit: int = 2000) -> str:
"""Scrub secret-shaped substrings, then keep the tail (where the real error
lands), bounded so a runaway log can't bloat the payload. Every raw
error/stderr string goes through here before it leaves the machine."""
if not text:
return ""
for pat in _TELEMETRY_SECRET_PATTERNS:
text = pat.sub("[redacted]", text)
return text[-limit:]
# Patterns that indicate an upstream transient problem (overload / rate limit /
# infra blip), safe to silently retry with backoff. Checked against the
# stringified exception from claude_agent_sdk / Claude CLI.
@@ -12,10 +37,30 @@ _TRANSIENT_CAPACITY_PATTERNS = re.compile(
r"|internal\s+server\s+error"
r"|rate[_\s-]?limit(?:_error)?"
r"|ECONNRESET|ETIMEDOUT|ENETUNREACH|fetch\s+failed"
r"|resource[_\s-]?exhausted"
r"|upstream\s+connect\s+error)",
re.IGNORECASE,
)
# A first message ships the full tool schema; 9Router rewrites Anthropic
# tools[].input_schema into Gemini function_declarations / OpenAI params, and a
# construct it can't translate makes the provider 400 (INVALID_ARGUMENT) with
# zero tokens. That is NOT auth, reconnecting won't help, the request shape is
# wrong, so we classify it apart and stop the catch-all from showing a
# "reconnect your subscription" card for a tool-schema 400.
_TRANSLATION_ERROR_PATTERNS = re.compile(
r"(?:function_declarations"
r"|invalid_argument"
r"|invalid\s+json\s+payload"
r"|unknown\s+name\b"
r"|cannot\s+find\s+field"
r"|proto\s+field"
r"|input_schema"
r"|\btools\[\d+\]"
r")",
re.IGNORECASE,
)
# Patterns that look rate-limit-ish but are actually non-transient (user quota,
# auth, context-window tier gate). Must NOT retry, upgrading, reauthing, or
# trimming context is required. The long-context-required variant is what
@@ -87,6 +132,17 @@ def p_is_out_of_tokens(exc: BaseException, extra_text: str = "") -> bool:
))
def p_is_translation_error(exc: BaseException, extra_text: str = "") -> bool:
"""True when the upstream 400 is a tool-schema / protocol translation
failure (9Router rewriting Anthropic tools into Gemini function_declarations
or OpenAI params), not auth or capacity. Kept distinct so the catch-all
stops mislabeling a schema 400 as an expired-subscription reconnect card."""
combined = f"{exc!s}\n{extra_text}".strip()
if not combined:
return False
return bool(_TRANSLATION_ERROR_PATTERNS.search(combined))
def p_extract_reset_hint(text: str) -> str:
"""Pull a human reset phrase ('at 7:42 AM', 'in 2h 30m', 'after 1m 59s') out of
a provider usage error so we can tell the user when their limit comes back.
@@ -109,6 +165,10 @@ def p_is_auth_error(exc: BaseException, extra_text: str = "") -> bool:
combined = f"{exc!s}\n{extra_text}".strip()
if not combined:
return False
# A tool-schema translation 400 can carry provider/connection wording that
# trips the auth regex below; it isn't auth, so don't claim it is.
if p_is_translation_error(exc, extra_text):
return False
return bool(re.search(
r"\b(401|403)\b"
r"|invalid\s+authentication\s+credentials"
@@ -142,6 +202,24 @@ def p_is_unknown_model_error(exc: BaseException, extra_text: str = "") -> bool:
))
def parse_retry_after(exc: BaseException, extra_text: str = "") -> int | None:
"""Best-effort seconds-until-retry pulled from a throttle error; None if the
upstream didn't say. Only used to label the rate-limit pill, so a miss just
means the pill shows no countdown, never anything load-bearing."""
combined = f"{exc!s}\n{extra_text}"
# "1m 59s" / "2m" / "45s" (reset-window phrasing Codex/Anthropic use).
m = re.search(r"\b(?:(\d{1,2})\s*m(?:in)?)?\s*(\d{1,3})\s*s(?:ec)?\b", combined, re.IGNORECASE)
if m and (m.group(1) or m.group(2)):
return int(m.group(1) or 0) * 60 + int(m.group(2) or 0)
# "retry-after: 30" / "try again in 2 minutes".
m = re.search(r"(?:retry[-\s]?after|try\s+again\s+in)\D{0,8}(\d{1,4})\s*(m|min|minute|s|sec|second)?", combined, re.IGNORECASE)
if m:
n = int(m.group(1))
unit = (m.group(2) or "s").lower()
return n * 60 if unit.startswith("m") else n
return None
def p_is_transient_capacity_error(exc: BaseException, extra_text: str = "") -> bool:
# The Claude CLI's underlying ProcessError stringifies to a generic
# "Command failed with exit code 1 / Check stderr output for details";
backend/apps/agents/core/mcp_preflight.py
+31 -3
View File
@@ -12,6 +12,7 @@ from backend.apps.agents.providers.registry import resolve_aux_model
from backend.apps.settings.credentials import get_anthropic_client_for_model
from backend.apps.settings.settings import load_settings
from backend.apps.tools_lib.tools_lib import _load_all as load_all_tools
from backend.apps.tools_lib.mcp_config import _sanitize_server_name
logger = logging.getLogger(__name__)
@@ -55,6 +56,11 @@ CURATED_SHORTLIST: list[CuratedEntry] = [
"title": "Airtable",
"description": "Read and write records, manage bases, tables, and fields in the user's Airtable.",
},
{
"id": "GitHub",
"title": "GitHub",
"description": "Repos, issues, pull requests, Actions, code search, gists; when the task involves the user's GitHub.",
},
{
"id": "Reddit",
"title": "Reddit",
@@ -85,8 +91,10 @@ def _is_obviously_local(prompt: str) -> bool:
return False
async def run_preflight(prompt: str, timeout_s: float = 2.0, task_id: str | None = None) -> dict:
"""Classify the prompt and return {is_vague, suggestions}; never raises."""
async def run_preflight(prompt: str, timeout_s: float = 8.0, task_id: str | None = None, require_vague: bool = True) -> dict:
"""Classify the prompt and return {is_vague, suggestions}; never raises. require_vague=False
keeps suggestions even on a concrete prompt: used when the agent already proved it needs an
integration (it called MCPSearch), so the "don't interrupt concrete tasks" guard no longer applies."""
default: dict[str, Any] = {"is_vague": False, "suggestions": []}
if not prompt or not prompt.strip():
@@ -112,7 +120,7 @@ async def run_preflight(prompt: str, timeout_s: float = 2.0, task_id: str | None
result["suggestions"] = [s for s in result["suggestions"] if s is not None]
result["is_vague"] = bool(result.get("is_vague"))
# Suppress on concrete prompts; false-positives feel broken (interrupting "refactor foo.ts" to suggest GitHub MCP).
if not result["is_vague"]:
if require_vague and not result["is_vague"]:
result["suggestions"] = []
return result
except asyncio.TimeoutError:
@@ -138,6 +146,26 @@ def _build_available_shortlist(settings) -> list[CuratedEntry]:
]
def offer_for_gated_server(server_name: str, settings) -> CuratedEntry | None:
"""Mid-run a running agent may reach for a vetted MCP it isn't granted; this maps that
server to a one-click connect offer to SHOW the user. Suggest-only by construction: it
returns data to display, never an action that grants access, so it cannot widen the MCP
surface (activation stays behind MCPActivate + the dispatch gate). Returns None unless the
server is vetted AND inactive AND not dismissed, reusing the same filter as the preflight."""
if not server_name or not isinstance(server_name, str):
return None
# The hot-path hands us a sanitized slug ("google-workspace"); curated ids are display names
# ("Google Workspace"). Match on the slug of both sides so neither form is a load-bearing string.
slug = _sanitize_server_name(server_name)
entry = next(
(e for e in _build_available_shortlist(settings) if _sanitize_server_name(e["id"]) == slug),
None,
)
if entry is None:
return None
return {"id": entry["id"], "title": entry["title"], "description": entry["description"], "reason": ""}
def _decorate(llm_suggestion: dict, available: list[CuratedEntry]) -> dict | None:
"""Expand an LLM-returned {id, reason} into the full frontend shape."""
entry = next((e for e in available if e["id"] == llm_suggestion["id"]), None)
backend/apps/agents/core/models.py
+4
View File
@@ -42,6 +42,10 @@ class ApprovalResponse(BaseModel):
# (from ApprovalRequest.sensitive_pattern) to disk so future writes
# against the same pattern skip the modal.
trust_pattern: bool = False
# "Always approve" button: persist this tool's policy to always_allow so
# the same tool stops prompting (the catastrophic/sensitive guards still
# fire, so this can't blanket-approve an rm -rf or a sensitive-path write).
set_always_allow: bool = False
class Message(BaseModel):
id: str = Field(default_factory=lambda: uuid4().hex)
backend/apps/agents/manager/prompt/prompt_context.py
+86 -3
View File
@@ -98,6 +98,35 @@ def _build_connected_tools_context(allowed_tools: list[str], get_all_tool_names:
)
# A run of this many ToolSearch calls with no other tool between them is the
# "looping on ToolSearch" wedge: the model hunts for a gated MCP server's tools,
# which ToolSearch can never see, gets empty results, and retries. Two free
# calls (a power user with many activated MCPs may legitimately ToolSearch to
# load a deferred tool); redirect on the third.
TOOLSEARCH_LOOP_THRESHOLD = 3
def toolsearch_loop_redirect(consecutive_toolsearch: int, gated_servers: list[str]) -> str | None:
"""The feedback to hand a model that's stuck calling ToolSearch in a row.
None until it crosses the threshold; then a steer toward MCPActivate (the
only path to a gated server) plus a reminder its other tools are already
loaded. Pure so the loop-break boundary is unit-testable."""
if consecutive_toolsearch < TOOLSEARCH_LOOP_THRESHOLD:
return None
reason = (
"ToolSearch can't load anything here, every tool you can use is already "
"active and callable by name, so there's nothing to search for. "
)
if gated_servers:
reason += (
"If you need an app you don't see yet (email, calendar, drive, etc.), "
"it's gated: call MCPActivate(server_name) with one of these and its "
f"tools become callable next turn: {', '.join(gated_servers)}. "
)
reason += "Stop calling ToolSearch."
return reason
def _build_browser_context(dashboard_id: str | None, selected_browser_ids: list[str] | None = None) -> str | None:
"""Build a context block listing browser cards and delegation instructions.
@@ -233,6 +262,28 @@ def _build_selected_app_context(selected_app_output_ids: list[str] | None) -> st
)
def _build_selected_settings_context(selected_setting_ids: list[str] | None) -> str | None:
"""Context block when the user points the agent at specific Settings rows.
A targeting aid, NOT a gate: the settings tools (SettingsRead/SettingsWrite)
are always available regardless. This just focuses the agent on the exact
fields the user clicked. Ids are AppSettings field names (e.g. 'theme',
'default_model'), so no label map to drift out of date."""
ids = [s for s in (selected_setting_ids or []) if s]
if not ids:
return None
bullets = "\n".join(f"- {fid}" for fid in ids)
return (
"<selected_settings>\n"
"The user pointed you at these specific OpenSwarm Settings fields. Focus "
"on them: call SettingsRead to see their current values, then "
"SettingsWrite to change what the user asked for. Leave unrelated "
"settings alone.\n"
f"{bullets}\n"
"</selected_settings>"
)
def _build_mcp_registry_summary(allowed_tools: list[str], active_mcps: list[str], get_all_tool_names: Callable[[], list[str]]) -> str | None:
"""Compact registry of installed MCP servers, one line per server.
@@ -304,6 +355,12 @@ def _build_mcp_registry_summary(allowed_tools: list[str], active_mcps: list[str]
"Calendar/Drive, the equivalent OpenSwarm server is listed below; "
"activate that one via MCPActivate instead."
)
sections.append(
"1b. The native `ToolSearch` tool CANNOT see these servers, they're "
"hidden from it until activated, so searching for them returns nothing "
"and just burns turns. Never ToolSearch for an app/integration; go "
"straight to MCPActivate."
)
sections.append(
"2. After MCPActivate returns, end the turn, a follow-up turn fires "
"automatically with the new tools available."
@@ -402,13 +459,39 @@ def _resolve_forced_tools(forced_tools: list[str] | None) -> str:
def _resolve_attached_skills(attached_skills: list | None) -> str:
"""Build a context block injecting attached skill content into the prompt."""
"""Build a context block injecting attached skill content into the prompt.
For a multi-file (folder) skill we inject the SKILL.md body as text AND point
the agent at the folder so it can read supporting files (scripts, templates)
on demand with the normal Read/Glob/Bash tools. That keeps skills fully
provider-agnostic: plain prompt text plus universal file tools, identical on
Claude, OpenAI, Gemini, or any custom model routed through 9router. The
folder lookup is resolved backend-side from the skill id so the frontend
send payload stays a simple {id, name, content}."""
if not attached_skills:
return ""
folder_by_id: dict[str, str] = {}
try:
from backend.apps.skills.skills import _sync_skills
for s in _sync_skills():
if s.dir_path and s.has_supporting_files:
folder_by_id[s.id] = s.dir_path
except Exception:
folder_by_id = {}
sections = []
for skill in attached_skills:
name = skill.get("name", "Unknown")
content = skill.get("content", "")
if content:
sections.append(f"[Using skill: {name}]\n\n{content}")
if not content:
continue
block = f"[Using skill: {name}]\n\n{content}"
folder = folder_by_id.get(skill.get("id", ""))
if folder:
block += (
f"\n\nThis skill bundles supporting files in {folder}. "
"Read them with your normal file tools (Read / Glob / Bash) when "
"the steps above call for one; don't guess their contents."
)
sections.append(block)
return "\n\n".join(sections)
backend/apps/agents/providers/registry.py
+3 -1
View File
@@ -312,8 +312,10 @@ async def resolve_aux_model(
paying for (Codex chat → Codex aux, OR chat → OR aux, etc.).
Returns (model_id, base_url); base_url=None means default Anthropic.
"""
# Must track the canonical Anthropic entries in BUILTIN_MODELS (sonnet/haiku); a stale id here
# 404s every aux call (sonnet was pinned to the long-dead 4.0 "20250514" and silently broke).
haiku_bare = "claude-haiku-4-5-20251001"
sonnet_bare = "claude-sonnet-4-20250514"
sonnet_bare = "claude-sonnet-4-6"
or_haiku = "openrouter/anthropic/claude-haiku-4.5"
or_sonnet = "openrouter/anthropic/claude-sonnet-4.5"
bare = haiku_bare if preferred_tier == "haiku" else sonnet_bare
backend/apps/agents/session_credential.py
+198
View File
@@ -0,0 +1,198 @@
"""Which credential keeps a live agent session alive, as one typed value.
The settings-meta tool lets an agent edit its own Settings autonomously. The
single hard rule is "no suicide": it must never disconnect the credential that
powers its own run. We enforce that structurally, not with a scattered if-check,
by resolving the powering credential to a small closed value HERE, in one place,
and having the write guard key off it.
Add a provider lane and you add a case here; the exhaustive enumeration in
test_settings_meta_guard.py walks every (provider x route x connection_mode)
combo and fails until the new lane is classified, so a wrong/forgotten state
can't ship silently.
Honest scope: only API keys live in writable settings fields, so they're the
only credential the guard can be asked to protect. Subscriptions (OpenSwarm
Pro/free-trial, and the 9router OAuth lanes for Claude/Codex/Gemini) are either
server-owned or live entirely outside settings.json, so the settings-meta tool
cannot touch them at all, a stronger protection than the guard itself.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Literal, TYPE_CHECKING
from backend.apps.agents.providers.registry import (
_CUSTOM_VALUE_PREFIX,
_custom_provider_slug_for_lookup,
_find_builtin_model,
_find_custom_provider_for_value,
get_api_type,
)
if TYPE_CHECKING:
from backend.apps.settings.models import AppSettings
# AppSettings fields holding a user-writable API key, keyed by provider api-type.
# Blanking whichever of these powers the current run is the one suicide the guard
# stops. Anything not here (subscription tokens, bearers) is not settings-writable.
_API_KEY_FIELD_BY_API: dict[str, str] = {
"anthropic": "anthropic_api_key",
"openai": "openai_api_key",
"codex": "openai_api_key",
"gemini": "google_api_key",
"openrouter": "openrouter_api_key",
}
# Every settings field that can hold an API key (the full guarded set). Custom
# providers keep their keys inside the custom_providers list, guarded separately.
ALL_API_KEY_FIELDS: frozenset[str] = frozenset(_API_KEY_FIELD_BY_API.values())
CredentialKind = Literal["api_key", "subscription", "unknown"]
@dataclass(frozen=True)
class PoweringCredential:
"""The credential keeping THIS run alive, resolved to a closed value.
kind=="api_key" -> protected_field (or custom slug) names exactly what
the guard must keep alive.
kind=="subscription" -> the live credential isn't a settings field at all
(Pro/free-trial/9router OAuth), so no api-key field
needs guarding; clearing OTHER keys stays allowed.
kind=="unknown" -> we couldn't classify the run; fail safe by treating
ALL credential fields as protected.
"""
kind: CredentialKind
provider: str
protected_field: str | None = None
protected_custom_slug: str | None = None
label: str = ""
def _custom_slug_for_model(model_value: str, settings: AppSettings) -> str | None:
cp = _find_custom_provider_for_value(settings, model_value)
if cp is not None:
return _custom_provider_slug_for_lookup(getattr(cp, "name", ""))
# Fall back to the slug encoded in the picker value itself.
if isinstance(model_value, str) and model_value.startswith(_CUSTOM_VALUE_PREFIX):
slug = model_value[len(_CUSTOM_VALUE_PREFIX):].partition("/")[0]
return slug or None
return None
def resolve_powering_credential(model_value: str, settings: AppSettings) -> PoweringCredential:
"""Resolve the credential powering a run on `model_value` to a typed value.
`model_value` is the session's short model name (e.g. "opus-4-8", "sonnet-api",
"custom/lmstudio/llama"), exactly what AgentSession.model holds.
"""
entry = _find_builtin_model(model_value)
api = (entry or {}).get("api") or get_api_type(model_value)
route = (entry or {}).get("route")
mode = getattr(settings, "connection_mode", "own_key")
# Custom provider (LM Studio, Ollama, Together, ...). Local servers use a
# placeholder key, so suicide is removing the provider ENTRY, not blanking
# its key; the guard keys off the slug.
if api == "custom":
slug = _custom_slug_for_model(model_value, settings)
return PoweringCredential(
kind="api_key", provider="custom",
protected_custom_slug=slug,
label=f"custom provider '{slug}'" if slug else "custom provider",
)
# Explicit API-key route: the matching *_api_key field is the live one.
if route == "api":
field = _API_KEY_FIELD_BY_API.get(api)
if field:
return PoweringCredential(kind="api_key", provider=api, protected_field=field,
label=f"{field} (powers this run)")
return PoweringCredential(kind="unknown", provider=api,
label=f"{api} api route (unclassified)")
# Subscription-only routes (cx/ Codex, gc/ Gemini CLI) and pinned cc/ Claude:
# these lanes live in 9router, never in settings.
if route == "cc" or (entry or {}).get("subscription_only"):
return PoweringCredential(kind="subscription", provider=api,
label=f"{api} subscription")
# OpenRouter (its own `openrouter` route, plus xai/meta/deepseek/etc routed
# through it): always an API key, never a subscription.
if api == "openrouter":
return PoweringCredential(kind="api_key", provider="openrouter",
protected_field="openrouter_api_key",
label="OpenRouter API key (powers this run)")
# Default Anthropic rows (route is None): connection_mode picks the lane.
if api == "anthropic":
if mode in ("openswarm-pro", "free-trial"):
label = "OpenSwarm Pro" if mode == "openswarm-pro" else "OpenSwarm free trial"
return PoweringCredential(kind="subscription", provider="anthropic", label=label)
if getattr(settings, "anthropic_api_key", None):
return PoweringCredential(kind="api_key", provider="anthropic",
protected_field="anthropic_api_key",
label="Anthropic API key (powers this run)")
# No key, no proxy mode -> the user's Claude subscription via 9router.
return PoweringCredential(kind="subscription", provider="anthropic",
label="Claude subscription")
# Default Gemini rows (api gemini-cli, route None): the AG/gc OAuth lane is a
# subscription. A bare AI Studio key only powers the explicit -api rows above.
if api in ("gemini", "gemini-cli"):
return PoweringCredential(kind="subscription", provider="gemini",
label="Gemini subscription")
# Anything we can't place: protect everything (fail safe), never fail open.
return PoweringCredential(kind="unknown", provider=api or "unknown",
label=f"{api or 'unknown'} provider (unclassified)")
def _is_blank(value: Any) -> bool:
"""A credential write that removes the credential: None, "", or whitespace."""
if value is None:
return True
if isinstance(value, str):
return value.strip() == ""
return False
def _powering_custom_slug_present(new_providers: Any, slug: str) -> bool:
"""True if the powering custom provider's entry still exists after the write."""
if not isinstance(new_providers, list):
return False
for cp in new_providers:
name = cp.get("name") if isinstance(cp, dict) else getattr(cp, "name", None)
if name and _custom_provider_slug_for_lookup(name) == slug:
return True
return False
def write_would_suicide(field: str, new_value: Any, powering: PoweringCredential) -> bool:
"""True if writing `new_value` to `field` would disconnect the live credential.
Pure and total: every (field, value, powering) maps to a definite yes/no, so
the guard can't be tricked by an unhandled path. Only blanking/removing a
credential counts; SETTING a fresh key is a (re)connect, never suicide.
"""
if field == "custom_providers":
# Removing the entry that powers a custom-provider run is suicide; a
# local provider's placeholder key being blanked is not. When the run is
# unknown, any custom run could be the live one, so refuse a vanish.
if powering.kind == "api_key" and powering.provider == "custom" and powering.protected_custom_slug:
return not _powering_custom_slug_present(new_value, powering.protected_custom_slug)
if powering.kind == "unknown":
return not _powering_custom_slug_present(new_value, powering.protected_custom_slug or "")
return False
if field in ALL_API_KEY_FIELDS:
if not _is_blank(new_value):
return False
if powering.kind == "unknown":
return True
return powering.kind == "api_key" and field == powering.protected_field
return False
backend/apps/agents/settings_meta_server.py
+185
View File
@@ -0,0 +1,185 @@
#!/usr/bin/env python3
"""Stdio MCP server letting an agent read and edit its own OpenSwarm Settings.
Two tools, SettingsRead and SettingsWrite, backed by /api/settings-meta. Always
on, no activation gate (Settings is the agent's own house). The backend enforces
the only hard rule: it can change anything EXCEPT disconnect the credential
powering its own run ("no suicide"), and reads come back with secrets redacted
to configured/not, never the value. Both guards live server-side so this thin
client can't weaken them."""
import json
import os
import sys
import urllib.error
import urllib.request
BACKEND_PORT = os.environ.get("OPENSWARM_PORT", "8324")
BACKEND_AUTH = os.environ.get("OPENSWARM_AUTH_TOKEN", "")
BACKEND_URL = f"http://127.0.0.1:{BACKEND_PORT}/api/settings-meta"
PARENT_SESSION_ID = os.environ.get("OPENSWARM_PARENT_SESSION_ID", "")
TOOLS = [
{
"name": "SettingsRead",
"description": (
"Read the user's OpenSwarm Settings (model defaults, theme, prompts, "
"connected providers, toggles). Secrets come back as configured/not, "
"never the actual key. Call this before SettingsWrite so you change "
"the right field to the right value."
),
"inputSchema": {
"type": "object",
"properties": {},
"additionalProperties": False,
},
},
{
"name": "SettingsWrite",
"description": (
"Change one or more OpenSwarm Settings. Pass `changes` as a map of "
"setting field name to new value (use the exact field names from "
"SettingsRead, e.g. {\"theme\": \"light\", \"default_model\": \"opus-4-8\"}). "
"You can set or clear API keys too. Two things you cannot do: clear the "
"credential currently powering YOU (it's refused so you don't cut your "
"own run off), and touch subscription/connection state (managed by the "
"Subscription section; tell the user to use it). The result reports each "
"field as applied / refused / unknown, so relay what actually changed."
),
"inputSchema": {
"type": "object",
"properties": {
"changes": {
"type": "object",
"description": "Field name -> new value. e.g. {\"theme\": \"light\"}.",
"additionalProperties": True,
},
},
"required": ["changes"],
"additionalProperties": False,
},
},
]
def send_response(id_, result=None, error=None):
msg = {"jsonrpc": "2.0", "id": id_}
if error is not None:
msg["error"] = error
else:
msg["result"] = result
sys.stdout.write(json.dumps(msg) + "\n")
sys.stdout.flush()
def call_backend(action: str, payload: dict) -> dict:
full = {**payload, "parent_session_id": PARENT_SESSION_ID}
body = json.dumps(full).encode()
headers = {"Content-Type": "application/json"}
if BACKEND_AUTH:
headers["Authorization"] = f"Bearer {BACKEND_AUTH}"
req = urllib.request.Request(
f"{BACKEND_URL}/{action}", data=body, headers=headers, method="POST"
)
try:
with urllib.request.urlopen(req, timeout=60) as resp:
return json.loads(resp.read().decode())
except urllib.error.HTTPError as e:
detail = e.read().decode() if e.fp else str(e)
return {"error": f"HTTP {e.code}: {detail}"}
except Exception as e:
return {"error": str(e)}
def _format_read(settings: dict) -> str:
"""Render redacted settings compactly so the model spends tokens on the
values it can act on, not on JSON punctuation."""
lines = ["Current OpenSwarm Settings (secrets shown as configured/not):"]
for key in sorted(settings.keys()):
val = settings[key]
if isinstance(val, dict) and "configured" in val:
state = f"configured (…{val['last4']})" if val.get("configured") else "not configured"
lines.append(f"- {key}: {state}")
else:
lines.append(f"- {key}: {json.dumps(val)}")
return "\n".join(lines)
def _format_write(outcomes: dict) -> str:
applied = [f for f, o in outcomes.items() if o.get("status") == "applied"]
parts = []
if applied:
parts.append("Applied: " + ", ".join(sorted(applied)))
for field, o in outcomes.items():
status = o.get("status")
if status in ("applied", None):
continue
# "error" is transient (retryable); "refused"/"unknown" are not.
verb = "Failed" if status == "error" else "Refused"
parts.append(f"{verb} {field}: {o.get('reason', status)}")
if not parts:
return "No changes were applied."
return "\n".join(parts)
def handle_tool_call(tool_name: str, arguments: dict) -> dict:
if tool_name == "SettingsRead":
result = call_backend("read", {})
if "error" in result:
return {"content": [{"type": "text", "text": f"Error: {result['error']}"}], "isError": True}
return {"content": [{"type": "text", "text": _format_read(result.get("settings", {}))}]}
if tool_name == "SettingsWrite":
changes = arguments.get("changes")
if not isinstance(changes, dict) or not changes:
return {"content": [{"type": "text", "text": "Error: `changes` must be a non-empty object of field -> value."}], "isError": True}
result = call_backend("write", {"changes": changes})
if "error" in result:
return {"content": [{"type": "text", "text": f"Error: {result['error']}"}], "isError": True}
return {"content": [{"type": "text", "text": _format_write(result.get("outcomes", {}))}]}
return {"content": [{"type": "text", "text": f"Unknown tool: {tool_name}"}], "isError": True}
def main():
for line in sys.stdin:
line = line.strip()
if not line:
continue
try:
msg = json.loads(line)
except json.JSONDecodeError:
continue
method = msg.get("method")
id_ = msg.get("id")
params = msg.get("params", {})
if method == "initialize":
send_response(id_, {
"protocolVersion": "2024-11-05",
"capabilities": {"tools": {}},
"serverInfo": {"name": "openswarm-settings-meta", "version": "1.0.0"},
})
elif method == "notifications/initialized":
pass
elif method == "tools/list":
send_response(id_, {"tools": TOOLS})
elif method == "tools/call":
tool_name = params.get("name", "")
arguments = params.get("arguments", {})
try:
send_response(id_, handle_tool_call(tool_name, arguments))
except Exception as e:
send_response(id_, error={"code": -32000, "message": str(e)})
elif method == "resources/list":
send_response(id_, {"resources": []})
elif method == "prompts/list":
send_response(id_, {"prompts": []})
elif id_ is not None:
send_response(id_, error={"code": -32601, "message": f"Method not found: {method}"})
if __name__ == "__main__":
main()
backend/apps/dashboards/dashboards.py
+36 -1
View File
@@ -363,10 +363,45 @@ async def generate_name(dashboard_id: str):
return {"name": dashboard.name, "auto_named": True}
def _strip_orphan_session_cards(data: dict) -> None:
"""Drop layout cards (and expanded ids) whose agent session no longer exists
anywhere, in memory OR on disk. The frontend mounts an AgentChat per card and
GETs its session; a card pointing at a vanished session (e.g. an empty
never-saved session) 404s on every load and flashes a dead "connect a model"
card before the client reconciles it away. The `gone()` test is the exact
condition that makes GET /sessions/{id} 404, so it removes precisely those
cards and nothing else. Filtering the RESPONSE (never the stored file) is
non-destructive: a wrong check can only hide a card for one response, not
delete it. Drafts have no backend session yet, so they're always kept."""
from backend.apps.agents.agent_manager import agent_manager
from backend.apps.agents.manager.session.session_store import _load_session_data
layout = data.get("layout")
if not isinstance(layout, dict):
return
cards = layout.get("cards")
if not isinstance(cards, dict):
return
def gone(sid: str) -> bool:
if sid.startswith("draft-") or sid in agent_manager.sessions:
return False
return _load_session_data(sid) is None
orphans = [sid for sid in cards if gone(sid)]
for sid in orphans:
cards.pop(sid, None)
if orphans:
exp = layout.get("expanded_session_ids")
if isinstance(exp, list):
layout["expanded_session_ids"] = [s for s in exp if s not in orphans]
@dashboards.router.get("/{dashboard_id}")
async def get_dashboard(dashboard_id: str):
dashboard = _load(dashboard_id)
return dashboard.model_dump(mode="json")
data = dashboard.model_dump(mode="json")
_strip_orphan_session_cards(data)
return data
@dashboards.router.put("/{dashboard_id}")
backend/apps/dashboards/models.py
+4
View File
@@ -40,6 +40,10 @@ class BrowserCardPosition(BaseModel):
# Used by the frontend to auto-remove the browser when its owner agent
# reaches a terminal completed/error state.
spawned_by: Optional[str] = None
# When the agent leaves the deliverable on the page (a video playing, a page
# to read), it sets this so the frontend's auto-close on parent finish skips
# the card and the browser stays put.
keep_open: bool = False
class NotePosition(BaseModel):
backend/apps/nine_router/process.py
+152 -51
View File
@@ -16,7 +16,9 @@ import logging
import os
import secrets
import shutil
import socket
import subprocess
import tempfile
import time
from typing import Any
@@ -54,8 +56,41 @@ NINE_ROUTER_V1 = f"{NINE_ROUTER_URL}/v1"
# routed via an `openai-compatible` node that honors `baseUrl`) STAYS necessary.
NINE_ROUTER_NPM_VERSION = os.environ.get("OPENSWARM_ROUTER_VERSION", "0.3.60")
# 9Router (our pinned 0.3.60) appends every request to ~/.9router/request-details.json and
# reloads the WHOLE file on each write; once it reaches tens of MB the router's node process
# OOM-aborts and takes the app down, even while idle (verified from crash dumps). Two cheap,
# pin-safe guards until the real fix (a 9Router bump past 0.4.66, which moved off this file):
# 1. rotate that log before we spawn 9Router when it gets large, so growth can't run away;
# 2. give node an explicit, generous heap ceiling for legitimate large multimodal bodies.
# Neither touches routing, so WebSearch/WebFetch translation and the 0.3.60 pin are unaffected.
_REQUEST_LOG_PATH = os.path.expanduser("~/.9router/request-details.json")
_REQUEST_LOG_MAX_BYTES = 5 * 1024 * 1024
_NODE_HEAP_MB = 4096
def _rotate_request_log() -> None:
"""Rotate ~/.9router/request-details.json to a single .0 backup when it grows past the cap,
BEFORE 9Router is spawned (never racing a live writer). 9Router recreates a fresh file, exactly
like a clean install. The only consumer is the 'most recent 5' reasoning-token lookup, which
already tolerates an empty/missing file, so no feature loses data it depends on."""
try:
if os.path.exists(_REQUEST_LOG_PATH) and os.path.getsize(_REQUEST_LOG_PATH) > _REQUEST_LOG_MAX_BYTES:
os.replace(_REQUEST_LOG_PATH, _REQUEST_LOG_PATH + ".0")
logger.info(
"9Router request log rotated (exceeded %d MB) to avoid the router OOM",
_REQUEST_LOG_MAX_BYTES // (1024 * 1024),
)
except Exception as e:
logger.debug("9Router request-log rotation skipped: %s", e)
_process: subprocess.Popen | None = None
# Serializes ensure_running() so a background auto-start and a concurrent
# dispatch-time ensure can't both spawn 9Router (double-bind on :20128). Lazily
# created so module import doesn't require a running event loop.
_start_lock: "asyncio.Lock | None" = None
# Short TTL cache for positive is_running() results. The probe is a sync
# httpx.get that blocks the event loop, and under load (9Router busy
# streaming inference) it can exceed its 2s timeout and return False even
@@ -68,13 +103,29 @@ _is_running_last_ok: float = 0.0
def is_running() -> bool:
"""Check if 9Router is running."""
"""Check if 9Router is running.
Fast-fail when down. is_running() is called ~5x on the cold boot path (the
settings key-sync sequence + ensure_running) BEFORE 9Router is up. The old
body did a synchronous httpx.get to "localhost:20128"; on Windows a dead-port
connect to "localhost" stalls multiple seconds (it tries ::1 first and the
loopback refusal is slow), so those probes froze the asyncio event loop ~18s
and dominated cold startup (faulthandler caught the loop stuck in
socket.create_connection here). Fix: probe 127.0.0.1 with a 0.3s TCP timeout
first; a down 9Router is detected in <~0.3s instead of ~7s. Only when the
port is open do we do the HTTP confirm. 9Router binds 0.0.0.0 (the warm app
reaches it via 127.0.0.1 today), so this changes timing, not reachability."""
global _is_running_last_ok
now = time.monotonic()
if now - _is_running_last_ok < _IS_RUNNING_TTL:
return True
try:
r = httpx.get(f"{NINE_ROUTER_V1}/models", timeout=2.0)
with socket.create_connection(("127.0.0.1", NINE_ROUTER_PORT), timeout=0.3):
pass
except OSError:
return False
try:
r = httpx.get(f"http://127.0.0.1:{NINE_ROUTER_PORT}/v1/models", timeout=2.0)
if r.status_code == 200:
_is_running_last_ok = now
return True
@@ -297,7 +348,52 @@ def _ensure_router_cached() -> str | None:
return server_js if os.path.exists(server_js) else None
def _read_capture_tail(path: str, limit: int = 6000) -> str:
"""Tail of the 9Router start-capture file, where the real spawn error lands.
Best-effort; empty string on any hiccup so telemetry never breaks boot."""
try:
with open(path, "rb") as f:
f.seek(0, os.SEEK_END)
size = f.tell()
f.seek(max(0, size - limit))
return f.read().decode("utf-8", "replace")
except OSError:
return ""
def _report_start_failure(reason: str, *, detail: str = "", **fields: Any) -> None:
"""9Router didn't come up. Log it and ship a scrubbed diagnostic so a user's
'every model exits 1' is finally explained from our side instead of a silent
warning. The stderr tail can echo an own_key, so it rides the same scrub as
every other telemetry string. Never raises."""
logger.warning("9Router start failed (%s)", reason)
try:
from backend.apps.agents.core.error_classify import redact_for_telemetry
from backend.apps.service.client import submit_diagnostic
payload: dict[str, Any] = {
"kind": "9router_start_failed",
"reason": reason,
"packaged": os.environ.get("OPENSWARM_PACKAGED") == "1",
**fields,
}
if detail:
payload["stderr_tail"] = redact_for_telemetry(detail)
submit_diagnostic(payload)
except Exception:
logger.debug("9router start-failure diagnostic submit failed", exc_info=True)
async def ensure_running():
"""Start 9Router if not already running. Serialized so concurrent callers
(the background auto-start + a dispatch-time ensure) can't double-spawn."""
global _start_lock
if _start_lock is None:
_start_lock = asyncio.Lock()
async with _start_lock:
await _ensure_running_impl()
async def _ensure_running_impl():
"""Start 9Router if not already running."""
global _process
_is_packaged = os.environ.get("OPENSWARM_PACKAGED") == "1"
@@ -325,100 +421,105 @@ async def ensure_running():
else:
logger.info("9Router already running on port %d", NINE_ROUTER_PORT)
return
_rotate_request_log()
_9router_dir = _find_9router_dir()
_patch = _gpt5_patch_path()
if _is_packaged and _9router_dir:
# Packaged mode; run the pre-built standalone server staged at
# <resources>/router/server.js by scripts/fetch-router.sh at build time.
if _is_packaged:
# Packaged: run the pre-built standalone server staged at
# <resources>/router/server.js by fetch-router at build time. We do NOT
# fall back to the dev npm path here, a user machine has no npm, so that
# only ever fails silently; every miss is reported instead.
if not _9router_dir:
_report_start_failure("router_not_bundled")
return
standalone_server = os.path.join(_9router_dir, "server.js")
if not os.path.exists(standalone_server):
standalone_server = os.path.join(_9router_dir, ".next", "standalone", "server.js")
if not os.path.exists(standalone_server):
logger.warning("9Router standalone build not found in %s", _9router_dir)
_report_start_failure("server_missing", router_dir_found=True)
return
node = _find_node()
if not node:
logger.warning("Node.js not found; cannot start 9Router in packaged mode.")
_report_start_failure("node_not_found", router_dir_found=True, server_found=True)
return
logger.info("Starting 9Router (production) on port %d...", NINE_ROUTER_PORT)
cmd = [node]
_patch = _gpt5_patch_path()
if _patch:
cmd += ["--require", _patch]
cmd.append(standalone_server)
cmd = [node, f"--max-old-space-size={_NODE_HEAP_MB}"] + (["--require", _patch] if _patch else []) + [standalone_server]
cwd = os.path.dirname(standalone_server)
env = {**os.environ, "PORT": str(NINE_ROUTER_PORT), "NODE_ENV": "production"}
if node == os.environ.get("OPENSWARM_ELECTRON_PATH"):
env["ELECTRON_RUN_AS_NODE"] = "1"
else:
# Dev mode; install the pinned 9router npm package into a local
# cache the first time run.sh boots, then spawn `node app/server.js`
# directly on subsequent launches. Bypassing the package's cli.js
# avoids its menu-bar tray icon (which users confusingly quit,
# silently killing their subscription routing), its update-check
# spinner, and the interactive TUI.
# Dev: install the pinned npm package into a local cache once, then spawn
# `node app/server.js` directly (bypasses the package cli.js tray icon
# users confusingly quit, its update-check spinner, and the TUI).
cached_server = _ensure_router_cached()
if not cached_server:
return
node = _find_node()
if not node:
logger.warning("Node.js not found; cannot start 9Router in dev mode.")
return
logger.info(
"Starting 9Router (dev cache, 9router@%s) on port %d...",
NINE_ROUTER_NPM_VERSION, NINE_ROUTER_PORT,
)
cmd = [node]
_patch = _gpt5_patch_path()
if _patch:
cmd += ["--require", _patch]
cmd.append(cached_server)
cmd = [node, f"--max-old-space-size={_NODE_HEAP_MB}"] + (["--require", _patch] if _patch else []) + [cached_server]
cwd = os.path.dirname(cached_server)
env = {**os.environ, "PORT": str(NINE_ROUTER_PORT), "NODE_ENV": "production"}
# By default, 9Router's stdout/stderr go to /dev/null (Next.js dev mode
# is extremely chatty and floods the openswarm console otherwise). When
# debugging is needed, set OPENSWARM_DEBUG_9ROUTER=1 in the environment
# before launching the backend; output will then be appended to
# backend/data/9router.log line-buffered, which can be `tail -f`'d.
if os.environ.get("OPENSWARM_DEBUG_9ROUTER"):
# Capture stdout+stderr so a failed start can tell us WHY (the old DEVNULL
# default made every "router never came up" a silent mystery, which is the
# whole reason #90 was un-diagnosable). Packaged prod (NODE_ENV=production
# standalone) is quiet, so one fixed temp file, truncated each start attempt,
# won't grow; dev keeps its chatty-Next.js DEVNULL unless debug is set.
_cap_path = os.path.join(tempfile.gettempdir(), "openswarm-9router-start.log")
_cap_file = None
if _is_packaged:
try:
_cap_file = open(_cap_path, "wb")
_stdout, _stderr = _cap_file, subprocess.STDOUT
except OSError:
_stdout, _stderr = subprocess.DEVNULL, subprocess.DEVNULL
elif os.environ.get("OPENSWARM_DEBUG_9ROUTER"):
_log_path = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
"data",
"9router.log",
"data", "9router.log",
)
os.makedirs(os.path.dirname(_log_path), exist_ok=True)
_stdout = open(_log_path, "a", buffering=1) # line-buffered
_stderr = subprocess.STDOUT
_stdout, _stderr = open(_log_path, "a", buffering=1), subprocess.STDOUT
logger.info(f"9Router debug logging enabled → {_log_path}")
else:
_stdout = subprocess.DEVNULL
_stderr = subprocess.DEVNULL
_stdout, _stderr = subprocess.DEVNULL, subprocess.DEVNULL
try:
_process = subprocess.Popen(
cmd,
cwd=cwd,
stdout=_stdout,
stderr=_stderr,
env=env,
)
_process = subprocess.Popen(cmd, cwd=cwd, stdout=_stdout, stderr=_stderr, env=env)
if _cap_file is not None:
_cap_file.close() # the child holds its own fd; the parent copy isn't needed
timeout = 20 if _is_packaged else 30
for _ in range(timeout * 2):
await asyncio.sleep(0.5)
if is_running():
logger.info("9Router started successfully")
return
logger.warning("9Router did not start within %ds", timeout)
# Verify-at-boot: it never answered. Report with the captured tail + the
# exit code (non-None = it crashed; None = wedged or just slow).
_report_start_failure(
"not_ready_in_time",
detail=_read_capture_tail(_cap_path) if _is_packaged else "",
returncode=_process.poll(),
timeout_s=timeout,
)
except Exception as e:
logger.warning(f"Failed to start 9Router: {e}")
if _cap_file is not None and not _cap_file.closed:
try:
_cap_file.close()
except OSError:
pass
_report_start_failure(
"spawn_exception",
detail=f"{e}\n{_read_capture_tail(_cap_path) if _is_packaged else ''}",
)
def stop():
backend/apps/outputs/app_builder_skill.md
+39
View File
@@ -325,6 +325,45 @@ export const JOBS_LIST = '/api/jobs/list';
---
## Publishable AI + compute — `window.OUTPUT_LLM` / `window.OUTPUT_COMPUTE`
The FastAPI backend above runs in preview but is **not hosted when an app is
published** to the web. For features that should keep working on a published
`{slug}.openswarm.host` link, use these two runtime calls instead of a backend.
They run on the published site (same-origin, no credentials). In the App Builder
**preview** they throw a clear "available once published" error, preview can't run
them without embedding a credential into your app, so test these by publishing.
**AI (Claude):** call `window.OUTPUT_LLM` with an Anthropic-style messages body.
The model is chosen for you (a cheap default), so don't pass one.
```ts
const res = await window.OUTPUT_LLM({
messages: [{ role: 'user', content: prompt }],
max_tokens: 512,
});
const data = await res.json();
const text = data.content[0].text;
```
**Data-shaping compute:** put pure Python (json/math/csv/datetime only — no
network, no files) in a top-level `backend.py` that reads `input_data` and assigns
`result`, then call `window.OUTPUT_COMPUTE(input)`:
```python
# backend.py
result = {"total": sum(input_data["nums"])}
```
```ts
const out = await window.OUTPUT_COMPUTE({ nums: [1, 2, 3] }); // -> { total: 6 }
```
Rule of thumb: if the app should be publishable, reach for `OUTPUT_LLM` /
`OUTPUT_COMPUTE` first; only use the FastAPI backend for preview-only tools or
things those two can't do (it won't be there once published).
---
## Debugging — use `swarm_debug`, not `print()`
The backend has `swarm_debug` pre-installed. It's a colored frame-aware
backend/apps/outputs/html_inject.py
+20 -4
View File
@@ -54,20 +54,36 @@ def _validate_against_schema(data: dict, schema: dict) -> str | None:
return f"Schema validation failed at {path}: {exc.message}"
def _build_data_injection(input_json: str, result_json: str, backend_url_json: str = "null") -> str:
def _runtime_helpers_js() -> str:
"""OUTPUT_COMPUTE / OUTPUT_LLM only run for real on the published edge, where they
are same-origin and carry NO credentials. In the App Builder preview we
deliberately do NOT wire them to the authenticated backend: doing so would embed
this install's token into the app's own JS (the exact exposure SECURITY.md item A
is about). Preview defines readable stubs instead, the app degrades with a clear
message rather than crashing or leaking a credential."""
return (
" window.OUTPUT_COMPUTE = async function () { throw new Error('OUTPUT_COMPUTE runs once this app is published.'); };\n"
" window.OUTPUT_LLM = async function () { throw new Error('OUTPUT_LLM runs once this app is published.'); };\n"
)
def _build_data_injection(input_json: str, result_json: str, backend_url_json: str = "null", with_runtime: bool = False) -> str:
"""Build a <script> tag that sets OUTPUT_INPUT / OUTPUT_BACKEND_RESULT /
OUTPUT_BACKEND_URL and listens for postMessage updates.
OUTPUT_BACKEND_URL, optionally wires OUTPUT_COMPUTE / OUTPUT_LLM, and listens
for postMessage updates.
OUTPUT_BACKEND_URL is `null` when the app has no live `backend.py`
process; otherwise it's `http://localhost:<port>` and app code can
`fetch(window.OUTPUT_BACKEND_URL + '/route')` to hit the persistent
backend's endpoints."""
helpers = _runtime_helpers_js() if with_runtime else ""
return (
"<script>\n"
"(function() {\n"
" window.OUTPUT_INPUT = " + input_json + ";\n"
" window.OUTPUT_BACKEND_RESULT = " + result_json + ";\n"
" window.OUTPUT_BACKEND_URL = " + backend_url_json + ";\n"
+ helpers +
" window.addEventListener('message', function(e) {\n"
" if (e.data && e.data.type === 'OUTPUT_DATA') {\n"
" window.OUTPUT_INPUT = e.data.input || {};\n"
@@ -81,8 +97,8 @@ def _build_data_injection(input_json: str, result_json: str, backend_url_json: s
)
def _inject_data_into_html(html: str, input_json: str = "{}", result_json: str = "null", backend_url_json: str = "null") -> str:
injection = _build_data_injection(input_json, result_json, backend_url_json)
def _inject_data_into_html(html: str, input_json: str = "{}", result_json: str = "null", backend_url_json: str = "null", with_runtime: bool = False) -> str:
injection = _build_data_injection(input_json, result_json, backend_url_json, with_runtime)
if "</head>" in html:
return html.replace("</head>", f"{injection}\n</head>", 1)
if "<body" in html:
backend/apps/outputs/models.py
+52
View File
@@ -25,6 +25,12 @@ class Output(BaseModel):
workspace_id: Optional[str] = None
created_at: str = Field(default_factory=lambda: datetime.now().isoformat())
updated_at: str = Field(default_factory=lambda: datetime.now().isoformat())
# App publishing to {slug}.openswarm.host. Server-managed: set by the publish
# endpoint, never accepted from OutputUpdate (so a client can't spoof a live URL).
published_slug: Optional[str] = None
published_url: Optional[str] = None
publish_status: Optional[Literal["publishing", "published", "error"]] = None
publish_error: Optional[str] = None
@model_validator(mode="before")
@classmethod
@@ -55,6 +61,20 @@ class Output(BaseModel):
return self.files.get("backend.py")
class OutputVersion(BaseModel):
"""A saved point in an app's history. The list endpoint returns these; the
heavy bits (the serialized app metadata + the workspace byte tree) live in
the version's manifest plus content-addressed blobs on disk, not here."""
id: str = Field(default_factory=lambda: uuid4().hex)
created_at: str = Field(default_factory=lambda: datetime.now().isoformat())
label: str = ""
# auto: saved after a builder edit run. manual: user clicked Save this version.
# pre_restore: the automatic backup taken right before a restore (so restore undoes).
source: Literal["auto", "manual", "pre_restore"] = "auto"
parent_id: Optional[str] = None
thumbnail: Optional[str] = None
class OutputCreate(BaseModel):
name: str
description: str = ""
@@ -197,3 +217,35 @@ class VibeCodeRequest(BaseModel):
current_schema: str = ""
name: str = ""
description: str = ""
class PublishReview(BaseModel):
# Same JSON shape the frontend shareTypes.ReviewSummary expects.
verdict: Literal["clean", "warn", "block"] = "clean"
findings: list[str] = Field(default_factory=list)
scanned_files: list[str] = Field(default_factory=list)
class PublishPreflightRequest(BaseModel):
output_id: str
class PublishRequest(BaseModel):
output_id: str
slug: Optional[str] = None
force: bool = False
class PublishPreflightResponse(BaseModel):
review: PublishReview
class PublishResult(BaseModel):
ok: bool = True
published_slug: Optional[str] = None
published_url: Optional[str] = None
# When the AST safety net blocks a non-force publish, carry the findings so
# the UI shows the review modal instead of a generic error toast.
blocked: bool = False
review: Optional[PublishReview] = None
error: Optional[str] = None
backend/apps/outputs/outputs.py
+121 -4
View File
@@ -12,8 +12,14 @@ from backend.config.Apps import SubApp
from backend.apps.outputs.models import (
Output, OutputCreate, OutputUpdate, OutputExecute, OutputExecuteResult,
VibeCodeRequest, WorkspaceSeedRequest,
PublishPreflightRequest, PublishRequest, PublishPreflightResponse,
PublishResult, PublishReview,
)
from backend.apps.outputs.executor import execute_backend_code, get_code_warnings
from backend.apps.outputs.publish_common import slugify, PublishError
from backend.apps.outputs.publish_scan import scan_for_publish, quick_ast_gate
from backend.apps.outputs.publish_build import build_static, collect_bundle
from backend.apps.outputs.publish_cloud import upload_to_cloud, unpublish_from_cloud
from backend.apps.outputs.view_builder_templates import (
VIEW_TEMPLATE_FILES,
load_app_builder_skill,
@@ -87,7 +93,7 @@ async def serve_workspace_file(workspace_id: str, filepath: str, _d: str = ""):
if filepath == "index.html":
input_json, result_json = _decode_data_param(_d) if _d else ("{}", "null")
backend_url_json = _backend_url_for_workspace(workspace_id)
content = _inject_data_into_html(content, input_json, result_json, backend_url_json)
content = _inject_data_into_html(content, input_json, result_json, backend_url_json, with_runtime=True)
# Iframe sub-resource fetches (<link>, <script src>, <img>) drop the
# parent's ?token= query string, so rewrite the HTML to put the token
# back on every relative URL; otherwise sub-resources 401.
@@ -108,7 +114,7 @@ async def serve_output_file(output_id: str, filepath: str, _d: str = ""):
if filepath == "index.html":
input_json, result_json = _decode_data_param(_d) if _d else ("{}", "null")
backend_url_json = _backend_url_for_workspace(output.workspace_id) if output.workspace_id else "null"
content = _inject_data_into_html(content, input_json, result_json, backend_url_json)
content = _inject_data_into_html(content, input_json, result_json, backend_url_json, with_runtime=True)
content = _inject_token_into_relative_urls(content, get_auth_token())
mime, _ = mimetypes.guess_type(filepath)
@@ -454,6 +460,33 @@ async def runtime_get_status(workspace_id: str):
return _runtime_status_payload(workspace_id)
@outputs.router.post("/workspace/{workspace_id}/runtime/report-error")
async def runtime_report_error(workspace_id: str, body: dict):
from backend.apps.outputs.runtime import manager as runtime_manager
rt = runtime_manager.get(workspace_id)
if rt is None:
return {"ok": False, "recorded": 0}
message = (body.get("message") or "").strip()
component_stack = (body.get("componentStack") or "").strip()
if not message:
return {"ok": False, "recorded": 0}
composed = message
if component_stack:
composed = f"{composed}\n{component_stack}"
rt.set_render_error(composed)
return {"ok": True, "recorded": 1}
@outputs.router.post("/workspace/{workspace_id}/runtime/report-ready")
async def runtime_report_ready(workspace_id: str):
from backend.apps.outputs.runtime import manager as runtime_manager
rt = runtime_manager.get(workspace_id)
if rt is None:
return {"ok": False}
rt.set_render_ok()
return {"ok": True}
@outputs.router.post("/shutdown-all")
async def runtime_shutdown_all():
"""Reap every workspace subprocess. Electron POSTs this during
@@ -527,7 +560,6 @@ async def create_output(body: OutputCreate):
updated_at=now,
)
_save(output)
pass
return {"ok": True, "output": output.model_dump()}
@@ -555,6 +587,8 @@ async def delete_output(output_id: str):
path = os.path.join(DATA_DIR, f"{output_id}.json")
if os.path.exists(path):
os.remove(path)
from backend.apps.outputs import versions
versions.delete_all(output_id)
return {"ok": True}
@@ -620,7 +654,6 @@ async def vibe_code(body: VibeCodeRequest):
raw = raw[:-3]
result = json.loads(raw)
pass
return {
"message": result.get("message", "View updated."),
"frontend_code": result.get("frontend_code", body.current_frontend_code),
@@ -707,3 +740,87 @@ async def execute_output(body: OutputExecute):
).model_dump()
# ---------------------------------------------------------------------------
# Publishing to {slug}.openswarm.host
# ---------------------------------------------------------------------------
@outputs.router.post("/publish/preflight")
async def publish_preflight(body: PublishPreflightRequest):
"""Scan the app (AST + an aux-LLM pass on the user's own creds) and return a
review. No build, no cloud call; this just drives the security modal."""
output = _load(body.output_id)
review = await scan_for_publish(output, load_settings())
return PublishPreflightResponse(review=review).model_dump()
@outputs.router.post("/publish")
async def publish_output(body: PublishRequest):
"""Build (webapp) + bundle + upload to the cloud host. `force` skips the
cheap AST safety net (the user already saw the findings in the review modal)."""
output = _load(body.output_id)
settings = load_settings()
if not body.force:
ast = quick_ast_gate(output)
if ast:
return PublishResult(
ok=False,
blocked=True,
review=PublishReview(verdict="warn", findings=ast),
).model_dump()
output.publish_status = "publishing"
output.publish_error = None
_save(output)
try:
dist = await build_static(output)
bundle = collect_bundle(output, dist)
slug_hint = slugify(body.slug or output.name)
res = await upload_to_cloud(
settings,
output_id=output.id,
name=output.name,
slug_hint=slug_hint,
bundle=bundle,
override=body.force,
)
except PublishError as e:
output.publish_status = "error"
output.publish_error = str(e)
_save(output)
return PublishResult(ok=False, error=str(e)).model_dump()
except Exception as e:
logger.exception("publish failed for %s", output.id)
output.publish_status = "error"
output.publish_error = "Something went wrong while publishing."
_save(output)
return PublishResult(ok=False, error=output.publish_error).model_dump()
output.published_slug = res.get("slug")
output.published_url = res.get("url")
output.publish_status = "published"
output.publish_error = None
_save(output)
return PublishResult(
ok=True,
published_slug=output.published_slug,
published_url=output.published_url,
).model_dump()
@outputs.router.post("/unpublish")
async def unpublish_output(body: PublishPreflightRequest):
"""Take the app offline and clear its publish state."""
output = _load(body.output_id)
if output.published_slug:
try:
await unpublish_from_cloud(load_settings(), output.published_slug)
except PublishError as e:
return {"ok": False, "error": str(e)}
output.published_slug = None
output.published_url = None
output.publish_status = None
output.publish_error = None
_save(output)
return {"ok": True}
backend/apps/outputs/publish_build.py
+148
View File
@@ -0,0 +1,148 @@
"""Build + bundle the static artifact the cloud will host. Webapp-mode runs the
bundled node on `vite build`; flat-mode is already the artifact. Secret-shaped
files (.env, private keys) are dropped because a published bundle is world-readable."""
from __future__ import annotations
import asyncio
import io
import logging
import os
import re
import shutil
import tarfile
from typing import Optional
from backend.apps.outputs.models import Output
from backend.apps.outputs.publish_common import PublishError, is_webapp, workspace_dir
logger = logging.getLogger(__name__)
_BUILD_TIMEOUT = 180 # vite build on a cold-ish node_modules can be slow
_MAX_BUNDLE_FILE = 25 * 1024 * 1024
_SECRET_KEY_EXTS = (".pem", ".key", ".p12", ".pfx", ".keystore")
def _node_bin() -> Optional[str]:
return os.environ.get("OPENSWARM_NODE_PATH") or shutil.which("node")
def _safe_build_config(fe: str) -> tuple[list[str], Optional[str]]:
"""vite-plugin-terminal injects a dev-only `virtual:terminal` module that
breaks `vite build` in older workspaces (the template later gated it to dev,
but apps seeded before that still carry the ungated plugin). Build against a
temp config that makes that plugin a no-op (Vite drops null plugins) so ANY
workspace builds clean. The user's own vite.config is never touched.
Returns (extra build args, temp-config path to delete) or ([], None)."""
cfg_name = next(
(n for n in ("vite.config.ts", "vite.config.js", "vite.config.mjs")
if os.path.exists(os.path.join(fe, n))),
None,
)
if not cfg_name:
return [], None
with open(os.path.join(fe, cfg_name), "r", encoding="utf-8") as f:
content = f.read()
if "vite-plugin-terminal" not in content:
return [], None
patched = re.sub(
r"import\s+terminal\s+from\s+['\"]vite-plugin-terminal['\"];?",
"const terminal = () => null;",
content,
)
ext = os.path.splitext(cfg_name)[1]
temp_name = f"vite.config.openswarm-publish{ext}"
with open(os.path.join(fe, temp_name), "w", encoding="utf-8") as f:
f.write(patched)
return ["--config", temp_name], os.path.join(fe, temp_name)
async def build_static(output: Output) -> Optional[str]:
"""Webapp apps -> build `frontend/dist`, return its path. Flat apps need no
build (the files dict is the artifact), return None. Raises PublishError with
a user-safe message on any failure."""
if not is_webapp(output):
return None
fe = os.path.join(workspace_dir(output), "frontend")
vite = os.path.join(fe, "node_modules", "vite", "bin", "vite.js")
node = _node_bin()
if not node or not os.path.exists(vite):
raise PublishError(
"This app isn't set up to build yet. Open it once in the editor, then try publishing again."
)
config_args, temp_cfg = _safe_build_config(fe)
proc = await asyncio.create_subprocess_exec(
node, "node_modules/vite/bin/vite.js", "build", *config_args,
cwd=fe,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
env={**os.environ, "NODE_ENV": "production"},
)
try:
_out, err = await asyncio.wait_for(proc.communicate(), timeout=_BUILD_TIMEOUT)
except asyncio.TimeoutError:
proc.kill()
await proc.wait()
raise PublishError("Building your app took too long and was stopped.")
finally:
if temp_cfg:
try:
os.remove(temp_cfg)
except OSError:
pass
if proc.returncode != 0:
logger.error("vite build failed (%s): %s", output.id, err.decode(errors="replace")[-2000:])
raise PublishError("We couldn't build your app. Make sure it runs in the editor, then try again.")
dist = os.path.join(fe, "dist")
if not os.path.isdir(dist):
raise PublishError("The build finished but produced no files.")
return dist
def _is_secret_file(rel_path: str) -> bool:
"""This bundle is served publicly, so anything secret-shaped must never make it
in. dotenv files and private-key material are the realistic leaks; the webapp
path already ships only the built dist, this also covers a hand-built flat app."""
base = rel_path.rsplit("/", 1)[-1].lower()
return (
base == ".env"
or base.startswith(".env.")
or base.endswith(_SECRET_KEY_EXTS)
or base in (".npmrc", ".git-credentials", ".htpasswd")
)
def collect_bundle(output: Output, dist_dir: Optional[str]) -> bytes:
"""tar.gz of what the cloud should host. Webapp -> the built dist tree.
Flat -> the files dict, including backend.py (the edge runs it on the shared
sandbox; the edge refuses to serve .py as a static file). Secret-shaped files
(.env, private keys) are dropped: a published bundle is world-readable."""
buf = io.BytesIO()
with tarfile.open(fileobj=buf, mode="w:gz") as tar:
if dist_dir:
for root, _dirs, files in os.walk(dist_dir):
for fn in files:
full = os.path.join(root, fn)
if os.path.islink(full):
continue
rel = os.path.relpath(full, dist_dir).replace(os.sep, "/")
if _is_secret_file(rel):
continue
try:
if os.path.getsize(full) > _MAX_BUNDLE_FILE:
continue
except OSError:
continue
tar.add(full, arcname=rel)
else:
for name, content in (output.files or {}).items():
rel = name.replace(os.sep, "/")
if _is_secret_file(rel):
continue
data = content.encode("utf-8")
if len(data) > _MAX_BUNDLE_FILE:
continue
info = tarfile.TarInfo(name=rel)
info.size = len(data)
tar.addfile(info, io.BytesIO(data))
return buf.getvalue()
backend/apps/outputs/publish_cloud.py
+67
View File
@@ -0,0 +1,67 @@
"""Cloud client for the publish pipeline: ships the bundle to the host and takes
it back down. Reads the bearer directly (publish works for any signed-in account,
not just pro/free-trial), matching the cloud's requireAuthedUser gate."""
from __future__ import annotations
from typing import Optional
import httpx
from backend.apps.outputs.publish_common import PublishError
from backend.apps.settings.credentials import OPENSWARM_DEFAULT_PROXY_URL
def _cloud_auth(settings) -> tuple[Optional[str], str]:
base = (getattr(settings, "openswarm_proxy_url", None) or OPENSWARM_DEFAULT_PROXY_URL).rstrip("/")
token = getattr(settings, "openswarm_bearer_token", None)
return token, base
def _safe_detail(resp: httpx.Response, fallback: str) -> str:
try:
body = resp.json()
msg = body.get("message") or body.get("error")
if isinstance(msg, str) and msg:
return msg
except Exception:
pass
return fallback
async def upload_to_cloud(
settings, *, output_id: str, name: str, slug_hint: str, bundle: bytes, override: bool
) -> dict:
token, base = _cloud_auth(settings)
if not token:
raise PublishError("Sign in to your OpenSwarm account to publish apps.")
try:
async with httpx.AsyncClient(timeout=120.0) as client:
r = await client.post(
f"{base}/api/apps/publish",
headers={"Authorization": f"Bearer {token}"},
# output_id lets the cloud reuse this app's slug on republish instead
# of minting a duplicate; override marks a publish past a non-clean scan.
data={"name": name, "slug": slug_hint, "output_id": output_id, "override": "1" if override else "0"},
files={"bundle": ("app.tar.gz", bundle, "application/gzip")},
)
except httpx.HTTPError:
raise PublishError("Couldn't reach the publishing service. Check your connection and try again.")
if r.status_code >= 400:
raise PublishError(_safe_detail(r, "Publishing failed. Please try again."))
return r.json()
async def unpublish_from_cloud(settings, slug: str) -> None:
token, base = _cloud_auth(settings)
if not token:
raise PublishError("Sign in to your OpenSwarm account to manage published apps.")
try:
async with httpx.AsyncClient(timeout=30.0) as client:
r = await client.post(
f"{base}/api/apps/{slug}/delete",
headers={"Authorization": f"Bearer {token}"},
)
except httpx.HTTPError:
raise PublishError("Couldn't reach the publishing service. Check your connection and try again.")
if r.status_code >= 400 and r.status_code != 404:
raise PublishError(_safe_detail(r, "Couldn't unpublish. Please try again."))
backend/apps/outputs/publish_common.py
+29
View File
@@ -0,0 +1,29 @@
"""Shared low-level bits for the publish pipeline (scan / build / cloud). Kept
tiny and dependency-light so scan, build, and cloud_client can all lean on it
without reaching sideways into each other."""
from __future__ import annotations
import os
import re
from backend.apps.outputs.models import Output
from backend.config.paths import OUTPUTS_WORKSPACE_DIR
class PublishError(Exception):
"""User-facing publish failure; message is safe to show in a toast."""
def slugify(name: str) -> str:
"""A url-safe slug hint from the app name; the cloud guarantees uniqueness."""
s = re.sub(r"[^a-z0-9]+", "-", (name or "app").lower()).strip("-")
s = s[:32].strip("-")
return s or "app"
def is_webapp(output: Output) -> bool:
return bool(output.workspace_id)
def workspace_dir(output: Output) -> str:
return os.path.join(OUTPUTS_WORKSPACE_DIR, output.workspace_id or "")
backend/apps/outputs/publish_scan.py
+179
View File
@@ -0,0 +1,179 @@
"""Pre-publish security scan: a free AST pass (reuses the executor's
`get_code_warnings`) plus a best-effort aux-LLM semantic pass, both on the user's
OWN creds so it costs us nothing and the code never leaves the machine until they
ship. The JSON shape matches the frontend `ReviewSummary`.
The full scan (`scan_for_publish`) is memoized on a hash of the collected source:
reopening the publish modal on unchanged code returns the cached review instead of
billing the user's aux model again."""
from __future__ import annotations
import hashlib
import json
import logging
import os
from collections import OrderedDict
from typing import Literal
from backend.apps.outputs.executor import get_code_warnings
from backend.apps.outputs.models import Output, PublishReview
from backend.apps.outputs.publish_common import is_webapp, workspace_dir
from backend.apps.outputs.workspace_io import _WALK_SKIP_DIRS
logger = logging.getLogger(__name__)
_SCAN_CODE_BUDGET = 60_000 # chars of source we hand the aux model
_SCAN_EXTS = (".py", ".html", ".ts", ".tsx", ".js", ".jsx", ".vue", ".svelte", ".css")
_MEMO_MAX = 32
_SCAN_SYSTEM_PROMPT = (
"You are a security reviewer for a no-code app host. The app below will be "
"served publicly at a *.openswarm.host subdomain. Read the source and report "
"only concrete, real risks a reviewer would act on: hardcoded secrets or API "
"keys, phishing or credential-harvesting forms, sending user data to a "
"third-party endpoint, obvious XSS or injection, or anything malicious. Do "
"NOT nitpick style or speculate. Reply ONLY with JSON: "
'{"severity": "clean|warn|block", "findings": ["short dev-readable line", ...]}. '
"Use block only for clearly malicious or credential-harvesting code. Empty "
"findings means clean."
)
# slug-content-hash -> PublishReview, so a reopened modal doesn't re-bill the LLM.
_memo: "OrderedDict[str, PublishReview]" = OrderedDict()
def _collect_source(output: Output) -> dict[str, str]:
"""Gather human-readable source text for the scan. Flat apps come from the
files dict; webapp apps walk the workspace skipping node_modules/.venv/dist."""
src: dict[str, str] = {}
for name, content in (output.files or {}).items():
if name.lower().endswith(_SCAN_EXTS):
src[name] = content
if is_webapp(output):
root = workspace_dir(output)
for base, _dirs, fnames in os.walk(root):
_dirs[:] = [d for d in _dirs if d not in _WALK_SKIP_DIRS]
for fn in fnames:
if not fn.lower().endswith(_SCAN_EXTS):
continue
full = os.path.join(base, fn)
if os.path.islink(full):
continue
try:
if os.path.getsize(full) > 512 * 1024:
continue
with open(full, "r", encoding="utf-8", errors="replace") as f:
rel = os.path.relpath(full, root).replace(os.sep, "/")
src[rel] = f.read()
except OSError:
continue
return src
def _source_hash(src: dict[str, str]) -> str:
h = hashlib.sha256()
for path in sorted(src):
h.update(path.encode("utf-8"))
h.update(b"\0")
h.update(src[path].encode("utf-8", errors="replace"))
h.update(b"\0")
return h.hexdigest()
def _scan_blob(src: dict[str, str]) -> str:
parts: list[str] = []
total = 0
for path, code in src.items():
chunk = f"=== {path} ===\n{code}\n"
if total + len(chunk) > _SCAN_CODE_BUDGET:
chunk = chunk[: max(0, _SCAN_CODE_BUDGET - total)]
parts.append(chunk)
total += len(chunk)
if total >= _SCAN_CODE_BUDGET:
break
return "".join(parts)
def _ast_findings(src: dict[str, str]) -> tuple[list[str], list[str]]:
findings: list[str] = []
scanned: list[str] = []
for path, code in src.items():
if path.lower().endswith(".py"):
scanned.append(path)
for w in get_code_warnings(code):
findings.append(f"{path}: {w}")
return findings, scanned
async def _llm_findings(src: dict[str, str], settings) -> tuple[list[str], str]:
"""Aux-tier semantic pass. Best-effort: if no aux model is configured or the
call fails, return clean so the AST pass still gates. Runs on the user's creds."""
blob = _scan_blob(src)
if not blob.strip():
return [], "clean"
from backend.apps.agents.providers.registry import resolve_aux_model
from backend.apps.settings.credentials import get_anthropic_client_for_model
from backend.apps.agents.core.aux_llm import _safe_resp_text
try:
model, _base = await resolve_aux_model(settings, preferred_tier="haiku")
except Exception:
return [], "clean"
client = get_anthropic_client_for_model(settings, model)
try:
resp = await client.messages.create(
model=model,
max_tokens=1200,
system=_SCAN_SYSTEM_PROMPT,
messages=[{"role": "user", "content": blob}],
)
except Exception:
logger.exception("publish LLM scan call failed; AST-only result stands")
return [], "clean"
text = _safe_resp_text(resp).strip()
if text.startswith("```"):
text = text.split("\n", 1)[1] if "\n" in text else text[3:]
if text.endswith("```"):
text = text[:-3]
try:
parsed = json.loads(text)
except (json.JSONDecodeError, ValueError):
return [], "clean"
findings = [str(f) for f in parsed.get("findings", []) if str(f).strip()][:20]
severity = parsed.get("severity", "clean")
if severity not in ("clean", "warn", "block"):
severity = "warn" if findings else "clean"
return findings, severity
async def scan_for_publish(output: Output, settings) -> PublishReview:
src = _collect_source(output)
key = _source_hash(src)
cached = _memo.get(key)
if cached is not None:
_memo.move_to_end(key)
return cached
ast_findings, scanned = _ast_findings(src)
llm_findings, llm_sev = await _llm_findings(src, settings)
findings = ast_findings + llm_findings
verdict: Literal["clean", "warn", "block"] = "clean"
if findings:
verdict = "warn"
if llm_sev == "block":
verdict = "block"
review = PublishReview(
verdict=verdict,
findings=findings,
scanned_files=scanned or sorted(src.keys()),
)
_memo[key] = review
_memo.move_to_end(key)
while len(_memo) > _MEMO_MAX:
_memo.popitem(last=False)
return review
def quick_ast_gate(output: Output) -> list[str]:
"""Cheap, free safety net used by /publish when force is not set: flags the
AST-visible 'runs code outside the sandbox' findings without an LLM call."""
findings, _ = _ast_findings(_collect_source(output))
return findings
backend/apps/outputs/runtime.py
+64 -7
View File
@@ -99,6 +99,8 @@ class AppRuntime:
# vite/babel/uvicorn errors in its next turn and can self-fix
# instead of leaving the user with a red iframe overlay.
self.recent_errors: deque[str] = deque(maxlen=_RECENT_ERRORS_MAX)
self.render_state: Optional[str] = None
self.render_error_text: str = ""
self._stdout_task: Optional[asyncio.Task] = None
self._stderr_task: Optional[asyncio.Task] = None
self._wait_task: Optional[asyncio.Task] = None
@@ -113,6 +115,18 @@ class AppRuntime:
self.recent_errors.clear()
return out
def set_render_ok(self) -> None:
self.render_state = "ok"
self.render_error_text = ""
def set_render_error(self, text: str) -> None:
self.render_state = "error"
self.render_error_text = (text or "").strip()
def reset_render_state(self) -> None:
self.render_state = None
self.render_error_text = ""
@property
def running(self) -> bool:
return self.process is not None and self.process.returncode is None
@@ -250,12 +264,13 @@ class AppRuntime:
env["OPENSWARM_DEBUGGER_PATH"] = _DEBUGGER_PATH
env["OPENSWARM_TEMPLATE_BACKEND_PATH"] = _TEMPLATE_BACKEND_PATH
cmd, spawn_cwd, launch_desc = self._resolve_launch(env)
try:
self.process = await asyncio.create_subprocess_exec(
_resolve_bash(), "run.sh",
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
cwd=self.workspace_path,
cwd=spawn_cwd,
env=env,
**_background_priority_kwargs(),
)
@@ -267,7 +282,7 @@ class AppRuntime:
self.process = None
return False
backend_note = f" + backend on {self.port}" if self.port else ""
self._broadcast(LogLine("runtime", f"[runtime] bash run.sh started; frontend on {self.frontend_port}{backend_note} (pid {self.process.pid})"))
self._broadcast(LogLine("runtime", f"[runtime] {launch_desc} started; frontend on {self.frontend_port}{backend_note} (pid {self.process.pid})"))
self._stdout_task = asyncio.create_task(self._pipe_stream(self.process.stdout, "stdout"))
self._stderr_task = asyncio.create_task(self._pipe_stream(self.process.stderr, "stderr"))
self._wait_task = asyncio.create_task(self._await_exit())
@@ -277,6 +292,33 @@ class AppRuntime:
self._frontend_ready_task = asyncio.create_task(self._await_frontend_bind())
return True
def _resolve_launch(self, env: dict) -> tuple[list[str], str, str]:
"""Pick the new-mode launch command.
Default is `bash run.sh` at the workspace root, which handles both
frontend-only and backend-enabled apps. On Windows we take a fast path
for frontend-only apps (the common case): run vite directly through the
bundled node, with no system `bash` at all. The packaged Windows build
ships node but not bash, so a user without Git for Windows hit
[WinError 2] on `bash run.sh` and the preview never started. We only
take this path when vite is actually present (node_modules linked);
otherwise fall back to bash so behavior is unchanged everywhere else.
vite.config.ts reads FRONTEND_PORT / BACKEND_PORT from the environment."""
if os.name == "nt" and self.port is None:
node = env.get("OPENSWARM_NODE_PATH") or shutil.which("node")
vite_bin = os.path.join(
self.workspace_path, "frontend", "node_modules", "vite", "bin", "vite.js"
)
if node and os.path.exists(node) and os.path.exists(vite_bin):
env["FRONTEND_PORT"] = str(self.frontend_port)
env["BACKEND_PORT"] = "NONE"
return (
[node, "node_modules/vite/bin/vite.js"],
os.path.join(self.workspace_path, "frontend"),
"vite (bundled node, no bash)",
)
return [_resolve_bash(), "run.sh"], self.workspace_path, "bash run.sh"
async def _await_frontend_bind(self) -> None:
"""Poll `frontend_port` every _FRONTEND_BIND_POLL_INTERVAL until
something binds (Vite dev server) or we hit the timeout. Emits a
@@ -460,13 +502,16 @@ class AppRuntime:
pass
def _maybe_capture_error(self, text: str) -> None:
"""If a stderr/stdout line matches a known build-error pattern,
record it for the next agent-tool drain. Tests every line ,
cheap (single regex search) and only the matching ones land in
the buffer."""
if _ERROR_PATTERNS.search(text):
self.recent_errors.append(text.rstrip())
def p_maybe_capture_render_beacon(self, text: str) -> None:
if "[openswarm:app-ready]" in text:
self.set_render_ok()
elif "[openswarm:app-error]" in text:
idx = text.index("[openswarm:app-error]") + len("[openswarm:app-error]")
self.set_render_error(text[idx:].strip())
async def _pipe_stream(self, stream: Optional[asyncio.StreamReader], name: str) -> None:
if stream is None:
return
@@ -480,6 +525,7 @@ class AppRuntime:
self._broadcast(LogLine(name, text))
if name == "stderr" or name == "stdout":
self._maybe_capture_error(text)
self.p_maybe_capture_render_beacon(text)
except Exception:
logger.exception("log pipe error (%s) for %s", name, self.workspace_id)
@@ -638,6 +684,17 @@ class AppRuntimeManager:
return rt.drain_errors()
return []
def get_render_state_for_workspace(self, workspace_id: str) -> tuple[Optional[str], str]:
rt = self.runtimes.get(workspace_id) or self._idle_lru.get(workspace_id)
if rt is None:
return None, ""
return rt.render_state, rt.render_error_text
def reset_render_state_for_workspace(self, workspace_id: str) -> None:
rt = self.runtimes.get(workspace_id) or self._idle_lru.get(workspace_id)
if rt is not None:
rt.reset_render_state()
async def restart(self, workspace_id: str, workspace_path: Optional[str] = None) -> Optional[AppRuntime]:
rt = self.runtimes.get(workspace_id) or self._idle_lru.get(workspace_id)
if rt is None:
backend/apps/outputs/versions.py
+306
View File
@@ -0,0 +1,306 @@
"""Per-app version history, stored the way git stores history: content-addressed.
Each unique file's bytes are written ONCE to a per-app blob store (sha256 name,
zlib-compressed); a version is a tiny manifest mapping path -> blob digest. So a
run that changes one file out of fifty costs one new blob, not fifty, and 500
versions of a barely-changing app stay small. That keeps this feature from
making OpenSwarm feel heavier than it already is. We reuse the .swarm app
serializer (captures flat-inline AND webapp_template workspace apps; skips
node_modules/.venv/dist/.git, excludes .env), but never .swarm's pack() (it
refuses on secret-shaped fields, and a local snapshot must never decline to save
the user's own app). No git binary: it isn't guaranteed on a packaged Mac/Win."""
from __future__ import annotations
import hashlib
import json
import logging
import os
import shutil
import zlib
from datetime import datetime
from uuid import uuid4
from backend.apps.outputs.models import Output, OutputVersion
from backend.apps.outputs.workspace_io import _WALK_SKIP_DIRS, _save, load_output
from backend.apps.swarm.entities.apps import AppExportable
from backend.config.paths import OUTPUTS_VERSIONS_DIR, OUTPUTS_WORKSPACE_DIR
logger = logging.getLogger(__name__)
_MAX_FILE_BYTES = 25 * 1024 * 1024 # don't snapshot giant build artifacts
class _NullExportCtx:
"""serialize() wants an ExportContext but apps have no cross-refs to rewrite."""
def bundle_id_for(self, etype, local_id): # noqa: ARG002
return None
_NULL_CTX = _NullExportCtx()
def _app_dir(output_id: str) -> str:
return os.path.join(OUTPUTS_VERSIONS_DIR, output_id)
def _blobs_dir(output_id: str) -> str:
return os.path.join(_app_dir(output_id), "blobs")
def _manifests_dir(output_id: str) -> str:
return os.path.join(_app_dir(output_id), "manifests")
def _digest(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def _safe_join(folder: str, rel: str) -> str:
dest = os.path.realpath(os.path.join(folder, rel))
root = os.path.realpath(folder)
if dest != root and not dest.startswith(root + os.sep):
raise ValueError("version file path escapes the workspace")
return dest
def _write_blob(output_id: str, data: bytes, digest: str) -> None:
"""Store bytes once, content-addressed. A blob that already exists is the
whole point: that's a file unchanged since an earlier version, stored zero
extra times."""
path = os.path.join(_blobs_dir(output_id), digest)
if os.path.exists(path):
return
os.makedirs(_blobs_dir(output_id), exist_ok=True)
tmp = path + ".tmp"
with open(tmp, "wb") as f:
f.write(zlib.compress(data, 6))
os.replace(tmp, path)
def _read_blob(output_id: str, digest: str) -> bytes | None:
try:
with open(os.path.join(_blobs_dir(output_id), digest), "rb") as f:
return zlib.decompress(f.read())
except (OSError, zlib.error):
return None
def _read_manifest(output_id: str, version_id: str) -> dict | None:
try:
with open(os.path.join(_manifests_dir(output_id), f"{version_id}.json"), encoding="utf-8") as f:
return json.load(f)
except (OSError, json.JSONDecodeError, UnicodeDecodeError):
return None
def _latest_manifest(output_id: str) -> dict | None:
"""Newest manifest by write time; only read for the dedupe check so capture
stays O(1) reads rather than scanning every version's content."""
folder = _manifests_dir(output_id)
if not os.path.isdir(folder):
return None
js = [os.path.join(folder, f) for f in os.listdir(folder) if f.endswith(".json")]
if not js:
return None
newest = max(js, key=os.path.getmtime)
try:
with open(newest, encoding="utf-8") as f:
return json.load(f)
except (OSError, json.JSONDecodeError, UnicodeDecodeError):
return None
def _tree_hash(app_meta: dict, file_map: dict[str, str]) -> str:
"""Dedupe key over the snapshot's content: app metadata + path->digest map.
Cheap because the per-file hashing already happened to name the blobs."""
h = hashlib.sha256()
h.update(json.dumps(app_meta, sort_keys=True).encode("utf-8"))
h.update(b"\0")
for path in sorted(file_map):
h.update(path.encode("utf-8"))
h.update(b"\0")
h.update(file_map[path].encode("utf-8"))
h.update(b"\0")
return h.hexdigest()
def _snapshot(output: Output) -> tuple[dict, dict[str, bytes]]:
app = AppExportable(output)
return app.serialize(_NULL_CTX), app.files() # files keyed workspace/<rel>
def list_versions(output_id: str) -> list[OutputVersion]:
folder = _manifests_dir(output_id)
if not os.path.isdir(folder):
return []
out: list[OutputVersion] = []
for fname in os.listdir(folder):
if not fname.endswith(".json"):
continue
m = _read_manifest(output_id, fname[:-5])
if m is None:
continue
try:
out.append(OutputVersion(**m)) # ignores the heavy keys (app_meta, files)
except Exception:
logger.warning("skipping unreadable version manifest %s", fname)
out.sort(key=lambda v: v.created_at, reverse=True)
return out
def capture(
output_id: str,
*,
source: str = "auto",
label: str = "",
thumbnail: str | None = None,
) -> OutputVersion | None:
"""Snapshot current app state. Returns the existing latest version (no new
manifest) when nothing changed, so unchanged runs don't pile up. None only if
the app is gone."""
output = load_output(output_id)
if output is None:
return None
app_meta, files = _snapshot(output)
file_map: dict[str, str] = {}
for path, data in files.items():
if len(data) > _MAX_FILE_BYTES:
continue
file_map[path] = _digest(data)
tree = _tree_hash(app_meta, file_map)
latest = _latest_manifest(output_id)
parent_id = latest.get("id") if latest else None
if latest is not None and latest.get("tree_hash") == tree:
try:
return OutputVersion(**latest)
except Exception:
pass # corrupt latest: fall through and write a fresh manifest
for path, data in files.items():
d = file_map.get(path)
if d is not None:
_write_blob(output_id, data, d)
vid = uuid4().hex
manifest = {
"id": vid,
"created_at": datetime.now().isoformat(),
"label": label or "",
"source": source,
"parent_id": parent_id,
"thumbnail": thumbnail if thumbnail is not None else output.thumbnail,
"tree_hash": tree,
"app_meta": app_meta,
"files": file_map,
}
os.makedirs(_manifests_dir(output_id), exist_ok=True)
dest = os.path.join(_manifests_dir(output_id), f"{vid}.json")
tmp = dest + ".tmp"
with open(tmp, "w", encoding="utf-8") as f:
json.dump(manifest, f)
os.replace(tmp, dest)
return OutputVersion(**manifest)
def _restore_workspace(output_id: str, workspace_id: str, file_map: dict[str, str]) -> None:
"""Make the workspace match the snapshot. Deletes current authored files not
in it, then writes the rest FROM BLOBS, skipping any file already byte-correct
(so a restore writes only the diff). Keeps the live .env and build/cache dirs.
Per-file writes: a crash mid-restore leaves a mixed tree, but the pre_restore
backup restore() takes first is the real safety net."""
folder = os.path.join(OUTPUTS_WORKSPACE_DIR, workspace_id)
os.makedirs(folder, exist_ok=True)
targets: dict[str, str] = {
key[len("workspace/"):]: dig for key, dig in file_map.items() if key.startswith("workspace/")
}
for root, dirs, fnames in os.walk(folder):
dirs[:] = [d for d in dirs if d not in _WALK_SKIP_DIRS]
for fname in fnames:
if fname == ".env":
continue
full = os.path.join(root, fname)
rel = os.path.relpath(full, folder).replace(os.sep, "/")
if rel not in targets:
try:
os.remove(full)
except OSError:
pass
for rel, digest in targets.items():
dest = _safe_join(folder, rel)
if os.path.exists(dest):
try:
with open(dest, "rb") as f:
if _digest(f.read()) == digest:
continue # already correct: don't rewrite
except OSError:
pass
data = _read_blob(output_id, digest)
if data is None:
continue
os.makedirs(os.path.dirname(dest), exist_ok=True)
with open(dest, "wb") as f:
f.write(data)
def restore(output_id: str, version_id: str) -> Output | None:
"""Bring the app back to an earlier version, in place. Saves the current
state as a pre_restore version first so this is always undoable."""
output = load_output(output_id)
if output is None:
return None
manifest = _read_manifest(output_id, version_id)
if manifest is None:
return None
target_label = manifest.get("label") or "an earlier version"
capture(output_id, source="pre_restore", label=f"Before restoring '{target_label}'")
app_meta = manifest.get("app_meta") or {}
output.name = app_meta.get("name", output.name)
output.description = app_meta.get("description", output.description)
output.icon = app_meta.get("icon", output.icon)
# Presence, not truthiness: a snapshot's empty {} schema must restore as empty.
if app_meta.get("input_schema") is not None:
output.input_schema = app_meta["input_schema"]
output.files = app_meta.get("files") or {}
if manifest.get("thumbnail") is not None:
output.thumbnail = manifest["thumbnail"]
now = datetime.now().isoformat()
output.updated_at = now
output.preview_updated_at = now
if output.workspace_id:
_restore_workspace(output_id, output.workspace_id, manifest.get("files") or {})
_save(output)
return output
def branch(output_id: str, version_id: str) -> str | None:
"""Make a brand-new app from an earlier version. Reuses the .swarm importer,
which mints a fresh output id + workspace id and localizes a fresh .env."""
manifest = _read_manifest(output_id, version_id)
if manifest is None:
return None
app_meta = dict(manifest.get("app_meta") or {})
app_meta["name"] = f"{app_meta.get('name') or 'App'} (copy)"
files: dict[str, bytes] = {}
for path, digest in (manifest.get("files") or {}).items():
data = _read_blob(output_id, digest)
if data is not None:
files[path] = data
from backend.apps.swarm.exportable import RemapTable
try:
return AppExportable.import_(app_meta, files, RemapTable())
except Exception:
# A partial import leaves an orphan workspace dir (small); better than a 500.
logger.exception("branch import failed for %s/%s", output_id, version_id)
return None
def delete_all(output_id: str) -> None:
shutil.rmtree(_app_dir(output_id), ignore_errors=True)
backend/apps/outputs/versions_routes.py
+80
View File
@@ -0,0 +1,80 @@
"""HTTP surface for app version history. Thin: each route validates then
delegates to versions.py. Its own SubApp so the already-large outputs.py doesn't
grow, and so the agent-running restore guard lives next to the route, not in the
pure store. Prefix: /api/output_versions."""
import logging
import os
from contextlib import asynccontextmanager
from typing import Literal, Optional
from fastapi import HTTPException
from pydantic import BaseModel
from backend.apps.outputs import versions
from backend.apps.outputs.workspace_io import load_output
from backend.config.Apps import SubApp
from backend.config.paths import OUTPUTS_VERSIONS_DIR
logger = logging.getLogger(__name__)
@asynccontextmanager
async def output_versions_lifespan():
os.makedirs(OUTPUTS_VERSIONS_DIR, exist_ok=True)
yield
output_versions = SubApp("output_versions", output_versions_lifespan)
class CaptureRequest(BaseModel):
# Clients can only ask for auto/manual; pre_restore is set internally by restore.
source: Literal["auto", "manual"] = "manual"
label: str = ""
thumbnail: Optional[str] = None
@output_versions.router.get("/{output_id}")
async def list_output_versions(output_id: str):
return {"versions": [v.model_dump() for v in versions.list_versions(output_id)]}
@output_versions.router.post("/{output_id}")
async def capture_output_version(output_id: str, body: CaptureRequest):
v = versions.capture(
output_id, source=body.source, label=body.label, thumbnail=body.thumbnail
)
if v is None:
raise HTTPException(status_code=404, detail="Output not found")
return {"ok": True, "version": v.model_dump()}
@output_versions.router.post("/{output_id}/{version_id}/restore")
async def restore_output_version(output_id: str, version_id: str):
output = load_output(output_id)
if output is None:
raise HTTPException(status_code=404, detail="Output not found")
# Don't restore out from under a live builder run. The frontend disables the
# button while the agent is active; this is the backend half of that guard.
if output.session_id:
from backend.apps.agents.agent_manager import agent_manager
session = agent_manager.sessions.get(output.session_id)
if session and getattr(session, "status", None) in ("running", "waiting_approval"):
raise HTTPException(
status_code=409,
detail="This app is still being edited. Wait for the current change to finish, then try again.",
)
restored = versions.restore(output_id, version_id)
if restored is None:
raise HTTPException(status_code=404, detail="Version not found")
return {"ok": True, "output": restored.model_dump()}
@output_versions.router.post("/{output_id}/{version_id}/branch")
async def branch_output_version(output_id: str, version_id: str):
if load_output(output_id) is None:
raise HTTPException(status_code=404, detail="Output not found")
new_id = versions.branch(output_id, version_id)
if new_id is None:
raise HTTPException(status_code=404, detail="Version not found")
return {"ok": True, "new_output_id": new_id}
backend/apps/outputs/view_builder_templates.py
+89 -12
View File
@@ -214,6 +214,19 @@ def _bundled_archive_path_for(digest: str) -> str:
return os.path.join(_BUNDLED_ARCHIVE_DIR, f"node_modules.{digest}.tar.gz")
def _bundled_extracted_modules() -> str | None:
"""A node_modules tree shipped ALREADY EXTRACTED in resources (digest-tagged),
so a workspace can junction straight at it with ZERO extract. This skips the
~14s first-app tar-extract on Windows (the extract is dominated by Defender
scanning ~tens of thousands of small files as they're written; shipping it
extracted moves that scan to install time, once). Returns the read-only path
or None when no extracted tree is shipped (e.g. the Mac build, which ships
the .tar.gz and uses the extract path instead). vite only reads node_modules
(its optimize cache lives elsewhere), so a read-only shared tree is safe."""
cand = os.path.join(_BUNDLED_ARCHIVE_DIR, _warm_cache_digest(), "node_modules")
return cand if os.path.isdir(cand) else None
def _try_extract_bundled_archive(cache_dir: str, digest: str) -> bool:
"""Unpack the sha-tagged bundled archive into `cache_dir` if one
exists for the current template digest. Returns True on success,
@@ -273,27 +286,54 @@ def _warm_cache_dir() -> str:
return os.path.join(base, _warm_cache_digest())
def _warm_cache_is_complete(cache_modules: str) -> bool:
"""A populated node_modules/ dir is not proof of a *finished* install.
npm links package bins (node_modules/.bin/*) in the final phase, so an
install killed partway (e.g. Electron quit mid-warm) leaves the package
trees on disk but no .bin/. The old `os.path.isdir(node_modules)` check
then trusted that half-tree forever, every app symlinked to it, and
`npm run dev` died with `vite: command not found`. Require the one bin
every webapp-template app actually launches with so a partial cache is
treated as not-ready and repopulated instead of cached as good."""
return os.path.exists(os.path.join(cache_modules, ".bin", "vite"))
def _ensure_warm_cache() -> str | None:
"""Populate the warm-cache node_modules if missing. Returns the
absolute path to the populated `node_modules` directory, or None on
failure. Thread-safe; concurrent callers block on a single install
instead of racing. Idempotent and fast after the first call."""
"""Populate the warm-cache node_modules if missing or incomplete.
Returns the absolute path to the populated `node_modules` directory, or
None on failure. Thread-safe; concurrent callers block on a single
install instead of racing. Idempotent and fast after the first call."""
cache_dir = _warm_cache_dir()
cache_modules = os.path.join(cache_dir, "node_modules")
if os.path.isdir(cache_modules):
if _warm_cache_is_complete(cache_modules):
return cache_modules
# Prefer a pre-extracted bundled tree: junction the workspace straight at it,
# no tar-extract and no npm. This is the #9 first-app speed win on Windows.
bundled = _bundled_extracted_modules()
if bundled:
logger.info("webapp-template: using bundled pre-extracted node_modules (zero extract)")
return bundled
with _warm_cache_lock:
if os.path.isdir(cache_modules):
if _warm_cache_is_complete(cache_modules):
return cache_modules
# A node_modules that exists but flunks the completeness check is a
# half-finished install; wipe it so the rebuild below starts on clean
# ground instead of layering onto a broken tree.
if os.path.isdir(cache_modules):
shutil.rmtree(cache_modules, ignore_errors=True)
# Fast path: pre-built archive shipped inside the release. The
# build script generates this so users hitting OpenSwarm for the
# first time skip the ~22 s live `npm install`. Falls through on
# any failure so dev installs (no archive) keep working.
if _try_extract_bundled_archive(cache_dir, _warm_cache_digest()):
logger.info("webapp-template: warm cache ready from bundled archive")
return cache_modules
if _warm_cache_is_complete(cache_modules):
logger.info("webapp-template: warm cache ready from bundled archive")
return cache_modules
# Archive unpacked a tree without the launch bin; don't trust it.
shutil.rmtree(cache_modules, ignore_errors=True)
try:
os.makedirs(cache_dir, exist_ok=True)
# Copy package.json + lockfile (if it exists) into the cache
@@ -338,12 +378,48 @@ def _ensure_warm_cache() -> str | None:
(result.stderr or "")[-1500:],
)
return None
# Never hand back a tree the workspace can't actually launch from.
if not _warm_cache_is_complete(cache_modules):
logger.warning("webapp-template: warm-cache install left no .bin/vite; not caching")
return None
return cache_modules
except Exception as exc:
logger.warning("webapp-template warm-cache failed: %s", exc)
return None
def _try_link_dir(src: str, target: str) -> bool:
"""Point `target` at `src` as cheaply as possible. Prefer a symlink (instant,
shared, zero disk). On Windows os.symlink needs admin / Developer Mode, which
a normal user account lacks, so fall back to a directory junction (mklink /J,
no privilege required), then to a full copy as a last resort so even a
locked-down Windows box ends up with a usable node_modules. Returns True if
`target` now resolves to the dependency tree."""
try:
os.symlink(src, target)
return True
except OSError:
pass
if os.name == "nt":
try:
r = subprocess.run(
["cmd", "/c", "mklink", "/J", target, src],
capture_output=True, text=True, timeout=15,
)
if r.returncode == 0 and os.path.isdir(target):
return True
except Exception:
pass
try:
# Slow + uses disk, but guarantees the workspace can boot vite even when
# neither symlink nor junction is available.
shutil.copytree(src, target, dirs_exist_ok=True)
return True
except OSError as exc:
logger.warning("webapp-template link/copy failed (%s) for %s", exc, target)
return False
def _link_node_modules(workspace_dir: str) -> None:
"""After copytree, point the workspace's frontend/node_modules at
the warm-cache directory. Safe fallback; if the cache isn't ready,
@@ -379,10 +455,11 @@ def _link_node_modules(workspace_dir: str) -> None:
return
try:
os.makedirs(os.path.dirname(target), exist_ok=True)
os.symlink(cache_modules, target)
logger.info("webapp-template: linked %s -> %s", target, cache_modules)
except OSError as exc:
logger.warning("webapp-template symlink failed (%s) for %s", exc, workspace_dir)
logger.warning("webapp-template mkdir failed (%s) for %s", exc, workspace_dir)
return
if _try_link_dir(cache_modules, target):
logger.info("webapp-template: linked %s -> %s", target, cache_modules)
# ---------------------------------------------------------------------------
@@ -484,7 +561,7 @@ def warm_cache_in_background() -> None:
global _warm_cache_thread
if _warm_cache_thread is not None and _warm_cache_thread.is_alive():
return
node_done = os.path.isdir(os.path.join(_warm_cache_dir(), "node_modules"))
node_done = _warm_cache_is_complete(os.path.join(_warm_cache_dir(), "node_modules"))
venv_done = os.path.isfile(os.path.join(_warm_venv_dir(), ".populated"))
if node_done and venv_done:
return
backend/apps/outputs/webapp_template/frontend/run.sh
+12 -6
View File
@@ -40,12 +40,18 @@ fi
# Fast path: the seeder usually symlinks node_modules to a shared warm
# cache (~/.openswarm/cache/webapp_template_node_modules/<hash>), so the
# dependency install has already been done once and we can skip straight
# to vite. Only run npm install when node_modules is genuinely missing
# or empty — e.g. a workspace seeded before the warm-cache existed, or
# the user's cache was cleared.
if [ -d node_modules ] && [ -n "$(ls -A node_modules 2>/dev/null)" ]; then
echo "Dependencies already present — skipping install."
# to vite. A non-empty node_modules is NOT proof of a finished install
# (npm links .bin/* last, so a killed install leaves trees but no bin and
# vite dies with "command not found"); gate on the bin we actually launch.
if [ -e node_modules/.bin/vite ]; then
echo "Dependencies already present - skipping install."
else
# Incomplete tree. If node_modules is a SYMLINK to the shared warm cache,
# never install through it: that writes into the cache every other app
# shares (corruption) and stampedes when several apps boot at once. Drop
# the link and install a private tree so this app heals alone while the
# backend's background warmer rebuilds the shared cache for everyone else.
[ -L node_modules ] && rm -f node_modules
echo "Installing dependencies..."
"$NPM" install --prefer-offline --no-audit --no-fund
fi
@@ -54,7 +60,7 @@ echo "Building with development mode..."
# Prefer `npm run dev` (honors package.json script + flags). But the
# packaged build ships node.exe WITHOUT npm, so on a machine with no
# system npm we fall back to invoking vite directly through the bundled
# node — node_modules is already populated (warm-cache symlink or seed),
# node; node_modules is already populated (warm-cache symlink or seed),
# so vite's bin is present and this needs no package manager at all.
if command -v "$NPM" &>/dev/null || [[ "$NPM" != "npm" ]]; then
"$NPM" run dev
backend/apps/outputs/webapp_template/frontend/src/app/components/ErrorBoundary.tsx
+26
View File
@@ -35,8 +35,34 @@ class ErrorBoundary extends React.Component<ErrorBoundaryProps, ErrorBoundarySta
return { error };
}
componentDidMount(): void {
// Clean first mount: nothing caught, so the app is in a good render
// state and the ready beacon is allowed.
if (!this.state.error) {
window.__openswarm_render_failed = false;
window.__openswarm_last_error = '';
}
}
componentDidUpdate(_prevProps: ErrorBoundaryProps, prevState: ErrorBoundaryState): void {
// Fast Refresh retried the previously-broken subtree and it rendered:
// re-allow the ready beacon so index.tsx's vite:afterUpdate can report ok.
if (prevState.error && !this.state.error) {
window.__openswarm_render_failed = false;
window.__openswarm_last_error = '';
}
}
componentDidCatch(error: Error, errorInfo: React.ErrorInfo): void {
this.setState({ errorInfo });
// Suppress the post-mount "ready" beacon in index.tsx and re-arm the
// window-error gate: the app is not in a rendered state right now.
// Stash the message so the HMR handler can re-assert this error after
// an unrelated edit (which resets the host's render-state to None).
window.__openswarm_render_failed = true;
window.__openswarm_rendered = false;
window.__openswarm_last_error =
`${error?.message ?? String(error)}\n${errorInfo?.componentStack ?? ''}`.trim();
// Two channels so the OpenSwarm host's webview-preload bridge can
// pick this up regardless of which one it taps:
// 1. console.error — forwarded as a `[FRONTEND]` line into the
backend/apps/outputs/webapp_template/frontend/src/index.tsx
+52 -6
View File
@@ -3,21 +3,67 @@ import { createRoot } from 'react-dom/client';
import Main from './app/Main';
import ErrorBoundary from './app/components/ErrorBoundary';
console.log('[App] Bootstrapping React app');
// Render-health beacons the OpenSwarm host reads (via the forwarded preview
// console) to decide, at the end of an agent turn, whether the app renders.
// The ErrorBoundary covers React render crashes; the listeners here cover
// what never reaches a boundary: module-load / pre-mount throws and vite
// transform errors.
function reportRender(ok: boolean, detail?: string) {
if (ok) {
window.__openswarm_rendered = true;
// eslint-disable-next-line no-console
console.log('[openswarm:app-ready]');
} else {
// eslint-disable-next-line no-console
console.error('[openswarm:app-error]', detail ?? '');
}
}
// Gate on __openswarm_rendered so a throw inside a click handler after a good
// render (a bug, but not "the app won't render") doesn't block the turn.
window.addEventListener('error', (e) => {
if (!window.__openswarm_rendered) reportRender(false, e.message || String(e.error ?? e));
});
window.addEventListener('unhandledrejection', (e) => {
if (!window.__openswarm_rendered) reportRender(false, String(e.reason ?? e));
});
if (import.meta.hot) {
const hot = import.meta.hot;
hot.on('vite:error', (payload) => {
const err = payload?.err;
reportRender(false, err?.message || err?.plugin || 'vite error');
});
// Re-assert the real state after every HMR update: if the ErrorBoundary is
// still showing its fallback, report the error again (an unrelated edit that
// didn't fix it must not flip the gate to "ready"); otherwise report ready.
hot.on('vite:afterUpdate', () => {
if (window.__openswarm_render_failed) {
reportRender(false, window.__openswarm_last_error || 'app still failing to render');
} else {
reportRender(true);
}
});
}
const rootEl = document.getElementById('root');
if (!rootEl) {
console.error('[App] FATAL: #root element not found in DOM');
console.error('[openswarm:app-error]', '#root element not found in DOM');
} else {
// Wrap Main in an ErrorBoundary so any runtime crash from agent
// edits (missing imports, hook-rules violations, etc.) shows a
// readable error card in the preview pane instead of unmounting
// to a blank screen. The boundary also forwards the error via
// console.error + postMessage so the agent sees it on its next
// turn.
// to a blank screen. The boundary forwards the error via
// console.error + postMessage so the agent sees it on its next turn.
createRoot(rootEl).render(
<ErrorBoundary>
<Main />
</ErrorBoundary>,
);
console.log('[App] React root mounted');
// Defer a frame so a synchronous render crash sets __openswarm_render_failed
// (via the boundary) before we'd wrongly report ready.
requestAnimationFrame(() => {
if (window.__openswarm_render_failed) return;
reportRender(true);
});
}
backend/apps/outputs/webapp_template/frontend/src/vite-env.d.ts
Vendored
+7
View File
@@ -1,2 +1,9 @@
/// <reference types="vite/client" />
/// <reference types="vite-plugin-pages/client-react" />
// Render-health beacon flags the OpenSwarm App Builder host reads off the preview.
interface Window {
__openswarm_rendered?: boolean;
__openswarm_render_failed?: boolean;
__openswarm_last_error?: string;
}
backend/apps/outputs/webapp_template/frontend/vite.config.ts
+7 -1
View File
@@ -52,7 +52,13 @@ export default defineConfig(({ mode }) => {
plugins: [
react(),
Pages({ dirs: 'src/pages', extensions: ['tsx'] }),
terminal({ console: 'terminal', output: ['terminal', 'console'] }),
// vite-plugin-terminal provides a `virtual:terminal/console` module
// that only exists in dev; loading it during `vite build` errors
// out, so the End-of-turn build-verify gate would fail on every
// brand-new workspace.
...(mode === 'development'
? [terminal({ console: 'terminal', output: ['terminal', 'console'] })]
: []),
],
resolve: {
alias: {
backend/apps/service/service.py
+17 -2
View File
@@ -33,6 +33,7 @@ logger = logging.getLogger(__name__)
_pulse_task: asyncio.Task | None = None
_drain_task: asyncio.Task | None = None
_9r_start_task: asyncio.Task | None = None
_last_9r_cost: float | None = None
_last_9r_prompt_tokens: int | None = None
@@ -121,7 +122,7 @@ async def _drain_loop():
@asynccontextmanager
async def service_lifespan():
global _pulse_task, _drain_task
global _pulse_task, _drain_task, _9r_start_task
try:
from backend.apps.settings.settings import load_settings, _save_settings
@@ -193,7 +194,13 @@ async def service_lifespan():
try:
from backend.apps.nine_router import ensure_running as ensure_9router
await ensure_9router()
# Start 9Router in the BACKGROUND instead of awaiting it here. Awaiting
# it was ~7s (up to ~18s cold) of the startup critical path, blocking the
# HTTP bind and the whole UI behind it. 9Router is only needed when the
# user sends an agent message, and the dispatch path calls ensure_running()
# itself (now serialized, so no double-spawn), so the first message waits
# for readiness lazily. This is the single biggest warm-startup win.
_9r_start_task = asyncio.create_task(ensure_9router())
except Exception as e:
logger.debug(f"9Router auto-start skipped: {e}")
@@ -218,6 +225,14 @@ async def service_lifespan():
pass
_drain_task = None
if _9r_start_task and not _9r_start_task.done():
_9r_start_task.cancel()
try:
await _9r_start_task
except (asyncio.CancelledError, Exception):
pass
_9r_start_task = None
try:
from backend.apps.nine_router import stop as stop_9router
stop_9router()
backend/apps/settings/models.py
+3
View File
@@ -76,6 +76,9 @@ class AppSettings(BaseModel):
free_trial_token: Optional[str] = None
free_trial_remaining: Optional[int] = None
free_trial_runs_limit: Optional[int] = None
# Epoch seconds when the rolling window refills to a fresh allotment; lets the spent-trial
# nudge say "fresh runs in ~3h" instead of a vague "for now". Server-owned.
free_trial_resets_at: Optional[float] = None
openswarm_subscription_plan: Optional[str] = None
openswarm_subscription_expires: Optional[str] = None
openswarm_usage_cached: Optional[dict] = None
backend/apps/settings/redaction.py
+68
View File
@@ -0,0 +1,68 @@
"""Redact secrets out of a settings view before an agent ever sees it.
The settings-meta read tool is always-on, so an always-on exfiltration risk: a
prompt-injected agent that could read raw settings could mail your API keys out.
So the read tool returns shape + state, never a secret VALUE. Keys are write-only
from the agent's side: it can SET a new one, never SEE the old.
The secret set is derived from field NAMES, not a hand-kept list that silently
drifts the day someone adds a new credential. Rule: a field whose name ends in
`_key`, `_token`, or `_secret` is a secret, plus installation_id (a stable
machine fingerprint that isn't a credential but still shouldn't leak). A test
asserts every field the settings PUT path already treats as secret is caught
here, so the two can't diverge.
"""
from __future__ import annotations
from typing import Any
from backend.common.secret_scan import looks_secret
_SECRET_NAME_SUFFIXES = ("_key", "_token", "_secret")
# Not a credential and doesn't match the suffix rule, but a stable hardware-ish
# fingerprint used for cohorting/abuse; keep it out of the agent's eyes too.
_SECRET_EXTRA_FIELDS = frozenset({"installation_id"})
def is_secret_field(name: str) -> bool:
return name.endswith(_SECRET_NAME_SUFFIXES) or name in _SECRET_EXTRA_FIELDS
def _value_is_secret_shaped(value: Any) -> bool:
"""Fail-safe behind the name rule: a field the name rule misses (a future
secret with an off-convention name) is still caught if its VALUE looks like
a credential (sk-..., ghp_..., Bearer ...). So a leak needs BOTH a bad name
AND a non-credential-shaped value, not just one."""
return isinstance(value, str) and looks_secret(value)
def _redact_value(value: Any) -> dict[str, Any]:
"""A secret rendered as state, never content: configured + last 4 only."""
if value is None or (isinstance(value, str) and value.strip() == ""):
return {"configured": False}
last4 = value[-4:] if isinstance(value, str) and len(value) >= 4 else None
return {"configured": True, "last4": last4}
def redact_settings(raw: dict[str, Any]) -> dict[str, Any]:
"""Return a copy of a settings dict with every secret value collapsed to
{configured, last4}. Nested custom-provider api_keys are redacted too."""
out: dict[str, Any] = {}
for key, value in raw.items():
if is_secret_field(key) or _value_is_secret_shaped(value):
out[key] = _redact_value(value)
elif key == "custom_providers" and isinstance(value, list):
out[key] = [_redact_custom_provider(cp) for cp in value]
else:
out[key] = value
return out
def _redact_custom_provider(cp: Any) -> Any:
if not isinstance(cp, dict):
return cp
out = dict(cp)
if "api_key" in out:
out["api_key"] = _redact_value(out.get("api_key"))
return out
backend/apps/settings/settings.py
+101 -1
View File
@@ -137,6 +137,7 @@ SERVER_OWNED_FIELDS = (
"free_trial_token",
"free_trial_remaining",
"free_trial_runs_limit",
"free_trial_resets_at",
"user_id",
"signin_method",
"installation_id",
@@ -146,14 +147,86 @@ SERVER_OWNED_FIELDS = (
)
import weakref as _weakref
# One serialization point for EVERY settings write (renderer PUT/PATCH + agent
# tool), so two writes can't interleave and clobber each other mid read-modify-
# write. Callers hold it across read->build->save; apply_settings_update itself
# does NOT acquire it (would deadlock the agent path that reads under it), so
# every caller wraps apply in it. Created lazily PER event loop: prod has one
# loop so it's effectively a singleton, but a module-level asyncio.Lock binds to
# the first loop that uses it and then errors on reuse from another loop (every
# async test spins a fresh one). WeakKeyDictionary auto-drops a loop's lock once
# the loop is gone.
_settings_write_locks: "_weakref.WeakKeyDictionary" = _weakref.WeakKeyDictionary()
def settings_write_lock() -> asyncio.Lock:
loop = asyncio.get_running_loop()
lock = _settings_write_locks.get(loop)
if lock is None:
lock = asyncio.Lock()
_settings_write_locks[loop] = lock
return lock
@settings.router.put("")
async def update_settings(body: AppSettings):
async with settings_write_lock():
saved = await apply_settings_update(body)
return {"ok": True, "settings": saved.model_dump()}
@settings.router.patch("")
async def patch_settings(changes: dict):
"""Save only the fields the user changed, merged onto the CURRENT on-disk
state. The renderer sends a diff (not a stale full object), so a save can't
clobber a field something else, an agent, an OAuth connect, changed
underneath it. Makes the lost-update unrepresentable: you can't overwrite a
field you never sent."""
async with settings_write_lock():
saved = await apply_settings_patch(changes)
return {"ok": True, "settings": saved.model_dump()}
async def apply_settings_patch(changes: dict) -> AppSettings:
"""Merge `changes` onto fresh on-disk settings and persist. Caller holds
settings_write_lock so the read is current. Reuses apply_settings_update for
every side effect: the object it hands over IS current state plus the diff,
which is exactly what a non-clobbering save means."""
valid = set(AppSettings.model_fields.keys())
data = load_settings().model_dump()
for k, v in changes.items():
if k in valid:
data[k] = v
return await apply_settings_update(AppSettings(**data))
async def apply_settings_update(body: AppSettings, protect_fields: set[str] | None = None) -> AppSettings:
"""Persist a full settings object with all the safety side effects: restore
server-owned fields, hand the wheel back from the free trial when a real
model is connected, reconcile 9router provider connections, and sync
analytics/identity. The PUT route and the agent settings tool both call this
so the write semantics can't drift between them. Returns the saved body.
Caller must hold settings_write_lock. `protect_fields` names credential fields
that must never be blanked by this write (the agent tool passes the field
powering the live run): a SECOND, independent wall behind the endpoint's
suicide-guard, so a guard bug still can't disconnect a run."""
from backend.apps.service.client import sync as _sync
old = load_settings()
for k in SERVER_OWNED_FIELDS:
setattr(body, k, getattr(old, k, None))
# Second wall: if a write tries to clear a credential that's currently set and
# flagged as powering this run, restore it (like server-owned fields). The
# endpoint guard already strips these; this is the backstop that can't be
# bypassed by a logic slip upstream.
for f in (protect_fields or ()):
if getattr(old, f, None) and not getattr(body, f, None):
setattr(body, f, getattr(old, f, None))
# If the user connects their own model while the free trial is armed, hand
# the wheel back to their provider. Without this, connection_mode (server-
# owned, so the loop above just restored it to "free-trial") would keep them
@@ -271,7 +344,7 @@ async def update_settings(body: AppSettings):
any_keyed_added,
))
return {"ok": True, "settings": body.model_dump()}
return body
class AppThemeOverridePayload(BaseModel):
@@ -320,6 +393,33 @@ async def reset_system_prompt():
return {"ok": True, "settings": current.model_dump()}
# A preferences reset (the iOS "Reset All Settings" analogue): everything back to
# defaults EXCEPT the things a "reset my preferences" click must never silently
# sever, your connections (server-owned subscription fields AND your pasted
# provider credentials) and your identity. Hard-erase is the separate flow.
_RESET_PRESERVE_FIELDS = SERVER_OWNED_FIELDS + (
"anthropic_api_key",
"openai_api_key",
"google_api_key",
"openrouter_api_key",
"custom_providers",
"user_name",
"user_email",
"analytics_opt_in",
"first_opened_at",
)
@settings.router.post("/reset-to-defaults")
async def reset_to_defaults():
old = load_settings()
fresh = AppSettings()
for k in _RESET_PRESERVE_FIELDS:
setattr(fresh, k, getattr(old, k, None))
await save_settings_async(fresh)
return {"ok": True, "settings": fresh.model_dump()}
class BrowseResponse(BaseModel):
current: str
parent: Optional[str]
backend/apps/skill_registry/skill_registry.py
+301 -5
View File
@@ -1,12 +1,15 @@
import asyncio
import json
import logging
import os
import re
import time
from contextlib import asynccontextmanager
from typing import Optional
import httpx
from fastapi import Query
from fastapi import HTTPException, Query
from pydantic import BaseModel
from backend.config.Apps import SubApp
logger = logging.getLogger(__name__)
@@ -17,12 +20,62 @@ RAW_BASE = f"https://raw.githubusercontent.com/{REPO}/{BRANCH}"
MANIFEST_URL = f"{RAW_BASE}/.claude-plugin/marketplace.json"
REFRESH_INTERVAL_S = 3600
CONCURRENT_FETCHES = 15
# Retry the startup fetch on this short backoff (capped) until the FIRST success,
# instead of waiting a full REFRESH_INTERVAL_S after a cold/slow/failed fetch.
# That 1h gap was the "skills empty until reboot" bug on cold Windows networks.
_RETRY_BACKOFF_START_S = 2
_RETRY_BACKOFF_MAX_S = 60
# Catalog ships in the repo so a brand-new install shows skills with zero network
# (build snapshot), and every successful live fetch is persisted to the user's
# cache so subsequent launches are instant + offline-safe. The live fetch always
# overwrites both once it lands, so neither can go stale at runtime.
_BUNDLED_SNAPSHOT = os.path.join(os.path.dirname(__file__), "skills_snapshot.json")
_cache: dict[str, dict] = {}
_cache_updated_at: float = 0
_refresh_task: Optional[asyncio.Task] = None
def _disk_cache_path() -> str:
base = os.environ.get("OPENSWARM_SKILL_CACHE_DIR") or os.path.expanduser(
"~/.openswarm/cache"
)
return os.path.join(base, "skill_registry.json")
def _load_seed_cache() -> dict[str, dict]:
"""Return a non-empty catalog from the on-disk last-good cache, falling back
to the bundled snapshot, so the registry is never empty on a cold/offline
start. Returns {} only if neither source is present/valid."""
for path in (_disk_cache_path(), _BUNDLED_SNAPSHOT):
try:
with open(path, encoding="utf-8") as f:
data = json.load(f)
if isinstance(data, dict) and data:
logger.info(f"Skill registry: seeded {len(data)} skills from {os.path.basename(path)}")
return data
except (OSError, ValueError):
continue
return {}
def _save_disk_cache(skills: dict[str, dict]) -> None:
"""Persist the last good live fetch so the next launch is instant. Atomic
replace so a crash mid-write can't leave a truncated cache."""
if not skills:
return
path = _disk_cache_path()
try:
os.makedirs(os.path.dirname(path), exist_ok=True)
tmp = f"{path}.tmp"
with open(tmp, "w", encoding="utf-8") as f:
json.dump(skills, f)
os.replace(tmp, path)
except OSError:
logger.debug("Skill registry: could not persist disk cache", exc_info=True)
def _parse_frontmatter(raw: str) -> tuple[dict, str]:
"""Split YAML frontmatter from markdown body."""
if not raw.startswith("---"):
@@ -114,18 +167,37 @@ async def _fetch_all_skills() -> dict[str, dict]:
async def _refresh_loop():
global _cache, _cache_updated_at
backoff = _RETRY_BACKOFF_START_S
while True:
ok = False
try:
_cache = await _fetch_all_skills()
_cache_updated_at = time.time()
fetched = await _fetch_all_skills()
if fetched:
_cache = fetched
_cache_updated_at = time.time()
_save_disk_cache(_cache)
ok = True
except Exception as e:
logger.exception(f"Skill registry refresh error: {e}")
await asyncio.sleep(REFRESH_INTERVAL_S)
if ok:
# Settle to the slow hourly refresh once we have a good catalog.
backoff = _RETRY_BACKOFF_START_S
await asyncio.sleep(REFRESH_INTERVAL_S)
else:
# Cold/slow/failed fetch: retry soon (capped) until the first success
# so a transient network hiccup doesn't leave the catalog empty for
# an hour. The seeded snapshot keeps it non-empty meanwhile.
await asyncio.sleep(backoff)
backoff = min(backoff * 2, _RETRY_BACKOFF_MAX_S)
@asynccontextmanager
async def skill_registry_lifespan():
global _refresh_task
global _refresh_task, _cache
# Seed instantly from disk/bundled snapshot so the very first request never
# sees an empty catalog (the live fetch below overwrites it when it lands).
if not _cache:
_cache = _load_seed_cache()
_refresh_task = asyncio.create_task(_refresh_loop())
yield
if _refresh_task:
@@ -159,7 +231,16 @@ async def registry_search(
offset: int = Query(0, ge=0),
sort: str = Query("name", description="Sort by: name"),
category: str = Query("", description="Filter by category"),
source: str = Query("curated", description="curated (vetted) | community (skills.sh wild registry)"),
):
# The wild registry is a remote 600k-entry index, searched live, not mirrored.
if source == "community":
try:
return await _community_search(q, limit)
except Exception as e:
logger.warning(f"community skill search failed: {e}")
return {"skills": [], "total": 0, "offset": 0, "limit": limit, "source": "community", "error": "skills.sh unreachable"}
pool = list(_cache.values())
if category:
cat_lower = category.lower()
@@ -197,3 +278,218 @@ async def registry_detail(skill_name: str):
if not sk:
return {"error": "Skill not found"}, 404
return {"skill": sk}
# ---------------------------------------------------------------------------
# Community source: the skills.sh wild registry (~600k+ telemetry-ranked,
# zero-curation community skills, GitHub-repo backed). The curated source above
# (anthropics/skills) stays the default; community is opt-in via ?source=community
# and the UI flags it as unvetted. See .claude/SECURITY.md for the posture: this
# installs INERT files only (never executes), discloses scripts before commit,
# and any skill script later runs through the same gated Bash path as anything.
# ---------------------------------------------------------------------------
_COMMUNITY_SEARCH_URL = "https://skills.sh/api/search"
_GH_API = "https://api.github.com"
_GH_RAW = "https://raw.githubusercontent.com"
_MAX_SKILL_FILES = 60
_SCRIPT_EXTS = (".sh", ".py", ".js", ".mjs", ".cjs", ".ts", ".rb", ".pl", ".ps1", ".bat", ".php")
def _is_script_path(rel: str) -> bool:
"""Whether a skill file is executable code worth disclosing before install."""
if rel.lower().endswith(_SCRIPT_EXTS):
return True
head = rel.split("/", 1)[0].lower()
return head in ("scripts", "bin", "hooks")
def _github_headers() -> dict:
"""GitHub request headers, with auth if a token is set. Unauthenticated is
60 req/hr/IP (fine for the odd install, the wall for a power user); a token
(OPENSWARM_GITHUB_TOKEN or GITHUB_TOKEN) raises it to 5000/hr."""
headers = {"User-Agent": "openswarm-skill-registry", "Accept": "application/vnd.github+json"}
token = os.environ.get("OPENSWARM_GITHUB_TOKEN") or os.environ.get("GITHUB_TOKEN")
if token:
headers["Authorization"] = f"Bearer {token}"
return headers
def _select_skill_paths(tree: list[dict], skill_id: str) -> tuple[str, list[str]]:
"""From a GitHub recursive tree, pick the SKILL.md for `skill_id` and every
file beside it. Pure, so the resolution logic is unit-tested without a network
round-trip. When a repo has several `<x>/<skill_id>/SKILL.md` matches the pick
is deterministic: prefer a top-level `<skill_id>/`, then `skills/<skill_id>/`,
then the shallowest, then alphabetical, never an arbitrary tie."""
blobs = [t["path"] for t in tree if t.get("type") == "blob" and isinstance(t.get("path"), str)]
candidates = [p for p in blobs if p.endswith(f"/{skill_id}/SKILL.md") or p == f"{skill_id}/SKILL.md"]
if not candidates:
raise ValueError(f"no SKILL.md for '{skill_id}' in this repo")
def _rank(p: str) -> tuple:
if p == f"{skill_id}/SKILL.md":
return (0, 0, p)
if p == f"skills/{skill_id}/SKILL.md":
return (1, p.count("/"), p)
return (2, p.count("/"), p)
skill_md = min(candidates, key=_rank)
skill_dir = skill_md[: -len("/SKILL.md")] if "/" in skill_md else ""
prefix = (skill_dir + "/") if skill_dir else ""
members = [p for p in blobs if (p.startswith(prefix) if prefix else "/" not in p)]
return skill_md, members[:_MAX_SKILL_FILES]
class RegistryRateLimited(Exception):
"""GitHub's unauthenticated API (60/hr) is exhausted; the caller surfaces a
'try again shortly' rather than a generic failure."""
async def _tree_at(client: httpx.AsyncClient, owner: str, repo: str, branch: str):
"""(tree | None) for a branch. None on 404 (branch absent); raises on 403."""
r = await client.get(f"{_GH_API}/repos/{owner}/{repo}/git/trees/{branch}?recursive=1")
if r.status_code == 200:
return r.json().get("tree", [])
if r.status_code == 403:
raise RegistryRateLimited()
return None
async def _fetch_repo_tree(client: httpx.AsyncClient, owner: str, repo: str) -> tuple[str, list[dict]]:
"""Recursive tree of owner/repo. Tries main then master first (one call, the
99% case, no quota wasted on a repo-meta lookup); only if BOTH are absent
does it ask the repo for its real default branch (handles develop/trunk/etc).
Raises RegistryRateLimited on a 403, ValueError if no branch resolves."""
for branch in ("main", "master"):
tree = await _tree_at(client, owner, repo, branch)
if tree is not None:
return branch, tree
meta = await client.get(f"{_GH_API}/repos/{owner}/{repo}")
if meta.status_code == 403:
raise RegistryRateLimited()
if meta.status_code == 200:
default = meta.json().get("default_branch")
if default and default not in ("main", "master"):
tree = await _tree_at(client, owner, repo, default)
if tree is not None:
return default, tree
raise ValueError(f"repo {owner}/{repo} has no resolvable default branch")
async def resolve_community_skill(source: str, skill_id: str) -> dict:
"""Resolve a skills.sh entry (source='owner/repo', skill_id=folder name) to
its files via the GitHub trees API. Returns name/description/repo_url plus
{relpath: content} and the list of script files. Fetches text only; never
runs anything. Raises ValueError on a bad source or a missing skill, and
RegistryRateLimited when GitHub's anon API is exhausted."""
owner, _, repo = source.partition("/")
if not owner or not repo:
raise ValueError(f"unrecognized source '{source}' (expected owner/repo)")
async with httpx.AsyncClient(timeout=30.0, headers=_github_headers()) as client:
branch, tree = await _fetch_repo_tree(client, owner, repo)
skill_md, members = _select_skill_paths(tree, skill_id)
skill_dir = skill_md[: -len("/SKILL.md")] if "/" in skill_md else ""
prefix = (skill_dir + "/") if skill_dir else ""
files: dict[str, str] = {}
for p in members:
rel = p[len(prefix):] if prefix else p
raw = await client.get(f"{_GH_RAW}/{owner}/{repo}/{branch}/{p}")
if raw.status_code == 200:
files[rel] = raw.text
if "SKILL.md" not in files:
raise ValueError("SKILL.md could not be fetched")
meta, _body = _parse_frontmatter(files["SKILL.md"])
# Reuse the .swarm importer's content scan: flag files holding secret-shaped
# literals (the author's leaked key, or a sketchy skill) so the user sees it
# before installing from an unvetted repo.
from backend.common.secret_scan import find_secrets_in_files
secret_findings = find_secrets_in_files({rel: data.encode("utf-8", "ignore") for rel, data in files.items()})
return {
"name": meta.get("name") or skill_id,
"description": meta.get("description", ""),
"repo_url": f"https://github.com/{owner}/{repo}/tree/{branch}/{skill_dir}".rstrip("/"),
"skill_id": skill_id,
"files": files,
"scripts": sorted(rel for rel in files if _is_script_path(rel)),
"secret_findings": secret_findings,
}
async def _community_search(q: str, limit: int) -> dict:
"""Live-proxy a query to the skills.sh wild registry. Not cached: it's a
600k-entry remote index, so we search it on demand rather than mirror it."""
async with httpx.AsyncClient(timeout=15.0, headers={"User-Agent": "openswarm"}) as client:
r = await client.get(_COMMUNITY_SEARCH_URL, params={"q": q or "skill"})
r.raise_for_status()
data = r.json()
skills = []
for s in (data.get("skills") or [])[:limit]:
src = s.get("source", "")
try:
installs = int(s.get("installs") or 0)
except (TypeError, ValueError):
installs = 0
skills.append({
"name": s.get("name", ""),
"description": f"{installs:,} installs",
"folder": s.get("skillId", ""),
"category": src,
"repositoryUrl": f"https://github.com/{src}" if src else "",
"source": src,
"skillId": s.get("skillId", ""),
"installs": installs,
"community": True,
})
return {"skills": skills, "total": len(skills), "offset": 0, "limit": limit, "source": "community"}
class _InstallRequest(BaseModel):
source: str
skill_id: str
confirm: bool = False
@skill_registry.router.post("/install")
async def registry_install(req: _InstallRequest):
"""Install a community (skills.sh) skill, in two honest steps.
confirm=false (default): resolve + return a disclosure (the SKILL.md and the
list of files, flagging scripts) WITHOUT writing anything, so the user sees
exactly what they're about to install from an unvetted repo.
confirm=true: write the skill folder to ~/.claude/skills/. Files only; no
script is executed here. Curated skills install via the normal skills CRUD;
this endpoint is the wild-registry path."""
try:
resolved = await resolve_community_skill(req.source, req.skill_id)
except RegistryRateLimited:
raise HTTPException(status_code=429, detail="GitHub rate limit hit fetching this skill; try again in a few minutes.")
except ValueError as e:
raise HTTPException(status_code=404, detail=str(e))
except Exception as e:
raise HTTPException(status_code=502, detail=f"could not fetch skill: {e}")
disclosure = {
"name": resolved["name"],
"description": resolved["description"],
"repo_url": resolved["repo_url"],
"skill_md": resolved["files"].get("SKILL.md", ""),
"files": sorted(resolved["files"].keys()),
"scripts": resolved["scripts"],
"has_scripts": bool(resolved["scripts"]),
"secret_findings": resolved.get("secret_findings", []),
}
if not req.confirm:
return {"installed": False, "disclosure": disclosure}
from backend.apps.skills.skills import write_folder_skill, unique_skill_slug
# Never clobber an existing local skill that happens to share this slug; a
# wild-registry name collision lands as a copy instead of overwriting.
slug = unique_skill_slug(resolved["skill_id"])
skill = write_folder_skill(
slug,
resolved["files"],
{"name": resolved["name"], "description": resolved["description"]},
)
return {"installed": True, "skill": skill.model_dump(), "disclosure": disclosure}
backend/apps/skill_registry/skills_snapshot.json
+138
View File
@@ -0,0 +1,138 @@
{
"algorithmic-art": {
"category": "Example Skills",
"content": "",
"description": "Creating algorithmic art using p5.js with seeded randomness and interactive parameter exploration. Use this when users request creating art using code, generative art, algorithmic art, flow fields, or particle systems. Create original algorithmic art rather than copying existing artists' work to avoid copyright violations.",
"folder": "skills/algorithmic-art",
"name": "algorithmic-art",
"repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/algorithmic-art"
},
"brand-guidelines": {
"category": "Example Skills",
"content": "",
"description": "Applies Anthropic's official brand colors and typography to any sort of artifact that may benefit from having Anthropic's look-and-feel. Use it when brand colors or style guidelines, visual formatting, or company design standards apply.",
"folder": "skills/brand-guidelines",
"name": "brand-guidelines",
"repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/brand-guidelines"
},
"canvas-design": {
"category": "Example Skills",
"content": "",
"description": "Create beautiful visual art in .png and .pdf documents using design philosophy. You should use this skill when the user asks to create a poster, piece of art, design, or other static piece. Create original visual designs, never copying existing artists' work to avoid copyright violations.",
"folder": "skills/canvas-design",
"name": "canvas-design",
"repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/canvas-design"
},
"claude-api": {
"category": "Claude Api",
"content": "",
"description": "|-",
"folder": "skills/claude-api",
"name": "claude-api",
"repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/claude-api"
},
"doc-coauthoring": {
"category": "Example Skills",
"content": "",
"description": "Guide users through a structured workflow for co-authoring documentation. Use when user wants to write documentation, proposals, technical specs, decision docs, or similar structured content. This workflow helps users efficiently transfer context, refine content through iteration, and verify the doc works for readers. Trigger when user mentions writing docs, creating proposals, drafting specs, or similar documentation tasks.",
"folder": "skills/doc-coauthoring",
"name": "doc-coauthoring",
"repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/doc-coauthoring"
},
"docx": {
"category": "Document Skills",
"content": "",
"description": "Use this skill whenever the user wants to create, read, edit, or manipulate Word documents (.docx files). Triggers include: any mention of 'Word doc', 'word document', '.docx', or requests to produce professional documents with formatting like tables of contents, headings, page numbers, or letterheads. Also use when extracting or reorganizing content from .docx files, inserting or replacing images in documents, performing find-and-replace in Word files, working with tracked changes or comments, or converting content into a polished Word document. If the user asks for a 'report', 'memo', 'letter', 'template', or similar deliverable as a Word or .docx file, use this skill. Do NOT use for PDFs, spreadsheets, Google Docs, or general coding tasks unrelated to document generation.",
"folder": "skills/docx",
"name": "docx",
"repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/docx"
},
"frontend-design": {
"category": "Example Skills",
"content": "",
"description": "Guidance for distinctive, intentional visual design when building new UI or reshaping an existing one. Helps with aesthetic direction, typography, and making choices that don't read as templated defaults.",
"folder": "skills/frontend-design",
"name": "frontend-design",
"repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/frontend-design"
},
"internal-comms": {
"category": "Example Skills",
"content": "",
"description": "A set of resources to help me write all kinds of internal communications, using the formats that my company likes to use. Claude should use this skill whenever asked to write some sort of internal communications (status reports, leadership updates, 3P updates, company newsletters, FAQs, incident reports, project updates, etc.).",
"folder": "skills/internal-comms",
"name": "internal-comms",
"repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/internal-comms"
},
"mcp-builder": {
"category": "Example Skills",
"content": "",
"description": "Guide for creating high-quality MCP (Model Context Protocol) servers that enable LLMs to interact with external services through well-designed tools. Use when building MCP servers to integrate external APIs or services, whether in Python (FastMCP) or Node/TypeScript (MCP SDK).",
"folder": "skills/mcp-builder",
"name": "mcp-builder",
"repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/mcp-builder"
},
"pdf": {
"category": "Document Skills",
"content": "",
"description": "Use this skill whenever the user wants to do anything with PDF files. This includes reading or extracting text/tables from PDFs, combining or merging multiple PDFs into one, splitting PDFs apart, rotating pages, adding watermarks, creating new PDFs, filling PDF forms, encrypting/decrypting PDFs, extracting images, and OCR on scanned PDFs to make them searchable. If the user mentions a .pdf file or asks to produce one, use this skill.",
"folder": "skills/pdf",
"name": "pdf",
"repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/pdf"
},
"pptx": {
"category": "Document Skills",
"content": "",
"description": "Use this skill any time a .pptx file is involved in any way \u2014 as input, output, or both. This includes: creating slide decks, pitch decks, or presentations; reading, parsing, or extracting text from any .pptx file (even if the extracted content will be used elsewhere, like in an email or summary); editing, modifying, or updating existing presentations; combining or splitting slide files; working with templates, layouts, speaker notes, or comments. Trigger whenever the user mentions \\\"deck,\\\" \\\"slides,\\\" \\\"presentation,\\\" or references a .pptx filename, regardless of what they plan to do with the content afterward. If a .pptx file needs to be opened, created, or touched, use this skill.",
"folder": "skills/pptx",
"name": "pptx",
"repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/pptx"
},
"skill-creator": {
"category": "Example Skills",
"content": "",
"description": "Create new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy.",
"folder": "skills/skill-creator",
"name": "skill-creator",
"repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/skill-creator"
},
"slack-gif-creator": {
"category": "Example Skills",
"content": "",
"description": "Knowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like \"make me a GIF of X doing Y for Slack.",
"folder": "skills/slack-gif-creator",
"name": "slack-gif-creator",
"repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/slack-gif-creator"
},
"theme-factory": {
"category": "Example Skills",
"content": "",
"description": "Toolkit for styling artifacts with a theme. These artifacts can be slides, docs, reportings, HTML landing pages, etc. There are 10 pre-set themes with colors/fonts that you can apply to any artifact that has been creating, or can generate a new theme on-the-fly.",
"folder": "skills/theme-factory",
"name": "theme-factory",
"repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/theme-factory"
},
"web-artifacts-builder": {
"category": "Example Skills",
"content": "",
"description": "Suite of tools for creating elaborate, multi-component claude.ai HTML artifacts using modern frontend web technologies (React, Tailwind CSS, shadcn/ui). Use for complex artifacts requiring state management, routing, or shadcn/ui components - not for simple single-file HTML/JSX artifacts.",
"folder": "skills/web-artifacts-builder",
"name": "web-artifacts-builder",
"repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/web-artifacts-builder"
},
"webapp-testing": {
"category": "Example Skills",
"content": "",
"description": "Toolkit for interacting with and testing local web applications using Playwright. Supports verifying frontend functionality, debugging UI behavior, capturing browser screenshots, and viewing browser logs.",
"folder": "skills/webapp-testing",
"name": "webapp-testing",
"repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/webapp-testing"
},
"xlsx": {
"category": "Document Skills",
"content": "",
"description": "Use this skill any time a spreadsheet file is the primary input or output. This means any task where the user wants to: open, read, edit, or fix an existing .xlsx, .xlsm, .csv, or .tsv file (e.g., adding columns, computing formulas, formatting, charting, cleaning messy data); create a new spreadsheet from scratch or from other data sources; or convert between tabular file formats. Trigger especially when the user references a spreadsheet file by name or path \u2014 even casually (like \\\"the xlsx in my downloads\\\") \u2014 and wants something done to it or produced from it. Also trigger for cleaning or restructuring messy tabular data files (malformed rows, misplaced headers, junk data) into proper spreadsheets. The deliverable must be a spreadsheet file. Do NOT trigger when the primary deliverable is a Word document, HTML report, standalone Python script, database pipeline, or Google Sheets API integration, even if tabular data is involved.",
"folder": "skills/xlsx",
"name": "xlsx",
"repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/xlsx"
}
}
backend/apps/skills/models.py
+5
View File
@@ -12,6 +12,11 @@ class Skill(BaseModel):
command: str = ""
# Platform-shipped skills (e.g. App Builder): UI hides delete and DELETE returns 409, but content stays editable so users can tune them.
built_in: bool = False
# Multi-file skills live in ~/.claude/skills/<id>/ with a SKILL.md plus supporting files (scripts, templates).
# dir_path is set for those; empty for a legacy flat <id>.md skill. has_supporting_files flags extra files
# beyond SKILL.md so the prompt layer knows to point the agent at the folder for on-demand reading.
dir_path: str = ""
has_supporting_files: bool = False
class SkillCreate(BaseModel):
backend/apps/skills/skills.py
+203 -57
View File
@@ -2,6 +2,9 @@ import os
import json
import logging
import re
import tempfile
import threading
import time
from contextlib import asynccontextmanager
from fastapi import HTTPException
from backend.config.Apps import SubApp
@@ -16,15 +19,58 @@ from backend.config.paths import SKILLS_WORKSPACE_DIR
def _load_index() -> dict[str, dict]:
if os.path.exists(INDEX_PATH):
with open(INDEX_PATH) as f:
return json.load(f)
"""Read the skill index, never raising on a corrupt file. A truncated/garbled
index (e.g. a crash mid-write before atomic writes existed) is moved aside so
it's recoverable, and we start empty rather than bricking every skill op,
skills still list from their files with frontmatter/filename-derived names."""
if not os.path.exists(INDEX_PATH):
return {}
try:
with open(INDEX_PATH, encoding="utf-8") as f:
data = json.load(f)
if isinstance(data, dict):
return data
logger.warning("skills index was not an object; ignoring")
except (OSError, ValueError):
logger.warning("skills index unreadable; preserving aside and starting empty", exc_info=True)
try:
os.replace(INDEX_PATH, INDEX_PATH + ".corrupt")
except OSError:
pass
return {}
# Guards the index write so an atomic replace is never interleaved by another
# writer. Today every index write runs on the single backend event-loop thread
# (no await between a load and its save, so no lost-update race), but this stays
# correct if a save ever moves to a thread pool the way settings' did.
_index_write_lock = threading.Lock()
def _save_index(index: dict[str, dict]):
with open(INDEX_PATH, "w") as f:
json.dump(index, f, indent=2)
"""Atomic index write: tmp file + os.replace so a crash mid-write can't leave
a truncated index. Mirrors the settings store's write discipline."""
with _index_write_lock:
os.makedirs(SKILLS_DIR, exist_ok=True)
fd, tmp = tempfile.mkstemp(prefix=".skills_index.", suffix=".tmp", dir=SKILLS_DIR)
try:
with os.fdopen(fd, "w", encoding="utf-8") as f:
json.dump(index, f, indent=2)
# Windows: Defender can briefly lock the destination; one retry covers it.
for attempt in range(2):
try:
os.replace(tmp, INDEX_PATH)
return
except PermissionError:
if attempt == 1:
raise
time.sleep(0.05)
except Exception:
try:
os.unlink(tmp)
except OSError:
pass
raise
# Built-in skills shipped with OpenSwarm itself. Each entry describes a
@@ -122,30 +168,79 @@ async def skills_lifespan():
skills = SubApp("skills", skills_lifespan)
def _skill_md_path(skill_id: str) -> tuple[str | None, str]:
"""Resolve where a skill's markdown lives: (path, kind).
A skill is either a folder (~/.claude/skills/<id>/SKILL.md, multi-file) or a
legacy flat file (~/.claude/skills/<id>.md). Folder wins if both exist. The
one place that knows the layout, so get/update/delete never re-guess it."""
folder_md = os.path.join(SKILLS_DIR, skill_id, "SKILL.md")
if os.path.isfile(folder_md):
return folder_md, "folder"
flat_md = os.path.join(SKILLS_DIR, f"{skill_id}.md")
if os.path.isfile(flat_md):
return flat_md, "flat"
return None, "flat"
def _has_supporting_files(skill_dir: str) -> bool:
"""True if a skill folder ships anything beyond its SKILL.md (scripts, templates)."""
try:
return any(e != "SKILL.md" and not e.startswith(".") for e in os.listdir(skill_dir))
except OSError:
return False
def _build_skill(skill_id: str, content: str, md_path: str, kind: str, index: dict) -> Skill:
"""Assemble a Skill from disk + index, falling back to SKILL.md frontmatter
for a folder skill the index hasn't catalogued (e.g. hand-dropped)."""
meta = dict(index.get(skill_id, {}))
if kind == "folder" and ("name" not in meta or "description" not in meta):
fm = _parse_skill_frontmatter(content)
meta.setdefault("name", fm.get("name", ""))
meta.setdefault("description", fm.get("description", ""))
pretty = skill_id.replace("-", " ").replace("_", " ").title()
skill_dir = os.path.join(SKILLS_DIR, skill_id)
return Skill(
id=skill_id,
name=meta.get("name") or pretty,
description=meta.get("description", ""),
content=content,
file_path=md_path,
command=meta.get("command", skill_id),
built_in=bool(meta.get("built_in", False)),
dir_path=skill_dir if kind == "folder" else "",
has_supporting_files=(kind == "folder" and _has_supporting_files(skill_dir)),
)
def _sync_skills() -> list[Skill]:
"""Sync skills from the filesystem, updating the index."""
"""Sync skills from the filesystem, updating the index. Reads both layouts:
legacy flat <id>.md files and multi-file <id>/SKILL.md folders."""
index = _load_index()
result = []
seen: set[str] = set()
if os.path.exists(SKILLS_DIR):
for fname in os.listdir(SKILLS_DIR):
if fname.endswith(".md"):
fpath = os.path.join(SKILLS_DIR, fname)
with open(fpath) as f:
content = f.read()
if not os.path.exists(SKILLS_DIR):
return result
skill_id = fname.replace(".md", "")
meta = index.get(skill_id, {})
skill = Skill(
id=skill_id,
name=meta.get("name", fname.replace(".md", "").replace("-", " ").replace("_", " ").title()),
description=meta.get("description", ""),
content=content,
file_path=fpath,
command=meta.get("command", fname.replace(".md", "")),
built_in=bool(meta.get("built_in", False)),
)
result.append(skill)
for entry in os.listdir(SKILLS_DIR):
full = os.path.join(SKILLS_DIR, entry)
if os.path.isdir(full):
skill_id = entry
elif entry.endswith(".md"):
skill_id = entry[: -len(".md")]
else:
continue
if skill_id in seen:
continue
md_path, kind = _skill_md_path(skill_id)
if not md_path:
continue
with open(md_path, encoding="utf-8") as f:
content = f.read()
seen.add(skill_id)
result.append(_build_skill(skill_id, content, md_path, kind, index))
return result
@@ -224,42 +319,95 @@ async def get_skill(skill_id: str):
raise HTTPException(status_code=404, detail="Skill not found")
@skills.router.post("/create")
async def create_skill(body: SkillCreate):
slug = body.name.lower().replace(" ", "-")
fpath = os.path.join(SKILLS_DIR, f"{slug}.md")
def _safe_slug(raw: str) -> str:
slug = re.sub(r"[^a-zA-Z0-9_-]+", "-", (raw or "").strip().lower()).strip("-")
return slug or "skill"
with open(fpath, "w") as f:
f.write(body.content)
def _skill_exists(slug: str) -> bool:
return (
slug in _load_index()
or os.path.isfile(os.path.join(SKILLS_DIR, f"{slug}.md"))
or os.path.isdir(os.path.join(SKILLS_DIR, slug))
)
def unique_skill_slug(base: str) -> str:
"""A free slug for `base`, suffixing -2, -3, ... on collision. Lets a
registry install land beside a same-named skill instead of silently
overwriting the user's existing one."""
slug = _safe_slug(base)
if not _skill_exists(slug):
return slug
i = 2
while _skill_exists(f"{slug}-{i}"):
i += 1
return f"{slug}-{i}"
def write_folder_skill(skill_id: str, files: dict[str, str], meta: dict) -> Skill:
"""Write a multi-file skill folder (relpath -> content) under SKILLS_DIR and
index it. `files` must include a 'SKILL.md'. Shared by registry install and
zip/.swarm import. Relpaths that try to escape the skill folder (../, abs
paths) are dropped, an untrusted registry archive can't write outside its
own dir."""
slug = _safe_slug(skill_id)
base = os.path.join(SKILLS_DIR, slug)
base_abs = os.path.abspath(base)
# A folder write supersedes any legacy flat <slug>.md, so we never leave a
# phantom flat file shadowed by the folder (folder wins in _skill_md_path).
legacy_flat = os.path.join(SKILLS_DIR, f"{slug}.md")
if os.path.isfile(legacy_flat):
try:
os.remove(legacy_flat)
except OSError:
pass
os.makedirs(base, exist_ok=True)
for rel, content in files.items():
dest = os.path.abspath(os.path.join(base, rel))
if os.path.commonpath([base_abs, dest]) != base_abs:
logger.warning("skill import: dropped path-escape entry %r", rel)
continue
os.makedirs(os.path.dirname(dest), exist_ok=True)
with open(dest, "w", encoding="utf-8") as f:
f.write(content)
index = _load_index()
index[slug] = {
"name": body.name,
"description": body.description,
"command": body.command or slug,
"name": meta.get("name") or slug,
"description": meta.get("description", ""),
"command": meta.get("command", slug),
}
_save_index(index)
skill = Skill(
id=slug,
name=body.name,
description=body.description,
content=body.content,
file_path=fpath,
command=body.command or slug,
)
pass
md_path, kind = _skill_md_path(slug)
if not md_path:
raise HTTPException(status_code=400, detail="skill had no SKILL.md")
with open(md_path, encoding="utf-8") as f:
content = f.read()
return _build_skill(slug, content, md_path, kind, index)
@skills.router.post("/create")
async def create_skill(body: SkillCreate):
# All user skills are folders now (<id>/SKILL.md); flat files stay readable
# but are no longer written, so a skill's on-disk shape no longer depends on
# how it was created vs imported.
meta = {"name": body.name, "description": body.description}
if body.command:
meta["command"] = body.command
skill = write_folder_skill(body.name, {"SKILL.md": body.content}, meta)
return {"ok": True, "skill": skill.model_dump()}
@skills.router.put("/{skill_id}")
async def update_skill(skill_id: str, body: SkillUpdate):
fpath = os.path.join(SKILLS_DIR, f"{skill_id}.md")
if not os.path.exists(fpath):
md_path, kind = _skill_md_path(skill_id)
if not md_path:
raise HTTPException(status_code=404, detail="Skill not found")
if body.content is not None:
with open(fpath, "w") as f:
with open(md_path, "w", encoding="utf-8") as f:
f.write(body.content)
index = _load_index()
@@ -273,17 +421,10 @@ async def update_skill(skill_id: str, body: SkillUpdate):
index[skill_id] = meta
_save_index(index)
with open(fpath) as f:
with open(md_path, encoding="utf-8") as f:
content = f.read()
skill = Skill(
id=skill_id,
name=meta.get("name", skill_id),
description=meta.get("description", ""),
content=content,
file_path=fpath,
command=meta.get("command", skill_id),
)
skill = _build_skill(skill_id, content, md_path, kind, index)
return {"ok": True, "skill": skill.model_dump()}
@@ -299,9 +440,14 @@ async def delete_skill(skill_id: str):
"the next agent turn)."
),
)
fpath = os.path.join(SKILLS_DIR, f"{skill_id}.md")
if os.path.exists(fpath):
os.remove(fpath)
# Remove whichever layout exists: the whole folder, or the flat file.
import shutil
skill_dir = os.path.join(SKILLS_DIR, skill_id)
flat = os.path.join(SKILLS_DIR, f"{skill_id}.md")
if os.path.isdir(skill_dir):
shutil.rmtree(skill_dir, ignore_errors=True)
if os.path.isfile(flat):
os.remove(flat)
index.pop(skill_id, None)
_save_index(index)
return {"ok": True}
backend/apps/subscription/free_trial.py
+45 -2
View File
@@ -10,12 +10,14 @@ forced cheap model; this module only mirrors state into settings and 9Router.
from __future__ import annotations
import asyncio
import hashlib
import logging
import os
import platform
import re
import subprocess
import time
import httpx
@@ -138,6 +140,13 @@ async def clear_free_trial(settings_obj) -> None:
(so the UI knows it's spent) and never touches a real paid mode."""
if getattr(settings_obj, "connection_mode", "own_key") == "free-trial":
settings_obj.connection_mode = "own_key"
# arm() pinned default_model to "haiku" for the free run; once the wheel is
# handed back, don't let that forced pick linger (it'd silently default a
# real subscription user to Haiku). "sonnet" is the fresh default; the
# frontend's DefaultModelGuard reconciles it to a reachable model if the
# connected provider isn't Anthropic.
if getattr(settings_obj, "default_model", None) == "haiku":
settings_obj.default_model = "sonnet"
settings_obj.free_trial_token = None
await save_settings_async(settings_obj)
await _sync_routing(settings_obj)
@@ -151,7 +160,35 @@ async def arm_free_trial(settings_obj) -> dict:
mode = getattr(settings_obj, "connection_mode", "own_key")
if mode not in ("own_key", "free-trial"):
return {"armed": False, "reason": "other_mode"}
if _has_own_model(settings_obj) or await _has_connected_subscription():
own = _has_own_model(settings_obj)
has_sub = False
if not own:
# A subscription lives in 9Router, not settings, and 9Router now starts in
# the BACKGROUND (non-blocking boot), so at first-launch mint time it isn't
# up yet. Without this wait _has_connected_subscription() reads False and
# we'd arm the free trial OVER a real Claude/ChatGPT/Gemini sub, pinning the
# user to Haiku until they manually reload. Bring 9Router up so the sub is
# actually visible before we decide. Bounded + idempotent (shares the start
# lock with the boot auto-start), and skipped when a settings-level model
# already proves there's nothing to shadow.
try:
from backend.apps.nine_router import ensure_running as _ensure_9r
await _ensure_9r()
except Exception:
pass
# 9Router's /api/providers can lag /v1/models (what is_running probes) by a
# beat on a cold start, so a real sub can read as absent for a sub-second
# window. Re-check a few times before concluding "no sub", so we never arm
# over a sub that's merely still loading. CAPPED on purpose: a genuinely
# sub-less user exhausts these in ~1.2s and falls through to arm, so this
# never waits on a subscription that doesn't exist.
for _i in range(5):
if await _has_connected_subscription():
has_sub = True
break
if _i < 4:
await asyncio.sleep(0.3)
if own or has_sub:
# A real model exists now (key, custom provider, or a 9Router sub). If we
# were on the free lane, hand the wheel back instead of re-arming.
if mode == "free-trial":
@@ -227,8 +264,14 @@ async def refresh_free_trial(settings_obj) -> dict:
data = r.json()
remaining = int(data.get("runs_remaining") or 0)
settings_obj.free_trial_remaining = remaining
# Stash an absolute refill time so the spent nudge can say "fresh runs in ~3h". Set before
# clearing (clear keeps it) so it survives the hand-back to own_key. Relative -> absolute here
# because the client reads it much later than we fetched it.
resets_in = data.get("resets_in_seconds")
if isinstance(resets_in, (int, float)) and resets_in > 0:
settings_obj.free_trial_resets_at = time.time() + float(resets_in)
if remaining <= 0:
await clear_free_trial(settings_obj)
return {"connected": False, "runs_remaining": 0}
return {"connected": False, "runs_remaining": 0, "resets_at": getattr(settings_obj, "free_trial_resets_at", None)}
await save_settings_async(settings_obj)
return {"connected": True, "runs_remaining": remaining, "runs_limit": getattr(settings_obj, "free_trial_runs_limit", None)}
backend/apps/swarm/closure.py
+48 -5
View File
@@ -170,6 +170,26 @@ def swarm_filename(name: str) -> str:
# ---------- import: staging ----------
def validate_manifest(manifest: Manifest) -> None:
"""Structural integrity of the untrusted part of a .swarm. The checksum
covers entity payloads + files but NOT the manifest itself, so an attacker
can rewrite root/edges/paths freely; catch the breakages that would import
silently wrong (a root pointing nowhere, a duplicate id that drops an
entity, an edge or path that doesn't resolve inside the bundle)."""
seen: set[str] = set()
for e in manifest.entities:
if e.bundle_id in seen:
raise BundleError("bundle manifest has duplicate entity ids")
seen.add(e.bundle_id)
if not e.path.startswith("entities/") or ".." in e.path.split("/"):
raise BundleError("bundle manifest has an out-of-tree entity path")
if manifest.root.bundle_id not in seen:
raise BundleError("bundle manifest root is not one of its entities")
for edge in manifest.edges:
if edge.from_ not in seen or edge.to not in seen:
raise BundleError("bundle manifest has an edge to an unknown entity")
def stage_upload(raw: bytes, filename: str) -> tuple[str, Manifest, list[str]]:
warnings: list[str] = []
if is_zip(raw):
@@ -179,6 +199,7 @@ def stage_upload(raw: bytes, filename: str) -> tuple[str, Manifest, list[str]]:
raw_manifest = read_manifest(sandbox)
verify_checksum(sandbox, raw_manifest)
manifest = Manifest(**raw_manifest)
validate_manifest(manifest)
except BundleError:
shutil.rmtree(sandbox, ignore_errors=True)
raise
@@ -215,13 +236,27 @@ def _stage_skill_from_zip(raw: bytes, filename: str, warnings: list[str]):
if target is None:
raise BundleError("zip has no SKILL.md")
content = zf.read(target).decode("utf-8", errors="replace")
others = [n for n in zf.namelist() if not n.endswith("/") and n != target]
if others:
warnings.append("supporting files were not imported (a skill is a single markdown file)")
return _synth_single_skill(content, _name_from_filename(filename), warnings)
# Carry supporting files (scripts, templates) through as a folder skill,
# keyed relative to the SKILL.md's directory so a nested layout flattens
# onto the skill folder. Cap count + per-file size so a hostile zip can't
# balloon the install.
base_dir = target.rsplit("/", 1)[0] + "/" if "/" in target else ""
extra_files: dict[str, bytes] = {}
for n in zf.namelist():
if n.endswith("/") or n == target:
continue
rel = n[len(base_dir):] if base_dir and n.startswith(base_dir) else os.path.basename(n)
if not rel or rel.startswith("."):
continue
info = zf.getinfo(n)
if info.file_size > 2_000_000 or len(extra_files) >= 50:
warnings.append("some oversized/extra supporting files were skipped")
continue
extra_files[rel] = zf.read(n)
return _synth_single_skill(content, _name_from_filename(filename), warnings, extra_files)
def _synth_single_skill(content: str, name: str, warnings: list[str]):
def _synth_single_skill(content: str, name: str, warnings: list[str], extra_files: dict[str, bytes] | None = None):
bid = uuid4().hex
sandbox = tempfile.mkdtemp(prefix="swarm-import-")
edir = os.path.join(sandbox, "entities", bid)
@@ -230,6 +265,14 @@ def _synth_single_skill(content: str, name: str, warnings: list[str]):
payload = {"slug": slug, "name": name, "description": "", "command": slug, "content": content, "builtin": False}
with open(os.path.join(edir, "payload.json"), "w", encoding="utf-8") as f:
json.dump(payload, f)
# Supporting files ride the same entities/<bid>/files/<rel> channel the
# commit reader (_read_files) feeds into import_, so a zip-of-SKILL.md
# round-trips as a folder skill instead of getting flattened.
for rel, data in (extra_files or {}).items():
dest = _safe_join(edir, os.path.join("files", rel))
os.makedirs(os.path.dirname(dest), exist_ok=True)
with open(dest, "wb") as f:
f.write(data)
ref = EntityRef(type=EntityType.skill, bundle_id=bid, name=name, path=f"entities/{bid}")
manifest = Manifest(
bundle_id=uuid4().hex,
backend/apps/swarm/entities/dashboards.py
+12 -2
View File
@@ -39,7 +39,13 @@ class DashboardExportable:
for oid, card in (layout.get("view_cards") or {}).items():
bid = ctx.bundle_id_for(EntityType.app, oid)
if bid:
view_cards[bid] = {**card, "output_id": bid}
# parent_session_id tethers the app card to the agent that built it;
# it's a session id, so it remaps like spawned_by on browser cards.
parent = card.get("parent_session_id")
view_cards[bid] = {
**card, "output_id": bid,
"parent_session_id": ctx.bundle_id_for(EntityType.session, parent) if parent else None,
}
browser_cards = {}
for bkey, card in (layout.get("browser_cards") or {}).items():
c = dict(card)
@@ -78,7 +84,11 @@ class DashboardExportable:
for bid, card in (layout.get("view_cards") or {}).items():
noid = remap.local(bid)
if noid:
view_cards[noid] = {**card, "output_id": noid}
parent = card.get("parent_session_id")
view_cards[noid] = {
**card, "output_id": noid,
"parent_session_id": remap.local(parent) if parent else None,
}
browser_cards = {}
for _bkey, card in (layout.get("browser_cards") or {}).items():
nbid = "browser-" + uuid4().hex[:10]
backend/apps/swarm/entities/sessions.py
+40 -13
View File
@@ -1,10 +1,12 @@
"""SessionExportable: an agent card on a shared dashboard. We carry only the
recipe (name, model, mode, system prompt, allowed tools) and deliberately DROP
the chat transcript (privacy + size), runtime state, costs, the worktree path,
and active_mcps (importing must never silently grant tool access, per the gate).
Its MCP/actions, provider, and built-in mode become import requirements so the
importer is walked through enabling them. The dashboard re-points dashboard_id
after import."""
"""SessionExportable: an agent card on a shared dashboard. We carry the recipe
(name, model, mode, system prompt, allowed tools) AND the chat transcript so a
shared agent arrives with the conversation that produced it, that's the whole
point of sharing one. The transcript rides through the same scrub layer as every
payload, so any secret-shaped string in it is redacted before it leaves. We still
DROP runtime state, costs, the worktree path, and active_mcps: importing must
never silently grant tool access, per the gate. Its MCP/actions, provider, and
built-in mode become import requirements so the importer is walked through
enabling them. The dashboard re-points dashboard_id after import."""
from __future__ import annotations
from datetime import datetime, timezone
@@ -14,7 +16,14 @@ from ..exportable import DepRef, ExportContext, RemapTable
from ..models import EntityType, Requirement, RequirementKind
_BUILTIN_MODES = {"agent", "ask", "plan", "view-builder", "skill-builder"}
_KEEP = ("name", "provider", "model", "mode", "system_prompt", "allowed_tools", "max_turns", "thinking_level")
# Transcript fields ride along so the shared agent keeps its history; ids inside
# (message ids, branch ids, their parent/fork refs) are self-consistent within
# the one session file, so they carry verbatim with no remap.
_KEEP = (
"name", "provider", "model", "mode", "system_prompt", "allowed_tools",
"max_turns", "thinking_level",
"messages", "branches", "active_branch_id", "tool_group_meta",
)
class SessionExportable:
@@ -27,8 +36,17 @@ class SessionExportable:
@classmethod
def load(cls, local_id: str) -> "SessionExportable | None":
from backend.apps.agents.manager.session.session_store import _load_session_data
d = _load_session_data(local_id)
# Memory first, disk fallback, the same order duplicate_session uses.
# The live session holds the freshest transcript; a disk-only read would
# ship a stale one (missing the latest turns) or drop a just-created
# agent that hasn't flushed yet, so its card vanishes from the bundle.
from backend.apps.agents.agent_manager import agent_manager
sess = agent_manager.sessions.get(local_id)
if sess is not None:
d = sess.model_dump(mode="json")
else:
from backend.apps.agents.manager.session.session_store import _load_session_data
d = _load_session_data(local_id)
if d is None:
return None
return cls(local_id, d.get("name") or "Agent", d)
@@ -70,6 +88,14 @@ class SessionExportable:
from backend.apps.agents.manager.session.session_store import _save_session
sid = uuid4().hex
now = datetime.now(timezone.utc).isoformat()
# Older bundles (made before transcripts were carried) have no messages;
# fall back to a single empty main branch so the imported agent is valid.
branches = payload.get("branches") or {
"main": {"id": "main", "parent_branch_id": None, "fork_point_message_id": None, "created_at": now}
}
active_branch_id = payload.get("active_branch_id") or "main"
if active_branch_id not in branches:
active_branch_id = next(iter(branches), "main")
doc = {
"id": sid,
"name": payload.get("name") or "Agent",
@@ -81,9 +107,10 @@ class SessionExportable:
"allowed_tools": payload.get("allowed_tools") or [],
"max_turns": payload.get("max_turns"),
"thinking_level": payload.get("thinking_level") or "auto",
"messages": [],
"branches": {"main": {"id": "main", "parent_branch_id": None, "fork_point_message_id": None, "created_at": now}},
"active_branch_id": "main",
"messages": payload.get("messages") or [],
"branches": branches,
"active_branch_id": active_branch_id,
"tool_group_meta": payload.get("tool_group_meta") or {},
"active_mcps": [],
"dashboard_id": None, # the dashboard import re-points this
"browser_id": None,
backend/apps/swarm/entities/skills.py
+53 -25
View File
@@ -1,10 +1,13 @@
"""SkillExportable: skills are leaves (no deps, no requirements). An installed
skill is just a markdown file plus index metadata, so this also powers the
generic "import a .md or a zip-of-SKILL.md" path. Nothing here is secret, but
the body still rides the central scrub in case someone pasted a token into it."""
"""SkillExportable: skills are leaves (no deps, no requirements). A skill is
either a single markdown file or a folder (SKILL.md + supporting files like
scripts/templates), so this powers both the .swarm round-trip AND the generic
"import a .md or a zip-of-SKILL.md" path. Folder skills ride the entity files()
channel so their supporting files survive export/import. Nothing here is secret,
but the body still rides the central scrub in case someone pasted a token in."""
from __future__ import annotations
import os
import shutil
from backend.apps.skills import skills as store
from ..exportable import DepRef, ExportContext, RemapTable
@@ -14,17 +17,18 @@ from ..models import EntityType, Requirement
class SkillExportable:
type = EntityType.skill
def __init__(self, local_id: str, name: str, payload: dict):
def __init__(self, local_id: str, name: str, payload: dict, files: dict[str, bytes] | None = None):
self.local_id = local_id
self.name = name
self._payload = payload
self._files = files or {}
@classmethod
def load(cls, local_id: str) -> "SkillExportable | None":
fpath = os.path.join(store.SKILLS_DIR, f"{local_id}.md")
if not os.path.isfile(fpath):
md_path, kind = store._skill_md_path(local_id)
if not md_path:
return None
with open(fpath, encoding="utf-8") as f:
with open(md_path, encoding="utf-8") as f:
content = f.read()
meta = store._load_index().get(local_id, {})
name = meta.get("name") or local_id.replace("-", " ").replace("_", " ").title()
@@ -36,13 +40,16 @@ class SkillExportable:
"content": content,
"builtin": bool(meta.get("built_in", False)),
}
return cls(local_id, name, payload)
files: dict[str, bytes] = {}
if kind == "folder":
files = _read_supporting_files(os.path.join(store.SKILLS_DIR, local_id))
return cls(local_id, name, payload, files)
def serialize(self, ctx: ExportContext) -> dict:
return dict(self._payload)
def files(self) -> dict[str, bytes]:
return {}
return dict(self._files)
def dependencies(self) -> list[DepRef]:
return []
@@ -61,35 +68,56 @@ class SkillExportable:
def import_(cls, payload: dict, files: dict[str, bytes], remap: RemapTable) -> str:
base = (payload.get("slug") or payload.get("name") or "skill").lower().replace(" ", "-")
slug = _free_slug(base)
os.makedirs(store.SKILLS_DIR, exist_ok=True)
fpath = os.path.join(store.SKILLS_DIR, f"{slug}.md")
with open(fpath, "w", encoding="utf-8") as f:
f.write(payload.get("content", ""))
index = store._load_index()
# Imported skills are never builtin, even if the source tagged them so.
index[slug] = {
meta = {
"name": payload.get("name", slug),
"description": payload.get("description", ""),
"command": payload.get("command", slug),
}
store._save_index(index)
return slug
# Every imported skill lands as a folder (SKILL.md + any supporting files),
# one path for one-file and multi-file skills alike. write_folder_skill is
# path-traversal-safe, so an untrusted bundle can't escape the skill dir.
bundle = {"SKILL.md": payload.get("content", "")}
for rel, data in files.items():
bundle[rel] = data.decode("utf-8", errors="replace")
skill = store.write_folder_skill(slug, bundle, meta)
return skill.id
@classmethod
def rollback(cls, local_id: str) -> None:
fpath = os.path.join(store.SKILLS_DIR, f"{local_id}.md")
if os.path.exists(fpath):
os.remove(fpath)
skill_dir = os.path.join(store.SKILLS_DIR, local_id)
flat = os.path.join(store.SKILLS_DIR, f"{local_id}.md")
if os.path.isdir(skill_dir):
shutil.rmtree(skill_dir, ignore_errors=True)
if os.path.isfile(flat):
os.remove(flat)
index = store._load_index()
if local_id in index:
index.pop(local_id, None)
store._save_index(index)
def _read_supporting_files(skill_dir: str) -> dict[str, bytes]:
"""Every file in a skill folder except SKILL.md, as {relpath: bytes}."""
out: dict[str, bytes] = {}
for root, _dirs, names in os.walk(skill_dir):
for n in names:
full = os.path.join(root, n)
rel = os.path.relpath(full, skill_dir)
if rel == "SKILL.md" or n.startswith("."):
continue
try:
with open(full, "rb") as f:
out[rel] = f.read()
except OSError:
continue
return out
def _slug_taken(slug: str) -> bool:
return slug in store._load_index() or os.path.isfile(
os.path.join(store.SKILLS_DIR, f"{slug}.md")
return (
slug in store._load_index()
or os.path.isfile(os.path.join(store.SKILLS_DIR, f"{slug}.md"))
or os.path.isdir(os.path.join(store.SKILLS_DIR, slug))
)
backend/apps/swarm/redact.py
+11 -15
View File
@@ -23,15 +23,13 @@ _DENY_EXACT = {
"credentials", "sdk_session_id",
}
REDACTED = "[redacted]"
# Literal-secret shapes someone might paste into a file or skill body.
_CONTENT_PATTERNS = (
re.compile(r"sk-ant-[A-Za-z0-9_\-]{16,}"),
re.compile(r"sk-[A-Za-z0-9_\-]{16,}"),
re.compile(r"AIza[A-Za-z0-9_\-]{20,}"), # Google API key shape
re.compile(r"gh[pousr]_[A-Za-z0-9]{20,}"), # GitHub tokens
re.compile(r"Bearer\s+[A-Za-z0-9._\-]{16,}"),
# The secret-shape scanner moved to backend.common so skills + settings reuse it
# without reaching into swarm; re-exported here so ziputil/closure keep their API.
from backend.common.secret_scan import ( # noqa: E402
REDACTED,
find_secrets_in_files,
looks_secret as _looks_secret,
redact_secret_shapes as scrub_text,
)
@@ -42,12 +40,6 @@ def is_denied_key(key: str) -> bool:
return any(sub in k for sub in _DENY_SUBSTRINGS)
def scrub_text(text: str) -> str:
for pat in _CONTENT_PATTERNS:
text = pat.sub(REDACTED, text)
return text
def scrub_payload(value: Any) -> Any:
"""Recursively drop denied keys and redact secret-shaped strings in a
JSON-able structure. Returns a new structure; never mutates the input."""
@@ -79,3 +71,7 @@ def find_denied_keys(value: Any, _path: str = "") -> list[str]:
for i, v in enumerate(value):
found.extend(find_denied_keys(v, f"{_path}[{i}]"))
return found
# _looks_secret + find_secrets_in_files now come from backend.common.secret_scan
# (imported at the top); kept re-exported so ziputil's audit import is unchanged.
backend/apps/swarm/review.py
+1 -1
View File
@@ -30,5 +30,5 @@ def scan_app_files(files: dict[str, bytes]) -> ReviewSummary:
verdict = "warn" if findings else "clean"
if runnable:
verdict = "warn"
findings.insert(0, "This app runs code on your computer when you open it. Only import apps you trust.")
findings.insert(0, "This app runs code on your computer. Only import apps you trust.")
return ReviewSummary(verdict=verdict, findings=findings, scanned_files=scanned)
backend/apps/swarm/ziputil.py
+7 -1
View File
@@ -12,7 +12,7 @@ import shutil
import tempfile
import zipfile
from .redact import find_denied_keys
from .redact import find_denied_keys, find_secrets_in_files
MANIFEST_NAME = "manifest.json"
@@ -46,6 +46,12 @@ def pack(manifest: dict, payloads: dict[str, dict], files: dict[str, bytes]) ->
raise BundleError(
f"refusing to export: secret-shaped field(s) in {bid}: {leaked[:3]}"
)
leaky_files = find_secrets_in_files(files)
if leaky_files:
raise BundleError(
f"refusing to export: a secret-shaped value is in {leaky_files[0]}; "
"remove it (use an environment variable) and try again"
)
entries: dict[str, bytes] = {}
for bid, payload in payloads.items():
entries[f"entities/{bid}/payload.json"] = json.dumps(payload, indent=2).encode("utf-8")
backend/apps/tools_lib/oauth_tokens.py
+7
View File
@@ -27,6 +27,7 @@ _TOOL_NAME_TO_PROVIDER = {
"hubspot": "hubspot",
"discord": "discord",
"notion": "notion",
"github": "github",
# Built-in Google tool's name is "Google Workspace"; accept the bare
# "google" alias too for forward compatibility.
"google workspace": "google",
@@ -61,6 +62,12 @@ def _persist_cloud_tokens(tool: ToolDefinition, tokens: dict) -> None:
elif name == "notion":
tool.oauth_tokens = {"access_token": tokens.get("access_token", "")}
tool.connected_account_email = tokens.get("workspace_name", "Notion workspace")
elif name == "github":
# GitHub OAuth-App tokens don't expire and carry no refresh_token, so
# store the bare token; the cloud callback enriches `login` for the label.
tool.oauth_tokens = {"access_token": tokens.get("access_token", "")}
login = tokens.get("login")
tool.connected_account_email = f"@{login}" if login else ""
else:
tool.oauth_tokens = {
"access_token": tokens.get("access_token", ""),
backend/apps/tools_lib/tools_lib.py
+63 -7
View File
@@ -2,10 +2,11 @@ import asyncio
import json
import logging
import os
import re
import shutil
import time
from contextlib import asynccontextmanager
from typing import Any
from typing import Any, NamedTuple
from urllib.parse import urlencode
import httpx
@@ -51,11 +52,16 @@ async def tools_lib_lifespan():
tools_lib = SubApp("tools", tools_lib_lifespan)
# Bash defaults to "ask" because it can execute untrusted text from MCP tool
# outputs (Gmail, WebFetch); every other built-in is sandboxed by domain.
# Must match agent_manager._DEFAULTS so the Settings UI and the agent agree
# on what "no policy set" means.
_DEFAULT_BUILTIN_POLICIES = {"Bash": "ask"}
# Every built-in seeds to always_allow for a frictionless run. The agent's
# runtime guards in agent_manager (catastrophic-command match, OS-scheduling,
# sensitive-path gate) STILL force a prompt for the dangerous shapes even on
# always_allow, so the poisoned-MCP-output -> destructive-command case is
# still caught. Must match agent_manager._DEFAULTS (empty -> always_allow) so
# the Settings UI and the agent agree on what "no policy set" means.
_DEFAULT_BUILTIN_POLICIES: dict[str, str] = {}
# One-time marker: older installs seeded Bash="ask"; we lift them once.
_BASH_AUTOALLOW_MARKER = os.path.join(DATA_DIR, ".bash_autoallow_migrated")
def _ensure_default_permissions() -> None:
@@ -72,6 +78,17 @@ def _ensure_default_permissions() -> None:
for t in BUILTIN_TOOLS
}
merged = {**desired, **existing}
# One-time lift: installs seeded under the old default carry Bash="ask";
# raise them to always_allow once so shell commands stop prompting. The
# marker means a deliberate "ask" set afterward sticks (never re-flipped).
if not os.path.exists(_BASH_AUTOALLOW_MARKER):
if merged.get("Bash") == "ask":
merged["Bash"] = "always_allow"
try:
with open(_BASH_AUTOALLOW_MARKER, "w") as f:
f.write("1")
except OSError:
pass
if merged != existing:
save_builtin_permissions(merged)
@@ -188,6 +205,40 @@ async def list_builtin_tools():
return {"tools": [t.model_dump() for t in BUILTIN_TOOLS]}
class PolicySlot(NamedTuple):
"""Where a tool's permission policy is stored.
store == "builtin": policy lives in builtin_permissions under `key`.
store == "mcp": policy lives on the owning tool's tool_permissions[action];
`key` is that tool's id, or None when no such tool exists.
"""
store: str
key: str | None
action: str | None
def resolve_policy_slot(tool_name: str, tools: list[ToolDefinition]) -> PolicySlot:
"""Single source of truth for WHERE a tool's permission policy is stored, so the
dispatch gate (read) and the 'Always approve' writer (write) can never key it
differently. That divergence was the bug behind 'Always approve' acting like a
one-time accept: writes landed under the raw mcp__server__action name while the
gate read the parsed inner action, so the next call never saw the policy."""
bm = re.match(r"mcp__openswarm-browser-agent__(.+)", tool_name)
if bm:
return PolicySlot("builtin", bm.group(1), None)
im = re.match(r"mcp__openswarm-invoke-agent__(.+)", tool_name)
if im:
return PolicySlot("builtin", im.group(1), None)
m = re.match(r"mcp__([^_]+(?:-[^_]+)*)__(.+)", tool_name)
if m:
server_slug, action = m.group(1), m.group(2)
for t in tools:
if t.mcp_config and t.enabled and _sanitize_server_name(t.name) == server_slug:
return PolicySlot("mcp", t.id, action)
return PolicySlot("mcp", None, action)
return PolicySlot("builtin", tool_name, None)
def load_builtin_permissions() -> dict[str, str]:
if not os.path.exists(BUILTIN_PERMS_PATH):
return {}
@@ -383,7 +434,12 @@ async def discover_tools(tool_id: str):
tool_names = [t["name"] for t in raw_tools]
services, service_groups, all_read, all_write = _classify_services(tool_names, tool.name)
permissions: dict[str, Any] = {n: tool.tool_permissions.get(n, "ask") for n in tool_names}
# Read-only actions auto-allow by default (no prompt for safe, scoped reads);
# writes still default to "ask". Any choice the user already made is kept.
permissions: dict[str, Any] = {
n: tool.tool_permissions.get(n, "always_allow" if n in all_read else "ask")
for n in tool_names
}
permissions["_categories"] = {"read": all_read, "write": all_write}
permissions["_services"] = services
permissions["_service_groups"] = service_groups
backend/common/__init__.py
View File
backend/common/secret_scan.py
+46
View File
@@ -0,0 +1,46 @@
"""Shared secret-shape scanner: spot credential-shaped literals in text/files.
Lives in backend.common so the .swarm importer, the skills registry, and the
settings redactor all pull it DOWN from one place instead of one feature app
reaching sideways into another. It catches a secret by its SHAPE (sk-ant-...,
ghp_..., AIza...), which is the fail-safe behind name-based redaction: a key
that's misnamed (so a name rule misses it) still gets caught by its shape."""
from __future__ import annotations
import re
REDACTED = "[redacted]"
# Literal-secret shapes someone might paste into a file, skill body, or setting.
SECRET_SHAPE_PATTERNS = (
re.compile(r"sk-ant-[A-Za-z0-9_\-]{16,}"),
re.compile(r"sk-[A-Za-z0-9_\-]{16,}"),
re.compile(r"AIza[A-Za-z0-9_\-]{20,}"), # Google API key shape
re.compile(r"gh[pousr]_[A-Za-z0-9]{20,}"), # GitHub tokens
re.compile(r"Bearer\s+[A-Za-z0-9._\-]{16,}"),
)
def looks_secret(text: str) -> bool:
"""True if `text` contains a credential-shaped literal."""
return any(p.search(text) for p in SECRET_SHAPE_PATTERNS)
def redact_secret_shapes(text: str) -> str:
"""Replace every secret-shaped literal in `text` with the redacted marker."""
for p in SECRET_SHAPE_PATTERNS:
text = p.sub(REDACTED, text)
return text
def find_secrets_in_files(files: dict[str, bytes]) -> list[str]:
"""Paths of any file whose text body holds a secret-shaped literal. Binary
files (a null byte in the first 4KB) are skipped, they aren't pasted text."""
hits: list[str] = []
for path, data in files.items():
if b"\x00" in data[:4096]:
continue
if looks_secret(data.decode("utf-8", errors="ignore")):
hits.append(path)
return hits
backend/config/Apps.py
+12
View File
@@ -1,4 +1,5 @@
import os
import time
from fastapi import FastAPI, APIRouter
import debug
@@ -29,9 +30,20 @@ class MainApp:
@asynccontextmanager
async def lifespan(app: FastAPI):
async with AsyncExitStack() as stack:
# [perf] per-lifespan boot timing. debug() is a no-op in the
# packaged build, so without this the packaged backend.log has no
# per-SubApp markers and a cold-start stall can only be guessed at.
# One perf_counter + flushed print per app pins exactly which
# lifespan (or the cold first-touch I/O entering it) dominates.
_boot_t0 = time.perf_counter()
for sub_app in sub_apps:
debug(sub_app.name)
_t0 = time.perf_counter()
await stack.enter_async_context(sub_app.lifespan())
_dt = (time.perf_counter() - _t0) * 1000
if _dt > 50: # only flag a slow lifespan; keeps boot logs quiet
print(f"[perf] lifespan {sub_app.name} t={_dt:.0f}ms", flush=True)
print(f"[perf] lifespans-total t={(time.perf_counter() - _boot_t0) * 1000:.0f}ms", flush=True)
_port = os.environ.get("OPENSWARM_PORT", "8324")
print(f"\nCheck out the API docs at: http://127.0.0.1:{_port}/docs\n")
yield
backend/config/paths.py
+1
View File
@@ -25,6 +25,7 @@ MODES_DIR = os.path.join(DATA_ROOT, "modes")
DASHBOARDS_DIR = os.path.join(DATA_ROOT, "dashboards")
OUTPUTS_DIR = os.path.join(DATA_ROOT, "outputs")
OUTPUTS_WORKSPACE_DIR = os.path.join(DATA_ROOT, "outputs_workspace")
OUTPUTS_VERSIONS_DIR = os.path.join(DATA_ROOT, "outputs_versions")
SKILLS_WORKSPACE_DIR = os.path.join(DATA_ROOT, "skills_workspace")
DASHBOARD_LAYOUT_DIR = os.path.join(DATA_ROOT, "dashboard_layout")
BUILTIN_PERMISSIONS_PATH = os.path.join(DATA_ROOT, "builtin_permissions.json")
backend/main.py
+113 -1
View File
@@ -38,6 +38,7 @@ from backend.apps.settings.settings import settings
from backend.apps.mcp_registry.mcp_registry import mcp_registry
from backend.apps.skill_registry.skill_registry import skill_registry
from backend.apps.outputs.outputs import outputs
from backend.apps.outputs.versions_routes import output_versions
from backend.apps.dashboards.dashboards import dashboards
from backend.apps.swarm.swarm import swarm
from backend.apps.service.service import service
@@ -50,7 +51,7 @@ from fastapi.middleware.cors import CORSMiddleware
from fastapi import WebSocket, WebSocketDisconnect
import json
main_app = MainApp([health, agents, skills, tools_lib, modes, settings, mcp_registry, skill_registry, outputs, dashboards, swarm, service, subscription, auth, web, anthropic_proxy, workflows])
main_app = MainApp([health, agents, skills, tools_lib, modes, settings, mcp_registry, skill_registry, outputs, output_versions, dashboards, swarm, service, subscription, auth, web, anthropic_proxy, workflows])
app = main_app.app
# Generate per-install auth token BEFORE we bind the HTTP port. By the
@@ -743,6 +744,117 @@ async def mcp_meta(action: str, request: Request):
return JSONResponse({"error": f"unknown action: {action}"}, status_code=400)
@app.post("/api/settings-meta/{action}")
async def settings_meta(action: str, request: Request):
"""Back the openswarm-settings-meta stdio MCP server (agent-editable Settings).
Actions:
- read: the full settings object with every secret redacted to
configured/not (never the value), so an always-on read is never an
exfiltration path.
- write: apply a field -> value map. Three things can't be written, in
priority order: an unknown field (reported, not invented), a server-owned
subscription/connection field (managed by its dedicated flow), and the
credential powering THIS run (the no-suicide rule, enforced structurally
via resolve_powering_credential). Everything else is applied through the
same path PUT /api/settings uses, so 9router reconciliation and the
server-owned restore behave identically.
"""
from backend.apps.settings.store import load_settings
from backend.apps.settings.models import AppSettings
from backend.apps.settings.redaction import redact_settings
from backend.apps.settings.settings import SERVER_OWNED_FIELDS, apply_settings_update, settings_write_lock
from backend.apps.agents.session_credential import (
ALL_API_KEY_FIELDS, PoweringCredential, resolve_powering_credential, write_would_suicide,
)
from backend.apps.agents.agent_manager import agent_manager
from pydantic import ValidationError
body = await request.json()
parent_session_id = body.get("parent_session_id", "")
if action == "read":
return JSONResponse({"settings": redact_settings(load_settings().model_dump())})
if action == "write":
changes = body.get("changes")
if not isinstance(changes, dict) or not changes:
return JSONResponse({"error": "changes must be a non-empty object of field -> value"}, status_code=400)
valid_fields = set(AppSettings.model_fields.keys())
outcomes: dict[str, dict] = {}
# Serialize the read-modify-write: SettingsWrite goes through apply_settings_update,
# which awaits (so two autonomous agents would interleave and clobber each
# other's fields while BOTH got an "applied" result). The lock makes agent
# writes serial so the last load always sees the prior write. (Agent vs the
# renderer's own PUT stays the pre-existing full-object-replace race.)
async with settings_write_lock():
settings = load_settings()
session = agent_manager.sessions.get(parent_session_id) if parent_session_id else None
if session is not None:
powering = resolve_powering_credential(session.model, settings)
else:
# No live session to anchor the guard: fail safe, protect every credential.
powering = PoweringCredential(kind="unknown", provider="unknown", label="this run")
# The credential field(s) the second-wall restore in apply_settings_update
# must never let a write blank (independent of the per-field guard below).
if powering.kind == "unknown":
protect_fields = set(ALL_API_KEY_FIELDS)
elif powering.kind == "api_key" and powering.protected_field:
protect_fields = {powering.protected_field}
else:
protect_fields = set()
staged: dict = {}
for field, value in changes.items():
if field not in valid_fields:
outcomes[field] = {"status": "unknown", "reason": "not a settings field"}
elif field in SERVER_OWNED_FIELDS:
outcomes[field] = {"status": "refused", "reason": "managed by your subscription/connection; change it in the Subscription section"}
elif write_would_suicide(field, value, powering):
outcomes[field] = {"status": "refused", "reason": f"would disconnect {powering.label}, which is powering this run"}
else:
staged[field] = value
if staged:
merged = settings.model_dump()
merged.update(staged)
try:
new_body = AppSettings(**merged)
except ValidationError as e:
bad = {str(err["loc"][0]) for err in e.errors() if err.get("loc")}
for f in bad & set(staged.keys()):
outcomes[f] = {"status": "refused", "reason": "invalid value for this field"}
staged.pop(f, None)
new_body = None
if staged:
merged = settings.model_dump()
merged.update(staged)
new_body = AppSettings(**merged)
if staged and new_body is not None:
try:
await apply_settings_update(new_body, protect_fields=protect_fields)
for f in staged:
outcomes[f] = {"status": "applied"}
except Exception as e:
# Don't hand the agent an opaque 500; tell it which writes failed.
for f in staged:
outcomes[f] = {"status": "error", "reason": f"write failed: {e}"}
if any(o.get("status") == "applied" for o in outcomes.values()):
# An agent wrote settings (not the user via the modal), so nudge every
# open window to refetch instead of waiting for the next window-focus.
# Pure signal: the renderer refetches the authoritative state, so nothing
# (least of all a secret) needs to ride the broadcast.
from backend.apps.agents.core.ws_manager import ws_manager as _wsm
await _wsm.broadcast_global("settings:changed", {})
return JSONResponse({"outcomes": outcomes})
return JSONResponse({"error": f"unknown action: {action}"}, status_code=400)
@app.post("/api/agents/sessions/{session_id}/compact")
async def session_compact(session_id: str):
"""Force a compaction pass on a session (Phase 2 /compact slash cmd).
backend/tests/formal/README.md
+20
View File
@@ -0,0 +1,20 @@
# Formal proofs
Machine-checked proofs of safety/security invariants that the unit/property
tests can only *sample*. A property test tries thousands of cases; an SMT proof
is exhaustive over the modeled domain (assert the negation, `unsat` => theorem).
Not wired into prod or CI, and excluded from the packaged build (under `tests/`).
Run manually:
```
pip install z3-solver
python backend/tests/formal/mcp_gate_proof.py
```
- **`mcp_gate_proof.py`** , the MCP dispatch-gate invariant (`agent_manager._build_mcp_servers`):
a gated session forwards a server *only if* it was activated, an empty
activation list forwards zero, and a denied server is never forwarded. Sampled
by `tests/test_v2_invariants.py::test_mcp_gate_only_forwards_activated_servers`;
proven for all inputs here. The script also refutes a deliberately-buggy gate
(activation check dropped) so the proof can't be vacuous.
backend/tests/formal/mcp_gate_proof.py
+83
View File
@@ -0,0 +1,83 @@
"""Formal proof (Z3 / SMT) of the MCP dispatch-gate security invariant.
The product rule "MCP tools are reachable only after MCPActivate" is enforced at
dispatch in agent_manager._build_mcp_servers: for a gated session a server is
forwarded to the model only if its sanitized name is in session.active_mcps.
tests/test_v2_invariants.py::test_mcp_gate_only_forwards_activated_servers
SAMPLES that contract (400 random cases). This SMT proof is exhaustive over the
modeled domain: we assert the negation of each property and ask Z3 for a
counterexample. `unsat` means none can exist, so the property is a theorem,
true for every possible input, not just the ones a test happened to try.
Not wired into prod or CI. Run manually:
pip install z3-solver && python backend/tests/formal/mcp_gate_proof.py
"""
from z3 import And, Bool, Implies, Not, Or, Solver, sat, unsat
def forwarded(installed, allowed, denied, active_is_none, active_t):
"""Faithful model of the gate decision for one arbitrary server `t`
(agent_manager.py:165-203). A server ships to the model iff it is an
installed+configured MCP tool, passes the permission gate, isn't fully
denied, and EITHER the session is legacy (active_mcps is None) OR the
server is in active_mcps. Proving it for an arbitrary symbolic `t` proves
it for all servers."""
return And(installed, allowed, Not(denied), Or(active_is_none, active_t))
def buggy_forwarded(installed, allowed, denied, active_is_none, active_t):
"""The same gate with the activation check dropped, used to show the proof
has teeth: Z3 must be able to refute the no-leak property for this variant."""
return And(installed, allowed, Not(denied))
def prove(name: str, claim) -> bool:
"""`claim` should be valid (true for every input). Proven by showing its
negation is unsatisfiable."""
s = Solver()
s.add(Not(claim))
if s.check() == unsat:
print(f" PROVED: {name}")
return True
print(f" FAILED: {name} counterexample: {s.model()}")
return False
def main() -> None:
installed = Bool("installed")
allowed = Bool("allowed")
denied = Bool("denied")
active_is_none = Bool("active_is_none") # legacy session (no activation gate)
active_t = Bool("active_t") # server t is in active_mcps
fwd = forwarded(installed, allowed, denied, active_is_none, active_t)
gated = Not(active_is_none)
print("Proving MCP dispatch-gate invariants (exhaustive over all inputs):")
ok = True
# A. No leak: a gated session never forwards a non-activated server.
ok &= prove("gated => (forwarded(t) -> activated(t))",
Implies(And(gated, fwd), active_t))
# B. Empty activation => zero servers (no t is active, so none ship).
ok &= prove("gated & !activated(t) => !forwarded(t)",
Implies(And(gated, Not(active_t)), Not(fwd)))
# C. The permission gate still binds: a denied server is never forwarded.
ok &= prove("denied(t) => !forwarded(t)", Implies(denied, Not(fwd)))
# Teeth: the buggy gate (activation check dropped) MUST be refutable, else
# the proof above would be vacuous.
print("Sanity-checking the proof has teeth (a buggy gate must be refuted):")
bug = buggy_forwarded(installed, allowed, denied, active_is_none, active_t)
s = Solver()
s.add(Not(Implies(And(gated, bug), active_t)))
assert s.check() == sat, "buggy gate should leak but Z3 couldn't refute it"
print(f" REFUTED (as expected): a gate without the activation check leaks; "
f"counterexample = {s.model()}")
print("\nALL GATE PROPERTIES PROVED" if ok else "\nPROOF FAILED")
raise SystemExit(0 if ok else 1)
if __name__ == "__main__":
main()
backend/tests/test_bundled_extracted_modules.py
+68
View File
@@ -0,0 +1,68 @@
"""Tests for the #9 item 2 pre-extracted node_modules path.
The packaged Windows build ships node_modules ALREADY EXTRACTED in resources so a
workspace junctions straight at it with zero first-app tar-extract. _ensure_warm_cache
must prefer that tree over the .tar.gz / npm paths, and must be a no-op (return None)
when no tree is shipped so Mac and older builds fall back unchanged.
"""
import os
from backend.apps.outputs import view_builder_templates as vt
def test_prefers_bundled_extracted_tree(monkeypatch, tmp_path):
digest = vt._warm_cache_digest()
# Force a home-cache miss so we exercise the bundled path.
monkeypatch.setenv("OPENSWARM_WEBAPP_CACHE_DIR", str(tmp_path / "home"))
bundle = tmp_path / "resources_cache"
monkeypatch.setattr(vt, "_BUNDLED_ARCHIVE_DIR", str(bundle))
nm = bundle / digest / "node_modules" / "vite" / "bin"
nm.mkdir(parents=True)
(nm / "vite.js").write_text("// fake")
expected = str(bundle / digest / "node_modules")
assert vt._bundled_extracted_modules() == expected
# Zero extract / zero npm: returns the read-only resources tree directly.
assert vt._ensure_warm_cache() == expected
def test_no_bundled_tree_returns_none(monkeypatch, tmp_path):
# No extracted tree shipped (Mac / older builds): must not select it, so the
# caller falls through to the .tar.gz extract or live npm.
monkeypatch.setattr(vt, "_BUNDLED_ARCHIVE_DIR", str(tmp_path / "empty"))
assert vt._bundled_extracted_modules() is None
def test_warm_cache_is_complete_requires_launch_bin(tmp_path):
# A package tree on disk is NOT a finished install; the .bin/vite launch
# shim is what proves npm finished its bin-linking phase.
nm = tmp_path / "node_modules"
(nm / "vite" / "bin").mkdir(parents=True)
(nm / "vite" / "bin" / "vite.js").write_text("// vite")
assert vt._warm_cache_is_complete(str(nm)) is False
bindir = nm / ".bin"
bindir.mkdir()
(bindir / "vite").symlink_to("../vite/bin/vite.js")
assert vt._warm_cache_is_complete(str(nm)) is True
def test_ensure_warm_cache_wipes_partial_and_never_returns_incomplete(monkeypatch, tmp_path):
# A half-finished cache (package tree present, .bin/vite missing) must be
# WIPED and never handed back, so no workspace symlinks to an unlaunchable
# tree and run.sh is never pushed into installing through the shared cache.
digest = vt._warm_cache_digest()
home = tmp_path / "home"
monkeypatch.setenv("OPENSWARM_WEBAPP_CACHE_DIR", str(home))
cache_modules = home / digest / "node_modules"
(cache_modules / "vite" / "bin").mkdir(parents=True)
(cache_modules / "vite" / "bin" / "vite.js").write_text("// vite")
assert vt._warm_cache_is_complete(str(cache_modules)) is False
# No bundled tree, no archive, no npm: the only honest answer is "not ready"
# (None), and the broken tree must be gone, not cached for the next caller.
monkeypatch.setattr(vt, "_BUNDLED_ARCHIVE_DIR", str(tmp_path / "noresources"))
monkeypatch.setattr(vt, "_try_extract_bundled_archive", lambda *a, **k: False)
monkeypatch.setattr(vt, "_resolve_npm", lambda: None)
assert vt._ensure_warm_cache() is None
assert not cache_modules.exists()
backend/tests/test_error_classify.py
+45
View File
@@ -0,0 +1,45 @@
"""redact_for_telemetry is the wall between a model_error diagnostic and a key
leak: in own_key mode the subprocess stderr we now attach can echo the user's
provider key, so these tests pin that no secret shape survives while the actual
error text (the whole point of capturing stderr) does.
The secret-shaped inputs are built by concatenation on purpose: no contiguous
key-shaped literal lands in this source file (so it never trips gitleaks or
alarms a reader), yet the runtime values are still key-shaped enough to exercise
the scrub. None of these are real keys; they unlock nothing."""
from backend.apps.agents.core.error_classify import redact_for_telemetry
def test_redacts_provider_key_shapes_keeps_context():
anthropic = "sk-" + "ant-" + "A" * 28
openai = "sk-" + "B" * 24
google = "AIza" + "C" * 30
github = "ghp" + "_" + "D" * 24
s = f"9router: invalid x-api-key {anthropic} {openai} {google} {github}"
out = redact_for_telemetry(s)
for secret in (anthropic, openai, google, github):
assert secret not in out
assert "[redacted]" in out
# The diagnostic signal survives, that's the reason we capture stderr at all.
assert "9router: invalid x-api-key" in out
def test_redacts_bearer_and_key_value():
bearer_token = "E" * 24
kv_value = "F" * 16
s = "Authorization: " + "Bearer " + bearer_token + "\n" + "api_key=" + kv_value
out = redact_for_telemetry(s)
assert bearer_token not in out
assert kv_value not in out
def test_keeps_tail_and_bounds_length():
# The real error lands at the end of the stderr stream, so we keep the tail.
s = "old noise\n" * 500 + "Command failed: ENOENT spawn 9router"
out = redact_for_telemetry(s, limit=120)
assert len(out) <= 120
assert "Command failed: ENOENT spawn 9router" in out
def test_empty_is_safe():
assert redact_for_telemetry("") == ""
backend/tests/test_free_trial.py
+113 -1
View File
@@ -2,6 +2,8 @@
import backend # noqa: F401 (path sanity asserted below)
import pytest
from backend.apps.settings.models import AppSettings
from backend.apps.settings.credentials import proxy_auth
from backend.apps.agents.core.error_classify import (
@@ -9,7 +11,8 @@ from backend.apps.agents.core.error_classify import (
p_is_transient_capacity_error,
)
from backend.apps.agents.providers.registry import resolve_model_id_for_sdk
from backend.apps.subscription.free_trial import _has_own_model
from backend.apps.subscription import free_trial as ft
from backend.apps.subscription.free_trial import _has_own_model, arm_free_trial, clear_free_trial
def test_proxy_auth_for_each_mode():
@@ -66,3 +69,112 @@ def test_has_own_model_never_shadows_a_real_provider():
assert _has_own_model(
AppSettings(connection_mode="openswarm-pro", openswarm_bearer_token="b")
)
@pytest.mark.asyncio
async def test_arm_waits_for_9router_before_shadowing_a_background_started_sub(monkeypatch):
"""The regression: 9Router starts in the background, so at first-boot mint time
a real Claude sub is invisible. arm() must bring 9Router up (so the sub becomes
visible) BEFORE deciding, instead of arming the free trial over it."""
saved: list = []
monkeypatch.setattr(ft, "save_settings_async", _record(saved))
monkeypatch.setattr(ft, "_sync_routing", _noop)
started = {"called": False}
async def fake_ensure_running():
started["called"] = True # 9Router comes up here; the sub is now visible
# The sub is only reachable AFTER ensure_running ran (mirrors the real race).
async def sub_visible_after_start():
return started["called"]
import backend.apps.nine_router as nr
monkeypatch.setattr(nr, "ensure_running", fake_ensure_running)
monkeypatch.setattr(ft, "_has_connected_subscription", sub_visible_after_start)
s = AppSettings() # no key, own_key mode: a subscription-only user
out = await arm_free_trial(s)
assert started["called"], "arm must start 9Router before trusting the sub check"
assert out["armed"] is False and out["reason"] == "has_model"
assert s.connection_mode == "own_key"
assert s.default_model != "haiku"
@pytest.mark.asyncio
async def test_arm_tolerates_provider_load_lag(monkeypatch):
"""9Router's /api/providers can lag is_running on a cold start. arm must re-check
a few times so a sub that loads a beat late is still caught, not shadowed."""
monkeypatch.setattr(ft, "save_settings_async", _noop)
monkeypatch.setattr(ft, "_sync_routing", _noop)
async def fake_ensure_running():
return None
calls = {"n": 0}
async def lagging_sub():
calls["n"] += 1
return calls["n"] >= 3 # empty for the first two probes, then the sub appears
import backend.apps.nine_router as nr
monkeypatch.setattr(nr, "ensure_running", fake_ensure_running)
monkeypatch.setattr(ft, "_has_connected_subscription", lagging_sub)
s = AppSettings()
res = await ft.arm_free_trial(s)
assert res["reason"] == "has_model", res
assert s.default_model != "haiku"
assert calls["n"] >= 3, "should have re-checked past the lagging-empty probes"
@pytest.mark.asyncio
async def test_arm_with_no_sub_is_bounded_and_falls_through_to_arm(monkeypatch):
"""The 'don't poll for something that doesn't exist' guarantee: a genuinely
sub-less user must exhaust the re-checks quickly and PROCEED to arm, never hang."""
async def fake_ensure_running():
return None
async def never_sub():
return False
import time
import backend.apps.nine_router as nr
monkeypatch.setattr(nr, "ensure_running", fake_ensure_running)
monkeypatch.setattr(ft, "_has_connected_subscription", never_sub)
# Short-circuit before the cloud mint so the test stays offline + deterministic;
# reaching this branch proves arm did NOT falsely conclude has_model.
monkeypatch.setattr(ft, "_fingerprint", lambda _s: None)
s = AppSettings()
t = time.monotonic()
res = await ft.arm_free_trial(s)
elapsed = time.monotonic() - t
assert res["reason"] == "no_fingerprint", res # got past the sub guard to the arm path
assert elapsed < 3.0, f"re-check budget not bounded: {elapsed:.2f}s"
@pytest.mark.asyncio
async def test_clear_reverts_forced_haiku_so_it_doesnt_outlive_the_trial(monkeypatch):
monkeypatch.setattr(ft, "save_settings_async", _noop)
monkeypatch.setattr(ft, "_sync_routing", _noop)
s = AppSettings(connection_mode="free-trial", free_trial_token="ftk", default_model="haiku")
await clear_free_trial(s)
assert s.connection_mode == "own_key"
assert s.default_model == "sonnet" # forced free-run pick handed back, not left on Haiku
assert s.free_trial_token is None
# A user who deliberately picked haiku OUTSIDE free-trial mode is left alone.
s2 = AppSettings(connection_mode="own_key", default_model="haiku")
await clear_free_trial(s2)
assert s2.default_model == "haiku"
async def _noop(*_a, **_k):
return None
def _record(bucket):
async def _inner(obj, *_a, **_k):
bucket.append(obj)
return _inner
backend/tests/test_mcp_offer.py
+103
View File
@@ -0,0 +1,103 @@
"""Gate-safety invariants for the Phase 2 in-task connect offer (offer_for_gated_server).
The whole point of the offer is that it can ONLY ever suggest, never grant: it must surface a
vetted, inactive, not-dismissed MCP for the user to one-click-connect, and it must never carry
anything that could widen the MCP surface on its own. These tests make a bad offer state fail
loudly instead of shipping a silent gate bypass.
"""
import asyncio
from types import SimpleNamespace
import backend.apps.agents.core.mcp_preflight as pf
from backend.apps.agents.core.mcp_preflight import (
CURATED_SHORTLIST,
offer_for_gated_server,
run_preflight,
)
VETTED = {e["id"] for e in CURATED_SHORTLIST}
OFFER_SHAPE = {"id", "title", "description", "reason"}
def _settings(dismissed=None):
return SimpleNamespace(dismissed_mcp_suggestions=dismissed or {})
def test_offer_resolves_both_display_name_and_hotpath_slug(monkeypatch):
# The hot-path passes a sanitized slug ("google-workspace"); the curated id is a display
# name ("Google Workspace"). Both must resolve, so the wiring isn't a load-bearing string.
monkeypatch.setattr(pf, "load_all_tools", lambda: []) # nothing enabled
s = _settings()
for name in ("Google Workspace", "google-workspace"):
o = offer_for_gated_server(name, s)
assert o is not None, f"{name!r} should resolve to the vetted entry"
assert o["id"] == "Google Workspace"
assert o["id"] in VETTED
def test_offer_rejects_unvetted_and_empty(monkeypatch):
monkeypatch.setattr(pf, "load_all_tools", lambda: [])
s = _settings()
assert offer_for_gated_server("NotAVettedServer", s) is None
assert offer_for_gated_server("", s) is None
assert offer_for_gated_server(None, s) is None # type: ignore[arg-type]
def test_offer_suppressed_when_dismissed(monkeypatch):
monkeypatch.setattr(pf, "load_all_tools", lambda: [])
s = _settings({"Google Workspace": "2026-01-01T00:00:00Z"})
assert offer_for_gated_server("Google Workspace", s) is None
def test_offer_suppressed_when_already_active(monkeypatch):
monkeypatch.setattr(
pf, "load_all_tools",
lambda: [SimpleNamespace(name="Google Workspace", enabled=True)],
)
s = _settings()
assert offer_for_gated_server("Google Workspace", s) is None
def test_offer_carries_no_activate_capability(monkeypatch):
# The security invariant: an offer is data to display, never an action that grants access.
monkeypatch.setattr(pf, "load_all_tools", lambda: [])
s = _settings()
for entry in CURATED_SHORTLIST:
o = offer_for_gated_server(entry["id"], s)
assert o is not None
assert set(o.keys()) == OFFER_SHAPE, f"offer for {entry['id']} grew an unexpected field"
# --- require_vague: the MCPSearch path keeps suggestions on a concrete prompt ----------------
def _stub_classifier(is_vague, ids):
async def _fake(settings, prompt, available, task_id=None):
return {"is_vague": is_vague, "suggestions": [{"id": i, "reason": "fits"} for i in ids]}
return _fake
def test_preflight_default_suppresses_suggestions_on_concrete_prompt(monkeypatch):
# Launch path: a concrete (non-vague) prompt must NOT interrupt with a card.
monkeypatch.setattr(pf, "load_all_tools", lambda: [])
monkeypatch.setattr(pf, "_call_classifier", _stub_classifier(False, ["Google Workspace"]))
out = asyncio.run(run_preflight("refactor foo.ts to use the new client", timeout_s=5))
assert out["suggestions"] == []
def test_preflight_require_vague_false_keeps_suggestions(monkeypatch):
# MCPSearch path: the agent already proved it needs an integration, so keep the suggestion
# even though the prompt is concrete (is_vague False).
monkeypatch.setattr(pf, "load_all_tools", lambda: [])
monkeypatch.setattr(pf, "_call_classifier", _stub_classifier(False, ["Google Workspace"]))
out = asyncio.run(run_preflight("check my unread emails", timeout_s=5, require_vague=False))
assert [s["id"] for s in out["suggestions"]] == ["Google Workspace"]
assert set(out["suggestions"][0].keys()) == OFFER_SHAPE
def test_preflight_require_vague_false_still_drops_hallucinated_ids(monkeypatch):
# require_vague=False must NOT loosen the vetted-id revalidation: a made-up id is still dropped.
monkeypatch.setattr(pf, "load_all_tools", lambda: [])
monkeypatch.setattr(pf, "_call_classifier", _stub_classifier(False, ["TotallyFakeServer"]))
out = asyncio.run(run_preflight("do the thing", timeout_s=5, require_vague=False))
assert out["suggestions"] == []
backend/tests/test_publish.py
+187
View File
@@ -0,0 +1,187 @@
"""Unit tests for app-publishing build/scan/bundle logic (the locally-testable
core of Workstream A). The build step (vite) and the cloud upload need node /
network and are exercised in the staging E2E, not here.
What this proves:
1. slugify makes url-safe, length-capped slugs and never empties.
2. quick_ast_gate flags backend code that reaches outside the sandbox allowlist
and stays silent for allowlist-only code.
3. _collect_source picks up flat files and skips binary/non-source.
4. collect_bundle (flat) tars exactly the files dict; (webapp) tars a dist tree
and skips symlinks; secret-shaped files never make it into a public bundle.
5. scan_for_publish merges AST findings into the review when the LLM pass is a
no-op, reports a clean verdict for a benign app, and memoizes by source so an
unchanged reopen never re-bills the aux model.
Run with: backend/.venv/bin/python backend/tests/test_publish.py
"""
import asyncio
import io
import os
import sys
import tarfile
import tempfile
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
from backend.apps.outputs.models import Output
from backend.apps.outputs import publish_common, publish_scan, publish_build
def test_slugify():
assert publish_common.slugify("My Cool App!!") == "my-cool-app"
assert publish_common.slugify(" ") == "app"
assert publish_common.slugify("") == "app"
assert publish_common.slugify("a" * 100) == "a" * 32
assert publish_common.slugify("Café ☕ Menu") == "caf-menu"
def test_ast_gate_flags_unsafe_and_clean():
unsafe = Output(name="x", files={"backend.py": "import os\nresult={'c': os.getcwd()}\n"})
findings = publish_scan.quick_ast_gate(unsafe)
assert findings and any("os" in f for f in findings)
clean = Output(name="x", files={"backend.py": "import math\nresult={'p': math.pi}\n"})
assert publish_scan.quick_ast_gate(clean) == []
no_backend = Output(name="x", files={"index.html": "<html>hi</html>"})
assert publish_scan.quick_ast_gate(no_backend) == []
def test_collect_source_filters():
o = Output(name="x", files={
"index.html": "<html></html>",
"backend.py": "result={}",
"data.bin": "not source",
"notes.txt": "ignore me",
})
src = publish_scan._collect_source(o)
assert set(src.keys()) == {"index.html", "backend.py"}
def test_collect_bundle_flat():
o = Output(name="x", files={
"index.html": "<html>hi</html>",
"backend.py": "result={}",
})
blob = publish_build.collect_bundle(o, None)
with tarfile.open(fileobj=io.BytesIO(blob), mode="r:gz") as t:
assert sorted(t.getnames()) == ["backend.py", "index.html"]
idx = t.extractfile("index.html").read().decode()
assert idx == "<html>hi</html>"
def test_collect_bundle_drops_secret_files():
# A public bundle must never carry secrets, in either mode.
o = Output(name="x", files={
"index.html": "<html>hi</html>",
".env": "OPENAI_API_KEY=sk-secret",
".env.local": "X=1",
"server.pem": "-----BEGIN PRIVATE KEY-----",
".npmrc": "//registry/:_authToken=abc",
"app.js": "console.log(1)",
})
blob = publish_build.collect_bundle(o, None)
with tarfile.open(fileobj=io.BytesIO(blob), mode="r:gz") as t:
names = set(t.getnames())
assert names == {"index.html", "app.js"}
assert not (names & {".env", ".env.local", "server.pem", ".npmrc"})
def test_collect_bundle_webapp_dist_skips_symlink():
o = Output(name="x", workspace_id="ws123")
with tempfile.TemporaryDirectory() as dist:
os.makedirs(os.path.join(dist, "assets"))
with open(os.path.join(dist, "index.html"), "w") as f:
f.write("<html>built</html>")
with open(os.path.join(dist, "assets", "app.js"), "w") as f:
f.write("console.log(1)")
with open(os.path.join(dist, ".env"), "w") as f:
f.write("SECRET=1")
try:
os.symlink(os.path.join(dist, "index.html"), os.path.join(dist, "link.html"))
except OSError:
pass
blob = publish_build.collect_bundle(o, dist)
with tarfile.open(fileobj=io.BytesIO(blob), mode="r:gz") as t:
names = sorted(t.getnames())
assert "index.html" in names
assert "assets/app.js" in names
assert "link.html" not in names # symlinks are skipped
assert ".env" not in names # secrets are dropped
def test_scan_for_publish_merges_ast():
# Force the LLM pass to a deterministic no-op so the test is hermetic.
async def _no_llm(src, settings):
return [], "clean"
orig = publish_scan._llm_findings
publish_scan._llm_findings = _no_llm
publish_scan._memo.clear()
try:
unsafe = Output(name="x", files={"backend.py": "import socket\nresult={}\n"})
review = asyncio.run(publish_scan.scan_for_publish(unsafe, settings=object()))
assert review.verdict == "warn"
assert any("socket" in f for f in review.findings)
clean = Output(name="x", files={"index.html": "<html>hi</html>"})
review2 = asyncio.run(publish_scan.scan_for_publish(clean, settings=object()))
assert review2.verdict == "clean"
assert review2.findings == []
finally:
publish_scan._llm_findings = orig
publish_scan._memo.clear()
def test_scan_memo_skips_second_llm_call():
# Unchanged source must not re-invoke the (paid) LLM pass on a reopen.
calls = {"n": 0}
async def _counting_llm(src, settings):
calls["n"] += 1
return ["from the llm"], "warn"
orig = publish_scan._llm_findings
publish_scan._llm_findings = _counting_llm
publish_scan._memo.clear()
try:
app = Output(name="x", files={"index.html": "<html>same</html>"})
r1 = asyncio.run(publish_scan.scan_for_publish(app, settings=object()))
r2 = asyncio.run(publish_scan.scan_for_publish(app, settings=object()))
assert calls["n"] == 1, "second scan of identical source should hit the memo"
assert r1.findings == r2.findings
changed = Output(name="x", files={"index.html": "<html>different</html>"})
asyncio.run(publish_scan.scan_for_publish(changed, settings=object()))
assert calls["n"] == 2, "changed source must bust the memo"
finally:
publish_scan._llm_findings = orig
publish_scan._memo.clear()
def test_runtime_injection():
from backend.apps.outputs.html_inject import _build_data_injection, _inject_data_into_html
base = _build_data_injection("{}", "null")
assert "OUTPUT_COMPUTE" not in base and "OUTPUT_LLM" not in base # off by default
rt = _build_data_injection("{}", "null", "null", with_runtime=True)
assert "OUTPUT_COMPUTE" in rt and "OUTPUT_LLM" in rt # preview stubs are defined
# Preview must NEVER embed the install token into app JS (SECURITY.md item A).
assert "Bearer" not in rt and "Authorization" not in rt
assert "once this app is published" in rt
html = _inject_data_into_html("<html><head></head><body>x</body></html>", "{}", "null", "null", with_runtime=True)
assert "OUTPUT_LLM" in html and "</head>" in html
def _run_all():
fns = [v for k, v in sorted(globals().items()) if k.startswith("test_") and callable(v)]
for fn in fns:
fn()
print(f"ok {fn.__name__}")
print(f"\n{len(fns)} passed")
if __name__ == "__main__":
_run_all()
backend/tests/test_session_cleanup.py
+40
View File
@@ -0,0 +1,40 @@
"""Invariant: closing or deleting a session must strand NO per-session state.
The orchestration core keeps several maps keyed by session id (the session
record, its asyncio task, the live partial-stream mirror, and two module-level
view-builder retry/dirty structures). Removal used to pop only `sessions` +
`tasks`, leaking the rest for the life of the process, an unbounded creep over
a long-running app. `_purge_session_memory` is the single chokepoint both the
close and delete paths route through; this pins the invariant that after it
runs the id is gone from EVERY structure, while a sibling session is untouched.
Run with: backend/.venv/bin/python -m pytest backend/tests/test_session_cleanup.py
"""
from backend.apps.agents import agent_manager as am
def test_purge_session_memory_clears_every_structure():
mgr = am.AgentManager()
mgr.sessions = {"dead": object(), "alive": object()}
mgr.tasks = {"dead": object()}
mgr._live_partial = {"dead": {"text": "half a reply"}}
am.p_view_builder_render_retry_counts["dead"] = 4
am.p_view_builder_dirty_sessions.add("dead")
mgr._purge_session_memory("dead")
assert "dead" not in mgr.sessions
assert "dead" not in mgr.tasks
assert "dead" not in mgr._live_partial
assert "dead" not in am.p_view_builder_render_retry_counts
assert "dead" not in am.p_view_builder_dirty_sessions
# Only the target id is purged; an unrelated live session survives.
assert "alive" in mgr.sessions
def test_purge_is_safe_on_an_untracked_id():
# Purging an id that was never tracked must be a quiet no-op, not a KeyError,
# so the delete/close paths can call it unconditionally.
mgr = am.AgentManager()
mgr._purge_session_memory("never-existed")
assert mgr.sessions == {}
backend/tests/test_settings_meta_concurrency.py
+62
View File
@@ -0,0 +1,62 @@
"""Concurrent SettingsWrite must not lose updates.
SettingsWrite is a read-modify-write that routes through update_settings (which
awaits), so two autonomous agents writing at the same time would interleave: each
loads the same snapshot, each writes the WHOLE object back, and the last writer
silently reverts the other's field, while BOTH agents are told "applied". This
drives two genuinely concurrent writes (different fields) through the ASGI app
and asserts neither is lost. It's the regression guard for the asyncio lock that
serializes these writes; without the lock this fails reproducibly.
"""
from __future__ import annotations
import asyncio
import httpx
import pytest
from backend.main import app
def _auth_headers():
import backend.auth as auth_mod
if not auth_mod._TOKEN:
import secrets
auth_mod._TOKEN = secrets.token_urlsafe(32)
return {"Authorization": f"Bearer {auth_mod._TOKEN}"}
@pytest.fixture
def reset_settings():
from backend.apps.settings.settings import load_settings, _save_settings
original = load_settings().model_copy(deep=True)
yield
_save_settings(original)
@pytest.mark.asyncio
async def test_concurrent_writes_to_different_fields_both_survive(reset_settings):
from backend.apps.settings.settings import load_settings, _save_settings
base = load_settings()
base.theme = "dark"
base.default_mode = "agent"
_save_settings(base)
headers = _auth_headers()
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport, base_url="http://test", headers=headers) as client:
r1, r2 = await asyncio.gather(
client.post("/api/settings-meta/write", json={"changes": {"theme": "light"}}),
client.post("/api/settings-meta/write", json={"changes": {"default_mode": "chat"}}),
)
assert r1.status_code == 200 and r2.status_code == 200
assert r1.json()["outcomes"]["theme"]["status"] == "applied"
assert r2.json()["outcomes"]["default_mode"]["status"] == "applied"
final = load_settings()
# Both concurrent edits must persist; neither agent's "applied" result is a lie.
assert final.theme == "light", "lost update: theme was clobbered by the concurrent write"
assert final.default_mode == "chat", "lost update: default_mode was clobbered"
backend/tests/test_settings_meta_endpoint.py
+145
View File
@@ -0,0 +1,145 @@
"""End-to-end coverage of /api/settings-meta (the agent-editable Settings tool).
Drives the real FastAPI route with a real in-memory AgentSession so the guard
runs against an actual run's model, exactly as it will in production. The unit
invariant lives in test_settings_meta_guard.py; this test proves the wiring:
redaction on read, the three write refusals, and a benign write actually landing.
"""
from __future__ import annotations
import pytest
from fastapi.testclient import TestClient
from backend.main import app
@pytest.mark.asyncio
async def test_second_wall_restores_protected_credential_even_if_body_blanks_it():
"""Defense in depth: even if a write reaches apply_settings_update with the
live credential blanked (a guard slip upstream), the second-wall restore puts
it back. Proves the api-key guard isn't a single point of failure."""
from backend.apps.settings.settings import (
apply_settings_update, settings_write_lock, load_settings, _save_settings,
)
original = load_settings().model_copy(deep=True)
try:
s = load_settings()
s.anthropic_api_key = "sk-live-KEEP-ME"
_save_settings(s)
# A body that (as if a guard bug let it through) clears the live key.
body = load_settings()
body.anthropic_api_key = ""
async with settings_write_lock():
saved = await apply_settings_update(body, protect_fields={"anthropic_api_key"})
assert saved.anthropic_api_key == "sk-live-KEEP-ME", "second wall failed to restore"
assert load_settings().anthropic_api_key == "sk-live-KEEP-ME"
# And a NON-protected blank still goes through (only the protected one is restored).
body2 = load_settings()
body2.openai_api_key = ""
async with settings_write_lock():
await apply_settings_update(body2, protect_fields={"anthropic_api_key"})
assert not load_settings().openai_api_key
finally:
_save_settings(original)
@pytest.fixture
def client():
import backend.auth as auth_mod
if not auth_mod._TOKEN:
import secrets
auth_mod._TOKEN = secrets.token_urlsafe(32)
return TestClient(app, headers={"Authorization": f"Bearer {auth_mod._TOKEN}"})
@pytest.fixture
def reset_settings():
from backend.apps.settings.settings import load_settings, _save_settings
original = load_settings().model_copy(deep=True)
yield
_save_settings(original)
@pytest.fixture
def session_on_anthropic_key():
"""A live run on opus-4-8 in own_key mode with an Anthropic key set: the
Anthropic key powers it. Registered in agent_manager so the guard sees it."""
from backend.apps.agents.agent_manager import agent_manager
from backend.apps.agents.core.models import AgentSession
from backend.apps.settings.settings import load_settings, _save_settings
s = load_settings()
s.connection_mode = "own_key"
s.anthropic_api_key = "sk-ant-test-LIVE"
s.openai_api_key = "sk-openai-test-OTHER"
_save_settings(s)
sess = AgentSession(id="settings-meta-test", name="t", model="opus-4-8")
agent_manager.sessions["settings-meta-test"] = sess
yield "settings-meta-test"
agent_manager.sessions.pop("settings-meta-test", None)
def test_read_redacts_every_secret(client, reset_settings):
r = client.post("/api/settings-meta/read", json={})
assert r.status_code == 200, r.text
settings = r.json()["settings"]
# Secret fields come back as state, never a raw string value.
for field in ("anthropic_api_key", "openai_api_key", "claude_subscription_token", "openswarm_bearer_token"):
if field in settings:
assert isinstance(settings[field], dict), f"{field} leaked as a raw value"
assert "configured" in settings[field]
# A non-secret field is passed through untouched.
assert settings["theme"] in ("dark", "light")
def test_benign_write_applies(client, reset_settings):
r = client.post("/api/settings-meta/write", json={"changes": {"theme": "light"}})
assert r.status_code == 200, r.text
assert r.json()["outcomes"]["theme"]["status"] == "applied"
from backend.apps.settings.settings import load_settings
assert load_settings().theme == "light"
def test_unknown_and_server_owned_fields_are_refused(client, reset_settings):
r = client.post("/api/settings-meta/write", json={"changes": {
"not_a_real_field": 1,
"connection_mode": "openswarm-pro",
"openswarm_bearer_token": "forged",
}})
assert r.status_code == 200, r.text
out = r.json()["outcomes"]
assert out["not_a_real_field"]["status"] == "unknown"
assert out["connection_mode"]["status"] == "refused"
assert out["openswarm_bearer_token"]["status"] == "refused"
# And the server-owned field is genuinely untouched on disk.
from backend.apps.settings.settings import load_settings
assert load_settings().connection_mode != "openswarm-pro"
def test_cannot_suicide_but_disconnects_others(client, reset_settings, session_on_anthropic_key):
"""The spec scenario over HTTP: run on the Anthropic key, asked to clear
every model key + flip a benign setting. It must refuse the live key,
clear the other one, and apply the benign change, all in one call."""
sid = session_on_anthropic_key
r = client.post("/api/settings-meta/write", json={
"parent_session_id": sid,
"changes": {
"anthropic_api_key": "",
"openai_api_key": "",
"theme": "light",
},
})
assert r.status_code == 200, r.text
out = r.json()["outcomes"]
assert out["anthropic_api_key"]["status"] == "refused", "blanked the live credential!"
assert "powering this run" in out["anthropic_api_key"]["reason"]
assert out["openai_api_key"]["status"] == "applied"
assert out["theme"]["status"] == "applied"
from backend.apps.settings.settings import load_settings
s = load_settings()
assert s.anthropic_api_key == "sk-ant-test-LIVE", "live key was cleared despite refusal"
assert not s.openai_api_key, "the other provider's key should have been cleared"
assert s.theme == "light"
backend/tests/test_settings_meta_guard.py
+213
View File
@@ -0,0 +1,213 @@
"""The no-suicide invariant for the agent-editable settings tool, proved by
exhaustive enumeration rather than a few hand-picked cases.
The state space here is small and finite (every shipped model row x every
connection mode x which keys are present), so we walk ALL of it deterministically
instead of reaching for randomized property testing. A failure is a concrete,
reproducible (model, mode, keys) tuple, not a flaky seed.
The one invariant under test: the settings-meta write guard must NEVER let an
agent blank the credential powering its own run, while still allowing it to
clear any OTHER provider's key. Plus two drift seals: every shipped model lane
classifies (no "unknown"), and the redactor catches every credential field.
"""
from __future__ import annotations
import itertools
import pytest
from backend.apps.settings.models import AppSettings, CustomProvider
from backend.apps.agents.providers.registry import BUILTIN_MODELS
from backend.apps.agents.session_credential import (
ALL_API_KEY_FIELDS,
resolve_powering_credential,
write_would_suicide,
)
from backend.apps.settings.redaction import is_secret_field, redact_settings
CONNECTION_MODES = ["own_key", "openswarm-pro", "free-trial"]
# Every credential field the settings PUT path already treats as secret. Kept
# here as the contract the redactor must honor; if PUT's notion of "secret"
# grows, this list should too, and the drift-seal test fails until the redactor
# also covers it.
KNOWN_SECRET_FIELDS = [
"anthropic_api_key", "openai_api_key", "google_api_key", "openrouter_api_key",
"claude_subscription_token", "openai_subscription_token", "gemini_subscription_token",
"openswarm_bearer_token", "free_trial_token", "installation_id",
]
def _all_model_values() -> list[str]:
vals = [m["value"] for rows in BUILTIN_MODELS.values() for m in rows]
# Plus synthesized lanes the resolver must also place.
vals += ["or:anthropic/claude-3.5", "custom/lmstudio/llama-3", "totally-made-up-model"]
return vals
def _settings_with(mode: str, keys: set[str], custom: bool = False) -> AppSettings:
s = AppSettings(connection_mode=mode)
if "anthropic" in keys:
s.anthropic_api_key = "sk-ant-live-aaaa"
if "openai" in keys:
s.openai_api_key = "sk-openai-live-bbbb"
if "google" in keys:
s.google_api_key = "goog-live-cccc"
if "openrouter" in keys:
s.openrouter_api_key = "or-live-dddd"
if mode in ("openswarm-pro", "free-trial"):
s.openswarm_bearer_token = "bearer-live-eeee"
if mode == "free-trial":
s.free_trial_token = "ft-live-ffff"
if custom:
s.custom_providers = [CustomProvider(name="LMStudio", base_url="http://localhost:1234/v1", api_key="local")]
return s
# ---------------------------------------------------------------------------
# The invariant: the live credential can never be blanked; others always can.
# ---------------------------------------------------------------------------
def test_live_api_key_can_never_be_blanked_but_others_can():
key_subsets = [set(c) for r in range(5)
for c in itertools.combinations(["anthropic", "openai", "google", "openrouter"], r)]
checked_api_key_runs = 0
for model in _all_model_values():
for mode in CONNECTION_MODES:
for keys in key_subsets:
for custom in (False, True):
s = _settings_with(mode, keys, custom=custom)
p = resolve_powering_credential(model, s)
if p.kind == "api_key" and p.protected_field:
checked_api_key_runs += 1
# Blanking the live key, in any blank form, is refused.
for blank in (None, "", " "):
assert write_would_suicide(p.protected_field, blank, p), (
f"suicide allowed: model={model} mode={mode} field={p.protected_field}={blank!r}"
)
# Replacing it with a real key is a reconnect, allowed.
assert not write_would_suicide(p.protected_field, "sk-fresh-9999", p)
# Clearing any OTHER provider's key stays allowed.
for other in ALL_API_KEY_FIELDS - {p.protected_field}:
assert not write_would_suicide(other, "", p), (
f"over-blocked unrelated key {other}: model={model} mode={mode}"
)
elif p.kind == "subscription":
# The live credential isn't a settings field, so clearing
# ANY api key is safe (it can't be the powering one).
for field in ALL_API_KEY_FIELDS:
assert not write_would_suicide(field, "", p), (
f"subscription run wrongly protected {field}: model={model} mode={mode}"
)
elif p.kind == "unknown":
# Fail safe: every credential field is protected.
for field in ALL_API_KEY_FIELDS:
assert write_would_suicide(field, "", p)
assert checked_api_key_runs > 0, "enumeration never exercised an api-key run; test is vacuous"
def test_custom_provider_run_protects_its_entry():
s = _settings_with("own_key", set(), custom=True)
p = resolve_powering_credential("custom/lmstudio/llama-3", s)
assert p.kind == "api_key" and p.provider == "custom"
# Dropping the powering provider's entry is suicide.
assert write_would_suicide("custom_providers", [], p)
# Keeping it (even with a blanked placeholder key, local servers don't need one) is fine.
keep = [{"name": "LMStudio", "base_url": "http://localhost:1234/v1", "api_key": ""}]
assert not write_would_suicide("custom_providers", keep, p)
# Swapping in a different provider but losing the live one is suicide.
other = [{"name": "Together", "base_url": "https://api.together.xyz/v1", "api_key": "k"}]
assert write_would_suicide("custom_providers", other, p)
def test_disconnect_all_models_spec_scenario():
"""The spec's worked example: Claude (api key) + OpenAI (api key) both
connected, run on an Anthropic model, asked to disconnect everything. It
must refuse to kill Claude (the live one) and allow killing OpenAI."""
s = _settings_with("own_key", {"anthropic", "openai"})
p = resolve_powering_credential("opus-4-8", s) # default Anthropic row, own_key -> api key
assert p.kind == "api_key" and p.protected_field == "anthropic_api_key"
assert write_would_suicide("anthropic_api_key", "", p) # refuse self
assert not write_would_suicide("openai_api_key", "", p) # allow the other
# Same connections, but the run is on the OpenAI key instead: mirror image.
p2 = resolve_powering_credential("gpt-5.5-api", s)
assert p2.kind == "api_key" and p2.protected_field == "openai_api_key"
assert write_would_suicide("openai_api_key", "", p2)
assert not write_would_suicide("anthropic_api_key", "", p2)
# ---------------------------------------------------------------------------
# Drift seals.
# ---------------------------------------------------------------------------
def test_every_shipped_model_lane_classifies():
"""A new model row that the resolver can't place would silently fall to the
fail-safe 'unknown' lane (over-blocking every key). Force every shipped row
to resolve to a real api_key/subscription so new lanes get classified."""
s_pro = _settings_with("openswarm-pro", {"anthropic", "openai", "google", "openrouter"})
s_key = _settings_with("own_key", {"anthropic", "openai", "google", "openrouter"})
for rows in BUILTIN_MODELS.values():
for m in rows:
for s in (s_pro, s_key):
p = resolve_powering_credential(m["value"], s)
assert p.kind in ("api_key", "subscription"), (
f"unclassified model lane {m['value']!r} -> {p.kind}"
)
def test_redactor_catches_every_known_secret():
for field in KNOWN_SECRET_FIELDS:
assert is_secret_field(field), f"redactor would leak {field}"
# And every AppSettings field that NAMES itself a secret is caught by the rule.
for name in AppSettings.model_fields:
if name.endswith(("_key", "_token", "_secret")):
assert is_secret_field(name)
def test_redaction_fail_safe_catches_misnamed_secret_by_value():
# The name rule (_key/_token/_secret) would MISS a field named off-convention.
# The value-shape backstop must still redact it, so a leak needs BOTH a bad
# name AND a non-credential-shaped value, not just one.
import json
raw = {"theme": "dark", "weird_field": "sk-ant-api03-AAAABBBBCCCCDDDDEEEEFFFF"}
red = redact_settings(raw)
assert red["theme"] == "dark"
assert isinstance(red["weird_field"], dict) and red["weird_field"]["configured"] is True
assert "sk-ant-api03" not in json.dumps(red)
def test_redact_settings_never_emits_a_raw_secret():
s = _settings_with("openswarm-pro", {"anthropic", "openai", "google", "openrouter"}, custom=True)
s.claude_subscription_token = "should-never-appear"
raw = s.model_dump()
red = redact_settings(raw)
for field in KNOWN_SECRET_FIELDS:
if field in red:
assert isinstance(red[field], dict), f"{field} not redacted to a state dict"
assert "configured" in red[field]
raw_val = raw.get(field)
if isinstance(raw_val, str) and raw_val.strip():
# Configured: state only, never the whole value (last4 at most).
assert red[field]["configured"] is True
assert red[field].get("last4") != raw_val
assert len(red[field].get("last4") or "") <= 4
# The nested custom-provider key is redacted too.
assert isinstance(red["custom_providers"][0]["api_key"], dict)
# Non-secret fields pass through untouched.
assert red["theme"] == raw["theme"]
assert red["connection_mode"] == raw["connection_mode"]
# The strongest check: the literal secret string appears nowhere in the output.
import json
assert "should-never-appear" not in json.dumps(red)
assert "sk-ant-live-aaaa" not in json.dumps(red)
backend/tests/test_settings_meta_stdio_live.py
+115
View File
@@ -0,0 +1,115 @@
"""Definitive proof: a SettingsWrite from the REAL stdio MCP server, against a
REAL running backend, on a REAL session whose model is powered by a specific
API key, refuses to clear THAT key but clears a different provider's key.
Earlier coverage either drove the endpoint with an injected session (skipping the
stdio subprocess) or drove the stdio subprocess with no session (hitting the
fail-safe). This closes the gap: it boots uvicorn in-process (so the subprocess's
HTTP call lands on a backend whose `agent_manager.sessions` we can populate),
then runs `settings_meta_server.py` exactly as an agent would. No model required,
the guard is provider-routing logic, not an LLM call.
"""
from __future__ import annotations
import json
import os
import secrets
import socket
import subprocess
import sys
import threading
import time
import pytest
import uvicorn
from backend.main import app
SERVER = os.path.join(os.path.dirname(os.path.dirname(__file__)), "apps", "agents", "settings_meta_server.py")
def _free_port() -> int:
s = socket.socket()
s.bind(("127.0.0.1", 0))
port = s.getsockname()[1]
s.close()
return port
@pytest.fixture
def live_backend():
import backend.auth as auth_mod
if not auth_mod._TOKEN:
auth_mod._TOKEN = secrets.token_urlsafe(32)
port = _free_port()
server = uvicorn.Server(uvicorn.Config(app, host="127.0.0.1", port=port, log_level="error"))
thread = threading.Thread(target=server.run, daemon=True)
thread.start()
for _ in range(200):
if getattr(server, "started", False):
break
time.sleep(0.05)
assert getattr(server, "started", False), "uvicorn did not start"
yield port, auth_mod._TOKEN
server.should_exit = True
thread.join(timeout=5)
@pytest.fixture
def reset_settings():
from backend.apps.settings.settings import load_settings, _save_settings
original = load_settings().model_copy(deep=True)
yield
_save_settings(original)
def _run_stdio(port: int, token: str, session_id: str, changes: dict) -> str:
env = {
**os.environ,
"OPENSWARM_PORT": str(port),
"OPENSWARM_AUTH_TOKEN": token,
"OPENSWARM_PARENT_SESSION_ID": session_id,
}
rpc = "\n".join([
json.dumps({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}}),
json.dumps({"jsonrpc": "2.0", "id": 2, "method": "tools/call",
"params": {"name": "SettingsWrite", "arguments": {"changes": changes}}}),
]) + "\n"
proc = subprocess.run([sys.executable, SERVER], input=rpc, capture_output=True, text=True, env=env, timeout=30)
msgs = [json.loads(l) for l in proc.stdout.splitlines() if l.strip()]
resp = next(m for m in msgs if m.get("id") == 2)
return resp["result"]["content"][0]["text"]
def test_live_stdio_settingswrite_refuses_live_key_clears_other(live_backend, reset_settings):
port, token = live_backend
from backend.apps.settings.settings import load_settings, _save_settings
from backend.apps.agents.agent_manager import agent_manager
from backend.apps.agents.core.models import AgentSession
# A real run on opus-4-8 in own_key mode: the Anthropic key powers it; an
# OpenAI key is also connected (the "other provider").
s = load_settings()
s.connection_mode = "own_key"
s.anthropic_api_key = "sk-ant-LIVE-do-not-clear"
s.openai_api_key = "sk-oai-OTHER-ok-to-clear"
_save_settings(s)
agent_manager.sessions["live-stdio-test"] = AgentSession(id="live-stdio-test", name="t", model="opus-4-8")
try:
text = _run_stdio(port, token, "live-stdio-test",
{"anthropic_api_key": "", "openai_api_key": "", "theme": "light"})
finally:
agent_manager.sessions.pop("live-stdio-test", None)
# The tool's own rendered result, exactly what the agent would read.
assert "Refused anthropic_api_key" in text, text
assert "powering this run" in text
assert "Applied" in text and "theme" in text
# And the truth on disk: live key kept, other cleared, benign applied.
final = load_settings()
assert final.anthropic_api_key == "sk-ant-LIVE-do-not-clear", "live key was cleared!"
assert not final.openai_api_key, "the other provider's key should have been cleared"
assert final.theme == "light"
backend/tests/test_settings_patch.py
+83
View File
@@ -0,0 +1,83 @@
"""PATCH settings merges a diff onto fresh state, so a renderer save can't
clobber a field it didn't send. This is the structural close on the last
renderer-vs-agent race: the lost update is now unrepresentable, you can't
overwrite a field you never put in the body.
"""
from __future__ import annotations
import asyncio
import httpx
import pytest
from fastapi.testclient import TestClient
from backend.main import app
def _auth_headers():
import backend.auth as auth_mod
if not auth_mod._TOKEN:
import secrets
auth_mod._TOKEN = secrets.token_urlsafe(32)
return {"Authorization": f"Bearer {auth_mod._TOKEN}"}
@pytest.fixture
def client():
return TestClient(app, headers=_auth_headers())
@pytest.fixture
def reset_settings():
from backend.apps.settings.settings import load_settings, _save_settings
original = load_settings().model_copy(deep=True)
yield
_save_settings(original)
def test_patch_changes_only_sent_fields(client, reset_settings):
from backend.apps.settings.settings import load_settings, _save_settings
s = load_settings()
s.theme = "dark"
s.default_mode = "chat" # as if something else had set this
_save_settings(s)
r = client.patch("/api/settings", json={"theme": "light"})
assert r.status_code == 200, r.text
final = load_settings()
assert final.theme == "light" # the field we sent changed
assert final.default_mode == "chat" # the field we DIDN'T send is untouched
def test_patch_ignores_unknown_fields(client, reset_settings):
r = client.patch("/api/settings", json={"theme": "light", "not_a_field": 123})
assert r.status_code == 200
# Unknown key is dropped, not stored; the real field still applied.
from backend.apps.settings.settings import load_settings
assert load_settings().theme == "light"
assert "not_a_field" not in load_settings().model_dump()
@pytest.mark.asyncio
async def test_concurrent_renderer_patch_and_agent_write_both_survive(reset_settings):
"""The renderer PATCHes one field while an autonomous agent writes another,
at the same time. Both must land: the renderer never sends the agent's field,
so it can't clobber it, and both reads happen fresh under the shared lock."""
from backend.apps.settings.settings import load_settings, _save_settings
base = load_settings()
base.theme = "dark"
base.default_mode = "agent"
_save_settings(base)
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport, base_url="http://test", headers=_auth_headers()) as client:
r1, r2 = await asyncio.gather(
client.patch("/api/settings", json={"theme": "light"}),
client.post("/api/settings-meta/write", json={"changes": {"default_mode": "chat"}}),
)
assert r1.status_code == 200 and r2.status_code == 200
final = load_settings()
assert final.theme == "light", "renderer's change lost"
assert final.default_mode == "chat", "agent's change clobbered by the renderer PATCH"
backend/tests/test_settings_select_and_send.py
+53
View File
@@ -0,0 +1,53 @@
"""Proof that selecting a Settings row and sending threads selected_setting_ids
from the HTTP boundary into the run, and produces the focused context block.
Covers the backend half definitively (HTTP -> send_message kwarg; the context
builder output). The browser half (click a row -> the POST body carries
selected_setting_ids) is proven live via CDP.
"""
from __future__ import annotations
import secrets
import pytest
from fastapi.testclient import TestClient
from backend.main import app
@pytest.fixture
def client():
import backend.auth as auth_mod
if not auth_mod._TOKEN:
auth_mod._TOKEN = secrets.token_urlsafe(32)
return TestClient(app, headers={"Authorization": f"Bearer {auth_mod._TOKEN}"})
def test_message_endpoint_threads_selected_setting_ids(client, monkeypatch):
from backend.apps.agents.agent_manager import agent_manager
captured: dict = {}
async def fake_send(session_id, prompt, **kwargs):
captured["session_id"] = session_id
captured["kwargs"] = kwargs
monkeypatch.setattr(agent_manager, "send_message", fake_send)
r = client.post(
"/api/agents/sessions/sess-x/message",
json={"prompt": "flip my theme to light", "selected_setting_ids": ["theme", "default_model"]},
)
assert r.status_code == 200, r.text
assert captured["kwargs"].get("selected_setting_ids") == ["theme", "default_model"]
def test_selected_settings_context_block_targets_the_fields():
from backend.apps.agents.manager.prompt.prompt_context import _build_selected_settings_context
assert _build_selected_settings_context(None) is None
assert _build_selected_settings_context([]) is None
block = _build_selected_settings_context(["theme", "default_model"])
assert "theme" in block and "default_model" in block
# It tells the agent to use the always-on settings tools on exactly these fields.
assert "SettingsRead" in block and "SettingsWrite" in block
# Targeting aid, not a gate: it focuses, never claims to unlock anything.
assert "Leave unrelated settings alone" in block
backend/tests/test_skill_registry_community.py
+193
View File
@@ -0,0 +1,193 @@
"""skills.sh wild-registry resolution + safe install.
The network parts (GitHub trees + raw fetch) are smoked manually; here we pin
the PURE logic that decides which files a skill is made of and the safety of
writing them: SKILL.md selection at arbitrary repo depth, script disclosure,
and the path-traversal guard that stops an untrusted archive escaping its dir.
"""
from __future__ import annotations
import os
import pytest
import backend.apps.skills.skills as skills_mod
from backend.apps.skill_registry.skill_registry import _select_skill_paths, _is_script_path
def test_selects_shortest_matching_skill_md_at_any_depth():
tree = [
{"type": "blob", "path": "README.md"},
{"type": "blob", "path": "plugins/x/skills/pdftk/SKILL.md"},
{"type": "blob", "path": "plugins/x/skills/pdftk/run.sh"},
{"type": "blob", "path": "plugins/x/skills/pdftk/templates/form.txt"},
{"type": "blob", "path": "plugins/x/skills/other/SKILL.md"},
]
skill_md, members = _select_skill_paths(tree, "pdftk")
assert skill_md == "plugins/x/skills/pdftk/SKILL.md"
assert set(members) == {
"plugins/x/skills/pdftk/SKILL.md",
"plugins/x/skills/pdftk/run.sh",
"plugins/x/skills/pdftk/templates/form.txt",
}
# The unrelated 'other' skill's files are excluded.
assert all("/other/" not in m for m in members)
def test_top_level_skill_md():
tree = [{"type": "blob", "path": "pdftk/SKILL.md"}, {"type": "blob", "path": "pdftk/x.py"}]
skill_md, members = _select_skill_paths(tree, "pdftk")
assert skill_md == "pdftk/SKILL.md"
assert "pdftk/x.py" in members
def test_missing_skill_raises():
with pytest.raises(ValueError):
_select_skill_paths([{"type": "blob", "path": "a/SKILL.md"}], "nonexistent")
def test_ambiguous_match_picks_deterministically():
# Several <x>/pdf/SKILL.md: a top-level pdf/ wins, else skills/pdf/, never arbitrary.
tree = [
{"type": "blob", "path": "plugins/z/pdf/SKILL.md"},
{"type": "blob", "path": "skills/pdf/SKILL.md"},
{"type": "blob", "path": "pdf/SKILL.md"},
]
skill_md, _ = _select_skill_paths(tree, "pdf")
assert skill_md == "pdf/SKILL.md"
# Without a top-level one, prefer skills/<id>/.
tree2 = [
{"type": "blob", "path": "plugins/z/pdf/SKILL.md"},
{"type": "blob", "path": "skills/pdf/SKILL.md"},
]
skill_md2, _ = _select_skill_paths(tree2, "pdf")
assert skill_md2 == "skills/pdf/SKILL.md"
def test_github_headers_adds_token_when_set(monkeypatch):
from backend.apps.skill_registry.skill_registry import _github_headers
monkeypatch.delenv("OPENSWARM_GITHUB_TOKEN", raising=False)
monkeypatch.delenv("GITHUB_TOKEN", raising=False)
assert "Authorization" not in _github_headers()
monkeypatch.setenv("OPENSWARM_GITHUB_TOKEN", "ghp_test")
assert _github_headers()["Authorization"] == "Bearer ghp_test"
def test_install_disclosure_flags_secret_shaped_files():
# The scan we wire into the install disclosure (reused from the .swarm importer)
# must flag a community skill shipping credentials, and leave clean files alone.
from backend.apps.swarm.redact import find_secrets_in_files
files = {
"SKILL.md": b"Renders PDFs. No secrets.",
"config.py": b'API_KEY = "sk-ant-api03-AAAABBBBCCCCDDDDEEEEFFFFGGGGHHHH"',
}
hits = find_secrets_in_files(files)
assert "config.py" in hits
assert "SKILL.md" not in hits
def test_script_classification():
assert _is_script_path("run.sh")
assert _is_script_path("helper.py")
assert _is_script_path("scripts/build.txt") # under a scripts/ dir
assert _is_script_path("bin/tool")
assert not _is_script_path("SKILL.md")
assert not _is_script_path("templates/form.html")
assert not _is_script_path("data.json")
# ---------------------------------------------------------------------------
# Safe install (write_folder_skill).
# ---------------------------------------------------------------------------
@pytest.fixture
def skills_dir(tmp_path, monkeypatch):
d = tmp_path / "skills"
d.mkdir()
monkeypatch.setattr(skills_mod, "SKILLS_DIR", str(d))
monkeypatch.setattr(skills_mod, "INDEX_PATH", str(d / ".skills_index.json"))
return d
def test_write_folder_skill_lands_files_and_indexes(skills_dir):
skill = skills_mod.write_folder_skill(
"PDF Tk",
{"SKILL.md": "---\nname: PDF Tk\n---\nbody", "scripts/run.sh": "echo hi"},
{"name": "PDF Tk", "description": "fill forms"},
)
assert skill.id == "pdf-tk"
assert skill.has_supporting_files is True
assert os.path.isfile(skills_dir / "pdf-tk" / "SKILL.md")
assert os.path.isfile(skills_dir / "pdf-tk" / "scripts" / "run.sh")
# Re-syncs and shows up in the list.
assert "pdf-tk" in {s.id for s in skills_mod._sync_skills()}
def test_install_dedups_instead_of_clobbering_existing_skill(skills_dir):
# A user already has a local skill named "pdf".
skills_mod.write_folder_skill("pdf", {"SKILL.md": "MINE"}, {"name": "My PDF"})
# A wild-registry install of a same-named skill must NOT overwrite it.
slug = skills_mod.unique_skill_slug("pdf")
assert slug == "pdf-2"
skills_mod.write_folder_skill(slug, {"SKILL.md": "THEIRS"}, {"name": "Registry PDF"})
with open(skills_dir / "pdf" / "SKILL.md", encoding="utf-8") as f:
assert f.read() == "MINE", "registry install clobbered the user's existing skill"
with open(skills_dir / "pdf-2" / "SKILL.md", encoding="utf-8") as f:
assert f.read() == "THEIRS"
ids = {s.id for s in skills_mod._sync_skills()}
assert {"pdf", "pdf-2"} <= ids
def test_confirm_install_writes_folder_lists_and_injects(skills_dir, monkeypatch):
"""End-to-end install->usable: confirm=true through the real /install endpoint
writes the folder skill, it shows up in /api/skills/list with supporting
files, and _resolve_attached_skills injects it with the folder path so an
agent can read its scripts. (resolve is mocked to skip the network; the live
GitHub resolve is proven separately.)"""
import secrets as _secrets
from fastapi.testclient import TestClient
from backend.main import app
from backend.apps.agents.manager.prompt.prompt_context import _resolve_attached_skills
import backend.auth as auth_mod
if not auth_mod._TOKEN:
auth_mod._TOKEN = _secrets.token_urlsafe(32)
client = TestClient(app, headers={"Authorization": f"Bearer {auth_mod._TOKEN}"})
async def fake_resolve(source, skill_id):
return {
"name": "PDF Tools", "description": "work with pdfs", "repo_url": "https://github.com/o/r",
"skill_id": skill_id,
"files": {"SKILL.md": "# PDF Tools\nRun scripts/extract.py to pull text.",
"scripts/extract.py": "print('extract')"},
"scripts": ["scripts/extract.py"], "secret_findings": [],
}
monkeypatch.setattr("backend.apps.skill_registry.skill_registry.resolve_community_skill", fake_resolve)
r = client.post("/api/skill-registry/install", json={"source": "o/r", "skill_id": "pdf-tools", "confirm": True})
assert r.status_code == 200 and r.json()["installed"] is True
slug = r.json()["skill"]["id"]
# Listed via the real skills API, flagged as multi-file.
listed = {s["id"]: s for s in client.get("/api/skills/list").json()["skills"]}
assert slug in listed and listed[slug]["has_supporting_files"] is True
# On disk as a folder with the script.
assert (skills_dir / slug / "SKILL.md").exists()
assert (skills_dir / slug / "scripts" / "extract.py").exists()
# Injectable: the agent gets the body AND a pointer to the folder for on-demand reads.
block = _resolve_attached_skills([{"id": slug, "name": "PDF Tools", "content": "# PDF Tools\nRun scripts/extract.py to pull text."}])
assert "[Using skill: PDF Tools]" in block
assert str(skills_dir / slug) in block
def test_write_folder_skill_blocks_path_traversal(skills_dir):
skills_mod.write_folder_skill(
"evil",
{"SKILL.md": "x", "../escape.txt": "pwned", "/etc/abs.txt": "pwned"},
{"name": "evil"},
)
# The escape attempts never landed outside the skill folder.
assert not (skills_dir.parent / "escape.txt").exists()
assert not os.path.exists("/etc/abs.txt") or open("/etc/abs.txt").read() != "pwned"
# The legitimate SKILL.md did land.
assert os.path.isfile(skills_dir / "evil" / "SKILL.md")
backend/tests/test_skill_registry_seed.py
+46
View File
@@ -0,0 +1,46 @@
"""Regression tests for the skill-registry never-empty seed (winv2 Bug #1).
The bug: the catalog was fetched from GitHub once at startup then only hourly,
so a cold/slow/failed network left it empty for the whole session, breaking the
Skills page and the onboarding "Install a skill" step (waitForSelector
"skill-item-pdf" timing out). Fix: seed from a bundled snapshot + on-disk
last-good cache so the catalog is never empty, even fully offline.
"""
import asyncio
import json
import os
from backend.apps.skill_registry import skill_registry as sr
def test_bundled_snapshot_exists_and_includes_pdf():
# The onboarding step targets the "pdf" skill via /pdf/i; it must be present
# in the shipped snapshot or the tour times out even with a populated list.
assert os.path.exists(sr._BUNDLED_SNAPSHOT)
data = json.load(open(sr._BUNDLED_SNAPSHOT, encoding="utf-8"))
assert isinstance(data, dict) and len(data) >= 10
assert any("pdf" in k.lower() or "pdf" in v.get("folder", "").lower()
for k, v in data.items())
def test_seed_makes_catalog_non_empty_offline(monkeypatch, tmp_path):
# Point the disk cache at an empty tmp dir so only the bundled snapshot can
# seed; this is the brand-new-install, no-network case.
monkeypatch.setenv("OPENSWARM_SKILL_CACHE_DIR", str(tmp_path))
seeded = sr._load_seed_cache()
assert len(seeded) >= 10
sr._cache = seeded
res = asyncio.run(sr.registry_search(q="", limit=100, offset=0, sort="name", category=""))
assert res["total"] >= 10 and len(res["skills"]) >= 10
def test_disk_cache_roundtrip_and_priority(monkeypatch, tmp_path):
# A saved last-good fetch must win over the bundled snapshot on next boot.
monkeypatch.setenv("OPENSWARM_SKILL_CACHE_DIR", str(tmp_path))
sentinel = {"only-skill": {"name": "only-skill", "description": "", "content": "",
"folder": "skills/only-skill", "category": "Test",
"repositoryUrl": ""}}
sr._save_disk_cache(sentinel)
assert os.path.exists(sr._disk_cache_path())
assert sr._load_seed_cache() == sentinel
backend/tests/test_skills_folders.py
+225
View File
@@ -0,0 +1,225 @@
"""Multi-file (folder) skills, plus backward compatibility with legacy flat skills.
A skill is now either ~/.claude/skills/<id>/SKILL.md (with optional supporting
files) or a legacy ~/.claude/skills/<id>.md. Both must list, read, and delete
correctly, and a folder skill with supporting files must get its folder path
appended to the prompt so the agent can read those files on demand.
"""
from __future__ import annotations
import os
import json
import pytest
import backend.apps.skills.skills as skills_mod
from backend.apps.agents.manager.prompt.prompt_context import _resolve_attached_skills
@pytest.fixture
def skills_dir(tmp_path, monkeypatch):
d = tmp_path / "skills"
d.mkdir()
monkeypatch.setattr(skills_mod, "SKILLS_DIR", str(d))
monkeypatch.setattr(skills_mod, "INDEX_PATH", str(d / ".skills_index.json"))
return d
def _write(path, text):
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
f.write(text)
def test_corrupt_index_does_not_brick_skills_and_is_preserved(skills_dir):
_write(str(skills_dir / "alpha.md"), "content")
with open(skills_dir / ".skills_index.json", "w") as f:
f.write("{ not valid json")
# Load returns empty instead of raising, and moves the bad file aside.
assert skills_mod._load_index() == {}
assert (skills_dir / ".skills_index.json.corrupt").exists()
# Skills still list (name falls back to the filename), so nothing is bricked.
assert "alpha" in {s.id for s in skills_mod._sync_skills()}
def test_non_object_index_is_rejected(skills_dir):
with open(skills_dir / ".skills_index.json", "w") as f:
f.write("[1, 2, 3]")
assert skills_mod._load_index() == {}
def test_save_index_is_atomic_no_temp_leftover(skills_dir):
skills_mod._save_index({"x": {"name": "X"}})
assert skills_mod._load_index() == {"x": {"name": "X"}}
leftovers = [n for n in __import__("os").listdir(skills_dir) if n.startswith(".skills_index.") and n.endswith(".tmp")]
assert leftovers == []
def test_flat_skill_still_syncs(skills_dir):
_write(str(skills_dir / "my-flat.md"), "do the flat thing")
skills = {s.id: s for s in skills_mod._sync_skills()}
assert "my-flat" in skills
s = skills["my-flat"]
assert s.content == "do the flat thing"
assert s.dir_path == ""
assert s.has_supporting_files is False
def test_folder_skill_syncs_with_supporting_files(skills_dir):
base = skills_dir / "remotion"
_write(str(base / "SKILL.md"), "---\nname: Remotion\ndescription: make videos\n---\nrender stuff")
_write(str(base / "helper.py"), "print('hi')")
skills = {s.id: s for s in skills_mod._sync_skills()}
assert "remotion" in skills
s = skills["remotion"]
assert "render stuff" in s.content
assert s.dir_path == str(base)
assert s.has_supporting_files is True
# Frontmatter fills name/description when the index hasn't catalogued it.
assert s.name == "Remotion"
assert s.description == "make videos"
def test_folder_skill_without_extra_files_flags_false(skills_dir):
base = skills_dir / "solo"
_write(str(base / "SKILL.md"), "just one file")
s = {x.id: x for x in skills_mod._sync_skills()}["solo"]
assert s.dir_path == str(base)
assert s.has_supporting_files is False
@pytest.mark.asyncio
async def test_delete_removes_folder(skills_dir):
base = skills_dir / "doomed"
_write(str(base / "SKILL.md"), "x")
_write(str(base / "data.txt"), "y")
assert base.is_dir()
await skills_mod.delete_skill("doomed")
assert not base.exists()
@pytest.mark.asyncio
async def test_update_writes_folder_skill_md(skills_dir):
base = skills_dir / "editable"
_write(str(base / "SKILL.md"), "old body")
from backend.apps.skills.models import SkillUpdate
res = await skills_mod.update_skill("editable", SkillUpdate(content="new body", description="d"))
assert res["ok"]
with open(base / "SKILL.md", encoding="utf-8") as f:
assert f.read() == "new body"
assert res["skill"]["dir_path"] == str(base)
def test_injection_points_at_folder_for_supporting_files(skills_dir):
base = skills_dir / "withfiles"
_write(str(base / "SKILL.md"), "use the template")
_write(str(base / "template.html"), "<html></html>")
block = _resolve_attached_skills([{"id": "withfiles", "name": "WithFiles", "content": "use the template"}])
assert "[Using skill: WithFiles]" in block
assert str(base) in block
assert "Read" in block # tells the agent to read supporting files
def test_skill_injection_is_provider_agnostic_by_construction(skills_dir):
"""The provider-agnostic claim, proven structurally (a live GPT/Gemini run
needs a key): the injector takes no provider arg so it CAN'T differ by model,
it points at supporting files via the universal Read tool, and Read is in the
builtin set every provider gets. So a non-Claude agent receives byte-identical
skill text and the same file-reading tools."""
import inspect
from backend.apps.agents.manager.prompt.prompt_context import _resolve_attached_skills
from backend.apps.agents.manager.prompt.tool_catalog import FULL_TOOLS
# 1. No provider/api parameter -> the injected text cannot branch on the model.
assert set(inspect.signature(_resolve_attached_skills).parameters) == {"attached_skills"}
# 2. A folder skill yields the body + a pointer to its folder via Read/Glob/Bash.
base = skills_dir / "vid"
_write(str(base / "SKILL.md"), "render it")
_write(str(base / "helper.py"), "x")
block = _resolve_attached_skills([{"id": "vid", "name": "Vid", "content": "render it"}])
assert "[Using skill: Vid]" in block
assert str(base) in block and "Read" in block
# 3. Those file tools are universal builtins, not an Anthropic-only set.
assert {"Read", "Glob", "Bash"} <= set(FULL_TOOLS)
def test_injection_no_folder_note_for_flat_skill(skills_dir):
_write(str(skills_dir / "plain.md"), "plain content")
block = _resolve_attached_skills([{"id": "plain", "name": "Plain", "content": "plain content"}])
assert "[Using skill: Plain]" in block
assert "supporting files" not in block.lower()
# ---------------------------------------------------------------------------
# .swarm round-trip for folder skills (export carries files, import rebuilds them).
# ---------------------------------------------------------------------------
def test_swarm_export_folder_skill_carries_supporting_files(skills_dir):
from backend.apps.swarm.entities.skills import SkillExportable
base = skills_dir / "vid"
_write(str(base / "SKILL.md"), "render")
_write(str(base / "scripts" / "go.py"), "print(1)")
exp = SkillExportable.load("vid")
assert exp is not None
files = exp.files()
assert "scripts/go.py" in files
assert files["scripts/go.py"] == b"print(1)"
assert exp._payload["content"] == "render"
def test_swarm_import_writes_folder_when_files_present(skills_dir):
from backend.apps.swarm.entities.skills import SkillExportable
payload = {"slug": "vid", "name": "Vid", "description": "d", "command": "vid", "content": "render"}
new_id = SkillExportable.import_(payload, {"scripts/go.py": b"print(1)"}, None)
assert os.path.isfile(skills_dir / new_id / "SKILL.md")
assert os.path.isfile(skills_dir / new_id / "scripts" / "go.py")
synced = {s.id: s for s in skills_mod._sync_skills()}
assert synced[new_id].has_supporting_files is True
def test_swarm_import_always_writes_folder(skills_dir):
# Unified storage: even a one-file skill imports as a folder, so a skill's
# on-disk shape never depends on whether it had supporting files.
from backend.apps.swarm.entities.skills import SkillExportable
payload = {"slug": "note", "name": "Note", "content": "just text"}
new_id = SkillExportable.import_(payload, {}, None)
assert os.path.isfile(skills_dir / new_id / "SKILL.md")
assert not (skills_dir / f"{new_id}.md").exists()
@pytest.mark.asyncio
async def test_create_writes_folder_and_supersedes_legacy_flat(skills_dir):
from backend.apps.skills.models import SkillCreate
# A pre-existing legacy flat skill of the same id...
_write(str(skills_dir / "notes.md"), "old flat")
# ...is superseded (not shadowed) when the user (re)creates it; folder wins,
# and the phantom flat file is removed so there's exactly one shape on disk.
res = await skills_mod.create_skill(SkillCreate(name="Notes", content="new body", description="d"))
sid = res["skill"]["id"]
assert sid == "notes"
assert os.path.isfile(skills_dir / "notes" / "SKILL.md")
assert not (skills_dir / "notes.md").exists()
only = [s for s in skills_mod._sync_skills() if s.id == "notes"]
assert len(only) == 1 and only[0].content == "new body"
def test_stage_zip_carries_supporting_files_into_sandbox():
import io as _io, zipfile, os as _os, shutil
from backend.apps.swarm.closure import _stage_skill_from_zip
buf = _io.BytesIO()
with zipfile.ZipFile(buf, "w") as zf:
zf.writestr("my-skill/SKILL.md", "do it")
zf.writestr("my-skill/scripts/run.sh", "echo hi")
sandbox, manifest, warnings = _stage_skill_from_zip(buf.getvalue(), "my-skill.zip", [])
try:
bid = manifest.entities[0].bundle_id
files_dir = _os.path.join(sandbox, "entities", bid, "files")
assert _os.path.isfile(_os.path.join(files_dir, "scripts", "run.sh"))
# SKILL.md is the payload body, not a supporting file.
assert not _os.path.exists(_os.path.join(files_dir, "SKILL.md"))
finally:
shutil.rmtree(sandbox, ignore_errors=True)
backend/tests/test_swarm_bundle.py
+323 -13
View File
@@ -49,8 +49,8 @@ def test_skill_export_import_round_trip(skill_store):
# Original is untouched, import lands under a fresh, non-clobbering slug.
assert root_type == EntityType.skill
assert root_id != "my-skill"
assert (skill_store / "my-skill.md").exists()
assert (skill_store / f"{root_id}.md").read_text(encoding="utf-8") == "# hello\nbody text"
assert (skill_store / "my-skill.md").exists() # original flat skill untouched
assert (skill_store / root_id / "SKILL.md").read_text(encoding="utf-8") == "# hello\nbody text"
assert created == {"skill": [root_id]}
@@ -63,7 +63,7 @@ def test_bare_markdown_import(skill_store):
finally:
import shutil
shutil.rmtree(sandbox, ignore_errors=True)
assert (skill_store / f"{root_id}.md").read_text(encoding="utf-8") == "# Just markdown"
assert (skill_store / root_id / "SKILL.md").read_text(encoding="utf-8") == "# Just markdown"
def test_content_secret_redacted_in_bundle(skill_store):
@@ -99,6 +99,19 @@ def test_pack_refuses_denied_key():
pack({"format_version": 1}, {"bid1": {"api_key": "leak"}}, {})
def test_pack_refuses_secret_in_workspace_file():
# A key hardcoded in app source (not .env) must not ride along; pack scans
# file bytes, not just payload keys.
leak = b"const KEY = 'sk-ant-api03-AAAAAAAAAAAAAAAAAAAAAAAA';\n"
with pytest.raises(BundleError):
pack({"format_version": 1}, {"bid1": {"name": "ok"}}, {"entities/bid1/files/config.js": leak})
def test_pack_allows_clean_workspace_file():
raw = pack({"format_version": 1}, {"bid1": {"name": "ok"}}, {"entities/bid1/files/app.js": b"export default 1"})
assert zipfile.is_zipfile(io.BytesIO(raw))
def test_app_export_drops_machine_env(tmp_path, monkeypatch):
# The live .env holds the source machine's absolute paths + pinned port; it
# must never ride along. .env.example (portable) does.
@@ -155,23 +168,194 @@ def test_workflow_unavailable_on_this_branch():
WorkflowExportable.import_({"title": "x"}, {}, RemapTable())
def test_session_export_strips_transcript_and_secrets():
def test_session_export_carries_transcript_drops_runtime_and_secrets():
from backend.apps.swarm.entities.sessions import SessionExportable
from backend.apps.swarm.redact import scrub_payload
data = {
"name": "A", "provider": "anthropic", "model": "sonnet", "mode": "agent",
"system_prompt": "hi", "allowed_tools": ["Read"],
"messages": [{"role": "user", "content": "private chat"}],
"messages": [
{"id": "m1", "role": "user", "content": "private chat", "branch_id": "main"},
{"id": "m2", "role": "assistant", "content": "token is sk-ant-abcdefghij0123456789"},
],
"branches": {"main": {"id": "main", "parent_branch_id": None, "fork_point_message_id": None}},
"active_branch_id": "main",
"tool_group_meta": {"g1": {"label": "x"}},
"active_mcps": ["Gmail"], "cwd": "/Users/me/repo", "cost_usd": 9.9, "sdk_session_id": "x",
}
ex = SessionExportable("s1", "A", data)
out = ex.serialize(None)
for gone in ("messages", "cwd", "active_mcps", "cost_usd", "sdk_session_id"):
# The transcript now rides along, that's the point of sharing an agent.
assert out["messages"][0]["content"] == "private chat"
assert out["active_branch_id"] == "main" and "main" in out["branches"]
assert out["tool_group_meta"] == {"g1": {"label": "x"}}
# Runtime, identity, and gate state still never leave.
for gone in ("cwd", "active_mcps", "cost_usd", "sdk_session_id"):
assert gone not in out
assert out["model"] == "sonnet" and out["mode"] == "agent"
# The closure runs scrub_payload on every payload, so a secret-shaped
# string sitting in the transcript is redacted before it ships.
assert "sk-ant-" not in json.dumps(scrub_payload(out))
reqs = ex.requirements()
assert any(r.kind.value == "mcp_action" and r.key == "Gmail" for r in reqs)
def test_session_import_restores_transcript_without_granting_mcp(monkeypatch):
from backend.apps.swarm.entities.sessions import SessionExportable
from backend.apps.swarm.exportable import RemapTable
from backend.apps.agents.manager.session import session_store
saved: dict = {}
monkeypatch.setattr(session_store, "_save_session", lambda sid, doc: saved.update({sid: doc}))
payload = {
"name": "A", "model": "sonnet", "mode": "agent",
"messages": [{"id": "m1", "role": "user", "content": "hi", "branch_id": "main"}],
"branches": {"main": {"id": "main", "parent_branch_id": None, "fork_point_message_id": None}},
"active_branch_id": "main",
"tool_group_meta": {"g1": {"label": "x"}},
}
sid = SessionExportable.import_(payload, {}, RemapTable())
doc = saved[sid]
assert doc["messages"][0]["content"] == "hi"
assert doc["active_branch_id"] == "main"
assert doc["tool_group_meta"] == {"g1": {"label": "x"}}
# The gate stays shut: a shared agent never arrives with MCP access.
assert doc["active_mcps"] == []
# The dashboard import re-points this; it must never be the sharer's id.
assert doc["dashboard_id"] is None
def test_session_import_old_bundle_without_transcript(monkeypatch):
# A bundle made before transcripts were carried has no messages; it must
# still import as a valid empty-history agent (single main branch), not crash.
from backend.apps.swarm.entities.sessions import SessionExportable
from backend.apps.swarm.exportable import RemapTable
from backend.apps.agents.manager.session import session_store
saved: dict = {}
monkeypatch.setattr(session_store, "_save_session", lambda sid, doc: saved.update({sid: doc}))
sid = SessionExportable.import_({"name": "Old", "model": "sonnet"}, {}, RemapTable())
doc = saved[sid]
assert doc["messages"] == []
assert doc["active_branch_id"] == "main" and "main" in doc["branches"]
def test_session_load_prefers_live_memory_over_stale_disk(tmp_path, monkeypatch):
# The freshest transcript lives in memory; a disk-only load would ship a
# stale one. load() must read the live session first, disk only as fallback.
from backend.apps.agents import agent_manager as am
from backend.apps.swarm.entities.sessions import SessionExportable
sdir = tmp_path / "sessions"
sdir.mkdir()
monkeypatch.setattr(am, "SESSIONS_DIR", str(sdir))
(sdir / "s1.json").write_text(json.dumps(
{"name": "Stale", "messages": [{"id": "old", "role": "user", "content": "old"}]}))
class FakeSess:
def model_dump(self, mode="json"):
return {"name": "Live", "messages": [
{"id": "old", "role": "user", "content": "old"},
{"id": "new", "role": "assistant", "content": "fresh turn"},
]}
monkeypatch.setattr(am.agent_manager, "sessions", {"s1": FakeSess()})
out = SessionExportable.load("s1").serialize(None)
assert out["name"] == "Live" # not the stale disk copy
assert len(out["messages"]) == 2 # the unflushed turn is included
def test_dashboard_export_import_carries_agent_cards_and_transcript(tmp_path, monkeypatch):
# The path the single-session tests missed: a whole dashboard with agent
# cards + a browser card. Both agents (with their transcripts) and the
# browser must survive export -> import. An empty-history import is the bug
# the user hit ("the chats didn't even show up, let alone the history").
import shutil
from backend.apps.agents import agent_manager as am
import backend.config.paths as paths
sdir = tmp_path / "sessions"
ddir = tmp_path / "dashboards"
sdir.mkdir()
ddir.mkdir()
monkeypatch.setattr(am, "SESSIONS_DIR", str(sdir))
monkeypatch.setattr(paths, "DASHBOARDS_DIR", str(ddir))
monkeypatch.setattr(am.agent_manager, "sessions", {}) # nothing live -> disk path
did, sid1, sid2, bkey = "d1", "sA", "sB", "browser-1"
def sess(sid, name, text):
return {
"id": sid, "name": name, "status": "completed", "provider": "anthropic",
"model": "sonnet", "mode": "agent", "allowed_tools": [],
"messages": [{"id": "m1", "role": "user", "content": text, "branch_id": "main"}],
"branches": {"main": {"id": "main", "parent_branch_id": None, "fork_point_message_id": None, "created_at": "2026-01-01"}},
"active_branch_id": "main", "tool_group_meta": {}, "active_mcps": [], "dashboard_id": did,
}
(sdir / f"{sid1}.json").write_text(json.dumps(sess(sid1, "Agent One", "from one")))
(sdir / f"{sid2}.json").write_text(json.dumps(sess(sid2, "Agent Two", "from two")))
(ddir / f"{did}.json").write_text(json.dumps({"id": did, "name": "Board", "layout": {
"cards": {sid1: {"session_id": sid1}, sid2: {"session_id": sid2}},
"view_cards": {},
"browser_cards": {bkey: {"browser_id": bkey, "url": "u", "spawned_by": None}},
"notes": {}, "expanded_session_ids": [sid1],
}}))
raw, _ = closure.build_bundle(EntityType.dashboard, did)
sandbox, manifest, _w = closure.stage_upload(raw, "board.swarm")
try:
_rt, root_id, _created, _u = closure.commit(sandbox, manifest, [])
finally:
shutil.rmtree(sandbox, ignore_errors=True)
L = json.loads((ddir / f"{root_id}.json").read_text())["layout"]
assert len(L["cards"]) == 2, "both agent cards must survive import"
assert len(L["browser_cards"]) == 1, "the browser card must survive too"
total_msgs = 0
for sid in L["cards"]:
doc = json.loads((sdir / f"{sid}.json").read_text())
total_msgs += len(doc.get("messages") or [])
assert doc["active_mcps"] == [], "import must not grant MCP access"
assert total_msgs == 2, "each agent's transcript must carry through"
# The bug behind "the chats didn't even show up": after import the sessions
# are on disk but not in memory, and the dashboard-open fetch
# (get_all_sessions) was memory-only, so the cards rendered blank. The fetch
# must now see the freshly-imported sessions straight off disk.
found = am.agent_manager.get_all_sessions(dashboard_id=root_id)
assert len(found) == 2, f"dashboard-open fetch must see imported agent sessions, got {len(found)}"
assert sum(len(s.messages) for s in found) == 2, "and with their transcripts"
def test_get_all_sessions_does_not_resurrect_deleted_cards(tmp_path, monkeypatch):
# Deleting a card removes it from the layout but the session keeps its
# dashboard_id on disk. get_all_sessions must surface only sessions the
# layout still has a card for, or deleted chats come back on every reopen.
from backend.apps.agents import agent_manager as am
import backend.config.paths as paths
sdir = tmp_path / "sessions"
ddir = tmp_path / "dashboards"
sdir.mkdir()
ddir.mkdir()
monkeypatch.setattr(am, "SESSIONS_DIR", str(sdir))
monkeypatch.setattr(paths, "DASHBOARDS_DIR", str(ddir))
monkeypatch.setattr(am.agent_manager, "sessions", {})
did = "d1"
def sess(sid):
return {
"id": sid, "name": sid, "status": "completed", "model": "sonnet",
"mode": "agent", "messages": [], "branches": {}, "active_branch_id": "main",
"dashboard_id": did,
}
(sdir / "kept.json").write_text(json.dumps(sess("kept")))
(sdir / "deleted.json").write_text(json.dumps(sess("deleted"))) # still tagged, card gone
# The layout has a card only for "kept" (the user deleted "deleted"'s card).
(ddir / f"{did}.json").write_text(json.dumps({"id": did, "layout": {"cards": {"kept": {"session_id": "kept"}}}}))
ids = {s.id for s in am.agent_manager.get_all_sessions(dashboard_id=did)}
assert "kept" in ids, "a session the layout still has a card for must surface"
assert "deleted" not in ids, "a session whose card was deleted must NOT resurrect"
def test_dashboard_serialize_rewrites_refs_to_bundle_ids():
from backend.apps.swarm.entities.dashboards import DashboardExportable
from backend.apps.swarm.models import EntityType
@@ -182,13 +366,15 @@ def test_dashboard_serialize_rewrites_refs_to_bundle_ids():
data = {"name": "D", "layout": {
"cards": {"S": {"session_id": "S", "x": 1}},
"view_cards": {"A": {"output_id": "A", "x": 2}},
"view_cards": {"A": {"output_id": "A", "x": 2, "parent_session_id": "S"}},
"browser_cards": {"b1": {"browser_id": "b1", "url": "u", "spawned_by": "S"}},
"expanded_session_ids": ["S"],
}}
L = DashboardExportable("d1", "D", data).serialize(Ctx())["layout"]
assert L["cards"]["SBID"]["session_id"] == "SBID"
assert L["view_cards"]["ABID"]["output_id"] == "ABID"
# the app card's tether to its builder agent is a session id, so it remaps too
assert L["view_cards"]["ABID"]["parent_session_id"] == "SBID"
assert L["browser_cards"]["b1"]["spawned_by"] == "SBID"
assert L["expanded_session_ids"] == ["SBID"]
@@ -205,18 +391,96 @@ def test_dashboard_import_remaps_to_fresh_local_ids(monkeypatch):
remap.assign("ABID", "newapp")
payload = {"name": "D", "layout": {
"cards": {"SBID": {"session_id": "SBID"}},
"view_cards": {"ABID": {"output_id": "ABID"}},
"view_cards": {
"ABID": {"output_id": "ABID", "parent_session_id": "SBID"},
"ABID2": {"output_id": "ABID2", "parent_session_id": "GONE"},
},
"browser_cards": {"b1": {"browser_id": "b1", "spawned_by": "SBID"}},
"expanded_session_ids": ["SBID", "ORPHAN"],
}}
remap.assign("ABID2", "newapp2")
did = dmod.DashboardExportable.import_(payload, {}, remap)
L = written[did]["layout"]
assert L["cards"]["newsess"]["session_id"] == "newsess"
assert "newapp" in L["view_cards"]
assert L["view_cards"]["newapp"]["parent_session_id"] == "newsess"
assert L["view_cards"]["newapp2"]["parent_session_id"] is None # parent not in bundle
assert list(L["browser_cards"].values())[0]["spawned_by"] == "newsess"
assert L["expanded_session_ids"] == ["newsess"] # the dangling ref is dropped
def test_dashboard_remap_invariant_generative(monkeypatch):
# The hand-written remap tests only check the id-bearing fields I remembered.
# Generate random dashboards and assert the real invariant on a serialize ->
# import round-trip: no source-local id and no bundle id survives into the
# imported layout, and every card id is a freshly-minted local id. This is
# what catches "someone adds a new layout field holding a session id and
# forgets to remap it."
import random
from backend.apps.swarm.entities import dashboards as dmod
from backend.apps.swarm.exportable import RemapTable
from backend.apps.swarm.models import EntityType
written: dict = {}
monkeypatch.setattr(dmod, "_write", lambda did, doc: written.update({did: doc}))
monkeypatch.setattr(dmod, "_retag_sessions", lambda ids, did: None)
rng = random.Random(1234)
for _ in range(60):
sess = [f"S{i}" for i in range(rng.randint(0, 5))]
apps = [f"A{i}" for i in range(rng.randint(0, 4))]
s_bid = {s: f"sbid{i}" for i, s in enumerate(sess)}
a_bid = {a: f"abid{i}" for i, a in enumerate(apps)}
class Ctx:
def bundle_id_for(self, t, lid):
if t == EntityType.session:
return s_bid.get(lid)
if t == EntityType.app:
return a_bid.get(lid)
return None
layout = {
"cards": {s: {"session_id": s, "x": rng.randint(0, 9)} for s in sess},
"view_cards": {
a: {"output_id": a,
"parent_session_id": (rng.choice(sess + ["ORPHAN"]) if sess and rng.random() < 0.7 else None)}
for a in apps
},
"browser_cards": {
f"b{i}": {"browser_id": f"b{i}", "url": "u",
"spawned_by": (rng.choice(sess) if sess and rng.random() < 0.7 else None)}
for i in range(rng.randint(0, 3))
},
"expanded_session_ids": (sess + ["ORPHAN"]) if rng.random() < 0.5 else list(sess),
}
payload = dmod.DashboardExportable("d-src", "D", {"name": "D", "layout": layout}).serialize(Ctx())
remap = RemapTable()
fresh_sess = {s: f"new-{s_bid[s]}" for s in sess}
fresh_apps = {a: f"new-{a_bid[a]}" for a in apps}
for s in sess:
remap.assign(s_bid[s], fresh_sess[s])
for a in apps:
remap.assign(a_bid[a], fresh_apps[a])
did = dmod.DashboardExportable.import_(payload, {}, remap)
L = written[did]["layout"]
forbidden = set(sess) | set(apps) | set(s_bid.values()) | set(a_bid.values())
assert set(L["cards"]) == set(fresh_sess.values())
assert set(L["view_cards"]) == set(fresh_apps.values())
for cid, card in L["cards"].items():
assert cid not in forbidden and card["session_id"] == cid
for oid, card in L["view_cards"].items():
assert oid not in forbidden and card["output_id"] == oid
p = card["parent_session_id"]
assert p is None or (p in set(fresh_sess.values()) and p not in forbidden)
assert set(L["expanded_session_ids"]) <= set(fresh_sess.values())
for card in L["browser_cards"].values():
assert card["spawned_by"] is None or card["spawned_by"] in set(fresh_sess.values())
def test_checksum_rejects_tampering(skill_store):
_make_skill(skill_store, "tmp", "Tmp", "# original")
raw, _ = closure.build_bundle(EntityType.skill, "tmp")
@@ -239,9 +503,9 @@ def test_skill_rollback_removes_it(skill_store):
from backend.apps.swarm.entities.skills import SkillExportable
from backend.apps.swarm.exportable import RemapTable
sid = SkillExportable.import_({"slug": "rbk", "name": "Rbk", "content": "x"}, {}, RemapTable())
assert (skill_store / f"{sid}.md").exists()
assert (skill_store / sid / "SKILL.md").exists()
SkillExportable.rollback(sid)
assert not (skill_store / f"{sid}.md").exists()
assert not (skill_store / sid).exists()
assert sid not in store._load_index()
@@ -264,7 +528,41 @@ def test_commit_rolls_back_created_on_failure(skill_store, tmp_path):
with pytest.raises(BundleError):
closure.commit(str(sb), manifest, [])
assert "rollme" not in store._load_index()
assert not (skill_store / "rollme.md").exists()
assert not (skill_store / "rollme").exists() # the imported folder was rolled back
def test_manifest_duplicate_ids_rejected():
# Two entities sharing a bundle_id silently collapse in the topo/summary
# dicts, dropping one; reject up front. (The manifest is outside the checksum.)
from backend.apps.swarm.closure import validate_manifest
from backend.apps.swarm.models import BundlePreview, EntityRef, Manifest
ref = EntityRef(type=EntityType.skill, bundle_id="dup", name="A", path="entities/dup")
m = Manifest(bundle_id="b", root=ref, entities=[ref, ref],
preview=BundlePreview(root_type=EntityType.skill, root_name="A"))
with pytest.raises(BundleError):
validate_manifest(m)
def test_manifest_root_not_in_entities_rejected():
from backend.apps.swarm.closure import validate_manifest
from backend.apps.swarm.models import BundlePreview, EntityRef, Manifest
root = EntityRef(type=EntityType.skill, bundle_id="root", name="A", path="entities/root")
other = EntityRef(type=EntityType.skill, bundle_id="other", name="B", path="entities/other")
m = Manifest(bundle_id="b", root=root, entities=[other],
preview=BundlePreview(root_type=EntityType.skill, root_name="A"))
with pytest.raises(BundleError):
validate_manifest(m)
def test_manifest_edge_to_unknown_entity_rejected():
from backend.apps.swarm.closure import validate_manifest
from backend.apps.swarm.models import BundlePreview, DependencyEdge, EntityRef, Manifest
ref = EntityRef(type=EntityType.dashboard, bundle_id="d", name="D", path="entities/d")
m = Manifest(bundle_id="b", root=ref, entities=[ref],
edges=[DependencyEdge(**{"from": "d", "to": "ghost"})],
preview=BundlePreview(root_type=EntityType.dashboard, root_name="D"))
with pytest.raises(BundleError):
validate_manifest(m)
def _zip_with(name, data=b"x"):
@@ -284,6 +582,18 @@ def test_absolute_path_rejected():
unpack(_zip_with("/etc/evil"))
def test_symlink_entry_rejected():
# A symlink entry could point outside the sandbox once followed; unpack must
# refuse it before writing anything.
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w") as zf:
zi = zipfile.ZipInfo("link")
zi.external_attr = 0o120777 << 16
zf.writestr(zi, "/etc/passwd")
with pytest.raises(BundleError):
unpack(buf.getvalue())
def test_too_many_entries_rejected():
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w") as zf:
backend/tests/test_tool_policy_slot.py
+141
View File
@@ -0,0 +1,141 @@
"""'Always approve' persistence invariant.
The bug: the dispatch gate READ a tool's policy from one slot while the
'Always approve' button WROTE it to another (the raw mcp__server__action name
in builtin_permissions vs the parsed inner action on the owning tool), so the
next call never saw the policy and the button behaved like a one-time accept.
The seal: both sides now resolve the slot through resolve_policy_slot(), so a
WRITE always lands where the READ looks. These tests pin that for every
tool-name shape, including the round-trip that the old code failed.
"""
from backend.apps.tools_lib.tools_lib import resolve_policy_slot, PolicySlot
from backend.apps.tools_lib.mcp_config import _sanitize_server_name
from backend.apps.tools_lib.models import ToolDefinition
def _mcp_tool(name: str) -> ToolDefinition:
return ToolDefinition(name=name, mcp_config={"command": "x"}, enabled=True, tool_permissions={})
def test_slot_for_builtin_tool():
assert resolve_policy_slot("Bash", []) == PolicySlot("builtin", "Bash", None)
assert resolve_policy_slot("Read", []) == PolicySlot("builtin", "Read", None)
def test_slot_for_our_browser_and_invoke_agents_uses_inner_name():
# These live in builtin_permissions under the INNER name, not the namespaced one.
assert resolve_policy_slot("mcp__openswarm-browser-agent__BrowserAgent", []) == \
PolicySlot("builtin", "BrowserAgent", None)
assert resolve_policy_slot("mcp__openswarm-invoke-agent__InvokeAgent", []) == \
PolicySlot("builtin", "InvokeAgent", None)
def test_slot_for_community_mcp_points_at_the_owning_tool():
tool = _mcp_tool("My Notion Server")
slug = _sanitize_server_name(tool.name)
assert resolve_policy_slot(f"mcp__{slug}__notion-fetch", [tool]) == \
PolicySlot("mcp", tool.id, "notion-fetch")
def test_slot_for_unknown_mcp_has_no_write_target():
assert resolve_policy_slot("mcp__ghostserver__do-thing", []) == \
PolicySlot("mcp", None, "do-thing")
# read/write mirror the dispatch-gate branches in agent_manager
# (_get_effective_policy / _set_tool_policy): both key through resolve_policy_slot.
def _read(tool_name, builtin_perms, tools):
slot = resolve_policy_slot(tool_name, tools)
if slot.store == "builtin":
return builtin_perms.get(slot.key, "ask")
if slot.key is not None:
for t in tools:
if t.id == slot.key:
return t.tool_permissions.get(slot.action, "ask")
return "ask"
def _write(tool_name, policy, builtin_perms, tools):
slot = resolve_policy_slot(tool_name, tools)
if slot.store == "builtin":
builtin_perms[slot.key] = policy
return
if slot.key is not None:
for t in tools:
if t.id == slot.key:
t.tool_permissions[slot.action] = policy
return
def test_always_approve_round_trips_for_every_tool_shape():
"""The invariant the old code violated: after WRITE(always_allow), the very
next READ returns always_allow, for builtin, our agents, and community MCP."""
notion = _mcp_tool("Notion")
slug = _sanitize_server_name("Notion")
tools = [notion]
builtin_perms: dict[str, str] = {}
shapes = [
"Bash",
"Read",
"mcp__openswarm-browser-agent__BrowserAgent",
"mcp__openswarm-invoke-agent__InvokeAgent",
f"mcp__{slug}__notion-fetch",
]
for tool_name in shapes:
assert _read(tool_name, builtin_perms, tools) != "always_allow"
_write(tool_name, "always_allow", builtin_perms, tools)
assert _read(tool_name, builtin_perms, tools) == "always_allow", \
f"{tool_name}: write did not land in the slot the gate reads"
def test_two_actions_on_the_same_mcp_server_are_independent():
"""Approving one action must not silently approve a sibling action."""
tool = _mcp_tool("Notion")
slug = _sanitize_server_name("Notion")
tools = [tool]
bp: dict[str, str] = {}
_write(f"mcp__{slug}__notion-fetch", "always_allow", bp, tools)
assert _read(f"mcp__{slug}__notion-fetch", bp, tools) == "always_allow"
assert _read(f"mcp__{slug}__notion-create-pages", bp, tools) == "ask"
# ---- Integration: the same round-trip through the REAL file persistence the gate
# uses (load_builtin_permissions / _save / _load_all), so 'write then re-read'
# survives a save+reload, not just an in-memory dict. ----
import backend.apps.tools_lib.tools_lib as tl
def test_builtin_policy_survives_a_real_file_reload(tmp_path, monkeypatch):
monkeypatch.setattr(tl, "BUILTIN_PERMS_PATH", str(tmp_path / "builtin_permissions.json"))
slot = tl.resolve_policy_slot("Read", [])
perms = tl.load_builtin_permissions()
perms[slot.key] = "always_allow"
tl.save_builtin_permissions(perms)
# Fresh read (what the next session / a reload does) finds it at the read key.
reloaded = tl.load_builtin_permissions()
assert reloaded.get(tl.resolve_policy_slot("Read", []).key) == "always_allow"
def test_mcp_policy_survives_a_real_tool_file_reload(tmp_path, monkeypatch):
monkeypatch.setattr(tl, "DATA_DIR", str(tmp_path))
monkeypatch.setattr(tl, "_tools_cache", None)
monkeypatch.setattr(tl, "_tools_cache_sig", None)
tl._save(_mcp_tool("Notion"))
slug = _sanitize_server_name("Notion")
name = f"mcp__{slug}__notion-fetch"
# WRITE via the resolver against the freshly loaded tool, then persist.
tools = tl._load_all()
slot = tl.resolve_policy_slot(name, tools)
target = next(t for t in tools if t.id == slot.key)
target.tool_permissions[slot.action] = "always_allow"
tl._save(target)
# RELOAD from disk and read via the resolver: the policy is there.
tools2 = tl._load_all()
rslot = tl.resolve_policy_slot(name, tools2)
got = next(t for t in tools2 if t.id == rslot.key)
assert got.tool_permissions.get(rslot.action) == "always_allow"
backend/tests/test_v2_invariants.py
+172
View File
@@ -210,6 +210,75 @@ async def test_gate_stress_random_activations():
)
# ===========================================================================
# Group A2, ToolSearch loop-breaker
# ===========================================================================
# Gated MCP servers are withheld from the SDK, so the CLI's native ToolSearch
# can never see them; small models loop (empty ToolSearch -> retry) until the
# user pauses. The break must (a) not fire on the first call or two (a power
# user may legitimately ToolSearch a deferred tool), (b) fire once it's clearly
# stuck, steering to MCPActivate, and (c) reset when any real tool runs.
def test_toolsearch_redirect_holds_below_threshold():
from backend.apps.agents.manager.prompt.prompt_context import (
toolsearch_loop_redirect,
TOOLSEARCH_LOOP_THRESHOLD,
)
for n in range(1, TOOLSEARCH_LOOP_THRESHOLD):
assert toolsearch_loop_redirect(n, ["gmail"]) is None, f"must not redirect at n={n}"
def test_toolsearch_redirect_fires_at_threshold_and_names_gated_servers():
from backend.apps.agents.manager.prompt.prompt_context import (
toolsearch_loop_redirect,
TOOLSEARCH_LOOP_THRESHOLD,
)
reason = toolsearch_loop_redirect(TOOLSEARCH_LOOP_THRESHOLD, ["google-workspace", "slack"])
assert reason is not None
assert "MCPActivate" in reason
assert "google-workspace" in reason and "slack" in reason
assert "Stop calling ToolSearch" in reason
def test_toolsearch_redirect_works_with_no_gated_servers():
# Even with nothing to activate, the steer must still tell the model its
# tools are already loaded so it stops searching (no crash on empty list).
from backend.apps.agents.manager.prompt.prompt_context import (
toolsearch_loop_redirect,
TOOLSEARCH_LOOP_THRESHOLD,
)
reason = toolsearch_loop_redirect(TOOLSEARCH_LOOP_THRESHOLD, [])
assert reason is not None
assert "MCPActivate" not in reason # nothing to point at
assert "Stop calling ToolSearch" in reason
@pytest.mark.asyncio
async def test_gated_server_names_surface_only_inactive_servers():
"""The steer list must mirror the gate: connected-but-not-active servers
only, never one that's already activated (callable) or denied."""
from backend.apps.agents.agent_manager import AgentManager
fake_tools = [_fake_tool("Gmail"), _fake_tool("Slack"), _fake_tool("Notion")]
with patch("backend.apps.agents.agent_manager.load_all_tools", return_value=fake_tools):
mgr = AgentManager()
names = mgr._gated_mcp_server_names(
allowed_tools=["mcp:Gmail", "mcp:Slack", "mcp:Notion"],
active_mcps=["gmail"], # already activated -> not "gated"
)
assert "gmail" not in names, "activated server must not appear as gated"
assert "slack" in names and "notion" in names
@pytest.mark.asyncio
async def test_gated_server_names_empty_when_all_active():
from backend.apps.agents.agent_manager import AgentManager
fake_tools = [_fake_tool("Gmail")]
with patch("backend.apps.agents.agent_manager.load_all_tools", return_value=fake_tools):
mgr = AgentManager()
assert mgr._gated_mcp_server_names(["mcp:Gmail"], ["gmail"]) == []
# ===========================================================================
# Group B, needs_fresh_session soft-restart
# ===========================================================================
@@ -398,6 +467,10 @@ async def test_resolve_aux_model_anthropic_pro_returns_proxy():
settings = AppSettings()
settings.connection_mode = "openswarm-pro"
settings.openswarm_proxy_url = "https://api.openswarm.test"
# A real Pro-connected user carries a bearer token; proxy_auth reads it.
# Without it the resolver can't see Pro and falls through to the raise,
# which is what made this test depend on live machine state.
settings.openswarm_bearer_token = "test-pro-token"
with patch("backend.apps.nine_router.is_running", return_value=False):
model_id, base = await registry.resolve_aux_model(settings)
assert "haiku" in model_id
@@ -523,6 +596,104 @@ def test_resolve_sdk_gemini_prefers_antigravity_over_api_key():
assert registry.resolve_model_id_for_sdk("gemini-3-flash", s2) == "gc/gemini-3-flash-preview"
def test_error_classify_schema_translation_400_is_not_auth():
"""A 9Router tool-schema translation 400 can carry provider/connection
wording that trips the auth regex, so it used to surface a misleading
'reconnect your subscription' card for what is really a schema bug. The
translation guard must win: schema 400 -> not auth; a real auth failure
with no translation signature still reads as auth."""
from backend.apps.agents.core.error_classify import p_is_auth_error, p_is_translation_error
both = Exception("provider not connected: 400 INVALID_ARGUMENT at "
"tools[0].function_declarations[0].parameters")
assert p_is_translation_error(both)
assert not p_is_auth_error(both), "schema-400 must not be classified as auth"
# Pure auth failures (no translation signature) still classify as auth.
assert p_is_auth_error(Exception("provider not connected: gemini"))
assert p_is_auth_error(Exception("401 invalid authentication credentials"))
assert not p_is_translation_error(Exception("401 invalid authentication credentials"))
def test_error_classify_gemini_resource_exhausted_is_transient():
"""gemini-cli's free-tier 429 surfaces as RESOURCE_EXHAUSTED; it must count
as transient so the existing backoff/retry catches it instead of dying as a
hard first-message error. A 403 (hard auth/quota) must still NOT retry."""
from backend.apps.agents.core.error_classify import p_is_transient_capacity_error
assert p_is_transient_capacity_error(Exception("429 RESOURCE_EXHAUSTED: Quota exceeded"))
assert p_is_transient_capacity_error(Exception("RESOURCE_EXHAUSTED"))
assert not p_is_transient_capacity_error(Exception("403 permission denied"))
@pytest.mark.asyncio
async def test_mcp_gate_only_forwards_activated_servers():
"""Dispatch-layer security invariant (the non-bypassable enforcement of
'MCP tools only via MCPActivate'): for a GATED session (active_mcps is a
list), _build_mcp_servers forwards ONLY servers whose sanitized name is in
active_mcps; an empty list forwards ZERO; None is the legacy all-allowed
path. The model cannot reach an unactivated server no matter what it asks
for. Property-checked over random installed sets and random activation
subsets, plus the two boundary cases."""
import random
from types import SimpleNamespace
from backend.apps.agents.agent_manager import AgentManager
mgr = AgentManager()
names = ["gmail", "drive", "slack", "reddit", "notion", "airtable"]
def installed():
return [SimpleNamespace(name=n, mcp_config={"x": 1}, enabled=True,
auth_status="configured", auth_type="apikey") for n in names]
# allowed_tools == get_all_tool_names() bypasses the (separate) permission
# gate so we isolate the ACTIVATION gate. _sanitize_server_name -> identity.
with patch("backend.apps.agents.agent_manager.load_all_tools", side_effect=installed), \
patch("backend.apps.agents.agent_manager.get_all_tool_names", return_value=["__ALL__"]), \
patch("backend.apps.agents.agent_manager._sanitize_server_name", side_effect=lambda n: n), \
patch("backend.apps.agents.agent_manager._is_fully_denied", return_value=False), \
patch("backend.apps.agents.agent_manager.derive_mcp_config", side_effect=lambda t: {"command": "x"}):
allowed = ["__ALL__"]
# Boundary 1: empty activation list -> zero servers, always.
assert await mgr._build_mcp_servers(allowed, active_mcps=[]) == {}
# Boundary 2: None (legacy) -> permission gate only, all forwarded.
assert set((await mgr._build_mcp_servers(allowed, active_mcps=None)).keys()) == set(names)
# Property: forwarded set is ALWAYS a subset of the activated set, and
# equals exactly the activated-and-installed intersection.
rng = random.Random(1234)
for _ in range(400):
active = rng.sample(names, rng.randint(0, len(names)))
# throw in a bogus name the gate must never invent a server for
if rng.random() < 0.3:
active = active + ["ghost-not-installed"]
forwarded = set((await mgr._build_mcp_servers(allowed, active_mcps=active)).keys())
assert forwarded <= set(active), f"leaked {forwarded - set(active)} for active={active}"
assert forwarded == (set(active) & set(names)), f"mismatch for active={active}"
def test_dashboard_get_strips_only_orphan_session_cards():
"""A layout card whose session vanished (gone from memory AND disk) makes the
frontend GET /sessions/{id} 404 on every load and flash a dead card. The
dashboard GET filters those orphan cards out of the response, but must keep
live (in-memory) cards, on-disk cards, and drafts. Non-destructive: only the
response is filtered, never the stored layout."""
from types import SimpleNamespace
from backend.apps.dashboards import dashboards as D
data = {"layout": {
"cards": {
"live": {"session_id": "live"}, # in memory
"ondisk": {"session_id": "ondisk"}, # closed but on disk
"draft-1": {"session_id": "draft-1"}, # unsent draft, no backend session yet
"ghost": {"session_id": "ghost"}, # gone from memory AND disk -> would 404
},
"expanded_session_ids": ["live", "ghost"],
}}
fake_mgr = SimpleNamespace(sessions={"live": object()})
on_disk = {"ondisk": {"id": "ondisk"}}
with patch("backend.apps.agents.agent_manager.agent_manager", fake_mgr), \
patch("backend.apps.agents.manager.session.session_store._load_session_data",
side_effect=lambda sid: on_disk.get(sid)):
D._strip_orphan_session_cards(data)
assert set(data["layout"]["cards"].keys()) == {"live", "ondisk", "draft-1"}, "only the ghost should be dropped"
assert data["layout"]["expanded_session_ids"] == ["live"], "ghost dropped from expanded too"
def test_banned_models_not_offered():
"""Claude Fable (banned) and Gemini 3.1 Pro (no working lane: AG can't serve
it, AI Studio key 429s pro-preview) were pulled from the picker. Guard so a
@@ -1038,6 +1209,7 @@ async def test_aux_failover_anthropic_to_codex():
settings = AppSettings()
settings.connection_mode = "openswarm-pro" # provides anthropic fallback
settings.openswarm_proxy_url = "https://api.openswarm.test"
settings.openswarm_bearer_token = "test-pro-token" # what a real Pro user carries
with patch("backend.apps.nine_router.is_running", return_value=True), \
patch("backend.apps.nine_router.get_providers",
new=AsyncMock(return_value=[])): # nothing connected
backend/tests/test_versions.py
+210
View File
@@ -0,0 +1,210 @@
"""App version history: capture / list / restore / branch round-trips for both
a flat (inline-files) app and a webapp_template (workspace-folder) app, plus the
load-bearing invariants: dedupe of unchanged state, the pre_restore safety net,
.env preserved across restore, and branch producing a fully independent app.
The path constants are module-level, so (like test_swarm_bundle) we monkeypatch
them per test into a temp tree."""
import os
import pytest
from backend.apps.outputs import versions, workspace_io
from backend.apps.outputs.models import Output
from backend.apps.swarm.entities import apps as appmod
@pytest.fixture
def stores(tmp_path, monkeypatch):
outputs_dir = tmp_path / "outputs"
ws_dir = tmp_path / "ws"
ver_dir = tmp_path / "versions"
for d in (outputs_dir, ws_dir, ver_dir):
d.mkdir()
monkeypatch.setattr(workspace_io, "DATA_DIR", str(outputs_dir))
monkeypatch.setattr(versions, "OUTPUTS_VERSIONS_DIR", str(ver_dir))
monkeypatch.setattr(versions, "OUTPUTS_WORKSPACE_DIR", str(ws_dir))
monkeypatch.setattr(appmod, "OUTPUTS_WORKSPACE_DIR", str(ws_dir))
monkeypatch.setattr(appmod, "OUTPUTS_DIR", str(outputs_dir))
return ws_dir
def _flat_app(html="<h1>v1</h1>"):
o = Output(name="Flat", files={"index.html": html})
workspace_io._save(o)
return o
def _webapp(ws_dir, files):
wsid = "wsid1"
folder = os.path.join(str(ws_dir), wsid)
os.makedirs(folder, exist_ok=True)
for rel, content in files.items():
p = os.path.join(folder, rel)
os.makedirs(os.path.dirname(p), exist_ok=True)
with open(p, "w") as f:
f.write(content)
o = Output(name="Web", workspace_id=wsid)
workspace_io._save(o)
return o, folder
def test_flat_capture_list_and_restore(stores):
o = _flat_app("<h1>v1</h1>")
v1 = versions.capture(o.id, source="manual", label="v1")
assert v1 is not None
o.files = {"index.html": "<h1>v2</h1>"}
workspace_io._save(o)
versions.capture(o.id, source="auto", label="made v2")
assert [v.label for v in versions.list_versions(o.id)] == ["made v2", "v1"]
# diverge the live state WITHOUT capturing, so restore must back it up.
o.files = {"index.html": "<h1>v3 uncaptured</h1>"}
workspace_io._save(o)
restored = versions.restore(o.id, v1.id)
assert restored.files["index.html"] == "<h1>v1</h1>"
after = versions.list_versions(o.id)
pre = [v for v in after if v.source == "pre_restore"]
assert len(pre) == 1
meta = versions._read_manifest(o.id, pre[0].id)
assert meta["app_meta"]["files"]["index.html"] == "<h1>v3 uncaptured</h1>"
def test_dedupe_unchanged_state(stores):
o = _flat_app()
v1 = versions.capture(o.id, label="a")
v2 = versions.capture(o.id, label="b") # nothing changed since v1
assert v1.id == v2.id
assert len(versions.list_versions(o.id)) == 1
def test_restore_no_redundant_backup_when_current_already_saved(stores):
o = _flat_app("<h1>v1</h1>")
versions.capture(o.id, label="v1")
o.files = {"index.html": "<h1>v2</h1>"}
workspace_io._save(o)
v2 = versions.capture(o.id, label="v2") # live state == latest version
versions.restore(o.id, v2.id)
# current already equalled the latest version, so no pre_restore junk.
assert not any(v.source == "pre_restore" for v in versions.list_versions(o.id))
def test_webapp_restore_preserves_env_and_removes_added_files(stores):
o, folder = _webapp(stores, {"index.html": "<h1>v1</h1>", "src/app.js": "console.log(1)"})
with open(os.path.join(folder, ".env"), "w") as f:
f.write("SECRET=keepme\nFRONTEND_PORT=51000\n")
v1 = versions.capture(o.id, label="v1")
with open(os.path.join(folder, "index.html"), "w") as f:
f.write("<h1>v2</h1>")
with open(os.path.join(folder, "added.txt"), "w") as f:
f.write("added later")
versions.capture(o.id, label="v2")
versions.restore(o.id, v1.id)
with open(os.path.join(folder, "index.html")) as f:
assert f.read() == "<h1>v1</h1>"
assert not os.path.exists(os.path.join(folder, "added.txt"))
with open(os.path.join(folder, ".env")) as f:
assert "SECRET=keepme" in f.read() # .env never snapshotted, never wiped
def test_branch_makes_independent_app(stores):
o, _ = _webapp(stores, {"index.html": "<h1>v1</h1>"})
v1 = versions.capture(o.id, label="v1")
new_id = versions.branch(o.id, v1.id)
assert new_id and new_id != o.id
new_o = workspace_io.load_output(new_id)
assert new_o is not None
assert new_o.workspace_id and new_o.workspace_id != o.workspace_id
assert new_o.name == "Web (copy)"
assert new_o.session_id is None
new_folder = os.path.join(str(stores), new_o.workspace_id)
with open(os.path.join(new_folder, "index.html")) as f:
assert f.read() == "<h1>v1</h1>"
def test_delete_all_removes_history(stores):
o = _flat_app()
versions.capture(o.id, label="v1")
assert versions.list_versions(o.id)
versions.delete_all(o.id)
assert versions.list_versions(o.id) == []
def test_restore_is_undoable(stores):
"""Invariant: after any restore, the state you were on is recoverable (the
pre_restore backup), so a wrong restore is never a dead end."""
o = _flat_app("<h1>v1</h1>")
v1 = versions.capture(o.id, label="v1")
o.files = {"index.html": "<h1>v2</h1>"}
workspace_io._save(o)
versions.capture(o.id, label="v2")
o.files = {"index.html": "<h1>v3 live</h1>"} # uncaptured live edit
workspace_io._save(o)
versions.restore(o.id, v1.id)
assert workspace_io.load_output(o.id).files["index.html"] == "<h1>v1</h1>"
backup = next(v for v in versions.list_versions(o.id) if v.source == "pre_restore")
versions.restore(o.id, backup.id)
assert workspace_io.load_output(o.id).files["index.html"] == "<h1>v3 live</h1>"
def test_branch_does_not_mutate_source(stores):
"""Invariant: branching, then editing the copy, never touches the original."""
o, folder = _webapp(stores, {"index.html": "<h1>source</h1>"})
v1 = versions.capture(o.id, label="v1")
new_id = versions.branch(o.id, v1.id)
new_o = workspace_io.load_output(new_id)
new_folder = os.path.join(str(stores), new_o.workspace_id)
with open(os.path.join(new_folder, "index.html"), "w") as f:
f.write("<h1>changed copy</h1>")
with open(os.path.join(folder, "index.html")) as f:
assert f.read() == "<h1>source</h1>"
assert workspace_io.load_output(o.id).name == "Web"
def test_restore_clears_empty_schema(stores):
"""Regression: an empty {} input_schema must actually restore as empty, not
fall back to the current one (the falsy-dict bug)."""
o = Output(name="S", files={"index.html": "x"}, input_schema={})
workspace_io._save(o)
v1 = versions.capture(o.id, label="empty schema")
o.input_schema = {"type": "object", "properties": {"a": {}}, "required": []}
workspace_io._save(o)
versions.capture(o.id, label="full schema")
versions.restore(o.id, v1.id)
assert workspace_io.load_output(o.id).input_schema == {}
def test_unchanged_files_are_not_duplicated(stores):
"""The whole efficiency point: changing 1 of 10 files across two versions
adds 1 blob, not 10. Storage is O(unique content), not O(files x versions)."""
o, folder = _webapp(stores, {f"f{i}.txt": f"content {i}" for i in range(10)})
versions.capture(o.id, label="v1")
with open(os.path.join(folder, "f0.txt"), "w") as f:
f.write("changed")
versions.capture(o.id, label="v2")
blobs = os.listdir(versions._blobs_dir(o.id))
assert len(blobs) == 11 # 10 originals shared + 1 changed; NOT 20
def test_capture_missing_app_returns_none(stores):
assert versions.capture("does-not-exist") is None
assert versions.restore("nope", "nope") is None
assert versions.branch("nope", "nope") is None
assert versions.list_versions("nope") == []
debugger/debug.py
+9
View File
@@ -7,6 +7,15 @@ from debugger_backend.color_adjuster import rgb_to_ansi, bold_and_italicize_text
from debugger_backend.debug_arg_parser import is_text, is_error
def debug(*args, mode:str='debug', override_max_chars:bool=False):
# Packaged/prod no-op: this frame-aware debugger is a dev tool, and its first
# call instantiates Debugleton() -> a recursive os.scandir project scan that
# runs synchronously on the backend's startup path. On a cold launch (uncached
# filesystem) that scan cost ~17s of the backend-http-ready time; warm it is
# ~80ms. Skipping it in the packaged build removes the cold cost entirely. Dev
# (OPENSWARM_PACKAGED unset) keeps the full debugger. Safe: debug() returns
# None and every caller ignores the return value.
if os.environ.get("OPENSWARM_PACKAGED") == "1":
return
frame = inspect.currentframe().f_back
code = frame.f_code
line_no = frame.f_lineno
docs/perf/winv2/README.md
+394
View File
@@ -0,0 +1,394 @@
# winv2: Windows startup + App Builder speed and bug fixes
Branch: `eric/winv2`. Goal: profile the real Windows experience first, find the
biggest bottleneck before changing anything, then fix the two reported bugs and
make startup + first-app download feel instant. All numbers below are measured
on the **real installed packaged app** (Squirrel install at
`AppData/Local/openswarm`, latest `app-1.2.82`), Windows 11, not dev mode.
Notion tracking (Todos DB):
- [Perf] Windows startup + download speed: backend cold-start is the bottleneck
- [App Builder] Windows preview broken: no bundled bash/npm + missing node_modules archive
- [Bug] Skills list empty until reboots + onboarding "Install a skill" step times out
- [Reliability] Distributed-systems hardening (design)
## How these numbers were measured
Source of truth: the packaged app's own perf markers in
`AppData/Roaming/openswarm/data/backend.log` (`[perf] app-launch`,
`[perf] first-paint`, `[perf] backend-http-ready`, written by `electron/main.js`).
These are wall-clock ms from process start, i.e. exactly what the user feels.
Raw extract: `baseline_startup.csv`. Re-run with `profile_startup.sh`.
Import cost measured with the bundled interpreter:
`python-env/python.exe -X importtime -c "import backend.main"`.
## Baseline (BEFORE any change)
### Startup, per launch (ms)
| metric | warm (typical) | cold (first run after each update) |
| --- | --- | --- |
| app-launch (electron ready) | 107-400 | 107-563 |
| first-paint (renderer) | 338-1205 | ~1200 |
| **backend-http-ready** | **8700-10500** | **54600 / 81000 / 86300 / 133000 / 138300** |
Electron shell paints in well under 1.5s every time. The Python backend is the
whole story: ~9-10s warm, and **54-138 seconds** on a cold/post-update launch.
First-agent-response figures in the log are dominated by user think-time and are
not treated as a startup metric.
### Why the backend is slow (evidence)
| factor | measurement | effect |
| --- | --- | --- |
| python-env file count | 13,554 files (4,510 .py/.pyd/.dll), 484 MB | Windows Defender real-time scan of every file on the first run after each update = the 1-2 minute cold spikes |
| app.asar size | 639 MB | cold disk read on first launch |
| backend.main import tree | ~2.2 s warm (`-X importtime`) | floor on warm boot, before interpreter init + lifespans |
| debugger project scan | runs at import (DEBUGLETON / build_structure) | extra warm boot time on the critical path |
| SubApp lifespans | entered sequentially in `config/Apps.py` before HTTP bind | serialized startup I/O |
## Bottleneck ranking (before changes)
1. **Python backend cold-start (dominant).** 9-10s warm, 54-138s cold. ~95% of
perceived startup. Cold case driven by Defender scanning 13.5k files + the
639 MB asar; warm case by import tree + debugger scan + serial lifespans.
2. **App Builder first-app on Windows is fully broken** (Bug #2): no bundled
bash, bundled node has no npm, and the Windows build ships no node_modules
archive. Confirmed against the installed binary. Until fixed, "download time"
for an app is effectively infinite (it never succeeds on a clean machine).
3. **Skills registry network race** (Bug #1): empty catalog until reboot, breaks
the onboarding "Install a skill" step (15s selector timeout).
## Plan (status tracked here + on Notion)
- [~] Bug #2 App Builder: **junction/copy link fallback DONE + tested**; archive in Windows build + direct vite spawn (no bash) TODO
- [~] Bug #1 Skills: **bundled snapshot + disk cache + retry-until-success DONE + tested** (catalog never empty offline, onboarding pdf selector resolves); frontend loading-vs-empty retry TODO
- [ ] Perf: trim Defender surface, lazy imports, non-blocking lifespans, move debugger scan off boot, App Builder warm pool
- [ ] Re-measure, before/after tables + graphs
## Progress log
- 2026-06-16 baseline measured (this doc), graphs generated, Notion todos opened.
- 2026-06-16 Bug #1 backend: `skill_registry.py` now seeds from bundled `skills_snapshot.json` + on-disk last-good cache and retries until first success. Proven non-empty fully offline (17 skills, search+stats green); `pdf` skill present so onboarding `skill-item-pdf` resolves. Regression test `backend/tests/test_skill_registry_seed.py` (3 cases green).
- 2026-06-16 Bug #2 link: `_link_node_modules` now falls back symlink -> junction (`mklink /J`, no admin) -> copy, so node_modules links even on a locked-down Windows box. Tested with forced symlink failure.
## Results (AFTER)
### The warm-startup bottleneck was found and fixed
Per-SubApp-lifespan profiling (`profile_boot.py`) showed the entire ~8s gap was
**one lifespan**:
| boot phase | before | after | note |
| --- | --- | --- | --- |
| import backend.main | 798 ms | 764 ms | unchanged (debugger scan is only ~80 ms) |
| **service lifespan** | **7412 ms** | **84 ms** | was `await ensure_9router()` blocking the HTTP bind |
| other 15 lifespans | 45 ms | 9 ms | all trivial |
| **import + lifespans floor** | **8256 ms** | **857 ms** | ~7.4 s removed (~90%) |
Fix: `service.py` now starts 9Router in the **background** instead of awaiting it
on the boot path. 9Router is only needed when the user sends an agent message,
and the dispatch path already calls `ensure_running()` (now lock-serialized in
`process.py` so the background start and a dispatch-time ensure can't
double-spawn). Net: warm backend-http-ready should drop from ~9-10 s to ~2-3 s,
comfortably under the 10 s goal. See `boot_breakdown.svg`.
### Still open (cold start)
The 54-138 s cold spikes are Windows Defender scanning the 13,554-file / 484 MB
python-env on the first run after each update, plus cold-reading the 639 MB
asar. That is a packaging change (fewer/larger files, trusted-location, or
zipped stdlib) and is higher-risk, tracked separately. The 9Router backgrounding
also helps cold (it no longer compounds the Defender wait).
### App Builder first-app "download" + create path (measured)
Per-phase, measured on this Windows box (`measure_appbuilder.py` + `measure_vite.py`),
isolated temp dirs, real warm caches. See `appbuilder_breakdown.svg`.
| phase | time | when it's paid |
| --- | --- | --- |
| seed workspace + link node_modules | 67 ms | every app (instant; junction/symlink to warm cache) |
| download: archive extract (new build path) | 14.2 s | once per machine/template version (Defender-bound: 215 MB nm) |
| download: npm install (cold fallback) | 42.7 s | once, only if no archive ships |
| vite bind: cold vite cache | 6.7 s | first app ever (esbuild pre-bundle) |
| vite bind: warm shared cache | 0.7 s | every subsequent app |
| build-time: tar nm -> archive | 6.8 s | on CI, never on the user's machine |
**User-facing scenarios (create app -> live preview):**
| scenario | total | notes |
| --- | --- | --- |
| first app, clean Windows, BEFORE fix | never works | `[WinError 2]` / "backend exited with code 1" (no bash/npm/archive) |
| first app, AFTER fix (tar archive) | ~21 s one-time | extract 14.2 + seed 0.07 + vite cold 6.7; and it actually works |
| **first app, AFTER fix (.tar.gz + background extract at startup)** | **~7 s typical / ~21 s worst case** | extract runs in the background at startup; if done before first create -> seed 0.07 + vite cold 6.7 ~= 7s, else +14.2s |
| first app, if we shipped npm instead | ~49 s | 42.7 + 6.7; the archive saves ~28 s and needs no npm |
| every subsequent app | ~0.8 s | seed 0.07 + vite warm 0.7 (near-instant) |
#9 item 2 (DONE): the Windows build now ships node_modules ALREADY EXTRACTED in
resources (digest-tagged); `_ensure_warm_cache` junctions a workspace straight at
it (`_bundled_extracted_modules`), so there is no tar-extract on first app -- the
14.2 s Defender-scanned write cost moves to install time, once. Verified by
`backend/tests/test_bundled_extracted_modules.py` (selection + Mac fallback) and
the build step `build-app-win.ps1` 4b now robocopies the tree into resources.
Takeaways: the archive (Bug #2 fix) turns a broken/∞ first-app into a working
~21s one-time, and ~0.8s for every app after. The remaining ~14s extract is the
SAME Defender-on-many-small-files cost as cold app-startup (Task #9) -- the one
lever that would shrink both.
### Task #10 — VERIFIED on the real code-signed build (v1.3.86)
Downloaded the signed draft-release installer, verified signature, installed, and
measured on this Windows 11 box. All numbers are from the packaged app, not dev.
(Shipped as v1.3.86 on the fixed code; the earlier v1.3.87 build was identical
bits and is abandoned. Numbers below are the v1.3.86 run; v1.3.87 matched.)
| metric | baseline (1.2.x) | signed v1.3.86 | result |
| --- | --- | --- | --- |
| installer download | n/a | 371.5 MB @ 27.1 MB/s (13.7 s) | signed: Authenticode **Valid** (CN=Eric Zeng) |
| install time | n/a | ~6.3 s | Squirrel |
| **cold backend-http-ready** | **54-138 s** | **22.6 s** | **~75-84% faster** |
| **warm backend-http-ready** | **9-10 s** | **5.0 s** | **~50% faster, under the 10s goal** |
| app.asar size | ~607 MB | **2.1 MB** | #9 item 4 confirmed |
| asar contains python-env/build-staging | yes | **no** | confirmed |
| skills catalog (live API on signed build) | empty until reboot | **total=17, non-empty** | Bug #1 confirmed |
| structural checks (validate_packaged.ps1) | 4 fail | **5/5 PASS** | snapshot + node tar + unpacked python-env |
Cold is 22.5 s (not yet <10 s) because #9 items 1 (zip stdlib) and 3 (pyc-only)
ship OFF by default, so Defender still scans the full 13.5k-file python-env on the
first post-update launch. Enabling those (next, build-gated) is the remaining cold
lever. Bug #2 (App Builder) is verified structurally (node_modules .tar.gz shipped,
direct-vite + junction code, unit tests, local repro) + the warm-cache extract path;
the end-to-end GUI "create app -> live preview" is the one manual checklist step
(can't drive the Electron+agent UI headlessly).
## [DISPROVEN 2026-06-17] hypothesis: residual ~17s cold = swarm-debug DEBUGLETON scan
> UPDATE: this hypothesis was WRONG. The `debug()` -> `OPENSWARM_PACKAGED=1` no-op
> shipped (commit 3d6fe483) and was verified live on the signed build ("Scanning
> Project" count=0, scan confirmed gone), yet **cold backend-http-ready stayed at
> 21.5s (no change)**. So the DEBUGLETON scan was NOT the cold driver. Kept the
> no-op anyway (it removes a real warm cost and is harmless), but the cold 16s is
> elsewhere. See the source-audit section below for what it actually is. The
> original (now-disproven) reasoning is preserved below for the record.
### original (disproven) reasoning
No-coding investigation (import profile + the app's own timestamped logs) pinned it:
- The cold launch has a ~17s SILENT, synchronous event-loop block during startup
(no async task ran). NOT import (1.1s), NOT Defender (proven twice), NOT
file-count, NOT network (the updater succeeded in the window), and every SubApp
lifespan is verified trivial (mkdir / early-return / yield).
- It is the swarm-debug DEBUGLETON: debug() -> Debugleton().find_file_info()
(debug.py:20); the first call instantiates the singleton -> update_debug_toggles()
-> Directory.build_structure() -> a recursive os.scandir() walk of the project
tree (Directory.py:74). debug() is called on the startup critical path
(config/Apps.py SubApp init + the lifespan loop), so the scan runs SYNCHRONOUSLY
and blocks the HTTP bind. Cold (uncached fs) = ~17s; warm (cached) = ~80ms. The
DEBUGLETON INIT log lines land exactly in the 17s gap.
- This also explains why items 1+3 (file count) and item 5 (Defender) did nothing:
the cost is a synchronous scandir tree-walk, not AV scanning or bytecode.
SAFE FIX (proposed): make debug() a no-op when OPENSWARM_PACKAGED=1 (early-return
before Debugleton() instantiates), so the scan never runs in the packaged build.
Dev keeps the debugger. Risk very low (debug() is non-critical logging that already
swallows errors). Expected cold ~22s -> ~5s (under the 10s goal). Confirm with a
cold rebuild+measure.
## #9 item 5 (Defender exclusion) measured: ALSO no cold benefit -> cold is NOT Defender
Applied the Defender exclusion (admin) for all 3 openswarm folders, rebuilt a
fresh-content lean v1.3.86 (so Defender would see new files), installed with the
exclusion active, measured cold:
| | cold backend-http-ready |
| --- | --- |
| no exclusion (items off) | 22.5 s |
| no exclusion (items 1+3 on) | 22.4 s |
| **Defender exclusion ON** | **21.4 s (no change)** |
Conclusion: TWO independent Defender-targeting interventions (file-count via
items 1+3, and a full AV exclusion) both moved cold by ~0. So the residual ~22 s
cold is NOT Defender real-time scanning. It is the first-launch-after-install cost
-- cold disk I/O of the imported native binaries + bundled-Python interpreter init
+ Squirrel first-run -- which neither AV-exclusion nor file-count tricks touch.
(Caveat: my non-admin shell can't read Get-MpPreference to re-confirm the
exclusion is live, but the result is consistent with the items-1+3 negative.)
ACTION: remove the exclusion -- it weakened AV for zero gain:
`& scripts\add-defender-exclusion.ps1 -Remove` (elevated).
The cold win was already banked by the asar trim (54-138 s -> ~22 s). Pushing
cold below ~22 s would need shrinking the startup-imported bytes (lazy-load heavy
native deps like lxml/PIL, or trim the 242 MB bundled claude.exe) or faster disk
-- bigger/riskier work with diminishing returns. Warm (5 s) is already under goal.
## #9 items 1+3 measured on the signed build: NO cold-start benefit (negative result)
Built v1.3.86 with items 1+3 ON (python-env 13,554 -> 9,285 files, ~31% fewer) and
measured the signed install:
| metric | items OFF (1.3.86) | items ON (1.3.86) |
| --- | --- | --- |
| cold backend-http-ready | 22.5 s | **22.4 s (no change)** |
| warm backend-http-ready | 5.0 s | 5.2 s (noise) |
| installer | 372 MB | 365 MB (~7 MB smaller) |
**The hypothesis was wrong.** Cutting the file COUNT 31% did nothing for cold,
because cold is dominated by Defender scanning the large NATIVE binaries imported
/ present at boot, not the many small .py files. The biggest are
`claude.exe` (242 MB!), `_rust.pyd` (9.4), `_avif...pyd` (7.5), `python313.dll`
(5.8), `libcrypto-3-x64.dll` (5.7), `mfc140u.dll` (5.4) -- none of which items 1+3
touch (zip/pyc only affect pure-python). So items 1+3 are a wash for cold (a tiny
installer-size win + import-clean, but not the goal).
The real remaining cold levers are byte/native-bound, not file-count:
- **#9 item 5 (Defender exclusion, opt-in)** -- the only thing that removes the
native-binary scan entirely; would bring cold toward the ~5 s warm number.
- Trim/lazy the heavy native deps (e.g., the 242 MB bundled claude.exe, PIL/lxml)
-- larger, riskier code/packaging work.
- Or accept cold 22.5 s: already 75-84% below the 54-138 s baseline, and warm 5 s
is already under the 10 s goal.
Recommendation: items 1+3 don't earn their build-time/complexity for cold; keep
them only for the marginal installer-size win, or revert step 2b to keep the build
lean. The meaningful cold work is item 5.
## Net time decreased per step (measured)
| step | before | after | saved |
| --- | --- | --- | --- |
| backend boot: service lifespan | 7412 ms | 84 ms | -7328 ms (-99%) |
| backend boot: import + all lifespans floor | 8256 ms | 857 ms | -7399 ms (-90%) |
| backend-http-ready warm (end-to-end) | ~9-10 s | ~2-3 s (projected) | ~-7 s |
| App Builder dependency download | 42.7 s npm | 14.2 s archive | -28.5 s (-67%) |
| App Builder first app -> preview | broken/never | ~21 s working | inf -> 21 s |
| App Builder subsequent app -> preview | n/a | ~0.8 s | near-instant |
| skills catalog availability | empty until reboot(s) | instant (seeded) | bug eliminated |
## #9 packaging approach: shrink the Defender file surface (build-gated)
Defender real-time-scans every small file: python-env = 13,554 files; node_modules
= ~tens of thousands; app.asar = 639 MB. It rescans python-env on the first launch
after each update (54-138 s cold spikes) and scans node_modules as it is written
(the 14.2 s extract). Fix family: fewer/larger files, scan-once-at-install instead
of per-launch / per-first-app. Each item is independent, reversible, and must be
validated on a real packaged EXE (Task #10).
1. [ENABLED + validated] Zip the Python stdlib -> python313.zip. scripts/zip-python-stdlib.ps1, now wired into build-app-win.ps1 step 2b. VALIDATED 2026-06-17: applied to a copy of the real shipped python-env and `import backend.main` (full app + deps) imported cleanly; combined with #3 the python-env drops 13,554 -> 9,287 files (~31%). Draft notes: Measured on the real env: 910 stdlib .py/.pyc files (15.1 MB) collapse into one zip. CPython auto-adds <prefix>/python313.zip to sys.path, so no python._pth is needed; site-packages + DLLs (native .pyd) stay loose; a keep-list keeps data-file stdlib dirs (lib2to3, idlelib, tkinter, ...) loose. Impact: ~7% of total python-env file count, but it collapses the stdlib import-time file-opens (the cold-launch Defender scan storm) into a single scanned file; bigger combined with #3. Validation (Task #10): -Apply on a copy, then import backend.main, importtime parity, boot the packaged backend, measure cold backend-http-ready vs baseline. Wire into build-app-win.ps1 behind an off-by-default -ZipStdlib switch only after it passes.
2. [DONE] Ship the webapp_template node_modules archive in the Windows build (build-app-win.ps1 step 4b builds node_modules.<digest>.tar.gz, mirroring the Mac build). Runtime _try_extract_bundled_archive unpacks it into the warm cache, kicked off in the BACKGROUND by warm_cache_in_background at startup so it is off the first-app create path. CORRECTION 2026-06-17: an earlier draft shipped node_modules PRE-EXTRACTED in resources (~30k files) -- that blew the Windows build past 50 min (Squirrel LZMA on tens of thousands of tiny files) and bloated the installer, so it was reverted to the single .tar.gz. The runtime keeps _bundled_extracted_modules() as a harmless preference (returns None when no tree is shipped -> falls back to the tar). Tests: test_bundled_extracted_modules.py still valid (selection + fallback).
3. [ENABLED + validated] Ship site-packages as sourceless .pyc only (drop .py). scripts/strip-py-to-pyc.ps1, now wired into build-app-win.ps1 step 2b. VALIDATED 2026-06-17 alongside #1 (backend.main imports clean from a transformed copy; 3,352 .py removed). Draft notes: Measured: 3,352 .py (26.9 MB) + 362 __pycache__ dirs strippable from site-packages (keep-list excludes pip/setuptools). compileall -b writes legacy module.pyc next to source; we delete the .py whose .pyc exists and drop __pycache__. Sourceless import proven with the bundled 3.13 interpreter. Scope: site-packages ONLY (NOT backend app code -- the swarm-debug debugger reads our own source for frame annotation). .pyc magic must match the shipped interpreter, so compile with the bundled python. Validate on a packaged EXE (Task #10); some packages use inspect.getsource and may need the keep-list. Combined with #1 + #2 this takes python-env from ~13,554 files toward ~9,300 (~31% fewer for Defender).
4. [APPLIED, build-gated] Trim app.asar. Inventory (docs/perf/winv2/inspect_asar.js) found the 607 MB asar is almost entirely DUPLICATION: python-env (408 MB, incl. a 242 MB bundled claude.exe) and build-staging (197 MB: node.exe 67 MB, uv.exe 65 MB, mcp-bundles, frontend) are packed into the asar AND already shipped UNPACKED in resources/ via extraResources. The runtime reads from resources/ (confirmed: "Starting backend: ...resources\python-env\python.exe"), never from inside the asar. Source maps were a red herring (0.4 MB). Fix: added a build.files exclusion in electron/package.json ("!python-env/**", "!build-staging/**") so those trees no longer pack into the asar -> ~607 MB -> ~2 MB (just main.js/preload/node_modules). Removes the entire 639 MB cold-read on first launch. Validate on a packaged EXE (Task #10): app still boots (python/node/router resolved from resources), asar size shrunk.
5. [DRAFTED, opt-in] Defender exclusion for OpenSwarm's dirs -- the nuclear cold-start fix (stops real-time scanning entirely, so it kills BOTH the 54-138s post-update launch and the ~14s extract). Draft: scripts/add-defender-exclusion.ps1 (dry-run by default; -Apply/-Remove need admin; -Status lists). Excludes %LOCALAPPDATA%\openswarm, %APPDATA%\openswarm, ~/.openswarm (verified the paths resolve). SECURITY: reduces AV coverage of those folders, so it must ALWAYS be an explicit user choice -- never auto-run, never a startup prompt. Proposed surface: an OFF-by-default Settings > Advanced toggle ("Faster Windows startup -- adds a Defender exclusion for OpenSwarm; one-time admin approval; reversible"), which on enable spawns an elevated `powershell Start-Process -Verb RunAs` to run the script -Apply (UAC), and -Remove on disable. This is a passive opt-in toggle, NOT a banner/tip/prompt, so it respects the no-user-action-UI rule. Not wired into the frontend yet (design only).
Recommended order: #2 (biggest UX win, lowest risk), then #1 (largest cold win, careful import testing), then #3/#4. Validation: re-run profile_startup.sh + a fresh-extract timing on the packaged EXE after each change, diff vs baseline_startup.csv.
### Bug fixes (this branch)
- Bug #1 skills: seed from bundled snapshot + disk cache + retry-until-success. Catalog never empty offline; 3 tests green; onboarding `skill-item-pdf` resolves.
- Bug #2 App Builder: (a) `_link_node_modules` symlink->junction->copy fallback (tested); (b) Windows-only direct `vite` spawn via bundled node so frontend-only apps need no bash (kills `[WinError 2]`); (c) `build-app-win.ps1` now pre-builds the node_modules archive natively. Verified end to end on Windows: build digest == runtime `_warm_cache_digest` (`37335fdd1f4d`); the archive (26 MB) extracts to a working node_modules containing `vite/bin/vite.js` and the Windows-native `@esbuild/win32-x64/esbuild.exe`.
## Residual cold ~16s: full source audit + boot instrumentation (2026-06-17)
After FOUR disproven cold hypotheses (file-count via items 1+3, Defender exclusion,
DEBUGLETON scan, and the asar trim which DID bank 138s->22s), I stopped guessing and
read the real signed-build cold log line by line, then audited every lifespan in source.
What the cold log (commit 3d6fe483, scan-free build) actually shows:
```
03:03:02 skill_registry: seeded 17 skills <- last backend log before the gap
... 16 seconds, NO backend log line ...
03:03:18 nine_router: Starting 9Router <- a backgrounded create_task finally runs
03:03:18 Application startup complete <- uvicorn; all lifespans entered
03:03:21 9Router started; GET /api/health 200; backend-http-ready t=21519
```
SubApp lifespan order (`backend/main.py:52`):
`health, agents, skills, tools_lib, modes, settings, mcp_registry, skill_registry,
outputs, dashboards, swarm, service, subscription, auth, web, anthropic_proxy`.
Source-audited EVERY one of the 16 lifespan bodies + the service client:
- outputs: two `os.makedirs` + yield (trivial)
- dashboards: `_migrate_if_needed()` early-returns when dashboards exist (they persist
across reinstall in AppData, so it's a no-op on the cold post-update launch)
- swarm: `_gc_staging()` over an empty dict + yield (trivial)
- service: builds a provider list + `svc.sync()` x2, then `create_task(ensure_9router)`,
`create_task(_pulse_loop)`, `create_task(_drain_loop)`, yield. `svc.sync()` is genuinely
fire-and-forget: `client.py:sync()` -> `_schedule()` -> `loop.create_task(_post_or_spool)`;
the actual httpx POST has a 5s timeout and runs in the task, never on the boot path.
- skill_registry: seed from disk + `create_task(_refresh_loop)` + yield (trivial)
- subscription / auth / anthropic_proxy: bare `yield`
- web: `debug("START")` (no-op packaged) + yield
So NO lifespan body blocks. This matches `profile_boot.py` warm (import + all 16
lifespans = 617ms, no lifespan over 95ms). The 16s is therefore NOT in our Python
startup logic; it is cold first-run demand-paging of native bytes (interpreter .pyc
cold reads, native .pyd / .dll first-touch, the 9Router/claude.exe binaries) that
the OS pages in during this window. That class of cost is exactly what the asar
trim already cut and what Defender-exclusion / file-count provably cannot move.
THE missing instrument: `debug(sub_app.name)` (Apps.py:33) is a no-op in the
packaged build, so the packaged log had ZERO per-lifespan markers, which is why
four hypotheses were guesses. Added permanent per-lifespan boot timing in
`backend/config/Apps.py` (one `time.perf_counter()` + flushed `print` per app, plus
a `lifespans-total`): `[perf] lifespan <name> t=<ms>ms`. Logging only, zero
functional risk; validated warm (correctly attributes a simulated 300ms blocker to
the one slow lifespan, others 0ms). On the NEXT cold packaged launch this pins the
16s to a single lifespan (=> a real fix) or shows it smeared across many (=> confirms
distributed cold I/O => accept 22s; warm 5s already meets the <10s goal).
Status: warm 5.0s (under goal), cold ~22s (75-84% below the 54-138s baseline), both
bugs fixed/verified on the signed build. The cold residual is either accepted as
first-run-only OS I/O, or pinned definitively by one more build that ships this
instrumentation. Build-gated (user manages tags/release), so not auto-built.
## [SOLVED 2026-06-18] cold ~22s -> 3.86s: synchronous is_running() froze the event loop
The per-lifespan instrumentation (v1.3.88) overturned every prior hypothesis: all
16 lifespans enter in ~120ms even COLD. The ~18s cold cost was entirely AFTER
lifespan startup, in a backgrounded create_task that synchronously blocked the
single asyncio event loop, so uvicorn could not answer the health probe.
Finer instrumentation (v1.3.89) split it into two stalls (~13s before any bg task,
~5s in 9Router ensure). faulthandler (`dump_traceback_later`, v1.3.90) on the
signed cold build caught the loop thread frozen, three times, in the SAME call:
```
socket.create_connection <- stuck >7s
httpx ... get
backend/apps/nine_router/process.py:83 is_running() <- synchronous httpx.get
<- sync_openswarm_pro_as_claude / sync_custom_providers (settings._boot_router_then_sync)
<- _ensure_running_impl (ensure_running)
```
ROOT CAUSE: `is_running()` did a synchronous `httpx.get("http://localhost:20128/...")`.
It is called ~5x on the cold boot path (the settings key-sync sequence + the
9Router ensure) BEFORE 9Router is up. On Windows a dead-port connect to
"localhost" stalls ~7s each: getaddrinfo returns `::1` first, and the loopback
refusal is slow (measured: a refused connect is ~2s/address, and localhost =
`::1`+`127.0.0.1` = ~4s; cold ~7s). ~5 serial probes = the ~18s freeze.
This is why every earlier hypothesis missed: it is not disk, not Defender, not
file-count, not the DEBUGLETON scan, not imports, not the lifespans. It is one
synchronous network probe on the event loop, repeated.
FIX (v1.3.91, `process.py` is_running): probe `127.0.0.1` with a 0.3s TCP timeout
first (a short timeout caps the slow Windows refusal: measured 306ms vs ~7s); only
HTTP-confirm when the port is open. 9Router binds `0.0.0.0` (the warm app reaches
it via `127.0.0.1`), so reachability is unchanged, only the dead-port wait dies.
VERIFIED on the real signed build (this Windows 11 box, fresh Squirrel install):
| metric | baseline | before fix (1.3.90) | after fix (1.3.91) |
| --- | --- | --- | --- |
| cold backend-http-ready | 54-138s | 23.5s | **3.86s** |
| warm backend-http-ready | 9-10s | 5.0s | **3.32s** |
Cold is now ~97% below baseline and well under the 10s goal; warm improved too
(the same localhost stall taxed it). 9Router still starts successfully via the new
probe (no regression). The diagnostic `[perf] bg` logs + faulthandler were removed
after diagnosis; the lightweight per-lifespan timer stays (prints only a lifespan
over 50ms + the total) as a cheap regression tripwire.
docs/perf/winv2/TASK10_CHECKLIST.md
+55
View File
@@ -0,0 +1,55 @@
# Task #10 — validate the winv2 changes on the REAL signed Windows build
Do not call winv2 "done" until a **code-signed, downloaded, installed, launched**
build passes this. Unit tests + dry-runs are necessary but not sufficient — the
build-script changes (asar exclusion, node_modules pre-extract) and the cold-start
wins only exist in a packaged EXE.
## 0. Produce the signed build
- Tag the winv2 HEAD and push: `git tag v1.3.86 && git push origin v1.3.86`.
- This runs `.github/workflows/release-windows.yml` (Azure code-signing) and creates
a **draft** GitHub release (drafts do NOT auto-update existing users).
- Watch it: `gh run watch` / `gh run list --workflow=release-windows.yml`.
- If `build-app-win.ps1` errors on the new step 4b (pre-extract) or the asar
`files` exclusion, fix and re-tag (delete the draft + tag first; never force-push
an existing release tag).
## 1. Download + verify the signature (must be real signed bits)
- `gh release download v1.3.86 --pattern "*Setup*.exe" --dir .` (or from the draft release page).
- Verify Authenticode: `Get-AuthenticodeSignature .\OpenSwarm-Setup-x64.exe` → Status must be `Valid`, signer = the Azure Trusted Signing cert. NOT "NotSigned"/"UnknownError".
## 2. Install + first (COLD) launch — the headline metric
- Install the downloaded EXE (Squirrel → `%LOCALAPPDATA%\openswarm`).
- Launch once and let it fully load. This is the COLD launch (Defender scans fresh files).
- Then run the automated checker: `pwsh docs/perf/winv2/validate_packaged.ps1`.
- Acceptance (perf): cold `backend-http-ready` should be **far below the 54-138s baseline**
(target: well under the 10s goal even cold, given the 639MB asar read is gone +
fewer files to scan). Relaunch once for the warm number (target ~2-3s).
## 3. Automated structural checks (validate_packaged.ps1 must be all PASS)
- app.asar < 50 MB (was ~607 MB) — #9 item 4.
- app.asar does NOT contain python-env / build-staging — #9 item 4.
- `resources/python-env/python.exe` present (still shipped unpacked).
- `resources/node/x64/node.exe` present.
- `resources/backend/apps/skill_registry/skills_snapshot.json` present — Bug #1.
- webapp_template_cache has a pre-extracted `<digest>/node_modules/vite/bin/vite.js`
(#9 item 2) — or a `.tar.gz` fallback.
## 4. Manual GUI checks (can't be automated)
- **Skills (Bug #1):** open the Skills page on a fresh launch → the catalog shows
immediately (NOT empty). Run onboarding step 6/8 "Install a skill" → it finds the
pdf skill (no `waitForSelector "skill-item-pdf" 15000ms` timeout).
- **App Builder (Bug #2):** create an app → the preview goes LIVE. No `[WinError 2]`,
no "backend exited with code 1". First app should be quick (pre-extracted nm + vite).
Bonus: test on a machine WITHOUT Git Bash to confirm the no-bash vite path.
- Sanity: send an agent message (9Router now starts in the background → first message
may wait a moment for it; confirm it still answers).
## 5. Optional cold-start levers (only after 1-4 pass)
- #9 item 1 (`zip-python-stdlib.ps1 -Apply`) and item 3 (`strip-py-to-pyc.ps1 -Apply`)
on a build copy, then re-run 1-4 + re-measure. Enable in the build only if green.
- #9 item 5 (`add-defender-exclusion.ps1`) is a user opt-in, validate separately.
## 6. Sign-off
- All of 1-4 green on the signed build → publish the draft release (un-draft) to ship 1.3.86.
- Record the real cold/warm numbers in `boot_breakdown.csv` / README "Results (AFTER)".
docs/perf/winv2/appbuilder_breakdown.csv
+7
View File
@@ -0,0 +1,7 @@
phase,ms,note
"seed workspace + link node_modules (per app)",67,"nm linked, instant"
"download: archive extract (new build path, one-time)",14204,"215MB nm, defender-bound"
"download: npm install (cold fallback, one-time)",42684,"ok"
"vite bind: cold vite cache (first app)",6714,"bound"
"vite bind: warm shared cache (subsequent)",672,"bound"
"build-time: tar node_modules to archive (CI, not user)",6809,"26MB archive"
1 phase ms note
2 seed workspace + link node_modules (per app) 67 nm linked, instant
3 download: archive extract (new build path, one-time) 14204 215MB nm, defender-bound
4 download: npm install (cold fallback, one-time) 42684 ok
5 vite bind: cold vite cache (first app) 6714 bound
6 vite bind: warm shared cache (subsequent) 672 bound
7 build-time: tar node_modules to archive (CI, not user) 6809 26MB archive
docs/perf/winv2/appbuilder_breakdown.svg
+22
View File
@@ -0,0 +1,22 @@
<svg xmlns="http://www.w3.org/2000/svg" width="980" height="350" font-family="Segoe UI, sans-serif">
<text x="20" y="30" font-size="17" font-weight="600" fill="#1a1d27">App Builder "create app -> live preview" breakdown (ms)</text>
<text x="310" y="76" font-size="12" fill="#1a1d27" text-anchor="end">seed workspace + link node_modules (per app)</text>
<rect x="320" y="56" width="1.0" height="30" fill="#2e9e5b" rx="3"/>
<text x="329" y="76" font-size="12" fill="#1a1d27">67ms</text>
<text x="310" y="120" font-size="12" fill="#1a1d27" text-anchor="end">download: archive extract (new build path, one-time)</text>
<rect x="320" y="100" width="189.7" height="30" fill="#2e9e5b" rx="3"/>
<text x="518" y="120" font-size="12" fill="#1a1d27">14.20s</text>
<text x="310" y="164" font-size="12" fill="#1a1d27" text-anchor="end">download: npm install (cold fallback, one-time)</text>
<rect x="320" y="144" width="570.0" height="30" fill="#d64545" rx="3"/>
<text x="898" y="164" font-size="12" fill="#1a1d27">42.68s</text>
<text x="310" y="208" font-size="12" fill="#1a1d27" text-anchor="end">vite bind: cold vite cache (first app)</text>
<rect x="320" y="188" width="89.7" height="30" fill="#d64545" rx="3"/>
<text x="418" y="208" font-size="12" fill="#1a1d27">6.71s</text>
<text x="310" y="252" font-size="12" fill="#1a1d27" text-anchor="end">vite bind: warm shared cache (subsequent)</text>
<rect x="320" y="232" width="9.0" height="30" fill="#2e9e5b" rx="3"/>
<text x="337" y="252" font-size="12" fill="#1a1d27">672ms</text>
<text x="310" y="296" font-size="12" fill="#1a1d27" text-anchor="end">build-time: tar node_modules to archive (CI, not user)</text>
<rect x="320" y="276" width="90.9" height="30" fill="#2e9e5b" rx="3"/>
<text x="419" y="296" font-size="12" fill="#1a1d27">6.81s</text>
<text x="20" y="340" font-size="11" fill="#8892a4">green = warm/per-app cost; red = cold one-time download (npm with no archive)</text>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 1.9 KiB

docs/perf/winv2/baseline_phases.svg
+12
View File
@@ -0,0 +1,12 @@
<svg xmlns="http://www.w3.org/2000/svg" width="900" height="260" font-family="Segoe UI, sans-serif">
<text x="20" y="28" font-size="17" font-weight="600" fill="#1a1d27">where startup time goes (backend dwarfs the shell)</text>
<text x="20" y="77" font-size="13" fill="#1a1d27">typical warm launch</text>
<rect x="170" y="50" width="77.8" height="46" fill="#2e9e5b" rx="3"/>
<rect x="170" y="50" width="5.5" height="46" fill="#1a1d27" rx="3"/>
<text x="256" y="77" font-size="12" fill="#1a1d27">backend 9.6s (shell 0.68s)</text>
<text x="20" y="149" font-size="13" fill="#1a1d27">typical cold launch</text>
<rect x="170" y="122" width="700.0" height="46" fill="#d64545" rx="3"/>
<rect x="170" y="122" width="12.0" height="46" fill="#1a1d27" rx="3"/>
<text x="878" y="149" font-size="12" fill="#1a1d27">backend 86.3s (shell 1.48s)</text>
<text x="20" y="248" font-size="11" fill="#8892a4">dark = electron shell (app-launch + first-paint); colored = python backend</text>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 977 B

docs/perf/winv2/baseline_startup.csv
+15
View File
@@ -0,0 +1,15 @@
launch_ts,version,app_launch_ms,first_paint_ms,backend_http_ready_ms,class
2026-06-02T09:18:13Z,1.1.72,380,1097,54606,cold
2026-06-08T22:16:53Z,1.2.73,143,636,10463,warm
2026-06-08T23:03:47Z,1.2.73,317,625,10084,warm
2026-06-08T23:58:30Z,1.2.73,198,515,81041,cold
2026-06-09T03:13:05Z,1.2.73,147,559,10418,warm
2026-06-09T11:04:12Z,1.2.75,129,609,10041,warm
2026-06-09T11:53:18Z,1.2.75,108,338,8761,warm
2026-06-10T07:44:10Z,1.2.75,388,1100,86310,cold
2026-06-10T07:46:53Z,1.2.76,112,389,9342,warm
2026-06-10T23:32:57Z,1.2.76,563,1205,133070,cold
2026-06-10T23:35:13Z,1.2.77,114,391,9349,warm
2026-06-10T23:35:41Z,1.2.77,122,404,9303,warm
2026-06-14T00:07:35Z,1.2.77,159,1213,138335,cold
2026-06-14T00:09:58Z,1.2.82,107,702,9590,warm
1 launch_ts version app_launch_ms first_paint_ms backend_http_ready_ms class
2 2026-06-02T09:18:13Z 1.1.72 380 1097 54606 cold
3 2026-06-08T22:16:53Z 1.2.73 143 636 10463 warm
4 2026-06-08T23:03:47Z 1.2.73 317 625 10084 warm
5 2026-06-08T23:58:30Z 1.2.73 198 515 81041 cold
6 2026-06-09T03:13:05Z 1.2.73 147 559 10418 warm
7 2026-06-09T11:04:12Z 1.2.75 129 609 10041 warm
8 2026-06-09T11:53:18Z 1.2.75 108 338 8761 warm
9 2026-06-10T07:44:10Z 1.2.75 388 1100 86310 cold
10 2026-06-10T07:46:53Z 1.2.76 112 389 9342 warm
11 2026-06-10T23:32:57Z 1.2.76 563 1205 133070 cold
12 2026-06-10T23:35:13Z 1.2.77 114 391 9349 warm
13 2026-06-10T23:35:41Z 1.2.77 122 404 9303 warm
14 2026-06-14T00:07:35Z 1.2.77 159 1213 138335 cold
15 2026-06-14T00:09:58Z 1.2.82 107 702 9590 warm
docs/perf/winv2/baseline_startup.svg
+57
View File
@@ -0,0 +1,57 @@
<svg xmlns="http://www.w3.org/2000/svg" width="900" height="420" font-family="Segoe UI, sans-serif">
<text x="60" y="28" font-size="17" font-weight="600" fill="#1a1d27">backend-http-ready per launch (ms) - lower is better</text>
<line x1="60" y1="330" x2="880" y2="330" stroke="#e2e6ef"/>
<text x="52" y="334" font-size="11" fill="#8892a4" text-anchor="end">0s</text>
<line x1="60" y1="260" x2="880" y2="260" stroke="#e2e6ef"/>
<text x="52" y="264" font-size="11" fill="#8892a4" text-anchor="end">35s</text>
<line x1="60" y1="190" x2="880" y2="190" stroke="#e2e6ef"/>
<text x="52" y="194" font-size="11" fill="#8892a4" text-anchor="end">69s</text>
<line x1="60" y1="120" x2="880" y2="120" stroke="#e2e6ef"/>
<text x="52" y="124" font-size="11" fill="#8892a4" text-anchor="end">104s</text>
<line x1="60" y1="50" x2="880" y2="50" stroke="#e2e6ef"/>
<text x="52" y="54" font-size="11" fill="#8892a4" text-anchor="end">138s</text>
<rect x="68.8" y="219.5" width="41.0" height="110.5" fill="#d64545" rx="2"/>
<text x="89.3" y="214.5" font-size="10" fill="#1a1d27" text-anchor="middle">55s</text>
<text x="89.3" y="346" font-size="9" fill="#8892a4" text-anchor="end" transform="rotate(-40 89.3 346)">1.1.72</text>
<rect x="127.4" y="308.8" width="41.0" height="21.2" fill="#2e9e5b" rx="2"/>
<text x="147.9" y="303.8" font-size="10" fill="#1a1d27" text-anchor="middle">10s</text>
<text x="147.9" y="346" font-size="9" fill="#8892a4" text-anchor="end" transform="rotate(-40 147.9 346)">1.2.73</text>
<rect x="185.9" y="309.6" width="41.0" height="20.4" fill="#2e9e5b" rx="2"/>
<text x="206.4" y="304.6" font-size="10" fill="#1a1d27" text-anchor="middle">10s</text>
<text x="206.4" y="346" font-size="9" fill="#8892a4" text-anchor="end" transform="rotate(-40 206.4 346)">1.2.73</text>
<rect x="244.5" y="166.0" width="41.0" height="164.0" fill="#d64545" rx="2"/>
<text x="265.0" y="161.0" font-size="10" fill="#1a1d27" text-anchor="middle">81s</text>
<text x="265.0" y="346" font-size="9" fill="#8892a4" text-anchor="end" transform="rotate(-40 265.0 346)">1.2.73</text>
<rect x="303.1" y="308.9" width="41.0" height="21.1" fill="#2e9e5b" rx="2"/>
<text x="323.6" y="303.9" font-size="10" fill="#1a1d27" text-anchor="middle">10s</text>
<text x="323.6" y="346" font-size="9" fill="#8892a4" text-anchor="end" transform="rotate(-40 323.6 346)">1.2.73</text>
<rect x="361.6" y="309.7" width="41.0" height="20.3" fill="#2e9e5b" rx="2"/>
<text x="382.1" y="304.7" font-size="10" fill="#1a1d27" text-anchor="middle">10s</text>
<text x="382.1" y="346" font-size="9" fill="#8892a4" text-anchor="end" transform="rotate(-40 382.1 346)">1.2.75</text>
<rect x="420.2" y="312.3" width="41.0" height="17.7" fill="#2e9e5b" rx="2"/>
<text x="440.7" y="307.3" font-size="10" fill="#1a1d27" text-anchor="middle">9s</text>
<text x="440.7" y="346" font-size="9" fill="#8892a4" text-anchor="end" transform="rotate(-40 440.7 346)">1.2.75</text>
<rect x="478.8" y="155.3" width="41.0" height="174.7" fill="#d64545" rx="2"/>
<text x="499.3" y="150.3" font-size="10" fill="#1a1d27" text-anchor="middle">86s</text>
<text x="499.3" y="346" font-size="9" fill="#8892a4" text-anchor="end" transform="rotate(-40 499.3 346)">1.2.75</text>
<rect x="537.4" y="311.1" width="41.0" height="18.9" fill="#2e9e5b" rx="2"/>
<text x="557.9" y="306.1" font-size="10" fill="#1a1d27" text-anchor="middle">9s</text>
<text x="557.9" y="346" font-size="9" fill="#8892a4" text-anchor="end" transform="rotate(-40 557.9 346)">1.2.76</text>
<rect x="595.9" y="60.7" width="41.0" height="269.3" fill="#d64545" rx="2"/>
<text x="616.4" y="55.7" font-size="10" fill="#1a1d27" text-anchor="middle">133s</text>
<text x="616.4" y="346" font-size="9" fill="#8892a4" text-anchor="end" transform="rotate(-40 616.4 346)">1.2.76</text>
<rect x="654.5" y="311.1" width="41.0" height="18.9" fill="#2e9e5b" rx="2"/>
<text x="675.0" y="306.1" font-size="10" fill="#1a1d27" text-anchor="middle">9s</text>
<text x="675.0" y="346" font-size="9" fill="#8892a4" text-anchor="end" transform="rotate(-40 675.0 346)">1.2.77</text>
<rect x="713.1" y="311.2" width="41.0" height="18.8" fill="#2e9e5b" rx="2"/>
<text x="733.6" y="306.2" font-size="10" fill="#1a1d27" text-anchor="middle">9s</text>
<text x="733.6" y="346" font-size="9" fill="#8892a4" text-anchor="end" transform="rotate(-40 733.6 346)">1.2.77</text>
<rect x="771.6" y="50.0" width="41.0" height="280.0" fill="#d64545" rx="2"/>
<text x="792.1" y="45.0" font-size="10" fill="#1a1d27" text-anchor="middle">138s</text>
<text x="792.1" y="346" font-size="9" fill="#8892a4" text-anchor="end" transform="rotate(-40 792.1 346)">1.2.77</text>
<rect x="830.2" y="310.6" width="41.0" height="19.4" fill="#2e9e5b" rx="2"/>
<text x="850.7" y="305.6" font-size="10" fill="#1a1d27" text-anchor="middle">10s</text>
<text x="850.7" y="346" font-size="9" fill="#8892a4" text-anchor="end" transform="rotate(-40 850.7 346)">1.2.82</text>
<rect x="700" y="50" width="12" height="12" fill="#2e9e5b"/><text x="716" y="61" font-size="12" fill="#1a1d27">warm</text>
<rect x="770" y="50" width="12" height="12" fill="#d64545"/><text x="786" y="61" font-size="12" fill="#1a1d27">cold (post-update)</text>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 5.0 KiB

docs/perf/winv2/boot_breakdown.csv
+5
View File
@@ -0,0 +1,5 @@
phase,before_ms,after_ms
import backend.main,798,764
service lifespan (9router start),7412,84
other 15 lifespans,45,9
import + lifespans floor,8256,857
1 phase before_ms after_ms
2 import backend.main 798 764
3 service lifespan (9router start) 7412 84
4 other 15 lifespans 45 9
5 import + lifespans floor 8256 857
docs/perf/winv2/boot_breakdown.svg
+31
View File
@@ -0,0 +1,31 @@
<svg xmlns="http://www.w3.org/2000/svg" width="900" height="360" font-family="Segoe UI, sans-serif">
<text x="60" y="28" font-size="17" font-weight="600" fill="#1a1d27">warm boot breakdown: before vs after (ms) - the service lifespan was the bottleneck</text>
<line x1="60" y1="240" x2="870" y2="240" stroke="#e2e6ef"/>
<text x="52" y="244" font-size="11" fill="#8892a4" text-anchor="end">0.0s</text>
<line x1="60" y1="145" x2="870" y2="145" stroke="#e2e6ef"/>
<text x="52" y="149" font-size="11" fill="#8892a4" text-anchor="end">4.1s</text>
<line x1="60" y1="50" x2="870" y2="50" stroke="#e2e6ef"/>
<text x="52" y="54" font-size="11" fill="#8892a4" text-anchor="end">8.3s</text>
<rect x="89.0" y="221.6" width="68.9" height="18.4" fill="#d64545" rx="2"/>
<text x="123.4" y="217.6" font-size="10" fill="#1a1d27" text-anchor="middle">0.8s</text>
<rect x="157.8" y="222.4" width="68.9" height="17.6" fill="#2e9e5b" rx="2"/>
<text x="192.2" y="218.4" font-size="10" fill="#1a1d27" text-anchor="middle">0.8s</text>
<text x="161.2" y="258" font-size="10" fill="#8892a4" text-anchor="end" transform="rotate(-25 161.2 258)">import backend.main</text>
<rect x="291.5" y="69.4" width="68.9" height="170.6" fill="#d64545" rx="2"/>
<text x="325.9" y="65.4" font-size="10" fill="#1a1d27" text-anchor="middle">7.4s</text>
<rect x="360.3" y="238.1" width="68.9" height="1.9" fill="#2e9e5b" rx="2"/>
<text x="394.7" y="234.1" font-size="10" fill="#1a1d27" text-anchor="middle">0.1s</text>
<text x="363.8" y="258" font-size="10" fill="#8892a4" text-anchor="end" transform="rotate(-25 363.8 258)">service lifespan (9router start)</text>
<rect x="494.0" y="239.0" width="68.9" height="1.0" fill="#d64545" rx="2"/>
<text x="528.4" y="235.0" font-size="10" fill="#1a1d27" text-anchor="middle">0.0s</text>
<rect x="562.8" y="239.8" width="68.9" height="0.2" fill="#2e9e5b" rx="2"/>
<text x="597.2" y="235.8" font-size="10" fill="#1a1d27" text-anchor="middle">0.0s</text>
<text x="566.2" y="258" font-size="10" fill="#8892a4" text-anchor="end" transform="rotate(-25 566.2 258)">other 15 lifespans</text>
<rect x="696.5" y="50.0" width="68.9" height="190.0" fill="#d64545" rx="2"/>
<text x="730.9" y="46.0" font-size="10" fill="#1a1d27" text-anchor="middle">8.3s</text>
<rect x="765.3" y="220.3" width="68.9" height="19.7" fill="#2e9e5b" rx="2"/>
<text x="799.7" y="216.3" font-size="10" fill="#1a1d27" text-anchor="middle">0.9s</text>
<text x="768.8" y="258" font-size="10" fill="#8892a4" text-anchor="end" transform="rotate(-25 768.8 258)">import + lifespans floor</text>
<rect x="700" y="50" width="12" height="12" fill="#d64545"/><text x="716" y="61" font-size="12" fill="#1a1d27">before</text>
<rect x="780" y="50" width="12" height="12" fill="#2e9e5b"/><text x="796" y="61" font-size="12" fill="#1a1d27">after</text>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 2.7 KiB

docs/perf/winv2/inspect_asar.js
+57
View File
@@ -0,0 +1,57 @@
// #9 item 4: inventory an app.asar without extracting it. Parses the asar header
// (a Chromium Pickle: [u32 payloadSize][u32 headerSize] then [u32 payloadSize]
// [u32 jsonLen][json...]) and reports total size, biggest top-level dirs, biggest
// individual files, and trimmable categories (source maps, etc.).
// Usage: node inspect_asar.js <path-to-app.asar>
const fs = require('fs');
const asar = process.argv[2];
if (!asar) { console.error('usage: node inspect_asar.js <app.asar>'); process.exit(1); }
const fd = fs.openSync(asar, 'r');
const head = Buffer.alloc(8);
fs.readSync(fd, head, 0, 8, 0);
const headerSize = head.readUInt32LE(4); // size of the header pickle
const hp = Buffer.alloc(headerSize);
fs.readSync(fd, hp, 0, headerSize, 8);
const jsonLen = hp.readUInt32LE(4); // string length inside the pickle
const json = hp.slice(8, 8 + jsonLen).toString('utf8');
const header = JSON.parse(json);
fs.closeSync(fd);
let total = 0, fileCount = 0;
const byExt = {};
const files = []; // {path, size}
const topDirs = {}; // top-level entry -> size
function walk(node, parts) {
if (node.files) {
for (const [name, child] of Object.entries(node.files)) walk(child, parts.concat(name));
} else if (typeof node.size === 'number') {
const p = parts.join('/');
total += node.size; fileCount++;
files.push({ p, size: node.size });
const ext = (p.match(/\.[^./]+$/) || ['(none)'])[0].toLowerCase();
byExt[ext] = (byExt[ext] || 0) + node.size;
topDirs[parts[0]] = (topDirs[parts[0]] || 0) + node.size;
}
}
walk(header, []);
const mb = (b) => (b / 1048576).toFixed(1) + ' MB';
const sortObj = (o) => Object.entries(o).sort((a, b) => b[1] - a[1]);
console.log(`asar total: ${mb(total)} across ${fileCount} files\n`);
console.log('=== biggest top-level entries ===');
for (const [d, s] of sortObj(topDirs).slice(0, 15)) console.log(` ${mb(s).padStart(10)} ${d}`);
console.log('\n=== biggest single files ===');
for (const f of files.sort((a, b) => b.size - a.size).slice(0, 20)) console.log(` ${mb(f.size).padStart(10)} ${f.p}`);
console.log('\n=== by extension (top 15) ===');
for (const [e, s] of sortObj(byExt).slice(0, 15)) console.log(` ${mb(s).padStart(10)} ${e}`);
console.log('\n=== trimmable categories ===');
const cat = (re) => files.filter(f => re.test(f.p)).reduce((n, f) => n + f.size, 0);
console.log(` source maps (*.map): ${mb(cat(/\.map$/))}`);
console.log(` .ts/.tsx sources: ${mb(cat(/\.tsx?$/))}`);
console.log(` markdown/license/readme: ${mb(cat(/(\.md|license|readme|changelog)/i))}`);
console.log(` test/spec/__tests__: ${mb(cat(/(\/test\/|\/tests\/|__tests__|\.spec\.|\.test\.)/i))}`);
console.log(` node_modules inside asar: ${mb(cat(/(^|\/)node_modules\//))}`);
docs/perf/winv2/make_graphs.py
+226
View File
@@ -0,0 +1,226 @@
"""Dependency-free SVG charts for the winv2 perf baseline.
No matplotlib/pandas (not in the bundled env). Reads baseline_startup.csv and
writes two self-contained SVGs that render in a browser, GitHub, or Notion:
baseline_startup.svg - backend-http-ready per launch (warm vs cold)
baseline_phases.svg - where the time goes (app-launch / first-paint / backend)
Run: python make_graphs.py
"""
import csv
import os
HERE = os.path.dirname(os.path.abspath(__file__))
CSV = os.path.join(HERE, "baseline_startup.csv")
WARM = "#2e9e5b"
COLD = "#d64545"
INK = "#1a1d27"
MUTE = "#8892a4"
GRID = "#e2e6ef"
def rows():
with open(CSV, newline="", encoding="utf-8") as f:
return list(csv.DictReader(f))
def bars_chart(data):
w, h = 900, 420
pad_l, pad_b, pad_t, pad_r = 60, 90, 50, 20
plot_w = w - pad_l - pad_r
plot_h = h - pad_t - pad_b
vals = [int(r["backend_http_ready_ms"]) for r in data]
vmax = max(vals)
n = len(data)
bw = plot_w / n * 0.7
gap = plot_w / n
out = [f'<svg xmlns="http://www.w3.org/2000/svg" width="{w}" height="{h}" font-family="Segoe UI, sans-serif">']
out.append(f'<text x="{pad_l}" y="28" font-size="17" font-weight="600" fill="{INK}">'
'backend-http-ready per launch (ms) - lower is better</text>')
# y gridlines
for frac in (0, 0.25, 0.5, 0.75, 1.0):
yv = vmax * frac
y = pad_t + plot_h - plot_h * frac
out.append(f'<line x1="{pad_l}" y1="{y:.0f}" x2="{w-pad_r}" y2="{y:.0f}" stroke="{GRID}"/>')
out.append(f'<text x="{pad_l-8}" y="{y+4:.0f}" font-size="11" fill="{MUTE}" text-anchor="end">{yv/1000:.0f}s</text>')
for i, r in enumerate(data):
v = int(r["backend_http_ready_ms"])
bh = plot_h * v / vmax
x = pad_l + i * gap + (gap - bw) / 2
y = pad_t + plot_h - bh
color = COLD if r["class"] == "cold" else WARM
out.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bw:.1f}" height="{bh:.1f}" fill="{color}" rx="2"/>')
out.append(f'<text x="{x+bw/2:.1f}" y="{y-5:.1f}" font-size="10" fill="{INK}" text-anchor="middle">{v/1000:.0f}s</text>')
out.append(f'<text x="{x+bw/2:.1f}" y="{h-pad_b+16:.0f}" font-size="9" fill="{MUTE}" '
f'text-anchor="end" transform="rotate(-40 {x+bw/2:.1f} {h-pad_b+16:.0f})">{r["version"]}</text>')
out.append(f'<rect x="{w-200}" y="{pad_t}" width="12" height="12" fill="{WARM}"/>'
f'<text x="{w-184}" y="{pad_t+11}" font-size="12" fill="{INK}">warm</text>')
out.append(f'<rect x="{w-130}" y="{pad_t}" width="12" height="12" fill="{COLD}"/>'
f'<text x="{w-114}" y="{pad_t+11}" font-size="12" fill="{INK}">cold (post-update)</text>')
out.append('</svg>')
return "\n".join(out)
def phases_chart(data):
warm = [r for r in data if r["class"] == "warm"]
cold = [r for r in data if r["class"] == "cold"]
def med(rows_, key):
xs = sorted(int(r[key]) for r in rows_)
return xs[len(xs) // 2] if xs else 0
cases = [
("typical warm launch", med(warm, "app_launch_ms"), med(warm, "first_paint_ms"), med(warm, "backend_http_ready_ms")),
("typical cold launch", med(cold, "app_launch_ms"), med(cold, "first_paint_ms"), med(cold, "backend_http_ready_ms")),
]
w, h = 900, 260
pad_l, pad_r, pad_t = 170, 30, 50
plot_w = w - pad_l - pad_r
vmax = max(c[3] for c in cases)
out = [f'<svg xmlns="http://www.w3.org/2000/svg" width="{w}" height="{h}" font-family="Segoe UI, sans-serif">']
out.append(f'<text x="20" y="28" font-size="17" font-weight="600" fill="{INK}">'
'where startup time goes (backend dwarfs the shell)</text>')
row_h = 46
for i, (label, al, fp, br) in enumerate(cases):
y = pad_t + i * (row_h + 26)
out.append(f'<text x="20" y="{y+row_h/2+4:.0f}" font-size="13" fill="{INK}">{label}</text>')
# backend is the full bar; app-launch+first-paint are the tiny left slice
bw_backend = plot_w * br / vmax
out.append(f'<rect x="{pad_l}" y="{y}" width="{bw_backend:.1f}" height="{row_h}" fill="{COLD if i==1 else WARM}" rx="3"/>')
shell = al + fp
bw_shell = plot_w * shell / vmax
out.append(f'<rect x="{pad_l}" y="{y}" width="{max(bw_shell,2):.1f}" height="{row_h}" fill="{INK}" rx="3"/>')
out.append(f'<text x="{pad_l+bw_backend+8:.0f}" y="{y+row_h/2+4:.0f}" font-size="12" fill="{INK}">'
f'backend {br/1000:.1f}s (shell {shell/1000:.2f}s)</text>')
out.append(f'<text x="20" y="{h-12}" font-size="11" fill="{MUTE}">'
'dark = electron shell (app-launch + first-paint); colored = python backend</text>')
out.append('</svg>')
return "\n".join(out)
def boot_chart():
"""Before/after grouped bars for the boot-phase breakdown (profile_boot.py)."""
path = os.path.join(HERE, "boot_breakdown.csv")
with open(path, newline="", encoding="utf-8") as f:
data = list(csv.DictReader(f))
w, h = 900, 360
pad_l, pad_r, pad_t, pad_b = 60, 30, 50, 120
plot_w = w - pad_l - pad_r
plot_h = h - pad_t - pad_b
vmax = max(max(int(r["before_ms"]), int(r["after_ms"])) for r in data)
n = len(data)
group = plot_w / n
bw = group * 0.34
out = [f'<svg xmlns="http://www.w3.org/2000/svg" width="{w}" height="{h}" font-family="Segoe UI, sans-serif">']
out.append(f'<text x="{pad_l}" y="28" font-size="17" font-weight="600" fill="{INK}">'
'warm boot breakdown: before vs after (ms) - the service lifespan was the bottleneck</text>')
for frac in (0, 0.5, 1.0):
y = pad_t + plot_h - plot_h * frac
out.append(f'<line x1="{pad_l}" y1="{y:.0f}" x2="{w-pad_r}" y2="{y:.0f}" stroke="{GRID}"/>')
out.append(f'<text x="{pad_l-8}" y="{y+4:.0f}" font-size="11" fill="{MUTE}" text-anchor="end">{vmax*frac/1000:.1f}s</text>')
for i, r in enumerate(data):
bx = pad_l + i * group + group / 2
for j, (key, color, lab) in enumerate((("before_ms", COLD, "before"), ("after_ms", WARM, "after"))):
v = int(r[key])
bh = plot_h * v / vmax
x = bx + (j - 1) * bw - bw * 0.05
y = pad_t + plot_h - bh
out.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bw:.1f}" height="{bh:.1f}" fill="{color}" rx="2"/>')
out.append(f'<text x="{x+bw/2:.1f}" y="{y-4:.1f}" font-size="10" fill="{INK}" text-anchor="middle">{v/1000:.1f}s</text>')
out.append(f'<text x="{bx:.1f}" y="{h-pad_b+18:.0f}" font-size="10" fill="{MUTE}" text-anchor="end" '
f'transform="rotate(-25 {bx:.1f} {h-pad_b+18:.0f})">{r["phase"]}</text>')
out.append(f'<rect x="{w-200}" y="{pad_t}" width="12" height="12" fill="{COLD}"/><text x="{w-184}" y="{pad_t+11}" font-size="12" fill="{INK}">before</text>')
out.append(f'<rect x="{w-120}" y="{pad_t}" width="12" height="12" fill="{WARM}"/><text x="{w-104}" y="{pad_t+11}" font-size="12" fill="{INK}">after</text>')
out.append('</svg>')
return "\n".join(out)
def appbuilder_chart():
"""Horizontal bars for the App Builder create-path breakdown. Returns None
if the measurement CSV hasn't been generated yet."""
path = os.path.join(HERE, "appbuilder_breakdown.csv")
if not os.path.exists(path):
return None
with open(path, newline="", encoding="utf-8") as f:
raw = list(csv.DictReader(f))
# Keep only real timing phases (drop the boolean/skipped/-1 rows).
data = [r for r in raw if r["ms"].lstrip("-").isdigit() and int(r["ms"]) >= 0
and not r["phase"].strip().startswith("->")]
if not data:
return None
w = 980
row_h, gap, pad_t, pad_l, pad_r = 30, 14, 56, 320, 90
h = pad_t + len(data) * (row_h + gap) + 30
vmax = max(int(r["ms"]) for r in data) or 1
plot_w = w - pad_l - pad_r
out = [f'<svg xmlns="http://www.w3.org/2000/svg" width="{w}" height="{h}" font-family="Segoe UI, sans-serif">']
out.append(f'<text x="20" y="30" font-size="17" font-weight="600" fill="{INK}">'
'App Builder "create app -> live preview" breakdown (ms)</text>')
for i, r in enumerate(data):
v = int(r["ms"])
y = pad_t + i * (row_h + gap)
bw = max(plot_w * v / vmax, 1)
# download/npm = cold cost (red-ish), everything else = warm/per-app (green)
cold = ("npm" in r["phase"]) or ("cold" in r["phase"])
color = COLD if cold else WARM
out.append(f'<text x="{pad_l-10}" y="{y+row_h*0.68:.0f}" font-size="12" fill="{INK}" text-anchor="end">{r["phase"]}</text>')
out.append(f'<rect x="{pad_l}" y="{y}" width="{bw:.1f}" height="{row_h}" fill="{color}" rx="3"/>')
label = f'{v/1000:.2f}s' if v >= 1000 else f'{v}ms'
out.append(f'<text x="{pad_l+bw+8:.0f}" y="{y+row_h*0.68:.0f}" font-size="12" fill="{INK}">{label}</text>')
out.append(f'<text x="20" y="{h-10}" font-size="11" fill="{MUTE}">'
'green = warm/per-app cost; red = cold one-time download (npm with no archive)</text>')
out.append('</svg>')
return "\n".join(out)
def startup_beforeafter_chart():
"""Before/after grouped bars for the signed-build startup result (Task #10)."""
path = os.path.join(HERE, "startup_beforeafter.csv")
if not os.path.exists(path):
return None
with open(path, newline="", encoding="utf-8") as f:
data = list(csv.DictReader(f))
w, h = 900, 320
pad_l, pad_r, pad_t, pad_b = 60, 30, 56, 90
plot_w, plot_h = w - pad_l - pad_r, h - pad_t - pad_b
vmax = max(max(int(r["before_ms"]), int(r["after_ms"])) for r in data)
n = len(data); group = plot_w / n; bw = group * 0.30
out = [f'<svg xmlns="http://www.w3.org/2000/svg" width="{w}" height="{h}" font-family="Segoe UI, sans-serif">']
out.append(f'<text x="{pad_l}" y="30" font-size="17" font-weight="600" fill="{INK}">'
'signed v1.3.87 startup: before vs after (seconds, lower is better)</text>')
for frac in (0, 0.5, 1.0):
y = pad_t + plot_h - plot_h * frac
out.append(f'<line x1="{pad_l}" y1="{y:.0f}" x2="{w-pad_r}" y2="{y:.0f}" stroke="{GRID}"/>')
out.append(f'<text x="{pad_l-8}" y="{y+4:.0f}" font-size="11" fill="{MUTE}" text-anchor="end">{vmax*frac/1000:.0f}s</text>')
for i, r in enumerate(data):
bx = pad_l + i * group + group / 2
for j, (k, c, lab) in enumerate((("before_ms", COLD, "before"), ("after_ms", WARM, "after"))):
v = int(r[k]); bh = plot_h * v / vmax; x = bx + (j - 1) * bw - bw * 0.05; y = pad_t + plot_h - bh
out.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bw:.1f}" height="{bh:.1f}" fill="{c}" rx="2"/>')
out.append(f'<text x="{x+bw/2:.1f}" y="{y-4:.1f}" font-size="11" fill="{INK}" text-anchor="middle">{v/1000:.1f}s</text>')
out.append(f'<text x="{bx:.1f}" y="{h-pad_b+22:.0f}" font-size="11" fill="{MUTE}" text-anchor="middle">{r["metric"]}</text>')
out.append(f'<rect x="{w-200}" y="{pad_t}" width="12" height="12" fill="{COLD}"/><text x="{w-184}" y="{pad_t+11}" font-size="12" fill="{INK}">before (1.2.x)</text>')
out.append(f'<rect x="{w-95}" y="{pad_t}" width="12" height="12" fill="{WARM}"/><text x="{w-79}" y="{pad_t+11}" font-size="12" fill="{INK}">v1.3.87</text>')
out.append('</svg>')
return "\n".join(out)
def main():
data = rows()
open(os.path.join(HERE, "baseline_startup.svg"), "w", encoding="utf-8").write(bars_chart(data))
open(os.path.join(HERE, "baseline_phases.svg"), "w", encoding="utf-8").write(phases_chart(data))
open(os.path.join(HERE, "boot_breakdown.svg"), "w", encoding="utf-8").write(boot_chart())
wrote = "baseline_startup.svg + baseline_phases.svg + boot_breakdown.svg"
ab = appbuilder_chart()
if ab:
open(os.path.join(HERE, "appbuilder_breakdown.svg"), "w", encoding="utf-8").write(ab)
wrote += " + appbuilder_breakdown.svg"
sba = startup_beforeafter_chart()
if sba:
open(os.path.join(HERE, "startup_beforeafter.svg"), "w", encoding="utf-8").write(sba)
wrote += " + startup_beforeafter.svg"
print("wrote " + wrote)
if __name__ == "__main__":
main()
docs/perf/winv2/measure_appbuilder.py
+135
View File
@@ -0,0 +1,135 @@
"""Granular App Builder first-app create/"download" profiler (winv2 Task #3).
Incremental + bounded: each phase appends to appbuilder_breakdown.csv and flushes
the instant it finishes, so a slow/hung later phase can't erase earlier numbers.
Run UNBUFFERED (python -u) so progress is visible mid-run. Cheap phases first.
Phases:
1. seed workspace + link node_modules (per-app cost, uses real warm cache)
2. download: npm install (cold, no archive) (the "slow as bricks" download)
3. download: archive extract (new build path) (tar the just-installed nm, time extract)
4. vite bind: cold vite cache (first app ever)
5. vite bind: warm shared cache (subsequent apps)
Isolated temp dirs; never mutates the user's real caches (read-only link to the
warm node_modules cache; vite cache is overridden to temp for the cold case).
"""
import asyncio
import os
import shutil
import subprocess
import tarfile
import tempfile
import time
from backend.apps.outputs import view_builder_templates as vt
from backend.apps.outputs.runtime_proc import _find_free_port
from backend.apps.outputs.runtime import AppRuntime
HERE = os.path.dirname(os.path.abspath(__file__))
CSV = os.path.join(HERE, "appbuilder_breakdown.csv")
TMP = tempfile.mkdtemp(prefix="ab-measure-")
TMPL_FRONTEND = os.path.join(vt.WEBAPP_TEMPLATE_DIR, "frontend")
VITE_DEADLINE = 90
with open(CSV, "w", encoding="utf-8") as f:
f.write("phase,ms,note\n")
def lap(t):
return round((time.perf_counter() - t) * 1000)
def record(name, ms, note=""):
print(f"{ms:8d} ms {name}" + (f" ({note})" if note else ""), flush=True)
with open(CSV, "a", encoding="utf-8") as f:
f.write(f'"{name}",{ms},"{note}"\n')
f.flush()
def phase_seed():
ws = os.path.join(TMP, "ws-seed")
t = time.perf_counter()
vt.seed_webapp_template_workspace(ws, _find_free_port())
ms = lap(t)
present = os.path.exists(os.path.join(ws, "frontend", "node_modules"))
record("seed workspace + link node_modules (per app)", ms, "nm linked" if present else "NO nm")
def phase_npm_and_extract():
npm = vt._resolve_npm()
if not npm:
record("download: npm install (cold)", -1, "skipped: no npm")
return
work = os.path.join(TMP, "npm_cold")
os.makedirs(work, exist_ok=True)
shutil.copyfile(os.path.join(TMPL_FRONTEND, "package.json"), os.path.join(work, "package.json"))
lock = os.path.join(TMPL_FRONTEND, "package-lock.json")
cmd = [*npm, "install", "--prefer-offline", "--no-audit", "--no-fund", "--loglevel=error"]
if os.path.exists(lock):
shutil.copyfile(lock, os.path.join(work, "package-lock.json"))
cmd = [*npm, "ci", "--prefer-offline", "--no-audit", "--no-fund", "--loglevel=error"]
t = time.perf_counter()
try:
r = subprocess.run(cmd, cwd=work, capture_output=True, text=True, timeout=240)
record("download: npm install (cold, no archive)", lap(t), "ok" if r.returncode == 0 else f"rc={r.returncode}")
except subprocess.TimeoutExpired:
record("download: npm install (cold, no archive)", -1, "TIMEOUT 240s")
return
nm = os.path.join(work, "node_modules")
if not os.path.isdir(nm):
return
# Reuse that node_modules to time the archive build + extract (new path).
archive = os.path.join(TMP, "nm.tar.gz")
t = time.perf_counter()
with tarfile.open(archive, "w:gz") as tar:
tar.add(nm, arcname="node_modules")
record("build-time: tar node_modules -> archive", lap(t), f"{os.path.getsize(archive)//(1024*1024)}MB")
exd = os.path.join(TMP, "extract"); os.makedirs(exd, exist_ok=True)
t = time.perf_counter()
with tarfile.open(archive, "r:gz") as tar:
tar.extractall(exd)
record("download: archive extract (new build path)", lap(t))
async def _bind_once(label, vite_cache_dir):
ws = os.path.join(TMP, f"ws-{label}")
vt.seed_webapp_template_workspace(ws, _find_free_port())
if vite_cache_dir:
os.environ["OPENSWARM_VITE_CACHE_DIR"] = vite_cache_dir
else:
os.environ.pop("OPENSWARM_VITE_CACHE_DIR", None)
rt = AppRuntime(f"ws-{label}", ws)
t = time.perf_counter()
await rt.start()
deadline = time.perf_counter() + VITE_DEADLINE
while rt.frontend_url is None and time.perf_counter() < deadline:
await asyncio.sleep(0.1)
bound = rt.frontend_url is not None
ms = lap(t) if bound else -1
try:
await rt.stop()
except Exception:
pass
record(f"vite bind ({label})", ms, "bound" if bound else f"TIMEOUT {VITE_DEADLINE}s")
async def main():
print(f"temp: {TMP}", flush=True)
for fn in (phase_seed, phase_npm_and_extract):
try:
fn()
except Exception as e:
record(fn.__name__, -1, f"ERR {type(e).__name__}: {e}")
for label, cache in (("cold vite cache", os.path.join(TMP, "vite_cold")), ("warm shared cache", None)):
try:
await _bind_once(label, cache)
except Exception as e:
record(f"vite bind ({label})", -1, f"ERR {type(e).__name__}: {e}")
print("done", flush=True)
shutil.rmtree(TMP, ignore_errors=True)
if __name__ == "__main__":
asyncio.run(main())
docs/perf/winv2/measure_download_install.ps1
+35
View File
@@ -0,0 +1,35 @@
<#
.SYNOPSIS
Task #10: track the user-facing DOWNLOAD speed + verify the signature of the
signed build, then hand off to validate_packaged.ps1 for install/startup. Does
NOT publish anything (downloads from the draft release / run artifacts only).
.USAGE
pwsh docs\perf\winv2\measure_download_install.ps1 -Tag v1.3.86
#>
param(
[string]$Tag = 'v1.3.86',
[string]$WorkDir = (Join-Path $env:TEMP "os-dl-$Tag")
)
$ErrorActionPreference = 'Stop'
New-Item -ItemType Directory -Force -Path $WorkDir | Out-Null
# 1. Download the signed installer (timed) -> download speed.
$sw = [Diagnostics.Stopwatch]::StartNew()
gh release download $Tag --pattern '*Setup*.exe' --dir $WorkDir --clobber
$sw.Stop()
$exe = Get-ChildItem $WorkDir -Filter '*Setup*.exe' | Select-Object -First 1
if (-not $exe) { throw "no Setup .exe for $Tag (is the draft release built? try: gh run download <id>)" }
$mb = [math]::Round($exe.Length / 1MB, 1)
$secs = [math]::Round($sw.Elapsed.TotalSeconds, 1)
$mbps = if ($secs -gt 0) { [math]::Round($mb / $secs, 1) } else { 'inf' }
Write-Host ("DOWNLOAD: {0} MB in {1}s ({2} MB/s) -> {3}" -f $mb, $secs, $mbps, $exe.Name)
# 2. Verify it is really code-signed.
$sig = Get-AuthenticodeSignature $exe.FullName
Write-Host ("SIGNATURE: {0} signer={1}" -f $sig.Status, $sig.SignerCertificate.Subject)
if ($sig.Status -ne 'Valid') { Write-Warning "signature is NOT Valid -- stop and investigate before installing" }
Write-Host ""
Write-Host "Installer: $($exe.FullName)"
Write-Host "Next (install timing): note the clock, run the installer, then time until %APPDATA%\openswarm\data\backend.log appears."
Write-Host "Then: pwsh docs\perf\winv2\validate_packaged.ps1 (structural + cold/warm startup)"
docs/perf/winv2/measure_vite.py
+70
View File
@@ -0,0 +1,70 @@
"""Vite-bind-only measurement (winv2 Task #3, part 2).
Split out from measure_appbuilder.py because Python's tarfile gzip of a full
node_modules is pathologically slow and was eating the time budget before the
vite phases ran. This does ONLY the two vite binds (cold vite cache = first app
ever; warm shared cache = subsequent apps) and APPENDS to appbuilder_breakdown.csv.
No tar, no npm. Run unbuffered.
"""
import asyncio
import os
import shutil
import tempfile
import time
from backend.apps.outputs import view_builder_templates as vt
from backend.apps.outputs.runtime_proc import _find_free_port
from backend.apps.outputs.runtime import AppRuntime
HERE = os.path.dirname(os.path.abspath(__file__))
CSV = os.path.join(HERE, "appbuilder_breakdown.csv")
TMP = tempfile.mkdtemp(prefix="ab-vite-")
VITE_DEADLINE = 100
def record(name, ms, note=""):
print(f"{ms:8d} ms {name}" + (f" ({note})" if note else ""), flush=True)
with open(CSV, "a", encoding="utf-8") as f:
f.write(f'"{name}",{ms},"{note}"\n')
f.flush()
async def bind_once(label, vite_cache_dir):
ws = os.path.join(TMP, f"ws-{label.replace(' ', '_')}")
vt.seed_webapp_template_workspace(ws, _find_free_port())
if not os.path.exists(os.path.join(ws, "frontend", "node_modules")):
record(f"vite bind ({label})", -1, "no node_modules linked")
return
if vite_cache_dir:
os.environ["OPENSWARM_VITE_CACHE_DIR"] = vite_cache_dir
else:
os.environ.pop("OPENSWARM_VITE_CACHE_DIR", None)
rt = AppRuntime(f"ws-{label}", ws)
t = time.perf_counter()
await rt.start()
deadline = time.perf_counter() + VITE_DEADLINE
while rt.frontend_url is None and time.perf_counter() < deadline:
await asyncio.sleep(0.1)
bound = rt.frontend_url is not None
ms = round((time.perf_counter() - t) * 1000) if bound else -1
try:
await rt.stop()
except Exception:
pass
record(f"vite bind ({label})", ms, "bound" if bound else f"TIMEOUT {VITE_DEADLINE}s")
async def main():
print(f"temp: {TMP}", flush=True)
for label, cache in (("cold vite cache", os.path.join(TMP, "vite_cold")),
("warm shared cache", None)):
try:
await bind_once(label, cache)
except Exception as e:
record(f"vite bind ({label})", -1, f"ERR {type(e).__name__}: {e}")
print("done", flush=True)
shutil.rmtree(TMP, ignore_errors=True)
if __name__ == "__main__":
asyncio.run(main())
docs/perf/winv2/profile_boot.py
+67
View File
@@ -0,0 +1,67 @@
"""Per-phase + per-SubApp-lifespan boot profiler (winv2).
Warm import is ~1.3s but backend-http-ready is ~9-10s, so the gap is the
lifespan startup (SubApp lifespans are entered sequentially in config/Apps.py
before uvicorn serves). This times each one to find what blocks the HTTP bind.
Run with the bundled interpreter from the resources dir, e.g.:
python-env/python.exe docs/perf/winv2/profile_boot.py
It spawns the same subprocesses a real boot does (9router etc.); the
AsyncExitStack unwinds at the end. Kill any straggler node/9router after.
"""
import asyncio
import os
import time
os.environ.setdefault("OPENSWARM_AUTH_TOKEN", "x")
_t0 = time.perf_counter()
import backend.main # noqa: F401 (builds main_app; full import tree)
_import_ms = (time.perf_counter() - _t0) * 1000
from contextlib import AsyncExitStack # noqa: E402
from backend.apps.health.health import health # noqa: E402
from backend.apps.agents.agents import agents # noqa: E402
from backend.apps.skills.skills import skills # noqa: E402
from backend.apps.tools_lib.tools_lib import tools_lib # noqa: E402
from backend.apps.modes.modes import modes # noqa: E402
from backend.apps.settings.settings import settings # noqa: E402
from backend.apps.mcp_registry.mcp_registry import mcp_registry # noqa: E402
from backend.apps.skill_registry.skill_registry import skill_registry # noqa: E402
from backend.apps.outputs.outputs import outputs # noqa: E402
from backend.apps.dashboards.dashboards import dashboards # noqa: E402
from backend.apps.swarm.swarm import swarm # noqa: E402
from backend.apps.service.service import service # noqa: E402
from backend.apps.subscription.router import subscription # noqa: E402
from backend.apps.auth.router import auth # noqa: E402
from backend.apps.web.web import web # noqa: E402
from backend.apps.agents.proxy.anthropic_proxy import anthropic_proxy # noqa: E402
SUBS = [health, agents, skills, tools_lib, modes, settings, mcp_registry,
skill_registry, outputs, dashboards, swarm, service, subscription,
auth, web, anthropic_proxy]
async def main():
print(f"{_import_ms:8.0f} ms import backend.main (full tree)")
print("-" * 48)
total = 0.0
async with AsyncExitStack() as stack:
for s in SUBS:
t = time.perf_counter()
try:
await asyncio.wait_for(stack.enter_async_context(s.lifespan()), timeout=60)
except Exception as e:
print(f" ERR lifespan {s.name}: {type(e).__name__}")
continue
dt = (time.perf_counter() - t) * 1000
total += dt
print(f"{dt:8.0f} ms lifespan {s.name}")
print("-" * 48)
print(f"{total:8.0f} ms all lifespans")
print(f"{_import_ms + total:8.0f} ms import + lifespans (approx backend-ready floor)")
if __name__ == "__main__":
asyncio.run(main())
docs/perf/winv2/profile_startup.sh
+31
View File
@@ -0,0 +1,31 @@
#!/bin/bash
# Re-extract real packaged-app startup timings from the installed app's backend
# log. Prints one row per launch: timestamp, version, app-launch ms,
# first-paint ms, backend-http-ready ms. Pipe to a CSV for the metrics table.
#
# Usage: bash profile_startup.sh [path-to-backend.log]
# Default log: AppData/Roaming/openswarm/data/backend.log
LOG="${1:-$HOME/AppData/Roaming/openswarm/data/backend.log}"
if [[ ! -f "$LOG" ]]; then
echo "no backend.log at $LOG" >&2
exit 1
fi
echo "launch_ts,version,app_launch_ms,first_paint_ms,backend_http_ready_ms,class"
awk '
/===== launch/ {
if (ts != "") emit()
ts=$3; ver=""
for (i=1;i<=NF;i++) if ($i ~ /^\(app$/) { ver=$(i+1); gsub(/,/,"",ver) }
al=""; fp=""; br=""
}
/\[perf\] app-launch t=/ { sub(/.*t=/,""); al=$0 }
/\[perf\] first-paint t=/ { sub(/.*t=/,""); fp=$0 }
/\[perf\] backend-http-ready t=/ { sub(/.*t=/,""); br=$0 }
END { if (ts != "") emit() }
function emit() {
cls = (br+0 > 20000) ? "cold" : "warm"
printf "%s,%s,%s,%s,%s,%s\n", ts, ver, al, fp, br, cls
}
' "$LOG"
docs/perf/winv2/startup_beforeafter.csv
+3
View File
@@ -0,0 +1,3 @@
metric,before_ms,after_ms
cold backend-ready (54-138s -> 22.5s),96000,22500
warm backend-ready (9-10s -> 5.0s),9500,5000
1 metric before_ms after_ms
2 cold backend-ready (54-138s -> 22.5s) 96000 22500
3 warm backend-ready (9-10s -> 5.0s) 9500 5000
docs/perf/winv2/startup_beforeafter.svg
+21
View File
@@ -0,0 +1,21 @@
<svg xmlns="http://www.w3.org/2000/svg" width="900" height="320" font-family="Segoe UI, sans-serif">
<text x="60" y="30" font-size="17" font-weight="600" fill="#1a1d27">signed v1.3.87 startup: before vs after (seconds, lower is better)</text>
<line x1="60" y1="230" x2="870" y2="230" stroke="#e2e6ef"/>
<text x="52" y="234" font-size="11" fill="#8892a4" text-anchor="end">0s</text>
<line x1="60" y1="143" x2="870" y2="143" stroke="#e2e6ef"/>
<text x="52" y="147" font-size="11" fill="#8892a4" text-anchor="end">48s</text>
<line x1="60" y1="56" x2="870" y2="56" stroke="#e2e6ef"/>
<text x="52" y="60" font-size="11" fill="#8892a4" text-anchor="end">96s</text>
<rect x="134.9" y="56.0" width="121.5" height="174.0" fill="#d64545" rx="2"/>
<text x="195.7" y="52.0" font-size="11" fill="#1a1d27" text-anchor="middle">96.0s</text>
<rect x="256.4" y="189.2" width="121.5" height="40.8" fill="#2e9e5b" rx="2"/>
<text x="317.2" y="185.2" font-size="11" fill="#1a1d27" text-anchor="middle">22.5s</text>
<text x="262.5" y="252" font-size="11" fill="#8892a4" text-anchor="middle">cold backend-ready (54-138s -> 22.5s)</text>
<rect x="539.9" y="212.8" width="121.5" height="17.2" fill="#d64545" rx="2"/>
<text x="600.7" y="208.8" font-size="11" fill="#1a1d27" text-anchor="middle">9.5s</text>
<rect x="661.4" y="220.9" width="121.5" height="9.1" fill="#2e9e5b" rx="2"/>
<text x="722.2" y="216.9" font-size="11" fill="#1a1d27" text-anchor="middle">5.0s</text>
<text x="667.5" y="252" font-size="11" fill="#8892a4" text-anchor="middle">warm backend-ready (9-10s -> 5.0s)</text>
<rect x="700" y="56" width="12" height="12" fill="#d64545"/><text x="716" y="67" font-size="12" fill="#1a1d27">before (1.2.x)</text>
<rect x="805" y="56" width="12" height="12" fill="#2e9e5b"/><text x="821" y="67" font-size="12" fill="#1a1d27">v1.3.87</text>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 1.8 KiB

docs/perf/winv2/validate_packaged.ps1
+62
View File
@@ -0,0 +1,62 @@
<#
.SYNOPSIS
Task #10 automated checks: run AFTER installing the signed build. Confirms the
winv2 structural fixes landed in the packaged app and reads the REAL cold/warm
backend-http-ready from the app's own perf log. Manual GUI checks are in
TASK10_CHECKLIST.md (App Builder preview, Skills list, onboarding).
.USAGE
pwsh docs\perf\winv2\validate_packaged.ps1
#>
param(
[string]$InstallRoot = (Join-Path $env:LOCALAPPDATA 'openswarm'),
[string]$BackendLog = (Join-Path $env:APPDATA 'openswarm\data\backend.log')
)
$ErrorActionPreference = 'Stop'
$pass = 0; $fail = 0
function ok($m) { Write-Host " PASS $m" -ForegroundColor Green; $script:pass++ }
function bad($m) { Write-Host " FAIL $m" -ForegroundColor Red; $script:fail++ }
function info($m) { Write-Host " .. $m" -ForegroundColor DarkGray }
$app = Get-ChildItem $InstallRoot -Directory -Filter 'app-*' -EA SilentlyContinue | Sort-Object Name | Select-Object -Last 1
if (-not $app) { throw "no app-* under $InstallRoot (install the build first)" }
$res = Join-Path $app.FullName 'resources'
Write-Host "Validating packaged build: $res`n"
# #9 item 4: asar trimmed
$asar = Join-Path $res 'app.asar'
if (Test-Path $asar) {
$asarMB = [math]::Round((Get-Item $asar).Length / 1MB, 1)
if ($asarMB -lt 50) { ok "app.asar = ${asarMB} MB (trimmed; was ~607 MB)" } else { bad "app.asar = ${asarMB} MB (expected < 50)" }
$insp = Join-Path $PSScriptRoot 'inspect_asar.js'
if ((Get-Command node -EA SilentlyContinue) -and (Test-Path $insp)) {
$out = & node $insp $asar 2>&1 | Out-String
if ($out -match 'python-env|build-staging') { bad "asar STILL contains python-env/build-staging" } else { ok "asar excludes python-env + build-staging" }
}
} else { bad "app.asar not found" }
# still shipped unpacked (runtime reads these)
if (Test-Path (Join-Path $res 'python-env\python.exe')) { ok "python-env shipped unpacked" } else { bad "python-env\python.exe missing" }
if (Test-Path (Join-Path $res 'node\x64\node.exe')) { ok "node bundled" } else { bad "node\x64\node.exe missing" }
# Bug #1: skills snapshot
if (Test-Path (Join-Path $res 'backend\apps\skill_registry\skills_snapshot.json')) { ok "skills snapshot shipped (catalog never empty)" } else { bad "skills_snapshot.json missing" }
# #9 item 2 / Bug #2: webapp node_modules pre-extracted or archive
$cache = Join-Path $res 'backend\apps\outputs\webapp_template_cache'
if (Test-Path (Join-Path $cache '*\node_modules\vite\bin\vite.js')) { ok "webapp node_modules PRE-EXTRACTED (zero first-app extract)" }
elseif (Test-Path (Join-Path $cache 'node_modules.*.tar.gz')) { info "webapp node_modules shipped as .tar.gz (extract path, not pre-extracted)" }
else { bad "no webapp node_modules tree/archive in resources" }
# perf: real cold/warm backend-http-ready from the app's own log
if (Test-Path $BackendLog) {
$m = Select-String -Path $BackendLog -Pattern 'backend-http-ready t=(\d+)' -AllMatches
$vals = @($m.Matches | ForEach-Object { [int]$_.Groups[1].Value })
if ($vals.Count) {
$recent = ($vals | Select-Object -Last 6 | ForEach-Object { [math]::Round($_ / 1000, 1) }) -join 's, '
info "backend-http-ready recent: ${recent}s (baseline: warm ~9-10s, cold 54-138s)"
info "latest: $([math]::Round($vals[-1]/1000,1))s -- first launch after install = COLD; relaunch for warm"
} else { info "no backend-http-ready markers yet" }
} else { info "no backend.log yet (launch the app once first)" }
Write-Host "`n$pass passed, $fail failed. Manual GUI checks: TASK10_CHECKLIST.md (App Builder, Skills, onboarding)."
if ($fail) { exit 1 }
electron/main.js
+131 -44
View File
@@ -1225,7 +1225,15 @@ function createWindow() {
});
mainWindow.webContents.on('will-navigate', (event, url) => {
if (isDev && url.startsWith('http://localhost:3000')) return;
// Same-origin navigations are the app's own routing (reload, hash routes),
// never an external link to pop into a browser card. The old port-specific
// exemptions missed prod (renderer on 127.0.0.1:4173, not localhost:3000 or
// file://), so a reload, e.g. Restart tour, got intercepted and re-opened
// as a browser card loading the app itself (the recursive nested window).
try {
const current = mainWindow.webContents.getURL();
if (current && new URL(url).origin === new URL(current).origin) return;
} catch (_) {}
if (url.startsWith('file://')) return;
event.preventDefault();
mainWindow.webContents.send('webview-new-window', url, mainWindow.webContents.id);
@@ -1274,40 +1282,27 @@ function createWindow() {
// closing windows as part of its pipeline.
mainWindow.on('close', (e) => {
console.log(`[diag][main] mainWindow close (quitInitiated=${quitInitiated})`);
// macOS close-to-dock: a close that is not part of a real quit (Cmd+Q,
// dock Quit, logout, updater — all fire before-quit first, flipping
// quitInitiated) gets prevented and the window HIDES instead. This keeps
// the renderer, webviews, and running agents fully alive, so both the
// user's Cmd+W/red-X and the un-attributed programmatic closer behind
// the 1.2.77 self-quits cost nothing: the next dock click shows the
// same window back instantly (`activate` below). Real quits must pass
// through — preventing a close during app.quit() cancels the quit
// (Electron semantics), which is exactly what the flag guards against.
// isInstallingUpdate must pass through: native quitAndInstall TERMINATES the app
// by closing the window (with quitInitiated still false), so intercepting it here
// hides the window and strands the update uninstalled. That was THE bug behind
// "Restart & Update does nothing" on Mac. Let that close (and real quits) through.
// macOS: the only way to land here with quitInitiated still false is the red
// traffic-light button. Cmd+W is swallowed in before-input-event, renderer
// window.close() is neutered above, and crash-recovery uses destroy() (which
// skips 'close'). So a red-button click means "quit": route it through
// app.quit() so before-quit drains the App Builder subprocesses and will-quit
// kills the backend, instead of leaving a headless app running. Real quits
// (Cmd+Q, dock Quit, logout) flip quitInitiated via before-quit first and pass
// straight through. isInstallingUpdate must also pass through: native
// quitAndInstall closes the window with quitInitiated still false, and
// intercepting it strands the update (THE "Restart & Update does nothing" bug).
if (process.platform === 'darwin' && !quitInitiated && !isInstallingUpdate) {
e.preventDefault();
// A staged update waiting + a user close = "apply it on the way out". Kick off
// the install (arms ShipIt + drives a real quit) instead of just hiding, so the
// red button finally updates instead of looping.
// A staged update waiting + a user close = "apply it on the way out": the
// install arms ShipIt and drives its own quit, so update instead of quitting.
if (cachedUpdateStatus && cachedUpdateStatus.status === 'downloaded') {
console.log('[updater] close with a staged update; applying it');
installDownloadedUpdate();
return;
}
try {
if (thisWindow.isFullScreen()) {
// Hiding a fullscreen window strands a black space; leave
// fullscreen first, then hide once the transition lands.
thisWindow.once('leave-full-screen', () => { try { thisWindow.hide(); } catch (_) {} });
thisWindow.setFullScreen(false);
} else {
thisWindow.hide();
}
console.log('[diag][main] close intercepted, window hidden (app + agents stay alive)');
} catch (_) {}
console.log('[diag][main] red-button close, quitting app');
app.quit();
}
});
mainWindow.on('closed', () => {
@@ -1770,7 +1765,13 @@ app.whenReady().then(async () => {
console.log(`[drm-req] ${details.method} ${details.url}`);
for (const [k, v] of Object.entries(details.requestHeaders || {})) {
if (/content-type|origin|referer|auth|accept/i.test(k)) {
console.log(`[drm-req] ${k}: ${v}`);
// Keep the auth scheme for debugging, never the token itself.
let safe = v;
if (/authorization/i.test(k)) {
const sp = String(v).indexOf(' ');
safe = sp > 0 ? `${String(v).slice(0, sp)} <redacted>` : '<redacted>';
}
console.log(`[drm-req] ${k}: ${safe}`);
}
}
},
@@ -1929,7 +1930,34 @@ app.whenReady().then(async () => {
}
});
// Cmd+W is the default menu's "File > Close Window". Now that the red button
// routes a close into app.quit(), an unguarded Cmd+W would tear down the whole
// app + every running agent on a stray tab-close reflex (the exact 1.2.77
// self-quit class). preventDefault here also blocks the menu accelerator
// (electron/electron#19279), and because macOS dispatches that accelerator
// against whichever webContents is focused, we have to guard the main window AND
// its webview guests, not just one. mac-only; on Windows Ctrl+W is input.control
// so this no-ops there and leaves that platform's close-on-last-window intact.
function swallowCloseWindowShortcut(event, input) {
if (
input.type === 'keyDown' &&
process.platform === 'darwin' &&
input.meta && !input.control && !input.alt &&
(input.key || '').toLowerCase() === 'w'
) {
event.preventDefault();
}
}
app.on('web-contents-created', (_event, contents) => {
// Block Cmd+W from closing the main window, whether the window chrome or one of
// its embedded webviews has focus. OAuth popups (their own 'window' contents,
// created while isCreatingMainWindow is false) are left alone so the user can
// still Cmd+W them shut.
if (isCreatingMainWindow || contents.getType() === 'webview') {
contents.on('before-input-event', swallowCloseWindowShortcut);
}
// Override the user-agent on popup BrowserWindows (i.e. anything created
// via window.open from the renderer, which includes the OAuth popup for
// subscription connect flows). Electron's default UA includes an
@@ -2086,6 +2114,7 @@ app.on('web-contents-created', (_event, contents) => {
cdpAutoAttachWired.delete(contents.id);
cdpRoutesByWcId.delete(contents.id);
webviewConsoleErrors.delete(contents.id);
cdpTearingDown.delete(contents.id);
});
contents.on('render-process-gone', () => {
@@ -2095,6 +2124,7 @@ app.on('web-contents-created', (_event, contents) => {
cdpAutoAttachWired.delete(contents.id);
cdpRoutesByWcId.delete(contents.id);
webviewConsoleErrors.delete(contents.id);
cdpTearingDown.delete(contents.id);
});
// A heavy SPA can HANG the renderer without crashing it (a render-process-gone
@@ -2204,16 +2234,13 @@ app.on('web-contents-created', (_event, contents) => {
app.on('window-all-closed', () => {
console.log(`[diag][main] window-all-closed (platform=${process.platform}${process.platform === 'darwin' ? ', staying alive' : ', quitting'})`);
// macOS: stay alive like a standard Mac app. We never install a custom
// application menu, so Electron's DEFAULT menu ships File > Close Window
// (Cmd+W) — and with a single window, quitting here turned "close the
// window" into "tear down the backend and every running agent". The 1.2.77
// prod self-quits all carried this exact signature (window close with no
// preceding before-quit). Keeping the process alive de-fangs the whole
// class: the dock icon stays, `activate` below reopens against the warm
// backend in ~1s, and the [diag][main] close-cause logging identifies the
// closer. Explicit quits (Cmd+Q, dock Quit) are untouched — Electron's
// quit pipeline runs will-quit -> killBackend regardless of this handler.
// macOS: don't quit just because the window list hit zero. The red button now
// routes through app.quit() (which drives will-quit -> killBackend itself) and
// Cmd+W is swallowed, so the only window-vanish that ISN'T already a real quit
// is an unforeseen teardown (a renderer-level destroy that skipped 'close'). For
// that stray case we stay alive as a standard Mac app rather than self-quitting
// headless, and `activate` below rebuilds the window on the next dock click. The
// 1.2.77 self-quits lived exactly here (window close with no before-quit).
if (process.platform === 'darwin') {
// An update install closed the window (native quitAndInstall) and now needs the
// process to actually die so ShipIt can swap + relaunch; finish the quit instead
@@ -2297,9 +2324,10 @@ app.on('will-quit', () => {
});
app.on('activate', () => {
// Dock-click after close-to-dock: the common case is a HIDDEN (not
// destroyed) window — just show it again; renderer, webviews, and agents
// never stopped, so this is instant and lossless.
// Live window still around (minimized, or hidden by some stray path): surface
// it instead of building a new one. The red button quits now, so the usual
// dock-click-after-close lands in the destroyed-window fallback below; this
// branch is the cheap, lossless path for the cases where a window survived.
if (mainWindow && !mainWindow.isDestroyed()) {
try {
if (mainWindow.isMinimized()) {
@@ -2563,6 +2591,25 @@ ipcMain.handle('get-install-state', () => {
}
});
// Factory reset ("Erase all content and settings"). Stop the backend FIRST so
// nothing rewrites the dir mid-wipe (on Windows a live process even locks the
// files), wipe everything under userData/data, then relaunch into a clean first
// run. install.json lives OUTSIDE /data so the install + affiliate identity
// survives, exactly like a real reinstall would. Best-effort throughout: a
// failed kill or wipe still relaunches rather than wedging the user.
ipcMain.handle('hard-reset', async () => {
try { killBackend(); } catch (e) { console.error('[hard-reset] killBackend failed', e); }
try {
const dataDir = path.join(app.getPath('userData'), 'data');
fs.rmSync(dataDir, { recursive: true, force: true });
console.log('[hard-reset] wiped data dir');
} catch (e) {
console.error('[hard-reset] wipe failed', e);
}
app.relaunch();
app.exit(0);
});
// ---------------------------------------------------------------------------
// CDP debugger bridge for the browser sub-agent
// ---------------------------------------------------------------------------
@@ -2583,6 +2630,10 @@ const cdpChildSessions = new Map(); // wcId -> Map<sessionId, {frameId, parent
const cdpAutoAttachWired = new Set(); // wcIds whose 'message' listener is attached
const cdpRoutesByWcId = new Map(); // wcId -> Map<routeKey, entry> (tier-2 shadow-API capture)
const webviewConsoleErrors = new Map(); // wcId -> [{level,message,source,line}] capped warn+error, read via BrowserGetConsole
// wcIds whose CDP is being cleanly detached on the way to destruction. Blocks a
// late agent command from RE-attaching (and re-enabling Network/auto-attach) as
// the webview tears down, which would re-arm the freed-DevToolsSession SIGSEGV.
const cdpTearingDown = new Set();
function wireChildSessions(wc) {
const wcId = wc.id;
@@ -2612,11 +2663,14 @@ function wireChildSessions(wc) {
parentSessionId: sessionId || null,
url: info.url || '',
});
// Enable perception + network domains and propagate auto-attach into nested OOPIF.
// Enable perception domains on the child + propagate auto-attach into nested OOPIF.
// Deliberately NO Network.enable here: a child iframe churns constantly, and a
// Network notification arriving after the child detaches lands on a freed session
// and SIGSEGVs the browser process (the mid-browse crash). Root Network still
// captures the page's own routes; we only forgo transient child-iframe routes.
const sid = params.sessionId;
wc.debugger.sendCommand('Accessibility.enable', {}, sid).catch(() => {});
wc.debugger.sendCommand('DOM.enable', {}, sid).catch(() => {});
wc.debugger.sendCommand('Network.enable', {}, sid).catch(() => {});
wc.debugger.sendCommand('Target.setAutoAttach',
{ autoAttach: true, waitForDebuggerOnStart: false, flatten: true }, sid).catch(() => {});
} else if (method === 'Target.detachedFromTarget') {
@@ -2635,6 +2689,11 @@ async function ensureDebuggerAttached(wc) {
if (!wc || wc.isDestroyed()) {
throw new Error('webContents is destroyed');
}
// Once a clean teardown has started, never re-attach: a re-attach here would
// re-enable Network + auto-attach right as the session is being freed.
if (cdpTearingDown.has(wc.id)) {
throw new Error('webContents is tearing down');
}
if (wc.debugger.isAttached()) return;
try {
wc.debugger.attach('1.3');
@@ -2656,6 +2715,25 @@ async function ensureDebuggerAttached(wc) {
} catch (_) {}
}
// Cleanly tear the DevTools session down BEFORE the webContents is destroyed:
// turn off the two churn-prone domains (auto-attach, Network) so no child
// sessions or network observers are live when Chromium frees the session, then
// detach. Without this, a notification in the mojo pipe lands on a freed
// DevToolsSession on the browser main thread and SIGSEGVs the whole app.
// Bounded + fail-open: a wedged pipe must never block the card from closing.
async function detachCdpCleanly(wc) {
if (!wc || wc.isDestroyed()) return;
cdpTearingDown.add(wc.id);
let attached = false;
try { attached = wc.debugger.isAttached(); } catch (_) { return; }
if (!attached) return;
const drain = (method, params) =>
raceCdp(wc.debugger.sendCommand(method, params || {}), 1200, method).catch(() => {});
await drain('Target.setAutoAttach', { autoAttach: false, waitForDebuggerOnStart: false, flatten: true });
await drain('Network.disable', {});
try { wc.debugger.detach(); } catch (_) { /* already detached / gone */ }
}
// debugger.sendCommand can hang FOREVER when the target's pipe breaks without
// a detach event (renderer process swap, wedged guest). Unraced, one hung call
// poisons the per-card queue and every later command "times out" while
@@ -2720,6 +2798,15 @@ ipcMain.handle('send-cdp-command', async (_event, wcId, method, params, sessionI
}
});
// Called by the renderer right before a browser card unmounts, so its CDP
// session is drained + detached while the webContents is still alive.
ipcMain.handle('cdp-detach-clean', async (_event, wcId) => {
try {
await detachCdpCleanly(getWebContentsById(wcId));
} catch (_) { /* fail-open: never block the card's teardown */ }
return { ok: true };
});
// Renderer-side AX index cache helpers — the renderer stores its own copy
// keyed by (browser_id, tab_id). The main process only stores per-wcId for
// invalidation purposes.
electron/native/mouseclamp/mouseclamp.mm
+14 -3
View File
@@ -24,9 +24,20 @@ static NSEvent *ClampOffWindowRelease(NSEvent *event) {
return event;
}
NSWindow *win = [event window];
NSPoint p;
if (win && [win contentView]) {
// Normal case: the captured window rode along on the event.
p = [event locationInWindow];
} else {
// The original gap: a release off the source window (easy with a second
// display) can arrive with no window attached, so the old code fail-opened
// here and the crash slipped through. Fall back to the key/main window and
// map the screen-space location into it so we can still snap it.
win = [NSApp keyWindow] ?: [NSApp mainWindow];
if (!win || ![win contentView]) return event;
p = [win convertPointFromScreen:[event locationInWindow]];
}
NSView *content = [win contentView];
if (!content) return event;
NSPoint p = [event locationInWindow];
NSSize ws = [win frame].size;
NSRect cb = [content frame];
// all the misfire-prone arithmetic lives in clamp_decision() so the property
@@ -40,7 +51,7 @@ static NSEvent *ClampOffWindowRelease(NSEvent *event) {
location:NSMakePoint(d.x, d.y)
modifierFlags:[event modifierFlags]
timestamp:[event timestamp]
windowNumber:[event windowNumber]
windowNumber:[win windowNumber]
context:nil
eventNumber:[event eventNumber]
clickCount:[event clickCount]
electron/package.json
+9 -1
View File
@@ -1,6 +1,6 @@
{
"name": "openswarm",
"version": "1.2.84",
"version": "1.4.2",
"description": "OpenSwarm — AI Agent Orchestrator",
"author": "openswarm-ai",
"main": "main.js",
@@ -41,6 +41,13 @@
"directories": {
"output": "dist"
},
"files": [
"**/*",
"!python-env",
"!python-env/**",
"!build-staging",
"!build-staging/**"
],
"icon": "build/icon.png",
"mac": {
"icon": "build/icon.icns",
@@ -50,6 +57,7 @@
],
"category": "public.app-category.developer-tools",
"hardenedRuntime": true,
"notarize": false,
"entitlements": "build/entitlements.mac.plist",
"entitlementsInherit": "build/entitlements.mac.plist",
"extraResources": [
electron/preload.js
+3
View File
@@ -55,8 +55,11 @@ contextBridge.exposeInMainWorld('openswarm', {
// Renderer attaches the ref to Stripe checkout + sign-in flows so
// the cloud can credit the affiliate. Resolves to {} if no state yet.
getInstallState: () => ipcRenderer.invoke('get-install-state'),
// Factory reset: wipes the data dir and relaunches. Never resolves on success (the app exits first).
hardReset: () => ipcRenderer.invoke('hard-reset'),
connectSlack: () => ipcRenderer.invoke('connect-slack'),
sendCdpCommand: (wcId, method, params, sessionId) => ipcRenderer.invoke('send-cdp-command', wcId, method, params, sessionId),
cdpDetachClean: (wcId) => ipcRenderer.invoke('cdp-detach-clean', wcId),
cdpCacheSet: (wcId, indexMap) => ipcRenderer.invoke('cdp-cache-set', wcId, indexMap),
cdpCacheGet: (wcId) => ipcRenderer.invoke('cdp-cache-get', wcId),
cdpCacheClear: (wcId) => ipcRenderer.invoke('cdp-cache-clear', wcId),

Some files were not shown because too many files have changed in this diff Show More