From ace783f4ef45e914311f442a4863bf05a71f4bb6 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Sun, 14 Jun 2026 05:55:28 -0700 Subject: [PATCH] [eric] swarm: .swarm bundle engine + skill export/import endpoints --- backend/apps/swarm/__init__.py | 0 backend/apps/swarm/closure.py | 315 ++++++++++++++++++++++++ backend/apps/swarm/entities/__init__.py | 0 backend/apps/swarm/entities/skills.py | 95 +++++++ backend/apps/swarm/exportable.py | 52 ++++ backend/apps/swarm/models.py | 133 ++++++++++ backend/apps/swarm/redact.py | 81 ++++++ backend/apps/swarm/registry.py | 22 ++ backend/apps/swarm/swarm.py | 128 ++++++++++ backend/apps/swarm/ziputil.py | 121 +++++++++ backend/main.py | 3 +- 11 files changed, 949 insertions(+), 1 deletion(-) create mode 100644 backend/apps/swarm/__init__.py create mode 100644 backend/apps/swarm/closure.py create mode 100644 backend/apps/swarm/entities/__init__.py create mode 100644 backend/apps/swarm/entities/skills.py create mode 100644 backend/apps/swarm/exportable.py create mode 100644 backend/apps/swarm/models.py create mode 100644 backend/apps/swarm/redact.py create mode 100644 backend/apps/swarm/registry.py create mode 100644 backend/apps/swarm/swarm.py create mode 100644 backend/apps/swarm/ziputil.py diff --git a/backend/apps/swarm/__init__.py b/backend/apps/swarm/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/apps/swarm/closure.py b/backend/apps/swarm/closure.py new file mode 100644 index 00000000..937f3635 --- /dev/null +++ b/backend/apps/swarm/closure.py @@ -0,0 +1,315 @@ +"""Export = walk the dependency closure from a root, scrub, pack. Import = stage +into a sandbox, topo-sort leaves-first, assign fresh local ids, rewrite cross +refs through a RemapTable. The single-skill staging path lets a bare .md or a +zip-of-SKILL.md come in through the same commit machinery as a full .swarm.""" +from __future__ import annotations + +import io +import json +import os +import shutil +import tempfile +import zipfile +from datetime import datetime, timezone +from uuid import uuid4 + +from .exportable import RemapTable +from .models import ( + FORMAT_VERSION, + BundlePreview, + BundleSummary, + DependencyEdge, + EntityRef, + EntityType, + IncludeItem, + Manifest, + Requirement, + RequirementView, +) +from .redact import scrub_payload +from .registry import IMPORT_ORDER, get_exportable +from .ziputil import MANIFEST_NAME, BundleError, has_member, is_zip, pack, read_manifest, unpack + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _created_with() -> str: + return os.environ.get("OPENSWARM_VERSION") or "OpenSwarm" + + +class _Ctx: + def __init__(self, local_to_bundle: dict[tuple, str]): + self._m = local_to_bundle + + def bundle_id_for(self, etype: EntityType, local_id: str) -> str | None: + return self._m.get((etype, local_id)) + + +# ---------- export ---------- + +def _assemble(root_type: EntityType, root_id: str): + root_cls = get_exportable(root_type) + if root_cls is None: + raise BundleError(f"can't share a {root_type.value} yet") + root = root_cls.load(root_id) + if root is None: + raise BundleError("nothing found to share") + + nodes: dict[tuple, object] = {} + order: list[tuple] = [] + queue: list[tuple] = [(root_type, root_id, root)] + while queue: + etype, lid, inst = queue.pop(0) + key = (etype, lid) + if key in nodes: + continue + nodes[key] = inst + order.append(key) + for dep in inst.dependencies(): + dkey = (dep.type, dep.local_id) + if dkey in nodes: + continue + dcls = get_exportable(dep.type) + if dcls is None: + raise BundleError(f"can't bundle a dependency of type {dep.type.value} yet") + dinst = dcls.load(dep.local_id) + if dinst is not None: + queue.append((dep.type, dep.local_id, dinst)) + + local_to_bundle = {key: uuid4().hex for key in order} + ctx = _Ctx(local_to_bundle) + payloads: dict[str, dict] = {} + files: dict[str, bytes] = {} + entities: list[EntityRef] = [] + edges: list[DependencyEdge] = [] + requirements: list[Requirement] = [] + counts: dict[str, int] = {} + + for key in order: + etype, _lid = key + inst = nodes[key] + bid = local_to_bundle[key] + payloads[bid] = scrub_payload(inst.serialize(ctx)) + for rel, data in inst.files().items(): + files[f"entities/{bid}/files/{rel}"] = data + entities.append(EntityRef(type=etype, bundle_id=bid, name=inst.name, path=f"entities/{bid}")) + counts[etype.value] = counts.get(etype.value, 0) + 1 + for dep in inst.dependencies(): + dkey = (dep.type, dep.local_id) + if dkey in local_to_bundle: + edges.append(DependencyEdge(from_=bid, to=local_to_bundle[dkey], relation=dep.relation)) + requirements.extend(inst.requirements()) + + requirements = _dedupe_requirements(requirements) + root_bid = local_to_bundle[(root_type, root_id)] + manifest = Manifest( + created_with=_created_with(), + created_at=_now(), + bundle_id=uuid4().hex, + root=EntityRef(type=root_type, bundle_id=root_bid, name=root.name, path=f"entities/{root_bid}"), + entities=entities, + edges=edges, + requirements=requirements, + preview=BundlePreview( + root_type=root_type, + root_name=root.name, + counts=counts, + requirement_summary=[r.label for r in requirements], + ), + ) + return manifest, payloads, files + + +def build_manifest(root_type: EntityType, root_id: str) -> Manifest: + return _assemble(root_type, root_id)[0] + + +def build_bundle(root_type: EntityType, root_id: str) -> tuple[bytes, str]: + manifest, payloads, files = _assemble(root_type, root_id) + raw = pack(manifest.model_dump(by_alias=True, mode="json"), payloads, files) + return raw, manifest.root.name + + +def _dedupe_requirements(reqs: list[Requirement]) -> list[Requirement]: + out: dict[tuple, Requirement] = {} + for r in reqs: + k = (r.kind, r.key) + if k in out: + for ref in r.referenced_by: + if ref not in out[k].referenced_by: + out[k].referenced_by.append(ref) + else: + out[k] = r + return list(out.values()) + + +# ---------- summary (shared by export + import preflight) ---------- + +def summarize(manifest: Manifest) -> BundleSummary: + includes = [ + IncludeItem(type=e.type, name=e.name) + for e in manifest.entities + if e.bundle_id != manifest.root.bundle_id + ] + reqs = [RequirementView(kind=r.kind, key=r.key, label=r.label, detail=r.detail) for r in manifest.requirements] + return BundleSummary( + root=IncludeItem(type=manifest.root.type, name=manifest.root.name), + includes=includes, + requirements=reqs, + counts=manifest.preview.counts, + ) + + +def swarm_filename(name: str) -> str: + keep = "".join(c if (c.isalnum() or c in " -_") else "" for c in (name or "bundle")).strip() + slug = keep.replace(" ", "-").lower() or "bundle" + return f"{slug}.swarm" + + +# ---------- import: staging ---------- + +def stage_upload(raw: bytes, filename: str) -> tuple[str, Manifest, list[str]]: + warnings: list[str] = [] + if is_zip(raw): + if has_member(raw, MANIFEST_NAME): + sandbox = unpack(raw) + try: + manifest = Manifest(**read_manifest(sandbox)) + except BundleError: + shutil.rmtree(sandbox, ignore_errors=True) + raise + except Exception: + shutil.rmtree(sandbox, ignore_errors=True) + raise BundleError("bundle manifest is invalid") + if manifest.format_version > FORMAT_VERSION: + shutil.rmtree(sandbox, ignore_errors=True) + raise BundleError("this .swarm was made by a newer OpenSwarm; please update") + return sandbox, manifest, warnings + return _stage_skill_from_zip(raw, filename, warnings) + return _stage_skill_from_markdown(raw, filename, warnings) + + +def _name_from_filename(filename: str) -> str: + base = os.path.splitext(os.path.basename(filename or "skill"))[0] + return base.replace("-", " ").replace("_", " ").strip().title() or "Imported Skill" + + +def _stage_skill_from_markdown(raw: bytes, filename: str, warnings: list[str]): + try: + content = raw.decode("utf-8") + except UnicodeDecodeError: + raise BundleError("unrecognized file; expected a .swarm or a .md skill") + return _synth_single_skill(content, _name_from_filename(filename), warnings) + + +def _stage_skill_from_zip(raw: bytes, filename: str, warnings: list[str]): + with zipfile.ZipFile(io.BytesIO(raw)) as zf: + mds = [n for n in zf.namelist() if n.lower().endswith(".md") and not n.endswith("/")] + target = next((n for n in mds if os.path.basename(n).lower() == "skill.md"), None) + if target is None and mds: + target = mds[0] + if target is None: + raise BundleError("zip has no SKILL.md") + content = zf.read(target).decode("utf-8", errors="replace") + others = [n for n in zf.namelist() if not n.endswith("/") and n != target] + if others: + warnings.append("supporting files were not imported (a skill is a single markdown file)") + return _synth_single_skill(content, _name_from_filename(filename), warnings) + + +def _synth_single_skill(content: str, name: str, warnings: list[str]): + bid = uuid4().hex + sandbox = tempfile.mkdtemp(prefix="swarm-import-") + edir = os.path.join(sandbox, "entities", bid) + os.makedirs(edir, exist_ok=True) + slug = name.lower().replace(" ", "-") + payload = {"slug": slug, "name": name, "description": "", "command": slug, "content": content, "builtin": False} + with open(os.path.join(edir, "payload.json"), "w", encoding="utf-8") as f: + json.dump(payload, f) + ref = EntityRef(type=EntityType.skill, bundle_id=bid, name=name, path=f"entities/{bid}") + manifest = Manifest( + bundle_id=uuid4().hex, + root=ref, + entities=[ref], + preview=BundlePreview(root_type=EntityType.skill, root_name=name, counts={"skill": 1}), + ) + return sandbox, manifest, warnings + + +# ---------- import: commit ---------- + +def _safe_join(sandbox: str, rel: str) -> str: + dest = os.path.realpath(os.path.join(sandbox, rel)) + root = os.path.realpath(sandbox) + if dest != root and not dest.startswith(root + os.sep): + raise BundleError("bundle manifest references a path outside the bundle") + return dest + + +def _read_payload(sandbox: str, ref: EntityRef) -> dict: + path = _safe_join(sandbox, os.path.join(ref.path, "payload.json")) + with open(path, encoding="utf-8") as f: + return json.load(f) + + +def _read_files(sandbox: str, ref: EntityRef) -> dict[str, bytes]: + base = _safe_join(sandbox, os.path.join(ref.path, "files")) + out: dict[str, bytes] = {} + if not os.path.isdir(base): + return out + for root, _dirs, fnames in os.walk(base): + for fn in fnames: + full = os.path.join(root, fn) + with open(full, "rb") as f: + out[os.path.relpath(full, base)] = f.read() + return out + + +def detect_conflicts(sandbox: str, manifest: Manifest) -> list[IncludeItem]: + out: list[IncludeItem] = [] + for e in manifest.entities: + cls = get_exportable(e.type) + check = getattr(cls, "conflict", None) if cls else None + if not check: + continue + msg = check(_read_payload(sandbox, e)) + if msg: + out.append(IncludeItem(type=e.type, name=e.name, detail=msg)) + return out + + +def _topo_order(manifest: Manifest) -> list[EntityRef]: + entities = {e.bundle_id: e for e in manifest.entities} + deps: dict[str, set[str]] = {bid: set() for bid in entities} + for edge in manifest.edges: + if edge.from_ in entities and edge.to in entities: + deps[edge.from_].add(edge.to) + tier = {t: i for i, t in enumerate(IMPORT_ORDER)} + result: list[EntityRef] = [] + done: set[str] = set() + remaining = set(entities) + while remaining: + ready = [b for b in remaining if deps[b] <= done] or list(remaining) + ready.sort(key=lambda b: tier.get(entities[b].type, 99)) + nxt = ready[0] + result.append(entities[nxt]) + done.add(nxt) + remaining.discard(nxt) + return result + + +def commit(sandbox: str, manifest: Manifest, accept_requirements: list[str]): + remap = RemapTable() + created: dict[str, list[str]] = {} + for e in _topo_order(manifest): + cls = get_exportable(e.type) + if cls is None: + raise BundleError(f"can't import a {e.type.value} yet") + new_id = cls.import_(_read_payload(sandbox, e), _read_files(sandbox, e), remap) + remap.assign(e.bundle_id, new_id) + created.setdefault(e.type.value, []).append(new_id) + accepted = set(accept_requirements) + unresolved = [r for r in manifest.requirements if r.key not in accepted] + return manifest.root.type, remap.local(manifest.root.bundle_id), created, unresolved diff --git a/backend/apps/swarm/entities/__init__.py b/backend/apps/swarm/entities/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/apps/swarm/entities/skills.py b/backend/apps/swarm/entities/skills.py new file mode 100644 index 00000000..cb8fbf06 --- /dev/null +++ b/backend/apps/swarm/entities/skills.py @@ -0,0 +1,95 @@ +"""SkillExportable: skills are leaves (no deps, no requirements). An installed +skill is just a markdown file plus index metadata, so this also powers the +generic "import a .md or a zip-of-SKILL.md" path. Nothing here is secret, but +the body still rides the central scrub in case someone pasted a token into it.""" +from __future__ import annotations + +import os + +from backend.apps.skills import skills as store +from ..exportable import DepRef, ExportContext, RemapTable +from ..models import EntityType, Requirement + + +class SkillExportable: + type = EntityType.skill + + def __init__(self, local_id: str, name: str, payload: dict): + self.local_id = local_id + self.name = name + self._payload = payload + + @classmethod + def load(cls, local_id: str) -> "SkillExportable | None": + fpath = os.path.join(store.SKILLS_DIR, f"{local_id}.md") + if not os.path.isfile(fpath): + return None + with open(fpath, encoding="utf-8") as f: + content = f.read() + meta = store._load_index().get(local_id, {}) + name = meta.get("name") or local_id.replace("-", " ").replace("_", " ").title() + payload = { + "slug": local_id, + "name": name, + "description": meta.get("description", ""), + "command": meta.get("command", local_id), + "content": content, + "builtin": bool(meta.get("built_in", False)), + } + return cls(local_id, name, payload) + + def serialize(self, ctx: ExportContext) -> dict: + return dict(self._payload) + + def files(self) -> dict[str, bytes]: + return {} + + def dependencies(self) -> list[DepRef]: + return [] + + def requirements(self) -> list[Requirement]: + return [] + + @classmethod + def conflict(cls, payload: dict) -> str | None: + slug = payload.get("slug") or "" + if slug and _slug_taken(slug): + return "already exists; will be added as a copy" + return None + + @classmethod + def import_(cls, payload: dict, files: dict[str, bytes], remap: RemapTable) -> str: + base = (payload.get("slug") or payload.get("name") or "skill").lower().replace(" ", "-") + slug = _free_slug(base) + os.makedirs(store.SKILLS_DIR, exist_ok=True) + fpath = os.path.join(store.SKILLS_DIR, f"{slug}.md") + with open(fpath, "w", encoding="utf-8") as f: + f.write(payload.get("content", "")) + index = store._load_index() + # Imported skills are never builtin, even if the source tagged them so. + index[slug] = { + "name": payload.get("name", slug), + "description": payload.get("description", ""), + "command": payload.get("command", slug), + } + store._save_index(index) + return slug + + +def _slug_taken(slug: str) -> bool: + return slug in store._load_index() or os.path.isfile( + os.path.join(store.SKILLS_DIR, f"{slug}.md") + ) + + +def _free_slug(base: str) -> str: + base = base or "skill" + if not _slug_taken(base): + return base + cand = f"{base}-imported" + if not _slug_taken(cand): + return cand + i = 2 + while _slug_taken(f"{base}-imported-{i}"): + i += 1 + return f"{base}-imported-{i}" diff --git a/backend/apps/swarm/exportable.py b/backend/apps/swarm/exportable.py new file mode 100644 index 00000000..506662c2 --- /dev/null +++ b/backend/apps/swarm/exportable.py @@ -0,0 +1,52 @@ +"""The one abstraction every shareable thing implements. Export walks +dependencies() into a closure; import calls import_() leaves-first, rewiring +cross-refs through the RemapTable. Secret redaction is centralized in closure + +ziputil so a new entity physically can't forget to scrub itself.""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import ClassVar, Protocol, runtime_checkable + +from .models import EntityType, Requirement + + +@dataclass +class DepRef: + """A local reference one entity holds to another, before bundling.""" + type: EntityType + local_id: str + relation: str = "" + + +class ExportContext(Protocol): + # Lets an entity rewrite its own cross-refs from local ids to bundle ids. + def bundle_id_for(self, etype: EntityType, local_id: str) -> str | None: ... + + +class RemapTable: + """bundle_id -> fresh local id, filled as import walks entities leaves-first.""" + + def __init__(self) -> None: + self._m: dict[str, str] = {} + + def assign(self, bundle_id: str, local_id: str) -> None: + self._m[bundle_id] = local_id + + def local(self, bundle_id: str) -> str | None: + return self._m.get(bundle_id) + + +@runtime_checkable +class Exportable(Protocol): + type: ClassVar[EntityType] + local_id: str + name: str + + @classmethod + def load(cls, local_id: str) -> "Exportable | None": ... + def serialize(self, ctx: ExportContext) -> dict: ... + def files(self) -> dict[str, bytes]: ... + def dependencies(self) -> list[DepRef]: ... + def requirements(self) -> list[Requirement]: ... + @classmethod + def import_(cls, payload: dict, files: dict[str, bytes], remap: RemapTable) -> str: ... diff --git a/backend/apps/swarm/models.py b/backend/apps/swarm/models.py new file mode 100644 index 00000000..65ff68ef --- /dev/null +++ b/backend/apps/swarm/models.py @@ -0,0 +1,133 @@ +"""Schema for the .swarm bundle: a hardened zip whose manifest.json is a +dependency graph of entities with one designated root. The manifest never +carries secrets or payloads (payloads live as files in the zip). The *View +models are the lighter, frontend-facing shapes the share/import modals read.""" +from enum import Enum +from typing import Any, Literal, Optional + +from pydantic import BaseModel, ConfigDict, Field + +FORMAT_VERSION = 1 + + +class EntityType(str, Enum): + skill = "skill" + app = "app" + workflow = "workflow" + dashboard = "dashboard" + mode = "mode" + session = "session" + + +class RequirementKind(str, Enum): + mcp_action = "mcp_action" # an MCP/Action that must be reconnected (never auto) + setting = "setting" # a safe settings fragment the user confirms + builtin_mode = "builtin_mode" # a builtin mode that must already exist locally + api_key = "api_key" # a provider key the bundle needs but can't carry + custom_provider = "custom_provider" # OpenAI-compatible endpoint (URL ssrf-checked) + + +class EntityRef(BaseModel): + type: EntityType + bundle_id: str # uuid4 hex, stable only within this bundle + name: str + path: str # dir inside the zip holding this entity + + +class DependencyEdge(BaseModel): + model_config = ConfigDict(populate_by_name=True) + from_: str = Field(alias="from") + to: str + relation: str = "" + + +class Requirement(BaseModel): + kind: RequirementKind + key: str + label: str + detail: str = "" + referenced_by: list[str] = Field(default_factory=list) + proposal: dict[str, Any] = Field(default_factory=dict) # safe, non-secret hint only + + +class BundlePreview(BaseModel): + root_type: EntityType + root_name: str + counts: dict[str, int] = Field(default_factory=dict) + requirement_summary: list[str] = Field(default_factory=list) + + +class Manifest(BaseModel): + format_version: int = FORMAT_VERSION + created_with: str = "OpenSwarm" + created_at: str = "" + bundle_id: str + root: EntityRef + entities: list[EntityRef] = Field(default_factory=list) + edges: list[DependencyEdge] = Field(default_factory=list) + requirements: list[Requirement] = Field(default_factory=list) + preview: BundlePreview + + +# ---- frontend-facing summary (export + import preflight) ---- + +class IncludeItem(BaseModel): + type: EntityType + name: str + detail: str = "" + + +class RequirementView(BaseModel): + kind: RequirementKind + key: str + label: str + detail: str = "" + + +class BundleSummary(BaseModel): + root: IncludeItem + includes: list[IncludeItem] = Field(default_factory=list) + requirements: list[RequirementView] = Field(default_factory=list) + counts: dict[str, int] = Field(default_factory=dict) + + +class ReviewSummary(BaseModel): + verdict: Literal["clean", "warn", "block"] = "clean" + findings: list[str] = Field(default_factory=list) + scanned_files: list[str] = Field(default_factory=list) + + +# ---- endpoint request/response ---- + +class ExportRequest(BaseModel): + type: EntityType + id: str + + +class ExportPreflightResponse(BaseModel): + ok: bool = True + summary: BundleSummary + filename: str + link_supported: bool = False + + +class ImportPreflightResponse(BaseModel): + ok: bool = True + summary: BundleSummary + staging_token: str + conflicts: list[IncludeItem] = Field(default_factory=list) + review: Optional[ReviewSummary] = None + warnings: list[str] = Field(default_factory=list) + + +class ImportCommitRequest(BaseModel): + staging_token: str + accept_requirements: list[str] = Field(default_factory=list) + + +class ImportCommitResponse(BaseModel): + ok: bool = True + root_type: EntityType + root_id: str + created: dict[str, list[str]] = Field(default_factory=dict) + unresolved_requirements: list[RequirementView] = Field(default_factory=list) diff --git a/backend/apps/swarm/redact.py b/backend/apps/swarm/redact.py new file mode 100644 index 00000000..1a97d20c --- /dev/null +++ b/backend/apps/swarm/redact.py @@ -0,0 +1,81 @@ +"""Strip secrets before anything enters a .swarm. Two layers: closure scrubs +every payload + text body, and ziputil.pack refuses to write if anything denied +slipped through. Over-redacting a bundle is fine; shipping a stranger your API +key is not.""" +from __future__ import annotations + +import re +from typing import Any + +# Substrings that mark a field name as secret (matched case-insensitively). +_DENY_SUBSTRINGS = ( + "api_key", "apikey", "secret", "password", "passwd", "credential", "oauth", + "bearer", "subscription_token", "access_token", "refresh_token", + "session_token", "auth_token", "private_key", +) + +# Exact field names that are sensitive or per-install identity (the substring +# pass alone would miss these). +_DENY_EXACT = { + "token", "installation_id", "user_id", "free_trial_token", + "free_trial_remaining", "free_trial_runs_limit", "openswarm_bearer_token", + "openswarm_usage_cached", "connected_account_email", "oauth_tokens", + "credentials", "sdk_session_id", +} + +REDACTED = "[redacted]" + +# Literal-secret shapes someone might paste into a file or skill body. +_CONTENT_PATTERNS = ( + re.compile(r"sk-ant-[A-Za-z0-9_\-]{16,}"), + re.compile(r"sk-[A-Za-z0-9_\-]{16,}"), + re.compile(r"AIza[A-Za-z0-9_\-]{20,}"), # Google API key shape + re.compile(r"gh[pousr]_[A-Za-z0-9]{20,}"), # GitHub tokens + re.compile(r"Bearer\s+[A-Za-z0-9._\-]{16,}"), +) + + +def is_denied_key(key: str) -> bool: + k = key.lower() + if k in _DENY_EXACT: + return True + return any(sub in k for sub in _DENY_SUBSTRINGS) + + +def scrub_text(text: str) -> str: + for pat in _CONTENT_PATTERNS: + text = pat.sub(REDACTED, text) + return text + + +def scrub_payload(value: Any) -> Any: + """Recursively drop denied keys and redact secret-shaped strings in a + JSON-able structure. Returns a new structure; never mutates the input.""" + if isinstance(value, dict): + out: dict[str, Any] = {} + for k, v in value.items(): + if isinstance(k, str) and is_denied_key(k): + continue + out[k] = scrub_payload(v) + return out + if isinstance(value, list): + return [scrub_payload(v) for v in value] + if isinstance(value, str): + return scrub_text(value) + return value + + +def find_denied_keys(value: Any, _path: str = "") -> list[str]: + """Audit used by ziputil.pack as the last line of defense: the paths of any + denied key still present. Empty list means clean.""" + found: list[str] = [] + if isinstance(value, dict): + for k, v in value.items(): + here = f"{_path}.{k}" if _path else str(k) + if isinstance(k, str) and is_denied_key(k): + found.append(here) + found.extend(find_denied_keys(v, here)) + elif isinstance(value, list): + for i, v in enumerate(value): + found.extend(find_denied_keys(v, f"{_path}[{i}]")) + return found diff --git a/backend/apps/swarm/registry.py b/backend/apps/swarm/registry.py new file mode 100644 index 00000000..e2a6281b --- /dev/null +++ b/backend/apps/swarm/registry.py @@ -0,0 +1,22 @@ +"""Maps an EntityType to the Exportable that handles it, and the leaves-first +order import walks. Adding a shareable type is one entry here plus its module.""" +from .entities.skills import SkillExportable +from .models import EntityType + +REGISTRY: dict[EntityType, type] = { + EntityType.skill: SkillExportable, +} + +# Leaves first: a dependency must import before whatever references it. +IMPORT_ORDER = [ + EntityType.skill, + EntityType.mode, + EntityType.session, + EntityType.app, + EntityType.workflow, + EntityType.dashboard, +] + + +def get_exportable(etype: EntityType) -> type | None: + return REGISTRY.get(etype) diff --git a/backend/apps/swarm/swarm.py b/backend/apps/swarm/swarm.py new file mode 100644 index 00000000..119e64f4 --- /dev/null +++ b/backend/apps/swarm/swarm.py @@ -0,0 +1,128 @@ +"""SubApp for .swarm sharing. Three endpoints: export (returns the bundle bytes +as a download), import/preflight (parse + stage in a sandbox, no writes), and +import/commit (write the staged entities with fresh ids). Staging is in-process +with a TTL; a lost token just means re-open the file.""" +import logging +import shutil +import time +import uuid +from contextlib import asynccontextmanager + +from fastapi import File, HTTPException, Response, UploadFile + +from backend.config.Apps import SubApp + +from . import closure +from .models import ( + ExportPreflightResponse, + ExportRequest, + ImportCommitRequest, + ImportCommitResponse, + ImportPreflightResponse, + RequirementView, +) +from .ziputil import MAX_TOTAL_BYTES, BundleError + +logger = logging.getLogger(__name__) + +_STAGING: dict[str, dict] = {} +_STAGING_TTL = 30 * 60 # 30 minutes + + +def _gc_staging() -> None: + now = time.time() + for token in list(_STAGING): + if now - _STAGING[token]["created_at"] > _STAGING_TTL: + _discard(token) + + +def _discard(token: str) -> None: + entry = _STAGING.pop(token, None) + if entry: + shutil.rmtree(entry["sandbox"], ignore_errors=True) + + +@asynccontextmanager +async def swarm_lifespan(): + _gc_staging() + try: + yield + finally: + for token in list(_STAGING): + _discard(token) + + +swarm = SubApp("swarm", swarm_lifespan) + + +@swarm.router.post("/export/preflight") +async def export_preflight(body: ExportRequest) -> ExportPreflightResponse: + try: + manifest = closure.build_manifest(body.type, body.id) + except BundleError as e: + raise HTTPException(status_code=400, detail=str(e)) + return ExportPreflightResponse( + summary=closure.summarize(manifest), + filename=closure.swarm_filename(manifest.root.name), + link_supported=False, + ) + + +@swarm.router.post("/export") +async def export_bundle(body: ExportRequest) -> Response: + try: + raw, name = closure.build_bundle(body.type, body.id) + except BundleError as e: + raise HTTPException(status_code=400, detail=str(e)) + fname = closure.swarm_filename(name) + return Response( + content=raw, + media_type="application/zip", + headers={"Content-Disposition": f'attachment; filename="{fname}"'}, + ) + + +@swarm.router.post("/import/preflight") +async def import_preflight(file: UploadFile = File(...)) -> ImportPreflightResponse: + raw = await file.read() + if len(raw) > MAX_TOTAL_BYTES: + raise HTTPException(status_code=400, detail="file is too large") + try: + sandbox, manifest, warnings = closure.stage_upload(raw, file.filename or "") + conflicts = closure.detect_conflicts(sandbox, manifest) + except BundleError as e: + raise HTTPException(status_code=400, detail=str(e)) + _gc_staging() + token = uuid.uuid4().hex + _STAGING[token] = {"sandbox": sandbox, "manifest": manifest, "created_at": time.time()} + return ImportPreflightResponse( + summary=closure.summarize(manifest), + staging_token=token, + conflicts=conflicts, + warnings=warnings, + ) + + +@swarm.router.post("/import/commit") +async def import_commit(body: ImportCommitRequest) -> ImportCommitResponse: + entry = _STAGING.get(body.staging_token) + if not entry: + raise HTTPException(status_code=404, detail="import session expired; please re-open the file") + try: + root_type, root_id, created, unresolved = closure.commit( + entry["sandbox"], entry["manifest"], body.accept_requirements + ) + except BundleError as e: + raise HTTPException(status_code=400, detail=str(e)) + finally: + _discard(body.staging_token) + if root_id is None: + raise HTTPException(status_code=400, detail="bundle has no root entity") + return ImportCommitResponse( + root_type=root_type, + root_id=root_id, + created=created, + unresolved_requirements=[ + RequirementView(kind=r.kind, key=r.key, label=r.label, detail=r.detail) for r in unresolved + ], + ) diff --git a/backend/apps/swarm/ziputil.py b/backend/apps/swarm/ziputil.py new file mode 100644 index 00000000..d53033cd --- /dev/null +++ b/backend/apps/swarm/ziputil.py @@ -0,0 +1,121 @@ +"""Hardened zip <-> bytes for .swarm bundles. The zip arrives from an untrusted +party, so unpack defends against zip-slip, zip-bombs, symlinks, and lying size +headers, and only ever writes into a throwaway sandbox dir (never a real store). +pack re-checks that no secret slipped past redaction before writing a byte.""" +from __future__ import annotations + +import io +import json +import os +import shutil +import tempfile +import zipfile + +from .redact import find_denied_keys + +MANIFEST_NAME = "manifest.json" + +MAX_ENTRIES = 5000 +MAX_TOTAL_BYTES = 200 * 1024 * 1024 # 200 MB uncompressed +MAX_FILE_BYTES = 25 * 1024 * 1024 # 25 MB per entry +MAX_RATIO = 200 # uncompressed / compressed per entry + + +class BundleError(Exception): + """Bundle is malformed or unsafe. Message is safe to show the user.""" + + +def pack(manifest: dict, payloads: dict[str, dict], files: dict[str, bytes]) -> bytes: + """payloads: bundle_id -> JSON payload (-> entities//payload.json). + files: full zip path -> bytes (e.g. entities//files/).""" + for bid, payload in payloads.items(): + leaked = find_denied_keys(payload) + if leaked: + raise BundleError( + f"refusing to export: secret-shaped field(s) in {bid}: {leaked[:3]}" + ) + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: + zf.writestr(MANIFEST_NAME, json.dumps(manifest, indent=2)) + for bid, payload in sorted(payloads.items()): + zf.writestr(f"entities/{bid}/payload.json", json.dumps(payload, indent=2)) + for path, data in sorted(files.items()): + zf.writestr(path, data) + return buf.getvalue() + + +def _safe_member_path(name: str, sandbox: str) -> str: + if name.startswith(("/", "\\")) or (len(name) > 1 and name[1] == ":"): + raise BundleError("bundle contains an absolute path") + dest = os.path.realpath(os.path.join(sandbox, name)) + root = os.path.realpath(sandbox) + if dest != root and not dest.startswith(root + os.sep): + raise BundleError("bundle contains a path-traversal entry") + return dest + + +def is_zip(raw: bytes) -> bool: + return zipfile.is_zipfile(io.BytesIO(raw)) + + +def has_member(raw: bytes, name: str) -> bool: + with zipfile.ZipFile(io.BytesIO(raw)) as zf: + return name in zf.namelist() + + +def unpack(raw: bytes) -> str: + """Extract into a fresh sandbox temp dir and return it. Caller deletes it.""" + if len(raw) > MAX_TOTAL_BYTES: + raise BundleError("bundle is too large") + try: + zf = zipfile.ZipFile(io.BytesIO(raw)) + except zipfile.BadZipFile: + raise BundleError("not a valid .swarm file") + infos = zf.infolist() + if len(infos) > MAX_ENTRIES: + raise BundleError("bundle has too many entries") + total = 0 + for zi in infos: + if zi.file_size > MAX_FILE_BYTES: + raise BundleError("bundle has an oversized entry") + total += zi.file_size + if total > MAX_TOTAL_BYTES: + raise BundleError("bundle is too large uncompressed") + if zi.compress_size and zi.file_size / zi.compress_size > MAX_RATIO: + raise BundleError("bundle entry is suspiciously compressed") + mode = (zi.external_attr >> 16) & 0o170000 + if mode == 0o120000: + raise BundleError("bundle contains a symlink") + + sandbox = tempfile.mkdtemp(prefix="swarm-import-") + try: + written = 0 + for zi in infos: + if zi.is_dir(): + continue + dest = _safe_member_path(zi.filename, sandbox) + os.makedirs(os.path.dirname(dest), exist_ok=True) + with zf.open(zi) as src, open(dest, "wb") as out: + while True: + chunk = src.read(65536) + if not chunk: + break + written += len(chunk) + if written > MAX_TOTAL_BYTES: + raise BundleError("bundle exceeded size during extraction") + out.write(chunk) + except Exception: + shutil.rmtree(sandbox, ignore_errors=True) + raise + return sandbox + + +def read_manifest(sandbox: str) -> dict: + path = os.path.join(sandbox, MANIFEST_NAME) + if not os.path.isfile(path): + raise BundleError("bundle has no manifest") + try: + with open(path, encoding="utf-8") as f: + return json.load(f) + except (json.JSONDecodeError, UnicodeDecodeError): + raise BundleError("bundle manifest is unreadable") diff --git a/backend/main.py b/backend/main.py index d51267a8..42daf1d3 100644 --- a/backend/main.py +++ b/backend/main.py @@ -39,6 +39,7 @@ from backend.apps.mcp_registry.mcp_registry import mcp_registry from backend.apps.skill_registry.skill_registry import skill_registry from backend.apps.outputs.outputs import outputs from backend.apps.dashboards.dashboards import dashboards +from backend.apps.swarm.swarm import swarm from backend.apps.service.service import service from backend.apps.subscription.router import subscription from backend.apps.auth.router import auth @@ -48,7 +49,7 @@ from fastapi.middleware.cors import CORSMiddleware from fastapi import WebSocket, WebSocketDisconnect import json -main_app = MainApp([health, agents, skills, tools_lib, modes, settings, mcp_registry, skill_registry, outputs, dashboards, service, subscription, auth, web, anthropic_proxy]) +main_app = MainApp([health, agents, skills, tools_lib, modes, settings, mcp_registry, skill_registry, outputs, dashboards, swarm, service, subscription, auth, web, anthropic_proxy]) app = main_app.app # Generate per-install auth token BEFORE we bind the HTTP port. By the