From d714dfaee18596e3907d6fbe8c49ef3cc3b5e114 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Fri, 19 Jun 2026 03:17:44 -0700 Subject: [PATCH] [eric] skills: folder skills round-trip through .swarm/zip import (supporting files preserved via the entity files() channel, not flattened) --- backend/apps/swarm/closure.py | 32 +++++++++-- backend/apps/swarm/entities/skills.py | 80 ++++++++++++++++++++------- backend/tests/test_skills_folders.py | 53 ++++++++++++++++++ 3 files changed, 139 insertions(+), 26 deletions(-) diff --git a/backend/apps/swarm/closure.py b/backend/apps/swarm/closure.py index ef15cfac..becca123 100644 --- a/backend/apps/swarm/closure.py +++ b/backend/apps/swarm/closure.py @@ -236,13 +236,27 @@ def _stage_skill_from_zip(raw: bytes, filename: str, warnings: list[str]): if target is None: raise BundleError("zip has no SKILL.md") content = zf.read(target).decode("utf-8", errors="replace") - others = [n for n in zf.namelist() if not n.endswith("/") and n != target] - if others: - warnings.append("supporting files were not imported (a skill is a single markdown file)") - return _synth_single_skill(content, _name_from_filename(filename), warnings) + # Carry supporting files (scripts, templates) through as a folder skill, + # keyed relative to the SKILL.md's directory so a nested layout flattens + # onto the skill folder. Cap count + per-file size so a hostile zip can't + # balloon the install. + base_dir = target.rsplit("/", 1)[0] + "/" if "/" in target else "" + extra_files: dict[str, bytes] = {} + for n in zf.namelist(): + if n.endswith("/") or n == target: + continue + rel = n[len(base_dir):] if base_dir and n.startswith(base_dir) else os.path.basename(n) + if not rel or rel.startswith("."): + continue + info = zf.getinfo(n) + if info.file_size > 2_000_000 or len(extra_files) >= 50: + warnings.append("some oversized/extra supporting files were skipped") + continue + extra_files[rel] = zf.read(n) + return _synth_single_skill(content, _name_from_filename(filename), warnings, extra_files) -def _synth_single_skill(content: str, name: str, warnings: list[str]): +def _synth_single_skill(content: str, name: str, warnings: list[str], extra_files: dict[str, bytes] | None = None): bid = uuid4().hex sandbox = tempfile.mkdtemp(prefix="swarm-import-") edir = os.path.join(sandbox, "entities", bid) @@ -251,6 +265,14 @@ def _synth_single_skill(content: str, name: str, warnings: list[str]): payload = {"slug": slug, "name": name, "description": "", "command": slug, "content": content, "builtin": False} with open(os.path.join(edir, "payload.json"), "w", encoding="utf-8") as f: json.dump(payload, f) + # Supporting files ride the same entities//files/ channel the + # commit reader (_read_files) feeds into import_, so a zip-of-SKILL.md + # round-trips as a folder skill instead of getting flattened. + for rel, data in (extra_files or {}).items(): + dest = _safe_join(edir, os.path.join("files", rel)) + os.makedirs(os.path.dirname(dest), exist_ok=True) + with open(dest, "wb") as f: + f.write(data) ref = EntityRef(type=EntityType.skill, bundle_id=bid, name=name, path=f"entities/{bid}") manifest = Manifest( bundle_id=uuid4().hex, diff --git a/backend/apps/swarm/entities/skills.py b/backend/apps/swarm/entities/skills.py index e02b41b2..077e250a 100644 --- a/backend/apps/swarm/entities/skills.py +++ b/backend/apps/swarm/entities/skills.py @@ -1,10 +1,13 @@ -"""SkillExportable: skills are leaves (no deps, no requirements). An installed -skill is just a markdown file plus index metadata, so this also powers the -generic "import a .md or a zip-of-SKILL.md" path. Nothing here is secret, but -the body still rides the central scrub in case someone pasted a token into it.""" +"""SkillExportable: skills are leaves (no deps, no requirements). A skill is +either a single markdown file or a folder (SKILL.md + supporting files like +scripts/templates), so this powers both the .swarm round-trip AND the generic +"import a .md or a zip-of-SKILL.md" path. Folder skills ride the entity files() +channel so their supporting files survive export/import. Nothing here is secret, +but the body still rides the central scrub in case someone pasted a token in.""" from __future__ import annotations import os +import shutil from backend.apps.skills import skills as store from ..exportable import DepRef, ExportContext, RemapTable @@ -14,17 +17,18 @@ from ..models import EntityType, Requirement class SkillExportable: type = EntityType.skill - def __init__(self, local_id: str, name: str, payload: dict): + def __init__(self, local_id: str, name: str, payload: dict, files: dict[str, bytes] | None = None): self.local_id = local_id self.name = name self._payload = payload + self._files = files or {} @classmethod def load(cls, local_id: str) -> "SkillExportable | None": - fpath = os.path.join(store.SKILLS_DIR, f"{local_id}.md") - if not os.path.isfile(fpath): + md_path, kind = store._skill_md_path(local_id) + if not md_path: return None - with open(fpath, encoding="utf-8") as f: + with open(md_path, encoding="utf-8") as f: content = f.read() meta = store._load_index().get(local_id, {}) name = meta.get("name") or local_id.replace("-", " ").replace("_", " ").title() @@ -36,13 +40,16 @@ class SkillExportable: "content": content, "builtin": bool(meta.get("built_in", False)), } - return cls(local_id, name, payload) + files: dict[str, bytes] = {} + if kind == "folder": + files = _read_supporting_files(os.path.join(store.SKILLS_DIR, local_id)) + return cls(local_id, name, payload, files) def serialize(self, ctx: ExportContext) -> dict: return dict(self._payload) def files(self) -> dict[str, bytes]: - return {} + return dict(self._files) def dependencies(self) -> list[DepRef]: return [] @@ -61,35 +68,66 @@ class SkillExportable: def import_(cls, payload: dict, files: dict[str, bytes], remap: RemapTable) -> str: base = (payload.get("slug") or payload.get("name") or "skill").lower().replace(" ", "-") slug = _free_slug(base) + meta = { + "name": payload.get("name", slug), + "description": payload.get("description", ""), + "command": payload.get("command", slug), + } + # A folder skill arrives with supporting files; write the whole folder. + # A bare .md skill writes a single file as before. write_folder_skill is + # path-traversal-safe, so an untrusted bundle can't escape the skill dir. + if files: + bundle = {"SKILL.md": payload.get("content", "")} + for rel, data in files.items(): + bundle[rel] = data.decode("utf-8", errors="replace") + skill = store.write_folder_skill(slug, bundle, meta) + return skill.id os.makedirs(store.SKILLS_DIR, exist_ok=True) fpath = os.path.join(store.SKILLS_DIR, f"{slug}.md") with open(fpath, "w", encoding="utf-8") as f: f.write(payload.get("content", "")) index = store._load_index() # Imported skills are never builtin, even if the source tagged them so. - index[slug] = { - "name": payload.get("name", slug), - "description": payload.get("description", ""), - "command": payload.get("command", slug), - } + index[slug] = meta store._save_index(index) return slug - @classmethod def rollback(cls, local_id: str) -> None: - fpath = os.path.join(store.SKILLS_DIR, f"{local_id}.md") - if os.path.exists(fpath): - os.remove(fpath) + skill_dir = os.path.join(store.SKILLS_DIR, local_id) + flat = os.path.join(store.SKILLS_DIR, f"{local_id}.md") + if os.path.isdir(skill_dir): + shutil.rmtree(skill_dir, ignore_errors=True) + if os.path.isfile(flat): + os.remove(flat) index = store._load_index() if local_id in index: index.pop(local_id, None) store._save_index(index) +def _read_supporting_files(skill_dir: str) -> dict[str, bytes]: + """Every file in a skill folder except SKILL.md, as {relpath: bytes}.""" + out: dict[str, bytes] = {} + for root, _dirs, names in os.walk(skill_dir): + for n in names: + full = os.path.join(root, n) + rel = os.path.relpath(full, skill_dir) + if rel == "SKILL.md" or n.startswith("."): + continue + try: + with open(full, "rb") as f: + out[rel] = f.read() + except OSError: + continue + return out + + def _slug_taken(slug: str) -> bool: - return slug in store._load_index() or os.path.isfile( - os.path.join(store.SKILLS_DIR, f"{slug}.md") + return ( + slug in store._load_index() + or os.path.isfile(os.path.join(store.SKILLS_DIR, f"{slug}.md")) + or os.path.isdir(os.path.join(store.SKILLS_DIR, slug)) ) diff --git a/backend/tests/test_skills_folders.py b/backend/tests/test_skills_folders.py index ab46a3f3..dae4506c 100644 --- a/backend/tests/test_skills_folders.py +++ b/backend/tests/test_skills_folders.py @@ -103,3 +103,56 @@ def test_injection_no_folder_note_for_flat_skill(skills_dir): block = _resolve_attached_skills([{"id": "plain", "name": "Plain", "content": "plain content"}]) assert "[Using skill: Plain]" in block assert "supporting files" not in block.lower() + + +# --------------------------------------------------------------------------- +# .swarm round-trip for folder skills (export carries files, import rebuilds them). +# --------------------------------------------------------------------------- + +def test_swarm_export_folder_skill_carries_supporting_files(skills_dir): + from backend.apps.swarm.entities.skills import SkillExportable + base = skills_dir / "vid" + _write(str(base / "SKILL.md"), "render") + _write(str(base / "scripts" / "go.py"), "print(1)") + exp = SkillExportable.load("vid") + assert exp is not None + files = exp.files() + assert "scripts/go.py" in files + assert files["scripts/go.py"] == b"print(1)" + assert exp._payload["content"] == "render" + + +def test_swarm_import_writes_folder_when_files_present(skills_dir): + from backend.apps.swarm.entities.skills import SkillExportable + payload = {"slug": "vid", "name": "Vid", "description": "d", "command": "vid", "content": "render"} + new_id = SkillExportable.import_(payload, {"scripts/go.py": b"print(1)"}, None) + assert os.path.isfile(skills_dir / new_id / "SKILL.md") + assert os.path.isfile(skills_dir / new_id / "scripts" / "go.py") + synced = {s.id: s for s in skills_mod._sync_skills()} + assert synced[new_id].has_supporting_files is True + + +def test_swarm_import_flat_when_no_files(skills_dir): + from backend.apps.swarm.entities.skills import SkillExportable + payload = {"slug": "note", "name": "Note", "content": "just text"} + new_id = SkillExportable.import_(payload, {}, None) + assert os.path.isfile(skills_dir / f"{new_id}.md") + assert not (skills_dir / new_id).is_dir() + + +def test_stage_zip_carries_supporting_files_into_sandbox(): + import io as _io, zipfile, os as _os, shutil + from backend.apps.swarm.closure import _stage_skill_from_zip + buf = _io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + zf.writestr("my-skill/SKILL.md", "do it") + zf.writestr("my-skill/scripts/run.sh", "echo hi") + sandbox, manifest, warnings = _stage_skill_from_zip(buf.getvalue(), "my-skill.zip", []) + try: + bid = manifest.entities[0].bundle_id + files_dir = _os.path.join(sandbox, "entities", bid, "files") + assert _os.path.isfile(_os.path.join(files_dir, "scripts", "run.sh")) + # SKILL.md is the payload body, not a supporting file. + assert not _os.path.exists(_os.path.join(files_dir, "SKILL.md")) + finally: + shutil.rmtree(sandbox, ignore_errors=True)