diff --git a/backend/apps/skill_registry/skill_registry.py b/backend/apps/skill_registry/skill_registry.py index 05717d9c..ef99b10b 100644 --- a/backend/apps/skill_registry/skill_registry.py +++ b/backend/apps/skill_registry/skill_registry.py @@ -8,7 +8,8 @@ from contextlib import asynccontextmanager from typing import Optional import httpx -from fastapi import Query +from fastapi import HTTPException, Query +from pydantic import BaseModel from backend.config.Apps import SubApp logger = logging.getLogger(__name__) @@ -230,7 +231,16 @@ async def registry_search( offset: int = Query(0, ge=0), sort: str = Query("name", description="Sort by: name"), category: str = Query("", description="Filter by category"), + source: str = Query("curated", description="curated (vetted) | community (skills.sh wild registry)"), ): + # The wild registry is a remote 600k-entry index, searched live, not mirrored. + if source == "community": + try: + return await _community_search(q, limit) + except Exception as e: + logger.warning(f"community skill search failed: {e}") + return {"skills": [], "total": 0, "offset": 0, "limit": limit, "source": "community", "error": "skills.sh unreachable"} + pool = list(_cache.values()) if category: cat_lower = category.lower() @@ -268,3 +278,169 @@ async def registry_detail(skill_name: str): if not sk: return {"error": "Skill not found"}, 404 return {"skill": sk} + + +# --------------------------------------------------------------------------- +# Community source: the skills.sh wild registry (~600k+ telemetry-ranked, +# zero-curation community skills, GitHub-repo backed). The curated source above +# (anthropics/skills) stays the default; community is opt-in via ?source=community +# and the UI flags it as unvetted. See .claude/SECURITY.md for the posture: this +# installs INERT files only (never executes), discloses scripts before commit, +# and any skill script later runs through the same gated Bash path as anything. +# --------------------------------------------------------------------------- + +_COMMUNITY_SEARCH_URL = "https://skills.sh/api/search" +_GH_API = "https://api.github.com" +_GH_RAW = "https://raw.githubusercontent.com" +_MAX_SKILL_FILES = 60 +_SCRIPT_EXTS = (".sh", ".py", ".js", ".mjs", ".cjs", ".ts", ".rb", ".pl", ".ps1", ".bat", ".php") + + +def _is_script_path(rel: str) -> bool: + """Whether a skill file is executable code worth disclosing before install.""" + if rel.lower().endswith(_SCRIPT_EXTS): + return True + head = rel.split("/", 1)[0].lower() + return head in ("scripts", "bin", "hooks") + + +def _select_skill_paths(tree: list[dict], skill_id: str) -> tuple[str, list[str]]: + """From a GitHub recursive tree, pick the SKILL.md for `skill_id` (shortest + matching path) and every file living beside it. Pure so the resolution logic + is unit-tested without a network round-trip.""" + blobs = [t["path"] for t in tree if t.get("type") == "blob" and isinstance(t.get("path"), str)] + candidates = [p for p in blobs if p.endswith(f"/{skill_id}/SKILL.md") or p == f"{skill_id}/SKILL.md"] + if not candidates: + raise ValueError(f"no SKILL.md for '{skill_id}' in this repo") + skill_md = min(candidates, key=len) + skill_dir = skill_md[: -len("/SKILL.md")] if "/" in skill_md else "" + prefix = (skill_dir + "/") if skill_dir else "" + members = [p for p in blobs if (p.startswith(prefix) if prefix else "/" not in p)] + return skill_md, members[:_MAX_SKILL_FILES] + + +class RegistryRateLimited(Exception): + """GitHub's unauthenticated API (60/hr) is exhausted; the caller surfaces a + 'try again shortly' rather than a generic failure.""" + + +async def _fetch_repo_tree(client: httpx.AsyncClient, owner: str, repo: str) -> tuple[str, list[dict]]: + """Recursive tree of owner/repo, trying main then master (one API call each, + usually just one). Avoids a separate repo-meta call to halve GitHub API use. + Raises RegistryRateLimited on a 403, ValueError if no usable branch.""" + last_status = None + for branch in ("main", "master"): + r = await client.get(f"{_GH_API}/repos/{owner}/{repo}/git/trees/{branch}?recursive=1") + if r.status_code == 200: + return branch, r.json().get("tree", []) + if r.status_code == 403: + raise RegistryRateLimited() + last_status = r.status_code + raise ValueError(f"repo {owner}/{repo} has no main/master branch (last status {last_status})") + + +async def resolve_community_skill(source: str, skill_id: str) -> dict: + """Resolve a skills.sh entry (source='owner/repo', skill_id=folder name) to + its files via the GitHub trees API. Returns name/description/repo_url plus + {relpath: content} and the list of script files. Fetches text only; never + runs anything. Raises ValueError on a bad source or a missing skill, and + RegistryRateLimited when GitHub's anon API is exhausted.""" + owner, _, repo = source.partition("/") + if not owner or not repo: + raise ValueError(f"unrecognized source '{source}' (expected owner/repo)") + headers = {"User-Agent": "openswarm-skill-registry", "Accept": "application/vnd.github+json"} + async with httpx.AsyncClient(timeout=30.0, headers=headers) as client: + branch, tree = await _fetch_repo_tree(client, owner, repo) + skill_md, members = _select_skill_paths(tree, skill_id) + skill_dir = skill_md[: -len("/SKILL.md")] if "/" in skill_md else "" + prefix = (skill_dir + "/") if skill_dir else "" + + files: dict[str, str] = {} + for p in members: + rel = p[len(prefix):] if prefix else p + raw = await client.get(f"{_GH_RAW}/{owner}/{repo}/{branch}/{p}") + if raw.status_code == 200: + files[rel] = raw.text + if "SKILL.md" not in files: + raise ValueError("SKILL.md could not be fetched") + + meta, _body = _parse_frontmatter(files["SKILL.md"]) + return { + "name": meta.get("name") or skill_id, + "description": meta.get("description", ""), + "repo_url": f"https://github.com/{owner}/{repo}/tree/{branch}/{skill_dir}".rstrip("/"), + "skill_id": skill_id, + "files": files, + "scripts": sorted(rel for rel in files if _is_script_path(rel)), + } + + +async def _community_search(q: str, limit: int) -> dict: + """Live-proxy a query to the skills.sh wild registry. Not cached: it's a + 600k-entry remote index, so we search it on demand rather than mirror it.""" + async with httpx.AsyncClient(timeout=15.0, headers={"User-Agent": "openswarm"}) as client: + r = await client.get(_COMMUNITY_SEARCH_URL, params={"q": q or "skill"}) + r.raise_for_status() + data = r.json() + skills = [] + for s in (data.get("skills") or [])[:limit]: + src = s.get("source", "") + installs = s.get("installs", 0) + skills.append({ + "name": s.get("name", ""), + "description": f"{installs:,} installs", + "folder": s.get("skillId", ""), + "category": src, + "repositoryUrl": f"https://github.com/{src}" if src else "", + "source": src, + "skillId": s.get("skillId", ""), + "installs": installs, + "community": True, + }) + return {"skills": skills, "total": len(skills), "offset": 0, "limit": limit, "source": "community"} + + +class _InstallRequest(BaseModel): + source: str + skill_id: str + confirm: bool = False + + +@skill_registry.router.post("/install") +async def registry_install(req: _InstallRequest): + """Install a community (skills.sh) skill, in two honest steps. + + confirm=false (default): resolve + return a disclosure (the SKILL.md and the + list of files, flagging scripts) WITHOUT writing anything, so the user sees + exactly what they're about to install from an unvetted repo. + confirm=true: write the skill folder to ~/.claude/skills/. Files only; no + script is executed here. Curated skills install via the normal skills CRUD; + this endpoint is the wild-registry path.""" + try: + resolved = await resolve_community_skill(req.source, req.skill_id) + except RegistryRateLimited: + raise HTTPException(status_code=429, detail="GitHub rate limit hit fetching this skill; try again in a few minutes.") + except ValueError as e: + raise HTTPException(status_code=404, detail=str(e)) + except Exception as e: + raise HTTPException(status_code=502, detail=f"could not fetch skill: {e}") + + disclosure = { + "name": resolved["name"], + "description": resolved["description"], + "repo_url": resolved["repo_url"], + "skill_md": resolved["files"].get("SKILL.md", ""), + "files": sorted(resolved["files"].keys()), + "scripts": resolved["scripts"], + "has_scripts": bool(resolved["scripts"]), + } + if not req.confirm: + return {"installed": False, "disclosure": disclosure} + + from backend.apps.skills.skills import write_folder_skill + skill = write_folder_skill( + resolved["skill_id"], + resolved["files"], + {"name": resolved["name"], "description": resolved["description"]}, + ) + return {"installed": True, "skill": skill.model_dump(), "disclosure": disclosure} diff --git a/backend/apps/skills/skills.py b/backend/apps/skills/skills.py index 851d8993..4ebef13e 100644 --- a/backend/apps/skills/skills.py +++ b/backend/apps/skills/skills.py @@ -273,6 +273,46 @@ async def get_skill(skill_id: str): raise HTTPException(status_code=404, detail="Skill not found") +def _safe_slug(raw: str) -> str: + slug = re.sub(r"[^a-zA-Z0-9_-]+", "-", (raw or "").strip().lower()).strip("-") + return slug or "skill" + + +def write_folder_skill(skill_id: str, files: dict[str, str], meta: dict) -> Skill: + """Write a multi-file skill folder (relpath -> content) under SKILLS_DIR and + index it. `files` must include a 'SKILL.md'. Shared by registry install and + zip/.swarm import. Relpaths that try to escape the skill folder (../, abs + paths) are dropped, an untrusted registry archive can't write outside its + own dir.""" + slug = _safe_slug(skill_id) + base = os.path.join(SKILLS_DIR, slug) + base_abs = os.path.abspath(base) + os.makedirs(base, exist_ok=True) + for rel, content in files.items(): + dest = os.path.abspath(os.path.join(base, rel)) + if os.path.commonpath([base_abs, dest]) != base_abs: + logger.warning("skill import: dropped path-escape entry %r", rel) + continue + os.makedirs(os.path.dirname(dest), exist_ok=True) + with open(dest, "w", encoding="utf-8") as f: + f.write(content) + + index = _load_index() + index[slug] = { + "name": meta.get("name") or slug, + "description": meta.get("description", ""), + "command": meta.get("command", slug), + } + _save_index(index) + + md_path, kind = _skill_md_path(slug) + if not md_path: + raise HTTPException(status_code=400, detail="skill had no SKILL.md") + with open(md_path, encoding="utf-8") as f: + content = f.read() + return _build_skill(slug, content, md_path, kind, index) + + @skills.router.post("/create") async def create_skill(body: SkillCreate): slug = body.name.lower().replace(" ", "-") diff --git a/backend/tests/test_skill_registry_community.py b/backend/tests/test_skill_registry_community.py new file mode 100644 index 00000000..c910d75b --- /dev/null +++ b/backend/tests/test_skill_registry_community.py @@ -0,0 +1,97 @@ +"""skills.sh wild-registry resolution + safe install. + +The network parts (GitHub trees + raw fetch) are smoked manually; here we pin +the PURE logic that decides which files a skill is made of and the safety of +writing them: SKILL.md selection at arbitrary repo depth, script disclosure, +and the path-traversal guard that stops an untrusted archive escaping its dir. +""" + +from __future__ import annotations + +import os + +import pytest + +import backend.apps.skills.skills as skills_mod +from backend.apps.skill_registry.skill_registry import _select_skill_paths, _is_script_path + + +def test_selects_shortest_matching_skill_md_at_any_depth(): + tree = [ + {"type": "blob", "path": "README.md"}, + {"type": "blob", "path": "plugins/x/skills/pdftk/SKILL.md"}, + {"type": "blob", "path": "plugins/x/skills/pdftk/run.sh"}, + {"type": "blob", "path": "plugins/x/skills/pdftk/templates/form.txt"}, + {"type": "blob", "path": "plugins/x/skills/other/SKILL.md"}, + ] + skill_md, members = _select_skill_paths(tree, "pdftk") + assert skill_md == "plugins/x/skills/pdftk/SKILL.md" + assert set(members) == { + "plugins/x/skills/pdftk/SKILL.md", + "plugins/x/skills/pdftk/run.sh", + "plugins/x/skills/pdftk/templates/form.txt", + } + # The unrelated 'other' skill's files are excluded. + assert all("/other/" not in m for m in members) + + +def test_top_level_skill_md(): + tree = [{"type": "blob", "path": "pdftk/SKILL.md"}, {"type": "blob", "path": "pdftk/x.py"}] + skill_md, members = _select_skill_paths(tree, "pdftk") + assert skill_md == "pdftk/SKILL.md" + assert "pdftk/x.py" in members + + +def test_missing_skill_raises(): + with pytest.raises(ValueError): + _select_skill_paths([{"type": "blob", "path": "a/SKILL.md"}], "nonexistent") + + +def test_script_classification(): + assert _is_script_path("run.sh") + assert _is_script_path("helper.py") + assert _is_script_path("scripts/build.txt") # under a scripts/ dir + assert _is_script_path("bin/tool") + assert not _is_script_path("SKILL.md") + assert not _is_script_path("templates/form.html") + assert not _is_script_path("data.json") + + +# --------------------------------------------------------------------------- +# Safe install (write_folder_skill). +# --------------------------------------------------------------------------- + +@pytest.fixture +def skills_dir(tmp_path, monkeypatch): + d = tmp_path / "skills" + d.mkdir() + monkeypatch.setattr(skills_mod, "SKILLS_DIR", str(d)) + monkeypatch.setattr(skills_mod, "INDEX_PATH", str(d / ".skills_index.json")) + return d + + +def test_write_folder_skill_lands_files_and_indexes(skills_dir): + skill = skills_mod.write_folder_skill( + "PDF Tk", + {"SKILL.md": "---\nname: PDF Tk\n---\nbody", "scripts/run.sh": "echo hi"}, + {"name": "PDF Tk", "description": "fill forms"}, + ) + assert skill.id == "pdf-tk" + assert skill.has_supporting_files is True + assert os.path.isfile(skills_dir / "pdf-tk" / "SKILL.md") + assert os.path.isfile(skills_dir / "pdf-tk" / "scripts" / "run.sh") + # Re-syncs and shows up in the list. + assert "pdf-tk" in {s.id for s in skills_mod._sync_skills()} + + +def test_write_folder_skill_blocks_path_traversal(skills_dir): + skills_mod.write_folder_skill( + "evil", + {"SKILL.md": "x", "../escape.txt": "pwned", "/etc/abs.txt": "pwned"}, + {"name": "evil"}, + ) + # The escape attempts never landed outside the skill folder. + assert not (skills_dir.parent / "escape.txt").exists() + assert not os.path.exists("/etc/abs.txt") or open("/etc/abs.txt").read() != "pwned" + # The legitimate SKILL.md did land. + assert os.path.isfile(skills_dir / "evil" / "SKILL.md")