mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-26 19:44:51 +02:00
[eric] skills: skills.sh wild-registry source (live search) + GitHub-trees install resolver w/ script disclosure, inert install, path-traversal guard
This commit is contained in:
ciregenz
@@ -8,7 +8,8 @@ from contextlib import asynccontextmanager
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
from fastapi import Query
|
||||
from fastapi import HTTPException, Query
|
||||
from pydantic import BaseModel
|
||||
from backend.config.Apps import SubApp
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -230,7 +231,16 @@ async def registry_search(
|
||||
offset: int = Query(0, ge=0),
|
||||
sort: str = Query("name", description="Sort by: name"),
|
||||
category: str = Query("", description="Filter by category"),
|
||||
source: str = Query("curated", description="curated (vetted) | community (skills.sh wild registry)"),
|
||||
):
|
||||
# The wild registry is a remote 600k-entry index, searched live, not mirrored.
|
||||
if source == "community":
|
||||
try:
|
||||
return await _community_search(q, limit)
|
||||
except Exception as e:
|
||||
logger.warning(f"community skill search failed: {e}")
|
||||
return {"skills": [], "total": 0, "offset": 0, "limit": limit, "source": "community", "error": "skills.sh unreachable"}
|
||||
|
||||
pool = list(_cache.values())
|
||||
if category:
|
||||
cat_lower = category.lower()
|
||||
@@ -268,3 +278,169 @@ async def registry_detail(skill_name: str):
|
||||
if not sk:
|
||||
return {"error": "Skill not found"}, 404
|
||||
return {"skill": sk}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Community source: the skills.sh wild registry (~600k+ telemetry-ranked,
|
||||
# zero-curation community skills, GitHub-repo backed). The curated source above
|
||||
# (anthropics/skills) stays the default; community is opt-in via ?source=community
|
||||
# and the UI flags it as unvetted. See .claude/SECURITY.md for the posture: this
|
||||
# installs INERT files only (never executes), discloses scripts before commit,
|
||||
# and any skill script later runs through the same gated Bash path as anything.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_COMMUNITY_SEARCH_URL = "https://skills.sh/api/search"
|
||||
_GH_API = "https://api.github.com"
|
||||
_GH_RAW = "https://raw.githubusercontent.com"
|
||||
_MAX_SKILL_FILES = 60
|
||||
_SCRIPT_EXTS = (".sh", ".py", ".js", ".mjs", ".cjs", ".ts", ".rb", ".pl", ".ps1", ".bat", ".php")
|
||||
|
||||
|
||||
def _is_script_path(rel: str) -> bool:
|
||||
"""Whether a skill file is executable code worth disclosing before install."""
|
||||
if rel.lower().endswith(_SCRIPT_EXTS):
|
||||
return True
|
||||
head = rel.split("/", 1)[0].lower()
|
||||
return head in ("scripts", "bin", "hooks")
|
||||
|
||||
|
||||
def _select_skill_paths(tree: list[dict], skill_id: str) -> tuple[str, list[str]]:
|
||||
"""From a GitHub recursive tree, pick the SKILL.md for `skill_id` (shortest
|
||||
matching path) and every file living beside it. Pure so the resolution logic
|
||||
is unit-tested without a network round-trip."""
|
||||
blobs = [t["path"] for t in tree if t.get("type") == "blob" and isinstance(t.get("path"), str)]
|
||||
candidates = [p for p in blobs if p.endswith(f"/{skill_id}/SKILL.md") or p == f"{skill_id}/SKILL.md"]
|
||||
if not candidates:
|
||||
raise ValueError(f"no SKILL.md for '{skill_id}' in this repo")
|
||||
skill_md = min(candidates, key=len)
|
||||
skill_dir = skill_md[: -len("/SKILL.md")] if "/" in skill_md else ""
|
||||
prefix = (skill_dir + "/") if skill_dir else ""
|
||||
members = [p for p in blobs if (p.startswith(prefix) if prefix else "/" not in p)]
|
||||
return skill_md, members[:_MAX_SKILL_FILES]
|
||||
|
||||
|
||||
class RegistryRateLimited(Exception):
|
||||
"""GitHub's unauthenticated API (60/hr) is exhausted; the caller surfaces a
|
||||
'try again shortly' rather than a generic failure."""
|
||||
|
||||
|
||||
async def _fetch_repo_tree(client: httpx.AsyncClient, owner: str, repo: str) -> tuple[str, list[dict]]:
|
||||
"""Recursive tree of owner/repo, trying main then master (one API call each,
|
||||
usually just one). Avoids a separate repo-meta call to halve GitHub API use.
|
||||
Raises RegistryRateLimited on a 403, ValueError if no usable branch."""
|
||||
last_status = None
|
||||
for branch in ("main", "master"):
|
||||
r = await client.get(f"{_GH_API}/repos/{owner}/{repo}/git/trees/{branch}?recursive=1")
|
||||
if r.status_code == 200:
|
||||
return branch, r.json().get("tree", [])
|
||||
if r.status_code == 403:
|
||||
raise RegistryRateLimited()
|
||||
last_status = r.status_code
|
||||
raise ValueError(f"repo {owner}/{repo} has no main/master branch (last status {last_status})")
|
||||
|
||||
|
||||
async def resolve_community_skill(source: str, skill_id: str) -> dict:
|
||||
"""Resolve a skills.sh entry (source='owner/repo', skill_id=folder name) to
|
||||
its files via the GitHub trees API. Returns name/description/repo_url plus
|
||||
{relpath: content} and the list of script files. Fetches text only; never
|
||||
runs anything. Raises ValueError on a bad source or a missing skill, and
|
||||
RegistryRateLimited when GitHub's anon API is exhausted."""
|
||||
owner, _, repo = source.partition("/")
|
||||
if not owner or not repo:
|
||||
raise ValueError(f"unrecognized source '{source}' (expected owner/repo)")
|
||||
headers = {"User-Agent": "openswarm-skill-registry", "Accept": "application/vnd.github+json"}
|
||||
async with httpx.AsyncClient(timeout=30.0, headers=headers) as client:
|
||||
branch, tree = await _fetch_repo_tree(client, owner, repo)
|
||||
skill_md, members = _select_skill_paths(tree, skill_id)
|
||||
skill_dir = skill_md[: -len("/SKILL.md")] if "/" in skill_md else ""
|
||||
prefix = (skill_dir + "/") if skill_dir else ""
|
||||
|
||||
files: dict[str, str] = {}
|
||||
for p in members:
|
||||
rel = p[len(prefix):] if prefix else p
|
||||
raw = await client.get(f"{_GH_RAW}/{owner}/{repo}/{branch}/{p}")
|
||||
if raw.status_code == 200:
|
||||
files[rel] = raw.text
|
||||
if "SKILL.md" not in files:
|
||||
raise ValueError("SKILL.md could not be fetched")
|
||||
|
||||
meta, _body = _parse_frontmatter(files["SKILL.md"])
|
||||
return {
|
||||
"name": meta.get("name") or skill_id,
|
||||
"description": meta.get("description", ""),
|
||||
"repo_url": f"https://github.com/{owner}/{repo}/tree/{branch}/{skill_dir}".rstrip("/"),
|
||||
"skill_id": skill_id,
|
||||
"files": files,
|
||||
"scripts": sorted(rel for rel in files if _is_script_path(rel)),
|
||||
}
|
||||
|
||||
|
||||
async def _community_search(q: str, limit: int) -> dict:
|
||||
"""Live-proxy a query to the skills.sh wild registry. Not cached: it's a
|
||||
600k-entry remote index, so we search it on demand rather than mirror it."""
|
||||
async with httpx.AsyncClient(timeout=15.0, headers={"User-Agent": "openswarm"}) as client:
|
||||
r = await client.get(_COMMUNITY_SEARCH_URL, params={"q": q or "skill"})
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
skills = []
|
||||
for s in (data.get("skills") or [])[:limit]:
|
||||
src = s.get("source", "")
|
||||
installs = s.get("installs", 0)
|
||||
skills.append({
|
||||
"name": s.get("name", ""),
|
||||
"description": f"{installs:,} installs",
|
||||
"folder": s.get("skillId", ""),
|
||||
"category": src,
|
||||
"repositoryUrl": f"https://github.com/{src}" if src else "",
|
||||
"source": src,
|
||||
"skillId": s.get("skillId", ""),
|
||||
"installs": installs,
|
||||
"community": True,
|
||||
})
|
||||
return {"skills": skills, "total": len(skills), "offset": 0, "limit": limit, "source": "community"}
|
||||
|
||||
|
||||
class _InstallRequest(BaseModel):
|
||||
source: str
|
||||
skill_id: str
|
||||
confirm: bool = False
|
||||
|
||||
|
||||
@skill_registry.router.post("/install")
|
||||
async def registry_install(req: _InstallRequest):
|
||||
"""Install a community (skills.sh) skill, in two honest steps.
|
||||
|
||||
confirm=false (default): resolve + return a disclosure (the SKILL.md and the
|
||||
list of files, flagging scripts) WITHOUT writing anything, so the user sees
|
||||
exactly what they're about to install from an unvetted repo.
|
||||
confirm=true: write the skill folder to ~/.claude/skills/. Files only; no
|
||||
script is executed here. Curated skills install via the normal skills CRUD;
|
||||
this endpoint is the wild-registry path."""
|
||||
try:
|
||||
resolved = await resolve_community_skill(req.source, req.skill_id)
|
||||
except RegistryRateLimited:
|
||||
raise HTTPException(status_code=429, detail="GitHub rate limit hit fetching this skill; try again in a few minutes.")
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=404, detail=str(e))
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=502, detail=f"could not fetch skill: {e}")
|
||||
|
||||
disclosure = {
|
||||
"name": resolved["name"],
|
||||
"description": resolved["description"],
|
||||
"repo_url": resolved["repo_url"],
|
||||
"skill_md": resolved["files"].get("SKILL.md", ""),
|
||||
"files": sorted(resolved["files"].keys()),
|
||||
"scripts": resolved["scripts"],
|
||||
"has_scripts": bool(resolved["scripts"]),
|
||||
}
|
||||
if not req.confirm:
|
||||
return {"installed": False, "disclosure": disclosure}
|
||||
|
||||
from backend.apps.skills.skills import write_folder_skill
|
||||
skill = write_folder_skill(
|
||||
resolved["skill_id"],
|
||||
resolved["files"],
|
||||
{"name": resolved["name"], "description": resolved["description"]},
|
||||
)
|
||||
return {"installed": True, "skill": skill.model_dump(), "disclosure": disclosure}
|
||||
|
||||
@@ -273,6 +273,46 @@ async def get_skill(skill_id: str):
|
||||
raise HTTPException(status_code=404, detail="Skill not found")
|
||||
|
||||
|
||||
def _safe_slug(raw: str) -> str:
|
||||
slug = re.sub(r"[^a-zA-Z0-9_-]+", "-", (raw or "").strip().lower()).strip("-")
|
||||
return slug or "skill"
|
||||
|
||||
|
||||
def write_folder_skill(skill_id: str, files: dict[str, str], meta: dict) -> Skill:
|
||||
"""Write a multi-file skill folder (relpath -> content) under SKILLS_DIR and
|
||||
index it. `files` must include a 'SKILL.md'. Shared by registry install and
|
||||
zip/.swarm import. Relpaths that try to escape the skill folder (../, abs
|
||||
paths) are dropped, an untrusted registry archive can't write outside its
|
||||
own dir."""
|
||||
slug = _safe_slug(skill_id)
|
||||
base = os.path.join(SKILLS_DIR, slug)
|
||||
base_abs = os.path.abspath(base)
|
||||
os.makedirs(base, exist_ok=True)
|
||||
for rel, content in files.items():
|
||||
dest = os.path.abspath(os.path.join(base, rel))
|
||||
if os.path.commonpath([base_abs, dest]) != base_abs:
|
||||
logger.warning("skill import: dropped path-escape entry %r", rel)
|
||||
continue
|
||||
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
||||
with open(dest, "w", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
|
||||
index = _load_index()
|
||||
index[slug] = {
|
||||
"name": meta.get("name") or slug,
|
||||
"description": meta.get("description", ""),
|
||||
"command": meta.get("command", slug),
|
||||
}
|
||||
_save_index(index)
|
||||
|
||||
md_path, kind = _skill_md_path(slug)
|
||||
if not md_path:
|
||||
raise HTTPException(status_code=400, detail="skill had no SKILL.md")
|
||||
with open(md_path, encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
return _build_skill(slug, content, md_path, kind, index)
|
||||
|
||||
|
||||
@skills.router.post("/create")
|
||||
async def create_skill(body: SkillCreate):
|
||||
slug = body.name.lower().replace(" ", "-")
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
"""skills.sh wild-registry resolution + safe install.
|
||||
|
||||
The network parts (GitHub trees + raw fetch) are smoked manually; here we pin
|
||||
the PURE logic that decides which files a skill is made of and the safety of
|
||||
writing them: SKILL.md selection at arbitrary repo depth, script disclosure,
|
||||
and the path-traversal guard that stops an untrusted archive escaping its dir.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
import backend.apps.skills.skills as skills_mod
|
||||
from backend.apps.skill_registry.skill_registry import _select_skill_paths, _is_script_path
|
||||
|
||||
|
||||
def test_selects_shortest_matching_skill_md_at_any_depth():
|
||||
tree = [
|
||||
{"type": "blob", "path": "README.md"},
|
||||
{"type": "blob", "path": "plugins/x/skills/pdftk/SKILL.md"},
|
||||
{"type": "blob", "path": "plugins/x/skills/pdftk/run.sh"},
|
||||
{"type": "blob", "path": "plugins/x/skills/pdftk/templates/form.txt"},
|
||||
{"type": "blob", "path": "plugins/x/skills/other/SKILL.md"},
|
||||
]
|
||||
skill_md, members = _select_skill_paths(tree, "pdftk")
|
||||
assert skill_md == "plugins/x/skills/pdftk/SKILL.md"
|
||||
assert set(members) == {
|
||||
"plugins/x/skills/pdftk/SKILL.md",
|
||||
"plugins/x/skills/pdftk/run.sh",
|
||||
"plugins/x/skills/pdftk/templates/form.txt",
|
||||
}
|
||||
# The unrelated 'other' skill's files are excluded.
|
||||
assert all("/other/" not in m for m in members)
|
||||
|
||||
|
||||
def test_top_level_skill_md():
|
||||
tree = [{"type": "blob", "path": "pdftk/SKILL.md"}, {"type": "blob", "path": "pdftk/x.py"}]
|
||||
skill_md, members = _select_skill_paths(tree, "pdftk")
|
||||
assert skill_md == "pdftk/SKILL.md"
|
||||
assert "pdftk/x.py" in members
|
||||
|
||||
|
||||
def test_missing_skill_raises():
|
||||
with pytest.raises(ValueError):
|
||||
_select_skill_paths([{"type": "blob", "path": "a/SKILL.md"}], "nonexistent")
|
||||
|
||||
|
||||
def test_script_classification():
|
||||
assert _is_script_path("run.sh")
|
||||
assert _is_script_path("helper.py")
|
||||
assert _is_script_path("scripts/build.txt") # under a scripts/ dir
|
||||
assert _is_script_path("bin/tool")
|
||||
assert not _is_script_path("SKILL.md")
|
||||
assert not _is_script_path("templates/form.html")
|
||||
assert not _is_script_path("data.json")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Safe install (write_folder_skill).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest.fixture
|
||||
def skills_dir(tmp_path, monkeypatch):
|
||||
d = tmp_path / "skills"
|
||||
d.mkdir()
|
||||
monkeypatch.setattr(skills_mod, "SKILLS_DIR", str(d))
|
||||
monkeypatch.setattr(skills_mod, "INDEX_PATH", str(d / ".skills_index.json"))
|
||||
return d
|
||||
|
||||
|
||||
def test_write_folder_skill_lands_files_and_indexes(skills_dir):
|
||||
skill = skills_mod.write_folder_skill(
|
||||
"PDF Tk",
|
||||
{"SKILL.md": "---\nname: PDF Tk\n---\nbody", "scripts/run.sh": "echo hi"},
|
||||
{"name": "PDF Tk", "description": "fill forms"},
|
||||
)
|
||||
assert skill.id == "pdf-tk"
|
||||
assert skill.has_supporting_files is True
|
||||
assert os.path.isfile(skills_dir / "pdf-tk" / "SKILL.md")
|
||||
assert os.path.isfile(skills_dir / "pdf-tk" / "scripts" / "run.sh")
|
||||
# Re-syncs and shows up in the list.
|
||||
assert "pdf-tk" in {s.id for s in skills_mod._sync_skills()}
|
||||
|
||||
|
||||
def test_write_folder_skill_blocks_path_traversal(skills_dir):
|
||||
skills_mod.write_folder_skill(
|
||||
"evil",
|
||||
{"SKILL.md": "x", "../escape.txt": "pwned", "/etc/abs.txt": "pwned"},
|
||||
{"name": "evil"},
|
||||
)
|
||||
# The escape attempts never landed outside the skill folder.
|
||||
assert not (skills_dir.parent / "escape.txt").exists()
|
||||
assert not os.path.exists("/etc/abs.txt") or open("/etc/abs.txt").read() != "pwned"
|
||||
# The legitimate SKILL.md did land.
|
||||
assert os.path.isfile(skills_dir / "evil" / "SKILL.md")
|
||||
Reference in New Issue
Block a user