mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-27 03:54:49 +02:00
[eric] skills: split skill_registry.py (705->275) into sources/github/cache modules to clear the 300-line cap (no cycle, downward imports)
This commit is contained in:
ciregenz
@@ -1,191 +1,28 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
from fastapi import HTTPException, Query
|
||||
from pydantic import BaseModel
|
||||
|
||||
from backend.config.Apps import SubApp
|
||||
from backend.apps.skill_registry import skill_registry_sources as sources
|
||||
from backend.apps.skill_registry import skill_registry_cache as cache
|
||||
from backend.apps.skill_registry.skill_registry_github import folder_tree_sha, RegistryRateLimited
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
REPO = "anthropics/skills"
|
||||
BRANCH = "main"
|
||||
RAW_BASE = f"https://raw.githubusercontent.com/{REPO}/{BRANCH}"
|
||||
MANIFEST_URL = f"{RAW_BASE}/.claude-plugin/marketplace.json"
|
||||
REFRESH_INTERVAL_S = 3600
|
||||
CONCURRENT_FETCHES = 15
|
||||
# Retry the startup fetch on this short backoff (capped) until the FIRST success, instead of waiting a full REFRESH_INTERVAL_S after a cold/slow/failed fetch. That 1h gap was the "skills empty until reboot" bug on cold Windows networks.
|
||||
P_RETRY_BACKOFF_START_S = 2
|
||||
P_RETRY_BACKOFF_MAX_S = 60
|
||||
|
||||
# Catalog ships in the repo so a brand-new install shows skills with zero network (build snapshot), and every successful live fetch is persisted to the user's cache so subsequent launches are instant + offline-safe. The live fetch always overwrites both once it lands, so neither can go stale at runtime.
|
||||
BUNDLED_SNAPSHOT = os.path.join(os.path.dirname(__file__), "skills_snapshot.json")
|
||||
|
||||
p_cache: dict[str, dict] = {}
|
||||
p_cache_updated_at: float = 0
|
||||
p_refresh_task: Optional[asyncio.Task] = None
|
||||
|
||||
# The curated repo's recursive file tree, warmed hourly alongside the catalog. A curated install reads paths from here and fetches contents over raw, so it makes ZERO GitHub API calls in the normal case (the trees API is the 60/hr-limited part); update detection reads per-folder tree SHAs from it too. Empty until the first refresh warms it; install falls back to one live tree call then.
|
||||
p_curated_tree: list[dict] = []
|
||||
# Community repo trees for update detection, cached briefly (best-effort) so an updates check on skills.sh-installed skills doesn't refetch every page load nor burn the API.
|
||||
P_COMMUNITY_TREE_TTL = 600
|
||||
p_community_tree_cache: dict[str, tuple] = {}
|
||||
|
||||
|
||||
def disk_cache_path() -> str:
|
||||
base = os.environ.get("OPENSWARM_SKILL_CACHE_DIR") or os.path.expanduser(
|
||||
"~/.openswarm/cache"
|
||||
)
|
||||
return os.path.join(base, "skill_registry.json")
|
||||
|
||||
|
||||
def load_seed_cache() -> dict[str, dict]:
|
||||
"""Return a non-empty catalog from the on-disk last-good cache, falling back
|
||||
to the bundled snapshot, so the registry is never empty on a cold/offline
|
||||
start. Returns {} only if neither source is present/valid."""
|
||||
for path in (disk_cache_path(), BUNDLED_SNAPSHOT):
|
||||
try:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if isinstance(data, dict) and data:
|
||||
logger.info(f"Skill registry: seeded {len(data)} skills from {os.path.basename(path)}")
|
||||
return data
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
return {}
|
||||
|
||||
|
||||
def save_disk_cache(skills: dict[str, dict]) -> None:
|
||||
"""Persist the last good live fetch so the next launch is instant. Atomic
|
||||
replace so a crash mid-write can't leave a truncated cache."""
|
||||
if not skills:
|
||||
return
|
||||
path = disk_cache_path()
|
||||
try:
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
tmp = f"{path}.tmp"
|
||||
with open(tmp, "w", encoding="utf-8") as f:
|
||||
json.dump(skills, f)
|
||||
os.replace(tmp, path)
|
||||
except OSError:
|
||||
logger.debug("Skill registry: could not persist disk cache", exc_info=True)
|
||||
|
||||
|
||||
def p_parse_frontmatter(raw: str) -> tuple[dict, str]:
|
||||
"""Split YAML frontmatter from markdown body."""
|
||||
if not raw.startswith("---"):
|
||||
return {}, raw
|
||||
end = raw.find("---", 3)
|
||||
if end == -1:
|
||||
return {}, raw
|
||||
fm_block = raw[3:end].strip()
|
||||
body = raw[end + 3:].strip()
|
||||
meta: dict = {}
|
||||
for line in fm_block.splitlines():
|
||||
m = re.match(r"^(\w[\w_-]*)\s*:\s*(.+)$", line)
|
||||
if m:
|
||||
meta[m.group(1).strip()] = m.group(2).strip().strip('"').strip("'")
|
||||
return meta, body
|
||||
|
||||
|
||||
async def p_fetch_skill_paths(client: httpx.AsyncClient) -> list[tuple[str, str]]:
|
||||
"""Fetch the marketplace.json manifest and return (skill_folder, plugin_name) pairs.
|
||||
|
||||
Uses raw.githubusercontent.com; no GitHub API needed, no rate limiting.
|
||||
"""
|
||||
resp = await client.get(MANIFEST_URL)
|
||||
resp.raise_for_status()
|
||||
manifest = resp.json()
|
||||
|
||||
paths: list[tuple[str, str]] = []
|
||||
for plugin in manifest.get("plugins", []):
|
||||
plugin_name = plugin.get("name", "")
|
||||
for skill_ref in plugin.get("skills", []):
|
||||
folder = skill_ref.lstrip("./")
|
||||
paths.append((folder, plugin_name))
|
||||
return paths
|
||||
|
||||
|
||||
async def p_fetch_one_skill(
|
||||
client: httpx.AsyncClient,
|
||||
sem: asyncio.Semaphore,
|
||||
folder: str,
|
||||
plugin_name: str,
|
||||
) -> Optional[dict]:
|
||||
async with sem:
|
||||
try:
|
||||
resp = await client.get(f"{RAW_BASE}/{folder}/SKILL.md")
|
||||
if resp.status_code != 200:
|
||||
return None
|
||||
raw = resp.text
|
||||
except Exception as exc:
|
||||
logger.debug(f"Failed to fetch {folder}/SKILL.md: {exc}")
|
||||
return None
|
||||
|
||||
meta, body = p_parse_frontmatter(raw)
|
||||
name = meta.get("name", "")
|
||||
if not name:
|
||||
folder_name = folder.rsplit("/", 1)[-1]
|
||||
name = folder_name.replace("-", " ").replace("_", " ").title()
|
||||
|
||||
return {
|
||||
"name": name,
|
||||
"description": meta.get("description", ""),
|
||||
"content": body,
|
||||
"folder": folder,
|
||||
"category": plugin_name.replace("-", " ").replace("_", " ").title(),
|
||||
"repositoryUrl": f"https://github.com/{REPO}/tree/{BRANCH}/{folder}",
|
||||
}
|
||||
|
||||
|
||||
async def p_fetch_all_skills() -> dict[str, dict]:
|
||||
skills: dict[str, dict] = {}
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
try:
|
||||
paths = await p_fetch_skill_paths(client)
|
||||
except Exception as e:
|
||||
logger.warning(f"Skill registry manifest fetch failed: {e}")
|
||||
return skills
|
||||
|
||||
logger.info(f"Skill registry: found {len(paths)} skills in manifest, fetching content...")
|
||||
sem = asyncio.Semaphore(CONCURRENT_FETCHES)
|
||||
results = await asyncio.gather(
|
||||
*[p_fetch_one_skill(client, sem, folder, plugin) for folder, plugin in paths]
|
||||
)
|
||||
for rec in results:
|
||||
if rec:
|
||||
skills[rec["name"]] = rec
|
||||
|
||||
logger.info(f"Skill registry cache refreshed: {len(skills)} skills")
|
||||
return skills
|
||||
|
||||
|
||||
async def p_warm_curated_tree() -> None:
|
||||
"""Best-effort: list the anthropics/skills repo once and cache its file paths so
|
||||
curated installs need ZERO trees-API calls (they read paths here, fetch contents
|
||||
over raw). One cheap call per hourly refresh, reused by every install in that hour.
|
||||
Isolated, a failure here never touches the SKILL.md catalog; install falls back to
|
||||
a live tree call while the cache is cold."""
|
||||
global p_curated_tree
|
||||
owner, _, repo = REPO.partition("/")
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=30.0, headers=github_headers()) as client:
|
||||
tree = await p_tree_at(client, owner, repo, BRANCH)
|
||||
if tree:
|
||||
p_curated_tree = tree
|
||||
logger.info(f"Curated skill tree warmed: {len(p_tree_blob_paths(tree))} file paths cached")
|
||||
except RegistryRateLimited:
|
||||
# Visible on purpose: a rate-limited warm-up means installs stay on the slow live-call path until the IP's quota resets or a token is set.
|
||||
logger.warning("Curated tree warm-up rate-limited by GitHub (60/hr anon limit). Set GITHUB_TOKEN or wait for the hourly reset; installs use a live tree call meanwhile.")
|
||||
except Exception:
|
||||
logger.debug("curated tree warm-up failed; installs fall back to a live tree call", exc_info=True)
|
||||
|
||||
|
||||
async def p_refresh_loop():
|
||||
global p_cache, p_cache_updated_at
|
||||
@@ -193,17 +30,17 @@ async def p_refresh_loop():
|
||||
while True:
|
||||
ok = False
|
||||
try:
|
||||
fetched = await p_fetch_all_skills()
|
||||
fetched = await sources.fetch_all_skills()
|
||||
if fetched:
|
||||
p_cache = fetched
|
||||
p_cache_updated_at = time.time()
|
||||
save_disk_cache(p_cache)
|
||||
cache.save_disk_cache(p_cache)
|
||||
ok = True
|
||||
except Exception as e:
|
||||
logger.exception(f"Skill registry refresh error: {e}")
|
||||
if ok:
|
||||
# Warm the curated file-tree on the SLOW path only (never on the fast failure-retry below, which would burn the 60/hr quota in seconds).
|
||||
await p_warm_curated_tree()
|
||||
await sources.warm_curated_tree()
|
||||
# Settle to the slow hourly refresh once we have a good catalog.
|
||||
backoff = P_RETRY_BACKOFF_START_S
|
||||
await asyncio.sleep(REFRESH_INTERVAL_S)
|
||||
@@ -218,7 +55,7 @@ async def skill_registry_lifespan():
|
||||
global p_refresh_task, p_cache
|
||||
# Seed instantly from disk/bundled snapshot so the very first request never sees an empty catalog (the live fetch below overwrites it when it lands).
|
||||
if not p_cache:
|
||||
p_cache = load_seed_cache()
|
||||
p_cache = cache.load_seed_cache()
|
||||
p_refresh_task = asyncio.create_task(p_refresh_loop())
|
||||
yield
|
||||
if p_refresh_task:
|
||||
@@ -257,40 +94,11 @@ async def registry_search(
|
||||
# The wild registry is a remote 600k-entry index, searched live, not mirrored.
|
||||
if source == "community":
|
||||
try:
|
||||
return await p_community_search(q, limit)
|
||||
return await sources.community_search(q, limit)
|
||||
except Exception as e:
|
||||
logger.warning(f"community skill search failed: {e}")
|
||||
return {"skills": [], "total": 0, "offset": 0, "limit": limit, "source": "community", "error": "skills.sh unreachable"}
|
||||
|
||||
pool = list(p_cache.values())
|
||||
if category:
|
||||
cat_lower = category.lower()
|
||||
pool = [s for s in pool if s.get("category", "").lower() == cat_lower]
|
||||
|
||||
query_lower = q.lower().strip()
|
||||
if query_lower:
|
||||
filtered = []
|
||||
for sk in pool:
|
||||
searchable = f"{sk['name']} {sk['description']} {sk.get('category', '')}".lower()
|
||||
if query_lower in searchable:
|
||||
filtered.append(sk)
|
||||
pool = filtered
|
||||
|
||||
pool.sort(key=lambda s: s["name"].lower())
|
||||
total = len(pool)
|
||||
page = pool[offset : offset + limit]
|
||||
|
||||
summary = [
|
||||
{
|
||||
"name": s["name"],
|
||||
"description": s["description"],
|
||||
"folder": s["folder"],
|
||||
"category": s.get("category", "General"),
|
||||
"repositoryUrl": s.get("repositoryUrl", ""),
|
||||
}
|
||||
for s in page
|
||||
]
|
||||
return {"skills": summary, "total": total, "offset": offset, "limit": limit}
|
||||
return sources.search_curated(p_cache, q, category, offset, limit)
|
||||
|
||||
|
||||
@skill_registry.router.get("/detail/{skill_name:path}")
|
||||
@@ -301,224 +109,6 @@ async def registry_detail(skill_name: str):
|
||||
return {"skill": sk}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- Community source: the skills.sh wild registry (~600k+ telemetry-ranked, zero-curation community skills, GitHub-repo backed). The curated source above (anthropics/skills) stays the default; community is opt-in via ?source=community and the UI flags it as unvetted. See .claude/SECURITY.md for the posture: this installs INERT files only (never executes), discloses scripts before commit, and any skill script later runs through the same gated Bash path as anything. ---------------------------------------------------------------------------
|
||||
|
||||
P_COMMUNITY_SEARCH_URL = "https://skills.sh/api/search"
|
||||
P_GH_API = "https://api.github.com"
|
||||
P_GH_RAW = "https://raw.githubusercontent.com"
|
||||
P_MAX_SKILL_FILES = 60
|
||||
P_SCRIPT_EXTS = (".sh", ".py", ".js", ".mjs", ".cjs", ".ts", ".rb", ".pl", ".ps1", ".bat", ".php")
|
||||
|
||||
|
||||
def is_script_path(rel: str) -> bool:
|
||||
"""Whether a skill file is executable code worth disclosing before install."""
|
||||
if rel.lower().endswith(P_SCRIPT_EXTS):
|
||||
return True
|
||||
head = rel.split("/", 1)[0].lower()
|
||||
return head in ("scripts", "bin", "hooks")
|
||||
|
||||
|
||||
def github_headers() -> dict:
|
||||
"""GitHub request headers, with auth if a token is set. Unauthenticated is
|
||||
60 req/hr/IP (fine for the odd install, the wall for a power user); a token
|
||||
(OPENSWARM_GITHUB_TOKEN or GITHUB_TOKEN) raises it to 5000/hr."""
|
||||
headers = {"User-Agent": "openswarm-skill-registry", "Accept": "application/vnd.github+json"}
|
||||
token = os.environ.get("OPENSWARM_GITHUB_TOKEN") or os.environ.get("GITHUB_TOKEN")
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
return headers
|
||||
|
||||
|
||||
def select_skill_paths(tree: list[dict], skill_id: str) -> tuple[str, list[str]]:
|
||||
"""From a GitHub recursive tree, pick the SKILL.md for `skill_id` and every
|
||||
file beside it. Pure, so the resolution logic is unit-tested without a network
|
||||
round-trip. When a repo has several `<x>/<skill_id>/SKILL.md` matches the pick
|
||||
is deterministic: prefer a top-level `<skill_id>/`, then `skills/<skill_id>/`,
|
||||
then the shallowest, then alphabetical, never an arbitrary tie."""
|
||||
blobs = [t["path"] for t in tree if t.get("type") == "blob" and isinstance(t.get("path"), str)]
|
||||
candidates = [p for p in blobs if p.endswith(f"/{skill_id}/SKILL.md") or p == f"{skill_id}/SKILL.md"]
|
||||
if not candidates:
|
||||
raise ValueError(f"no SKILL.md for '{skill_id}' in this repo")
|
||||
|
||||
def p_rank(p: str) -> tuple:
|
||||
if p == f"{skill_id}/SKILL.md":
|
||||
return (0, 0, p)
|
||||
if p == f"skills/{skill_id}/SKILL.md":
|
||||
return (1, p.count("/"), p)
|
||||
return (2, p.count("/"), p)
|
||||
|
||||
skill_md = min(candidates, key=p_rank)
|
||||
skill_dir = skill_md[: -len("/SKILL.md")] if "/" in skill_md else ""
|
||||
prefix = (skill_dir + "/") if skill_dir else ""
|
||||
members = [p for p in blobs if (p.startswith(prefix) if prefix else "/" not in p)]
|
||||
return skill_md, members[:P_MAX_SKILL_FILES]
|
||||
|
||||
|
||||
class RegistryRateLimited(Exception):
|
||||
"""GitHub's unauthenticated API (60/hr) is exhausted; the caller surfaces a
|
||||
'try again shortly' rather than a generic failure."""
|
||||
|
||||
|
||||
def p_tree_blob_paths(tree: list[dict]) -> list[str]:
|
||||
"""The blob (file) paths from a GitHub recursive tree, ignoring tree (dir) entries."""
|
||||
return [t["path"] for t in tree if t.get("type") == "blob" and isinstance(t.get("path"), str)]
|
||||
|
||||
|
||||
def p_folder_tree_sha(tree: list[dict], folder: str) -> str:
|
||||
"""The git tree SHA of `folder` within a recursive tree: a per-folder fingerprint
|
||||
that changes iff something inside it changes, so one skill going stale never marks
|
||||
its siblings stale. '' when the folder isn't present as a tree entry."""
|
||||
for t in tree:
|
||||
if t.get("type") == "tree" and t.get("path") == folder:
|
||||
return t.get("sha", "") or ""
|
||||
return ""
|
||||
|
||||
|
||||
async def p_tree_at(client: httpx.AsyncClient, owner: str, repo: str, branch: str):
|
||||
"""(tree | None) for a branch. None on 404 (branch absent); raises on rate limit.
|
||||
GitHub signals the limit as 403 (primary) or 429 (secondary), so treat both."""
|
||||
r = await client.get(f"{P_GH_API}/repos/{owner}/{repo}/git/trees/{branch}?recursive=1")
|
||||
if r.status_code == 200:
|
||||
return r.json().get("tree", [])
|
||||
if r.status_code in (403, 429):
|
||||
raise RegistryRateLimited()
|
||||
return None
|
||||
|
||||
|
||||
async def p_fetch_repo_tree(client: httpx.AsyncClient, owner: str, repo: str) -> tuple[str, list[dict]]:
|
||||
"""Recursive tree of owner/repo. Tries main then master first (one call, the
|
||||
99% case, no quota wasted on a repo-meta lookup); only if BOTH are absent
|
||||
does it ask the repo for its real default branch (handles develop/trunk/etc).
|
||||
Raises RegistryRateLimited on a 403, ValueError if no branch resolves."""
|
||||
for branch in ("main", "master"):
|
||||
tree = await p_tree_at(client, owner, repo, branch)
|
||||
if tree is not None:
|
||||
return branch, tree
|
||||
meta = await client.get(f"{P_GH_API}/repos/{owner}/{repo}")
|
||||
if meta.status_code == 403:
|
||||
raise RegistryRateLimited()
|
||||
if meta.status_code == 200:
|
||||
default = meta.json().get("default_branch")
|
||||
if default and default not in ("main", "master"):
|
||||
tree = await p_tree_at(client, owner, repo, default)
|
||||
if tree is not None:
|
||||
return default, tree
|
||||
raise ValueError(f"repo {owner}/{repo} has no resolvable default branch")
|
||||
|
||||
|
||||
async def p_build_resolved_skill(
|
||||
client: httpx.AsyncClient,
|
||||
owner: str,
|
||||
repo: str,
|
||||
branch: str,
|
||||
skill_dir: str,
|
||||
members: list[str],
|
||||
skill_id: str,
|
||||
version: str,
|
||||
) -> dict:
|
||||
"""Fetch every member file of a resolved skill folder and assemble the install
|
||||
payload (relpaths, scripts list, secret scan, provenance). Shared by the community
|
||||
and curated resolvers so both install the WHOLE folder identically. Fetches text
|
||||
only; never runs anything. `version` is the folder's tree SHA, the update fingerprint."""
|
||||
prefix = (skill_dir + "/") if skill_dir else ""
|
||||
files: dict[str, str] = {}
|
||||
for p in members:
|
||||
rel = p[len(prefix):] if prefix else p
|
||||
raw = await client.get(f"{P_GH_RAW}/{owner}/{repo}/{branch}/{p}")
|
||||
if raw.status_code == 200:
|
||||
files[rel] = raw.text
|
||||
if "SKILL.md" not in files:
|
||||
raise ValueError("SKILL.md could not be fetched")
|
||||
|
||||
meta, _ = p_parse_frontmatter(files["SKILL.md"])
|
||||
# Reuse the .swarm importer's content scan: flag files holding secret-shaped literals (the author's leaked key, or a sketchy skill) so the user sees it before installing from an unvetted repo.
|
||||
from backend.common.secret_scan import find_secrets_in_files
|
||||
secret_findings = find_secrets_in_files({rel: data.encode("utf-8", "ignore") for rel, data in files.items()})
|
||||
return {
|
||||
"name": meta.get("name") or skill_id,
|
||||
"description": meta.get("description", ""),
|
||||
"repo_url": f"https://github.com/{owner}/{repo}/tree/{branch}/{skill_dir}".rstrip("/"),
|
||||
"skill_id": skill_id,
|
||||
"files": files,
|
||||
"scripts": sorted(rel for rel in files if is_script_path(rel)),
|
||||
"secret_findings": secret_findings,
|
||||
"source": f"{owner}/{repo}",
|
||||
"folder": skill_dir,
|
||||
"version": version,
|
||||
}
|
||||
|
||||
|
||||
async def resolve_community_skill(source: str, skill_id: str) -> dict:
|
||||
"""Resolve a skills.sh entry (source='owner/repo', skill_id=folder name) to
|
||||
its files via the GitHub trees API. Returns name/description/repo_url plus
|
||||
{relpath: content} and the list of script files. Fetches text only; never
|
||||
runs anything. Raises ValueError on a bad source or a missing skill, and
|
||||
RegistryRateLimited when GitHub's anon API is exhausted."""
|
||||
owner, _, repo = source.partition("/")
|
||||
if not owner or not repo:
|
||||
raise ValueError(f"unrecognized source '{source}' (expected owner/repo)")
|
||||
async with httpx.AsyncClient(timeout=30.0, headers=github_headers()) as client:
|
||||
branch, tree = await p_fetch_repo_tree(client, owner, repo)
|
||||
skill_md, members = select_skill_paths(tree, skill_id)
|
||||
skill_dir = skill_md[: -len("/SKILL.md")] if "/" in skill_md else ""
|
||||
version = p_folder_tree_sha(tree, skill_dir)
|
||||
return await p_build_resolved_skill(client, owner, repo, branch, skill_dir, members, skill_id, version)
|
||||
|
||||
|
||||
async def resolve_curated_skill(folder: str) -> dict:
|
||||
"""Resolve a curated (anthropics/skills) skill folder to ALL its files via the
|
||||
GitHub trees API, so multi-file curated skills (pdf/docx/pptx scripts, etc.)
|
||||
install whole instead of just their SKILL.md. The exact folder comes from our
|
||||
catalog, so we match it precisely (not by basename). Same payload shape as
|
||||
resolve_community_skill. Raises ValueError if the folder has no SKILL.md and
|
||||
RegistryRateLimited when GitHub's anon API is exhausted."""
|
||||
owner, _, repo = REPO.partition("/")
|
||||
skill_dir = folder.rstrip("/")
|
||||
skill_id = skill_dir.rsplit("/", 1)[-1]
|
||||
prefix = skill_dir + "/"
|
||||
async with httpx.AsyncClient(timeout=30.0, headers=github_headers()) as client:
|
||||
tree = p_curated_tree
|
||||
if not tree:
|
||||
# Cold cache (pre-first-refresh, or a failed/rate-limited warm-up): pay one live tree call this once.
|
||||
tree = await p_tree_at(client, owner, repo, BRANCH)
|
||||
if tree is None:
|
||||
raise ValueError(f"could not read {REPO}@{BRANCH} tree")
|
||||
blobs = p_tree_blob_paths(tree)
|
||||
if (prefix + "SKILL.md") not in blobs:
|
||||
raise ValueError(f"no SKILL.md at '{folder}'")
|
||||
members = [p for p in blobs if p.startswith(prefix)][:P_MAX_SKILL_FILES]
|
||||
version = p_folder_tree_sha(tree, skill_dir)
|
||||
return await p_build_resolved_skill(client, owner, repo, BRANCH, skill_dir, members, skill_id, version)
|
||||
|
||||
|
||||
async def p_community_search(q: str, limit: int) -> dict:
|
||||
"""Live-proxy a query to the skills.sh wild registry. Not cached: it's a
|
||||
600k-entry remote index, so we search it on demand rather than mirror it."""
|
||||
async with httpx.AsyncClient(timeout=15.0, headers={"User-Agent": "openswarm"}) as client:
|
||||
r = await client.get(P_COMMUNITY_SEARCH_URL, params={"q": q or "skill"})
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
skills = []
|
||||
for s in (data.get("skills") or [])[:limit]:
|
||||
src = s.get("source", "")
|
||||
try:
|
||||
installs = int(s.get("installs") or 0)
|
||||
except (TypeError, ValueError):
|
||||
installs = 0
|
||||
skills.append({
|
||||
"name": s.get("name", ""),
|
||||
"description": f"{installs:,} installs",
|
||||
"folder": s.get("skillId", ""),
|
||||
"category": src,
|
||||
"repositoryUrl": f"https://github.com/{src}" if src else "",
|
||||
"source": src,
|
||||
"skillId": s.get("skillId", ""),
|
||||
"installs": installs,
|
||||
"community": True,
|
||||
})
|
||||
return {"skills": skills, "total": len(skills), "offset": 0, "limit": limit, "source": "community"}
|
||||
|
||||
|
||||
class p_InstallRequest(BaseModel):
|
||||
source: str
|
||||
skill_id: str
|
||||
@@ -536,7 +126,7 @@ async def registry_install(req: p_InstallRequest):
|
||||
script is executed here. Curated skills install via the normal skills CRUD;
|
||||
this endpoint is the wild-registry path."""
|
||||
try:
|
||||
resolved = await resolve_community_skill(req.source, req.skill_id)
|
||||
resolved = await sources.resolve_community_skill(req.source, req.skill_id)
|
||||
except RegistryRateLimited:
|
||||
raise HTTPException(status_code=429, detail="GitHub rate limit hit fetching this skill; try again in a few minutes.")
|
||||
except ValueError as e:
|
||||
@@ -583,7 +173,7 @@ async def registry_install_curated(req: p_CuratedInstallRequest):
|
||||
the vetted source, so this is one-click; files are still written inert, never
|
||||
executed. Needs network at install time (the catalog only caches SKILL.md)."""
|
||||
try:
|
||||
resolved = await resolve_curated_skill(req.folder)
|
||||
resolved = await sources.resolve_curated_skill(req.folder)
|
||||
except RegistryRateLimited:
|
||||
raise HTTPException(status_code=429, detail="GitHub rate limit hit fetching this skill; try again in a few minutes.")
|
||||
except ValueError as e:
|
||||
@@ -609,26 +199,6 @@ async def registry_install_curated(req: p_CuratedInstallRequest):
|
||||
}
|
||||
|
||||
|
||||
async def p_safe_repo_tree(source: str):
|
||||
"""Recursive tree for a community 'owner/repo', cached briefly and best-effort
|
||||
(None on rate-limit / missing repo) so an updates check never fails the whole
|
||||
list because one repo is unreachable."""
|
||||
now = time.time()
|
||||
hit = p_community_tree_cache.get(source)
|
||||
if hit and now - hit[0] < P_COMMUNITY_TREE_TTL:
|
||||
return hit[1]
|
||||
owner, _, repo = source.partition("/")
|
||||
tree = None
|
||||
if owner and repo:
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=30.0, headers=github_headers()) as client:
|
||||
_, tree = await p_fetch_repo_tree(client, owner, repo)
|
||||
except Exception:
|
||||
tree = None
|
||||
p_community_tree_cache[source] = (now, tree)
|
||||
return tree
|
||||
|
||||
|
||||
@skill_registry.router.get("/updates")
|
||||
async def registry_updates():
|
||||
"""Which installed skills have a newer version upstream. Curated skills check
|
||||
@@ -643,16 +213,16 @@ async def registry_updates():
|
||||
for s in sync_skills():
|
||||
if not s.source or not s.folder or not s.version:
|
||||
continue
|
||||
if s.source == REPO:
|
||||
tree = p_curated_tree
|
||||
if s.source == sources.REPO:
|
||||
tree = sources.curated_tree
|
||||
else:
|
||||
if s.source not in community_trees:
|
||||
community_trees[s.source] = await p_safe_repo_tree(s.source)
|
||||
community_trees[s.source] = await sources.safe_repo_tree(s.source)
|
||||
tree = community_trees[s.source]
|
||||
if not tree:
|
||||
unknown.append(s.id)
|
||||
continue
|
||||
current = p_folder_tree_sha(tree, s.folder)
|
||||
current = folder_tree_sha(tree, s.folder)
|
||||
checked.append(s.id)
|
||||
if current and current != s.version:
|
||||
outdated.append(s.id)
|
||||
@@ -676,10 +246,10 @@ async def registry_update(req: p_UpdateRequest):
|
||||
if not target.source or not target.folder:
|
||||
raise HTTPException(status_code=400, detail="this skill has no upstream source to update from")
|
||||
try:
|
||||
if target.source == REPO:
|
||||
resolved = await resolve_curated_skill(target.folder)
|
||||
if target.source == sources.REPO:
|
||||
resolved = await sources.resolve_curated_skill(target.folder)
|
||||
else:
|
||||
resolved = await resolve_community_skill(target.source, target.folder.rsplit("/", 1)[-1])
|
||||
resolved = await sources.resolve_community_skill(target.source, target.folder.rsplit("/", 1)[-1])
|
||||
except RegistryRateLimited:
|
||||
raise HTTPException(status_code=429, detail="GitHub rate limit hit; try again in a few minutes.")
|
||||
except ValueError as e:
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Catalog ships in the repo so a brand-new install shows skills with zero network (build snapshot), and every successful live fetch is persisted to the user's cache so subsequent launches are instant + offline-safe. The live fetch always overwrites both once it lands, so neither can go stale at runtime.
|
||||
BUNDLED_SNAPSHOT = os.path.join(os.path.dirname(__file__), "skills_snapshot.json")
|
||||
|
||||
|
||||
def disk_cache_path() -> str:
|
||||
base = os.environ.get("OPENSWARM_SKILL_CACHE_DIR") or os.path.expanduser(
|
||||
"~/.openswarm/cache"
|
||||
)
|
||||
return os.path.join(base, "skill_registry.json")
|
||||
|
||||
|
||||
def load_seed_cache() -> dict[str, dict]:
|
||||
"""Return a non-empty catalog from the on-disk last-good cache, falling back
|
||||
to the bundled snapshot, so the registry is never empty on a cold/offline
|
||||
start. Returns {} only if neither source is present/valid."""
|
||||
for path in (disk_cache_path(), BUNDLED_SNAPSHOT):
|
||||
try:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if isinstance(data, dict) and data:
|
||||
logger.info(f"Skill registry: seeded {len(data)} skills from {os.path.basename(path)}")
|
||||
return data
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
return {}
|
||||
|
||||
|
||||
def save_disk_cache(skills: dict[str, dict]) -> None:
|
||||
"""Persist the last good live fetch so the next launch is instant. Atomic
|
||||
replace so a crash mid-write can't leave a truncated cache."""
|
||||
if not skills:
|
||||
return
|
||||
path = disk_cache_path()
|
||||
try:
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
tmp = f"{path}.tmp"
|
||||
with open(tmp, "w", encoding="utf-8") as f:
|
||||
json.dump(skills, f)
|
||||
os.replace(tmp, path)
|
||||
except OSError:
|
||||
logger.debug("Skill registry: could not persist disk cache", exc_info=True)
|
||||
@@ -0,0 +1,121 @@
|
||||
import os
|
||||
import re
|
||||
|
||||
import httpx
|
||||
|
||||
GH_API = "https://api.github.com"
|
||||
MAX_SKILL_FILES = 60
|
||||
SCRIPT_EXTS = (".sh", ".py", ".js", ".mjs", ".cjs", ".ts", ".rb", ".pl", ".ps1", ".bat", ".php")
|
||||
|
||||
|
||||
class RegistryRateLimited(Exception):
|
||||
"""GitHub's unauthenticated API (60/hr) is exhausted; the caller surfaces a
|
||||
'try again shortly' rather than a generic failure."""
|
||||
|
||||
|
||||
def parse_frontmatter(raw: str) -> tuple[dict, str]:
|
||||
"""Split YAML frontmatter from markdown body."""
|
||||
if not raw.startswith("---"):
|
||||
return {}, raw
|
||||
end = raw.find("---", 3)
|
||||
if end == -1:
|
||||
return {}, raw
|
||||
fm_block = raw[3:end].strip()
|
||||
body = raw[end + 3:].strip()
|
||||
meta: dict = {}
|
||||
for line in fm_block.splitlines():
|
||||
m = re.match(r"^(\w[\w_-]*)\s*:\s*(.+)$", line)
|
||||
if m:
|
||||
meta[m.group(1).strip()] = m.group(2).strip().strip('"').strip("'")
|
||||
return meta, body
|
||||
|
||||
|
||||
def is_script_path(rel: str) -> bool:
|
||||
"""Whether a skill file is executable code worth disclosing before install."""
|
||||
if rel.lower().endswith(SCRIPT_EXTS):
|
||||
return True
|
||||
head = rel.split("/", 1)[0].lower()
|
||||
return head in ("scripts", "bin", "hooks")
|
||||
|
||||
|
||||
def github_headers() -> dict:
|
||||
"""GitHub request headers, with auth if a token is set. Unauthenticated is
|
||||
60 req/hr/IP (fine for the odd install, the wall for a power user); a token
|
||||
(OPENSWARM_GITHUB_TOKEN or GITHUB_TOKEN) raises it to 5000/hr."""
|
||||
headers = {"User-Agent": "openswarm-skill-registry", "Accept": "application/vnd.github+json"}
|
||||
token = os.environ.get("OPENSWARM_GITHUB_TOKEN") or os.environ.get("GITHUB_TOKEN")
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
return headers
|
||||
|
||||
|
||||
def select_skill_paths(tree: list[dict], skill_id: str) -> tuple[str, list[str]]:
|
||||
"""From a GitHub recursive tree, pick the SKILL.md for `skill_id` and every
|
||||
file beside it. Pure, so the resolution logic is unit-tested without a network
|
||||
round-trip. When a repo has several `<x>/<skill_id>/SKILL.md` matches the pick
|
||||
is deterministic: prefer a top-level `<skill_id>/`, then `skills/<skill_id>/`,
|
||||
then the shallowest, then alphabetical, never an arbitrary tie."""
|
||||
blobs = [t["path"] for t in tree if t.get("type") == "blob" and isinstance(t.get("path"), str)]
|
||||
candidates = [p for p in blobs if p.endswith(f"/{skill_id}/SKILL.md") or p == f"{skill_id}/SKILL.md"]
|
||||
if not candidates:
|
||||
raise ValueError(f"no SKILL.md for '{skill_id}' in this repo")
|
||||
|
||||
def p_rank(p: str) -> tuple:
|
||||
if p == f"{skill_id}/SKILL.md":
|
||||
return (0, 0, p)
|
||||
if p == f"skills/{skill_id}/SKILL.md":
|
||||
return (1, p.count("/"), p)
|
||||
return (2, p.count("/"), p)
|
||||
|
||||
skill_md = min(candidates, key=p_rank)
|
||||
skill_dir = skill_md[: -len("/SKILL.md")] if "/" in skill_md else ""
|
||||
prefix = (skill_dir + "/") if skill_dir else ""
|
||||
members = [p for p in blobs if (p.startswith(prefix) if prefix else "/" not in p)]
|
||||
return skill_md, members[:MAX_SKILL_FILES]
|
||||
|
||||
|
||||
def tree_blob_paths(tree: list[dict]) -> list[str]:
|
||||
"""The blob (file) paths from a GitHub recursive tree, ignoring tree (dir) entries."""
|
||||
return [t["path"] for t in tree if t.get("type") == "blob" and isinstance(t.get("path"), str)]
|
||||
|
||||
|
||||
def folder_tree_sha(tree: list[dict], folder: str) -> str:
|
||||
"""The git tree SHA of `folder` within a recursive tree: a per-folder fingerprint
|
||||
that changes iff something inside it changes, so one skill going stale never marks
|
||||
its siblings stale. '' when the folder isn't present as a tree entry."""
|
||||
for t in tree:
|
||||
if t.get("type") == "tree" and t.get("path") == folder:
|
||||
return t.get("sha", "") or ""
|
||||
return ""
|
||||
|
||||
|
||||
async def tree_at(client: httpx.AsyncClient, owner: str, repo: str, branch: str):
|
||||
"""(tree | None) for a branch. None on 404 (branch absent); raises on rate limit.
|
||||
GitHub signals the limit as 403 (primary) or 429 (secondary), so treat both."""
|
||||
r = await client.get(f"{GH_API}/repos/{owner}/{repo}/git/trees/{branch}?recursive=1")
|
||||
if r.status_code == 200:
|
||||
return r.json().get("tree", [])
|
||||
if r.status_code in (403, 429):
|
||||
raise RegistryRateLimited()
|
||||
return None
|
||||
|
||||
|
||||
async def fetch_repo_tree(client: httpx.AsyncClient, owner: str, repo: str) -> tuple[str, list[dict]]:
|
||||
"""Recursive tree of owner/repo. Tries main then master first (one call, the
|
||||
99% case, no quota wasted on a repo-meta lookup); only if BOTH are absent
|
||||
does it ask the repo for its real default branch (handles develop/trunk/etc).
|
||||
Raises RegistryRateLimited on a 403, ValueError if no branch resolves."""
|
||||
for branch in ("main", "master"):
|
||||
tree = await tree_at(client, owner, repo, branch)
|
||||
if tree is not None:
|
||||
return branch, tree
|
||||
meta = await client.get(f"{GH_API}/repos/{owner}/{repo}")
|
||||
if meta.status_code == 403:
|
||||
raise RegistryRateLimited()
|
||||
if meta.status_code == 200:
|
||||
default = meta.json().get("default_branch")
|
||||
if default and default not in ("main", "master"):
|
||||
tree = await tree_at(client, owner, repo, default)
|
||||
if tree is not None:
|
||||
return default, tree
|
||||
raise ValueError(f"repo {owner}/{repo} has no resolvable default branch")
|
||||
@@ -0,0 +1,289 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
|
||||
from backend.apps.skill_registry.skill_registry_github import (
|
||||
RegistryRateLimited,
|
||||
parse_frontmatter,
|
||||
is_script_path,
|
||||
github_headers,
|
||||
select_skill_paths,
|
||||
tree_blob_paths,
|
||||
folder_tree_sha,
|
||||
tree_at,
|
||||
fetch_repo_tree,
|
||||
MAX_SKILL_FILES,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
REPO = "anthropics/skills"
|
||||
BRANCH = "main"
|
||||
RAW_BASE = f"https://raw.githubusercontent.com/{REPO}/{BRANCH}"
|
||||
MANIFEST_URL = f"{RAW_BASE}/.claude-plugin/marketplace.json"
|
||||
CONCURRENT_FETCHES = 15
|
||||
GH_RAW = "https://raw.githubusercontent.com"
|
||||
COMMUNITY_SEARCH_URL = "https://skills.sh/api/search"
|
||||
P_COMMUNITY_TREE_TTL = 600
|
||||
|
||||
# The curated repo's recursive file tree, warmed hourly alongside the catalog. A curated install reads paths from here and fetches contents over raw, so it makes ZERO GitHub API calls in the normal case (the trees API is the 60/hr-limited part); update detection reads per-folder tree SHAs from it too. Empty until the first refresh warms it; install falls back to one live tree call then.
|
||||
curated_tree: list[dict] = []
|
||||
# Community repo trees for update detection, cached briefly (best-effort) so an updates check on skills.sh-installed skills doesn't refetch every page load nor burn the API.
|
||||
p_community_tree_cache: dict[str, tuple] = {}
|
||||
|
||||
|
||||
async def p_fetch_skill_paths(client: httpx.AsyncClient) -> list[tuple[str, str]]:
|
||||
"""Fetch the marketplace.json manifest and return (skill_folder, plugin_name) pairs.
|
||||
|
||||
Uses raw.githubusercontent.com; no GitHub API needed, no rate limiting.
|
||||
"""
|
||||
resp = await client.get(MANIFEST_URL)
|
||||
resp.raise_for_status()
|
||||
manifest = resp.json()
|
||||
|
||||
paths: list[tuple[str, str]] = []
|
||||
for plugin in manifest.get("plugins", []):
|
||||
plugin_name = plugin.get("name", "")
|
||||
for skill_ref in plugin.get("skills", []):
|
||||
folder = skill_ref.lstrip("./")
|
||||
paths.append((folder, plugin_name))
|
||||
return paths
|
||||
|
||||
|
||||
async def p_fetch_one_skill(
|
||||
client: httpx.AsyncClient,
|
||||
sem: asyncio.Semaphore,
|
||||
folder: str,
|
||||
plugin_name: str,
|
||||
) -> Optional[dict]:
|
||||
async with sem:
|
||||
try:
|
||||
resp = await client.get(f"{RAW_BASE}/{folder}/SKILL.md")
|
||||
if resp.status_code != 200:
|
||||
return None
|
||||
raw = resp.text
|
||||
except Exception as exc:
|
||||
logger.debug(f"Failed to fetch {folder}/SKILL.md: {exc}")
|
||||
return None
|
||||
|
||||
meta, body = parse_frontmatter(raw)
|
||||
name = meta.get("name", "")
|
||||
if not name:
|
||||
folder_name = folder.rsplit("/", 1)[-1]
|
||||
name = folder_name.replace("-", " ").replace("_", " ").title()
|
||||
|
||||
return {
|
||||
"name": name,
|
||||
"description": meta.get("description", ""),
|
||||
"content": body,
|
||||
"folder": folder,
|
||||
"category": plugin_name.replace("-", " ").replace("_", " ").title(),
|
||||
"repositoryUrl": f"https://github.com/{REPO}/tree/{BRANCH}/{folder}",
|
||||
}
|
||||
|
||||
|
||||
async def fetch_all_skills() -> dict[str, dict]:
|
||||
skills: dict[str, dict] = {}
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
try:
|
||||
paths = await p_fetch_skill_paths(client)
|
||||
except Exception as e:
|
||||
logger.warning(f"Skill registry manifest fetch failed: {e}")
|
||||
return skills
|
||||
|
||||
logger.info(f"Skill registry: found {len(paths)} skills in manifest, fetching content...")
|
||||
sem = asyncio.Semaphore(CONCURRENT_FETCHES)
|
||||
results = await asyncio.gather(
|
||||
*[p_fetch_one_skill(client, sem, folder, plugin) for folder, plugin in paths]
|
||||
)
|
||||
for rec in results:
|
||||
if rec:
|
||||
skills[rec["name"]] = rec
|
||||
|
||||
logger.info(f"Skill registry cache refreshed: {len(skills)} skills")
|
||||
return skills
|
||||
|
||||
|
||||
async def warm_curated_tree() -> None:
|
||||
"""Best-effort: list the anthropics/skills repo once and cache its file paths so
|
||||
curated installs need ZERO trees-API calls (they read paths here, fetch contents
|
||||
over raw). One cheap call per hourly refresh, reused by every install in that hour.
|
||||
Isolated, a failure here never touches the SKILL.md catalog; install falls back to
|
||||
a live tree call while the cache is cold."""
|
||||
global curated_tree
|
||||
owner, _, repo = REPO.partition("/")
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=30.0, headers=github_headers()) as client:
|
||||
tree = await tree_at(client, owner, repo, BRANCH)
|
||||
if tree:
|
||||
curated_tree = tree
|
||||
logger.info(f"Curated skill tree warmed: {len(tree_blob_paths(tree))} file paths cached")
|
||||
except RegistryRateLimited:
|
||||
# Visible on purpose: a rate-limited warm-up means installs stay on the slow live-call path until the IP's quota resets or a token is set.
|
||||
logger.warning("Curated tree warm-up rate-limited by GitHub (60/hr anon limit). Set GITHUB_TOKEN or wait for the hourly reset; installs use a live tree call meanwhile.")
|
||||
except Exception:
|
||||
logger.debug("curated tree warm-up failed; installs fall back to a live tree call", exc_info=True)
|
||||
|
||||
|
||||
async def p_build_resolved_skill(
|
||||
client: httpx.AsyncClient,
|
||||
owner: str,
|
||||
repo: str,
|
||||
branch: str,
|
||||
skill_dir: str,
|
||||
members: list[str],
|
||||
skill_id: str,
|
||||
version: str,
|
||||
) -> dict:
|
||||
"""Fetch every member file of a resolved skill folder and assemble the install
|
||||
payload (relpaths, scripts list, secret scan, provenance). Shared by the community
|
||||
and curated resolvers so both install the WHOLE folder identically. Fetches text
|
||||
only; never runs anything. `version` is the folder's tree SHA, the update fingerprint."""
|
||||
prefix = (skill_dir + "/") if skill_dir else ""
|
||||
files: dict[str, str] = {}
|
||||
for p in members:
|
||||
rel = p[len(prefix):] if prefix else p
|
||||
raw = await client.get(f"{GH_RAW}/{owner}/{repo}/{branch}/{p}")
|
||||
if raw.status_code == 200:
|
||||
files[rel] = raw.text
|
||||
if "SKILL.md" not in files:
|
||||
raise ValueError("SKILL.md could not be fetched")
|
||||
|
||||
meta, _ = parse_frontmatter(files["SKILL.md"])
|
||||
# Reuse the .swarm importer's content scan: flag files holding secret-shaped literals (the author's leaked key, or a sketchy skill) so the user sees it before installing from an unvetted repo.
|
||||
from backend.common.secret_scan import find_secrets_in_files
|
||||
secret_findings = find_secrets_in_files({rel: data.encode("utf-8", "ignore") for rel, data in files.items()})
|
||||
return {
|
||||
"name": meta.get("name") or skill_id,
|
||||
"description": meta.get("description", ""),
|
||||
"repo_url": f"https://github.com/{owner}/{repo}/tree/{branch}/{skill_dir}".rstrip("/"),
|
||||
"skill_id": skill_id,
|
||||
"files": files,
|
||||
"scripts": sorted(rel for rel in files if is_script_path(rel)),
|
||||
"secret_findings": secret_findings,
|
||||
"source": f"{owner}/{repo}",
|
||||
"folder": skill_dir,
|
||||
"version": version,
|
||||
}
|
||||
|
||||
|
||||
async def resolve_community_skill(source: str, skill_id: str) -> dict:
|
||||
"""Resolve a skills.sh entry (source='owner/repo', skill_id=folder name) to
|
||||
its files via the GitHub trees API. Returns name/description/repo_url plus
|
||||
{relpath: content} and the list of script files. Fetches text only; never
|
||||
runs anything. Raises ValueError on a bad source or a missing skill, and
|
||||
RegistryRateLimited when GitHub's anon API is exhausted."""
|
||||
owner, _, repo = source.partition("/")
|
||||
if not owner or not repo:
|
||||
raise ValueError(f"unrecognized source '{source}' (expected owner/repo)")
|
||||
async with httpx.AsyncClient(timeout=30.0, headers=github_headers()) as client:
|
||||
branch, tree = await fetch_repo_tree(client, owner, repo)
|
||||
skill_md, members = select_skill_paths(tree, skill_id)
|
||||
skill_dir = skill_md[: -len("/SKILL.md")] if "/" in skill_md else ""
|
||||
version = folder_tree_sha(tree, skill_dir)
|
||||
return await p_build_resolved_skill(client, owner, repo, branch, skill_dir, members, skill_id, version)
|
||||
|
||||
|
||||
async def resolve_curated_skill(folder: str) -> dict:
|
||||
"""Resolve a curated (anthropics/skills) skill folder to ALL its files via the
|
||||
GitHub trees API, so multi-file curated skills (pdf/docx/pptx scripts, etc.)
|
||||
install whole instead of just their SKILL.md. The exact folder comes from our
|
||||
catalog, so we match it precisely (not by basename). Same payload shape as
|
||||
resolve_community_skill. Raises ValueError if the folder has no SKILL.md and
|
||||
RegistryRateLimited when GitHub's anon API is exhausted."""
|
||||
owner, _, repo = REPO.partition("/")
|
||||
skill_dir = folder.rstrip("/")
|
||||
skill_id = skill_dir.rsplit("/", 1)[-1]
|
||||
prefix = skill_dir + "/"
|
||||
async with httpx.AsyncClient(timeout=30.0, headers=github_headers()) as client:
|
||||
tree = curated_tree
|
||||
if not tree:
|
||||
# Cold cache (pre-first-refresh, or a failed/rate-limited warm-up): pay one live tree call this once.
|
||||
tree = await tree_at(client, owner, repo, BRANCH)
|
||||
if tree is None:
|
||||
raise ValueError(f"could not read {REPO}@{BRANCH} tree")
|
||||
blobs = tree_blob_paths(tree)
|
||||
if (prefix + "SKILL.md") not in blobs:
|
||||
raise ValueError(f"no SKILL.md at '{folder}'")
|
||||
members = [p for p in blobs if p.startswith(prefix)][:MAX_SKILL_FILES]
|
||||
version = folder_tree_sha(tree, skill_dir)
|
||||
return await p_build_resolved_skill(client, owner, repo, BRANCH, skill_dir, members, skill_id, version)
|
||||
|
||||
|
||||
def search_curated(cache: dict[str, dict], q: str, category: str, offset: int, limit: int) -> dict:
|
||||
"""Filter + paginate the in-memory curated catalog. Pure (cache passed in) so the
|
||||
route stays a thin wrapper and this layer owns all skill-data shaping."""
|
||||
pool = list(cache.values())
|
||||
if category:
|
||||
cat_lower = category.lower()
|
||||
pool = [s for s in pool if s.get("category", "").lower() == cat_lower]
|
||||
|
||||
query_lower = q.lower().strip()
|
||||
if query_lower:
|
||||
pool = [sk for sk in pool if query_lower in f"{sk['name']} {sk['description']} {sk.get('category', '')}".lower()]
|
||||
|
||||
pool.sort(key=lambda s: s["name"].lower())
|
||||
total = len(pool)
|
||||
page = pool[offset: offset + limit]
|
||||
summary = [
|
||||
{
|
||||
"name": s["name"],
|
||||
"description": s["description"],
|
||||
"folder": s["folder"],
|
||||
"category": s.get("category", "General"),
|
||||
"repositoryUrl": s.get("repositoryUrl", ""),
|
||||
}
|
||||
for s in page
|
||||
]
|
||||
return {"skills": summary, "total": total, "offset": offset, "limit": limit}
|
||||
|
||||
|
||||
async def community_search(q: str, limit: int) -> dict:
|
||||
"""Live-proxy a query to the skills.sh wild registry. Not cached: it's a
|
||||
600k-entry remote index, so we search it on demand rather than mirror it."""
|
||||
async with httpx.AsyncClient(timeout=15.0, headers={"User-Agent": "openswarm"}) as client:
|
||||
r = await client.get(COMMUNITY_SEARCH_URL, params={"q": q or "skill"})
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
skills = []
|
||||
for s in (data.get("skills") or [])[:limit]:
|
||||
src = s.get("source", "")
|
||||
try:
|
||||
installs = int(s.get("installs") or 0)
|
||||
except (TypeError, ValueError):
|
||||
installs = 0
|
||||
skills.append({
|
||||
"name": s.get("name", ""),
|
||||
"description": f"{installs:,} installs",
|
||||
"folder": s.get("skillId", ""),
|
||||
"category": src,
|
||||
"repositoryUrl": f"https://github.com/{src}" if src else "",
|
||||
"source": src,
|
||||
"skillId": s.get("skillId", ""),
|
||||
"installs": installs,
|
||||
"community": True,
|
||||
})
|
||||
return {"skills": skills, "total": len(skills), "offset": 0, "limit": limit, "source": "community"}
|
||||
|
||||
|
||||
async def safe_repo_tree(source: str):
|
||||
"""Recursive tree for a community 'owner/repo', cached briefly and best-effort
|
||||
(None on rate-limit / missing repo) so an updates check never fails the whole
|
||||
list because one repo is unreachable."""
|
||||
now = time.time()
|
||||
hit = p_community_tree_cache.get(source)
|
||||
if hit and now - hit[0] < P_COMMUNITY_TREE_TTL:
|
||||
return hit[1]
|
||||
owner, _, repo = source.partition("/")
|
||||
tree = None
|
||||
if owner and repo:
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=30.0, headers=github_headers()) as client:
|
||||
_, tree = await fetch_repo_tree(client, owner, repo)
|
||||
except Exception:
|
||||
tree = None
|
||||
p_community_tree_cache[source] = (now, tree)
|
||||
return tree
|
||||
@@ -13,7 +13,7 @@ import os
|
||||
import pytest
|
||||
|
||||
import backend.apps.skills.skills as skills_mod
|
||||
from backend.apps.skill_registry.skill_registry import select_skill_paths, is_script_path
|
||||
from backend.apps.skill_registry.skill_registry_github import select_skill_paths, is_script_path, tree_blob_paths
|
||||
|
||||
|
||||
def test_selects_shortest_matching_skill_md_at_any_depth():
|
||||
@@ -66,7 +66,7 @@ def test_ambiguous_match_picks_deterministically():
|
||||
|
||||
|
||||
def test_github_headers_adds_token_when_set(monkeypatch):
|
||||
from backend.apps.skill_registry.skill_registry import github_headers
|
||||
from backend.apps.skill_registry.skill_registry_github import github_headers
|
||||
monkeypatch.delenv("OPENSWARM_GITHUB_TOKEN", raising=False)
|
||||
monkeypatch.delenv("GITHUB_TOKEN", raising=False)
|
||||
assert "Authorization" not in github_headers()
|
||||
@@ -159,7 +159,7 @@ def test_confirm_install_writes_folder_lists_and_injects(skills_dir, monkeypatch
|
||||
"scripts/extract.py": "print('extract')"},
|
||||
"scripts": ["scripts/extract.py"], "secret_findings": [],
|
||||
}
|
||||
monkeypatch.setattr("backend.apps.skill_registry.skill_registry.resolve_community_skill", fake_resolve)
|
||||
monkeypatch.setattr("backend.apps.skill_registry.skill_registry_sources.resolve_community_skill", fake_resolve)
|
||||
|
||||
r = client.post("/api/skill-registry/install", json={"source": "o/r", "skill_id": "pdf-tools", "confirm": True})
|
||||
assert r.status_code == 200 and r.json()["installed"] is True
|
||||
@@ -204,8 +204,8 @@ def test_curated_resolve_fetches_exact_folder_only(monkeypatch):
|
||||
skills/pdf-extra) must not leak in. Here the cache is COLD, so it pays one live
|
||||
tree call."""
|
||||
import asyncio
|
||||
import backend.apps.skill_registry.skill_registry as sr
|
||||
monkeypatch.setattr(sr, "p_curated_tree", [])
|
||||
import backend.apps.skill_registry.skill_registry_sources as sr
|
||||
monkeypatch.setattr(sr, "curated_tree",[])
|
||||
|
||||
class FakeClient:
|
||||
async def __aenter__(self):
|
||||
@@ -233,8 +233,8 @@ def test_curated_resolve_uses_warm_cache_with_zero_api_calls(monkeypatch):
|
||||
it reads paths from the cache and pulls contents over raw only. It also records
|
||||
provenance (source/folder/version) for later update detection."""
|
||||
import asyncio
|
||||
import backend.apps.skill_registry.skill_registry as sr
|
||||
monkeypatch.setattr(sr, "p_curated_tree", CURATED_TREE["tree"])
|
||||
import backend.apps.skill_registry.skill_registry_sources as sr
|
||||
monkeypatch.setattr(sr, "curated_tree",CURATED_TREE["tree"])
|
||||
|
||||
class NoApiClient:
|
||||
async def __aenter__(self):
|
||||
@@ -261,8 +261,8 @@ def test_curated_resolve_uses_warm_cache_with_zero_api_calls(monkeypatch):
|
||||
def test_warm_curated_tree_populates_cache(monkeypatch):
|
||||
"""The hourly warm-up caches the repo's full tree so later installs skip the API."""
|
||||
import asyncio
|
||||
import backend.apps.skill_registry.skill_registry as sr
|
||||
monkeypatch.setattr(sr, "p_curated_tree", [])
|
||||
import backend.apps.skill_registry.skill_registry_sources as sr
|
||||
monkeypatch.setattr(sr, "curated_tree",[])
|
||||
|
||||
class TreeClient:
|
||||
async def __aenter__(self):
|
||||
@@ -276,8 +276,8 @@ def test_warm_curated_tree_populates_cache(monkeypatch):
|
||||
return FakeResp(200, payload=CURATED_TREE)
|
||||
|
||||
monkeypatch.setattr(sr.httpx, "AsyncClient", lambda *a, **k: TreeClient())
|
||||
asyncio.run(sr.p_warm_curated_tree())
|
||||
assert "skills/pdf/SKILL.md" in sr.p_tree_blob_paths(sr.p_curated_tree)
|
||||
asyncio.run(sr.warm_curated_tree())
|
||||
assert "skills/pdf/SKILL.md" in tree_blob_paths(sr.curated_tree)
|
||||
|
||||
|
||||
def test_skill_update_detection_and_apply(skills_dir, monkeypatch):
|
||||
@@ -288,14 +288,14 @@ def test_skill_update_detection_and_apply(skills_dir, monkeypatch):
|
||||
from fastapi.testclient import TestClient
|
||||
from backend.main import app
|
||||
import backend.auth as auth_mod
|
||||
import backend.apps.skill_registry.skill_registry as sr
|
||||
import backend.apps.skill_registry.skill_registry_sources as sr
|
||||
import backend.apps.skills.skills as skills_mod
|
||||
if not auth_mod.TOKEN:
|
||||
auth_mod.TOKEN = p_secrets.token_urlsafe(32)
|
||||
client = TestClient(app, headers={"Authorization": f"Bearer {auth_mod.TOKEN}"})
|
||||
|
||||
# Upstream folder SHA for skills/pdf is PDFSHA1 (from the warmed tree).
|
||||
monkeypatch.setattr(sr, "p_curated_tree", CURATED_TREE["tree"])
|
||||
monkeypatch.setattr(sr, "curated_tree",CURATED_TREE["tree"])
|
||||
# Installed copy recorded at a STALE version, so it should read as outdated.
|
||||
skills_mod.write_folder_skill("pdf", {"SKILL.md": "old"}, {
|
||||
"name": "PDF", "source": "anthropics/skills", "folder": "skills/pdf", "version": "OLDSHA",
|
||||
@@ -339,7 +339,7 @@ def test_curated_install_writes_full_folder(skills_dir, monkeypatch):
|
||||
"files": {"SKILL.md": "# PDF\nRun scripts/extract.py", "scripts/extract.py": "print('x')"},
|
||||
"scripts": ["scripts/extract.py"], "secret_findings": [],
|
||||
}
|
||||
monkeypatch.setattr("backend.apps.skill_registry.skill_registry.resolve_curated_skill", fake_resolve)
|
||||
monkeypatch.setattr("backend.apps.skill_registry.skill_registry_sources.resolve_curated_skill", fake_resolve)
|
||||
|
||||
r = client.post("/api/skill-registry/install-curated", json={"folder": "skills/pdf"})
|
||||
assert r.status_code == 200 and r.json()["installed"] is True
|
||||
|
||||
@@ -11,12 +11,13 @@ import json
|
||||
import os
|
||||
|
||||
from backend.apps.skill_registry import skill_registry as sr
|
||||
from backend.apps.skill_registry import skill_registry_cache as cache
|
||||
|
||||
|
||||
def test_bundled_snapshot_exists_and_includes_pdf():
|
||||
# The onboarding step targets the "pdf" skill via /pdf/i; it must be present in the shipped snapshot or the tour times out even with a populated list.
|
||||
assert os.path.exists(sr.BUNDLED_SNAPSHOT)
|
||||
data = json.load(open(sr.BUNDLED_SNAPSHOT, encoding="utf-8"))
|
||||
assert os.path.exists(cache.BUNDLED_SNAPSHOT)
|
||||
data = json.load(open(cache.BUNDLED_SNAPSHOT, encoding="utf-8"))
|
||||
assert isinstance(data, dict) and len(data) >= 10
|
||||
assert any("pdf" in k.lower() or "pdf" in v.get("folder", "").lower()
|
||||
for k, v in data.items())
|
||||
@@ -25,10 +26,10 @@ def test_bundled_snapshot_exists_and_includes_pdf():
|
||||
def test_seed_makes_catalog_non_empty_offline(monkeypatch, tmp_path):
|
||||
# Point the disk cache at an empty tmp dir so only the bundled snapshot can seed; this is the brand-new-install, no-network case.
|
||||
monkeypatch.setenv("OPENSWARM_SKILL_CACHE_DIR", str(tmp_path))
|
||||
seeded = sr.load_seed_cache()
|
||||
seeded = cache.load_seed_cache()
|
||||
assert len(seeded) >= 10
|
||||
|
||||
sr.CACHE = seeded
|
||||
sr.p_cache = seeded
|
||||
res = asyncio.run(sr.registry_search(q="", limit=100, offset=0, sort="name", category=""))
|
||||
assert res["total"] >= 10 and len(res["skills"]) >= 10
|
||||
|
||||
@@ -39,6 +40,6 @@ def test_disk_cache_roundtrip_and_priority(monkeypatch, tmp_path):
|
||||
sentinel = {"only-skill": {"name": "only-skill", "description": "", "content": "",
|
||||
"folder": "skills/only-skill", "category": "Test",
|
||||
"repositoryUrl": ""}}
|
||||
sr.save_disk_cache(sentinel)
|
||||
assert os.path.exists(sr.disk_cache_path())
|
||||
assert sr.load_seed_cache() == sentinel
|
||||
cache.save_disk_cache(sentinel)
|
||||
assert os.path.exists(cache.disk_cache_path())
|
||||
assert cache.load_seed_cache() == sentinel
|
||||
|
||||
Reference in New Issue
Block a user