From 717763771fae1fb1f7ce9a8872de9cf580093ec7 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Mon, 29 Jun 2026 22:27:19 -0700 Subject: [PATCH] [eric] skills: split skill_registry.py (705->275) into sources/github/cache modules to clear the 300-line cap (no cycle, downward imports) --- backend/apps/skill_registry/skill_registry.py | 468 +----------------- .../skill_registry/skill_registry_cache.py | 47 ++ .../skill_registry/skill_registry_github.py | 121 +++++ .../skill_registry/skill_registry_sources.py | 289 +++++++++++ .../tests/test_skill_registry_community.py | 28 +- backend/tests/test_skill_registry_seed.py | 15 +- 6 files changed, 498 insertions(+), 470 deletions(-) create mode 100644 backend/apps/skill_registry/skill_registry_cache.py create mode 100644 backend/apps/skill_registry/skill_registry_github.py create mode 100644 backend/apps/skill_registry/skill_registry_sources.py diff --git a/backend/apps/skill_registry/skill_registry.py b/backend/apps/skill_registry/skill_registry.py index 49a07801..d1731dae 100644 --- a/backend/apps/skill_registry/skill_registry.py +++ b/backend/apps/skill_registry/skill_registry.py @@ -1,191 +1,28 @@ import asyncio -import json import logging -import os -import re import time from contextlib import asynccontextmanager from typing import Optional -import httpx from fastapi import HTTPException, Query from pydantic import BaseModel + from backend.config.Apps import SubApp +from backend.apps.skill_registry import skill_registry_sources as sources +from backend.apps.skill_registry import skill_registry_cache as cache +from backend.apps.skill_registry.skill_registry_github import folder_tree_sha, RegistryRateLimited logger = logging.getLogger(__name__) -REPO = "anthropics/skills" -BRANCH = "main" -RAW_BASE = f"https://raw.githubusercontent.com/{REPO}/{BRANCH}" -MANIFEST_URL = f"{RAW_BASE}/.claude-plugin/marketplace.json" REFRESH_INTERVAL_S = 3600 -CONCURRENT_FETCHES = 15 # Retry the startup fetch on this short backoff (capped) until the FIRST success, instead of waiting a full REFRESH_INTERVAL_S after a cold/slow/failed fetch. That 1h gap was the "skills empty until reboot" bug on cold Windows networks. P_RETRY_BACKOFF_START_S = 2 P_RETRY_BACKOFF_MAX_S = 60 -# Catalog ships in the repo so a brand-new install shows skills with zero network (build snapshot), and every successful live fetch is persisted to the user's cache so subsequent launches are instant + offline-safe. The live fetch always overwrites both once it lands, so neither can go stale at runtime. -BUNDLED_SNAPSHOT = os.path.join(os.path.dirname(__file__), "skills_snapshot.json") - p_cache: dict[str, dict] = {} p_cache_updated_at: float = 0 p_refresh_task: Optional[asyncio.Task] = None -# The curated repo's recursive file tree, warmed hourly alongside the catalog. A curated install reads paths from here and fetches contents over raw, so it makes ZERO GitHub API calls in the normal case (the trees API is the 60/hr-limited part); update detection reads per-folder tree SHAs from it too. Empty until the first refresh warms it; install falls back to one live tree call then. -p_curated_tree: list[dict] = [] -# Community repo trees for update detection, cached briefly (best-effort) so an updates check on skills.sh-installed skills doesn't refetch every page load nor burn the API. -P_COMMUNITY_TREE_TTL = 600 -p_community_tree_cache: dict[str, tuple] = {} - - -def disk_cache_path() -> str: - base = os.environ.get("OPENSWARM_SKILL_CACHE_DIR") or os.path.expanduser( - "~/.openswarm/cache" - ) - return os.path.join(base, "skill_registry.json") - - -def load_seed_cache() -> dict[str, dict]: - """Return a non-empty catalog from the on-disk last-good cache, falling back - to the bundled snapshot, so the registry is never empty on a cold/offline - start. Returns {} only if neither source is present/valid.""" - for path in (disk_cache_path(), BUNDLED_SNAPSHOT): - try: - with open(path, encoding="utf-8") as f: - data = json.load(f) - if isinstance(data, dict) and data: - logger.info(f"Skill registry: seeded {len(data)} skills from {os.path.basename(path)}") - return data - except (OSError, ValueError): - continue - return {} - - -def save_disk_cache(skills: dict[str, dict]) -> None: - """Persist the last good live fetch so the next launch is instant. Atomic - replace so a crash mid-write can't leave a truncated cache.""" - if not skills: - return - path = disk_cache_path() - try: - os.makedirs(os.path.dirname(path), exist_ok=True) - tmp = f"{path}.tmp" - with open(tmp, "w", encoding="utf-8") as f: - json.dump(skills, f) - os.replace(tmp, path) - except OSError: - logger.debug("Skill registry: could not persist disk cache", exc_info=True) - - -def p_parse_frontmatter(raw: str) -> tuple[dict, str]: - """Split YAML frontmatter from markdown body.""" - if not raw.startswith("---"): - return {}, raw - end = raw.find("---", 3) - if end == -1: - return {}, raw - fm_block = raw[3:end].strip() - body = raw[end + 3:].strip() - meta: dict = {} - for line in fm_block.splitlines(): - m = re.match(r"^(\w[\w_-]*)\s*:\s*(.+)$", line) - if m: - meta[m.group(1).strip()] = m.group(2).strip().strip('"').strip("'") - return meta, body - - -async def p_fetch_skill_paths(client: httpx.AsyncClient) -> list[tuple[str, str]]: - """Fetch the marketplace.json manifest and return (skill_folder, plugin_name) pairs. - - Uses raw.githubusercontent.com; no GitHub API needed, no rate limiting. - """ - resp = await client.get(MANIFEST_URL) - resp.raise_for_status() - manifest = resp.json() - - paths: list[tuple[str, str]] = [] - for plugin in manifest.get("plugins", []): - plugin_name = plugin.get("name", "") - for skill_ref in plugin.get("skills", []): - folder = skill_ref.lstrip("./") - paths.append((folder, plugin_name)) - return paths - - -async def p_fetch_one_skill( - client: httpx.AsyncClient, - sem: asyncio.Semaphore, - folder: str, - plugin_name: str, -) -> Optional[dict]: - async with sem: - try: - resp = await client.get(f"{RAW_BASE}/{folder}/SKILL.md") - if resp.status_code != 200: - return None - raw = resp.text - except Exception as exc: - logger.debug(f"Failed to fetch {folder}/SKILL.md: {exc}") - return None - - meta, body = p_parse_frontmatter(raw) - name = meta.get("name", "") - if not name: - folder_name = folder.rsplit("/", 1)[-1] - name = folder_name.replace("-", " ").replace("_", " ").title() - - return { - "name": name, - "description": meta.get("description", ""), - "content": body, - "folder": folder, - "category": plugin_name.replace("-", " ").replace("_", " ").title(), - "repositoryUrl": f"https://github.com/{REPO}/tree/{BRANCH}/{folder}", - } - - -async def p_fetch_all_skills() -> dict[str, dict]: - skills: dict[str, dict] = {} - async with httpx.AsyncClient(timeout=30.0) as client: - try: - paths = await p_fetch_skill_paths(client) - except Exception as e: - logger.warning(f"Skill registry manifest fetch failed: {e}") - return skills - - logger.info(f"Skill registry: found {len(paths)} skills in manifest, fetching content...") - sem = asyncio.Semaphore(CONCURRENT_FETCHES) - results = await asyncio.gather( - *[p_fetch_one_skill(client, sem, folder, plugin) for folder, plugin in paths] - ) - for rec in results: - if rec: - skills[rec["name"]] = rec - - logger.info(f"Skill registry cache refreshed: {len(skills)} skills") - return skills - - -async def p_warm_curated_tree() -> None: - """Best-effort: list the anthropics/skills repo once and cache its file paths so - curated installs need ZERO trees-API calls (they read paths here, fetch contents - over raw). One cheap call per hourly refresh, reused by every install in that hour. - Isolated, a failure here never touches the SKILL.md catalog; install falls back to - a live tree call while the cache is cold.""" - global p_curated_tree - owner, _, repo = REPO.partition("/") - try: - async with httpx.AsyncClient(timeout=30.0, headers=github_headers()) as client: - tree = await p_tree_at(client, owner, repo, BRANCH) - if tree: - p_curated_tree = tree - logger.info(f"Curated skill tree warmed: {len(p_tree_blob_paths(tree))} file paths cached") - except RegistryRateLimited: - # Visible on purpose: a rate-limited warm-up means installs stay on the slow live-call path until the IP's quota resets or a token is set. - logger.warning("Curated tree warm-up rate-limited by GitHub (60/hr anon limit). Set GITHUB_TOKEN or wait for the hourly reset; installs use a live tree call meanwhile.") - except Exception: - logger.debug("curated tree warm-up failed; installs fall back to a live tree call", exc_info=True) - async def p_refresh_loop(): global p_cache, p_cache_updated_at @@ -193,17 +30,17 @@ async def p_refresh_loop(): while True: ok = False try: - fetched = await p_fetch_all_skills() + fetched = await sources.fetch_all_skills() if fetched: p_cache = fetched p_cache_updated_at = time.time() - save_disk_cache(p_cache) + cache.save_disk_cache(p_cache) ok = True except Exception as e: logger.exception(f"Skill registry refresh error: {e}") if ok: # Warm the curated file-tree on the SLOW path only (never on the fast failure-retry below, which would burn the 60/hr quota in seconds). - await p_warm_curated_tree() + await sources.warm_curated_tree() # Settle to the slow hourly refresh once we have a good catalog. backoff = P_RETRY_BACKOFF_START_S await asyncio.sleep(REFRESH_INTERVAL_S) @@ -218,7 +55,7 @@ async def skill_registry_lifespan(): global p_refresh_task, p_cache # Seed instantly from disk/bundled snapshot so the very first request never sees an empty catalog (the live fetch below overwrites it when it lands). if not p_cache: - p_cache = load_seed_cache() + p_cache = cache.load_seed_cache() p_refresh_task = asyncio.create_task(p_refresh_loop()) yield if p_refresh_task: @@ -257,40 +94,11 @@ async def registry_search( # The wild registry is a remote 600k-entry index, searched live, not mirrored. if source == "community": try: - return await p_community_search(q, limit) + return await sources.community_search(q, limit) except Exception as e: logger.warning(f"community skill search failed: {e}") return {"skills": [], "total": 0, "offset": 0, "limit": limit, "source": "community", "error": "skills.sh unreachable"} - - pool = list(p_cache.values()) - if category: - cat_lower = category.lower() - pool = [s for s in pool if s.get("category", "").lower() == cat_lower] - - query_lower = q.lower().strip() - if query_lower: - filtered = [] - for sk in pool: - searchable = f"{sk['name']} {sk['description']} {sk.get('category', '')}".lower() - if query_lower in searchable: - filtered.append(sk) - pool = filtered - - pool.sort(key=lambda s: s["name"].lower()) - total = len(pool) - page = pool[offset : offset + limit] - - summary = [ - { - "name": s["name"], - "description": s["description"], - "folder": s["folder"], - "category": s.get("category", "General"), - "repositoryUrl": s.get("repositoryUrl", ""), - } - for s in page - ] - return {"skills": summary, "total": total, "offset": offset, "limit": limit} + return sources.search_curated(p_cache, q, category, offset, limit) @skill_registry.router.get("/detail/{skill_name:path}") @@ -301,224 +109,6 @@ async def registry_detail(skill_name: str): return {"skill": sk} -# --------------------------------------------------------------------------- Community source: the skills.sh wild registry (~600k+ telemetry-ranked, zero-curation community skills, GitHub-repo backed). The curated source above (anthropics/skills) stays the default; community is opt-in via ?source=community and the UI flags it as unvetted. See .claude/SECURITY.md for the posture: this installs INERT files only (never executes), discloses scripts before commit, and any skill script later runs through the same gated Bash path as anything. --------------------------------------------------------------------------- - -P_COMMUNITY_SEARCH_URL = "https://skills.sh/api/search" -P_GH_API = "https://api.github.com" -P_GH_RAW = "https://raw.githubusercontent.com" -P_MAX_SKILL_FILES = 60 -P_SCRIPT_EXTS = (".sh", ".py", ".js", ".mjs", ".cjs", ".ts", ".rb", ".pl", ".ps1", ".bat", ".php") - - -def is_script_path(rel: str) -> bool: - """Whether a skill file is executable code worth disclosing before install.""" - if rel.lower().endswith(P_SCRIPT_EXTS): - return True - head = rel.split("/", 1)[0].lower() - return head in ("scripts", "bin", "hooks") - - -def github_headers() -> dict: - """GitHub request headers, with auth if a token is set. Unauthenticated is - 60 req/hr/IP (fine for the odd install, the wall for a power user); a token - (OPENSWARM_GITHUB_TOKEN or GITHUB_TOKEN) raises it to 5000/hr.""" - headers = {"User-Agent": "openswarm-skill-registry", "Accept": "application/vnd.github+json"} - token = os.environ.get("OPENSWARM_GITHUB_TOKEN") or os.environ.get("GITHUB_TOKEN") - if token: - headers["Authorization"] = f"Bearer {token}" - return headers - - -def select_skill_paths(tree: list[dict], skill_id: str) -> tuple[str, list[str]]: - """From a GitHub recursive tree, pick the SKILL.md for `skill_id` and every - file beside it. Pure, so the resolution logic is unit-tested without a network - round-trip. When a repo has several `//SKILL.md` matches the pick - is deterministic: prefer a top-level `/`, then `skills//`, - then the shallowest, then alphabetical, never an arbitrary tie.""" - blobs = [t["path"] for t in tree if t.get("type") == "blob" and isinstance(t.get("path"), str)] - candidates = [p for p in blobs if p.endswith(f"/{skill_id}/SKILL.md") or p == f"{skill_id}/SKILL.md"] - if not candidates: - raise ValueError(f"no SKILL.md for '{skill_id}' in this repo") - - def p_rank(p: str) -> tuple: - if p == f"{skill_id}/SKILL.md": - return (0, 0, p) - if p == f"skills/{skill_id}/SKILL.md": - return (1, p.count("/"), p) - return (2, p.count("/"), p) - - skill_md = min(candidates, key=p_rank) - skill_dir = skill_md[: -len("/SKILL.md")] if "/" in skill_md else "" - prefix = (skill_dir + "/") if skill_dir else "" - members = [p for p in blobs if (p.startswith(prefix) if prefix else "/" not in p)] - return skill_md, members[:P_MAX_SKILL_FILES] - - -class RegistryRateLimited(Exception): - """GitHub's unauthenticated API (60/hr) is exhausted; the caller surfaces a - 'try again shortly' rather than a generic failure.""" - - -def p_tree_blob_paths(tree: list[dict]) -> list[str]: - """The blob (file) paths from a GitHub recursive tree, ignoring tree (dir) entries.""" - return [t["path"] for t in tree if t.get("type") == "blob" and isinstance(t.get("path"), str)] - - -def p_folder_tree_sha(tree: list[dict], folder: str) -> str: - """The git tree SHA of `folder` within a recursive tree: a per-folder fingerprint - that changes iff something inside it changes, so one skill going stale never marks - its siblings stale. '' when the folder isn't present as a tree entry.""" - for t in tree: - if t.get("type") == "tree" and t.get("path") == folder: - return t.get("sha", "") or "" - return "" - - -async def p_tree_at(client: httpx.AsyncClient, owner: str, repo: str, branch: str): - """(tree | None) for a branch. None on 404 (branch absent); raises on rate limit. - GitHub signals the limit as 403 (primary) or 429 (secondary), so treat both.""" - r = await client.get(f"{P_GH_API}/repos/{owner}/{repo}/git/trees/{branch}?recursive=1") - if r.status_code == 200: - return r.json().get("tree", []) - if r.status_code in (403, 429): - raise RegistryRateLimited() - return None - - -async def p_fetch_repo_tree(client: httpx.AsyncClient, owner: str, repo: str) -> tuple[str, list[dict]]: - """Recursive tree of owner/repo. Tries main then master first (one call, the - 99% case, no quota wasted on a repo-meta lookup); only if BOTH are absent - does it ask the repo for its real default branch (handles develop/trunk/etc). - Raises RegistryRateLimited on a 403, ValueError if no branch resolves.""" - for branch in ("main", "master"): - tree = await p_tree_at(client, owner, repo, branch) - if tree is not None: - return branch, tree - meta = await client.get(f"{P_GH_API}/repos/{owner}/{repo}") - if meta.status_code == 403: - raise RegistryRateLimited() - if meta.status_code == 200: - default = meta.json().get("default_branch") - if default and default not in ("main", "master"): - tree = await p_tree_at(client, owner, repo, default) - if tree is not None: - return default, tree - raise ValueError(f"repo {owner}/{repo} has no resolvable default branch") - - -async def p_build_resolved_skill( - client: httpx.AsyncClient, - owner: str, - repo: str, - branch: str, - skill_dir: str, - members: list[str], - skill_id: str, - version: str, -) -> dict: - """Fetch every member file of a resolved skill folder and assemble the install - payload (relpaths, scripts list, secret scan, provenance). Shared by the community - and curated resolvers so both install the WHOLE folder identically. Fetches text - only; never runs anything. `version` is the folder's tree SHA, the update fingerprint.""" - prefix = (skill_dir + "/") if skill_dir else "" - files: dict[str, str] = {} - for p in members: - rel = p[len(prefix):] if prefix else p - raw = await client.get(f"{P_GH_RAW}/{owner}/{repo}/{branch}/{p}") - if raw.status_code == 200: - files[rel] = raw.text - if "SKILL.md" not in files: - raise ValueError("SKILL.md could not be fetched") - - meta, _ = p_parse_frontmatter(files["SKILL.md"]) - # Reuse the .swarm importer's content scan: flag files holding secret-shaped literals (the author's leaked key, or a sketchy skill) so the user sees it before installing from an unvetted repo. - from backend.common.secret_scan import find_secrets_in_files - secret_findings = find_secrets_in_files({rel: data.encode("utf-8", "ignore") for rel, data in files.items()}) - return { - "name": meta.get("name") or skill_id, - "description": meta.get("description", ""), - "repo_url": f"https://github.com/{owner}/{repo}/tree/{branch}/{skill_dir}".rstrip("/"), - "skill_id": skill_id, - "files": files, - "scripts": sorted(rel for rel in files if is_script_path(rel)), - "secret_findings": secret_findings, - "source": f"{owner}/{repo}", - "folder": skill_dir, - "version": version, - } - - -async def resolve_community_skill(source: str, skill_id: str) -> dict: - """Resolve a skills.sh entry (source='owner/repo', skill_id=folder name) to - its files via the GitHub trees API. Returns name/description/repo_url plus - {relpath: content} and the list of script files. Fetches text only; never - runs anything. Raises ValueError on a bad source or a missing skill, and - RegistryRateLimited when GitHub's anon API is exhausted.""" - owner, _, repo = source.partition("/") - if not owner or not repo: - raise ValueError(f"unrecognized source '{source}' (expected owner/repo)") - async with httpx.AsyncClient(timeout=30.0, headers=github_headers()) as client: - branch, tree = await p_fetch_repo_tree(client, owner, repo) - skill_md, members = select_skill_paths(tree, skill_id) - skill_dir = skill_md[: -len("/SKILL.md")] if "/" in skill_md else "" - version = p_folder_tree_sha(tree, skill_dir) - return await p_build_resolved_skill(client, owner, repo, branch, skill_dir, members, skill_id, version) - - -async def resolve_curated_skill(folder: str) -> dict: - """Resolve a curated (anthropics/skills) skill folder to ALL its files via the - GitHub trees API, so multi-file curated skills (pdf/docx/pptx scripts, etc.) - install whole instead of just their SKILL.md. The exact folder comes from our - catalog, so we match it precisely (not by basename). Same payload shape as - resolve_community_skill. Raises ValueError if the folder has no SKILL.md and - RegistryRateLimited when GitHub's anon API is exhausted.""" - owner, _, repo = REPO.partition("/") - skill_dir = folder.rstrip("/") - skill_id = skill_dir.rsplit("/", 1)[-1] - prefix = skill_dir + "/" - async with httpx.AsyncClient(timeout=30.0, headers=github_headers()) as client: - tree = p_curated_tree - if not tree: - # Cold cache (pre-first-refresh, or a failed/rate-limited warm-up): pay one live tree call this once. - tree = await p_tree_at(client, owner, repo, BRANCH) - if tree is None: - raise ValueError(f"could not read {REPO}@{BRANCH} tree") - blobs = p_tree_blob_paths(tree) - if (prefix + "SKILL.md") not in blobs: - raise ValueError(f"no SKILL.md at '{folder}'") - members = [p for p in blobs if p.startswith(prefix)][:P_MAX_SKILL_FILES] - version = p_folder_tree_sha(tree, skill_dir) - return await p_build_resolved_skill(client, owner, repo, BRANCH, skill_dir, members, skill_id, version) - - -async def p_community_search(q: str, limit: int) -> dict: - """Live-proxy a query to the skills.sh wild registry. Not cached: it's a - 600k-entry remote index, so we search it on demand rather than mirror it.""" - async with httpx.AsyncClient(timeout=15.0, headers={"User-Agent": "openswarm"}) as client: - r = await client.get(P_COMMUNITY_SEARCH_URL, params={"q": q or "skill"}) - r.raise_for_status() - data = r.json() - skills = [] - for s in (data.get("skills") or [])[:limit]: - src = s.get("source", "") - try: - installs = int(s.get("installs") or 0) - except (TypeError, ValueError): - installs = 0 - skills.append({ - "name": s.get("name", ""), - "description": f"{installs:,} installs", - "folder": s.get("skillId", ""), - "category": src, - "repositoryUrl": f"https://github.com/{src}" if src else "", - "source": src, - "skillId": s.get("skillId", ""), - "installs": installs, - "community": True, - }) - return {"skills": skills, "total": len(skills), "offset": 0, "limit": limit, "source": "community"} - - class p_InstallRequest(BaseModel): source: str skill_id: str @@ -536,7 +126,7 @@ async def registry_install(req: p_InstallRequest): script is executed here. Curated skills install via the normal skills CRUD; this endpoint is the wild-registry path.""" try: - resolved = await resolve_community_skill(req.source, req.skill_id) + resolved = await sources.resolve_community_skill(req.source, req.skill_id) except RegistryRateLimited: raise HTTPException(status_code=429, detail="GitHub rate limit hit fetching this skill; try again in a few minutes.") except ValueError as e: @@ -583,7 +173,7 @@ async def registry_install_curated(req: p_CuratedInstallRequest): the vetted source, so this is one-click; files are still written inert, never executed. Needs network at install time (the catalog only caches SKILL.md).""" try: - resolved = await resolve_curated_skill(req.folder) + resolved = await sources.resolve_curated_skill(req.folder) except RegistryRateLimited: raise HTTPException(status_code=429, detail="GitHub rate limit hit fetching this skill; try again in a few minutes.") except ValueError as e: @@ -609,26 +199,6 @@ async def registry_install_curated(req: p_CuratedInstallRequest): } -async def p_safe_repo_tree(source: str): - """Recursive tree for a community 'owner/repo', cached briefly and best-effort - (None on rate-limit / missing repo) so an updates check never fails the whole - list because one repo is unreachable.""" - now = time.time() - hit = p_community_tree_cache.get(source) - if hit and now - hit[0] < P_COMMUNITY_TREE_TTL: - return hit[1] - owner, _, repo = source.partition("/") - tree = None - if owner and repo: - try: - async with httpx.AsyncClient(timeout=30.0, headers=github_headers()) as client: - _, tree = await p_fetch_repo_tree(client, owner, repo) - except Exception: - tree = None - p_community_tree_cache[source] = (now, tree) - return tree - - @skill_registry.router.get("/updates") async def registry_updates(): """Which installed skills have a newer version upstream. Curated skills check @@ -643,16 +213,16 @@ async def registry_updates(): for s in sync_skills(): if not s.source or not s.folder or not s.version: continue - if s.source == REPO: - tree = p_curated_tree + if s.source == sources.REPO: + tree = sources.curated_tree else: if s.source not in community_trees: - community_trees[s.source] = await p_safe_repo_tree(s.source) + community_trees[s.source] = await sources.safe_repo_tree(s.source) tree = community_trees[s.source] if not tree: unknown.append(s.id) continue - current = p_folder_tree_sha(tree, s.folder) + current = folder_tree_sha(tree, s.folder) checked.append(s.id) if current and current != s.version: outdated.append(s.id) @@ -676,10 +246,10 @@ async def registry_update(req: p_UpdateRequest): if not target.source or not target.folder: raise HTTPException(status_code=400, detail="this skill has no upstream source to update from") try: - if target.source == REPO: - resolved = await resolve_curated_skill(target.folder) + if target.source == sources.REPO: + resolved = await sources.resolve_curated_skill(target.folder) else: - resolved = await resolve_community_skill(target.source, target.folder.rsplit("/", 1)[-1]) + resolved = await sources.resolve_community_skill(target.source, target.folder.rsplit("/", 1)[-1]) except RegistryRateLimited: raise HTTPException(status_code=429, detail="GitHub rate limit hit; try again in a few minutes.") except ValueError as e: diff --git a/backend/apps/skill_registry/skill_registry_cache.py b/backend/apps/skill_registry/skill_registry_cache.py new file mode 100644 index 00000000..e9ba3d2a --- /dev/null +++ b/backend/apps/skill_registry/skill_registry_cache.py @@ -0,0 +1,47 @@ +import json +import logging +import os + +logger = logging.getLogger(__name__) + +# Catalog ships in the repo so a brand-new install shows skills with zero network (build snapshot), and every successful live fetch is persisted to the user's cache so subsequent launches are instant + offline-safe. The live fetch always overwrites both once it lands, so neither can go stale at runtime. +BUNDLED_SNAPSHOT = os.path.join(os.path.dirname(__file__), "skills_snapshot.json") + + +def disk_cache_path() -> str: + base = os.environ.get("OPENSWARM_SKILL_CACHE_DIR") or os.path.expanduser( + "~/.openswarm/cache" + ) + return os.path.join(base, "skill_registry.json") + + +def load_seed_cache() -> dict[str, dict]: + """Return a non-empty catalog from the on-disk last-good cache, falling back + to the bundled snapshot, so the registry is never empty on a cold/offline + start. Returns {} only if neither source is present/valid.""" + for path in (disk_cache_path(), BUNDLED_SNAPSHOT): + try: + with open(path, encoding="utf-8") as f: + data = json.load(f) + if isinstance(data, dict) and data: + logger.info(f"Skill registry: seeded {len(data)} skills from {os.path.basename(path)}") + return data + except (OSError, ValueError): + continue + return {} + + +def save_disk_cache(skills: dict[str, dict]) -> None: + """Persist the last good live fetch so the next launch is instant. Atomic + replace so a crash mid-write can't leave a truncated cache.""" + if not skills: + return + path = disk_cache_path() + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + tmp = f"{path}.tmp" + with open(tmp, "w", encoding="utf-8") as f: + json.dump(skills, f) + os.replace(tmp, path) + except OSError: + logger.debug("Skill registry: could not persist disk cache", exc_info=True) diff --git a/backend/apps/skill_registry/skill_registry_github.py b/backend/apps/skill_registry/skill_registry_github.py new file mode 100644 index 00000000..d867c70a --- /dev/null +++ b/backend/apps/skill_registry/skill_registry_github.py @@ -0,0 +1,121 @@ +import os +import re + +import httpx + +GH_API = "https://api.github.com" +MAX_SKILL_FILES = 60 +SCRIPT_EXTS = (".sh", ".py", ".js", ".mjs", ".cjs", ".ts", ".rb", ".pl", ".ps1", ".bat", ".php") + + +class RegistryRateLimited(Exception): + """GitHub's unauthenticated API (60/hr) is exhausted; the caller surfaces a + 'try again shortly' rather than a generic failure.""" + + +def parse_frontmatter(raw: str) -> tuple[dict, str]: + """Split YAML frontmatter from markdown body.""" + if not raw.startswith("---"): + return {}, raw + end = raw.find("---", 3) + if end == -1: + return {}, raw + fm_block = raw[3:end].strip() + body = raw[end + 3:].strip() + meta: dict = {} + for line in fm_block.splitlines(): + m = re.match(r"^(\w[\w_-]*)\s*:\s*(.+)$", line) + if m: + meta[m.group(1).strip()] = m.group(2).strip().strip('"').strip("'") + return meta, body + + +def is_script_path(rel: str) -> bool: + """Whether a skill file is executable code worth disclosing before install.""" + if rel.lower().endswith(SCRIPT_EXTS): + return True + head = rel.split("/", 1)[0].lower() + return head in ("scripts", "bin", "hooks") + + +def github_headers() -> dict: + """GitHub request headers, with auth if a token is set. Unauthenticated is + 60 req/hr/IP (fine for the odd install, the wall for a power user); a token + (OPENSWARM_GITHUB_TOKEN or GITHUB_TOKEN) raises it to 5000/hr.""" + headers = {"User-Agent": "openswarm-skill-registry", "Accept": "application/vnd.github+json"} + token = os.environ.get("OPENSWARM_GITHUB_TOKEN") or os.environ.get("GITHUB_TOKEN") + if token: + headers["Authorization"] = f"Bearer {token}" + return headers + + +def select_skill_paths(tree: list[dict], skill_id: str) -> tuple[str, list[str]]: + """From a GitHub recursive tree, pick the SKILL.md for `skill_id` and every + file beside it. Pure, so the resolution logic is unit-tested without a network + round-trip. When a repo has several `//SKILL.md` matches the pick + is deterministic: prefer a top-level `/`, then `skills//`, + then the shallowest, then alphabetical, never an arbitrary tie.""" + blobs = [t["path"] for t in tree if t.get("type") == "blob" and isinstance(t.get("path"), str)] + candidates = [p for p in blobs if p.endswith(f"/{skill_id}/SKILL.md") or p == f"{skill_id}/SKILL.md"] + if not candidates: + raise ValueError(f"no SKILL.md for '{skill_id}' in this repo") + + def p_rank(p: str) -> tuple: + if p == f"{skill_id}/SKILL.md": + return (0, 0, p) + if p == f"skills/{skill_id}/SKILL.md": + return (1, p.count("/"), p) + return (2, p.count("/"), p) + + skill_md = min(candidates, key=p_rank) + skill_dir = skill_md[: -len("/SKILL.md")] if "/" in skill_md else "" + prefix = (skill_dir + "/") if skill_dir else "" + members = [p for p in blobs if (p.startswith(prefix) if prefix else "/" not in p)] + return skill_md, members[:MAX_SKILL_FILES] + + +def tree_blob_paths(tree: list[dict]) -> list[str]: + """The blob (file) paths from a GitHub recursive tree, ignoring tree (dir) entries.""" + return [t["path"] for t in tree if t.get("type") == "blob" and isinstance(t.get("path"), str)] + + +def folder_tree_sha(tree: list[dict], folder: str) -> str: + """The git tree SHA of `folder` within a recursive tree: a per-folder fingerprint + that changes iff something inside it changes, so one skill going stale never marks + its siblings stale. '' when the folder isn't present as a tree entry.""" + for t in tree: + if t.get("type") == "tree" and t.get("path") == folder: + return t.get("sha", "") or "" + return "" + + +async def tree_at(client: httpx.AsyncClient, owner: str, repo: str, branch: str): + """(tree | None) for a branch. None on 404 (branch absent); raises on rate limit. + GitHub signals the limit as 403 (primary) or 429 (secondary), so treat both.""" + r = await client.get(f"{GH_API}/repos/{owner}/{repo}/git/trees/{branch}?recursive=1") + if r.status_code == 200: + return r.json().get("tree", []) + if r.status_code in (403, 429): + raise RegistryRateLimited() + return None + + +async def fetch_repo_tree(client: httpx.AsyncClient, owner: str, repo: str) -> tuple[str, list[dict]]: + """Recursive tree of owner/repo. Tries main then master first (one call, the + 99% case, no quota wasted on a repo-meta lookup); only if BOTH are absent + does it ask the repo for its real default branch (handles develop/trunk/etc). + Raises RegistryRateLimited on a 403, ValueError if no branch resolves.""" + for branch in ("main", "master"): + tree = await tree_at(client, owner, repo, branch) + if tree is not None: + return branch, tree + meta = await client.get(f"{GH_API}/repos/{owner}/{repo}") + if meta.status_code == 403: + raise RegistryRateLimited() + if meta.status_code == 200: + default = meta.json().get("default_branch") + if default and default not in ("main", "master"): + tree = await tree_at(client, owner, repo, default) + if tree is not None: + return default, tree + raise ValueError(f"repo {owner}/{repo} has no resolvable default branch") diff --git a/backend/apps/skill_registry/skill_registry_sources.py b/backend/apps/skill_registry/skill_registry_sources.py new file mode 100644 index 00000000..60bc33b1 --- /dev/null +++ b/backend/apps/skill_registry/skill_registry_sources.py @@ -0,0 +1,289 @@ +import asyncio +import logging +import time +from typing import Optional + +import httpx + +from backend.apps.skill_registry.skill_registry_github import ( + RegistryRateLimited, + parse_frontmatter, + is_script_path, + github_headers, + select_skill_paths, + tree_blob_paths, + folder_tree_sha, + tree_at, + fetch_repo_tree, + MAX_SKILL_FILES, +) + +logger = logging.getLogger(__name__) + +REPO = "anthropics/skills" +BRANCH = "main" +RAW_BASE = f"https://raw.githubusercontent.com/{REPO}/{BRANCH}" +MANIFEST_URL = f"{RAW_BASE}/.claude-plugin/marketplace.json" +CONCURRENT_FETCHES = 15 +GH_RAW = "https://raw.githubusercontent.com" +COMMUNITY_SEARCH_URL = "https://skills.sh/api/search" +P_COMMUNITY_TREE_TTL = 600 + +# The curated repo's recursive file tree, warmed hourly alongside the catalog. A curated install reads paths from here and fetches contents over raw, so it makes ZERO GitHub API calls in the normal case (the trees API is the 60/hr-limited part); update detection reads per-folder tree SHAs from it too. Empty until the first refresh warms it; install falls back to one live tree call then. +curated_tree: list[dict] = [] +# Community repo trees for update detection, cached briefly (best-effort) so an updates check on skills.sh-installed skills doesn't refetch every page load nor burn the API. +p_community_tree_cache: dict[str, tuple] = {} + + +async def p_fetch_skill_paths(client: httpx.AsyncClient) -> list[tuple[str, str]]: + """Fetch the marketplace.json manifest and return (skill_folder, plugin_name) pairs. + + Uses raw.githubusercontent.com; no GitHub API needed, no rate limiting. + """ + resp = await client.get(MANIFEST_URL) + resp.raise_for_status() + manifest = resp.json() + + paths: list[tuple[str, str]] = [] + for plugin in manifest.get("plugins", []): + plugin_name = plugin.get("name", "") + for skill_ref in plugin.get("skills", []): + folder = skill_ref.lstrip("./") + paths.append((folder, plugin_name)) + return paths + + +async def p_fetch_one_skill( + client: httpx.AsyncClient, + sem: asyncio.Semaphore, + folder: str, + plugin_name: str, +) -> Optional[dict]: + async with sem: + try: + resp = await client.get(f"{RAW_BASE}/{folder}/SKILL.md") + if resp.status_code != 200: + return None + raw = resp.text + except Exception as exc: + logger.debug(f"Failed to fetch {folder}/SKILL.md: {exc}") + return None + + meta, body = parse_frontmatter(raw) + name = meta.get("name", "") + if not name: + folder_name = folder.rsplit("/", 1)[-1] + name = folder_name.replace("-", " ").replace("_", " ").title() + + return { + "name": name, + "description": meta.get("description", ""), + "content": body, + "folder": folder, + "category": plugin_name.replace("-", " ").replace("_", " ").title(), + "repositoryUrl": f"https://github.com/{REPO}/tree/{BRANCH}/{folder}", + } + + +async def fetch_all_skills() -> dict[str, dict]: + skills: dict[str, dict] = {} + async with httpx.AsyncClient(timeout=30.0) as client: + try: + paths = await p_fetch_skill_paths(client) + except Exception as e: + logger.warning(f"Skill registry manifest fetch failed: {e}") + return skills + + logger.info(f"Skill registry: found {len(paths)} skills in manifest, fetching content...") + sem = asyncio.Semaphore(CONCURRENT_FETCHES) + results = await asyncio.gather( + *[p_fetch_one_skill(client, sem, folder, plugin) for folder, plugin in paths] + ) + for rec in results: + if rec: + skills[rec["name"]] = rec + + logger.info(f"Skill registry cache refreshed: {len(skills)} skills") + return skills + + +async def warm_curated_tree() -> None: + """Best-effort: list the anthropics/skills repo once and cache its file paths so + curated installs need ZERO trees-API calls (they read paths here, fetch contents + over raw). One cheap call per hourly refresh, reused by every install in that hour. + Isolated, a failure here never touches the SKILL.md catalog; install falls back to + a live tree call while the cache is cold.""" + global curated_tree + owner, _, repo = REPO.partition("/") + try: + async with httpx.AsyncClient(timeout=30.0, headers=github_headers()) as client: + tree = await tree_at(client, owner, repo, BRANCH) + if tree: + curated_tree = tree + logger.info(f"Curated skill tree warmed: {len(tree_blob_paths(tree))} file paths cached") + except RegistryRateLimited: + # Visible on purpose: a rate-limited warm-up means installs stay on the slow live-call path until the IP's quota resets or a token is set. + logger.warning("Curated tree warm-up rate-limited by GitHub (60/hr anon limit). Set GITHUB_TOKEN or wait for the hourly reset; installs use a live tree call meanwhile.") + except Exception: + logger.debug("curated tree warm-up failed; installs fall back to a live tree call", exc_info=True) + + +async def p_build_resolved_skill( + client: httpx.AsyncClient, + owner: str, + repo: str, + branch: str, + skill_dir: str, + members: list[str], + skill_id: str, + version: str, +) -> dict: + """Fetch every member file of a resolved skill folder and assemble the install + payload (relpaths, scripts list, secret scan, provenance). Shared by the community + and curated resolvers so both install the WHOLE folder identically. Fetches text + only; never runs anything. `version` is the folder's tree SHA, the update fingerprint.""" + prefix = (skill_dir + "/") if skill_dir else "" + files: dict[str, str] = {} + for p in members: + rel = p[len(prefix):] if prefix else p + raw = await client.get(f"{GH_RAW}/{owner}/{repo}/{branch}/{p}") + if raw.status_code == 200: + files[rel] = raw.text + if "SKILL.md" not in files: + raise ValueError("SKILL.md could not be fetched") + + meta, _ = parse_frontmatter(files["SKILL.md"]) + # Reuse the .swarm importer's content scan: flag files holding secret-shaped literals (the author's leaked key, or a sketchy skill) so the user sees it before installing from an unvetted repo. + from backend.common.secret_scan import find_secrets_in_files + secret_findings = find_secrets_in_files({rel: data.encode("utf-8", "ignore") for rel, data in files.items()}) + return { + "name": meta.get("name") or skill_id, + "description": meta.get("description", ""), + "repo_url": f"https://github.com/{owner}/{repo}/tree/{branch}/{skill_dir}".rstrip("/"), + "skill_id": skill_id, + "files": files, + "scripts": sorted(rel for rel in files if is_script_path(rel)), + "secret_findings": secret_findings, + "source": f"{owner}/{repo}", + "folder": skill_dir, + "version": version, + } + + +async def resolve_community_skill(source: str, skill_id: str) -> dict: + """Resolve a skills.sh entry (source='owner/repo', skill_id=folder name) to + its files via the GitHub trees API. Returns name/description/repo_url plus + {relpath: content} and the list of script files. Fetches text only; never + runs anything. Raises ValueError on a bad source or a missing skill, and + RegistryRateLimited when GitHub's anon API is exhausted.""" + owner, _, repo = source.partition("/") + if not owner or not repo: + raise ValueError(f"unrecognized source '{source}' (expected owner/repo)") + async with httpx.AsyncClient(timeout=30.0, headers=github_headers()) as client: + branch, tree = await fetch_repo_tree(client, owner, repo) + skill_md, members = select_skill_paths(tree, skill_id) + skill_dir = skill_md[: -len("/SKILL.md")] if "/" in skill_md else "" + version = folder_tree_sha(tree, skill_dir) + return await p_build_resolved_skill(client, owner, repo, branch, skill_dir, members, skill_id, version) + + +async def resolve_curated_skill(folder: str) -> dict: + """Resolve a curated (anthropics/skills) skill folder to ALL its files via the + GitHub trees API, so multi-file curated skills (pdf/docx/pptx scripts, etc.) + install whole instead of just their SKILL.md. The exact folder comes from our + catalog, so we match it precisely (not by basename). Same payload shape as + resolve_community_skill. Raises ValueError if the folder has no SKILL.md and + RegistryRateLimited when GitHub's anon API is exhausted.""" + owner, _, repo = REPO.partition("/") + skill_dir = folder.rstrip("/") + skill_id = skill_dir.rsplit("/", 1)[-1] + prefix = skill_dir + "/" + async with httpx.AsyncClient(timeout=30.0, headers=github_headers()) as client: + tree = curated_tree + if not tree: + # Cold cache (pre-first-refresh, or a failed/rate-limited warm-up): pay one live tree call this once. + tree = await tree_at(client, owner, repo, BRANCH) + if tree is None: + raise ValueError(f"could not read {REPO}@{BRANCH} tree") + blobs = tree_blob_paths(tree) + if (prefix + "SKILL.md") not in blobs: + raise ValueError(f"no SKILL.md at '{folder}'") + members = [p for p in blobs if p.startswith(prefix)][:MAX_SKILL_FILES] + version = folder_tree_sha(tree, skill_dir) + return await p_build_resolved_skill(client, owner, repo, BRANCH, skill_dir, members, skill_id, version) + + +def search_curated(cache: dict[str, dict], q: str, category: str, offset: int, limit: int) -> dict: + """Filter + paginate the in-memory curated catalog. Pure (cache passed in) so the + route stays a thin wrapper and this layer owns all skill-data shaping.""" + pool = list(cache.values()) + if category: + cat_lower = category.lower() + pool = [s for s in pool if s.get("category", "").lower() == cat_lower] + + query_lower = q.lower().strip() + if query_lower: + pool = [sk for sk in pool if query_lower in f"{sk['name']} {sk['description']} {sk.get('category', '')}".lower()] + + pool.sort(key=lambda s: s["name"].lower()) + total = len(pool) + page = pool[offset: offset + limit] + summary = [ + { + "name": s["name"], + "description": s["description"], + "folder": s["folder"], + "category": s.get("category", "General"), + "repositoryUrl": s.get("repositoryUrl", ""), + } + for s in page + ] + return {"skills": summary, "total": total, "offset": offset, "limit": limit} + + +async def community_search(q: str, limit: int) -> dict: + """Live-proxy a query to the skills.sh wild registry. Not cached: it's a + 600k-entry remote index, so we search it on demand rather than mirror it.""" + async with httpx.AsyncClient(timeout=15.0, headers={"User-Agent": "openswarm"}) as client: + r = await client.get(COMMUNITY_SEARCH_URL, params={"q": q or "skill"}) + r.raise_for_status() + data = r.json() + skills = [] + for s in (data.get("skills") or [])[:limit]: + src = s.get("source", "") + try: + installs = int(s.get("installs") or 0) + except (TypeError, ValueError): + installs = 0 + skills.append({ + "name": s.get("name", ""), + "description": f"{installs:,} installs", + "folder": s.get("skillId", ""), + "category": src, + "repositoryUrl": f"https://github.com/{src}" if src else "", + "source": src, + "skillId": s.get("skillId", ""), + "installs": installs, + "community": True, + }) + return {"skills": skills, "total": len(skills), "offset": 0, "limit": limit, "source": "community"} + + +async def safe_repo_tree(source: str): + """Recursive tree for a community 'owner/repo', cached briefly and best-effort + (None on rate-limit / missing repo) so an updates check never fails the whole + list because one repo is unreachable.""" + now = time.time() + hit = p_community_tree_cache.get(source) + if hit and now - hit[0] < P_COMMUNITY_TREE_TTL: + return hit[1] + owner, _, repo = source.partition("/") + tree = None + if owner and repo: + try: + async with httpx.AsyncClient(timeout=30.0, headers=github_headers()) as client: + _, tree = await fetch_repo_tree(client, owner, repo) + except Exception: + tree = None + p_community_tree_cache[source] = (now, tree) + return tree diff --git a/backend/tests/test_skill_registry_community.py b/backend/tests/test_skill_registry_community.py index 67a14c0c..118ace42 100644 --- a/backend/tests/test_skill_registry_community.py +++ b/backend/tests/test_skill_registry_community.py @@ -13,7 +13,7 @@ import os import pytest import backend.apps.skills.skills as skills_mod -from backend.apps.skill_registry.skill_registry import select_skill_paths, is_script_path +from backend.apps.skill_registry.skill_registry_github import select_skill_paths, is_script_path, tree_blob_paths def test_selects_shortest_matching_skill_md_at_any_depth(): @@ -66,7 +66,7 @@ def test_ambiguous_match_picks_deterministically(): def test_github_headers_adds_token_when_set(monkeypatch): - from backend.apps.skill_registry.skill_registry import github_headers + from backend.apps.skill_registry.skill_registry_github import github_headers monkeypatch.delenv("OPENSWARM_GITHUB_TOKEN", raising=False) monkeypatch.delenv("GITHUB_TOKEN", raising=False) assert "Authorization" not in github_headers() @@ -159,7 +159,7 @@ def test_confirm_install_writes_folder_lists_and_injects(skills_dir, monkeypatch "scripts/extract.py": "print('extract')"}, "scripts": ["scripts/extract.py"], "secret_findings": [], } - monkeypatch.setattr("backend.apps.skill_registry.skill_registry.resolve_community_skill", fake_resolve) + monkeypatch.setattr("backend.apps.skill_registry.skill_registry_sources.resolve_community_skill", fake_resolve) r = client.post("/api/skill-registry/install", json={"source": "o/r", "skill_id": "pdf-tools", "confirm": True}) assert r.status_code == 200 and r.json()["installed"] is True @@ -204,8 +204,8 @@ def test_curated_resolve_fetches_exact_folder_only(monkeypatch): skills/pdf-extra) must not leak in. Here the cache is COLD, so it pays one live tree call.""" import asyncio - import backend.apps.skill_registry.skill_registry as sr - monkeypatch.setattr(sr, "p_curated_tree", []) + import backend.apps.skill_registry.skill_registry_sources as sr + monkeypatch.setattr(sr, "curated_tree",[]) class FakeClient: async def __aenter__(self): @@ -233,8 +233,8 @@ def test_curated_resolve_uses_warm_cache_with_zero_api_calls(monkeypatch): it reads paths from the cache and pulls contents over raw only. It also records provenance (source/folder/version) for later update detection.""" import asyncio - import backend.apps.skill_registry.skill_registry as sr - monkeypatch.setattr(sr, "p_curated_tree", CURATED_TREE["tree"]) + import backend.apps.skill_registry.skill_registry_sources as sr + monkeypatch.setattr(sr, "curated_tree",CURATED_TREE["tree"]) class NoApiClient: async def __aenter__(self): @@ -261,8 +261,8 @@ def test_curated_resolve_uses_warm_cache_with_zero_api_calls(monkeypatch): def test_warm_curated_tree_populates_cache(monkeypatch): """The hourly warm-up caches the repo's full tree so later installs skip the API.""" import asyncio - import backend.apps.skill_registry.skill_registry as sr - monkeypatch.setattr(sr, "p_curated_tree", []) + import backend.apps.skill_registry.skill_registry_sources as sr + monkeypatch.setattr(sr, "curated_tree",[]) class TreeClient: async def __aenter__(self): @@ -276,8 +276,8 @@ def test_warm_curated_tree_populates_cache(monkeypatch): return FakeResp(200, payload=CURATED_TREE) monkeypatch.setattr(sr.httpx, "AsyncClient", lambda *a, **k: TreeClient()) - asyncio.run(sr.p_warm_curated_tree()) - assert "skills/pdf/SKILL.md" in sr.p_tree_blob_paths(sr.p_curated_tree) + asyncio.run(sr.warm_curated_tree()) + assert "skills/pdf/SKILL.md" in tree_blob_paths(sr.curated_tree) def test_skill_update_detection_and_apply(skills_dir, monkeypatch): @@ -288,14 +288,14 @@ def test_skill_update_detection_and_apply(skills_dir, monkeypatch): from fastapi.testclient import TestClient from backend.main import app import backend.auth as auth_mod - import backend.apps.skill_registry.skill_registry as sr + import backend.apps.skill_registry.skill_registry_sources as sr import backend.apps.skills.skills as skills_mod if not auth_mod.TOKEN: auth_mod.TOKEN = p_secrets.token_urlsafe(32) client = TestClient(app, headers={"Authorization": f"Bearer {auth_mod.TOKEN}"}) # Upstream folder SHA for skills/pdf is PDFSHA1 (from the warmed tree). - monkeypatch.setattr(sr, "p_curated_tree", CURATED_TREE["tree"]) + monkeypatch.setattr(sr, "curated_tree",CURATED_TREE["tree"]) # Installed copy recorded at a STALE version, so it should read as outdated. skills_mod.write_folder_skill("pdf", {"SKILL.md": "old"}, { "name": "PDF", "source": "anthropics/skills", "folder": "skills/pdf", "version": "OLDSHA", @@ -339,7 +339,7 @@ def test_curated_install_writes_full_folder(skills_dir, monkeypatch): "files": {"SKILL.md": "# PDF\nRun scripts/extract.py", "scripts/extract.py": "print('x')"}, "scripts": ["scripts/extract.py"], "secret_findings": [], } - monkeypatch.setattr("backend.apps.skill_registry.skill_registry.resolve_curated_skill", fake_resolve) + monkeypatch.setattr("backend.apps.skill_registry.skill_registry_sources.resolve_curated_skill", fake_resolve) r = client.post("/api/skill-registry/install-curated", json={"folder": "skills/pdf"}) assert r.status_code == 200 and r.json()["installed"] is True diff --git a/backend/tests/test_skill_registry_seed.py b/backend/tests/test_skill_registry_seed.py index 5f8dd115..b9c366e2 100644 --- a/backend/tests/test_skill_registry_seed.py +++ b/backend/tests/test_skill_registry_seed.py @@ -11,12 +11,13 @@ import json import os from backend.apps.skill_registry import skill_registry as sr +from backend.apps.skill_registry import skill_registry_cache as cache def test_bundled_snapshot_exists_and_includes_pdf(): # The onboarding step targets the "pdf" skill via /pdf/i; it must be present in the shipped snapshot or the tour times out even with a populated list. - assert os.path.exists(sr.BUNDLED_SNAPSHOT) - data = json.load(open(sr.BUNDLED_SNAPSHOT, encoding="utf-8")) + assert os.path.exists(cache.BUNDLED_SNAPSHOT) + data = json.load(open(cache.BUNDLED_SNAPSHOT, encoding="utf-8")) assert isinstance(data, dict) and len(data) >= 10 assert any("pdf" in k.lower() or "pdf" in v.get("folder", "").lower() for k, v in data.items()) @@ -25,10 +26,10 @@ def test_bundled_snapshot_exists_and_includes_pdf(): def test_seed_makes_catalog_non_empty_offline(monkeypatch, tmp_path): # Point the disk cache at an empty tmp dir so only the bundled snapshot can seed; this is the brand-new-install, no-network case. monkeypatch.setenv("OPENSWARM_SKILL_CACHE_DIR", str(tmp_path)) - seeded = sr.load_seed_cache() + seeded = cache.load_seed_cache() assert len(seeded) >= 10 - sr.CACHE = seeded + sr.p_cache = seeded res = asyncio.run(sr.registry_search(q="", limit=100, offset=0, sort="name", category="")) assert res["total"] >= 10 and len(res["skills"]) >= 10 @@ -39,6 +40,6 @@ def test_disk_cache_roundtrip_and_priority(monkeypatch, tmp_path): sentinel = {"only-skill": {"name": "only-skill", "description": "", "content": "", "folder": "skills/only-skill", "category": "Test", "repositoryUrl": ""}} - sr.save_disk_cache(sentinel) - assert os.path.exists(sr.disk_cache_path()) - assert sr.load_seed_cache() == sentinel + cache.save_disk_cache(sentinel) + assert os.path.exists(cache.disk_cache_path()) + assert cache.load_seed_cache() == sentinel