diff --git a/backend/apps/skill_registry/skill_registry.py b/backend/apps/skill_registry/skill_registry.py index 306880e4..05717d9c 100644 --- a/backend/apps/skill_registry/skill_registry.py +++ b/backend/apps/skill_registry/skill_registry.py @@ -1,5 +1,7 @@ import asyncio +import json import logging +import os import re import time from contextlib import asynccontextmanager @@ -17,12 +19,62 @@ RAW_BASE = f"https://raw.githubusercontent.com/{REPO}/{BRANCH}" MANIFEST_URL = f"{RAW_BASE}/.claude-plugin/marketplace.json" REFRESH_INTERVAL_S = 3600 CONCURRENT_FETCHES = 15 +# Retry the startup fetch on this short backoff (capped) until the FIRST success, +# instead of waiting a full REFRESH_INTERVAL_S after a cold/slow/failed fetch. +# That 1h gap was the "skills empty until reboot" bug on cold Windows networks. +_RETRY_BACKOFF_START_S = 2 +_RETRY_BACKOFF_MAX_S = 60 + +# Catalog ships in the repo so a brand-new install shows skills with zero network +# (build snapshot), and every successful live fetch is persisted to the user's +# cache so subsequent launches are instant + offline-safe. The live fetch always +# overwrites both once it lands, so neither can go stale at runtime. +_BUNDLED_SNAPSHOT = os.path.join(os.path.dirname(__file__), "skills_snapshot.json") _cache: dict[str, dict] = {} _cache_updated_at: float = 0 _refresh_task: Optional[asyncio.Task] = None +def _disk_cache_path() -> str: + base = os.environ.get("OPENSWARM_SKILL_CACHE_DIR") or os.path.expanduser( + "~/.openswarm/cache" + ) + return os.path.join(base, "skill_registry.json") + + +def _load_seed_cache() -> dict[str, dict]: + """Return a non-empty catalog from the on-disk last-good cache, falling back + to the bundled snapshot, so the registry is never empty on a cold/offline + start. Returns {} only if neither source is present/valid.""" + for path in (_disk_cache_path(), _BUNDLED_SNAPSHOT): + try: + with open(path, encoding="utf-8") as f: + data = json.load(f) + if isinstance(data, dict) and data: + logger.info(f"Skill registry: seeded {len(data)} skills from {os.path.basename(path)}") + return data + except (OSError, ValueError): + continue + return {} + + +def _save_disk_cache(skills: dict[str, dict]) -> None: + """Persist the last good live fetch so the next launch is instant. Atomic + replace so a crash mid-write can't leave a truncated cache.""" + if not skills: + return + path = _disk_cache_path() + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + tmp = f"{path}.tmp" + with open(tmp, "w", encoding="utf-8") as f: + json.dump(skills, f) + os.replace(tmp, path) + except OSError: + logger.debug("Skill registry: could not persist disk cache", exc_info=True) + + def _parse_frontmatter(raw: str) -> tuple[dict, str]: """Split YAML frontmatter from markdown body.""" if not raw.startswith("---"): @@ -114,18 +166,37 @@ async def _fetch_all_skills() -> dict[str, dict]: async def _refresh_loop(): global _cache, _cache_updated_at + backoff = _RETRY_BACKOFF_START_S while True: + ok = False try: - _cache = await _fetch_all_skills() - _cache_updated_at = time.time() + fetched = await _fetch_all_skills() + if fetched: + _cache = fetched + _cache_updated_at = time.time() + _save_disk_cache(_cache) + ok = True except Exception as e: logger.exception(f"Skill registry refresh error: {e}") - await asyncio.sleep(REFRESH_INTERVAL_S) + if ok: + # Settle to the slow hourly refresh once we have a good catalog. + backoff = _RETRY_BACKOFF_START_S + await asyncio.sleep(REFRESH_INTERVAL_S) + else: + # Cold/slow/failed fetch: retry soon (capped) until the first success + # so a transient network hiccup doesn't leave the catalog empty for + # an hour. The seeded snapshot keeps it non-empty meanwhile. + await asyncio.sleep(backoff) + backoff = min(backoff * 2, _RETRY_BACKOFF_MAX_S) @asynccontextmanager async def skill_registry_lifespan(): - global _refresh_task + global _refresh_task, _cache + # Seed instantly from disk/bundled snapshot so the very first request never + # sees an empty catalog (the live fetch below overwrites it when it lands). + if not _cache: + _cache = _load_seed_cache() _refresh_task = asyncio.create_task(_refresh_loop()) yield if _refresh_task: diff --git a/backend/apps/skill_registry/skills_snapshot.json b/backend/apps/skill_registry/skills_snapshot.json new file mode 100644 index 00000000..36cfd9c4 --- /dev/null +++ b/backend/apps/skill_registry/skills_snapshot.json @@ -0,0 +1,138 @@ +{ + "algorithmic-art": { + "category": "Example Skills", + "content": "", + "description": "Creating algorithmic art using p5.js with seeded randomness and interactive parameter exploration. Use this when users request creating art using code, generative art, algorithmic art, flow fields, or particle systems. Create original algorithmic art rather than copying existing artists' work to avoid copyright violations.", + "folder": "skills/algorithmic-art", + "name": "algorithmic-art", + "repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/algorithmic-art" + }, + "brand-guidelines": { + "category": "Example Skills", + "content": "", + "description": "Applies Anthropic's official brand colors and typography to any sort of artifact that may benefit from having Anthropic's look-and-feel. Use it when brand colors or style guidelines, visual formatting, or company design standards apply.", + "folder": "skills/brand-guidelines", + "name": "brand-guidelines", + "repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/brand-guidelines" + }, + "canvas-design": { + "category": "Example Skills", + "content": "", + "description": "Create beautiful visual art in .png and .pdf documents using design philosophy. You should use this skill when the user asks to create a poster, piece of art, design, or other static piece. Create original visual designs, never copying existing artists' work to avoid copyright violations.", + "folder": "skills/canvas-design", + "name": "canvas-design", + "repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/canvas-design" + }, + "claude-api": { + "category": "Claude Api", + "content": "", + "description": "|-", + "folder": "skills/claude-api", + "name": "claude-api", + "repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/claude-api" + }, + "doc-coauthoring": { + "category": "Example Skills", + "content": "", + "description": "Guide users through a structured workflow for co-authoring documentation. Use when user wants to write documentation, proposals, technical specs, decision docs, or similar structured content. This workflow helps users efficiently transfer context, refine content through iteration, and verify the doc works for readers. Trigger when user mentions writing docs, creating proposals, drafting specs, or similar documentation tasks.", + "folder": "skills/doc-coauthoring", + "name": "doc-coauthoring", + "repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/doc-coauthoring" + }, + "docx": { + "category": "Document Skills", + "content": "", + "description": "Use this skill whenever the user wants to create, read, edit, or manipulate Word documents (.docx files). Triggers include: any mention of 'Word doc', 'word document', '.docx', or requests to produce professional documents with formatting like tables of contents, headings, page numbers, or letterheads. Also use when extracting or reorganizing content from .docx files, inserting or replacing images in documents, performing find-and-replace in Word files, working with tracked changes or comments, or converting content into a polished Word document. If the user asks for a 'report', 'memo', 'letter', 'template', or similar deliverable as a Word or .docx file, use this skill. Do NOT use for PDFs, spreadsheets, Google Docs, or general coding tasks unrelated to document generation.", + "folder": "skills/docx", + "name": "docx", + "repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/docx" + }, + "frontend-design": { + "category": "Example Skills", + "content": "", + "description": "Guidance for distinctive, intentional visual design when building new UI or reshaping an existing one. Helps with aesthetic direction, typography, and making choices that don't read as templated defaults.", + "folder": "skills/frontend-design", + "name": "frontend-design", + "repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/frontend-design" + }, + "internal-comms": { + "category": "Example Skills", + "content": "", + "description": "A set of resources to help me write all kinds of internal communications, using the formats that my company likes to use. Claude should use this skill whenever asked to write some sort of internal communications (status reports, leadership updates, 3P updates, company newsletters, FAQs, incident reports, project updates, etc.).", + "folder": "skills/internal-comms", + "name": "internal-comms", + "repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/internal-comms" + }, + "mcp-builder": { + "category": "Example Skills", + "content": "", + "description": "Guide for creating high-quality MCP (Model Context Protocol) servers that enable LLMs to interact with external services through well-designed tools. Use when building MCP servers to integrate external APIs or services, whether in Python (FastMCP) or Node/TypeScript (MCP SDK).", + "folder": "skills/mcp-builder", + "name": "mcp-builder", + "repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/mcp-builder" + }, + "pdf": { + "category": "Document Skills", + "content": "", + "description": "Use this skill whenever the user wants to do anything with PDF files. This includes reading or extracting text/tables from PDFs, combining or merging multiple PDFs into one, splitting PDFs apart, rotating pages, adding watermarks, creating new PDFs, filling PDF forms, encrypting/decrypting PDFs, extracting images, and OCR on scanned PDFs to make them searchable. If the user mentions a .pdf file or asks to produce one, use this skill.", + "folder": "skills/pdf", + "name": "pdf", + "repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/pdf" + }, + "pptx": { + "category": "Document Skills", + "content": "", + "description": "Use this skill any time a .pptx file is involved in any way \u2014 as input, output, or both. This includes: creating slide decks, pitch decks, or presentations; reading, parsing, or extracting text from any .pptx file (even if the extracted content will be used elsewhere, like in an email or summary); editing, modifying, or updating existing presentations; combining or splitting slide files; working with templates, layouts, speaker notes, or comments. Trigger whenever the user mentions \\\"deck,\\\" \\\"slides,\\\" \\\"presentation,\\\" or references a .pptx filename, regardless of what they plan to do with the content afterward. If a .pptx file needs to be opened, created, or touched, use this skill.", + "folder": "skills/pptx", + "name": "pptx", + "repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/pptx" + }, + "skill-creator": { + "category": "Example Skills", + "content": "", + "description": "Create new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy.", + "folder": "skills/skill-creator", + "name": "skill-creator", + "repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/skill-creator" + }, + "slack-gif-creator": { + "category": "Example Skills", + "content": "", + "description": "Knowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like \"make me a GIF of X doing Y for Slack.", + "folder": "skills/slack-gif-creator", + "name": "slack-gif-creator", + "repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/slack-gif-creator" + }, + "theme-factory": { + "category": "Example Skills", + "content": "", + "description": "Toolkit for styling artifacts with a theme. These artifacts can be slides, docs, reportings, HTML landing pages, etc. There are 10 pre-set themes with colors/fonts that you can apply to any artifact that has been creating, or can generate a new theme on-the-fly.", + "folder": "skills/theme-factory", + "name": "theme-factory", + "repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/theme-factory" + }, + "web-artifacts-builder": { + "category": "Example Skills", + "content": "", + "description": "Suite of tools for creating elaborate, multi-component claude.ai HTML artifacts using modern frontend web technologies (React, Tailwind CSS, shadcn/ui). Use for complex artifacts requiring state management, routing, or shadcn/ui components - not for simple single-file HTML/JSX artifacts.", + "folder": "skills/web-artifacts-builder", + "name": "web-artifacts-builder", + "repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/web-artifacts-builder" + }, + "webapp-testing": { + "category": "Example Skills", + "content": "", + "description": "Toolkit for interacting with and testing local web applications using Playwright. Supports verifying frontend functionality, debugging UI behavior, capturing browser screenshots, and viewing browser logs.", + "folder": "skills/webapp-testing", + "name": "webapp-testing", + "repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/webapp-testing" + }, + "xlsx": { + "category": "Document Skills", + "content": "", + "description": "Use this skill any time a spreadsheet file is the primary input or output. This means any task where the user wants to: open, read, edit, or fix an existing .xlsx, .xlsm, .csv, or .tsv file (e.g., adding columns, computing formulas, formatting, charting, cleaning messy data); create a new spreadsheet from scratch or from other data sources; or convert between tabular file formats. Trigger especially when the user references a spreadsheet file by name or path \u2014 even casually (like \\\"the xlsx in my downloads\\\") \u2014 and wants something done to it or produced from it. Also trigger for cleaning or restructuring messy tabular data files (malformed rows, misplaced headers, junk data) into proper spreadsheets. The deliverable must be a spreadsheet file. Do NOT trigger when the primary deliverable is a Word document, HTML report, standalone Python script, database pipeline, or Google Sheets API integration, even if tabular data is involved.", + "folder": "skills/xlsx", + "name": "xlsx", + "repositoryUrl": "https://github.com/anthropics/skills/tree/main/skills/xlsx" + } +} \ No newline at end of file diff --git a/backend/tests/test_skill_registry_seed.py b/backend/tests/test_skill_registry_seed.py new file mode 100644 index 00000000..ed94d11a --- /dev/null +++ b/backend/tests/test_skill_registry_seed.py @@ -0,0 +1,46 @@ +"""Regression tests for the skill-registry never-empty seed (winv2 Bug #1). + +The bug: the catalog was fetched from GitHub once at startup then only hourly, +so a cold/slow/failed network left it empty for the whole session, breaking the +Skills page and the onboarding "Install a skill" step (waitForSelector +"skill-item-pdf" timing out). Fix: seed from a bundled snapshot + on-disk +last-good cache so the catalog is never empty, even fully offline. +""" +import asyncio +import json +import os + +from backend.apps.skill_registry import skill_registry as sr + + +def test_bundled_snapshot_exists_and_includes_pdf(): + # The onboarding step targets the "pdf" skill via /pdf/i; it must be present + # in the shipped snapshot or the tour times out even with a populated list. + assert os.path.exists(sr._BUNDLED_SNAPSHOT) + data = json.load(open(sr._BUNDLED_SNAPSHOT, encoding="utf-8")) + assert isinstance(data, dict) and len(data) >= 10 + assert any("pdf" in k.lower() or "pdf" in v.get("folder", "").lower() + for k, v in data.items()) + + +def test_seed_makes_catalog_non_empty_offline(monkeypatch, tmp_path): + # Point the disk cache at an empty tmp dir so only the bundled snapshot can + # seed; this is the brand-new-install, no-network case. + monkeypatch.setenv("OPENSWARM_SKILL_CACHE_DIR", str(tmp_path)) + seeded = sr._load_seed_cache() + assert len(seeded) >= 10 + + sr._cache = seeded + res = asyncio.run(sr.registry_search(q="", limit=100, offset=0, sort="name", category="")) + assert res["total"] >= 10 and len(res["skills"]) >= 10 + + +def test_disk_cache_roundtrip_and_priority(monkeypatch, tmp_path): + # A saved last-good fetch must win over the bundled snapshot on next boot. + monkeypatch.setenv("OPENSWARM_SKILL_CACHE_DIR", str(tmp_path)) + sentinel = {"only-skill": {"name": "only-skill", "description": "", "content": "", + "folder": "skills/only-skill", "category": "Test", + "repositoryUrl": ""}} + sr._save_disk_cache(sentinel) + assert os.path.exists(sr._disk_cache_path()) + assert sr._load_seed_cache() == sentinel