From f012faae94bab658bd0f83de938fc53c6b2ffabe Mon Sep 17 00:00:00 2001 From: haikdc Date: Sun, 5 Apr 2026 18:36:20 -0700 Subject: [PATCH] [Haik]: ckpt on cleanup, made a SkillStore singelton to handle db logic, also abstracted helper functions but now likely gonna make this a singelton too --- backend/apps/skills/SkillStore.py | 119 +++++++++ backend/apps/skills/parse_frontmatter.py | 19 ++ .../fetch_all_registry_skills.py | 47 ++++ .../utils/fetch_one_skill.py | 39 +++ .../utils/fetch_skill_paths.py | 21 ++ .../registry_refresh_loop.py | 35 +++ backend/apps/skills/skills.py | 231 +++--------------- 7 files changed, 308 insertions(+), 203 deletions(-) create mode 100644 backend/apps/skills/SkillStore.py create mode 100644 backend/apps/skills/parse_frontmatter.py create mode 100644 backend/apps/skills/registry_refresh_loop/fetch_all_registry_skills/fetch_all_registry_skills.py create mode 100644 backend/apps/skills/registry_refresh_loop/fetch_all_registry_skills/utils/fetch_one_skill.py create mode 100644 backend/apps/skills/registry_refresh_loop/fetch_all_registry_skills/utils/fetch_skill_paths.py create mode 100644 backend/apps/skills/registry_refresh_loop/registry_refresh_loop.py diff --git a/backend/apps/skills/SkillStore.py b/backend/apps/skills/SkillStore.py new file mode 100644 index 00000000..85460cc1 --- /dev/null +++ b/backend/apps/skills/SkillStore.py @@ -0,0 +1,119 @@ +import json +from backend.apps.skills.Skill import Skill +from typeguard import typechecked +from pydantic import BaseModel +import os +from typing import Dict, Optional + + +class SkillStore(BaseModel): + """File-backed persistence layer for local skills.""" + skills_dir: str + index_path: str + + @typechecked + def __init__(self, skills_dir: str, index_filename: str = ".skills_index.json"): + os.makedirs(skills_dir, exist_ok=True) + super().__init__( + skills_dir=skills_dir, + index_path=os.path.join(skills_dir, index_filename), + ) + + @typechecked + def p_load_index(self) -> Dict[str, dict]: + if os.path.exists(self.index_path): + with open(self.index_path) as f: + return json.load(f) + return {} + + @typechecked + def p_save_index(self, index: Dict[str, dict]) -> None: + with open(self.index_path, "w") as f: + json.dump(index, f, indent=2) + + @staticmethod + @typechecked + def slug(name: str) -> str: + return name.lower().replace(" ", "-") + + @typechecked + def p_skill_path(self, skill_id: str) -> str: + return os.path.join(self.skills_dir, f"{skill_id}.md") + + @typechecked + def list_all(self) -> list[Skill]: + index = self.p_load_index() + result: list[Skill] = [] + if not os.path.exists(self.skills_dir): + return result + for fname in os.listdir(self.skills_dir): + if not fname.endswith(".md"): + continue + fpath = os.path.join(self.skills_dir, fname) + with open(fpath) as f: + content = f.read() + skill_id = fname.removesuffix(".md") + meta = index.get(skill_id, {}) + result.append(Skill( + id=skill_id, + name=meta.get("name", skill_id.replace("-", " ").replace("_", " ").title()), + description=meta.get("description", ""), + content=content, + file_path=fpath, + command=meta.get("command", skill_id), + )) + return result + + @typechecked + def get(self, skill_id: str) -> Optional[Skill]: + for s in self.list_all(): + if s.id == skill_id: + return s + return None + + @typechecked + def create(self, name: str, description: str, content: str, command: str = "") -> Skill: + slug = self.slug(name) + fpath = self.p_skill_path(slug) + with open(fpath, "w") as f: + f.write(content) + index = self.p_load_index() + index[slug] = {"name": name, "description": description, "command": command or slug} + self.p_save_index(index) + return Skill(id=slug, name=name, description=description, + content=content, file_path=fpath, command=command or slug) + + @typechecked + def update(self, skill_id: str, *, name: Optional[str] = None, + description: Optional[str] = None, content: Optional[str] = None, + command: Optional[str] = None) -> Skill: + fpath = self.p_skill_path(skill_id) + if not os.path.exists(fpath): + raise FileNotFoundError(skill_id) + if content is not None: + with open(fpath, "w") as f: + f.write(content) + index = self.p_load_index() + meta = index.get(skill_id, {}) + if name is not None: + meta["name"] = name + if description is not None: + meta["description"] = description + if command is not None: + meta["command"] = command + index[skill_id] = meta + self.p_save_index(index) + with open(fpath) as f: + content = f.read() + return Skill(id=skill_id, name=meta.get("name", skill_id), + description=meta.get("description", ""), + content=content, file_path=fpath, command=meta.get("command", skill_id)) + + @typechecked + def delete(self, skill_id: str) -> None: + fpath = self.p_skill_path(skill_id) + if os.path.exists(fpath): + os.remove(fpath) + index = self.p_load_index() + index.pop(skill_id, None) + self.p_save_index(index) \ No newline at end of file diff --git a/backend/apps/skills/parse_frontmatter.py b/backend/apps/skills/parse_frontmatter.py new file mode 100644 index 00000000..d5f6a8ee --- /dev/null +++ b/backend/apps/skills/parse_frontmatter.py @@ -0,0 +1,19 @@ +from typeguard import typechecked +from typing import Tuple +import re + +@typechecked +def parse_frontmatter(raw: str) -> Tuple[dict, str]: + if not raw.startswith("---"): + return {}, raw + end = raw.find("---", 3) + if end == -1: + return {}, raw + fm_block = raw[3:end].strip() + body = raw[end + 3:].strip() + meta: dict = {} + for line in fm_block.splitlines(): + m = re.match(r"^(\w[\w_-]*)\s*:\s*(.+)$", line) + if m: + meta[m.group(1).strip()] = m.group(2).strip().strip('"').strip("'") + return meta, body \ No newline at end of file diff --git a/backend/apps/skills/registry_refresh_loop/fetch_all_registry_skills/fetch_all_registry_skills.py b/backend/apps/skills/registry_refresh_loop/fetch_all_registry_skills/fetch_all_registry_skills.py new file mode 100644 index 00000000..9ba9de14 --- /dev/null +++ b/backend/apps/skills/registry_refresh_loop/fetch_all_registry_skills/fetch_all_registry_skills.py @@ -0,0 +1,47 @@ + + + + +import asyncio +import httpx +from typeguard import typechecked +from backend.apps.skills.registry_refresh_loop.fetch_all_registry_skills.utils.fetch_skill_paths import fetch_skill_paths +from backend.apps.skills.registry_refresh_loop.fetch_all_registry_skills.utils.fetch_one_skill import fetch_one_skill + + +@typechecked +async def fetch_all_registry_skills( + num_concurrent_fetches: int, + manifest_url: str, + raw_base: str, + repo: str, + branch: str, +) -> dict[str, dict]: + result: dict[str, dict] = {} + async with httpx.AsyncClient(timeout=30.0) as client: + try: + paths = await fetch_skill_paths( + client=client, + manifest_url=manifest_url + ) + except Exception as e: + print(f"[fetch_all_registry_skills] Skill registry manifest fetch failed: {e}") + return result + print(f"[fetch_all_registry_skills] Skill registry: found {len(paths)} skills in manifest, fetching...") + sem = asyncio.Semaphore(num_concurrent_fetches) + records = await asyncio.gather( + *[fetch_one_skill( + client=client, + sem=sem, + folder=folder, + plugin_name=plugin, + raw_base=raw_base, + repo=repo, + branch=branch + ) for folder, plugin in paths] + ) + for rec in records: + if rec: + result[rec["name"]] = rec + print(f"[fetch_all_registry_skills] Skill registry cache refreshed: {len(result)} skills") + return result diff --git a/backend/apps/skills/registry_refresh_loop/fetch_all_registry_skills/utils/fetch_one_skill.py b/backend/apps/skills/registry_refresh_loop/fetch_all_registry_skills/utils/fetch_one_skill.py new file mode 100644 index 00000000..8aaba244 --- /dev/null +++ b/backend/apps/skills/registry_refresh_loop/fetch_all_registry_skills/utils/fetch_one_skill.py @@ -0,0 +1,39 @@ +import asyncio +from typing import Optional +import httpx +from backend.apps.skills.parse_frontmatter import parse_frontmatter +from typeguard import typechecked + +@typechecked +async def fetch_one_skill( + client: httpx.AsyncClient, + sem: asyncio.Semaphore, + folder: str, + plugin_name: str, + raw_base: str, + repo: str, + branch: str, +) -> Optional[dict]: + async with sem: + try: + resp = await client.get(f"{raw_base}/{folder}/SKILL.md") + if resp.status_code != 200: + return None + raw = resp.text + except Exception as exc: + print(f"[fetch_one_skill] Failed to fetch {folder}/SKILL.md: {exc}") + return None + + meta, body = parse_frontmatter(raw) + name = meta.get("name", "") + if not name: + name = folder.rsplit("/", 1)[-1].replace("-", " ").replace("_", " ").title() + + return { + "name": name, + "description": meta.get("description", ""), + "content": body, + "folder": folder, + "category": plugin_name.replace("-", " ").replace("_", " ").title(), + "repositoryUrl": f"https://github.com/{repo}/tree/{branch}/{folder}", + } \ No newline at end of file diff --git a/backend/apps/skills/registry_refresh_loop/fetch_all_registry_skills/utils/fetch_skill_paths.py b/backend/apps/skills/registry_refresh_loop/fetch_all_registry_skills/utils/fetch_skill_paths.py new file mode 100644 index 00000000..4720c93c --- /dev/null +++ b/backend/apps/skills/registry_refresh_loop/fetch_all_registry_skills/utils/fetch_skill_paths.py @@ -0,0 +1,21 @@ + +import httpx +from typeguard import typechecked + +@typechecked +async def fetch_skill_paths( + client: httpx.AsyncClient, + manifest_url: str + ) -> list[tuple[str, str]]: + + """Fetch marketplace.json and return (folder, plugin_name) pairs.""" + resp = await client.get(manifest_url) + resp.raise_for_status() + manifest = resp.json() + paths: list[tuple[str, str]] = [] + for plugin in manifest.get("plugins", []): + plugin_name = plugin.get("name", "") + for skill_ref in plugin.get("skills", []): + paths.append((skill_ref.lstrip("./"), plugin_name)) + + return paths diff --git a/backend/apps/skills/registry_refresh_loop/registry_refresh_loop.py b/backend/apps/skills/registry_refresh_loop/registry_refresh_loop.py new file mode 100644 index 00000000..83eb24f1 --- /dev/null +++ b/backend/apps/skills/registry_refresh_loop/registry_refresh_loop.py @@ -0,0 +1,35 @@ + + + +import asyncio +from datetime import time + +from typeguard import typechecked +from backend.apps.skills.registry_refresh_loop.fetch_all_registry_skills.fetch_all_registry_skills import fetch_all_registry_skills + +@typechecked +async def registry_refresh_loop( + # Args needed for the loop + refresh_interval_s: int, + registry_cache: dict[str, dict], + registry_updated_at: float, + # Args needed for fetch_all_registry_skills + num_concurrent_fetches: int, + manifest_url: str, + raw_base: str, + repo: str, + branch: str, +) -> None: + while True: + try: + registry_cache = await fetch_all_registry_skills( + num_concurrent_fetches=num_concurrent_fetches, + manifest_url=manifest_url, + raw_base=raw_base, + repo=repo, + branch=branch, + ) + registry_updated_at = time.time() + except Exception as e: + print(f"[registry_refresh_loop] Skill registry refresh error: {e}") + await asyncio.sleep(refresh_interval_s) \ No newline at end of file diff --git a/backend/apps/skills/skills.py b/backend/apps/skills/skills.py index f7a3a8d3..f7ea0461 100644 --- a/backend/apps/skills/skills.py +++ b/backend/apps/skills/skills.py @@ -4,29 +4,29 @@ import asyncio import json import logging import os -import re -import time from contextlib import asynccontextmanager from typing import Any, Optional -import httpx from fastapi import HTTPException, Query from pydantic import BaseModel from backend.config.Apps import SubApp from backend.config.paths import DB_ROOT -from backend.apps.skills.Skill import Skill +from backend.apps.skills.SkillStore import SkillStore +from backend.apps.skills.parse_frontmatter import parse_frontmatter +from backend.apps.skills.registry_refresh_loop.registry_refresh_loop import registry_refresh_loop logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- -# Paths +# Paths & singleton store # --------------------------------------------------------------------------- SKILLS_DIR = os.path.expanduser("~/.claude/skills") -INDEX_PATH = os.path.join(SKILLS_DIR, ".skills_index.json") SKILLS_WORKSPACE_DIR = os.path.join(DB_ROOT, "skills_workspace") +SKILL_STORE = SkillStore(skills_dir=SKILLS_DIR) + # --------------------------------------------------------------------------- # Registry constants # --------------------------------------------------------------------------- @@ -50,9 +50,15 @@ _refresh_task: Optional[asyncio.Task] = None @asynccontextmanager async def skills_lifespan(): global _refresh_task - os.makedirs(SKILLS_DIR, exist_ok=True) os.makedirs(SKILLS_WORKSPACE_DIR, exist_ok=True) - _refresh_task = asyncio.create_task(_registry_refresh_loop()) + _refresh_task = asyncio.create_task(registry_refresh_loop( + refresh_interval_s=_REFRESH_INTERVAL_S, + registry_cache=_registry_cache, + registry_updated_at=_registry_updated_at, + num_concurrent_fetches=_CONCURRENT_FETCHES, + manifest_url=_MANIFEST_URL, + raw_base=_RAW_BASE, + )) yield if _refresh_task: _refresh_task.cancel() @@ -65,69 +71,6 @@ async def skills_lifespan(): skills = SubApp("skills", skills_lifespan) -# =========================================================================== -# Local skill helpers -# =========================================================================== - - -def _load_index() -> dict[str, dict]: - if os.path.exists(INDEX_PATH): - with open(INDEX_PATH) as f: - return json.load(f) - return {} - - -def _save_index(index: dict[str, dict]) -> None: - with open(INDEX_PATH, "w") as f: - json.dump(index, f, indent=2) - - -def _slug(name: str) -> str: - return name.lower().replace(" ", "-") - - -def _sync_skills() -> list[Skill]: - """Scan ~/.claude/skills/ for .md files and reconcile with the sidecar index.""" - index = _load_index() - result: list[Skill] = [] - if not os.path.exists(SKILLS_DIR): - return result - for fname in os.listdir(SKILLS_DIR): - if not fname.endswith(".md"): - continue - fpath = os.path.join(SKILLS_DIR, fname) - with open(fpath) as f: - content = f.read() - skill_id = fname.removesuffix(".md") - meta = index.get(skill_id, {}) - result.append(Skill( - id=skill_id, - name=meta.get("name", skill_id.replace("-", " ").replace("_", " ").title()), - description=meta.get("description", ""), - content=content, - file_path=fpath, - command=meta.get("command", skill_id), - )) - return result - - -def _parse_frontmatter(raw: str) -> tuple[dict, str]: - """Split YAML frontmatter from markdown body.""" - if not raw.startswith("---"): - return {}, raw - end = raw.find("---", 3) - if end == -1: - return {}, raw - fm_block = raw[3:end].strip() - body = raw[end + 3:].strip() - meta: dict = {} - for line in fm_block.splitlines(): - m = re.match(r"^(\w[\w_-]*)\s*:\s*(.+)$", line) - if m: - meta[m.group(1).strip()] = m.group(2).strip().strip('"').strip("'") - return meta, body - - # =========================================================================== # Local skill routes # =========================================================================== @@ -135,7 +78,7 @@ def _parse_frontmatter(raw: str) -> tuple[dict, str]: @skills.router.get("/list") async def list_skills(): - return {"skills": [s.model_dump() for s in _sync_skills()]} + return {"skills": [s.model_dump() for s in SKILL_STORE.list_all()]} @skills.router.get("/workspace/{workspace_id}") @@ -159,7 +102,7 @@ async def read_skill_workspace(workspace_id: str): except json.JSONDecodeError: pass - frontmatter, _ = _parse_frontmatter(skill_content) if skill_content else ({}, "") + frontmatter, _ = parse_frontmatter(skill_content) if skill_content else ({}, "") return {"skill_content": skill_content, "meta": meta, "frontmatter": frontmatter} @@ -185,10 +128,10 @@ async def seed_skill_workspace(body: _WorkspaceSeedBody): @skills.router.get("/detail/{skill_id}") async def get_skill(skill_id: str): - for s in _sync_skills(): - if s.id == skill_id: - return s.model_dump() - raise HTTPException(status_code=404, detail="Skill not found") + s = SKILL_STORE.get(skill_id) + if not s: + raise HTTPException(status_code=404, detail="Skill not found") + return s.model_dump() class _SkillCreateBody(BaseModel): @@ -200,19 +143,7 @@ class _SkillCreateBody(BaseModel): @skills.router.post("/create") async def create_skill(body: _SkillCreateBody): - slug = _slug(body.name) - fpath = os.path.join(SKILLS_DIR, f"{slug}.md") - with open(fpath, "w") as f: - f.write(body.content) - - index = _load_index() - index[slug] = {"name": body.name, "description": body.description, "command": body.command or slug} - _save_index(index) - - skill = Skill( - id=slug, name=body.name, description=body.description, - content=body.content, file_path=fpath, command=body.command or slug, - ) + skill = SKILL_STORE.create(body.name, body.description, body.content, body.command) return {"ok": True, "skill": skill.model_dump()} @@ -225,127 +156,21 @@ class _SkillUpdateBody(BaseModel): @skills.router.put("/{skill_id}") async def update_skill(skill_id: str, body: _SkillUpdateBody): - fpath = os.path.join(SKILLS_DIR, f"{skill_id}.md") - if not os.path.exists(fpath): + try: + skill = SKILL_STORE.update( + skill_id, name=body.name, description=body.description, + content=body.content, command=body.command, + ) + except FileNotFoundError: raise HTTPException(status_code=404, detail="Skill not found") - - if body.content is not None: - with open(fpath, "w") as f: - f.write(body.content) - - index = _load_index() - meta = index.get(skill_id, {}) - if body.name is not None: - meta["name"] = body.name - if body.description is not None: - meta["description"] = body.description - if body.command is not None: - meta["command"] = body.command - index[skill_id] = meta - _save_index(index) - - with open(fpath) as f: - content = f.read() - - skill = Skill( - id=skill_id, name=meta.get("name", skill_id), - description=meta.get("description", ""), - content=content, file_path=fpath, command=meta.get("command", skill_id), - ) return {"ok": True, "skill": skill.model_dump()} @skills.router.delete("/{skill_id}") async def delete_skill(skill_id: str): - fpath = os.path.join(SKILLS_DIR, f"{skill_id}.md") - if os.path.exists(fpath): - os.remove(fpath) - index = _load_index() - index.pop(skill_id, None) - _save_index(index) + SKILL_STORE.delete(skill_id) return {"ok": True} - -# =========================================================================== -# Registry helpers -# =========================================================================== - - -async def _fetch_skill_paths(client: httpx.AsyncClient) -> list[tuple[str, str]]: - """Fetch marketplace.json and return (folder, plugin_name) pairs.""" - resp = await client.get(_MANIFEST_URL) - resp.raise_for_status() - manifest = resp.json() - paths: list[tuple[str, str]] = [] - for plugin in manifest.get("plugins", []): - plugin_name = plugin.get("name", "") - for skill_ref in plugin.get("skills", []): - paths.append((skill_ref.lstrip("./"), plugin_name)) - return paths - - -async def _fetch_one_skill( - client: httpx.AsyncClient, - sem: asyncio.Semaphore, - folder: str, - plugin_name: str, -) -> Optional[dict]: - async with sem: - try: - resp = await client.get(f"{_RAW_BASE}/{folder}/SKILL.md") - if resp.status_code != 200: - return None - raw = resp.text - except Exception as exc: - logger.debug("Failed to fetch %s/SKILL.md: %s", folder, exc) - return None - - meta, body = _parse_frontmatter(raw) - name = meta.get("name", "") - if not name: - name = folder.rsplit("/", 1)[-1].replace("-", " ").replace("_", " ").title() - - return { - "name": name, - "description": meta.get("description", ""), - "content": body, - "folder": folder, - "category": plugin_name.replace("-", " ").replace("_", " ").title(), - "repositoryUrl": f"https://github.com/{_REPO}/tree/{_BRANCH}/{folder}", - } - - -async def _fetch_all_registry_skills() -> dict[str, dict]: - result: dict[str, dict] = {} - async with httpx.AsyncClient(timeout=30.0) as client: - try: - paths = await _fetch_skill_paths(client) - except Exception as e: - logger.warning("Skill registry manifest fetch failed: %s", e) - return result - logger.info("Skill registry: found %d skills in manifest, fetching...", len(paths)) - sem = asyncio.Semaphore(_CONCURRENT_FETCHES) - records = await asyncio.gather( - *[_fetch_one_skill(client, sem, folder, plugin) for folder, plugin in paths] - ) - for rec in records: - if rec: - result[rec["name"]] = rec - logger.info("Skill registry cache refreshed: %d skills", len(result)) - return result - - -async def _registry_refresh_loop() -> None: - global _registry_cache, _registry_updated_at - while True: - try: - _registry_cache = await _fetch_all_registry_skills() - _registry_updated_at = time.time() - except Exception as e: - logger.exception("Skill registry refresh error: %s", e) - await asyncio.sleep(_REFRESH_INTERVAL_S) - - # =========================================================================== # Registry routes # ===========================================================================