[Haik]: ckpt on cleanup, made a SkillStore singelton to handle db logic, also abstracted helper functions but now likely gonna make this a singelton too

backend/apps/skills/SkillStore.py

@@ -0,0 +1,119 @@
+import json
+from backend.apps.skills.Skill import Skill
+from typeguard import typechecked
+from pydantic import BaseModel
+import os
+from typing import Dict, Optional
+
+
+class SkillStore(BaseModel):
+    """File-backed persistence layer for local skills."""
+    skills_dir: str
+    index_path: str
+
+    @typechecked
+    def __init__(self, skills_dir: str, index_filename: str = ".skills_index.json"):
+        os.makedirs(skills_dir, exist_ok=True)
+        super().__init__(
+            skills_dir=skills_dir,
+            index_path=os.path.join(skills_dir, index_filename),
+        )
+
+    @typechecked
+    def p_load_index(self) -> Dict[str, dict]:
+        if os.path.exists(self.index_path):
+            with open(self.index_path) as f:
+                return json.load(f)
+        return {}
+
+    @typechecked
+    def p_save_index(self, index: Dict[str, dict]) -> None:
+        with open(self.index_path, "w") as f:
+            json.dump(index, f, indent=2)
+
+    @staticmethod
+    @typechecked
+    def slug(name: str) -> str:
+        return name.lower().replace(" ", "-")
+
+    @typechecked
+    def p_skill_path(self, skill_id: str) -> str:
+        return os.path.join(self.skills_dir, f"{skill_id}.md")
+
+    @typechecked
+    def list_all(self) -> list[Skill]:
+        index = self.p_load_index()
+        result: list[Skill] = []
+        if not os.path.exists(self.skills_dir):
+            return result
+        for fname in os.listdir(self.skills_dir):
+            if not fname.endswith(".md"):
+                continue
+            fpath = os.path.join(self.skills_dir, fname)
+            with open(fpath) as f:
+                content = f.read()
+            skill_id = fname.removesuffix(".md")
+            meta = index.get(skill_id, {})
+            result.append(Skill(
+                id=skill_id,
+                name=meta.get("name", skill_id.replace("-", " ").replace("_", " ").title()),
+                description=meta.get("description", ""),
+                content=content,
+                file_path=fpath,
+                command=meta.get("command", skill_id),
+            ))
+        return result
+
+    @typechecked
+    def get(self, skill_id: str) -> Optional[Skill]:
+        for s in self.list_all():
+            if s.id == skill_id:
+                return s
+        return None
+
+    @typechecked
+    def create(self, name: str, description: str, content: str, command: str = "") -> Skill:
+        slug = self.slug(name)
+        fpath = self.p_skill_path(slug)
+        with open(fpath, "w") as f:
+            f.write(content)
+        index = self.p_load_index()
+        index[slug] = {"name": name, "description": description, "command": command or slug}
+        self.p_save_index(index)
+        return Skill(id=slug, name=name, description=description,
+                     content=content, file_path=fpath, command=command or slug)
+
+    @typechecked
+    def update(self, skill_id: str, *, name: Optional[str] = None,
+               description: Optional[str] = None, content: Optional[str] = None,
+               command: Optional[str] = None) -> Skill:
+        fpath = self.p_skill_path(skill_id)
+        if not os.path.exists(fpath):
+            raise FileNotFoundError(skill_id)
+        if content is not None:
+            with open(fpath, "w") as f:
+                f.write(content)
+        index = self.p_load_index()
+        meta = index.get(skill_id, {})
+        if name is not None:
+            meta["name"] = name
+        if description is not None:
+            meta["description"] = description
+        if command is not None:
+            meta["command"] = command
+        index[skill_id] = meta
+        self.p_save_index(index)
+        with open(fpath) as f:
+            content = f.read()
+        return Skill(id=skill_id, name=meta.get("name", skill_id),
+                     description=meta.get("description", ""),
+                     content=content, file_path=fpath, command=meta.get("command", skill_id))
+
+    @typechecked
+    def delete(self, skill_id: str) -> None:
+        fpath = self.p_skill_path(skill_id)
+        if os.path.exists(fpath):
+            os.remove(fpath)
+        index = self.p_load_index()
+        index.pop(skill_id, None)
+        self.p_save_index(index)

backend/apps/skills/parse_frontmatter.py

@@ -0,0 +1,19 @@
+from typeguard import typechecked
+from typing import Tuple
+import re
+
+@typechecked
+def parse_frontmatter(raw: str) -> Tuple[dict, str]:
+    if not raw.startswith("---"):
+        return {}, raw
+    end = raw.find("---", 3)
+    if end == -1:
+        return {}, raw
+    fm_block = raw[3:end].strip()
+    body = raw[end + 3:].strip()
+    meta: dict = {}
+    for line in fm_block.splitlines():
+        m = re.match(r"^(\w[\w_-]*)\s*:\s*(.+)$", line)
+        if m:
+            meta[m.group(1).strip()] = m.group(2).strip().strip('"').strip("'")
+    return meta, body

backend/apps/skills/registry_refresh_loop/fetch_all_registry_skills/fetch_all_registry_skills.py

@@ -0,0 +1,47 @@
+
+
+
+
+import asyncio
+import httpx
+from typeguard import typechecked
+from backend.apps.skills.registry_refresh_loop.fetch_all_registry_skills.utils.fetch_skill_paths import fetch_skill_paths
+from backend.apps.skills.registry_refresh_loop.fetch_all_registry_skills.utils.fetch_one_skill import fetch_one_skill
+
+
+@typechecked
+async def fetch_all_registry_skills(
+    num_concurrent_fetches: int,
+    manifest_url: str,
+    raw_base: str,
+    repo: str,
+    branch: str,
+) -> dict[str, dict]:
+    result: dict[str, dict] = {}
+    async with httpx.AsyncClient(timeout=30.0) as client:
+        try:
+            paths = await fetch_skill_paths(
+                client=client, 
+                manifest_url=manifest_url
+            )
+        except Exception as e:
+            print(f"[fetch_all_registry_skills] Skill registry manifest fetch failed: {e}")
+            return result
+        print(f"[fetch_all_registry_skills] Skill registry: found {len(paths)} skills in manifest, fetching...")
+        sem = asyncio.Semaphore(num_concurrent_fetches)
+        records = await asyncio.gather(
+            *[fetch_one_skill(
+                client=client, 
+                sem=sem, 
+                folder=folder, 
+                plugin_name=plugin,
+                raw_base=raw_base,
+                repo=repo,
+                branch=branch
+            ) for folder, plugin in paths]
+        )
+        for rec in records:
+            if rec:
+                result[rec["name"]] = rec
+    print(f"[fetch_all_registry_skills] Skill registry cache refreshed: {len(result)} skills")
+    return result

backend/apps/skills/registry_refresh_loop/fetch_all_registry_skills/utils/fetch_one_skill.py

@@ -0,0 +1,39 @@
+import asyncio
+from typing import Optional
+import httpx
+from backend.apps.skills.parse_frontmatter import parse_frontmatter
+from typeguard import typechecked
+
+@typechecked
+async def fetch_one_skill(
+    client: httpx.AsyncClient,
+    sem: asyncio.Semaphore,
+    folder: str,
+    plugin_name: str,
+    raw_base: str,
+    repo: str,
+    branch: str,
+) -> Optional[dict]:
+    async with sem:
+        try:
+            resp = await client.get(f"{raw_base}/{folder}/SKILL.md")
+            if resp.status_code != 200:
+                return None
+            raw = resp.text
+        except Exception as exc:
+            print(f"[fetch_one_skill] Failed to fetch {folder}/SKILL.md: {exc}")
+            return None
+
+    meta, body = parse_frontmatter(raw)
+    name = meta.get("name", "")
+    if not name:
+        name = folder.rsplit("/", 1)[-1].replace("-", " ").replace("_", " ").title()
+
+    return {
+        "name": name,
+        "description": meta.get("description", ""),
+        "content": body,
+        "folder": folder,
+        "category": plugin_name.replace("-", " ").replace("_", " ").title(),
+        "repositoryUrl": f"https://github.com/{repo}/tree/{branch}/{folder}",
+    }

backend/apps/skills/registry_refresh_loop/fetch_all_registry_skills/utils/fetch_skill_paths.py

@@ -0,0 +1,21 @@
+
+import httpx
+from typeguard import typechecked
+
+@typechecked
+async def fetch_skill_paths(
+        client: httpx.AsyncClient, 
+        manifest_url: str
+    ) -> list[tuple[str, str]]:
+
+    """Fetch marketplace.json and return (folder, plugin_name) pairs."""
+    resp = await client.get(manifest_url)
+    resp.raise_for_status()
+    manifest = resp.json()
+    paths: list[tuple[str, str]] = []
+    for plugin in manifest.get("plugins", []):
+        plugin_name = plugin.get("name", "")
+        for skill_ref in plugin.get("skills", []):
+            paths.append((skill_ref.lstrip("./"), plugin_name))
+    
+    return paths

backend/apps/skills/registry_refresh_loop/registry_refresh_loop.py

@@ -0,0 +1,35 @@
+
+
+
+import asyncio
+from datetime import time
+
+from typeguard import typechecked
+from backend.apps.skills.registry_refresh_loop.fetch_all_registry_skills.fetch_all_registry_skills import fetch_all_registry_skills
+
+@typechecked
+async def registry_refresh_loop(
+    # Args needed for the loop
+    refresh_interval_s: int,
+    registry_cache: dict[str, dict],
+    registry_updated_at: float,
+    # Args needed for fetch_all_registry_skills
+    num_concurrent_fetches: int,
+    manifest_url: str,
+    raw_base: str,
+    repo: str,
+    branch: str,
+) -> None:
+    while True:
+        try:
+            registry_cache = await fetch_all_registry_skills(
+                num_concurrent_fetches=num_concurrent_fetches,
+                manifest_url=manifest_url,
+                raw_base=raw_base,
+                repo=repo,
+                branch=branch,
+            )
+            registry_updated_at = time.time()
+        except Exception as e:
+            print(f"[registry_refresh_loop] Skill registry refresh error: {e}")
+        await asyncio.sleep(refresh_interval_s)

backend/apps/skills/skills.py

@@ -4,29 +4,29 @@ import asyncio
 import json
 import logging
 import os
-import re
-import time
 from contextlib import asynccontextmanager
 from typing import Any, Optional
 
-import httpx
 from fastapi import HTTPException, Query
 from pydantic import BaseModel
 
 from backend.config.Apps import SubApp
 from backend.config.paths import DB_ROOT
-from backend.apps.skills.Skill import Skill
+from backend.apps.skills.SkillStore import SkillStore
+from backend.apps.skills.parse_frontmatter import parse_frontmatter
+from backend.apps.skills.registry_refresh_loop.registry_refresh_loop import registry_refresh_loop
 
 logger = logging.getLogger(__name__)
 
 # ---------------------------------------------------------------------------
-# Paths
+# Paths & singleton store
 # ---------------------------------------------------------------------------
 
 SKILLS_DIR = os.path.expanduser("~/.claude/skills")
-INDEX_PATH = os.path.join(SKILLS_DIR, ".skills_index.json")
 SKILLS_WORKSPACE_DIR = os.path.join(DB_ROOT, "skills_workspace")
 
+SKILL_STORE = SkillStore(skills_dir=SKILLS_DIR)
+
 # ---------------------------------------------------------------------------
 # Registry constants
 # ---------------------------------------------------------------------------
@@ -50,9 +50,15 @@ _refresh_task: Optional[asyncio.Task] = None
 @asynccontextmanager
 async def skills_lifespan():
     global _refresh_task
-    os.makedirs(SKILLS_DIR, exist_ok=True)
     os.makedirs(SKILLS_WORKSPACE_DIR, exist_ok=True)
-    _refresh_task = asyncio.create_task(_registry_refresh_loop())
+    _refresh_task = asyncio.create_task(registry_refresh_loop(
+        refresh_interval_s=_REFRESH_INTERVAL_S,
+        registry_cache=_registry_cache,
+        registry_updated_at=_registry_updated_at,
+        num_concurrent_fetches=_CONCURRENT_FETCHES,
+        manifest_url=_MANIFEST_URL,
+        raw_base=_RAW_BASE,
+    ))
     yield
     if _refresh_task:
         _refresh_task.cancel()
@@ -65,69 +71,6 @@ async def skills_lifespan():
 skills = SubApp("skills", skills_lifespan)
 
 
-# ===========================================================================
-# Local skill helpers
-# ===========================================================================
-
-
-def _load_index() -> dict[str, dict]:
-    if os.path.exists(INDEX_PATH):
-        with open(INDEX_PATH) as f:
-            return json.load(f)
-    return {}
-
-
-def _save_index(index: dict[str, dict]) -> None:
-    with open(INDEX_PATH, "w") as f:
-        json.dump(index, f, indent=2)
-
-
-def _slug(name: str) -> str:
-    return name.lower().replace(" ", "-")
-
-
-def _sync_skills() -> list[Skill]:
-    """Scan ~/.claude/skills/ for .md files and reconcile with the sidecar index."""
-    index = _load_index()
-    result: list[Skill] = []
-    if not os.path.exists(SKILLS_DIR):
-        return result
-    for fname in os.listdir(SKILLS_DIR):
-        if not fname.endswith(".md"):
-            continue
-        fpath = os.path.join(SKILLS_DIR, fname)
-        with open(fpath) as f:
-            content = f.read()
-        skill_id = fname.removesuffix(".md")
-        meta = index.get(skill_id, {})
-        result.append(Skill(
-            id=skill_id,
-            name=meta.get("name", skill_id.replace("-", " ").replace("_", " ").title()),
-            description=meta.get("description", ""),
-            content=content,
-            file_path=fpath,
-            command=meta.get("command", skill_id),
-        ))
-    return result
-
-
-def _parse_frontmatter(raw: str) -> tuple[dict, str]:
-    """Split YAML frontmatter from markdown body."""
-    if not raw.startswith("---"):
-        return {}, raw
-    end = raw.find("---", 3)
-    if end == -1:
-        return {}, raw
-    fm_block = raw[3:end].strip()
-    body = raw[end + 3:].strip()
-    meta: dict = {}
-    for line in fm_block.splitlines():
-        m = re.match(r"^(\w[\w_-]*)\s*:\s*(.+)$", line)
-        if m:
-            meta[m.group(1).strip()] = m.group(2).strip().strip('"').strip("'")
-    return meta, body
-
-
 # ===========================================================================
 # Local skill routes
 # ===========================================================================
@@ -135,7 +78,7 @@ def _parse_frontmatter(raw: str) -> tuple[dict, str]:
 
 @skills.router.get("/list")
 async def list_skills():
-    return {"skills": [s.model_dump() for s in _sync_skills()]}
+    return {"skills": [s.model_dump() for s in SKILL_STORE.list_all()]}
 
 
 @skills.router.get("/workspace/{workspace_id}")
@@ -159,7 +102,7 @@ async def read_skill_workspace(workspace_id: str):
         except json.JSONDecodeError:
             pass
 
-    frontmatter, _ = _parse_frontmatter(skill_content) if skill_content else ({}, "")
+    frontmatter, _ = parse_frontmatter(skill_content) if skill_content else ({}, "")
 
     return {"skill_content": skill_content, "meta": meta, "frontmatter": frontmatter}
 
@@ -185,10 +128,10 @@ async def seed_skill_workspace(body: _WorkspaceSeedBody):
 
 @skills.router.get("/detail/{skill_id}")
 async def get_skill(skill_id: str):
-    for s in _sync_skills():
-        if s.id == skill_id:
-            return s.model_dump()
-    raise HTTPException(status_code=404, detail="Skill not found")
+    s = SKILL_STORE.get(skill_id)
+    if not s:
+        raise HTTPException(status_code=404, detail="Skill not found")
+    return s.model_dump()
 
 
 class _SkillCreateBody(BaseModel):
@@ -200,19 +143,7 @@ class _SkillCreateBody(BaseModel):
 
 @skills.router.post("/create")
 async def create_skill(body: _SkillCreateBody):
-    slug = _slug(body.name)
-    fpath = os.path.join(SKILLS_DIR, f"{slug}.md")
-    with open(fpath, "w") as f:
-        f.write(body.content)
-
-    index = _load_index()
-    index[slug] = {"name": body.name, "description": body.description, "command": body.command or slug}
-    _save_index(index)
-
-    skill = Skill(
-        id=slug, name=body.name, description=body.description,
-        content=body.content, file_path=fpath, command=body.command or slug,
-    )
+    skill = SKILL_STORE.create(body.name, body.description, body.content, body.command)
     return {"ok": True, "skill": skill.model_dump()}
 
 
@@ -225,127 +156,21 @@ class _SkillUpdateBody(BaseModel):
 
 @skills.router.put("/{skill_id}")
 async def update_skill(skill_id: str, body: _SkillUpdateBody):
-    fpath = os.path.join(SKILLS_DIR, f"{skill_id}.md")
-    if not os.path.exists(fpath):
+    try:
+        skill = SKILL_STORE.update(
+            skill_id, name=body.name, description=body.description,
+            content=body.content, command=body.command,
+        )
+    except FileNotFoundError:
         raise HTTPException(status_code=404, detail="Skill not found")
-
-    if body.content is not None:
-        with open(fpath, "w") as f:
-            f.write(body.content)
-
-    index = _load_index()
-    meta = index.get(skill_id, {})
-    if body.name is not None:
-        meta["name"] = body.name
-    if body.description is not None:
-        meta["description"] = body.description
-    if body.command is not None:
-        meta["command"] = body.command
-    index[skill_id] = meta
-    _save_index(index)
-
-    with open(fpath) as f:
-        content = f.read()
-
-    skill = Skill(
-        id=skill_id, name=meta.get("name", skill_id),
-        description=meta.get("description", ""),
-        content=content, file_path=fpath, command=meta.get("command", skill_id),
-    )
     return {"ok": True, "skill": skill.model_dump()}
 
 
 @skills.router.delete("/{skill_id}")
 async def delete_skill(skill_id: str):
-    fpath = os.path.join(SKILLS_DIR, f"{skill_id}.md")
-    if os.path.exists(fpath):
-        os.remove(fpath)
-    index = _load_index()
-    index.pop(skill_id, None)
-    _save_index(index)
+    SKILL_STORE.delete(skill_id)
     return {"ok": True}
 
-
-# ===========================================================================
-# Registry helpers
-# ===========================================================================
-
-
-async def _fetch_skill_paths(client: httpx.AsyncClient) -> list[tuple[str, str]]:
-    """Fetch marketplace.json and return (folder, plugin_name) pairs."""
-    resp = await client.get(_MANIFEST_URL)
-    resp.raise_for_status()
-    manifest = resp.json()
-    paths: list[tuple[str, str]] = []
-    for plugin in manifest.get("plugins", []):
-        plugin_name = plugin.get("name", "")
-        for skill_ref in plugin.get("skills", []):
-            paths.append((skill_ref.lstrip("./"), plugin_name))
-    return paths
-
-
-async def _fetch_one_skill(
-    client: httpx.AsyncClient,
-    sem: asyncio.Semaphore,
-    folder: str,
-    plugin_name: str,
-) -> Optional[dict]:
-    async with sem:
-        try:
-            resp = await client.get(f"{_RAW_BASE}/{folder}/SKILL.md")
-            if resp.status_code != 200:
-                return None
-            raw = resp.text
-        except Exception as exc:
-            logger.debug("Failed to fetch %s/SKILL.md: %s", folder, exc)
-            return None
-
-    meta, body = _parse_frontmatter(raw)
-    name = meta.get("name", "")
-    if not name:
-        name = folder.rsplit("/", 1)[-1].replace("-", " ").replace("_", " ").title()
-
-    return {
-        "name": name,
-        "description": meta.get("description", ""),
-        "content": body,
-        "folder": folder,
-        "category": plugin_name.replace("-", " ").replace("_", " ").title(),
-        "repositoryUrl": f"https://github.com/{_REPO}/tree/{_BRANCH}/{folder}",
-    }
-
-
-async def _fetch_all_registry_skills() -> dict[str, dict]:
-    result: dict[str, dict] = {}
-    async with httpx.AsyncClient(timeout=30.0) as client:
-        try:
-            paths = await _fetch_skill_paths(client)
-        except Exception as e:
-            logger.warning("Skill registry manifest fetch failed: %s", e)
-            return result
-        logger.info("Skill registry: found %d skills in manifest, fetching...", len(paths))
-        sem = asyncio.Semaphore(_CONCURRENT_FETCHES)
-        records = await asyncio.gather(
-            *[_fetch_one_skill(client, sem, folder, plugin) for folder, plugin in paths]
-        )
-        for rec in records:
-            if rec:
-                result[rec["name"]] = rec
-    logger.info("Skill registry cache refreshed: %d skills", len(result))
-    return result
-
-
-async def _registry_refresh_loop() -> None:
-    global _registry_cache, _registry_updated_at
-    while True:
-        try:
-            _registry_cache = await _fetch_all_registry_skills()
-            _registry_updated_at = time.time()
-        except Exception as e:
-            logger.exception("Skill registry refresh error: %s", e)
-        await asyncio.sleep(_REFRESH_INTERVAL_S)
-
-
 # ===========================================================================
 # Registry routes
 # ===========================================================================