[eric] mcp-registry: the 21k-server crawl arms on first request instead of at boot, so an idle app crawls nothing (ENG-288)

backend/apps/mcp_registry/mcp_registry.py

@@ -306,13 +306,29 @@ async def p_refresh_loop():
         await asyncio.sleep(REFRESH_INTERVAL_S)
 
 
+def p_start_refresh_task() -> None:
+    global p_refresh_task
+    p_refresh_task = asyncio.create_task(p_refresh_loop())
+
+
+def arm_registry_refresh() -> None:
+    """Start the crawl, once, the first time anyone actually asks for the registry.
+
+    Arming at boot cost every user ~215 sequential requests an hour plus a GitHub star pass
+    to populate a browser most of them never open, and the in-memory cache re-paid it on
+    every restart. Only a request can arm it now, so an idle app crawls nothing.
+    """
+    if p_refresh_task is None:
+        p_start_refresh_task()
+
+
 @asynccontextmanager
 async def mcp_registry_lifespan():
     global p_refresh_task
-    p_refresh_task = asyncio.create_task(p_refresh_loop())
     yield
     if p_refresh_task:
         p_refresh_task.cancel()
+        p_refresh_task = None
         try:
             await p_refresh_task
         except asyncio.CancelledError:
@@ -324,6 +340,7 @@ mcp_registry = SubApp("mcp-registry", mcp_registry_lifespan)
 
 @mcp_registry.router.get("/stats")
 async def registry_stats():
+    arm_registry_refresh()
     google = sum(1 for s in p_cache.values() if s.get("source") == "google")
     community = sum(1 for s in p_cache.values() if s.get("source") == "community")
     return {
@@ -342,6 +359,7 @@ async def registry_search(
     sort: str = Query("name", description="Sort by: name, stars"),
     source: str = Query("", description="Filter by source: google, community, or empty for all"),
 ):
+    arm_registry_refresh()
     pool = p_cache.values()
     if source:
         pool = [s for s in pool if s.get("source") == source]
@@ -387,6 +405,7 @@ async def registry_search(
 
 @mcp_registry.router.get("/detail/{server_name:path}")
 async def registry_detail(server_name: str):
+    arm_registry_refresh()
     srv = p_cache.get(server_name)
     if not srv:
         return {"error": "Server not found"}, 404

backend/tests/test_mcp_registry_crawls_on_demand.py

@@ -0,0 +1,74 @@
+"""The 21k-server MCP registry crawl must be demand-driven, not unconditional (ENG-288).
+
+Measured on the shipped tree: `mcp_registry_lifespan` armed the refresh task at boot, and
+the loop then fetched the whole official registry (100 entries a page, ~215 sequential
+requests against a live count of 21,403), scraped Google's catalogue, and ran a GitHub
+star batch. Immediately, then every 3600s, forever, for every user, whether or not they
+ever opened the Tools page that consumes it.
+
+The cache is an in-memory dict with no disk persistence, so a restart pays the full cold
+crawl again. And the star pass cannot finish by construction: unauthenticated
+`GITHUB_BATCH` is 50 repos/hour against 21,403, which is 400+ hours of uninterrupted
+uptime into a cache that resets on restart.
+
+The browser IS a real consumer (11 imports, live in Tools, ungated), so this must stay
+lazy rather than be deleted: the fix is that only a request can arm the crawl.
+
+Run:
+    backend/.venv/bin/python -m pytest backend/tests/test_mcp_registry_crawls_on_demand.py -v
+"""
+
+from typing import Any
+
+import pytest
+
+from backend.apps.mcp_registry import mcp_registry
+
+
+@pytest.fixture(autouse=True)
+def p_no_network(monkeypatch: Any) -> None:
+    """Nothing here may touch the network; we are counting intent, not results."""
+    async def p_refuse(*args: Any, **kwargs: Any) -> dict:
+        raise AssertionError("the crawl reached the network during a test")
+    monkeypatch.setattr(mcp_registry, "p_fetch_all_servers", p_refuse)
+    monkeypatch.setattr(mcp_registry, "p_fetch_google_servers", p_refuse)
+
+
+@pytest.mark.asyncio
+async def test_boot_does_not_start_the_crawl(monkeypatch: Any) -> None:
+    """An app that boots and sits idle must issue zero registry requests."""
+    started: list[str] = []
+    monkeypatch.setattr(mcp_registry, "arm_registry_refresh", lambda: started.append("armed"))
+    async with mcp_registry.mcp_registry_lifespan():
+        pass
+    assert started == [], (
+        "boot armed the registry crawl; an idle app pays ~215 requests/hour for a page "
+        "the user never opened"
+    )
+
+
+@pytest.mark.asyncio
+async def test_asking_for_the_registry_arms_it(monkeypatch: Any) -> None:
+    """The other direction: the feature must still work when someone opens Tools."""
+    started: list[str] = []
+    monkeypatch.setattr(mcp_registry, "p_start_refresh_task", lambda: started.append("armed"))
+    mcp_registry.p_refresh_task = None
+    mcp_registry.arm_registry_refresh()
+    assert started == ["armed"], "opening the registry did not start the crawl, so search stays empty"
+
+
+@pytest.mark.asyncio
+async def test_arming_twice_starts_one_crawl(monkeypatch: Any) -> None:
+    """Bounded by construction: N requests must not mean N crawlers."""
+    started: list[str] = []
+
+    def p_fake_start() -> None:
+        started.append("armed")
+        mcp_registry.p_refresh_task = object()
+
+    monkeypatch.setattr(mcp_registry, "p_start_refresh_task", p_fake_start)
+    mcp_registry.p_refresh_task = None
+    for _ in range(5):
+        mcp_registry.arm_registry_refresh()
+    assert started == ["armed"], f"armed {len(started)} crawlers for 5 requests"
+    mcp_registry.p_refresh_task = None