From 41aa8b02a570ae466cdded7a67a64a0bb76ecd9d Mon Sep 17 00:00:00 2001 From: ciregenz Date: Thu, 13 Aug 2026 00:58:41 -0700 Subject: [PATCH] [eric] mcp-registry: the 21k-server crawl arms on first request instead of at boot, so an idle app crawls nothing (ENG-288) --- backend/apps/mcp_registry/mcp_registry.py | 21 +++++- .../test_mcp_registry_crawls_on_demand.py | 74 +++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 backend/tests/test_mcp_registry_crawls_on_demand.py diff --git a/backend/apps/mcp_registry/mcp_registry.py b/backend/apps/mcp_registry/mcp_registry.py index 96fed0f8..7f175c9b 100644 --- a/backend/apps/mcp_registry/mcp_registry.py +++ b/backend/apps/mcp_registry/mcp_registry.py @@ -306,13 +306,29 @@ async def p_refresh_loop(): await asyncio.sleep(REFRESH_INTERVAL_S) +def p_start_refresh_task() -> None: + global p_refresh_task + p_refresh_task = asyncio.create_task(p_refresh_loop()) + + +def arm_registry_refresh() -> None: + """Start the crawl, once, the first time anyone actually asks for the registry. + + Arming at boot cost every user ~215 sequential requests an hour plus a GitHub star pass + to populate a browser most of them never open, and the in-memory cache re-paid it on + every restart. Only a request can arm it now, so an idle app crawls nothing. + """ + if p_refresh_task is None: + p_start_refresh_task() + + @asynccontextmanager async def mcp_registry_lifespan(): global p_refresh_task - p_refresh_task = asyncio.create_task(p_refresh_loop()) yield if p_refresh_task: p_refresh_task.cancel() + p_refresh_task = None try: await p_refresh_task except asyncio.CancelledError: @@ -324,6 +340,7 @@ mcp_registry = SubApp("mcp-registry", mcp_registry_lifespan) @mcp_registry.router.get("/stats") async def registry_stats(): + arm_registry_refresh() google = sum(1 for s in p_cache.values() if s.get("source") == "google") community = sum(1 for s in p_cache.values() if s.get("source") == "community") return { @@ -342,6 +359,7 @@ async def registry_search( sort: str = Query("name", description="Sort by: name, stars"), source: str = Query("", description="Filter by source: google, community, or empty for all"), ): + arm_registry_refresh() pool = p_cache.values() if source: pool = [s for s in pool if s.get("source") == source] @@ -387,6 +405,7 @@ async def registry_search( @mcp_registry.router.get("/detail/{server_name:path}") async def registry_detail(server_name: str): + arm_registry_refresh() srv = p_cache.get(server_name) if not srv: return {"error": "Server not found"}, 404 diff --git a/backend/tests/test_mcp_registry_crawls_on_demand.py b/backend/tests/test_mcp_registry_crawls_on_demand.py new file mode 100644 index 00000000..be7b1300 --- /dev/null +++ b/backend/tests/test_mcp_registry_crawls_on_demand.py @@ -0,0 +1,74 @@ +"""The 21k-server MCP registry crawl must be demand-driven, not unconditional (ENG-288). + +Measured on the shipped tree: `mcp_registry_lifespan` armed the refresh task at boot, and +the loop then fetched the whole official registry (100 entries a page, ~215 sequential +requests against a live count of 21,403), scraped Google's catalogue, and ran a GitHub +star batch. Immediately, then every 3600s, forever, for every user, whether or not they +ever opened the Tools page that consumes it. + +The cache is an in-memory dict with no disk persistence, so a restart pays the full cold +crawl again. And the star pass cannot finish by construction: unauthenticated +`GITHUB_BATCH` is 50 repos/hour against 21,403, which is 400+ hours of uninterrupted +uptime into a cache that resets on restart. + +The browser IS a real consumer (11 imports, live in Tools, ungated), so this must stay +lazy rather than be deleted: the fix is that only a request can arm the crawl. + +Run: + backend/.venv/bin/python -m pytest backend/tests/test_mcp_registry_crawls_on_demand.py -v +""" + +from typing import Any + +import pytest + +from backend.apps.mcp_registry import mcp_registry + + +@pytest.fixture(autouse=True) +def p_no_network(monkeypatch: Any) -> None: + """Nothing here may touch the network; we are counting intent, not results.""" + async def p_refuse(*args: Any, **kwargs: Any) -> dict: + raise AssertionError("the crawl reached the network during a test") + monkeypatch.setattr(mcp_registry, "p_fetch_all_servers", p_refuse) + monkeypatch.setattr(mcp_registry, "p_fetch_google_servers", p_refuse) + + +@pytest.mark.asyncio +async def test_boot_does_not_start_the_crawl(monkeypatch: Any) -> None: + """An app that boots and sits idle must issue zero registry requests.""" + started: list[str] = [] + monkeypatch.setattr(mcp_registry, "arm_registry_refresh", lambda: started.append("armed")) + async with mcp_registry.mcp_registry_lifespan(): + pass + assert started == [], ( + "boot armed the registry crawl; an idle app pays ~215 requests/hour for a page " + "the user never opened" + ) + + +@pytest.mark.asyncio +async def test_asking_for_the_registry_arms_it(monkeypatch: Any) -> None: + """The other direction: the feature must still work when someone opens Tools.""" + started: list[str] = [] + monkeypatch.setattr(mcp_registry, "p_start_refresh_task", lambda: started.append("armed")) + mcp_registry.p_refresh_task = None + mcp_registry.arm_registry_refresh() + assert started == ["armed"], "opening the registry did not start the crawl, so search stays empty" + + +@pytest.mark.asyncio +async def test_arming_twice_starts_one_crawl(monkeypatch: Any) -> None: + """Bounded by construction: N requests must not mean N crawlers.""" + started: list[str] = [] + + def p_fake_start() -> None: + started.append("armed") + mcp_registry.p_refresh_task = object() + + monkeypatch.setattr(mcp_registry, "p_start_refresh_task", p_fake_start) + mcp_registry.p_refresh_task = None + for _ in range(5): + mcp_registry.arm_registry_refresh() + assert started == ["armed"], f"armed {len(started)} crawlers for 5 requests" + mcp_registry.p_refresh_task = None