[eric] tools_lib: leading-_ -> p_/public across tools_lib/mcp_config/mcp_discovery/oauth_tokens/tool_taxonomy; promote cross-file public (load_all_tools/sanitize_server_name/discover_mcp_tools_*), fix re-export + attr test refs

This commit is contained in:
ciregenz
2026-06-23 20:30:14 -07:00
parent deeab52ed4
commit 71b2b33fae
16 changed files with 187 additions and 187 deletions
+2 -2
View File
@@ -13,8 +13,8 @@ from backend.apps.agents.core.models import (
from backend.apps.agents.core.ws_manager import ws_manager
from backend.apps.settings.settings import load_settings
from backend.apps.tools_lib.tools_lib import (
_load_all as load_all_tools,
_sanitize_server_name as sanitize_server_name,
load_all_tools as load_all_tools,
sanitize_server_name as sanitize_server_name,
load_builtin_permissions,
)
from backend.apps.agents.core.error_classify import (
+4 -4
View File
@@ -11,8 +11,8 @@ from typing import Any
from backend.apps.agents.providers.registry import resolve_aux_model
from backend.apps.settings.credentials import get_anthropic_client_for_model
from backend.apps.settings.settings import load_settings
from backend.apps.tools_lib.tools_lib import _load_all as load_all_tools
from backend.apps.tools_lib.mcp_config import _sanitize_server_name
from backend.apps.tools_lib.tools_lib import load_all_tools as load_all_tools
from backend.apps.tools_lib.mcp_config import sanitize_server_name
logger = logging.getLogger(__name__)
@@ -156,9 +156,9 @@ def offer_for_gated_server(server_name: str, settings) -> CuratedEntry | None:
return None
# The hot-path hands us a sanitized slug ("google-workspace"); curated ids are display names
# ("Google Workspace"). Match on the slug of both sides so neither form is a load-bearing string.
slug = _sanitize_server_name(server_name)
slug = sanitize_server_name(server_name)
entry = next(
(e for e in p_build_available_shortlist(settings) if _sanitize_server_name(e["id"]) == slug),
(e for e in p_build_available_shortlist(settings) if sanitize_server_name(e["id"]) == slug),
None,
)
if entry is None:
@@ -26,8 +26,8 @@ from backend.apps.agents.manager.prompt.attachments import (
resolve_context_paths,
)
from backend.apps.tools_lib.tools_lib import (
_load_all as load_all_tools,
_sanitize_server_name as sanitize_server_name,
load_all_tools as load_all_tools,
sanitize_server_name as sanitize_server_name,
derive_mcp_config,
refresh_airtable_token,
refresh_google_token,
@@ -17,8 +17,8 @@ from backend.apps.agents.core.ws_manager import ws_manager
from backend.apps.agents.manager.permissions import path_gate
from backend.apps.agents.manager.permissions.ApprovalDecision import ApprovalDecision
from backend.apps.tools_lib.tools_lib import (
_load_all as load_all_tools,
_save as save_tool,
load_all_tools as load_all_tools,
save as save_tool,
load_builtin_permissions,
load_trusted_sensitive_paths,
resolve_policy_slot,
@@ -14,8 +14,8 @@ from backend.apps.agents.manager.prompt.tool_catalog import (
get_denied_tool_names,
)
from backend.apps.tools_lib.tools_lib import (
_load_all as load_all_tools,
_sanitize_server_name as sanitize_server_name,
load_all_tools as load_all_tools,
sanitize_server_name as sanitize_server_name,
)
@@ -4,8 +4,8 @@ from typeguard import typechecked
from backend.apps.modes.modes import load_mode
from backend.apps.tools_lib.tools_lib import (
_load_all as load_all_tools,
_sanitize_server_name as sanitize_server_name,
load_all_tools as load_all_tools,
sanitize_server_name as sanitize_server_name,
)
from backend.apps.agents.manager.prompt.tool_catalog import is_fully_denied
@@ -5,8 +5,8 @@ from typeguard import typechecked
from backend.apps.tools_lib.models import ToolDefinition
from backend.apps.tools_lib.tools_lib import (
_load_all as load_all_tools,
_sanitize_server_name as sanitize_server_name,
load_all_tools as load_all_tools,
sanitize_server_name as sanitize_server_name,
load_builtin_permissions,
)
+9 -9
View File
@@ -11,12 +11,12 @@ from backend.apps.tools_lib.oauth_config import OPENSWARM_OAUTH_BASE_URL
logger = logging.getLogger(__name__)
def _sanitize_server_name(name: str) -> str:
def sanitize_server_name(name: str) -> str:
"""Convert a tool name into a valid MCP server identifier (alphanumeric + hyphens)."""
return re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-")
def _extra_bin_dirs() -> list[str]:
def p_extra_bin_dirs() -> list[str]:
"""Well-known user-local bin directories that may not be on PATH in packaged apps."""
home = os.path.expanduser("~")
# Bundled uv-bin (ships uvx for non-dev users)
@@ -46,14 +46,14 @@ def _extra_bin_dirs() -> list[str]:
return dirs
def _resolve_command(command: str) -> str | None:
def resolve_command(command: str) -> str | None:
"""Find a command on PATH, falling back to common user-local bin directories
and bundled binaries (uv-bin for uvx/uv)."""
found = shutil.which(command)
if found:
return found
# Windows binaries need an extension. shutil.which() handles PATHEXT for
# PATH lookups, but we manually scan _extra_bin_dirs below; replicate
# PATH lookups, but we manually scan p_extra_bin_dirs below; replicate
# the suffix probing here so `uvx` finds `uvx.exe`, etc.
if sys.platform == "win32":
suffixes = [""] + os.environ.get("PATHEXT", ".COM;.EXE;.BAT;.CMD").lower().split(os.pathsep)
@@ -65,7 +65,7 @@ def _resolve_command(command: str) -> str | None:
if os.path.isfile(candidate) and os.access(candidate, os.X_OK):
return candidate
return None
for d in _extra_bin_dirs():
for d in p_extra_bin_dirs():
hit = _probe(d)
if hit:
return hit
@@ -74,9 +74,9 @@ def _resolve_command(command: str) -> str | None:
return _probe(os.path.join(_backend, "uv-bin"))
def _augmented_path() -> str:
def augmented_path() -> str:
"""Return PATH with extra bin dirs prepended (for child process environments)."""
extra = [d for d in _extra_bin_dirs() if os.path.isdir(d)]
extra = [d for d in p_extra_bin_dirs() if os.path.isdir(d)]
current = os.environ.get("PATH", "")
seen: set[str] = set()
parts: list[str] = []
@@ -262,13 +262,13 @@ def derive_mcp_config(tool: ToolDefinition) -> Optional[dict]:
logger.info(f"Using pre-installed npm MCP server for {pkg_name}")
if not os.path.isabs(config.get("command", "")):
resolved = _resolve_command(config["command"])
resolved = resolve_command(config["command"])
if resolved:
config["command"] = resolved
else:
logger.warning(f"Command '{config['command']}' not found on PATH or bundled directories")
env = config.setdefault("env", {})
env.setdefault("PATH", _augmented_path())
env.setdefault("PATH", augmented_path())
env.setdefault("PYTHONPATH", "")
# Point uv/uvx at our bundled Python; avoids macOS CLT popup on fresh Macs
# and avoids downloading Python at runtime
+13 -13
View File
@@ -8,12 +8,12 @@ import shutil
import httpx
from fastapi import HTTPException
from backend.apps.tools_lib.mcp_config import _augmented_path, _resolve_command
from backend.apps.tools_lib.mcp_config import augmented_path, resolve_command
logger = logging.getLogger(__name__)
def _parse_sse_json(text: str) -> dict | None:
def p_parse_sse_json(text: str) -> dict | None:
"""Extract JSON from an SSE response body (handles `data: {...}` lines)."""
for line in text.splitlines():
stripped = line.strip()
@@ -30,7 +30,7 @@ def _parse_sse_json(text: str) -> dict | None:
return None
async def _discover_mcp_tools_http(url: str, headers: dict | None = None) -> list[dict]:
async def discover_mcp_tools_http(url: str, headers: dict | None = None) -> list[dict]:
"""Connect to a Streamable HTTP MCP server and call tools/list via JSON-RPC POST."""
h = {
"Content-Type": "application/json",
@@ -62,7 +62,7 @@ async def _discover_mcp_tools_http(url: str, headers: dict | None = None) -> lis
ct = list_resp.headers.get("content-type", "")
if "text/event-stream" in ct:
data = _parse_sse_json(list_resp.text)
data = p_parse_sse_json(list_resp.text)
else:
data = list_resp.json()
@@ -73,7 +73,7 @@ async def _discover_mcp_tools_http(url: str, headers: dict | None = None) -> lis
return [{"name": t.get("name", ""), "description": t.get("description", ""), "inputSchema": t.get("inputSchema")} for t in tools_list]
async def _discover_mcp_tools_sse(url: str, headers: dict | None = None) -> list[dict]:
async def discover_mcp_tools_sse(url: str, headers: dict | None = None) -> list[dict]:
"""Connect to a legacy SSE MCP server (GET event-stream + POST messages) and call tools/list."""
from mcp.client.sse import sse_client
from mcp import ClientSession
@@ -99,10 +99,10 @@ async def _discover_mcp_tools_sse(url: str, headers: dict | None = None) -> list
raise HTTPException(status_code=502, detail=f"SSE discovery failed: {first}") from first
_NPX_CACHE_RE = re.compile(r"_npx[/\\]([0-9a-f]{8,})[/\\]")
P_NPX_CACHE_RE = re.compile(r"_npx[/\\]([0-9a-f]{8,})[/\\]")
def _try_heal_npx_cache(stderr: str) -> str | None:
def p_try_heal_npx_cache(stderr: str) -> str | None:
"""On `ERR_MODULE_NOT_FOUND` pointing into `~/.npm/_npx/<hash>/`, wipe that one dir.
Why: interrupted npx installs leave a `package-lock.json` in the cache dir so
@@ -112,7 +112,7 @@ def _try_heal_npx_cache(stderr: str) -> str | None:
"""
if "ERR_MODULE_NOT_FOUND" not in stderr:
return None
m = _NPX_CACHE_RE.search(stderr)
m = P_NPX_CACHE_RE.search(stderr)
if not m:
return None
hash_ = m.group(1)
@@ -124,18 +124,18 @@ def _try_heal_npx_cache(stderr: str) -> str | None:
return hash_
async def _discover_mcp_tools_stdio(command: str, args: list[str] | None = None, env: dict | None = None, _attempt: int = 0) -> list[dict]:
async def discover_mcp_tools_stdio(command: str, args: list[str] | None = None, env: dict | None = None, _attempt: int = 0) -> list[dict]:
"""Spawn a stdio MCP server process and call tools/list via JSON-RPC over stdin/stdout.
On the first attempt, a failure that looks like corrupted npx cache
(`ERR_MODULE_NOT_FOUND` pointing into `~/.npm/_npx/<hash>/`) triggers one
auto-heal + retry. No heal on `_attempt >= 1`.
"""
cmd_path = _resolve_command(command)
cmd_path = resolve_command(command)
if not cmd_path:
raise HTTPException(status_code=400, detail=f"Command '{command}' not found on PATH or common install locations")
proc_env = {**os.environ, **(env or {}), "PATH": _augmented_path()}
proc_env = {**os.environ, **(env or {}), "PATH": augmented_path()}
proc_env.pop("PYTHONPATH", None)
proc = await asyncio.create_subprocess_exec(
@@ -231,8 +231,8 @@ async def _discover_mcp_tools_stdio(command: str, args: list[str] | None = None,
# Heal-on-corrupt-npx-cache still triggers from the EOF branch,
# which now includes the full stderr tail in `e.detail`; so the
# ERR_MODULE_NOT_FOUND signature is still discoverable here.
if _attempt == 0 and _try_heal_npx_cache(str(e.detail) if e.detail is not None else ""):
return await _discover_mcp_tools_stdio(command, args, env, _attempt=1)
if _attempt == 0 and p_try_heal_npx_cache(str(e.detail) if e.detail is not None else ""):
return await discover_mcp_tools_stdio(command, args, env, _attempt=1)
raise
except asyncio.TimeoutError:
# Most common cause: cold npx cache on Windows. The npm install
+13 -13
View File
@@ -13,7 +13,7 @@ from backend.apps.tools_lib.oauth_config import OPENSWARM_OAUTH_BASE_URL
logger = logging.getLogger(__name__)
def _save(tool: ToolDefinition) -> None:
def save(tool: ToolDefinition) -> None:
with open(os.path.join(DATA_DIR, f"{tool.id}.json"), "w") as f:
json.dump(tool.model_dump(), f, indent=2)
@@ -22,7 +22,7 @@ def _save(tool: ToolDefinition) -> None:
# through the Fly cloud-proxy so client_secret values never ship inside the
# desktop binary. v1.0.28 was the last release that used a local Google
# callback with the client_secret in backend/.env.
_TOOL_NAME_TO_PROVIDER = {
P_TOOL_NAME_TO_PROVIDER = {
"airtable": "airtable",
"hubspot": "hubspot",
"discord": "discord",
@@ -35,11 +35,11 @@ _TOOL_NAME_TO_PROVIDER = {
}
def _proxied_provider_for(tool: ToolDefinition) -> Optional[str]:
return _TOOL_NAME_TO_PROVIDER.get(tool.name.lower())
def proxied_provider_for(tool: ToolDefinition) -> Optional[str]:
return P_TOOL_NAME_TO_PROVIDER.get(tool.name.lower())
def _persist_cloud_tokens(tool: ToolDefinition, tokens: dict) -> None:
def persist_cloud_tokens(tool: ToolDefinition, tokens: dict) -> None:
"""Normalise the cloud's claim response into tool.oauth_tokens.
Per-provider shaping mirrors what the v1.0.25 local-callback flow used
@@ -84,7 +84,7 @@ def _persist_cloud_tokens(tool: ToolDefinition, tokens: dict) -> None:
tool.auth_status = "connected"
async def _refresh_via_proxy(provider: str, tool: ToolDefinition, default_expiry: int) -> Optional[str]:
async def p_refresh_via_proxy(provider: str, tool: ToolDefinition, default_expiry: int) -> Optional[str]:
"""Refresh an OAuth access_token by POSTing the refresh_token to the
helper service. Per-provider wrappers below pass a default expires_in
fallback for providers that don't return one.
@@ -108,7 +108,7 @@ async def _refresh_via_proxy(provider: str, tool: ToolDefinition, default_expiry
# Provider rejected; user revoked at the provider's side. Mark
# as needing re-auth so the UI prompts a Reconnect.
tool.auth_status = "expired"
_save(tool)
save(tool)
logger.warning(f"{provider} refresh rejected (user revoked); marking tool as expired")
return None
if resp.status_code != 200:
@@ -128,7 +128,7 @@ async def _refresh_via_proxy(provider: str, tool: ToolDefinition, default_expiry
# Backfill identity label on first successful refresh after upgrade.
if not tool.connected_account_email and data.get("email"):
tool.connected_account_email = data["email"]
_save(tool)
save(tool)
return new_token
except Exception as e:
logger.warning(f"{provider} cloud refresh exception for tool {tool.id}: {e}")
@@ -142,20 +142,20 @@ async def refresh_google_token(tool: ToolDefinition) -> Optional[str]:
refresh_token. Same pattern as Airtable/HubSpot. Pre-v1.0.29 builds
held the secret in their bundled .env; v1.0.29 removed it.
"""
return await _refresh_via_proxy("google", tool, default_expiry=3600)
return await p_refresh_via_proxy("google", tool, default_expiry=3600)
async def refresh_airtable_token(tool: ToolDefinition) -> Optional[str]:
"""Refresh an expired Airtable OAuth access_token."""
return await _refresh_via_proxy("airtable", tool, default_expiry=7200)
return await p_refresh_via_proxy("airtable", tool, default_expiry=7200)
async def refresh_hubspot_token(tool: ToolDefinition) -> Optional[str]:
"""Refresh an expired HubSpot OAuth access_token."""
return await _refresh_via_proxy("hubspot", tool, default_expiry=1800)
return await p_refresh_via_proxy("hubspot", tool, default_expiry=1800)
def _m365_server_script() -> str:
def m365_server_script() -> str:
"""Return the on-disk path to the bundled MS365 MCP server entry.
v1.0.26 replaced the heavy backend/npm-servers/softeria-ms-365-mcp-server/
@@ -180,7 +180,7 @@ def _m365_server_script() -> str:
)
def _m365_cache_env() -> dict[str, str]:
def m365_cache_env() -> dict[str, str]:
cache_dir = os.path.join(os.path.expanduser("~"), ".openswarm")
os.makedirs(cache_dir, exist_ok=True)
return {
+14 -14
View File
@@ -1,8 +1,8 @@
_READ_PREFIXES = ("get", "list", "read", "search", "fetch", "find", "query", "count", "check", "describe", "show", "download", "browse", "analy", "explain")
_WRITE_PREFIXES = ("create", "write", "delete", "update", "send", "remove", "modify", "add", "set", "put", "post", "patch", "insert", "move", "copy", "rename", "archive", "trash", "publish", "approve", "reject")
P_READ_PREFIXES = ("get", "list", "read", "search", "fetch", "find", "query", "count", "check", "describe", "show", "download", "browse", "analy", "explain")
P_WRITE_PREFIXES = ("create", "write", "delete", "update", "send", "remove", "modify", "add", "set", "put", "post", "patch", "insert", "move", "copy", "rename", "archive", "trash", "publish", "approve", "reject")
_SERVICE_RULES: list[tuple[list[str], str, str]] = [
P_SERVICE_RULES: list[tuple[list[str], str, str]] = [
# (keywords, service_name, group)
# Google Workspace
(["gmail"], "Gmail", "Google"),
@@ -31,20 +31,20 @@ _SERVICE_RULES: list[tuple[list[str], str, str]] = [
]
def _categorize_tool(name: str) -> str:
def p_categorize_tool(name: str) -> str:
lower = name.lower().replace("_", " ").replace("-", " ").strip()
for word in lower.split():
for prefix in _READ_PREFIXES:
for prefix in P_READ_PREFIXES:
if word.startswith(prefix):
return "read"
for prefix in _WRITE_PREFIXES:
for prefix in P_WRITE_PREFIXES:
if word.startswith(prefix):
return "write"
return "write"
def _integration_domain(integration: str) -> str:
"""Which curated _SERVICE_RULES set applies to this integration, if any. The Google rules use
def p_integration_domain(integration: str) -> str:
"""Which curated P_SERVICE_RULES set applies to this integration, if any. The Google rules use
generic words (message/table/page/doc/script) that otherwise mis-tag Slack/Notion/Airtable/M365."""
n = (integration or "").lower()
if "google" in n:
@@ -56,13 +56,13 @@ def _integration_domain(integration: str) -> str:
return ""
def _extract_service(name: str, integration: str) -> tuple[str, str]:
def p_extract_service(name: str, integration: str) -> tuple[str, str]:
"""Map a tool name to (service, group). Curated rulesets apply only to the integration they were
written for; every other integration groups under its own name so it isn't mislabeled as Google."""
domain = _integration_domain(integration)
domain = p_integration_domain(integration)
if domain:
lower = name.lower()
for keywords, display, group in _SERVICE_RULES:
for keywords, display, group in P_SERVICE_RULES:
if group != domain:
continue
for kw in keywords:
@@ -73,15 +73,15 @@ def _extract_service(name: str, integration: str) -> tuple[str, str]:
return (integration or "Other"), ""
def _classify_services(
def classify_services(
tool_names: list[str], integration: str
) -> tuple[dict[str, dict[str, list[str]]], dict[str, list[str]], list[str], list[str]]:
"""Bucket tool names into services + service groups + read/write categories for one integration."""
services: dict[str, dict[str, list[str]]] = {}
service_groups: dict[str, list[str]] = {}
for name in tool_names:
cat = _categorize_tool(name)
svc, group = _extract_service(name, integration)
cat = p_categorize_tool(name)
svc, group = p_extract_service(name, integration)
services.setdefault(svc, {"read": [], "write": []})
services[svc][cat].append(name)
if group:
+75 -75
View File
@@ -19,23 +19,23 @@ from backend.config.paths import DATA_ROOT, TOOLS_DIR as DATA_DIR, BUILTIN_PERMI
# oauth_config runs the dotenv load (leaf) so OPENSWARM_OAUTH_BASE_URL is set
# before anything reads it; re-exported here for the route handlers below.
from backend.apps.tools_lib.oauth_config import OPENSWARM_OAUTH_BASE_URL
# _sanitize_server_name + derive_mcp_config re-exported for agent_manager/main.
from backend.apps.tools_lib.mcp_config import _sanitize_server_name, derive_mcp_config
# sanitize_server_name + derive_mcp_config re-exported for agent_manager/main.
from backend.apps.tools_lib.mcp_config import sanitize_server_name, derive_mcp_config
from backend.apps.tools_lib.mcp_discovery import (
_discover_mcp_tools_http,
_discover_mcp_tools_sse,
_discover_mcp_tools_stdio,
discover_mcp_tools_http,
discover_mcp_tools_sse,
discover_mcp_tools_stdio,
)
from backend.apps.tools_lib.tool_taxonomy import _classify_services
from backend.apps.tools_lib.tool_taxonomy import classify_services
# refresh_* re-exported for agent_manager.
from backend.apps.tools_lib.oauth_tokens import (
_proxied_provider_for,
_persist_cloud_tokens,
proxied_provider_for,
persist_cloud_tokens,
refresh_google_token,
refresh_airtable_token,
refresh_hubspot_token,
_m365_server_script,
_m365_cache_env,
m365_server_script,
m365_cache_env,
)
logger = logging.getLogger(__name__)
@@ -44,8 +44,8 @@ logger = logging.getLogger(__name__)
@asynccontextmanager
async def tools_lib_lifespan():
os.makedirs(DATA_DIR, exist_ok=True)
_ensure_default_permissions()
_reclassify_existing_tools()
p_ensure_default_permissions()
p_reclassify_existing_tools()
yield
@@ -58,13 +58,13 @@ tools_lib = SubApp("tools", tools_lib_lifespan)
# always_allow, so the poisoned-MCP-output -> destructive-command case is
# still caught. Must match agent_manager._DEFAULTS (empty -> always_allow) so
# the Settings UI and the agent agree on what "no policy set" means.
_DEFAULT_BUILTIN_POLICIES: dict[str, str] = {}
P_DEFAULT_BUILTIN_POLICIES: dict[str, str] = {}
# One-time marker: older installs seeded Bash="ask"; we lift them once.
_BASH_AUTOALLOW_MARKER = os.path.join(DATA_DIR, ".bash_autoallow_migrated")
P_BASH_AUTOALLOW_MARKER = os.path.join(DATA_DIR, ".bash_autoallow_migrated")
def _ensure_default_permissions() -> None:
def p_ensure_default_permissions() -> None:
"""Seed BUILTIN_PERMISSIONS_PATH so the user's Settings toggles persist
cleanly. Without this the file is missing on first run, load returns {},
every PUT-from-the-UI overwrites with the partial payload the click
@@ -74,18 +74,18 @@ def _ensure_default_permissions() -> None:
"""
existing = load_builtin_permissions()
desired = {
t.name: _DEFAULT_BUILTIN_POLICIES.get(t.name, "always_allow")
t.name: P_DEFAULT_BUILTIN_POLICIES.get(t.name, "always_allow")
for t in BUILTIN_TOOLS
}
merged = {**desired, **existing}
# One-time lift: installs seeded under the old default carry Bash="ask";
# raise them to always_allow once so shell commands stop prompting. The
# marker means a deliberate "ask" set afterward sticks (never re-flipped).
if not os.path.exists(_BASH_AUTOALLOW_MARKER):
if not os.path.exists(P_BASH_AUTOALLOW_MARKER):
if merged.get("Bash") == "ask":
merged["Bash"] = "always_allow"
try:
with open(_BASH_AUTOALLOW_MARKER, "w") as f:
with open(P_BASH_AUTOALLOW_MARKER, "w") as f:
f.write("1")
except OSError:
pass
@@ -93,7 +93,7 @@ def _ensure_default_permissions() -> None:
save_builtin_permissions(merged)
def _reclassify_existing_tools() -> None:
def p_reclassify_existing_tools() -> None:
"""One-time correction for tools discovered before service rules were integration-scoped: most
integrations got mislabeled under a bogus 'Google' group (generic keyword rules applied globally).
Recompute services/groups from each tool's stored tool names. Idempotent; rewrites only on change.
@@ -104,7 +104,7 @@ def _reclassify_existing_tools() -> None:
if not fname.endswith(".json"):
continue
try:
tool = _load(fname[:-5])
tool = p_load(fname[:-5])
except Exception:
continue
perms = tool.tool_permissions or {}
@@ -113,7 +113,7 @@ def _reclassify_existing_tools() -> None:
names = [k for k in perms if not k.startswith("_")]
if not names:
continue
services, service_groups, all_read, all_write = _classify_services(names, tool.name)
services, service_groups, all_read, all_write = classify_services(names, tool.name)
if perms.get("_services") == services and perms.get("_service_groups") == service_groups:
continue
perms["_services"] = services
@@ -121,7 +121,7 @@ def _reclassify_existing_tools() -> None:
perms["_categories"] = {"read": all_read, "write": all_write}
tool.tool_permissions = perms
try:
_save(tool)
save(tool)
except Exception:
pass
@@ -131,15 +131,15 @@ def _reclassify_existing_tools() -> None:
GOOGLE_USERINFO_URL = "https://www.googleapis.com/oauth2/v2/userinfo"
# Tool JSONs total ~1.5MB and _load_all runs on every dispatch, prompt build, and
# Tool JSONs total ~1.5MB and load_all_tools runs on every dispatch, prompt build, and
# MCPSearch keystroke; the cache skips re-parsing, revalidated by a per-file stat
# signature so any write (ours or external) invalidates instantly. Callers treat
# the returned ToolDefinitions as immutable; mutate via _load(tool_id) + _save.
_tools_cache: list[ToolDefinition] | None = None
_tools_cache_sig: tuple | None = None
# the returned ToolDefinitions as immutable; mutate via p_load(tool_id) + save.
p_tools_cache: list[ToolDefinition] | None = None
p_tools_cache_sig: tuple | None = None
def _tools_sig() -> tuple | None:
def p_tools_sig() -> tuple | None:
if not os.path.exists(DATA_DIR):
return ()
try:
@@ -153,11 +153,11 @@ def _tools_sig() -> tuple | None:
return None
def _load_all() -> list[ToolDefinition]:
global _tools_cache, _tools_cache_sig
sig = _tools_sig()
if sig is not None and _tools_cache is not None and sig == _tools_cache_sig:
return list(_tools_cache)
def load_all_tools() -> list[ToolDefinition]:
global p_tools_cache, p_tools_cache_sig
sig = p_tools_sig()
if sig is not None and p_tools_cache is not None and sig == p_tools_cache_sig:
return list(p_tools_cache)
result = []
if not os.path.exists(DATA_DIR):
return result
@@ -166,17 +166,17 @@ def _load_all() -> list[ToolDefinition]:
with open(os.path.join(DATA_DIR, fname)) as f:
result.append(ToolDefinition(**json.load(f)))
if sig is not None:
_tools_cache = list(result)
_tools_cache_sig = sig
p_tools_cache = list(result)
p_tools_cache_sig = sig
return result
def _save(tool: ToolDefinition):
def save(tool: ToolDefinition):
with open(os.path.join(DATA_DIR, f"{tool.id}.json"), "w") as f:
json.dump(tool.model_dump(), f, indent=2)
def _load(tool_id: str) -> ToolDefinition:
def p_load(tool_id: str) -> ToolDefinition:
path = os.path.join(DATA_DIR, f"{tool_id}.json")
if not os.path.exists(path):
raise HTTPException(status_code=404, detail="Tool not found")
@@ -196,7 +196,7 @@ def _load(tool_id: str) -> ToolDefinition:
"command": "python",
"args": ["-m", "backend.apps.discord_mcp_shim"],
}
_save(tool)
save(tool)
return tool
@@ -233,7 +233,7 @@ def resolve_policy_slot(tool_name: str, tools: list[ToolDefinition]) -> PolicySl
if m:
server_slug, action = m.group(1), m.group(2)
for t in tools:
if t.mcp_config and t.enabled and _sanitize_server_name(t.name) == server_slug:
if t.mcp_config and t.enabled and sanitize_server_name(t.name) == server_slug:
return PolicySlot("mcp", t.id, action)
return PolicySlot("mcp", None, action)
return PolicySlot("builtin", tool_name, None)
@@ -312,7 +312,7 @@ async def update_builtin_permissions(body: dict):
@tools_lib.router.get("/list")
async def list_tools():
tools = []
for t in _load_all():
for t in load_all_tools():
d = t.model_dump()
# Heal pre-fix tools whose persisted email is the "{name} account" placeholder so the pill stops reading like a name and falls back to plain "Connected".
placeholder = f"{t.name} account"
@@ -322,7 +322,7 @@ async def list_tools():
return {"tools": tools}
def _connected_html() -> HTMLResponse:
def p_connected_html() -> HTMLResponse:
"""v1.0.25-style auto-close page. Same markup so the UX is unchanged."""
return HTMLResponse("""
<html><body>
@@ -338,7 +338,7 @@ def _connected_html() -> HTMLResponse:
@tools_lib.router.get("/{tool_id}")
async def get_tool(tool_id: str):
return _load(tool_id).model_dump()
return p_load(tool_id).model_dump()
@tools_lib.router.post("/create")
@@ -352,16 +352,16 @@ async def create_tool(body: ToolCreate):
auth_type=body.auth_type,
auth_status=body.auth_status,
)
_save(tool)
save(tool)
return {"ok": True, "tool": tool.model_dump()}
@tools_lib.router.put("/{tool_id}")
async def update_tool(tool_id: str, body: ToolUpdate):
tool = _load(tool_id)
tool = p_load(tool_id)
for k, v in body.model_dump(exclude_none=True).items():
setattr(tool, k, v)
_save(tool)
save(tool)
return {"ok": True, "tool": tool.model_dump()}
@@ -375,7 +375,7 @@ async def delete_tool(tool_id: str):
@tools_lib.router.post("/{tool_id}/discover")
async def discover_tools(tool_id: str):
tool = _load(tool_id)
tool = p_load(tool_id)
if tool.auth_type == "oauth2" and tool.auth_status == "connected":
if tool.oauth_tokens.get("refresh_token"):
@@ -404,7 +404,7 @@ async def discover_tools(tool_id: str):
command = config.get("command", "")
if not command:
raise HTTPException(status_code=400, detail="stdio transport requires a 'command' in MCP config")
raw_tools = await _discover_mcp_tools_stdio(
raw_tools = await discover_mcp_tools_stdio(
command=command,
args=config.get("args"),
env=config.get("env"),
@@ -414,13 +414,13 @@ async def discover_tools(tool_id: str):
if not url:
raise HTTPException(status_code=400, detail="HTTP/SSE transport requires a 'url' in MCP config")
if transport == "sse":
raw_tools = await _discover_mcp_tools_sse(url, config.get("headers"))
raw_tools = await discover_mcp_tools_sse(url, config.get("headers"))
else:
try:
raw_tools = await _discover_mcp_tools_http(url, config.get("headers"))
raw_tools = await discover_mcp_tools_http(url, config.get("headers"))
except HTTPException:
logger.info(f"Streamable HTTP failed for {tool.name}, retrying with SSE transport")
raw_tools = await _discover_mcp_tools_sse(url, config.get("headers"))
raw_tools = await discover_mcp_tools_sse(url, config.get("headers"))
else:
raise HTTPException(status_code=400, detail=f"Unsupported MCP transport type: '{transport}'. Use 'stdio', 'http', or 'sse'.")
except HTTPException:
@@ -433,7 +433,7 @@ async def discover_tools(tool_id: str):
raise HTTPException(status_code=502, detail=f"Discovery failed: {msg}")
tool_names = [t["name"] for t in raw_tools]
services, service_groups, all_read, all_write = _classify_services(tool_names, tool.name)
services, service_groups, all_read, all_write = classify_services(tool_names, tool.name)
# Read-only actions auto-allow by default (no prompt for safe, scoped reads);
# writes still default to "ask". Any choice the user already made is kept.
permissions: dict[str, Any] = {
@@ -447,7 +447,7 @@ async def discover_tools(tool_id: str):
permissions["_tool_schemas"] = {t["name"]: t.get("inputSchema") for t in raw_tools if t.get("inputSchema")}
tool.tool_permissions = permissions
_save(tool)
save(tool)
return {"ok": True, "tool": tool.model_dump()}
@@ -456,7 +456,7 @@ async def discover_tools(tool_id: str):
# Microsoft 365 device-code login (runs in the backend, not the MCP server)
# ---------------------------------------------------------------------------
_m365_login_processes: dict[str, dict] = {} # tool_id -> {proc, device_code, status, email}
p_m365_login_processes: dict[str, dict] = {} # tool_id -> {proc, device_code, status, email}
@tools_lib.router.post("/{tool_id}/m365/device-login")
@@ -468,8 +468,8 @@ async def m365_device_login(tool_id: str):
"""
import subprocess
tool = _load(tool_id)
script = _m365_server_script()
tool = p_load(tool_id)
script = m365_server_script()
if not os.path.isfile(script):
raise HTTPException(status_code=500, detail="M365 MCP server not installed")
@@ -483,12 +483,12 @@ async def m365_device_login(tool_id: str):
if not cmd:
raise HTTPException(status_code=500, detail="No node/electron found")
env = {**os.environ, **_m365_cache_env()}
env = {**os.environ, **m365_cache_env()}
if cmd == electron:
env["ELECTRON_RUN_AS_NODE"] = "1"
# Kill any existing login process for this tool
existing = _m365_login_processes.pop(tool_id, None)
existing = p_m365_login_processes.pop(tool_id, None)
if existing and existing.get("proc"):
try:
existing["proc"].kill()
@@ -534,11 +534,11 @@ async def m365_device_login(tool_id: str):
pass
# Update tool status
try:
t = _load(tool_id)
t = p_load(tool_id)
t.auth_status = "connected"
if login_state.get("email"):
t.connected_account_email = login_state["email"]
_save(t)
save(t)
except Exception:
pass
else:
@@ -547,7 +547,7 @@ async def m365_device_login(tool_id: str):
thread = threading.Thread(target=_read_output, daemon=True)
thread.start()
_m365_login_processes[tool_id] = login_state
p_m365_login_processes[tool_id] = login_state
# Wait briefly for device code to appear
for _ in range(30):
@@ -568,13 +568,13 @@ async def m365_device_login(tool_id: str):
@tools_lib.router.get("/{tool_id}/m365/device-login/status")
async def m365_device_login_status(tool_id: str):
"""Poll the status of a pending M365 device-code login."""
state = _m365_login_processes.get(tool_id)
state = p_m365_login_processes.get(tool_id)
if not state:
# Check if already connected via cached token
cache_env = _m365_cache_env()
cache_env = m365_cache_env()
cache_path = cache_env["MS365_MCP_TOKEN_CACHE_PATH"]
if os.path.isfile(cache_path):
tool = _load(tool_id)
tool = p_load(tool_id)
if tool.auth_status == "connected":
return {"status": "connected", "email": tool.connected_account_email}
return {"status": "no_login_in_progress"}
@@ -583,10 +583,10 @@ async def m365_device_login_status(tool_id: str):
result: dict = {"status": status}
if status == "connected":
result["email"] = state.get("email")
_m365_login_processes.pop(tool_id, None)
p_m365_login_processes.pop(tool_id, None)
elif status == "error":
result["message"] = "Login failed"
_m365_login_processes.pop(tool_id, None)
p_m365_login_processes.pop(tool_id, None)
return result
@@ -594,21 +594,21 @@ async def m365_device_login_status(tool_id: str):
@tools_lib.router.post("/{tool_id}/m365/disconnect")
async def m365_disconnect(tool_id: str):
"""Disconnect M365 by clearing the cached token."""
tool = _load(tool_id)
cache_env = _m365_cache_env()
tool = p_load(tool_id)
cache_env = m365_cache_env()
for path in cache_env.values():
if os.path.isfile(path):
os.remove(path)
tool.auth_status = "configured"
tool.connected_account_email = None
_save(tool)
save(tool)
return {"ok": True, "tool": tool.model_dump()}
@tools_lib.router.post("/{tool_id}/oauth/disconnect")
async def oauth_disconnect(tool_id: str):
"""Clear OAuth tokens and reset auth status so the user can reconnect with a different account."""
tool = _load(tool_id)
tool = p_load(tool_id)
access_token = tool.oauth_tokens.get("access_token")
if access_token and tool.name.lower() != "notion":
@@ -626,7 +626,7 @@ async def oauth_disconnect(tool_id: str):
tool.oauth_tokens = {}
tool.auth_status = "configured"
tool.connected_account_email = None
_save(tool)
save(tool)
return {"ok": True, "tool": tool.model_dump()}
@@ -634,8 +634,8 @@ async def oauth_disconnect(tool_id: str):
async def oauth_start(tool_id: str):
"""Return the OAuth start URL for this tool. All built-in providers
proxy through Fly so client_secret values stay server-side."""
tool = _load(tool_id)
proxied = _proxied_provider_for(tool)
tool = p_load(tool_id)
proxied = proxied_provider_for(tool)
if not proxied:
raise HTTPException(
status_code=400,
@@ -704,7 +704,7 @@ async def oauth_cloud_claim(
data = resp.json()
tokens = data.get("tokens", {}) or {}
tool = _load(tool_id)
tool = p_load(tool_id)
# Google's token endpoint doesn't include the user's email; fetch it
# from userinfo so the UI can show "you connected you@gmail.com"
# rather than the generic "Google account" placeholder.
@@ -719,9 +719,9 @@ async def oauth_cloud_claim(
tokens["email"] = info_resp.json().get("email") or ""
except Exception as e:
logger.warning("Google userinfo lookup post-claim failed: %s", e)
_persist_cloud_tokens(tool, tokens)
_save(tool)
return _connected_html()
persist_cloud_tokens(tool, tokens)
save(tool)
return p_connected_html()
@tools_lib.router.post("/google-oauth-token")
+3 -3
View File
@@ -554,7 +554,7 @@ async def mcp_meta(action: str, request: Request):
valid options instead of activating (anti-hallucination).
"""
from backend.apps.agents.agent_manager import agent_manager
from backend.apps.tools_lib.tools_lib import _load_all as load_all_tools, _sanitize_server_name
from backend.apps.tools_lib.tools_lib import load_all_tools as load_all_tools, sanitize_server_name
body = await request.json()
parent_session_id = body.get("parent_session_id", "")
@@ -590,7 +590,7 @@ async def mcp_meta(action: str, request: Request):
for t in load_all_tools():
if not (t.mcp_config and t.enabled and t.auth_status in ("configured", "connected")):
continue
sanitized = _sanitize_server_name(t.name)
sanitized = sanitize_server_name(t.name)
# Pull tool sub-action names from tool_permissions._tool_descriptions
# so MCPSearch can match against capability names (e.g. "send_email").
action_names: list[str] = []
@@ -715,7 +715,7 @@ async def mcp_meta(action: str, request: Request):
tool_hint = ""
try:
for t in load_all_tools():
if _sanitize_server_name(t.name) != server_name:
if sanitize_server_name(t.name) != server_name:
continue
descs = (t.tool_permissions or {}).get("_tool_descriptions", {}) or {}
if not descs:
+17 -17
View File
@@ -25,8 +25,8 @@ def tools_tmp(tmp_path, monkeypatch):
d = tmp_path / "tools"
d.mkdir()
monkeypatch.setattr(tools_lib, "DATA_DIR", str(d))
monkeypatch.setattr(tools_lib, "_tools_cache", None)
monkeypatch.setattr(tools_lib, "_tools_cache_sig", None)
monkeypatch.setattr(tools_lib, "p_tools_cache", None)
monkeypatch.setattr(tools_lib, "p_tools_cache_sig", None)
return d
@@ -71,40 +71,40 @@ def test_settings_file_deleted_falls_back_to_defaults(settings_tmp):
def test_tools_write_then_list_is_fresh(tools_tmp):
assert tools_lib._load_all() == []
assert tools_lib.load_all_tools() == []
t = ToolDefinition(name="Alpha", description="a")
tools_lib._save(t)
tools_lib.save(t)
_bump_mtime(tools_tmp / f"{t.id}.json")
names = [x.name for x in tools_lib._load_all()]
names = [x.name for x in tools_lib.load_all_tools()]
assert names == ["Alpha"]
t2 = ToolDefinition(name="Beta", description="b")
tools_lib._save(t2)
assert sorted(x.name for x in tools_lib._load_all()) == ["Alpha", "Beta"]
tools_lib.save(t2)
assert sorted(x.name for x in tools_lib.load_all_tools()) == ["Alpha", "Beta"]
def test_tools_delete_detected(tools_tmp):
t = ToolDefinition(name="Gone", description="g")
tools_lib._save(t)
assert [x.name for x in tools_lib._load_all()] == ["Gone"]
tools_lib.save(t)
assert [x.name for x in tools_lib.load_all_tools()] == ["Gone"]
os.remove(tools_tmp / f"{t.id}.json")
assert tools_lib._load_all() == []
assert tools_lib.load_all_tools() == []
def test_tools_in_place_rewrite_detected(tools_tmp):
t = ToolDefinition(name="Old", description="x")
tools_lib._save(t)
assert [x.name for x in tools_lib._load_all()] == ["Old"]
tools_lib.save(t)
assert [x.name for x in tools_lib.load_all_tools()] == ["Old"]
t.name = "New"
tools_lib._save(t)
tools_lib.save(t)
_bump_mtime(tools_tmp / f"{t.id}.json")
assert [x.name for x in tools_lib._load_all()] == ["New"]
assert [x.name for x in tools_lib.load_all_tools()] == ["New"]
def test_tools_cached_hit_skips_reparse(tools_tmp, monkeypatch):
tools_lib._save(ToolDefinition(name="Once", description="o"))
tools_lib._load_all()
tools_lib.save(ToolDefinition(name="Once", description="o"))
tools_lib.load_all_tools()
def boom(*a, **k):
raise AssertionError("disk re-parse on unchanged dir")
monkeypatch.setattr(json, "load", boom)
assert [x.name for x in tools_lib._load_all()] == ["Once"]
assert [x.name for x in tools_lib.load_all_tools()] == ["Once"]
+11 -11
View File
@@ -11,7 +11,7 @@ tool-name shape, including the round-trip that the old code failed.
"""
from backend.apps.tools_lib.tools_lib import resolve_policy_slot, PolicySlot
from backend.apps.tools_lib.mcp_config import _sanitize_server_name
from backend.apps.tools_lib.mcp_config import sanitize_server_name
from backend.apps.tools_lib.models import ToolDefinition
@@ -34,7 +34,7 @@ def test_slot_for_our_browser_and_invoke_agents_uses_inner_name():
def test_slot_for_community_mcp_points_at_the_owning_tool():
tool = _mcp_tool("My Notion Server")
slug = _sanitize_server_name(tool.name)
slug = sanitize_server_name(tool.name)
assert resolve_policy_slot(f"mcp__{slug}__notion-fetch", [tool]) == \
PolicySlot("mcp", tool.id, "notion-fetch")
@@ -73,7 +73,7 @@ def test_always_approve_round_trips_for_every_tool_shape():
"""The invariant the old code violated: after WRITE(always_allow), the very
next READ returns always_allow, for builtin, our agents, and community MCP."""
notion = _mcp_tool("Notion")
slug = _sanitize_server_name("Notion")
slug = sanitize_server_name("Notion")
tools = [notion]
builtin_perms: dict[str, str] = {}
@@ -94,7 +94,7 @@ def test_always_approve_round_trips_for_every_tool_shape():
def test_two_actions_on_the_same_mcp_server_are_independent():
"""Approving one action must not silently approve a sibling action."""
tool = _mcp_tool("Notion")
slug = _sanitize_server_name("Notion")
slug = sanitize_server_name("Notion")
tools = [tool]
bp: dict[str, str] = {}
_write(f"mcp__{slug}__notion-fetch", "always_allow", bp, tools)
@@ -121,21 +121,21 @@ def test_builtin_policy_survives_a_real_file_reload(tmp_path, monkeypatch):
def test_mcp_policy_survives_a_real_tool_file_reload(tmp_path, monkeypatch):
monkeypatch.setattr(tl, "DATA_DIR", str(tmp_path))
monkeypatch.setattr(tl, "_tools_cache", None)
monkeypatch.setattr(tl, "_tools_cache_sig", None)
tl._save(_mcp_tool("Notion"))
slug = _sanitize_server_name("Notion")
monkeypatch.setattr(tl, "p_tools_cache", None)
monkeypatch.setattr(tl, "p_tools_cache_sig", None)
tl.save(_mcp_tool("Notion"))
slug = sanitize_server_name("Notion")
name = f"mcp__{slug}__notion-fetch"
# WRITE via the resolver against the freshly loaded tool, then persist.
tools = tl._load_all()
tools = tl.load_all_tools()
slot = tl.resolve_policy_slot(name, tools)
target = next(t for t in tools if t.id == slot.key)
target.tool_permissions[slot.action] = "always_allow"
tl._save(target)
tl.save(target)
# RELOAD from disk and read via the resolver: the policy is there.
tools2 = tl._load_all()
tools2 = tl.load_all_tools()
rslot = tl.resolve_policy_slot(name, tools2)
got = next(t for t in tools2 if t.id == rslot.key)
assert got.tool_permissions.get(rslot.action) == "always_allow"
+16 -16
View File
@@ -641,7 +641,7 @@ async def test_mcp_gate_only_forwards_activated_servers():
auth_status="configured", auth_type="apikey") for n in names]
# allowed_tools == get_all_tool_names() bypasses the (separate) permission
# gate so we isolate the ACTIVATION gate. _sanitize_server_name -> identity.
# gate so we isolate the ACTIVATION gate. sanitize_server_name -> identity.
with patch("backend.apps.agents.manager.RunSupportMixin.load_all_tools", side_effect=installed), \
patch("backend.apps.agents.manager.RunSupportMixin.get_all_tool_names", return_value=["__ALL__"]), \
patch("backend.apps.agents.manager.RunSupportMixin.sanitize_server_name", side_effect=lambda n: n), \
@@ -810,9 +810,9 @@ def test_mcp_brand_covers_curated_servers():
"google-workspace", "microsoft-365", "slack", "discord",
"notion", "airtable", "hubspot", "reddit", "youtube",
}
from backend.apps.tools_lib.tools_lib import _sanitize_server_name
from backend.apps.tools_lib.tools_lib import sanitize_server_name
for slug in curated:
assert _sanitize_server_name(slug) == slug, (
assert sanitize_server_name(slug) == slug, (
f"curated slug {slug!r} is not in sanitized form"
)
@@ -828,30 +828,30 @@ def test_curated_server_aliases_in_main():
def test_sanitize_server_name_idempotent():
"""_sanitize_server_name must be idempotent (sanitize twice = sanitize once)."""
from backend.apps.tools_lib.tools_lib import _sanitize_server_name
"""sanitize_server_name must be idempotent (sanitize twice = sanitize once)."""
from backend.apps.tools_lib.tools_lib import sanitize_server_name
test_inputs = [
"Google Workspace", "Microsoft 365", "Slack", "Discord",
"Notion", "Airtable", "HubSpot", "Reddit", "YouTube",
"GitHub", "GitLab", "Jira",
]
for raw in test_inputs:
once = _sanitize_server_name(raw)
twice = _sanitize_server_name(once)
once = sanitize_server_name(raw)
twice = sanitize_server_name(once)
assert once == twice, f"{raw}: sanitize not idempotent ({once} != {twice})"
def test_sanitize_server_name_lowercase():
from backend.apps.tools_lib.tools_lib import _sanitize_server_name
assert _sanitize_server_name("Gmail") == "gmail"
assert _sanitize_server_name("UPPERCASE") == "uppercase"
from backend.apps.tools_lib.tools_lib import sanitize_server_name
assert sanitize_server_name("Gmail") == "gmail"
assert sanitize_server_name("UPPERCASE") == "uppercase"
def test_sanitize_server_name_strips_special_chars():
from backend.apps.tools_lib.tools_lib import _sanitize_server_name
assert _sanitize_server_name("Foo Bar!") == "foo-bar"
assert _sanitize_server_name("@x/y") == "x-y"
assert _sanitize_server_name("a__b") == "a-b"
from backend.apps.tools_lib.tools_lib import sanitize_server_name
assert sanitize_server_name("Foo Bar!") == "foo-bar"
assert sanitize_server_name("@x/y") == "x-y"
assert sanitize_server_name("a__b") == "a-b"
# ===========================================================================
@@ -864,10 +864,10 @@ def test_mcp_activate_handler_unknown_server():
# We test the response shape independently of the FastAPI plumbing.
# The handler is a closure inside main.py:mcp_meta_handler, so we
# instead exercise the contract: invalid name surfaces alternatives.
from backend.apps.tools_lib.tools_lib import _sanitize_server_name
from backend.apps.tools_lib.tools_lib import sanitize_server_name
valid = {"gmail", "slack", "google-workspace"}
requested = "Gmail" # raw, needs sanitize
sanitized = _sanitize_server_name(requested)
sanitized = sanitize_server_name(requested)
if sanitized in valid:
status = "would_activate"
else: