From 54ab6863ab0e07b040cd5f5ada2ab99beb2017ac Mon Sep 17 00:00:00 2001 From: ciregenz Date: Thu, 27 Aug 2026 14:00:20 -0700 Subject: [PATCH] [eric] tls: the Node children get the OS roots, so a proxy or endpoint-tool root no longer kills every model call (ENG-408) --- .../agents/manager/configure_provider_env.py | 11 ++ backend/apps/nine_router/process.py | 24 ++- backend/config/node_trust.py | 155 ++++++++++++++++++ backend/tests/test_node_trust.py | 133 +++++++++++++++ 4 files changed, 320 insertions(+), 3 deletions(-) create mode 100644 backend/config/node_trust.py create mode 100644 backend/tests/test_node_trust.py diff --git a/backend/apps/agents/manager/configure_provider_env.py b/backend/apps/agents/manager/configure_provider_env.py index bc4ce34c..d87b2e5d 100644 --- a/backend/apps/agents/manager/configure_provider_env.py +++ b/backend/apps/agents/manager/configure_provider_env.py @@ -251,6 +251,17 @@ async def configure_provider_env( p_env = options_kwargs.get("env") if isinstance(p_env, dict): p_env.setdefault("ENABLE_TOOL_SEARCH", "auto") + # The CLI is Node and ignores the OS trust store, so on a machine whose endpoint tool or + # corporate proxy installed a root, sign-in succeeds (ENG-407 armed truststore for THIS + # process) and the first model call still dies on TLS. Additive and fail-open: empty dict + # when nothing can be exported, which is every build before this (ENG-408). + try: + from backend.config.node_trust import node_ca_env + from backend.config.paths import DATA_ROOT + for k, v in node_ca_env(os.path.join(DATA_ROOT, "node-ca-roots.pem")).items(): + p_env.setdefault(k, v) + except Exception as e: + logger.debug("node trust: skipped for the CLI (%s)", e) # Fault-injection seam: lets a QA harness front the provider with a local proxy (mid-run 401 drills, ENG-302 family). Absent in prod, so every branch's real base URL stands. p_base_override = os.environ.get("OPENSWARM_ANTHROPIC_BASE_OVERRIDE") diff --git a/backend/apps/nine_router/process.py b/backend/apps/nine_router/process.py index 5f30b15c..58b24e9e 100644 --- a/backend/apps/nine_router/process.py +++ b/backend/apps/nine_router/process.py @@ -23,7 +23,7 @@ import subprocess import sys import tempfile import time -from typing import Any, List, Optional +from typing import Any, Dict, List, Optional import httpx @@ -71,6 +71,22 @@ P_REQUEST_LOG_MAX_BYTES = 5 * 1024 * 1024 P_NODE_HEAP_MB = 4096 +def p_node_ca_env() -> Dict[str, str]: + """OS roots for the router, because Node ignores the OS trust store (ENG-408). + + Additive: NODE_EXTRA_CA_CERTS adds to Node's bundled roots rather than replacing them, so a + machine that works today cannot be broken by it. Empty dict on any failure, which is exactly the + behaviour of every build before this. + """ + try: + from backend.config.node_trust import node_ca_env + from backend.config.paths import DATA_ROOT + return node_ca_env(os.path.join(DATA_ROOT, "node-ca-roots.pem")) + except Exception as e: + logger.debug("node trust: skipped for the router (%s)", e) + return {} + + def p_rotate_request_log() -> None: """Rotate ~/.9router/request-details.json to a single .0 backup when it grows past the cap, BEFORE 9Router is spawned (never racing a live writer). 9Router recreates a fresh file, exactly @@ -600,7 +616,8 @@ async def p_ensure_running_impl(): logger.info("Starting 9Router (production) on port %d...", NINE_ROUTER_PORT) cmd = [node, f"--max-old-space-size={P_NODE_HEAP_MB}"] + (["--require", p_patch] if p_patch else []) + [standalone_server] cwd = os.path.dirname(standalone_server) - env = {**os.environ, "PORT": str(NINE_ROUTER_PORT), "NODE_ENV": "production"} + env = {**os.environ, "PORT": str(NINE_ROUTER_PORT), "NODE_ENV": "production", + **p_node_ca_env()} if node == os.environ.get("OPENSWARM_ELECTRON_PATH"): env["ELECTRON_RUN_AS_NODE"] = "1" else: @@ -618,7 +635,8 @@ async def p_ensure_running_impl(): ) cmd = [node, f"--max-old-space-size={P_NODE_HEAP_MB}"] + (["--require", p_patch] if p_patch else []) + [cached_server] cwd = os.path.dirname(cached_server) - env = {**os.environ, "PORT": str(NINE_ROUTER_PORT), "NODE_ENV": "production"} + env = {**os.environ, "PORT": str(NINE_ROUTER_PORT), "NODE_ENV": "production", + **p_node_ca_env()} # Capture stdout+stderr so a failed start can tell us WHY (the old DEVNULL default made every "router never came up" a silent mystery, which is the whole reason #90 was un-diagnosable). Packaged prod (NODE_ENV=production standalone) is quiet, so one fixed temp file, truncated each start attempt, won't grow; dev keeps its chatty-Next.js DEVNULL unless debug is set. p_cap_path = os.path.join(tempfile.gettempdir(), "openswarm-9router-start.log") diff --git a/backend/config/node_trust.py b/backend/config/node_trust.py new file mode 100644 index 00000000..1cc02b45 --- /dev/null +++ b/backend/config/node_trust.py @@ -0,0 +1,155 @@ +"""Give the Node children the OS roots, because Node ignores the OS trust store. + +ENG-407 armed `truststore` so THIS process trusts what the user's browser trusts. That was only the +first wall. 9router and the claude CLI are Node, and Node verifies against its own compiled-in root +list on both Windows and macOS, so on a machine whose endpoint tool or corporate proxy installed a +root, sign-in now succeeds and the first model call still dies with the ENG-218 certificate card +(ENG-408). + +`NODE_USE_SYSTEM_CA=1` would make this file unnecessary, and it is the right long-term answer, but it +landed in Node 22.15 and the bundled runtime is v20.18.1 (`scripts/build-app.sh`). So we export the +OS roots to a PEM and point `NODE_EXTRA_CA_CERTS` at it. + +Two properties this must have, and both are the reason it is safe to ship: + +- **Additive, never substitutive.** `NODE_EXTRA_CA_CERTS` ADDS to Node's bundled roots rather than + replacing them, so a machine that works today cannot be broken by this. Trust becomes "what Node + shipped, plus what this machine's own OS store already trusts", and nothing wider. +- **Fails to today's behaviour.** Any failure (no export, empty export, unwritable path) leaves the + variable unset, which is exactly the current build. A guard that cannot arm says so and gets out of + the way; it never guesses. +""" + +import logging +import os +import platform +import subprocess +from typing import List, Optional + +from typeguard import typechecked + +logger = logging.getLogger(__name__) + +# Linux Node already reads OpenSSL's default paths, which is the system store there. +NEEDS_EXPORT = frozenset({"Darwin", "Windows"}) + +# macOS keeps the shipped roots and the admin-installed ones in different keychains, and an endpoint +# tool can land in either. The user's login keychain is included because that is where a per-user +# proxy root goes. +P_MAC_KEYCHAINS = ( + "/System/Library/Keychains/SystemRootCertificates.keychain", + "/Library/Keychains/System.keychain", +) + +PEM_HEADER = "-----BEGIN CERTIFICATE-----" +# A store this size is a sign we read something that is not a root list; refuse rather than hand Node +# a multi-megabyte file to parse on every spawn. +MAX_CERTS = 1000 + + +@typechecked +def p_mac_roots() -> List[str]: + """Every PEM block in the system keychains plus the user's login keychain.""" + p_out: List[str] = [] + p_chains = list(P_MAC_KEYCHAINS) + p_login = os.path.expanduser("~/Library/Keychains/login.keychain-db") + if os.path.exists(p_login): + p_chains.append(p_login) + for chain in p_chains: + if not os.path.exists(chain): + continue + try: + r = subprocess.run( + ["security", "find-certificate", "-a", "-p", chain], + capture_output=True, text=True, timeout=20, + ) + except Exception as e: + logger.debug("node trust: could not read %s (%s)", chain, e) + continue + if r.returncode == 0 and PEM_HEADER in r.stdout: + p_out.append(r.stdout) + return p_out + + +@typechecked +def p_windows_roots() -> List[str]: + """The Windows ROOT and CA stores, via stdlib; no shelling out and no extra dependency.""" + import ssl + p_out: List[str] = [] + # enum_certificates exists only on Windows, so it is fetched by name; a checker on any other + # platform is right that the attribute is absent, and this function never runs there. + p_enum = getattr(ssl, "enum_certificates", None) + if p_enum is None: + return p_out + for store in ("ROOT", "CA"): + try: + for cert, enc, p_trust in p_enum(store): + if enc == "x509_asn" and p_trust: + p_out.append(ssl.DER_cert_to_PEM_cert(cert)) + except Exception as e: + logger.debug("node trust: could not read the %s store (%s)", store, e) + return p_out + + +@typechecked +def export_os_roots(dest: str) -> Optional[str]: + """Write the OS roots to `dest` and return the path, or None when there is nothing to hand Node. + + None is the safe answer everywhere: the caller leaves NODE_EXTRA_CA_CERTS unset, and Node keeps + exactly the roots it ships with, which is what every build does today. + """ + system = platform.system() + if system not in NEEDS_EXPORT: + return None + p_blocks = p_mac_roots() if system == "Darwin" else p_windows_roots() + p_pem = "\n".join(b.strip() for b in p_blocks if b.strip()) + p_count = p_pem.count(PEM_HEADER) + if p_count == 0: + logger.warning( + "node trust: read no roots out of the %s store, so 9router and the CLI keep Node's " + "bundled roots only; a proxy or security-tool root will still be refused by them", system, + ) + return None + if p_count > MAX_CERTS: + logger.warning("node trust: %d certificates is not a root list; refusing to export", p_count) + return None + try: + os.makedirs(os.path.dirname(dest), exist_ok=True) + with open(dest, "w", encoding="utf-8") as f: + f.write(p_pem + "\n") + except Exception as e: + logger.warning( + "node trust: could not write %s (%s); 9router and the CLI keep Node's bundled roots only", + dest, e, + ) + return None + logger.info("node trust: exported %d OS roots for the Node children -> %s", p_count, dest) + return dest + + +# Exported once per process. Every spawn used to shell out to `security find-certificate` and rewrite +# ~250KB; measured three times in five seconds on one turn. The OS store does not change mid-session, +# and if a user installs a root while the app is open, a restart picks it up -- the same contract +# truststore already has for this process. +P_CACHED: Optional[dict] = None + + +@typechecked +def node_ca_env(dest: str) -> dict: + """`{NODE_EXTRA_CA_CERTS: }` when there is something to add, else `{}`. + + Returning a dict rather than mutating os.environ keeps this out of the parent process: the + backend's own TLS is truststore's job, and two mechanisms for one concern is how they drift. + """ + global P_CACHED + if P_CACHED is not None: + return dict(P_CACHED) + p_path = export_os_roots(dest) + P_CACHED = {"NODE_EXTRA_CA_CERTS": p_path} if p_path else {} + return dict(P_CACHED) + + +@typechecked +def reset_cache_for_test() -> None: + global P_CACHED + P_CACHED = None diff --git a/backend/tests/test_node_trust.py b/backend/tests/test_node_trust.py new file mode 100644 index 00000000..14a69492 --- /dev/null +++ b/backend/tests/test_node_trust.py @@ -0,0 +1,133 @@ +"""The Node children get the OS roots, because Node ignores the OS trust store. + +ENG-407 armed truststore so the PYTHON backend trusts what the browser trusts. 9router and the claude +CLI are Node and verify against Node's compiled-in list, so on a machine with an endpoint tool's or a +corporate proxy's root, sign-in now works and the first model call still dies (ENG-408). + +`NODE_USE_SYSTEM_CA=1` would make the whole file unnecessary; it landed in Node 22.15 and the bundled +runtime is v20.18.1, so we export the OS roots and point NODE_EXTRA_CA_CERTS at them. + +DRILLED 2026-08-27 against a real Node TLS server signed by a throwaway root: + stock Node -> REFUSED UNABLE_TO_VERIFY_LEAF_SIGNATURE (the ThinkPad bug) + NODE_EXTRA_CA_CERTS incl. root -> OK 200 (the cure) + NODE_EXTRA_CA_CERTS OS-only -> REFUSED (trust not widened) +""" + +import os +import platform + +import pytest + +from backend.config import node_trust + + +@pytest.fixture(autouse=True) +def p_no_cache(): + """The export is cached once per process; a cached answer would make every case below a no-op.""" + node_trust.reset_cache_for_test() + yield + node_trust.reset_cache_for_test() + +ROUTER = "backend/apps/nine_router/process.py" +CLI = "backend/apps/agents/manager/configure_provider_env.py" + + +def test_an_export_that_yields_nothing_sets_no_variable(tmp_path, monkeypatch): + """Fail to today's behaviour. An unset variable is Node's bundled roots, which is every build + before this; a broken export must never be louder than that.""" + monkeypatch.setattr(node_trust, "p_mac_roots", lambda: []) + monkeypatch.setattr(node_trust, "p_windows_roots", lambda: []) + monkeypatch.setattr(platform, "system", lambda: "Darwin") + assert node_trust.node_ca_env(str(tmp_path / "r.pem")) == {} + + +def test_an_unwritable_destination_sets_no_variable(monkeypatch): + monkeypatch.setattr(platform, "system", lambda: "Darwin") + monkeypatch.setattr(node_trust, "p_mac_roots", lambda: ["-----BEGIN CERTIFICATE-----\nx\n"]) + assert node_trust.node_ca_env("/proc/nope/cannot/write.pem") == {} + + +def test_linux_is_left_alone(monkeypatch, tmp_path): + """Node on Linux already reads OpenSSL's default paths, which IS the system store there.""" + monkeypatch.setattr(platform, "system", lambda: "Linux") + assert node_trust.export_os_roots(str(tmp_path / "r.pem")) is None + + +def test_an_absurd_store_is_refused_rather_than_handed_to_node(monkeypatch, tmp_path): + monkeypatch.setattr(platform, "system", lambda: "Darwin") + monkeypatch.setattr(node_trust, "p_mac_roots", + lambda: ["-----BEGIN CERTIFICATE-----\nx\n" * (node_trust.MAX_CERTS + 1)]) + assert node_trust.export_os_roots(str(tmp_path / "r.pem")) is None + + +def test_a_real_export_on_this_machine_produces_parseable_pem(tmp_path): + """Skipped off the two platforms that need it; on them it must produce real certificates.""" + if platform.system() not in node_trust.NEEDS_EXPORT: + return + p = node_trust.export_os_roots(str(tmp_path / "roots.pem")) + assert p and os.path.exists(p) + body = open(p).read() + assert body.count(node_trust.PEM_HEADER) > 10, "an OS root store is not this small" + assert body.count(node_trust.PEM_HEADER) == body.count("-----END CERTIFICATE-----") + + +def test_both_node_children_are_wired_not_just_one(): + """The recurring defect: a fix applied to one spawn site and not the other. The router and the + CLI are two separate Node processes and BOTH verify provider TLS.""" + for path in (ROUTER, CLI): + src = open(path).read() + assert "node_ca_env" in src, f"{path} spawns Node without the OS roots" + + +def test_it_is_additive_and_never_replaces_the_env(): + """setdefault, not assignment: a lane that deliberately set its own CA path keeps it, and this + can never blank a variable someone else needed.""" + src = open(CLI).read() + i = src.index("node_ca_env(") + assert "p_env.setdefault(k, v)" in src[i:i + 300] + + +def test_the_export_happens_once_per_process(monkeypatch, tmp_path): + """It used to shell out to `security` and rewrite ~250KB on EVERY spawn: measured three times in + five seconds on one turn.""" + calls = {"n": 0} + + def p_count(): + calls["n"] += 1 + return ["-----BEGIN CERTIFICATE-----\nx\n-----END CERTIFICATE-----\n"] + + monkeypatch.setattr(platform, "system", lambda: "Darwin") + monkeypatch.setattr(node_trust, "p_mac_roots", p_count) + dest = str(tmp_path / "r.pem") + a = node_trust.node_ca_env(dest) + b = node_trust.node_ca_env(dest) + c = node_trust.node_ca_env(dest) + assert calls["n"] == 1, f"exported {calls['n']} times" + assert a == b == c and a + + +def test_a_failed_export_is_cached_too_and_does_not_retry_every_spawn(monkeypatch, tmp_path): + """The failure path is the one that would otherwise shell out forever on a machine where the + store cannot be read at all.""" + calls = {"n": 0} + + def p_empty(): + calls["n"] += 1 + return [] + + monkeypatch.setattr(platform, "system", lambda: "Darwin") + monkeypatch.setattr(node_trust, "p_mac_roots", p_empty) + dest = str(tmp_path / "r.pem") + assert node_trust.node_ca_env(dest) == {} + assert node_trust.node_ca_env(dest) == {} + assert calls["n"] == 1 + + +def test_the_caller_cannot_mutate_the_cache(monkeypatch, tmp_path): + monkeypatch.setattr(platform, "system", lambda: "Darwin") + monkeypatch.setattr(node_trust, "p_mac_roots", + lambda: ["-----BEGIN CERTIFICATE-----\nx\n-----END CERTIFICATE-----\n"]) + dest = str(tmp_path / "r.pem") + first = node_trust.node_ca_env(dest) + first["NODE_EXTRA_CA_CERTS"] = "/tmp/evil.pem" + assert node_trust.node_ca_env(dest)["NODE_EXTRA_CA_CERTS"] != "/tmp/evil.pem"