[eric] tls: the Node children get the OS roots, so a proxy or endpoint-tool root no longer kills every model call (ENG-408)

backend/apps/agents/manager/configure_provider_env.py

@@ -251,6 +251,17 @@ async def configure_provider_env(
     p_env = options_kwargs.get("env")
     if isinstance(p_env, dict):
         p_env.setdefault("ENABLE_TOOL_SEARCH", "auto")
+        # The CLI is Node and ignores the OS trust store, so on a machine whose endpoint tool or
+        # corporate proxy installed a root, sign-in succeeds (ENG-407 armed truststore for THIS
+        # process) and the first model call still dies on TLS. Additive and fail-open: empty dict
+        # when nothing can be exported, which is every build before this (ENG-408).
+        try:
+            from backend.config.node_trust import node_ca_env
+            from backend.config.paths import DATA_ROOT
+            for k, v in node_ca_env(os.path.join(DATA_ROOT, "node-ca-roots.pem")).items():
+                p_env.setdefault(k, v)
+        except Exception as e:
+            logger.debug("node trust: skipped for the CLI (%s)", e)
 
     # Fault-injection seam: lets a QA harness front the provider with a local proxy (mid-run 401 drills, ENG-302 family). Absent in prod, so every branch's real base URL stands.
     p_base_override = os.environ.get("OPENSWARM_ANTHROPIC_BASE_OVERRIDE")

backend/apps/nine_router/process.py

@@ -23,7 +23,7 @@ import subprocess
 import sys
 import tempfile
 import time
-from typing import Any, List, Optional
+from typing import Any, Dict, List, Optional
 
 import httpx
 
@@ -71,6 +71,22 @@ P_REQUEST_LOG_MAX_BYTES = 5 * 1024 * 1024
 P_NODE_HEAP_MB = 4096
 
 
+def p_node_ca_env() -> Dict[str, str]:
+    """OS roots for the router, because Node ignores the OS trust store (ENG-408).
+
+    Additive: NODE_EXTRA_CA_CERTS adds to Node's bundled roots rather than replacing them, so a
+    machine that works today cannot be broken by it. Empty dict on any failure, which is exactly the
+    behaviour of every build before this.
+    """
+    try:
+        from backend.config.node_trust import node_ca_env
+        from backend.config.paths import DATA_ROOT
+        return node_ca_env(os.path.join(DATA_ROOT, "node-ca-roots.pem"))
+    except Exception as e:
+        logger.debug("node trust: skipped for the router (%s)", e)
+        return {}
+
+
 def p_rotate_request_log() -> None:
     """Rotate ~/.9router/request-details.json to a single .0 backup when it grows past the cap,
     BEFORE 9Router is spawned (never racing a live writer). 9Router recreates a fresh file, exactly
@@ -600,7 +616,8 @@ async def p_ensure_running_impl():
         logger.info("Starting 9Router (production) on port %d...", NINE_ROUTER_PORT)
         cmd = [node, f"--max-old-space-size={P_NODE_HEAP_MB}"] + (["--require", p_patch] if p_patch else []) + [standalone_server]
         cwd = os.path.dirname(standalone_server)
-        env = {**os.environ, "PORT": str(NINE_ROUTER_PORT), "NODE_ENV": "production"}
+        env = {**os.environ, "PORT": str(NINE_ROUTER_PORT), "NODE_ENV": "production",
+               **p_node_ca_env()}
         if node == os.environ.get("OPENSWARM_ELECTRON_PATH"):
             env["ELECTRON_RUN_AS_NODE"] = "1"
     else:
@@ -618,7 +635,8 @@ async def p_ensure_running_impl():
         )
         cmd = [node, f"--max-old-space-size={P_NODE_HEAP_MB}"] + (["--require", p_patch] if p_patch else []) + [cached_server]
         cwd = os.path.dirname(cached_server)
-        env = {**os.environ, "PORT": str(NINE_ROUTER_PORT), "NODE_ENV": "production"}
+        env = {**os.environ, "PORT": str(NINE_ROUTER_PORT), "NODE_ENV": "production",
+               **p_node_ca_env()}
 
     # Capture stdout+stderr so a failed start can tell us WHY (the old DEVNULL default made every "router never came up" a silent mystery, which is the whole reason #90 was un-diagnosable). Packaged prod (NODE_ENV=production standalone) is quiet, so one fixed temp file, truncated each start attempt, won't grow; dev keeps its chatty-Next.js DEVNULL unless debug is set.
     p_cap_path = os.path.join(tempfile.gettempdir(), "openswarm-9router-start.log")

backend/config/node_trust.py

@@ -0,0 +1,155 @@
+"""Give the Node children the OS roots, because Node ignores the OS trust store.
+
+ENG-407 armed `truststore` so THIS process trusts what the user's browser trusts. That was only the
+first wall. 9router and the claude CLI are Node, and Node verifies against its own compiled-in root
+list on both Windows and macOS, so on a machine whose endpoint tool or corporate proxy installed a
+root, sign-in now succeeds and the first model call still dies with the ENG-218 certificate card
+(ENG-408).
+
+`NODE_USE_SYSTEM_CA=1` would make this file unnecessary, and it is the right long-term answer, but it
+landed in Node 22.15 and the bundled runtime is v20.18.1 (`scripts/build-app.sh`). So we export the
+OS roots to a PEM and point `NODE_EXTRA_CA_CERTS` at it.
+
+Two properties this must have, and both are the reason it is safe to ship:
+
+- **Additive, never substitutive.** `NODE_EXTRA_CA_CERTS` ADDS to Node's bundled roots rather than
+  replacing them, so a machine that works today cannot be broken by this. Trust becomes "what Node
+  shipped, plus what this machine's own OS store already trusts", and nothing wider.
+- **Fails to today's behaviour.** Any failure (no export, empty export, unwritable path) leaves the
+  variable unset, which is exactly the current build. A guard that cannot arm says so and gets out of
+  the way; it never guesses.
+"""
+
+import logging
+import os
+import platform
+import subprocess
+from typing import List, Optional
+
+from typeguard import typechecked
+
+logger = logging.getLogger(__name__)
+
+# Linux Node already reads OpenSSL's default paths, which is the system store there.
+NEEDS_EXPORT = frozenset({"Darwin", "Windows"})
+
+# macOS keeps the shipped roots and the admin-installed ones in different keychains, and an endpoint
+# tool can land in either. The user's login keychain is included because that is where a per-user
+# proxy root goes.
+P_MAC_KEYCHAINS = (
+    "/System/Library/Keychains/SystemRootCertificates.keychain",
+    "/Library/Keychains/System.keychain",
+)
+
+PEM_HEADER = "-----BEGIN CERTIFICATE-----"
+# A store this size is a sign we read something that is not a root list; refuse rather than hand Node
+# a multi-megabyte file to parse on every spawn.
+MAX_CERTS = 1000
+
+
+@typechecked
+def p_mac_roots() -> List[str]:
+    """Every PEM block in the system keychains plus the user's login keychain."""
+    p_out: List[str] = []
+    p_chains = list(P_MAC_KEYCHAINS)
+    p_login = os.path.expanduser("~/Library/Keychains/login.keychain-db")
+    if os.path.exists(p_login):
+        p_chains.append(p_login)
+    for chain in p_chains:
+        if not os.path.exists(chain):
+            continue
+        try:
+            r = subprocess.run(
+                ["security", "find-certificate", "-a", "-p", chain],
+                capture_output=True, text=True, timeout=20,
+            )
+        except Exception as e:
+            logger.debug("node trust: could not read %s (%s)", chain, e)
+            continue
+        if r.returncode == 0 and PEM_HEADER in r.stdout:
+            p_out.append(r.stdout)
+    return p_out
+
+
+@typechecked
+def p_windows_roots() -> List[str]:
+    """The Windows ROOT and CA stores, via stdlib; no shelling out and no extra dependency."""
+    import ssl
+    p_out: List[str] = []
+    # enum_certificates exists only on Windows, so it is fetched by name; a checker on any other
+    # platform is right that the attribute is absent, and this function never runs there.
+    p_enum = getattr(ssl, "enum_certificates", None)
+    if p_enum is None:
+        return p_out
+    for store in ("ROOT", "CA"):
+        try:
+            for cert, enc, p_trust in p_enum(store):
+                if enc == "x509_asn" and p_trust:
+                    p_out.append(ssl.DER_cert_to_PEM_cert(cert))
+        except Exception as e:
+            logger.debug("node trust: could not read the %s store (%s)", store, e)
+    return p_out
+
+
+@typechecked
+def export_os_roots(dest: str) -> Optional[str]:
+    """Write the OS roots to `dest` and return the path, or None when there is nothing to hand Node.
+
+    None is the safe answer everywhere: the caller leaves NODE_EXTRA_CA_CERTS unset, and Node keeps
+    exactly the roots it ships with, which is what every build does today.
+    """
+    system = platform.system()
+    if system not in NEEDS_EXPORT:
+        return None
+    p_blocks = p_mac_roots() if system == "Darwin" else p_windows_roots()
+    p_pem = "\n".join(b.strip() for b in p_blocks if b.strip())
+    p_count = p_pem.count(PEM_HEADER)
+    if p_count == 0:
+        logger.warning(
+            "node trust: read no roots out of the %s store, so 9router and the CLI keep Node's "
+            "bundled roots only; a proxy or security-tool root will still be refused by them", system,
+        )
+        return None
+    if p_count > MAX_CERTS:
+        logger.warning("node trust: %d certificates is not a root list; refusing to export", p_count)
+        return None
+    try:
+        os.makedirs(os.path.dirname(dest), exist_ok=True)
+        with open(dest, "w", encoding="utf-8") as f:
+            f.write(p_pem + "\n")
+    except Exception as e:
+        logger.warning(
+            "node trust: could not write %s (%s); 9router and the CLI keep Node's bundled roots only",
+            dest, e,
+        )
+        return None
+    logger.info("node trust: exported %d OS roots for the Node children -> %s", p_count, dest)
+    return dest
+
+
+# Exported once per process. Every spawn used to shell out to `security find-certificate` and rewrite
+# ~250KB; measured three times in five seconds on one turn. The OS store does not change mid-session,
+# and if a user installs a root while the app is open, a restart picks it up -- the same contract
+# truststore already has for this process.
+P_CACHED: Optional[dict] = None
+
+
+@typechecked
+def node_ca_env(dest: str) -> dict:
+    """`{NODE_EXTRA_CA_CERTS: <path>}` when there is something to add, else `{}`.
+
+    Returning a dict rather than mutating os.environ keeps this out of the parent process: the
+    backend's own TLS is truststore's job, and two mechanisms for one concern is how they drift.
+    """
+    global P_CACHED
+    if P_CACHED is not None:
+        return dict(P_CACHED)
+    p_path = export_os_roots(dest)
+    P_CACHED = {"NODE_EXTRA_CA_CERTS": p_path} if p_path else {}
+    return dict(P_CACHED)
+
+
+@typechecked
+def reset_cache_for_test() -> None:
+    global P_CACHED
+    P_CACHED = None

backend/tests/test_node_trust.py

@@ -0,0 +1,133 @@
+"""The Node children get the OS roots, because Node ignores the OS trust store.
+
+ENG-407 armed truststore so the PYTHON backend trusts what the browser trusts. 9router and the claude
+CLI are Node and verify against Node's compiled-in list, so on a machine with an endpoint tool's or a
+corporate proxy's root, sign-in now works and the first model call still dies (ENG-408).
+
+`NODE_USE_SYSTEM_CA=1` would make the whole file unnecessary; it landed in Node 22.15 and the bundled
+runtime is v20.18.1, so we export the OS roots and point NODE_EXTRA_CA_CERTS at them.
+
+DRILLED 2026-08-27 against a real Node TLS server signed by a throwaway root:
+    stock Node                      -> REFUSED UNABLE_TO_VERIFY_LEAF_SIGNATURE   (the ThinkPad bug)
+    NODE_EXTRA_CA_CERTS incl. root  -> OK 200                                    (the cure)
+    NODE_EXTRA_CA_CERTS OS-only     -> REFUSED                                   (trust not widened)
+"""
+
+import os
+import platform
+
+import pytest
+
+from backend.config import node_trust
+
+
+@pytest.fixture(autouse=True)
+def p_no_cache():
+    """The export is cached once per process; a cached answer would make every case below a no-op."""
+    node_trust.reset_cache_for_test()
+    yield
+    node_trust.reset_cache_for_test()
+
+ROUTER = "backend/apps/nine_router/process.py"
+CLI = "backend/apps/agents/manager/configure_provider_env.py"
+
+
+def test_an_export_that_yields_nothing_sets_no_variable(tmp_path, monkeypatch):
+    """Fail to today's behaviour. An unset variable is Node's bundled roots, which is every build
+    before this; a broken export must never be louder than that."""
+    monkeypatch.setattr(node_trust, "p_mac_roots", lambda: [])
+    monkeypatch.setattr(node_trust, "p_windows_roots", lambda: [])
+    monkeypatch.setattr(platform, "system", lambda: "Darwin")
+    assert node_trust.node_ca_env(str(tmp_path / "r.pem")) == {}
+
+
+def test_an_unwritable_destination_sets_no_variable(monkeypatch):
+    monkeypatch.setattr(platform, "system", lambda: "Darwin")
+    monkeypatch.setattr(node_trust, "p_mac_roots", lambda: ["-----BEGIN CERTIFICATE-----\nx\n"])
+    assert node_trust.node_ca_env("/proc/nope/cannot/write.pem") == {}
+
+
+def test_linux_is_left_alone(monkeypatch, tmp_path):
+    """Node on Linux already reads OpenSSL's default paths, which IS the system store there."""
+    monkeypatch.setattr(platform, "system", lambda: "Linux")
+    assert node_trust.export_os_roots(str(tmp_path / "r.pem")) is None
+
+
+def test_an_absurd_store_is_refused_rather_than_handed_to_node(monkeypatch, tmp_path):
+    monkeypatch.setattr(platform, "system", lambda: "Darwin")
+    monkeypatch.setattr(node_trust, "p_mac_roots",
+                        lambda: ["-----BEGIN CERTIFICATE-----\nx\n" * (node_trust.MAX_CERTS + 1)])
+    assert node_trust.export_os_roots(str(tmp_path / "r.pem")) is None
+
+
+def test_a_real_export_on_this_machine_produces_parseable_pem(tmp_path):
+    """Skipped off the two platforms that need it; on them it must produce real certificates."""
+    if platform.system() not in node_trust.NEEDS_EXPORT:
+        return
+    p = node_trust.export_os_roots(str(tmp_path / "roots.pem"))
+    assert p and os.path.exists(p)
+    body = open(p).read()
+    assert body.count(node_trust.PEM_HEADER) > 10, "an OS root store is not this small"
+    assert body.count(node_trust.PEM_HEADER) == body.count("-----END CERTIFICATE-----")
+
+
+def test_both_node_children_are_wired_not_just_one():
+    """The recurring defect: a fix applied to one spawn site and not the other. The router and the
+    CLI are two separate Node processes and BOTH verify provider TLS."""
+    for path in (ROUTER, CLI):
+        src = open(path).read()
+        assert "node_ca_env" in src, f"{path} spawns Node without the OS roots"
+
+
+def test_it_is_additive_and_never_replaces_the_env():
+    """setdefault, not assignment: a lane that deliberately set its own CA path keeps it, and this
+    can never blank a variable someone else needed."""
+    src = open(CLI).read()
+    i = src.index("node_ca_env(")
+    assert "p_env.setdefault(k, v)" in src[i:i + 300]
+
+
+def test_the_export_happens_once_per_process(monkeypatch, tmp_path):
+    """It used to shell out to `security` and rewrite ~250KB on EVERY spawn: measured three times in
+    five seconds on one turn."""
+    calls = {"n": 0}
+
+    def p_count():
+        calls["n"] += 1
+        return ["-----BEGIN CERTIFICATE-----\nx\n-----END CERTIFICATE-----\n"]
+
+    monkeypatch.setattr(platform, "system", lambda: "Darwin")
+    monkeypatch.setattr(node_trust, "p_mac_roots", p_count)
+    dest = str(tmp_path / "r.pem")
+    a = node_trust.node_ca_env(dest)
+    b = node_trust.node_ca_env(dest)
+    c = node_trust.node_ca_env(dest)
+    assert calls["n"] == 1, f"exported {calls['n']} times"
+    assert a == b == c and a
+
+
+def test_a_failed_export_is_cached_too_and_does_not_retry_every_spawn(monkeypatch, tmp_path):
+    """The failure path is the one that would otherwise shell out forever on a machine where the
+    store cannot be read at all."""
+    calls = {"n": 0}
+
+    def p_empty():
+        calls["n"] += 1
+        return []
+
+    monkeypatch.setattr(platform, "system", lambda: "Darwin")
+    monkeypatch.setattr(node_trust, "p_mac_roots", p_empty)
+    dest = str(tmp_path / "r.pem")
+    assert node_trust.node_ca_env(dest) == {}
+    assert node_trust.node_ca_env(dest) == {}
+    assert calls["n"] == 1
+
+
+def test_the_caller_cannot_mutate_the_cache(monkeypatch, tmp_path):
+    monkeypatch.setattr(platform, "system", lambda: "Darwin")
+    monkeypatch.setattr(node_trust, "p_mac_roots",
+                        lambda: ["-----BEGIN CERTIFICATE-----\nx\n-----END CERTIFICATE-----\n"])
+    dest = str(tmp_path / "r.pem")
+    first = node_trust.node_ca_env(dest)
+    first["NODE_EXTRA_CA_CERTS"] = "/tmp/evil.pem"
+    assert node_trust.node_ca_env(dest)["NODE_EXTRA_CA_CERTS"] != "/tmp/evil.pem"