[eric] auth: backend https trusts the OS root store, ending the Windows sign-in 502 (ENG-407)

.github/workflows/os-trust-drill.yml

@@ -0,0 +1,60 @@
+name: os-trust-drill
+
+# ENG-407: on Windows ThinkPads sign-in 502'd with CERTIFICATE_VERIFY_FAILED because the Python backend trusted
+# certifi's file while the browser trusted the machine's own root store. The fix routes the backend through the
+# OS store. That is only provable on a real OS with a real root installed, so this job mints a throwaway root,
+# installs it the way an endpoint tool would, and proves both directions plus the real sign-in endpoint.
+
+on:
+  push:
+    paths:
+      - 'backend/config/os_trust.py'
+      - 'backend/main.py'
+      - 'backend/apps/auth/router.py'
+      - 'backend/requirements.lock'
+      - 'scripts/os_trust_drill.py'
+      - '.github/workflows/os-trust-drill.yml'
+  workflow_dispatch:
+
+jobs:
+  drill:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [windows-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+          cache: pip
+          cache-dependency-path: backend/requirements.lock
+
+      - name: Install the backend's locked dependencies (the same set the packaged python-env ships)
+        run: pip install -r backend/requirements.lock
+
+      - name: Mint a throwaway root and a leaf signed under it
+        run: python scripts/os_trust_drill.py mint drill-ca
+
+      - name: Install the root into the OS store the way an endpoint tool would (Windows)
+        if: runner.os == 'Windows'
+        run: certutil -addstore -f Root drill-ca/root.pem
+
+      - name: Install the root into the OS store the way an endpoint tool would (macOS)
+        if: runner.os == 'macOS'
+        run: sudo security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain drill-ca/root.pem
+
+      - name: Stock certifi refuses the OS root, the armed backend accepts it, the cloud verifies
+        shell: bash
+        run: python scripts/os_trust_drill.py verify drill-ca
+
+      - name: The real sign-in endpoint, all three directions
+        shell: bash
+        run: |
+          set -e
+          python scripts/os_trust_drill.py signin armed
+          python scripts/os_trust_drill.py signin armed-foreign-ca
+          python scripts/os_trust_drill.py signin stock-foreign-ca

backend/config/os_trust.py

@@ -0,0 +1,45 @@
+"""Route every outbound TLS verification in this process through the OS trust store.
+
+httpx, and everything built on it, verifies against certifi's bundled CA file, which never sees a
+root that an endpoint security tool or a corporate proxy installed into the Windows or macOS store.
+Chromium and the updater honor those roots, so the app looks healthy while the Python backend refuses
+every https call it makes: sign-in, the cloud, telemetry. Arming the OS store here, before any client
+exists, makes the backend trust exactly what the user's browser trusts and nothing more; an untrusted,
+self-signed or expired certificate still fails closed.
+"""
+
+import logging
+import platform
+import ssl
+from typing import Literal
+
+from typeguard import typechecked
+
+logger = logging.getLogger(__name__)
+
+TrustSource = Literal["os-store", "certifi"]
+
+# Linux keeps certifi: truststore there only reads OpenSSL's default paths, which a bare container may not have.
+P_OS_STORE_PLATFORMS = frozenset({"Darwin", "Windows"})
+
+
+@typechecked
+def install_os_trust() -> TrustSource:
+    system = platform.system()
+    if system not in P_OS_STORE_PLATFORMS:
+        logger.info("tls trust: certifi bundle (%s has no OS store hook)", system)
+        return "certifi"
+    try:
+        import truststore
+        truststore.inject_into_ssl()
+        # The factory httpx calls must hand back the OS-store class, or "armed" would be a lie.
+        if type(ssl.create_default_context()) is not truststore.SSLContext:
+            raise RuntimeError("ssl.create_default_context() still builds the stock SSLContext")
+    except Exception as e:
+        logger.warning(
+            "tls trust: OS store unavailable on %s (%s); https from this process trusts the certifi bundle only, so a proxy or security-tool root will be refused",
+            system, e,
+        )
+        return "certifi"
+    logger.info("tls trust: OS store on %s", system)
+    return "os-store"

backend/main.py

@@ -15,6 +15,11 @@ if not p_backend_logger.handlers:
 
 logger = logging.getLogger(__name__)
 
+from backend.config.os_trust import install_os_trust
+
+# Before any app module can build an httpx client, or the stock certifi-only context would be baked in.
+install_os_trust()
+
 from fastapi.responses import JSONResponse, HTMLResponse
 from fastapi import Request
 

backend/requirements.lock

@@ -278,7 +278,7 @@ click==8.4.1 \
     # via
     #   rich-toolkit
     #   uvicorn
-colorama==0.4.6 ; sys_platform == 'win32' or platform_system == 'Windows' \
+colorama==0.4.6 ; sys_platform == 'win32' \
     --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \
     --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6
     # via
@@ -1514,6 +1514,10 @@ trafilatura==2.0.0 \
     --hash=sha256:77eb5d1e993747f6f20938e1de2d840020719735690c840b9a1024803a4cd51d \
     --hash=sha256:ceb7094a6ecc97e72fea73c7dba36714c5c5b577b6470e4520dca893706d6247
     # via -r backend/requirements.txt
+truststore==0.10.4 \
+    --hash=sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301 \
+    --hash=sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981
+    # via -r backend/requirements.txt
 typeguard==4.4.2 \
     --hash=sha256:77a78f11f09777aeae7fa08585f33b5f4ef0e7335af40005b0c422ed398ff48c \
     --hash=sha256:a6f1065813e32ef365bc3b3f503af8a96f9dd4e0033a02c28c4a4983de8c6c49
@@ -1543,7 +1547,7 @@ typing-inspection==0.4.2 \
     #   mcp
     #   pydantic
     #   pydantic-settings
-tzdata==2026.2 ; platform_system == 'Windows' \
+tzdata==2026.2 ; sys_platform == 'win32' \
     --hash=sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10 \
     --hash=sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7
     # via tzlocal

backend/requirements.txt

@@ -15,6 +15,8 @@ typeguard==4.4.2
 python-dotenv==1.1.1
 Pillow==12.2.0
 httpx==0.28.1
+# truststore: verify https against the OS trust store (Windows/macOS) instead of certifi alone, so a root an endpoint tool or corporate proxy installed stops refusing sign-in and every other backend call; the browser already trusts those roots. ctypes only, 68KB, MIT.
+truststore==0.10.4
 trafilatura==2.0.0
 # curl_cffi: Chrome TLS-fingerprint impersonation for the keyless search rungs. Measured 8/8 against DuckDuckGo where plain httpx scored 4/8 on the same queries; abi3 wheels for macos arm64/x64 and win_amd64. Guarded import, so a missing wheel degrades to httpx instead of breaking the backend.
 curl_cffi==0.15.0

backend/tests/test_os_trust.py

@@ -0,0 +1,128 @@
+"""The backend's https trust follows the OS store on desktop platforms, and still fails closed.
+
+Sign-in on Windows ThinkPads died with `unable to get local issuer certificate`: an endpoint tool's
+root sat in the Windows store, the browser trusted it, and certifi (the only store httpx read) had
+never heard of it. These pin that the OS store is armed before any client exists, that a failure to
+arm is said out loud, and that a certificate nobody trusts is still refused with the OS store on.
+"""
+
+import datetime
+import http.server
+import logging
+import platform
+import ssl
+import threading
+from pathlib import Path
+
+import httpx
+import pytest
+import truststore
+
+from backend.config.os_trust import install_os_trust
+
+P_REPO = Path(__file__).resolve().parents[2]
+
+
+@pytest.fixture
+def stock_ssl_afterwards():
+    yield
+    truststore.extract_from_ssl()
+
+
+def test_desktop_platforms_arm_the_os_store_in_the_factory_httpx_calls(monkeypatch, stock_ssl_afterwards):
+    monkeypatch.setattr(platform, "system", lambda: "Windows")
+    assert install_os_trust() == "os-store"
+    # httpx builds its verify context with ssl.create_default_context(); that is the call that must change.
+    assert type(ssl.create_default_context()) is truststore.SSLContext
+
+
+def test_off_desktop_certifi_stays_and_says_so(monkeypatch, caplog):
+    monkeypatch.setattr(platform, "system", lambda: "Linux")
+    before = ssl.SSLContext
+    with caplog.at_level(logging.INFO, logger="backend.config.os_trust"):
+        assert install_os_trust() == "certifi"
+    assert ssl.SSLContext is before
+    assert any("certifi bundle" in r.getMessage() for r in caplog.records)
+
+
+def test_a_store_that_cannot_arm_is_reported_not_swallowed(monkeypatch, caplog):
+    monkeypatch.setattr(platform, "system", lambda: "Darwin")
+
+    def p_boom() -> None:
+        raise OSError("Security.framework missing")
+
+    monkeypatch.setattr(truststore, "inject_into_ssl", p_boom)
+    with caplog.at_level(logging.WARNING, logger="backend.config.os_trust"):
+        assert install_os_trust() == "certifi"
+    warned = [r for r in caplog.records if r.levelno == logging.WARNING]
+    assert warned and "certifi bundle only" in warned[0].getMessage()
+
+
+def p_self_signed_pem(tmp_path: Path) -> Path:
+    from cryptography import x509
+    from cryptography.hazmat.primitives import hashes, serialization
+    from cryptography.hazmat.primitives.asymmetric import ec
+    from cryptography.x509.oid import NameOID
+
+    key = ec.generate_private_key(ec.SECP256R1())
+    name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "127.0.0.1")])
+    now = datetime.datetime.now(datetime.timezone.utc)
+    cert = (
+        x509.CertificateBuilder()
+        .subject_name(name)
+        .issuer_name(name)
+        .public_key(key.public_key())
+        .serial_number(x509.random_serial_number())
+        .not_valid_before(now - datetime.timedelta(days=1))
+        .not_valid_after(now + datetime.timedelta(days=1))
+        .add_extension(x509.SubjectAlternativeName([x509.IPAddress(__import__("ipaddress").ip_address("127.0.0.1"))]), critical=False)
+        .sign(key, hashes.SHA256())
+    )
+    pem = tmp_path / "nobody-trusts-me.pem"
+    pem.write_bytes(
+        key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption())
+        + cert.public_bytes(serialization.Encoding.PEM)
+    )
+    return pem
+
+
+class P_QuietHandler(http.server.BaseHTTPRequestHandler):
+    def do_GET(self) -> None:
+        self.send_response(200)
+        self.end_headers()
+        self.wfile.write(b"hello")
+
+    def log_message(self, *args) -> None:
+        pass
+
+
+class P_QuietServer(http.server.ThreadingHTTPServer):
+    def handle_error(self, request, client_address) -> None:
+        pass
+
+
+@pytest.mark.skipif(platform.system() not in ("Darwin", "Windows"), reason="the OS store hook only arms on desktop platforms")
+def test_the_os_store_still_refuses_a_certificate_nobody_trusts(tmp_path, stock_ssl_afterwards):
+    pem = p_self_signed_pem(tmp_path)
+    server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+    server_ctx.load_cert_chain(pem)
+    server = P_QuietServer(("127.0.0.1", 0), P_QuietHandler)
+    server.socket = server_ctx.wrap_socket(server.socket, server_side=True)
+    threading.Thread(target=server.serve_forever, daemon=True).start()
+    url = f"https://127.0.0.1:{server.server_address[1]}/"
+    try:
+        assert install_os_trust() == "os-store"
+        with pytest.raises(httpx.ConnectError):
+            httpx.get(url, timeout=5)
+        # Control: the server is really up, so the refusal above was verification and not a dead port.
+        assert httpx.get(url, timeout=5, verify=False).text == "hello"
+    finally:
+        server.shutdown()
+
+
+def test_main_arms_os_trust_before_the_first_app_import():
+    """A client built at import time would keep the stock context forever, so the arming has to come
+    before any backend.apps module loads. Index order in the source is the assertion."""
+    src = (P_REPO / "backend" / "main.py").read_text()
+    assert "install_os_trust()" in src, "backend/main.py never arms the OS trust store"
+    assert src.index("install_os_trust()") < src.index("from backend.apps"), "OS trust is armed after an app import"

scripts/os_trust_drill.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+"""Closed-loop drill for ENG-407: does this machine's Python trust what its OS trusts, and does sign-in reach the cloud?
+
+  mint <dir>      write root.pem and leaf.pem (SAN localhost + 127.0.0.1) signed under that root
+  verify <dir>    serve leaf.pem on 127.0.0.1; stock certifi MUST refuse it, the armed backend trust MUST accept it
+                  (the workflow installed root.pem into the OS store first), and https://api.openswarm.com MUST verify armed
+  signin <mode>   POST the real /api/auth/signin-activate (no lifespan) with a junk token:
+                    armed             -> a 4xx FROM THE CLOUD (TLS passed, the service answered our junk token)
+                    armed-foreign-ca  -> still a cloud 4xx with SSL_CERT_FILE pointing at a CA file that cannot verify anything;
+                                         only the OS store can have done the verifying
+                    stock-foreign-ca  -> 502 CERTIFICATE_VERIFY_FAILED, the field failure reproduced on demand
+Exit 0 only when every direction holds; every assertion prints what it saw.
+"""
+
+import datetime
+import http.server
+import ipaddress
+import os
+import ssl
+import sys
+import tempfile
+import threading
+from pathlib import Path
+
+CLOUD = "https://api.openswarm.com/api/health"
+
+
+def p_say(ok: bool, what: str) -> None:
+    print(("PASS  " if ok else "FAIL  ") + what, flush=True)
+    if not ok:
+        sys.exit(1)
+
+
+def p_mint(out: Path) -> None:
+    from cryptography import x509
+    from cryptography.hazmat.primitives import hashes, serialization
+    from cryptography.hazmat.primitives.asymmetric import ec
+    from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID
+
+    now = datetime.datetime.now(datetime.timezone.utc)
+    root_key = ec.generate_private_key(ec.SECP256R1())
+    root_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "OpenSwarm ENG-407 throwaway drill root")])
+    root = (
+        x509.CertificateBuilder().subject_name(root_name).issuer_name(root_name).public_key(root_key.public_key())
+        .serial_number(x509.random_serial_number()).not_valid_before(now - datetime.timedelta(days=1)).not_valid_after(now + datetime.timedelta(days=2))
+        .add_extension(x509.BasicConstraints(ca=True, path_length=0), critical=True)
+        .add_extension(x509.KeyUsage(digital_signature=True, key_cert_sign=True, crl_sign=True, content_commitment=False, key_encipherment=False, data_encipherment=False, key_agreement=False, encipher_only=False, decipher_only=False), critical=True)
+        .add_extension(x509.SubjectKeyIdentifier.from_public_key(root_key.public_key()), critical=False)
+        .sign(root_key, hashes.SHA256())
+    )
+    leaf_key = ec.generate_private_key(ec.SECP256R1())
+    leaf = (
+        x509.CertificateBuilder().subject_name(x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "localhost")])).issuer_name(root_name)
+        .public_key(leaf_key.public_key()).serial_number(x509.random_serial_number())
+        .not_valid_before(now - datetime.timedelta(days=1)).not_valid_after(now + datetime.timedelta(days=2))
+        .add_extension(x509.SubjectAlternativeName([x509.DNSName("localhost"), x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]), critical=False)
+        .add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True)
+        .add_extension(x509.ExtendedKeyUsage([ExtendedKeyUsageOID.SERVER_AUTH]), critical=False)
+        .add_extension(x509.AuthorityKeyIdentifier.from_issuer_public_key(root_key.public_key()), critical=False)
+        .sign(root_key, hashes.SHA256())
+    )
+    out.mkdir(parents=True, exist_ok=True)
+    (out / "root.pem").write_bytes(root.public_bytes(serialization.Encoding.PEM))
+    (out / "leaf.pem").write_bytes(
+        leaf_key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption())
+        + leaf.public_bytes(serialization.Encoding.PEM)
+        + root.public_bytes(serialization.Encoding.PEM)
+    )
+    print(f"minted {out / 'root.pem'} and {out / 'leaf.pem'}")
+
+
+class P_Handler(http.server.BaseHTTPRequestHandler):
+    def do_GET(self) -> None:
+        self.send_response(200)
+        self.end_headers()
+        self.wfile.write(b"hello")
+
+    def log_message(self, *args) -> None:
+        pass
+
+
+class P_Server(http.server.ThreadingHTTPServer):
+    def handle_error(self, request, client_address) -> None:
+        pass
+
+
+def p_serve(leaf: Path) -> str:
+    ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+    ctx.load_cert_chain(leaf)
+    server = P_Server(("127.0.0.1", 0), P_Handler)
+    server.socket = ctx.wrap_socket(server.socket, server_side=True)
+    threading.Thread(target=server.serve_forever, daemon=True).start()
+    return f"https://localhost:{server.server_address[1]}/"
+
+
+def p_get(url: str) -> str:
+    import httpx
+    try:
+        r = httpx.get(url, timeout=15)
+        return f"HTTP {r.status_code}"
+    except Exception as e:
+        return f"{type(e).__name__}: {str(e)[:160]}"
+
+
+def p_verify(d: Path) -> None:
+    import platform
+    url = p_serve(d / "leaf.pem")
+    stock = p_get(url)
+    p_say("HTTP" not in stock, f"stock certifi refuses the OS-only root: {stock}")
+    from backend.config.os_trust import install_os_trust
+    source = install_os_trust()
+    p_say(source == "os-store", f"backend trust armed on {platform.system()}: {source}")
+    armed = p_get(url)
+    p_say(armed == "HTTP 200", f"armed backend trust accepts a leaf under the root the OS trusts: {armed}")
+    cloud = p_get(CLOUD)
+    p_say(cloud.startswith("HTTP"), f"armed backend trust verifies the sign-in service: {cloud}")
+
+
+def p_signin(mode: str) -> None:
+    data_root = tempfile.mkdtemp(prefix="osw-drill-")
+    os.environ["OPENSWARM_DATA_ROOT"] = data_root
+    os.environ["OPENSWARM_HEADLESS"] = "1"
+    if mode.endswith("foreign-ca"):
+        foreign = Path(data_root) / "foreign-ca"
+        p_mint(foreign)
+        os.environ["SSL_CERT_FILE"] = str(foreign / "root.pem")
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+    import backend.main as main
+    if mode.startswith("stock"):
+        import truststore
+        truststore.extract_from_ssl()
+    from fastapi.testclient import TestClient
+    r = TestClient(main.app).post("/api/auth/signin-activate", json={"token": "drill-" * 8, "signin_method": "google"})
+    detail = str(r.json().get("detail", ""))[:200] if r.headers.get("content-type", "").startswith("application/json") else r.text[:200]
+    seen = f"{mode}: HTTP {r.status_code} {detail}"
+    if mode.startswith("stock"):
+        p_say(r.status_code == 502 and "CERTIFICATE_VERIFY_FAILED" in detail, "field failure reproduced, " + seen)
+    else:
+        # A junk token earns a 4xx FROM THE CLOUD; the failure being drilled is our own 502 before any byte reached it.
+        p_say(r.status_code in (400, 401) and "Could not reach" not in detail, "sign-in reached the cloud over TLS, " + seen)
+
+
+if __name__ == "__main__":
+    cmd, arg = (sys.argv + [None, None])[1:3]
+    if cmd == "mint" and arg:
+        p_mint(Path(arg))
+    elif cmd == "verify" and arg:
+        sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+        p_verify(Path(arg))
+    elif cmd == "signin" and arg in ("armed", "armed-foreign-ca", "stock-foreign-ca"):
+        p_signin(arg)
+    else:
+        print(__doc__)
+        sys.exit(2)