[eric] agents: a quarantined runtime is put back from the installer package instead of asked about (ENG-422)

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01U6zrBsUCNzpMBnov3rTVYV

backend/apps/agents/agents.py

@@ -635,16 +635,29 @@ async def subscriptions_health():
     from backend.apps.nine_router.subscription_health import probe_subscription_health
     from backend.apps.agents.core.bundled_cli_missing import bundled_cli_missing
     # Independent of the router: the bundled-CLI integrity check rides the same boot fetch so an AV-quarantined runtime surfaces as a pill instead of dead turns.
-    p_cli_missing = bundled_cli_missing() is not None
+    p_gone = bundled_cli_missing()
+    p_heal = None
+    if p_gone is not None:
+        # Try to put it back before telling anyone it is broken. Detection alone left 22 of 25
+        # installs dead, because the fix we named is one almost no user can perform (ENG-422).
+        from backend.apps.agents.core.cli_self_heal import repair_bundled_cli
+        try:
+            p_result = repair_bundled_cli(p_gone)
+            p_heal = p_result.detail
+            if p_result.repaired and not p_result.retaken:
+                p_gone = bundled_cli_missing()
+        except Exception:
+            logger.exception("bundled-CLI self-heal failed; leaving the card standing")
+    p_cli_missing = p_gone is not None
     if not is_running():
-        return {"dead": [], "skipped": True, "cli_missing": p_cli_missing}
+        return {"dead": [], "skipped": True, "cli_missing": p_cli_missing, "cli_repair": p_heal}
     try:
         connections = await get_providers()
         dead = await probe_subscription_health(connections)
-        return {"dead": dead, "skipped": False, "cli_missing": p_cli_missing}
+        return {"dead": dead, "skipped": False, "cli_missing": p_cli_missing, "cli_repair": p_heal}
     except Exception as e:
         logger.debug(f"subscription health probe failed: {e}")
-        return {"dead": [], "skipped": True, "cli_missing": p_cli_missing}
+        return {"dead": [], "skipped": True, "cli_missing": p_cli_missing, "cli_repair": p_heal}
 
 
 @agents.router.get("/subscriptions/models")

backend/apps/agents/core/cli_self_heal.py

@@ -0,0 +1,120 @@
+"""Put the bundled agent runtime back when antivirus takes it, instead of asking the user to.
+
+The failure: Windows AV quarantines `claude.exe` out of the installed app, every agent reply stops,
+and the card tells the user to restore it from quarantine and add an exclusion. That is the correct
+fix and almost nobody performs it -- 22 of 25 affected installs never produced another agent reply,
+and a real user replied "don't know how to take a file out of quarantine" (ENG-422). Signing does not
+prevent it: the release gates on the binary being validly signed and it is taken anyway.
+
+The one link we control is that we cannot repair ourselves. We can: Squirrel keeps the installer
+package on disk under the user's own app data, and that package contains a pristine copy. Restoring
+is a local file copy with no download and no admin rights.
+
+Deliberately NOT silent. A repair that hides itself is how "it broke, then it worked, then it broke"
+becomes unreportable; the caller says what happened, and if AV takes it again immediately we say THAT
+rather than looping.
+"""
+
+import logging
+import os
+import platform
+import shutil
+import time
+import zipfile
+from typing import List, Optional
+
+from pydantic import BaseModel, ConfigDict
+from typeguard import typechecked
+
+logger = logging.getLogger(__name__)
+
+# Long enough that a real-time scanner has had its chance, short enough that a user is still watching.
+RETAKEN_CHECK_SECONDS = 2.0
+
+
+class RepairResult(BaseModel):
+    """What actually happened, in the caller's words. `repaired` alone is not enough: a file that
+    comes back and is immediately re-quarantined must not read as a fix."""
+
+    model_config = ConfigDict(validate_assignment=True)
+
+    repaired: bool = False
+    retaken: bool = False
+    source: Optional[str] = None
+    detail: str = ""
+
+
+@typechecked
+def package_cache_dirs() -> List[str]:
+    """Where the installer package lives, per platform. Windows is the only place this class has
+    been observed, but the search is written so a mac build can be repaired the same way if it ever
+    needs to be."""
+    p_out: List[str] = []
+    if platform.system() == "Windows":
+        for p_var in ("LOCALAPPDATA", "APPDATA"):
+            p_base = os.environ.get(p_var)
+            if p_base:
+                p_out.append(os.path.join(p_base, "openswarm", "packages"))
+    return [d for d in p_out if os.path.isdir(d)]
+
+
+@typechecked
+def find_pristine_copy(member_suffix: str, search_dirs: Optional[List[str]] = None) -> Optional[str]:
+    """The newest installer package that actually contains the missing file.
+
+    Newest first, because an older package holds an older binary and silently restoring THAT is how
+    a version mismatch becomes a second, stranger bug."""
+    p_dirs = package_cache_dirs() if search_dirs is None else search_dirs
+    p_packages: List[str] = []
+    for d in p_dirs:
+        for name in os.listdir(d):
+            if name.lower().endswith((".nupkg", ".zip")):
+                p_packages.append(os.path.join(d, name))
+    p_packages.sort(key=lambda p: os.path.getmtime(p), reverse=True)
+    for pkg in p_packages:
+        try:
+            with zipfile.ZipFile(pkg) as z:
+                for member in z.namelist():
+                    if member.replace("\\", "/").endswith(member_suffix):
+                        return pkg
+        except Exception as e:
+            logger.debug("self-heal: %s is not a readable package (%s)", pkg, e)
+    return None
+
+
+@typechecked
+def repair_bundled_cli(dest: str, member_suffix: str = "_bundled/claude.exe",
+                       search_dirs: Optional[List[str]] = None) -> RepairResult:
+    """Restore `dest` from the installer package, then check it survived.
+
+    Returns what happened rather than raising: a failed repair must leave the existing card standing,
+    not replace a explainable problem with a traceback."""
+    if os.path.isfile(dest):
+        return RepairResult(detail="the runtime is already present; nothing to repair")
+    pkg = find_pristine_copy(member_suffix, search_dirs)
+    if pkg is None:
+        return RepairResult(
+            detail="no installer package on disk holds a copy, so this needs a reinstall")
+    try:
+        os.makedirs(os.path.dirname(dest), exist_ok=True)
+        with zipfile.ZipFile(pkg) as z:
+            member = next(m for m in z.namelist()
+                          if m.replace("\\", "/").endswith(member_suffix))
+            with z.open(member) as src, open(dest, "wb") as out:
+                shutil.copyfileobj(src, out)
+        os.chmod(dest, 0o755)
+    except Exception as e:
+        logger.warning("self-heal: could not restore %s from %s (%s)", dest, pkg, e)
+        return RepairResult(source=pkg, detail=f"restoring it failed: {e}")
+
+    # The half that matters. A restore that is undone a second later is not a repair, and reporting
+    # it as one sends the user back to a broken app believing it is fixed.
+    time.sleep(RETAKEN_CHECK_SECONDS)
+    if not os.path.isfile(dest):
+        logger.warning("self-heal: %s was removed again right after restore; antivirus is holding it", dest)
+        return RepairResult(repaired=True, retaken=True, source=pkg,
+                            detail="it was restored and removed again straight away, so an "
+                                   "antivirus exclusion is needed before it will stay")
+    logger.info("self-heal: restored the bundled agent runtime from %s", pkg)
+    return RepairResult(repaired=True, source=pkg,
+                        detail="the bundled agent runtime was restored from your installer package")

backend/tests/test_cli_self_heal.py

@@ -0,0 +1,148 @@
+"""Putting the bundled runtime back, instead of asking the user to (ENG-422).
+
+Every case here is the real failure simulated on disk: a quarantined binary is just a missing file,
+and an installer package is just a zip, so the whole repair is exercisable on any platform even
+though the class has only ever been seen on Windows."""
+
+import os
+import zipfile
+
+import pytest
+
+from backend.apps.agents.core import cli_self_heal as heal
+
+MEMBER = "lib/net45/resources/python-env/Lib/site-packages/claude_agent_sdk/_bundled/claude.exe"
+SUFFIX = "_bundled/claude.exe"
+
+
+@pytest.fixture(autouse=True)
+def p_fast(monkeypatch):
+    """The re-take check is a real sleep in production; tests must not pay for it."""
+    monkeypatch.setattr(heal, "RETAKEN_CHECK_SECONDS", 0.01)
+
+
+def p_package(dirpath, body=b"PRISTINE-RUNTIME", name="openswarm-1.7.9-full.nupkg", member=MEMBER):
+    p = os.path.join(dirpath, name)
+    with zipfile.ZipFile(p, "w") as z:
+        z.writestr(member, body)
+    return p
+
+
+def test_it_restores_the_binary_from_the_installer_package(tmp_path):
+    cache = tmp_path / "packages"; cache.mkdir()
+    p_package(str(cache))
+    dest = tmp_path / "site-packages" / "claude_agent_sdk" / "_bundled" / "claude.exe"
+    r = heal.repair_bundled_cli(str(dest), SUFFIX, [str(cache)])
+    assert r.repaired and not r.retaken
+    assert dest.read_bytes() == b"PRISTINE-RUNTIME"
+    assert "restored" in r.detail
+
+
+def test_a_restore_that_is_undone_is_NOT_reported_as_a_fix(tmp_path, monkeypatch):
+    """The half that matters. Antivirus re-takes the file seconds later; calling that a repair sends
+    the user back to a broken app believing it works."""
+    cache = tmp_path / "packages"; cache.mkdir()
+    p_package(str(cache))
+    dest = tmp_path / "_bundled" / "claude.exe"
+
+    p_real_sleep = heal.time.sleep
+
+    def p_quarantine(_):
+        p_real_sleep(0)
+        if os.path.isfile(dest):
+            os.remove(dest)          # the scanner takes it back
+    monkeypatch.setattr(heal.time, "sleep", p_quarantine)
+
+    r = heal.repair_bundled_cli(str(dest), SUFFIX, [str(cache)])
+    assert r.repaired and r.retaken, "a re-taken file must say so"
+    assert "exclusion" in r.detail, "and must name what would actually make it stay"
+
+
+def test_no_package_on_disk_says_reinstall_rather_than_failing_silently(tmp_path):
+    empty = tmp_path / "packages"; empty.mkdir()
+    dest = tmp_path / "_bundled" / "claude.exe"
+    r = heal.repair_bundled_cli(str(dest), SUFFIX, [str(empty)])
+    assert not r.repaired and "reinstall" in r.detail
+    assert not dest.exists()
+
+
+def test_it_prefers_the_NEWEST_package(tmp_path):
+    """An older package holds an older binary; restoring that silently turns one bug into a version
+    mismatch nobody would think to look for."""
+    cache = tmp_path / "packages"; cache.mkdir()
+    old = p_package(str(cache), b"OLD-RUNTIME", "openswarm-1.7.8-full.nupkg")
+    new = p_package(str(cache), b"NEW-RUNTIME", "openswarm-1.7.9-full.nupkg")
+    os.utime(old, (1, 1))
+    os.utime(new, (10_000_000, 10_000_000))
+    dest = tmp_path / "_bundled" / "claude.exe"
+    r = heal.repair_bundled_cli(str(dest), SUFFIX, [str(cache)])
+    assert dest.read_bytes() == b"NEW-RUNTIME", "restored the stale copy"
+    assert r.source == new
+
+
+def test_a_package_without_the_binary_is_skipped_not_trusted(tmp_path):
+    cache = tmp_path / "packages"; cache.mkdir()
+    p_package(str(cache), b"irrelevant", "openswarm-1.7.9-full.nupkg", member="lib/net45/README.txt")
+    good = p_package(str(cache), b"PRISTINE-RUNTIME", "openswarm-1.7.9-delta.nupkg")
+    os.utime(good, (1, 1))
+    dest = tmp_path / "_bundled" / "claude.exe"
+    r = heal.repair_bundled_cli(str(dest), SUFFIX, [str(cache)])
+    assert r.repaired and r.source == good
+
+
+def test_a_corrupt_package_does_not_take_the_repair_down_with_it(tmp_path):
+    cache = tmp_path / "packages"; cache.mkdir()
+    (cache / "openswarm-broken.nupkg").write_bytes(b"not a zip at all")
+    good = p_package(str(cache), b"PRISTINE-RUNTIME", "openswarm-ok.nupkg")
+    os.utime(good, (1, 1))
+    dest = tmp_path / "_bundled" / "claude.exe"
+    assert heal.repair_bundled_cli(str(dest), SUFFIX, [str(cache)]).repaired
+    assert dest.read_bytes() == b"PRISTINE-RUNTIME"
+
+
+def test_a_present_binary_is_never_overwritten(tmp_path):
+    """The innocent case: this runs on a healthy install too, and must not clobber a good file with
+    an older packaged one."""
+    cache = tmp_path / "packages"; cache.mkdir()
+    p_package(str(cache), b"PACKAGED")
+    dest = tmp_path / "_bundled" / "claude.exe"
+    dest.parent.mkdir(parents=True)
+    dest.write_bytes(b"LIVE-AND-FINE")
+    r = heal.repair_bundled_cli(str(dest), SUFFIX, [str(cache)])
+    assert not r.repaired and dest.read_bytes() == b"LIVE-AND-FINE"
+
+
+@pytest.mark.asyncio
+async def test_detection_actually_CALLS_the_repair(monkeypatch):
+    """A repair nothing calls is this codebase's recurring defect: present, reachable, doing nothing.
+    Detection already existed and left 22 of 25 installs dead precisely because nothing acted on it.
+
+    Behavioural, not a grep: an earlier version of this test asserted the NAME appeared in the
+    source, and passed happily when the call was replaced with a raise."""
+    import backend.apps.agents.core.bundled_cli_missing as det
+    import backend.apps.agents.core.cli_self_heal as sh
+    from backend.apps.agents.agents import subscriptions_health
+
+    called = {}
+    monkeypatch.setattr(det, "bundled_cli_missing", lambda: "/gone/claude.exe")
+    def p_fake(dest, *a, **k):
+        called["dest"] = dest
+        return sh.RepairResult(repaired=True, detail="restored in the test")
+    monkeypatch.setattr(sh, "repair_bundled_cli", p_fake)
+    monkeypatch.setattr("backend.apps.nine_router.is_running", lambda: False)
+
+    out = await subscriptions_health()
+    assert called.get("dest") == "/gone/claude.exe", "the repair was never attempted"
+    assert out.get("cli_repair") == "restored in the test", "the endpoint hid what the repair did"
+
+
+def test_the_repair_never_heals_in_silence():
+    """Every return path of the health endpoint carries what the repair did. A fix the user is not
+    told about is indistinguishable from a flaky app that broke and un-broke itself."""
+    src = open("backend/apps/agents/agents.py", encoding="utf-8").read()
+    i = src.index("async def subscriptions_health")
+    body = src[i:src.index("@agents.router", i + 10)]
+    returns = [ln for ln in body.splitlines() if "return {" in ln and "cli_missing" in ln]
+    assert returns, "the health endpoint stopped reporting cli_missing"
+    for ln in returns:
+        assert "cli_repair" in ln, f"a return path hides the repair: {ln.strip()[:80]}"