From debc9eb6297166e4303a7bb8992901408736edf6 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Sun, 30 Aug 2026 10:26:52 -0700 Subject: [PATCH] [eric] agents: a quarantined runtime is put back from the installer package instead of asked about (ENG-422) Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01U6zrBsUCNzpMBnov3rTVYV --- backend/apps/agents/agents.py | 21 ++- backend/apps/agents/core/cli_self_heal.py | 120 ++++++++++++++++++ backend/tests/test_cli_self_heal.py | 148 ++++++++++++++++++++++ 3 files changed, 285 insertions(+), 4 deletions(-) create mode 100644 backend/apps/agents/core/cli_self_heal.py create mode 100644 backend/tests/test_cli_self_heal.py diff --git a/backend/apps/agents/agents.py b/backend/apps/agents/agents.py index ccb81c5e..a4e0b831 100644 --- a/backend/apps/agents/agents.py +++ b/backend/apps/agents/agents.py @@ -635,16 +635,29 @@ async def subscriptions_health(): from backend.apps.nine_router.subscription_health import probe_subscription_health from backend.apps.agents.core.bundled_cli_missing import bundled_cli_missing # Independent of the router: the bundled-CLI integrity check rides the same boot fetch so an AV-quarantined runtime surfaces as a pill instead of dead turns. - p_cli_missing = bundled_cli_missing() is not None + p_gone = bundled_cli_missing() + p_heal = None + if p_gone is not None: + # Try to put it back before telling anyone it is broken. Detection alone left 22 of 25 + # installs dead, because the fix we named is one almost no user can perform (ENG-422). + from backend.apps.agents.core.cli_self_heal import repair_bundled_cli + try: + p_result = repair_bundled_cli(p_gone) + p_heal = p_result.detail + if p_result.repaired and not p_result.retaken: + p_gone = bundled_cli_missing() + except Exception: + logger.exception("bundled-CLI self-heal failed; leaving the card standing") + p_cli_missing = p_gone is not None if not is_running(): - return {"dead": [], "skipped": True, "cli_missing": p_cli_missing} + return {"dead": [], "skipped": True, "cli_missing": p_cli_missing, "cli_repair": p_heal} try: connections = await get_providers() dead = await probe_subscription_health(connections) - return {"dead": dead, "skipped": False, "cli_missing": p_cli_missing} + return {"dead": dead, "skipped": False, "cli_missing": p_cli_missing, "cli_repair": p_heal} except Exception as e: logger.debug(f"subscription health probe failed: {e}") - return {"dead": [], "skipped": True, "cli_missing": p_cli_missing} + return {"dead": [], "skipped": True, "cli_missing": p_cli_missing, "cli_repair": p_heal} @agents.router.get("/subscriptions/models") diff --git a/backend/apps/agents/core/cli_self_heal.py b/backend/apps/agents/core/cli_self_heal.py new file mode 100644 index 00000000..5659dac0 --- /dev/null +++ b/backend/apps/agents/core/cli_self_heal.py @@ -0,0 +1,120 @@ +"""Put the bundled agent runtime back when antivirus takes it, instead of asking the user to. + +The failure: Windows AV quarantines `claude.exe` out of the installed app, every agent reply stops, +and the card tells the user to restore it from quarantine and add an exclusion. That is the correct +fix and almost nobody performs it -- 22 of 25 affected installs never produced another agent reply, +and a real user replied "don't know how to take a file out of quarantine" (ENG-422). Signing does not +prevent it: the release gates on the binary being validly signed and it is taken anyway. + +The one link we control is that we cannot repair ourselves. We can: Squirrel keeps the installer +package on disk under the user's own app data, and that package contains a pristine copy. Restoring +is a local file copy with no download and no admin rights. + +Deliberately NOT silent. A repair that hides itself is how "it broke, then it worked, then it broke" +becomes unreportable; the caller says what happened, and if AV takes it again immediately we say THAT +rather than looping. +""" + +import logging +import os +import platform +import shutil +import time +import zipfile +from typing import List, Optional + +from pydantic import BaseModel, ConfigDict +from typeguard import typechecked + +logger = logging.getLogger(__name__) + +# Long enough that a real-time scanner has had its chance, short enough that a user is still watching. +RETAKEN_CHECK_SECONDS = 2.0 + + +class RepairResult(BaseModel): + """What actually happened, in the caller's words. `repaired` alone is not enough: a file that + comes back and is immediately re-quarantined must not read as a fix.""" + + model_config = ConfigDict(validate_assignment=True) + + repaired: bool = False + retaken: bool = False + source: Optional[str] = None + detail: str = "" + + +@typechecked +def package_cache_dirs() -> List[str]: + """Where the installer package lives, per platform. Windows is the only place this class has + been observed, but the search is written so a mac build can be repaired the same way if it ever + needs to be.""" + p_out: List[str] = [] + if platform.system() == "Windows": + for p_var in ("LOCALAPPDATA", "APPDATA"): + p_base = os.environ.get(p_var) + if p_base: + p_out.append(os.path.join(p_base, "openswarm", "packages")) + return [d for d in p_out if os.path.isdir(d)] + + +@typechecked +def find_pristine_copy(member_suffix: str, search_dirs: Optional[List[str]] = None) -> Optional[str]: + """The newest installer package that actually contains the missing file. + + Newest first, because an older package holds an older binary and silently restoring THAT is how + a version mismatch becomes a second, stranger bug.""" + p_dirs = package_cache_dirs() if search_dirs is None else search_dirs + p_packages: List[str] = [] + for d in p_dirs: + for name in os.listdir(d): + if name.lower().endswith((".nupkg", ".zip")): + p_packages.append(os.path.join(d, name)) + p_packages.sort(key=lambda p: os.path.getmtime(p), reverse=True) + for pkg in p_packages: + try: + with zipfile.ZipFile(pkg) as z: + for member in z.namelist(): + if member.replace("\\", "/").endswith(member_suffix): + return pkg + except Exception as e: + logger.debug("self-heal: %s is not a readable package (%s)", pkg, e) + return None + + +@typechecked +def repair_bundled_cli(dest: str, member_suffix: str = "_bundled/claude.exe", + search_dirs: Optional[List[str]] = None) -> RepairResult: + """Restore `dest` from the installer package, then check it survived. + + Returns what happened rather than raising: a failed repair must leave the existing card standing, + not replace a explainable problem with a traceback.""" + if os.path.isfile(dest): + return RepairResult(detail="the runtime is already present; nothing to repair") + pkg = find_pristine_copy(member_suffix, search_dirs) + if pkg is None: + return RepairResult( + detail="no installer package on disk holds a copy, so this needs a reinstall") + try: + os.makedirs(os.path.dirname(dest), exist_ok=True) + with zipfile.ZipFile(pkg) as z: + member = next(m for m in z.namelist() + if m.replace("\\", "/").endswith(member_suffix)) + with z.open(member) as src, open(dest, "wb") as out: + shutil.copyfileobj(src, out) + os.chmod(dest, 0o755) + except Exception as e: + logger.warning("self-heal: could not restore %s from %s (%s)", dest, pkg, e) + return RepairResult(source=pkg, detail=f"restoring it failed: {e}") + + # The half that matters. A restore that is undone a second later is not a repair, and reporting + # it as one sends the user back to a broken app believing it is fixed. + time.sleep(RETAKEN_CHECK_SECONDS) + if not os.path.isfile(dest): + logger.warning("self-heal: %s was removed again right after restore; antivirus is holding it", dest) + return RepairResult(repaired=True, retaken=True, source=pkg, + detail="it was restored and removed again straight away, so an " + "antivirus exclusion is needed before it will stay") + logger.info("self-heal: restored the bundled agent runtime from %s", pkg) + return RepairResult(repaired=True, source=pkg, + detail="the bundled agent runtime was restored from your installer package") diff --git a/backend/tests/test_cli_self_heal.py b/backend/tests/test_cli_self_heal.py new file mode 100644 index 00000000..1b057c40 --- /dev/null +++ b/backend/tests/test_cli_self_heal.py @@ -0,0 +1,148 @@ +"""Putting the bundled runtime back, instead of asking the user to (ENG-422). + +Every case here is the real failure simulated on disk: a quarantined binary is just a missing file, +and an installer package is just a zip, so the whole repair is exercisable on any platform even +though the class has only ever been seen on Windows.""" + +import os +import zipfile + +import pytest + +from backend.apps.agents.core import cli_self_heal as heal + +MEMBER = "lib/net45/resources/python-env/Lib/site-packages/claude_agent_sdk/_bundled/claude.exe" +SUFFIX = "_bundled/claude.exe" + + +@pytest.fixture(autouse=True) +def p_fast(monkeypatch): + """The re-take check is a real sleep in production; tests must not pay for it.""" + monkeypatch.setattr(heal, "RETAKEN_CHECK_SECONDS", 0.01) + + +def p_package(dirpath, body=b"PRISTINE-RUNTIME", name="openswarm-1.7.9-full.nupkg", member=MEMBER): + p = os.path.join(dirpath, name) + with zipfile.ZipFile(p, "w") as z: + z.writestr(member, body) + return p + + +def test_it_restores_the_binary_from_the_installer_package(tmp_path): + cache = tmp_path / "packages"; cache.mkdir() + p_package(str(cache)) + dest = tmp_path / "site-packages" / "claude_agent_sdk" / "_bundled" / "claude.exe" + r = heal.repair_bundled_cli(str(dest), SUFFIX, [str(cache)]) + assert r.repaired and not r.retaken + assert dest.read_bytes() == b"PRISTINE-RUNTIME" + assert "restored" in r.detail + + +def test_a_restore_that_is_undone_is_NOT_reported_as_a_fix(tmp_path, monkeypatch): + """The half that matters. Antivirus re-takes the file seconds later; calling that a repair sends + the user back to a broken app believing it works.""" + cache = tmp_path / "packages"; cache.mkdir() + p_package(str(cache)) + dest = tmp_path / "_bundled" / "claude.exe" + + p_real_sleep = heal.time.sleep + + def p_quarantine(_): + p_real_sleep(0) + if os.path.isfile(dest): + os.remove(dest) # the scanner takes it back + monkeypatch.setattr(heal.time, "sleep", p_quarantine) + + r = heal.repair_bundled_cli(str(dest), SUFFIX, [str(cache)]) + assert r.repaired and r.retaken, "a re-taken file must say so" + assert "exclusion" in r.detail, "and must name what would actually make it stay" + + +def test_no_package_on_disk_says_reinstall_rather_than_failing_silently(tmp_path): + empty = tmp_path / "packages"; empty.mkdir() + dest = tmp_path / "_bundled" / "claude.exe" + r = heal.repair_bundled_cli(str(dest), SUFFIX, [str(empty)]) + assert not r.repaired and "reinstall" in r.detail + assert not dest.exists() + + +def test_it_prefers_the_NEWEST_package(tmp_path): + """An older package holds an older binary; restoring that silently turns one bug into a version + mismatch nobody would think to look for.""" + cache = tmp_path / "packages"; cache.mkdir() + old = p_package(str(cache), b"OLD-RUNTIME", "openswarm-1.7.8-full.nupkg") + new = p_package(str(cache), b"NEW-RUNTIME", "openswarm-1.7.9-full.nupkg") + os.utime(old, (1, 1)) + os.utime(new, (10_000_000, 10_000_000)) + dest = tmp_path / "_bundled" / "claude.exe" + r = heal.repair_bundled_cli(str(dest), SUFFIX, [str(cache)]) + assert dest.read_bytes() == b"NEW-RUNTIME", "restored the stale copy" + assert r.source == new + + +def test_a_package_without_the_binary_is_skipped_not_trusted(tmp_path): + cache = tmp_path / "packages"; cache.mkdir() + p_package(str(cache), b"irrelevant", "openswarm-1.7.9-full.nupkg", member="lib/net45/README.txt") + good = p_package(str(cache), b"PRISTINE-RUNTIME", "openswarm-1.7.9-delta.nupkg") + os.utime(good, (1, 1)) + dest = tmp_path / "_bundled" / "claude.exe" + r = heal.repair_bundled_cli(str(dest), SUFFIX, [str(cache)]) + assert r.repaired and r.source == good + + +def test_a_corrupt_package_does_not_take_the_repair_down_with_it(tmp_path): + cache = tmp_path / "packages"; cache.mkdir() + (cache / "openswarm-broken.nupkg").write_bytes(b"not a zip at all") + good = p_package(str(cache), b"PRISTINE-RUNTIME", "openswarm-ok.nupkg") + os.utime(good, (1, 1)) + dest = tmp_path / "_bundled" / "claude.exe" + assert heal.repair_bundled_cli(str(dest), SUFFIX, [str(cache)]).repaired + assert dest.read_bytes() == b"PRISTINE-RUNTIME" + + +def test_a_present_binary_is_never_overwritten(tmp_path): + """The innocent case: this runs on a healthy install too, and must not clobber a good file with + an older packaged one.""" + cache = tmp_path / "packages"; cache.mkdir() + p_package(str(cache), b"PACKAGED") + dest = tmp_path / "_bundled" / "claude.exe" + dest.parent.mkdir(parents=True) + dest.write_bytes(b"LIVE-AND-FINE") + r = heal.repair_bundled_cli(str(dest), SUFFIX, [str(cache)]) + assert not r.repaired and dest.read_bytes() == b"LIVE-AND-FINE" + + +@pytest.mark.asyncio +async def test_detection_actually_CALLS_the_repair(monkeypatch): + """A repair nothing calls is this codebase's recurring defect: present, reachable, doing nothing. + Detection already existed and left 22 of 25 installs dead precisely because nothing acted on it. + + Behavioural, not a grep: an earlier version of this test asserted the NAME appeared in the + source, and passed happily when the call was replaced with a raise.""" + import backend.apps.agents.core.bundled_cli_missing as det + import backend.apps.agents.core.cli_self_heal as sh + from backend.apps.agents.agents import subscriptions_health + + called = {} + monkeypatch.setattr(det, "bundled_cli_missing", lambda: "/gone/claude.exe") + def p_fake(dest, *a, **k): + called["dest"] = dest + return sh.RepairResult(repaired=True, detail="restored in the test") + monkeypatch.setattr(sh, "repair_bundled_cli", p_fake) + monkeypatch.setattr("backend.apps.nine_router.is_running", lambda: False) + + out = await subscriptions_health() + assert called.get("dest") == "/gone/claude.exe", "the repair was never attempted" + assert out.get("cli_repair") == "restored in the test", "the endpoint hid what the repair did" + + +def test_the_repair_never_heals_in_silence(): + """Every return path of the health endpoint carries what the repair did. A fix the user is not + told about is indistinguishable from a flaky app that broke and un-broke itself.""" + src = open("backend/apps/agents/agents.py", encoding="utf-8").read() + i = src.index("async def subscriptions_health") + body = src[i:src.index("@agents.router", i + 10)] + returns = [ln for ln in body.splitlines() if "return {" in ln and "cli_missing" in ln] + assert returns, "the health endpoint stopped reporting cli_missing" + for ln in returns: + assert "cli_repair" in ln, f"a return path hides the repair: {ln.strip()[:80]}"