diff --git a/backend/apps/outputs/reap_ghost_runtimes.py b/backend/apps/outputs/reap_ghost_runtimes.py index 427edb3f..62485fd2 100644 --- a/backend/apps/outputs/reap_ghost_runtimes.py +++ b/backend/apps/outputs/reap_ghost_runtimes.py @@ -72,6 +72,11 @@ def find_ghost_runtime_pids() -> List[int]: mine = os.getpid() owners = p_live_backend_pids() parents = p_ppid_map() + # WE are a backend, so a scan that finds no live backend has failed, not found ghosts: an empty + # owner set turns every working app into a "ghost" and the sweep would kill them all mid-use. + # Boot relied on running before anything spawned; the 10-minute sweep gets no such alibi. + if not owners or not parents: + return [] ghosts: List[int] = [] for line in (out.stdout or "").splitlines(): line = line.strip() @@ -83,14 +88,21 @@ def find_ghost_runtime_pids() -> List[int]: pid = int(head[0]) if pid == mine: continue - cur, owned, hops = pid, False, 0 - while cur > 1 and hops < 24: + cur, owned, broken = pid, False, False + for _ in range(24): if cur in owners or cur == mine: owned = True break - cur = parents.get(cur, 0) - hops += 1 - if not owned: + if cur <= 1: + break + nxt = parents.get(cur) + if nxt is None: + # The pid list and the ppid map are two separate ps snapshots; a process spawned + # between them has no entry here. Indeterminate is NOT ghost: skip, never kill. + broken = True + break + cur = nxt + if not owned and not broken: ghosts.append(pid) return ghosts diff --git a/backend/tests/test_reap_ghost_runtimes.py b/backend/tests/test_reap_ghost_runtimes.py index d9574a87..bdf5ce22 100644 --- a/backend/tests/test_reap_ghost_runtimes.py +++ b/backend/tests/test_reap_ghost_runtimes.py @@ -29,8 +29,11 @@ def test_runtime_owned_by_a_live_backend_is_never_reaped(): def test_runtime_whose_backend_died_is_reaped(): ws = os.path.abspath(mod.WORKSPACE_DIR) - args = f"200 node {ws}/app/vite\n" # no uvicorn anywhere - ppid = "200 1\n" # reparented to init + # The CALLER is always a live backend (this code runs inside one), so the scan must show at + # least ourselves; a fixture with "no uvicorn anywhere" models a world that cannot exist, and + # the fail-closed guard rightly refuses to reap in it. + args = f"50 python -m uvicorn backend.main:app\n200 node {ws}/app/vite\n" + ppid = "50 1\n200 1\n" # ghost reparented to init, NOT under the backend with patch.object(mod.subprocess, "run", side_effect=p_ps(args, ppid)): assert mod.find_ghost_runtime_pids() == [200] @@ -88,3 +91,30 @@ def test_stale_idle_runtimes_are_stopped_after_the_ttl(monkeypatch): assert old.stopped and not fresh.stopped assert "ws-old:1" not in m.idle_lru and "ws-new:1" in m.idle_lru assert "ws-old:1" not in m.p_idle_since + + +def test_a_failed_backend_scan_reaps_nothing(monkeypatch): + """We ARE a backend, so 'no live backends found' means the scan failed, not that everything is a + ghost; without this, one slow `ps` under load turned the 10-minute sweep into a kill-all.""" + from backend.apps.outputs import reap_ghost_runtimes as rg + monkeypatch.setattr(rg, "p_live_backend_pids", lambda: set()) + monkeypatch.setattr(rg, "p_ppid_map", lambda: {200: 1}) + class P_Out: + stdout = f"200 node {rg.os.path.abspath(rg.WORKSPACE_DIR)}/ws-x/run\n" + monkeypatch.setattr(rg.subprocess, "run", lambda *a, **k: P_Out()) + assert rg.find_ghost_runtime_pids() == [] + + +def test_indeterminate_ancestry_is_never_a_ghost(monkeypatch): + """A process missing from the ppid snapshot (spawned between the two ps calls) must be skipped, + not killed: mid-session, that is a runtime that just started.""" + from backend.apps.outputs import reap_ghost_runtimes as rg + monkeypatch.setattr(rg, "p_live_backend_pids", lambda: {50}) + monkeypatch.setattr(rg, "p_ppid_map", lambda: {300: 1}) + ws = rg.os.path.abspath(rg.WORKSPACE_DIR) + class P_Out: + stdout = f"300 node {ws}/ws-a/run\n999 node {ws}/ws-b/run\n" + monkeypatch.setattr(rg.subprocess, "run", lambda *a, **k: P_Out()) + ghosts = rg.find_ghost_runtime_pids() + assert 999 not in ghosts, "pid absent from the ppid map was treated as a ghost" + assert ghosts == [300], "a genuinely orphaned pid (walks to init, no backend) still reaps"