[eric] apps: the ghost sweep fails closed, a bad ps scan or a mid-spawn race can no longer read working apps as ghosts

backend/apps/outputs/reap_ghost_runtimes.py

@@ -72,6 +72,11 @@ def find_ghost_runtime_pids() -> List[int]:
     mine = os.getpid()
     owners = p_live_backend_pids()
     parents = p_ppid_map()
+    # WE are a backend, so a scan that finds no live backend has failed, not found ghosts: an empty
+    # owner set turns every working app into a "ghost" and the sweep would kill them all mid-use.
+    # Boot relied on running before anything spawned; the 10-minute sweep gets no such alibi.
+    if not owners or not parents:
+        return []
     ghosts: List[int] = []
     for line in (out.stdout or "").splitlines():
         line = line.strip()
@@ -83,14 +88,21 @@ def find_ghost_runtime_pids() -> List[int]:
         pid = int(head[0])
         if pid == mine:
             continue
-        cur, owned, hops = pid, False, 0
-        while cur > 1 and hops < 24:
+        cur, owned, broken = pid, False, False
+        for _ in range(24):
             if cur in owners or cur == mine:
                 owned = True
                 break
-            cur = parents.get(cur, 0)
-            hops += 1
-        if not owned:
+            if cur <= 1:
+                break
+            nxt = parents.get(cur)
+            if nxt is None:
+                # The pid list and the ppid map are two separate ps snapshots; a process spawned
+                # between them has no entry here. Indeterminate is NOT ghost: skip, never kill.
+                broken = True
+                break
+            cur = nxt
+        if not owned and not broken:
             ghosts.append(pid)
     return ghosts
 

backend/tests/test_reap_ghost_runtimes.py

@@ -29,8 +29,11 @@ def test_runtime_owned_by_a_live_backend_is_never_reaped():
 
 def test_runtime_whose_backend_died_is_reaped():
     ws = os.path.abspath(mod.WORKSPACE_DIR)
-    args = f"200 node {ws}/app/vite\n"          # no uvicorn anywhere
-    ppid = "200 1\n"                             # reparented to init
+    # The CALLER is always a live backend (this code runs inside one), so the scan must show at
+    # least ourselves; a fixture with "no uvicorn anywhere" models a world that cannot exist, and
+    # the fail-closed guard rightly refuses to reap in it.
+    args = f"50 python -m uvicorn backend.main:app\n200 node {ws}/app/vite\n"
+    ppid = "50 1\n200 1\n"                      # ghost reparented to init, NOT under the backend
     with patch.object(mod.subprocess, "run", side_effect=p_ps(args, ppid)):
         assert mod.find_ghost_runtime_pids() == [200]
 
@@ -88,3 +91,30 @@ def test_stale_idle_runtimes_are_stopped_after_the_ttl(monkeypatch):
     assert old.stopped and not fresh.stopped
     assert "ws-old:1" not in m.idle_lru and "ws-new:1" in m.idle_lru
     assert "ws-old:1" not in m.p_idle_since
+
+
+def test_a_failed_backend_scan_reaps_nothing(monkeypatch):
+    """We ARE a backend, so 'no live backends found' means the scan failed, not that everything is a
+    ghost; without this, one slow `ps` under load turned the 10-minute sweep into a kill-all."""
+    from backend.apps.outputs import reap_ghost_runtimes as rg
+    monkeypatch.setattr(rg, "p_live_backend_pids", lambda: set())
+    monkeypatch.setattr(rg, "p_ppid_map", lambda: {200: 1})
+    class P_Out:
+        stdout = f"200 node {rg.os.path.abspath(rg.WORKSPACE_DIR)}/ws-x/run\n"
+    monkeypatch.setattr(rg.subprocess, "run", lambda *a, **k: P_Out())
+    assert rg.find_ghost_runtime_pids() == []
+
+
+def test_indeterminate_ancestry_is_never_a_ghost(monkeypatch):
+    """A process missing from the ppid snapshot (spawned between the two ps calls) must be skipped,
+    not killed: mid-session, that is a runtime that just started."""
+    from backend.apps.outputs import reap_ghost_runtimes as rg
+    monkeypatch.setattr(rg, "p_live_backend_pids", lambda: {50})
+    monkeypatch.setattr(rg, "p_ppid_map", lambda: {300: 1})
+    ws = rg.os.path.abspath(rg.WORKSPACE_DIR)
+    class P_Out:
+        stdout = f"300 node {ws}/ws-a/run\n999 node {ws}/ws-b/run\n"
+    monkeypatch.setattr(rg.subprocess, "run", lambda *a, **k: P_Out())
+    ghosts = rg.find_ghost_runtime_pids()
+    assert 999 not in ghosts, "pid absent from the ppid map was treated as a ghost"
+    assert ghosts == [300], "a genuinely orphaned pid (walks to init, no backend) still reaps"