[eric] apps: a crash no longer leaves app runtimes running for days, boot reaps what no live backend owns

backend/apps/outputs/outputs.py

@@ -63,6 +63,17 @@ async def outputs_lifespan():
         recover_orphaned_apps()
     except Exception:
         logger.exception("orphaned-app recovery failed; apps stay hidden but nothing else breaks")
+    # Ghosts from a session that died badly keep running forever: stop_all only fires on a clean
+    # shutdown, and the port-collision path routes AROUND a squatter instead of killing it. Measured
+    # on a dev box: runtimes still alive after 2 days 19 hours. Boot is the one safe moment, since we
+    # have not spawned any of our own yet.
+    try:
+        from backend.apps.outputs.reap_ghost_runtimes import reap_ghost_runtimes
+        ghosts = reap_ghost_runtimes()
+        if ghosts:
+            logger.warning("outputs lifespan: reaped %d ghost runtime(s) from a previous session", ghosts)
+    except Exception:
+        logger.exception("ghost-runtime reap failed; stale processes stay but boot continues")
     try:
         yield
     finally:

backend/apps/outputs/reap_ghost_runtimes.py

@@ -0,0 +1,120 @@
+"""Kill app-runtime processes left behind by a previous OpenSwarm that died badly.
+
+`stop_all()` reaps runtimes on a CLEAN shutdown. A crash, a SIGKILL, or a force-quit skips it, and
+every `bash run.sh` plus its vite/uvicorn descendants reparents to PID 1 and keeps running: measured
+on a dev machine, ghosts had been alive for **2 days 19 hours**, still holding their ports. The only
+existing handling reallocates around a ghost that squats a port, so the ghost never dies at all and
+they accumulate across sessions.
+
+This runs at startup, before any runtime is spawned, which is the one moment when a workspace process
+cannot legitimately belong to us: we have not started any yet.
+"""
+
+import logging
+import os
+import subprocess
+from typing import List
+
+from typeguard import typechecked
+
+from backend.apps.outputs.runtime_proc import kill_descendant_tree
+from backend.config.paths import OUTPUTS_WORKSPACE_DIR as WORKSPACE_DIR
+
+logger = logging.getLogger(__name__)
+
+
+@typechecked
+def p_live_backend_pids() -> set:
+    """PIDs of every running backend. A workspace process descended from one of these is ALIVE and
+    owned, not a ghost; a first draft of this reaper matched on the workspace path alone and would
+    have killed 14 working app runtimes on a machine where the owning backend was up."""
+    try:
+        out = subprocess.run(["ps", "-eo", "pid=,args="], capture_output=True, text=True, timeout=8)
+    except Exception:
+        return set()
+    pids = set()
+    for line in (out.stdout or "").splitlines():
+        if "uvicorn" not in line or "backend.main" not in line:
+            continue
+        head = line.strip().split(None, 1)
+        if head and head[0].isdigit():
+            pids.add(int(head[0]))
+    return pids
+
+
+@typechecked
+def p_ppid_map() -> dict:
+    try:
+        out = subprocess.run(["ps", "-eo", "pid=,ppid="], capture_output=True, text=True, timeout=8)
+    except Exception:
+        return {}
+    m = {}
+    for line in (out.stdout or "").splitlines():
+        parts = line.split()
+        if len(parts) == 2 and parts[0].isdigit() and parts[1].isdigit():
+            m[int(parts[0])] = int(parts[1])
+    return m
+
+
+@typechecked
+def find_ghost_runtime_pids() -> List[int]:
+    """PIDs of workspace processes that NO live backend owns.
+
+    Matched on the absolute workspace path, so an unrelated `npm run dev` elsewhere is never touched,
+    then filtered by walking each candidate's ancestry: if a live backend is anywhere above it, it is
+    someone's working app and is left alone.
+    """
+    needle = os.path.abspath(WORKSPACE_DIR)
+    try:
+        out = subprocess.run(["ps", "-eo", "pid=,args="], capture_output=True, text=True, timeout=8)
+    except Exception:
+        return []
+    mine = os.getpid()
+    owners = p_live_backend_pids()
+    parents = p_ppid_map()
+    ghosts: List[int] = []
+    for line in (out.stdout or "").splitlines():
+        line = line.strip()
+        if needle not in line:
+            continue
+        head = line.split(None, 1)
+        if not head or not head[0].isdigit():
+            continue
+        pid = int(head[0])
+        if pid == mine:
+            continue
+        cur, owned, hops = pid, False, 0
+        while cur > 1 and hops < 24:
+            if cur in owners or cur == mine:
+                owned = True
+                break
+            cur = parents.get(cur, 0)
+            hops += 1
+        if not owned:
+            ghosts.append(pid)
+    return ghosts
+
+
+@typechecked
+def reap_ghost_runtimes() -> int:
+    """Reap them, leaves-first. Returns how many top-level processes were signalled.
+
+    Fire-and-forget by design: a machine where `ps` is restricted or a PID that vanishes between the
+    scan and the kill must never stop the backend from booting.
+    """
+    pids = find_ghost_runtime_pids()
+    if not pids:
+        return 0
+    logger.warning(
+        "reaping %d ghost app-runtime process(es) left by a previous session: %s",
+        len(pids), pids[:12],
+    )
+    killed = 0
+    for pid in pids:
+        try:
+            kill_descendant_tree(pid, "TERM")
+            os.kill(pid, 15)
+            killed += 1
+        except (ProcessLookupError, PermissionError, OSError):
+            continue
+    return killed

backend/tests/test_reap_ghost_runtimes.py

@@ -0,0 +1,60 @@
+"""A reaper that misjudges ownership kills working apps, so ownership is the thing under test.
+
+The first draft matched on the workspace path alone; a dry run on a live machine showed it would
+have killed 14 running app runtimes whose backend was up. These pin the discriminator.
+"""
+
+import os
+from unittest.mock import patch
+
+from backend.apps.outputs import reap_ghost_runtimes as mod
+
+
+def p_ps(pid_args: str, pid_ppid: str):
+    """Fake `ps` with two different outputs depending on the requested format."""
+    class R:
+        def __init__(self, out): self.stdout = out
+    def run(cmd, **kw):
+        return R(pid_args if "args=" in cmd[-1] or "pid=,args=" in " ".join(cmd) else pid_ppid)
+    return run
+
+
+def test_runtime_owned_by_a_live_backend_is_never_reaped():
+    ws = os.path.abspath(mod.WORKSPACE_DIR)
+    args = f"100 python -m uvicorn backend.main:app\n200 node {ws}/app/vite\n"
+    ppid = "100 1\n200 100\n"
+    with patch.object(mod.subprocess, "run", side_effect=p_ps(args, ppid)):
+        assert mod.find_ghost_runtime_pids() == []
+
+
+def test_runtime_whose_backend_died_is_reaped():
+    ws = os.path.abspath(mod.WORKSPACE_DIR)
+    args = f"200 node {ws}/app/vite\n"          # no uvicorn anywhere
+    ppid = "200 1\n"                             # reparented to init
+    with patch.object(mod.subprocess, "run", side_effect=p_ps(args, ppid)):
+        assert mod.find_ghost_runtime_pids() == [200]
+
+
+def test_ownership_is_inherited_through_the_bash_wrapper():
+    """run.sh sits between the backend and vite; the walk must climb past it."""
+    ws = os.path.abspath(mod.WORKSPACE_DIR)
+    args = f"100 python -m uvicorn backend.main:app\n150 bash run.sh\n200 node {ws}/app/vite\n"
+    ppid = "100 1\n150 100\n200 150\n"
+    with patch.object(mod.subprocess, "run", side_effect=p_ps(args, ppid)):
+        assert mod.find_ghost_runtime_pids() == []
+
+
+def test_unrelated_processes_are_never_matched():
+    """A user's own npm dev server elsewhere on the machine must be invisible to this."""
+    args = "300 node /Users/someone/other-project/vite\n"
+    ppid = "300 1\n"
+    with patch.object(mod.subprocess, "run", side_effect=p_ps(args, ppid)):
+        assert mod.find_ghost_runtime_pids() == []
+
+
+def test_a_broken_ps_reaps_nothing_rather_than_guessing():
+    def boom(*a, **k):
+        raise OSError("ps unavailable")
+    with patch.object(mod.subprocess, "run", side_effect=boom):
+        assert mod.find_ghost_runtime_pids() == []
+        assert mod.reap_ghost_runtimes() == 0