diff --git a/backend/apps/outputs/outputs.py b/backend/apps/outputs/outputs.py index 604e3d2b..ef2b5920 100644 --- a/backend/apps/outputs/outputs.py +++ b/backend/apps/outputs/outputs.py @@ -63,6 +63,17 @@ async def outputs_lifespan(): recover_orphaned_apps() except Exception: logger.exception("orphaned-app recovery failed; apps stay hidden but nothing else breaks") + # Ghosts from a session that died badly keep running forever: stop_all only fires on a clean + # shutdown, and the port-collision path routes AROUND a squatter instead of killing it. Measured + # on a dev box: runtimes still alive after 2 days 19 hours. Boot is the one safe moment, since we + # have not spawned any of our own yet. + try: + from backend.apps.outputs.reap_ghost_runtimes import reap_ghost_runtimes + ghosts = reap_ghost_runtimes() + if ghosts: + logger.warning("outputs lifespan: reaped %d ghost runtime(s) from a previous session", ghosts) + except Exception: + logger.exception("ghost-runtime reap failed; stale processes stay but boot continues") try: yield finally: diff --git a/backend/apps/outputs/reap_ghost_runtimes.py b/backend/apps/outputs/reap_ghost_runtimes.py new file mode 100644 index 00000000..427edb3f --- /dev/null +++ b/backend/apps/outputs/reap_ghost_runtimes.py @@ -0,0 +1,120 @@ +"""Kill app-runtime processes left behind by a previous OpenSwarm that died badly. + +`stop_all()` reaps runtimes on a CLEAN shutdown. A crash, a SIGKILL, or a force-quit skips it, and +every `bash run.sh` plus its vite/uvicorn descendants reparents to PID 1 and keeps running: measured +on a dev machine, ghosts had been alive for **2 days 19 hours**, still holding their ports. The only +existing handling reallocates around a ghost that squats a port, so the ghost never dies at all and +they accumulate across sessions. + +This runs at startup, before any runtime is spawned, which is the one moment when a workspace process +cannot legitimately belong to us: we have not started any yet. +""" + +import logging +import os +import subprocess +from typing import List + +from typeguard import typechecked + +from backend.apps.outputs.runtime_proc import kill_descendant_tree +from backend.config.paths import OUTPUTS_WORKSPACE_DIR as WORKSPACE_DIR + +logger = logging.getLogger(__name__) + + +@typechecked +def p_live_backend_pids() -> set: + """PIDs of every running backend. A workspace process descended from one of these is ALIVE and + owned, not a ghost; a first draft of this reaper matched on the workspace path alone and would + have killed 14 working app runtimes on a machine where the owning backend was up.""" + try: + out = subprocess.run(["ps", "-eo", "pid=,args="], capture_output=True, text=True, timeout=8) + except Exception: + return set() + pids = set() + for line in (out.stdout or "").splitlines(): + if "uvicorn" not in line or "backend.main" not in line: + continue + head = line.strip().split(None, 1) + if head and head[0].isdigit(): + pids.add(int(head[0])) + return pids + + +@typechecked +def p_ppid_map() -> dict: + try: + out = subprocess.run(["ps", "-eo", "pid=,ppid="], capture_output=True, text=True, timeout=8) + except Exception: + return {} + m = {} + for line in (out.stdout or "").splitlines(): + parts = line.split() + if len(parts) == 2 and parts[0].isdigit() and parts[1].isdigit(): + m[int(parts[0])] = int(parts[1]) + return m + + +@typechecked +def find_ghost_runtime_pids() -> List[int]: + """PIDs of workspace processes that NO live backend owns. + + Matched on the absolute workspace path, so an unrelated `npm run dev` elsewhere is never touched, + then filtered by walking each candidate's ancestry: if a live backend is anywhere above it, it is + someone's working app and is left alone. + """ + needle = os.path.abspath(WORKSPACE_DIR) + try: + out = subprocess.run(["ps", "-eo", "pid=,args="], capture_output=True, text=True, timeout=8) + except Exception: + return [] + mine = os.getpid() + owners = p_live_backend_pids() + parents = p_ppid_map() + ghosts: List[int] = [] + for line in (out.stdout or "").splitlines(): + line = line.strip() + if needle not in line: + continue + head = line.split(None, 1) + if not head or not head[0].isdigit(): + continue + pid = int(head[0]) + if pid == mine: + continue + cur, owned, hops = pid, False, 0 + while cur > 1 and hops < 24: + if cur in owners or cur == mine: + owned = True + break + cur = parents.get(cur, 0) + hops += 1 + if not owned: + ghosts.append(pid) + return ghosts + + +@typechecked +def reap_ghost_runtimes() -> int: + """Reap them, leaves-first. Returns how many top-level processes were signalled. + + Fire-and-forget by design: a machine where `ps` is restricted or a PID that vanishes between the + scan and the kill must never stop the backend from booting. + """ + pids = find_ghost_runtime_pids() + if not pids: + return 0 + logger.warning( + "reaping %d ghost app-runtime process(es) left by a previous session: %s", + len(pids), pids[:12], + ) + killed = 0 + for pid in pids: + try: + kill_descendant_tree(pid, "TERM") + os.kill(pid, 15) + killed += 1 + except (ProcessLookupError, PermissionError, OSError): + continue + return killed diff --git a/backend/tests/test_reap_ghost_runtimes.py b/backend/tests/test_reap_ghost_runtimes.py new file mode 100644 index 00000000..0ab4ae63 --- /dev/null +++ b/backend/tests/test_reap_ghost_runtimes.py @@ -0,0 +1,60 @@ +"""A reaper that misjudges ownership kills working apps, so ownership is the thing under test. + +The first draft matched on the workspace path alone; a dry run on a live machine showed it would +have killed 14 running app runtimes whose backend was up. These pin the discriminator. +""" + +import os +from unittest.mock import patch + +from backend.apps.outputs import reap_ghost_runtimes as mod + + +def p_ps(pid_args: str, pid_ppid: str): + """Fake `ps` with two different outputs depending on the requested format.""" + class R: + def __init__(self, out): self.stdout = out + def run(cmd, **kw): + return R(pid_args if "args=" in cmd[-1] or "pid=,args=" in " ".join(cmd) else pid_ppid) + return run + + +def test_runtime_owned_by_a_live_backend_is_never_reaped(): + ws = os.path.abspath(mod.WORKSPACE_DIR) + args = f"100 python -m uvicorn backend.main:app\n200 node {ws}/app/vite\n" + ppid = "100 1\n200 100\n" + with patch.object(mod.subprocess, "run", side_effect=p_ps(args, ppid)): + assert mod.find_ghost_runtime_pids() == [] + + +def test_runtime_whose_backend_died_is_reaped(): + ws = os.path.abspath(mod.WORKSPACE_DIR) + args = f"200 node {ws}/app/vite\n" # no uvicorn anywhere + ppid = "200 1\n" # reparented to init + with patch.object(mod.subprocess, "run", side_effect=p_ps(args, ppid)): + assert mod.find_ghost_runtime_pids() == [200] + + +def test_ownership_is_inherited_through_the_bash_wrapper(): + """run.sh sits between the backend and vite; the walk must climb past it.""" + ws = os.path.abspath(mod.WORKSPACE_DIR) + args = f"100 python -m uvicorn backend.main:app\n150 bash run.sh\n200 node {ws}/app/vite\n" + ppid = "100 1\n150 100\n200 150\n" + with patch.object(mod.subprocess, "run", side_effect=p_ps(args, ppid)): + assert mod.find_ghost_runtime_pids() == [] + + +def test_unrelated_processes_are_never_matched(): + """A user's own npm dev server elsewhere on the machine must be invisible to this.""" + args = "300 node /Users/someone/other-project/vite\n" + ppid = "300 1\n" + with patch.object(mod.subprocess, "run", side_effect=p_ps(args, ppid)): + assert mod.find_ghost_runtime_pids() == [] + + +def test_a_broken_ps_reaps_nothing_rather_than_guessing(): + def boom(*a, **k): + raise OSError("ps unavailable") + with patch.object(mod.subprocess, "run", side_effect=boom): + assert mod.find_ghost_runtime_pids() == [] + assert mod.reap_ghost_runtimes() == 0