[eric] apps: a frozen app runtime is thawed before it is reaped, so a parked ghost stops surviving every TERM we send it

backend/apps/outputs/reap_ghost_runtimes.py

@@ -12,7 +12,9 @@ cannot legitimately belong to us: we have not started any yet.
 
 import logging
 import os
+import signal
 import subprocess
+import time
 from typing import List
 
 from typeguard import typechecked
@@ -22,6 +24,10 @@ from backend.config.paths import OUTPUTS_WORKSPACE_DIR as WORKSPACE_DIR
 
 logger = logging.getLogger(__name__)
 
+# Grace between TERM and KILL. Long enough for a run.sh EXIT trap to clean up its ports, short enough
+# that boot does not visibly stall on it.
+REAP_GRACE_SECONDS = float(os.environ.get("OSW_REAP_GRACE_SECONDS", "1.5"))
+
 
 @typechecked
 def p_live_backend_pids() -> set:
@@ -156,10 +162,32 @@ def reap_ghost_runtimes() -> int:
     )
     killed = 0
     for pid in pids:
+        try:
+            # THAW FIRST. Idle app runtimes are frozen with SIGSTOP, and a stopped process never
+            # handles SIGTERM: it just queues it and stays alive forever. Measured live, a frozen
+            # ghost that had survived every reap for 2 days 21 hours.
+            kill_descendant_tree(pid, "CONT")
+            os.kill(pid, signal.SIGCONT)
+        except (ProcessLookupError, PermissionError, OSError):
+            pass
         try:
             kill_descendant_tree(pid, "TERM")
-            os.kill(pid, 15)
+            os.kill(pid, signal.SIGTERM)
             killed += 1
         except (ProcessLookupError, PermissionError, OSError):
             continue
+    # A ghost's run.sh traps EXIT but not TERM, so give the tree a moment, then take out whatever
+    # ignored us. A ghost has no work worth protecting, so escalation is always the right call.
+    time.sleep(REAP_GRACE_SECONDS)
+    for pid in pids:
+        try:
+            os.kill(pid, 0)
+        except OSError:
+            continue
+        try:
+            kill_descendant_tree(pid, "KILL")
+            os.kill(pid, signal.SIGKILL)
+            logger.warning("ghost %d ignored TERM; escalated to KILL", pid)
+        except (ProcessLookupError, PermissionError, OSError):
+            continue
     return killed

backend/tests/test_reap_ghost_runtimes.py

@@ -161,3 +161,27 @@ def test_a_cwd_orphan_owned_by_a_live_backend_is_spared(monkeypatch):
         stdout = "50 python -m uvicorn backend.main:app\n900 python3 -u backend.py\n"
     monkeypatch.setattr(rg.subprocess, "run", lambda *a, **k: P_Out())
     assert rg.find_ghost_runtime_pids() == [], "a live backend's own app runtime must never be killed"
+
+
+def test_a_frozen_ghost_is_thawed_before_being_signalled(monkeypatch):
+    """Idle app runtimes are parked with SIGSTOP, and a STOPPED process never handles SIGTERM: it
+    queues it and lives forever. Found live as a frozen `bash run.sh` that had survived every reap
+    for 2 days 21 hours. CONT must precede TERM, and anything still breathing gets KILL."""
+    import signal as sg
+    from backend.apps.outputs import reap_ghost_runtimes as rg
+    monkeypatch.setattr(rg, "find_ghost_runtime_pids", lambda: [4242])
+    monkeypatch.setattr(rg, "REAP_GRACE_SECONDS", 0.0)
+    monkeypatch.setattr(rg, "kill_descendant_tree", lambda pid, sig: None)
+    sent = []
+    alive = {4242: True}
+    def p_kill(pid, sig):
+        if sig == 0:
+            if not alive.get(pid): raise ProcessLookupError()
+            return
+        sent.append(sig)
+        if sig == sg.SIGKILL: alive[pid] = False
+    monkeypatch.setattr(rg.os, "kill", p_kill)
+    rg.reap_ghost_runtimes()
+    assert sg.SIGCONT in sent, "a stopped ghost never receives TERM unless it is thawed first"
+    assert sent.index(sg.SIGCONT) < sent.index(sg.SIGTERM), "CONT must come before TERM"
+    assert sg.SIGKILL in sent, "a ghost that ignored TERM must be escalated, not left running"