diff --git a/backend/apps/outputs/reap_ghost_runtimes.py b/backend/apps/outputs/reap_ghost_runtimes.py index b053875a..a72344b9 100644 --- a/backend/apps/outputs/reap_ghost_runtimes.py +++ b/backend/apps/outputs/reap_ghost_runtimes.py @@ -12,7 +12,9 @@ cannot legitimately belong to us: we have not started any yet. import logging import os +import signal import subprocess +import time from typing import List from typeguard import typechecked @@ -22,6 +24,10 @@ from backend.config.paths import OUTPUTS_WORKSPACE_DIR as WORKSPACE_DIR logger = logging.getLogger(__name__) +# Grace between TERM and KILL. Long enough for a run.sh EXIT trap to clean up its ports, short enough +# that boot does not visibly stall on it. +REAP_GRACE_SECONDS = float(os.environ.get("OSW_REAP_GRACE_SECONDS", "1.5")) + @typechecked def p_live_backend_pids() -> set: @@ -156,10 +162,32 @@ def reap_ghost_runtimes() -> int: ) killed = 0 for pid in pids: + try: + # THAW FIRST. Idle app runtimes are frozen with SIGSTOP, and a stopped process never + # handles SIGTERM: it just queues it and stays alive forever. Measured live, a frozen + # ghost that had survived every reap for 2 days 21 hours. + kill_descendant_tree(pid, "CONT") + os.kill(pid, signal.SIGCONT) + except (ProcessLookupError, PermissionError, OSError): + pass try: kill_descendant_tree(pid, "TERM") - os.kill(pid, 15) + os.kill(pid, signal.SIGTERM) killed += 1 except (ProcessLookupError, PermissionError, OSError): continue + # A ghost's run.sh traps EXIT but not TERM, so give the tree a moment, then take out whatever + # ignored us. A ghost has no work worth protecting, so escalation is always the right call. + time.sleep(REAP_GRACE_SECONDS) + for pid in pids: + try: + os.kill(pid, 0) + except OSError: + continue + try: + kill_descendant_tree(pid, "KILL") + os.kill(pid, signal.SIGKILL) + logger.warning("ghost %d ignored TERM; escalated to KILL", pid) + except (ProcessLookupError, PermissionError, OSError): + continue return killed diff --git a/backend/tests/test_reap_ghost_runtimes.py b/backend/tests/test_reap_ghost_runtimes.py index f3f96f40..ec9647fa 100644 --- a/backend/tests/test_reap_ghost_runtimes.py +++ b/backend/tests/test_reap_ghost_runtimes.py @@ -161,3 +161,27 @@ def test_a_cwd_orphan_owned_by_a_live_backend_is_spared(monkeypatch): stdout = "50 python -m uvicorn backend.main:app\n900 python3 -u backend.py\n" monkeypatch.setattr(rg.subprocess, "run", lambda *a, **k: P_Out()) assert rg.find_ghost_runtime_pids() == [], "a live backend's own app runtime must never be killed" + + +def test_a_frozen_ghost_is_thawed_before_being_signalled(monkeypatch): + """Idle app runtimes are parked with SIGSTOP, and a STOPPED process never handles SIGTERM: it + queues it and lives forever. Found live as a frozen `bash run.sh` that had survived every reap + for 2 days 21 hours. CONT must precede TERM, and anything still breathing gets KILL.""" + import signal as sg + from backend.apps.outputs import reap_ghost_runtimes as rg + monkeypatch.setattr(rg, "find_ghost_runtime_pids", lambda: [4242]) + monkeypatch.setattr(rg, "REAP_GRACE_SECONDS", 0.0) + monkeypatch.setattr(rg, "kill_descendant_tree", lambda pid, sig: None) + sent = [] + alive = {4242: True} + def p_kill(pid, sig): + if sig == 0: + if not alive.get(pid): raise ProcessLookupError() + return + sent.append(sig) + if sig == sg.SIGKILL: alive[pid] = False + monkeypatch.setattr(rg.os, "kill", p_kill) + rg.reap_ghost_runtimes() + assert sg.SIGCONT in sent, "a stopped ghost never receives TERM unless it is thawed first" + assert sent.index(sg.SIGCONT) < sent.index(sg.SIGTERM), "CONT must come before TERM" + assert sg.SIGKILL in sent, "a ghost that ignored TERM must be escalated, not left running"