From 803297146899fc24ab7acac892a3ff5203bbaeae Mon Sep 17 00:00:00 2001 From: ciregenz Date: Wed, 12 Aug 2026 00:02:04 -0700 Subject: [PATCH] [eric] service: a completed shutdown stands the hard-exit fuse down; it was firing os._exit(0) inside the test runner and silently killing 42% of the suite (cherry picked from commit 25fb3677cadd7de09dbe62a9950ce31e669a91f4) --- backend/apps/service/service.py | 6 +++ backend/apps/service/shutdown_fuse.py | 36 ++++++++++++++-- backend/tests/test_shutdown_fuse_disarms.py | 46 +++++++++++++++++++++ 3 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 backend/tests/test_shutdown_fuse_disarms.py diff --git a/backend/apps/service/service.py b/backend/apps/service/service.py index 6ff1c36f..1971a2fa 100644 --- a/backend/apps/service/service.py +++ b/backend/apps/service/service.py @@ -256,6 +256,12 @@ async def service_lifespan(): except Exception: pass + # Shutdown got all the way here on its own, so stand the fuse down before it fires on a live process. + try: + from backend.apps.service.shutdown_fuse import disarm_shutdown_fuse + disarm_shutdown_fuse() + except Exception: + pass logger.info("Service shut down") diff --git a/backend/apps/service/shutdown_fuse.py b/backend/apps/service/shutdown_fuse.py index 36a3c464..b66799fa 100644 --- a/backend/apps/service/shutdown_fuse.py +++ b/backend/apps/service/shutdown_fuse.py @@ -6,7 +6,7 @@ tree and exits. A daemon thread, so a wedged event loop cannot block it.""" import os import subprocess import threading -from typing import List +from typing import List, Optional from typeguard import typechecked @@ -48,11 +48,39 @@ def p_burn() -> None: os._exit(0) +p_armed: Optional[threading.Timer] = None + + @typechecked def arm_shutdown_fuse() -> None: """Called at lifespan-shutdown START (already past TERM), so no signal handling: just the timer. Touching signal.signal here would clobber uvicorn's asyncio-installed handlers.""" + global p_armed if os.name == "nt": return - timer = threading.Timer(FUSE_S, p_burn) - timer.daemon = True - timer.start() + disarm_shutdown_fuse() + p_armed = threading.Timer(FUSE_S, p_burn) + p_armed.daemon = True + p_armed.start() + + +@typechecked +def fuse_armed() -> bool: + """True while a fuse is still going to fire. cancel() only sets Timer.finished and leaves the + thread alive for a moment, so liveness is the wrong question to ask.""" + return p_armed is not None and not p_armed.finished.is_set() + + +@typechecked +def disarm_shutdown_fuse() -> None: + """Shutdown finished on its own, so the fuse has nothing left to save and must not go off. + + Nothing used to disarm it. Harmless in production (the process is leaving anyway) but lethal + anywhere the app's lifespan runs and the process keeps living: in the backend test suite one + lifespan exit armed a fuse that detonated 10s later mid-run, SIGKILLing children and calling + os._exit(0). pytest died with no summary and exit code 0, so ~42% of the suite silently never + ran and the run still looked like it had finished. + """ + global p_armed + if p_armed is not None: + p_armed.cancel() + p_armed = None diff --git a/backend/tests/test_shutdown_fuse_disarms.py b/backend/tests/test_shutdown_fuse_disarms.py new file mode 100644 index 00000000..f81e4eef --- /dev/null +++ b/backend/tests/test_shutdown_fuse_disarms.py @@ -0,0 +1,46 @@ +"""A completed shutdown must stand the hard-exit fuse down. + +The fuse (ENG-223) exists so a wedged quit cannot strand the process tree: 10s after shutdown starts +it SIGKILLs every descendant and calls os._exit(0). Nothing ever cancelled it. In production that is +invisible, because the process is leaving anyway. Anywhere the app's lifespan runs inside a process +that keeps living, it is lethal: one lifespan exit in the backend test suite armed a fuse that fired +mid-run, and pytest died with NO summary and exit code 0, so roughly 42% of the suite silently never +ran while the run still looked like it had finished. Same class as ENG-219 and the unmarked-async +skip: a green-looking run that never executed. +""" + +from backend.apps.service.shutdown_fuse import arm_shutdown_fuse, disarm_shutdown_fuse, fuse_armed + + +def test_arming_then_disarming_leaves_no_live_timer() -> None: + arm_shutdown_fuse() + assert fuse_armed() is True, "arming must actually start the fuse" + disarm_shutdown_fuse() + assert fuse_armed() is False, "a completed shutdown left a live os._exit timer behind" + + +def test_disarming_without_arming_is_harmless() -> None: + disarm_shutdown_fuse() + disarm_shutdown_fuse() + assert fuse_armed() is False + + +def test_arming_twice_does_not_leave_an_orphan_timer() -> None: + """The second arm must not abandon the first; an un-cancellable timer is exactly the thing that + kills the runner later, long after the code that armed it has been forgotten.""" + arm_shutdown_fuse() + arm_shutdown_fuse() + assert fuse_armed() is True + disarm_shutdown_fuse() + assert fuse_armed() is False + + +def test_the_service_lifespan_disarms_on_a_clean_shutdown() -> None: + """Wire-check the real caller, not just the helper: the fuse is armed at the top of the shutdown + block and must be stood down at the bottom of the same block.""" + import inspect + from backend.apps.service import service as svc + src = inspect.getsource(svc.service_lifespan) + assert "arm_shutdown_fuse()" in src + assert "disarm_shutdown_fuse()" in src, "the lifespan arms the fuse and never stands it down" + assert src.index("arm_shutdown_fuse()") < src.index("disarm_shutdown_fuse()")