[eric] service: a completed shutdown stands the hard-exit fuse down; it was firing os._exit(0) inside the test runner and silently killing 42% of the suite

backend/apps/service/service.py

@@ -256,6 +256,12 @@ async def service_lifespan():
     except Exception:
         pass
 
+    # Shutdown got all the way here on its own, so stand the fuse down before it fires on a live process.
+    try:
+        from backend.apps.service.shutdown_fuse import disarm_shutdown_fuse
+        disarm_shutdown_fuse()
+    except Exception:
+        pass
     logger.info("Service shut down")
 
 

backend/apps/service/shutdown_fuse.py

@@ -6,7 +6,7 @@ tree and exits. A daemon thread, so a wedged event loop cannot block it."""
 import os
 import subprocess
 import threading
-from typing import List
+from typing import List, Optional
 
 from typeguard import typechecked
 
@@ -48,11 +48,39 @@ def p_burn() -> None:
     os._exit(0)
 
 
+p_armed: Optional[threading.Timer] = None
+
+
 @typechecked
 def arm_shutdown_fuse() -> None:
     """Called at lifespan-shutdown START (already past TERM), so no signal handling: just the timer. Touching signal.signal here would clobber uvicorn's asyncio-installed handlers."""
+    global p_armed
     if os.name == "nt":
         return
-    timer = threading.Timer(FUSE_S, p_burn)
-    timer.daemon = True
-    timer.start()
+    disarm_shutdown_fuse()
+    p_armed = threading.Timer(FUSE_S, p_burn)
+    p_armed.daemon = True
+    p_armed.start()
+
+
+@typechecked
+def fuse_armed() -> bool:
+    """True while a fuse is still going to fire. cancel() only sets Timer.finished and leaves the
+    thread alive for a moment, so liveness is the wrong question to ask."""
+    return p_armed is not None and not p_armed.finished.is_set()
+
+
+@typechecked
+def disarm_shutdown_fuse() -> None:
+    """Shutdown finished on its own, so the fuse has nothing left to save and must not go off.
+
+    Nothing used to disarm it. Harmless in production (the process is leaving anyway) but lethal
+    anywhere the app's lifespan runs and the process keeps living: in the backend test suite one
+    lifespan exit armed a fuse that detonated 10s later mid-run, SIGKILLing children and calling
+    os._exit(0). pytest died with no summary and exit code 0, so ~42% of the suite silently never
+    ran and the run still looked like it had finished.
+    """
+    global p_armed
+    if p_armed is not None:
+        p_armed.cancel()
+        p_armed = None

backend/tests/test_shutdown_fuse_disarms.py

@@ -0,0 +1,46 @@
+"""A completed shutdown must stand the hard-exit fuse down.
+
+The fuse (ENG-223) exists so a wedged quit cannot strand the process tree: 10s after shutdown starts
+it SIGKILLs every descendant and calls os._exit(0). Nothing ever cancelled it. In production that is
+invisible, because the process is leaving anyway. Anywhere the app's lifespan runs inside a process
+that keeps living, it is lethal: one lifespan exit in the backend test suite armed a fuse that fired
+mid-run, and pytest died with NO summary and exit code 0, so roughly 42% of the suite silently never
+ran while the run still looked like it had finished. Same class as ENG-219 and the unmarked-async
+skip: a green-looking run that never executed.
+"""
+
+from backend.apps.service.shutdown_fuse import arm_shutdown_fuse, disarm_shutdown_fuse, fuse_armed
+
+
+def test_arming_then_disarming_leaves_no_live_timer() -> None:
+    arm_shutdown_fuse()
+    assert fuse_armed() is True, "arming must actually start the fuse"
+    disarm_shutdown_fuse()
+    assert fuse_armed() is False, "a completed shutdown left a live os._exit timer behind"
+
+
+def test_disarming_without_arming_is_harmless() -> None:
+    disarm_shutdown_fuse()
+    disarm_shutdown_fuse()
+    assert fuse_armed() is False
+
+
+def test_arming_twice_does_not_leave_an_orphan_timer() -> None:
+    """The second arm must not abandon the first; an un-cancellable timer is exactly the thing that
+    kills the runner later, long after the code that armed it has been forgotten."""
+    arm_shutdown_fuse()
+    arm_shutdown_fuse()
+    assert fuse_armed() is True
+    disarm_shutdown_fuse()
+    assert fuse_armed() is False
+
+
+def test_the_service_lifespan_disarms_on_a_clean_shutdown() -> None:
+    """Wire-check the real caller, not just the helper: the fuse is armed at the top of the shutdown
+    block and must be stood down at the bottom of the same block."""
+    import inspect
+    from backend.apps.service import service as svc
+    src = inspect.getsource(svc.service_lifespan)
+    assert "arm_shutdown_fuse()" in src
+    assert "disarm_shutdown_fuse()" in src, "the lifespan arms the fuse and never stands it down"
+    assert src.index("arm_shutdown_fuse()") < src.index("disarm_shutdown_fuse()")