[eric] system: watchdog deaths blame no workflow, card tails not owed, 20s probe (ENG-366)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01En8dRGsJPLrJCQBEkTH4Mp

backend/apps/agents/manager/session/SessionPersistence.py

@@ -41,7 +41,9 @@ class SessionPersistence(AgentManagerProtocol):
                 marked += 1
                 branch = data.get("active_branch_id") or "main"
                 p_msgs = [m for m in data.get("messages", []) if (m.get("branch_id") or "main") == branch]
-                p_owed = bool(p_msgs) and p_msgs[-1].get("role") != "assistant"
+                p_tail_role = p_msgs[-1].get("role") if p_msgs else None
+                # A system-role tail is a card (overflow, exhausted, a notice): terminal unless a continuation was armed behind it, so a finished chat is never poked back to life (ENG-366).
+                p_owed = bool(p_msgs) and p_tail_role != "assistant" and (p_tail_role != "system" or bool(data.get("pending_continuation")))
                 if p_was_running and p_owed and data.get("closed_at") is None:
                     count = int(data.get("crash_interrupt_count", 0) or 0) + 1
                     data["crash_interrupt_count"] = count

backend/apps/system/loop_liveness_watchdog.py

@@ -29,15 +29,27 @@ from backend.config.paths import DATA_ROOT
 logger = logging.getLogger(__name__)
 
 PROBE_INTERVAL_S = 30.0
-PROBE_TIMEOUT_S = 10.0
+# 20s, not 10s: a big session persist to a slow mount or the unwedger's 10s ps timeout can hold the loop that long and still be alive; a frozen loop is frozen forever, so the longer probe costs 30s of detection and nothing else (ENG-366).
+PROBE_TIMEOUT_S = 20.0
 MAX_STRIKES = 3
 # 75 = EX_TEMPFAIL, hermes's "restart me" exit language; Electron respawns any non-zero exit.
 RESTART_EXIT_CODE = 75
 DUMP_PATH = os.path.join(DATA_ROOT, "loop-watchdog-dump.log")
+# Left behind by a watchdog exit so the next boot knows the last life died of a frozen loop, not of whatever workflow happened to be mid-fire.
+WATCHDOG_EXIT_MARKER = os.path.join(DATA_ROOT, "loop-watchdog-exit")
 
 
 @typechecked
-def p_dump_and_exit(strikes: int) -> None:
+def consume_watchdog_exit_marker() -> bool:
+    try:
+        os.unlink(WATCHDOG_EXIT_MARKER)
+        return True
+    except OSError:
+        return False
+
+
+@typechecked
+def dump_and_exit(strikes: int) -> None:
     try:
         logger.critical(f"backend event loop missed {strikes} consecutive liveness probes; dumping stacks and exiting {RESTART_EXIT_CODE} so Electron respawns a working process")
     except Exception:
@@ -54,6 +66,11 @@ def p_dump_and_exit(strikes: int) -> None:
         faulthandler.dump_traceback(all_threads=True)
     except Exception:
         pass
+    try:
+        with open(WATCHDOG_EXIT_MARKER, "w", encoding="utf-8") as fh:
+            fh.write(f"{time.time():.0f}\n")
+    except Exception:
+        pass
     os._exit(RESTART_EXIT_CODE)
 
 
@@ -95,7 +112,7 @@ def start_loop_liveness_watchdog(loop: "asyncio.AbstractEventLoop") -> Optional[
             strikes += 1
             logger.warning(f"backend event loop missed liveness probe ({strikes}/{MAX_STRIKES})")
             if strikes >= MAX_STRIKES and not stop_event.is_set():
-                p_dump_and_exit(strikes)
+                dump_and_exit(strikes)
                 return
 
     try:

backend/apps/workflows/restart_loop_guard.py

@@ -19,6 +19,7 @@ from typing import Dict, List, Optional
 
 from typeguard import typechecked
 
+from backend.apps.system.loop_liveness_watchdog import consume_watchdog_exit_marker
 from backend.apps.workflows.storage import DATA_DIR
 
 logger = logging.getLogger(__name__)
@@ -64,7 +65,12 @@ def record_boot(now: Optional[float] = None) -> List[float]:
     boots = [float(t) for t in state.get("boots", []) if isinstance(t, (int, float)) and t >= ts - WINDOW_SECONDS]
     boots.append(ts)
     implicated = dict(state.get("implicated", {}))
+    # A death by our own loop watchdog is a frozen backend, not a workflow's doing; implicating whatever was mid-fire would pause innocent schedules (ENG-366).
+    p_watchdog_death = consume_watchdog_exit_marker()
     for wf_id in list(state.get("firing", {}) or {}):
+        if p_watchdog_death:
+            logger.warning(f"restart-loop guard: workflow {wf_id} was mid-fire when the loop watchdog restarted the backend; not implicated")
+            continue
         implicated[wf_id] = int(implicated.get(wf_id, 0)) + 1
         logger.warning(f"restart-loop guard: workflow {wf_id} was mid-fire when the last backend life died (implication #{implicated[wf_id]})")
     p_save({"boots": boots, "implicated": implicated, "firing": {}})

backend/tests/test_crash_auto_resume.py

@@ -93,3 +93,22 @@ def test_resume_failure_is_per_session_and_non_fatal(manager, tmp_path, monkeypa
     monkeypatch.setattr(manager, "send_message", p_flaky_send)
     asyncio.run(manager.auto_resume_crashed_turns())
     assert len(sent) == 1
+
+
+def test_a_terminal_system_card_tail_is_not_owed(manager, tmp_path, monkeypatch):
+    """ENG-366: an overflow card, an exhausted note or a notice is the END of that ask; a dirty death
+    after it must not poke a finished chat back to life with a hidden continue."""
+    p_write_session(tmp_path, monkeypatch, "s-card", "running",
+                    [p_msg("user", "do work"), p_msg("tool_call"), p_msg("system", "This chat exceeded the model's context window")])
+    asyncio.run(manager.reconcile_on_startup())
+    assert manager.crash_resume_queue == []
+
+
+def test_a_system_notice_with_an_armed_continuation_is_owed(manager, tmp_path, monkeypatch):
+    data = p_write_session(tmp_path, monkeypatch, "s-wait", "running",
+                           [p_msg("user", "do work"), p_msg("system", "token rotated, retrying in a minute")])
+    data["pending_continuation"] = True
+    with open(os.path.join(str(tmp_path), "s-wait.json"), "w") as f:
+        json.dump(data, f)
+    asyncio.run(manager.reconcile_on_startup())
+    assert manager.crash_resume_queue == ["s-wait"]

backend/tests/test_loop_liveness_watchdog.py

@@ -85,3 +85,19 @@ print("SURVIVED")
 """
     r = p_run_child(code)
     assert r.returncode == 0 and "SURVIVED" in r.stdout
+
+
+def test_the_exit_leaves_a_marker_the_next_boot_can_consume(tmp_path, monkeypatch):
+    """ENG-366: record_boot reads this marker to tell a watchdog restart from a workflow-caused death."""
+    import os
+    marker = str(tmp_path / "loop-watchdog-exit")
+    monkeypatch.setattr(w, "WATCHDOG_EXIT_MARKER", marker)
+    monkeypatch.setattr(w, "DUMP_PATH", str(tmp_path / "dump.log"))
+    exits = []
+    monkeypatch.setattr(os, "_exit", lambda code: exits.append(code))
+    w.dump_and_exit(3)
+    assert exits == [w.RESTART_EXIT_CODE]
+    assert os.path.exists(marker)
+    assert w.consume_watchdog_exit_marker() is True
+    assert not os.path.exists(marker)
+    assert w.consume_watchdog_exit_marker() is False

backend/tests/test_restart_loop_guard.py

@@ -74,3 +74,21 @@ def test_unwritable_dir_fails_open(tmp_path, monkeypatch):
     g.record_boot()
     g.mark_firing("wf1")
     g.clear_firing("wf1")
+
+
+def test_a_watchdog_death_implicates_nobody(tmp_path, monkeypatch):
+    """ENG-366: the loop watchdog's own hard exit is a frozen backend, not a workflow's doing; a
+    workflow mid-fire at three such deaths in a row must stay unimplicated and untripped, while the
+    same three deaths without the marker trip it (the control)."""
+    p_fresh(tmp_path, monkeypatch)
+    monkeypatch.setattr(g, "consume_watchdog_exit_marker", lambda: True)
+    for t in (1000.0, 1010.0, 1020.0):
+        g.mark_firing("wf1")
+        g.record_boot(now=t)
+    assert g.is_tripped("wf1", now=1020.0) is False
+    p_fresh(tmp_path, monkeypatch)
+    monkeypatch.setattr(g, "consume_watchdog_exit_marker", lambda: False)
+    for t in (1000.0, 1010.0, 1020.0):
+        g.mark_firing("wf1")
+        g.record_boot(now=t)
+    assert g.is_tripped("wf1", now=1020.0) is True