From e90abb7bfc162ffc89cd8b19e8b1304367134bf2 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Thu, 20 Aug 2026 22:22:49 -0700 Subject: [PATCH] [eric] system: watchdog deaths blame no workflow, card tails not owed, 20s probe (ENG-366) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01En8dRGsJPLrJCQBEkTH4Mp --- .../manager/session/SessionPersistence.py | 4 +++- backend/apps/system/loop_liveness_watchdog.py | 23 ++++++++++++++++--- backend/apps/workflows/restart_loop_guard.py | 6 +++++ backend/tests/test_crash_auto_resume.py | 19 +++++++++++++++ backend/tests/test_loop_liveness_watchdog.py | 16 +++++++++++++ backend/tests/test_restart_loop_guard.py | 18 +++++++++++++++ 6 files changed, 82 insertions(+), 4 deletions(-) diff --git a/backend/apps/agents/manager/session/SessionPersistence.py b/backend/apps/agents/manager/session/SessionPersistence.py index bbf4ff40..8665bc7b 100644 --- a/backend/apps/agents/manager/session/SessionPersistence.py +++ b/backend/apps/agents/manager/session/SessionPersistence.py @@ -41,7 +41,9 @@ class SessionPersistence(AgentManagerProtocol): marked += 1 branch = data.get("active_branch_id") or "main" p_msgs = [m for m in data.get("messages", []) if (m.get("branch_id") or "main") == branch] - p_owed = bool(p_msgs) and p_msgs[-1].get("role") != "assistant" + p_tail_role = p_msgs[-1].get("role") if p_msgs else None + # A system-role tail is a card (overflow, exhausted, a notice): terminal unless a continuation was armed behind it, so a finished chat is never poked back to life (ENG-366). + p_owed = bool(p_msgs) and p_tail_role != "assistant" and (p_tail_role != "system" or bool(data.get("pending_continuation"))) if p_was_running and p_owed and data.get("closed_at") is None: count = int(data.get("crash_interrupt_count", 0) or 0) + 1 data["crash_interrupt_count"] = count diff --git a/backend/apps/system/loop_liveness_watchdog.py b/backend/apps/system/loop_liveness_watchdog.py index fc287e21..1a08e247 100644 --- a/backend/apps/system/loop_liveness_watchdog.py +++ b/backend/apps/system/loop_liveness_watchdog.py @@ -29,15 +29,27 @@ from backend.config.paths import DATA_ROOT logger = logging.getLogger(__name__) PROBE_INTERVAL_S = 30.0 -PROBE_TIMEOUT_S = 10.0 +# 20s, not 10s: a big session persist to a slow mount or the unwedger's 10s ps timeout can hold the loop that long and still be alive; a frozen loop is frozen forever, so the longer probe costs 30s of detection and nothing else (ENG-366). +PROBE_TIMEOUT_S = 20.0 MAX_STRIKES = 3 # 75 = EX_TEMPFAIL, hermes's "restart me" exit language; Electron respawns any non-zero exit. RESTART_EXIT_CODE = 75 DUMP_PATH = os.path.join(DATA_ROOT, "loop-watchdog-dump.log") +# Left behind by a watchdog exit so the next boot knows the last life died of a frozen loop, not of whatever workflow happened to be mid-fire. +WATCHDOG_EXIT_MARKER = os.path.join(DATA_ROOT, "loop-watchdog-exit") @typechecked -def p_dump_and_exit(strikes: int) -> None: +def consume_watchdog_exit_marker() -> bool: + try: + os.unlink(WATCHDOG_EXIT_MARKER) + return True + except OSError: + return False + + +@typechecked +def dump_and_exit(strikes: int) -> None: try: logger.critical(f"backend event loop missed {strikes} consecutive liveness probes; dumping stacks and exiting {RESTART_EXIT_CODE} so Electron respawns a working process") except Exception: @@ -54,6 +66,11 @@ def p_dump_and_exit(strikes: int) -> None: faulthandler.dump_traceback(all_threads=True) except Exception: pass + try: + with open(WATCHDOG_EXIT_MARKER, "w", encoding="utf-8") as fh: + fh.write(f"{time.time():.0f}\n") + except Exception: + pass os._exit(RESTART_EXIT_CODE) @@ -95,7 +112,7 @@ def start_loop_liveness_watchdog(loop: "asyncio.AbstractEventLoop") -> Optional[ strikes += 1 logger.warning(f"backend event loop missed liveness probe ({strikes}/{MAX_STRIKES})") if strikes >= MAX_STRIKES and not stop_event.is_set(): - p_dump_and_exit(strikes) + dump_and_exit(strikes) return try: diff --git a/backend/apps/workflows/restart_loop_guard.py b/backend/apps/workflows/restart_loop_guard.py index 76e766c7..024db0a4 100644 --- a/backend/apps/workflows/restart_loop_guard.py +++ b/backend/apps/workflows/restart_loop_guard.py @@ -19,6 +19,7 @@ from typing import Dict, List, Optional from typeguard import typechecked +from backend.apps.system.loop_liveness_watchdog import consume_watchdog_exit_marker from backend.apps.workflows.storage import DATA_DIR logger = logging.getLogger(__name__) @@ -64,7 +65,12 @@ def record_boot(now: Optional[float] = None) -> List[float]: boots = [float(t) for t in state.get("boots", []) if isinstance(t, (int, float)) and t >= ts - WINDOW_SECONDS] boots.append(ts) implicated = dict(state.get("implicated", {})) + # A death by our own loop watchdog is a frozen backend, not a workflow's doing; implicating whatever was mid-fire would pause innocent schedules (ENG-366). + p_watchdog_death = consume_watchdog_exit_marker() for wf_id in list(state.get("firing", {}) or {}): + if p_watchdog_death: + logger.warning(f"restart-loop guard: workflow {wf_id} was mid-fire when the loop watchdog restarted the backend; not implicated") + continue implicated[wf_id] = int(implicated.get(wf_id, 0)) + 1 logger.warning(f"restart-loop guard: workflow {wf_id} was mid-fire when the last backend life died (implication #{implicated[wf_id]})") p_save({"boots": boots, "implicated": implicated, "firing": {}}) diff --git a/backend/tests/test_crash_auto_resume.py b/backend/tests/test_crash_auto_resume.py index 1c8af107..b8205b4a 100644 --- a/backend/tests/test_crash_auto_resume.py +++ b/backend/tests/test_crash_auto_resume.py @@ -93,3 +93,22 @@ def test_resume_failure_is_per_session_and_non_fatal(manager, tmp_path, monkeypa monkeypatch.setattr(manager, "send_message", p_flaky_send) asyncio.run(manager.auto_resume_crashed_turns()) assert len(sent) == 1 + + +def test_a_terminal_system_card_tail_is_not_owed(manager, tmp_path, monkeypatch): + """ENG-366: an overflow card, an exhausted note or a notice is the END of that ask; a dirty death + after it must not poke a finished chat back to life with a hidden continue.""" + p_write_session(tmp_path, monkeypatch, "s-card", "running", + [p_msg("user", "do work"), p_msg("tool_call"), p_msg("system", "This chat exceeded the model's context window")]) + asyncio.run(manager.reconcile_on_startup()) + assert manager.crash_resume_queue == [] + + +def test_a_system_notice_with_an_armed_continuation_is_owed(manager, tmp_path, monkeypatch): + data = p_write_session(tmp_path, monkeypatch, "s-wait", "running", + [p_msg("user", "do work"), p_msg("system", "token rotated, retrying in a minute")]) + data["pending_continuation"] = True + with open(os.path.join(str(tmp_path), "s-wait.json"), "w") as f: + json.dump(data, f) + asyncio.run(manager.reconcile_on_startup()) + assert manager.crash_resume_queue == ["s-wait"] diff --git a/backend/tests/test_loop_liveness_watchdog.py b/backend/tests/test_loop_liveness_watchdog.py index 50e9fdbf..dfc64da1 100644 --- a/backend/tests/test_loop_liveness_watchdog.py +++ b/backend/tests/test_loop_liveness_watchdog.py @@ -85,3 +85,19 @@ print("SURVIVED") """ r = p_run_child(code) assert r.returncode == 0 and "SURVIVED" in r.stdout + + +def test_the_exit_leaves_a_marker_the_next_boot_can_consume(tmp_path, monkeypatch): + """ENG-366: record_boot reads this marker to tell a watchdog restart from a workflow-caused death.""" + import os + marker = str(tmp_path / "loop-watchdog-exit") + monkeypatch.setattr(w, "WATCHDOG_EXIT_MARKER", marker) + monkeypatch.setattr(w, "DUMP_PATH", str(tmp_path / "dump.log")) + exits = [] + monkeypatch.setattr(os, "_exit", lambda code: exits.append(code)) + w.dump_and_exit(3) + assert exits == [w.RESTART_EXIT_CODE] + assert os.path.exists(marker) + assert w.consume_watchdog_exit_marker() is True + assert not os.path.exists(marker) + assert w.consume_watchdog_exit_marker() is False diff --git a/backend/tests/test_restart_loop_guard.py b/backend/tests/test_restart_loop_guard.py index c072f5b5..66040d30 100644 --- a/backend/tests/test_restart_loop_guard.py +++ b/backend/tests/test_restart_loop_guard.py @@ -74,3 +74,21 @@ def test_unwritable_dir_fails_open(tmp_path, monkeypatch): g.record_boot() g.mark_firing("wf1") g.clear_firing("wf1") + + +def test_a_watchdog_death_implicates_nobody(tmp_path, monkeypatch): + """ENG-366: the loop watchdog's own hard exit is a frozen backend, not a workflow's doing; a + workflow mid-fire at three such deaths in a row must stay unimplicated and untripped, while the + same three deaths without the marker trip it (the control).""" + p_fresh(tmp_path, monkeypatch) + monkeypatch.setattr(g, "consume_watchdog_exit_marker", lambda: True) + for t in (1000.0, 1010.0, 1020.0): + g.mark_firing("wf1") + g.record_boot(now=t) + assert g.is_tripped("wf1", now=1020.0) is False + p_fresh(tmp_path, monkeypatch) + monkeypatch.setattr(g, "consume_watchdog_exit_marker", lambda: False) + for t in (1000.0, 1010.0, 1020.0): + g.mark_firing("wf1") + g.record_boot(now=t) + assert g.is_tripped("wf1", now=1020.0) is True