[eric] agents: a frozen core tool recovers in 25s and the agent retries the step it lost

backend/apps/agents/manager/streaming/unwedge_sidecar.py

@@ -18,10 +18,11 @@ from typeguard import typechecked
 
 logger = logging.getLogger(__name__)
 
-# Generous by design: the quick class answers in milliseconds (memory, settings, schedule CRUD),
-# so a minute of silence is not slowness, it is a frozen process. Raising this only lengthens the
-# outage a user sits through; lowering it risks shooting a healthy-but-busy sidecar.
-WEDGE_SECONDS = 75.0
+# The quick class answers in MILLISECONDS (memory, settings, schedule CRUD), so 25s is already a
+# thousandfold margin: long enough that nothing healthy trips it, short enough that a user reads
+# the recovery as a hiccup rather than a hang. Anything that legitimately blocks (a human, a
+# delegated run) is exempt by name below, so this deadline never races real work.
+WEDGE_SECONDS = 25.0
 
 P_CORE_PREFIX = "mcp__openswarm-core__"
 
@@ -70,6 +71,27 @@ def find_sidecar_pids(session_id: str) -> list:
     return pids
 
 
+RETRY_PROMPT = (
+    "Your last tool call never returned because its server had frozen; that server has been "
+    "restarted and works now. Retry that one step, then carry on where you left off."
+)
+
+
+@typechecked
+def arm_retry(session: object) -> bool:
+    """Queue one hidden continuation so the agent redoes the lost step. Reuses the seam the
+    silent-quit nudge already owns, and never stacks on a continuation that is already pending.
+    Takes the live session the hook holds; there is no global registry to look one up in."""
+    if session is None or getattr(session, "pending_continuation", False):
+        return False
+    try:
+        session.pending_continuation = True          # type: ignore[attr-defined]
+        session.pending_continuation_prompt = RETRY_PROMPT   # type: ignore[attr-defined]
+        return True
+    except Exception:
+        return False
+
+
 @typechecked
 def unwedge(session_id: str, tool_name: str, outstanding_s: float) -> int:
     """CONT first: a STOPPED process queues TERM forever (the ghost-reaper lesson, ENG-196), so
@@ -120,7 +142,9 @@ def arm_wedge_watchdog(ctx: object, tool_use_id: str, tool_name: str) -> None:
         session_id = getattr(ctx, "session_id", "")
         if not session_id:
             return
-        # ps + kill are blocking; keep them off the event loop.
+        # ps + kill are blocking; keep them off the event loop. The retry is armed on the LIVE
+        # session object the hook holds, so the agent redoes the step the frozen server swallowed.
         loop.run_in_executor(None, unwedge, session_id, tool_name, time.time() - started)
+        arm_retry(getattr(ctx, "session", None))
 
     loop.call_later(WEDGE_SECONDS, p_check)

backend/tests/test_unwedge_sidecar.py

@@ -36,8 +36,48 @@ def test_non_core_tools_are_never_watched():
     assert not is_quick_core_tool("mcp__github__create_issue")
 
 
-def test_the_deadline_is_generous_not_twitchy():
-    assert WEDGE_SECONDS >= 60, "a quick tool answers in ms; anything under a minute risks healthy kills"
+def test_the_deadline_sits_between_twitchy_and_a_hang():
+    # The quick class answers in milliseconds, so the floor is about not shooting a healthy-but-busy
+    # sidecar, and the ceiling is about the user not reading recovery as a hang.
+    assert 10 <= WEDGE_SECONDS <= 45, (
+        f"{WEDGE_SECONDS}s is outside the band: under ~10s risks killing healthy work, "
+        "over ~45s and the user has already given up"
+    )
+
+
+def test_a_retry_is_armed_so_the_lost_step_is_redone():
+    from backend.apps.agents.manager.streaming.unwedge_sidecar import RETRY_PROMPT, arm_retry
+
+    class S:
+        pending_continuation = False
+        pending_continuation_prompt = ""
+
+    s = S()
+    assert arm_retry(s) is True
+    assert s.pending_continuation is True
+    assert s.pending_continuation_prompt == RETRY_PROMPT
+
+
+def test_a_retry_never_stacks_on_an_existing_continuation():
+    from backend.apps.agents.manager.streaming.unwedge_sidecar import arm_retry
+
+    class S:
+        pending_continuation = True
+        pending_continuation_prompt = "something else already queued"
+
+    s = S()
+    assert arm_retry(s) is False
+    assert s.pending_continuation_prompt == "something else already queued"
+
+
+def test_a_missing_session_is_survivable():
+    from backend.apps.agents.manager.streaming.unwedge_sidecar import arm_retry
+    assert arm_retry(None) is False
+
+
+def test_the_watchdog_arms_the_retry_on_the_live_session():
+    src = inspect.getsource(unwedge_sidecar.arm_wedge_watchdog)
+    assert "arm_retry" in src, "killing the sidecar frees the turn but loses the in-flight call"
 
 
 # --------------------------------------------------------------------- the kill choreography