[eric] agents: OSW_FAULT=sidecar_wedge finally does something; the sidecar never received the flag, so the fault armed nothing for weeks and its own liveness test exempted it

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012G8kyALnPjsA7aJFmMBq3R

backend/apps/agents/combined_meta_mcp_server.py

@@ -16,6 +16,7 @@ import json
 import os
 import sys
 import threading
+import time
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
@@ -109,11 +110,35 @@ def send_response(id_, result=None, error=None):
         sys.stdout.flush()
 
 
+P_FROZEN = False
+
+
+def p_fault_once(name: str) -> bool:
+    """Drill seam, marker-file one-shot: the parent backend arms OSW_FAULT, and the FIRST sidecar of a
+    session fires it; the sidecar the CLI respawns afterwards finds the marker and runs clean, which
+    is the road the watchdog's retry needs."""
+    import tempfile
+    if name not in {p.strip() for p in os.environ.get("OSW_FAULT", "").split(",") if p.strip()}:
+        return False
+    marker = os.path.join(tempfile.gettempdir(), f"osw-fault-{name}-{os.environ.get('OPENSWARM_PARENT_SESSION_ID', '')}")
+    if os.path.exists(marker):
+        return False
+    open(marker, "w").close()
+    return True
+
+
 def p_call_async(id_, tool_name: str, arguments: dict) -> None:
     mod = P_ROUTE.get(tool_name)
     if mod is None:
         send_response(id_, {"content": [{"type": "text", "text": f"Unknown tool: {tool_name}"}], "isError": True})
         return
+    # A wedged sidecar answers nothing AND stops breathing: the heartbeat is what tells the watchdog "alive but slow" from "dead", so a fault that kept beating would drill the wrong branch.
+    if p_fault_once("sidecar_wedge"):
+        global P_FROZEN
+        P_FROZEN = True
+        sys.stderr.write(f"[fault] sidecar_wedge: {tool_name} will never answer and the heartbeat has stopped\n")
+        time.sleep(3600)
+        return
     try:
         send_response(id_, p_call(mod, tool_name, arguments))
     except Exception as e:
@@ -134,11 +159,12 @@ def start_heartbeat():
 
     def p_beat():
         while True:
-            try:
-                with open(path, "a"):
-                    os.utime(path, None)
-            except Exception:
-                pass
+            if not P_FROZEN:
+                try:
+                    with open(path, "a"):
+                        os.utime(path, None)
+                except Exception:
+                    pass
             p_time.sleep(5)
 
     threading.Thread(target=p_beat, daemon=True, name="hb").start()

backend/apps/agents/manager/register_builtin_mcp_servers.py

@@ -106,6 +106,8 @@ def register_builtin_mcp_servers(
             "OPENSWARM_AGENT_MODEL": session.model,
             "OPENSWARM_PRE_SELECTED_BROWSER_IDS": ",".join(pre_selected_bids),
             "OPENSWARM_SELECTED_APP_IDS": ",".join(selected_app_ids),
+            # The drill flag never reached the sidecar, so `sidecar_wedge` armed nothing for weeks (found 2026-09-02).
+            "OSW_FAULT": os.environ.get("OSW_FAULT", ""),
         },
         "type": "stdio",
     }

backend/tests/test_fault_injection.py

@@ -64,6 +64,7 @@ WIRED_IN = {
     "cli_context_squeeze": "backend/apps/agents/manager/configure_provider_env.py",
     "stale_tool_schema": TURN_RUNNER,
     "unclassified_error": TURN_RUNNER,
+    "sidecar_wedge": "backend/apps/agents/combined_meta_mcp_server.py",
 }
 
 
@@ -151,8 +152,8 @@ def test_a_recoverable_fault_fires_once_so_the_retry_finds_a_clear_road(monkeypa
 
 
 def test_every_known_fault_is_wired_somewhere():
-    assert set(WIRED_IN) == KNOWN_FAULTS - {"sidecar_wedge"}, \
-        "a fault this build knows but wires nowhere is a guard that can never be drilled"
+    assert set(WIRED_IN) == KNOWN_FAULTS, \
+        "a fault this build knows but wires nowhere is a guard that can never be drilled (sidecar_wedge was exactly that for weeks)"
     for kind in WIRED_IN:
         assert p_block(kind).strip(), f"{kind} has an empty branch"
 
@@ -198,3 +199,14 @@ def test_the_unclassified_fault_is_owned_by_no_classifier():
         assert not pred(exc), pred.__name__
     assert not is_auth_error(exc)
     assert not is_connection_lost(exc)
+
+
+def test_the_sidecar_wedge_stops_answering_and_stops_breathing():
+    """The heartbeat is the watchdog's alive-versus-dead signal; a wedge that kept beating would drill
+    the "slow but alive" branch instead of the kill the fault exists to provoke."""
+    block = p_block("sidecar_wedge")
+    assert "P_FROZEN = True" in block and "time.sleep(" in block
+    src = open("backend/apps/agents/combined_meta_mcp_server.py").read()
+    assert "if not P_FROZEN:" in src.split("def p_beat")[1].split("threading.Thread")[0]
+    env = open("backend/apps/agents/manager/register_builtin_mcp_servers.py").read()
+    assert '"OSW_FAULT": os.environ.get("OSW_FAULT", "")' in env, "the sidecar must inherit the drill flag"