From 5e4d7cd096e7d4c6a9a51a647c9d17496cba37a5 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Wed, 2 Sep 2026 00:12:09 -0700 Subject: [PATCH] [eric] agents: OSW_FAULT=sidecar_wedge finally does something; the sidecar never received the flag, so the fault armed nothing for weeks and its own liveness test exempted it Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_012G8kyALnPjsA7aJFmMBq3R --- .../apps/agents/combined_meta_mcp_server.py | 36 ++++++++++++++++--- .../manager/register_builtin_mcp_servers.py | 2 ++ backend/tests/test_fault_injection.py | 16 +++++++-- 3 files changed, 47 insertions(+), 7 deletions(-) diff --git a/backend/apps/agents/combined_meta_mcp_server.py b/backend/apps/agents/combined_meta_mcp_server.py index b77a8902..bcf2b12a 100644 --- a/backend/apps/agents/combined_meta_mcp_server.py +++ b/backend/apps/agents/combined_meta_mcp_server.py @@ -16,6 +16,7 @@ import json import os import sys import threading +import time sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) @@ -109,11 +110,35 @@ def send_response(id_, result=None, error=None): sys.stdout.flush() +P_FROZEN = False + + +def p_fault_once(name: str) -> bool: + """Drill seam, marker-file one-shot: the parent backend arms OSW_FAULT, and the FIRST sidecar of a + session fires it; the sidecar the CLI respawns afterwards finds the marker and runs clean, which + is the road the watchdog's retry needs.""" + import tempfile + if name not in {p.strip() for p in os.environ.get("OSW_FAULT", "").split(",") if p.strip()}: + return False + marker = os.path.join(tempfile.gettempdir(), f"osw-fault-{name}-{os.environ.get('OPENSWARM_PARENT_SESSION_ID', '')}") + if os.path.exists(marker): + return False + open(marker, "w").close() + return True + + def p_call_async(id_, tool_name: str, arguments: dict) -> None: mod = P_ROUTE.get(tool_name) if mod is None: send_response(id_, {"content": [{"type": "text", "text": f"Unknown tool: {tool_name}"}], "isError": True}) return + # A wedged sidecar answers nothing AND stops breathing: the heartbeat is what tells the watchdog "alive but slow" from "dead", so a fault that kept beating would drill the wrong branch. + if p_fault_once("sidecar_wedge"): + global P_FROZEN + P_FROZEN = True + sys.stderr.write(f"[fault] sidecar_wedge: {tool_name} will never answer and the heartbeat has stopped\n") + time.sleep(3600) + return try: send_response(id_, p_call(mod, tool_name, arguments)) except Exception as e: @@ -134,11 +159,12 @@ def start_heartbeat(): def p_beat(): while True: - try: - with open(path, "a"): - os.utime(path, None) - except Exception: - pass + if not P_FROZEN: + try: + with open(path, "a"): + os.utime(path, None) + except Exception: + pass p_time.sleep(5) threading.Thread(target=p_beat, daemon=True, name="hb").start() diff --git a/backend/apps/agents/manager/register_builtin_mcp_servers.py b/backend/apps/agents/manager/register_builtin_mcp_servers.py index bd1f9cff..480d6372 100644 --- a/backend/apps/agents/manager/register_builtin_mcp_servers.py +++ b/backend/apps/agents/manager/register_builtin_mcp_servers.py @@ -106,6 +106,8 @@ def register_builtin_mcp_servers( "OPENSWARM_AGENT_MODEL": session.model, "OPENSWARM_PRE_SELECTED_BROWSER_IDS": ",".join(pre_selected_bids), "OPENSWARM_SELECTED_APP_IDS": ",".join(selected_app_ids), + # The drill flag never reached the sidecar, so `sidecar_wedge` armed nothing for weeks (found 2026-09-02). + "OSW_FAULT": os.environ.get("OSW_FAULT", ""), }, "type": "stdio", } diff --git a/backend/tests/test_fault_injection.py b/backend/tests/test_fault_injection.py index 4e275177..68339af9 100644 --- a/backend/tests/test_fault_injection.py +++ b/backend/tests/test_fault_injection.py @@ -64,6 +64,7 @@ WIRED_IN = { "cli_context_squeeze": "backend/apps/agents/manager/configure_provider_env.py", "stale_tool_schema": TURN_RUNNER, "unclassified_error": TURN_RUNNER, + "sidecar_wedge": "backend/apps/agents/combined_meta_mcp_server.py", } @@ -151,8 +152,8 @@ def test_a_recoverable_fault_fires_once_so_the_retry_finds_a_clear_road(monkeypa def test_every_known_fault_is_wired_somewhere(): - assert set(WIRED_IN) == KNOWN_FAULTS - {"sidecar_wedge"}, \ - "a fault this build knows but wires nowhere is a guard that can never be drilled" + assert set(WIRED_IN) == KNOWN_FAULTS, \ + "a fault this build knows but wires nowhere is a guard that can never be drilled (sidecar_wedge was exactly that for weeks)" for kind in WIRED_IN: assert p_block(kind).strip(), f"{kind} has an empty branch" @@ -198,3 +199,14 @@ def test_the_unclassified_fault_is_owned_by_no_classifier(): assert not pred(exc), pred.__name__ assert not is_auth_error(exc) assert not is_connection_lost(exc) + + +def test_the_sidecar_wedge_stops_answering_and_stops_breathing(): + """The heartbeat is the watchdog's alive-versus-dead signal; a wedge that kept beating would drill + the "slow but alive" branch instead of the kill the fault exists to provoke.""" + block = p_block("sidecar_wedge") + assert "P_FROZEN = True" in block and "time.sleep(" in block + src = open("backend/apps/agents/combined_meta_mcp_server.py").read() + assert "if not P_FROZEN:" in src.split("def p_beat")[1].split("threading.Thread")[0] + env = open("backend/apps/agents/manager/register_builtin_mcp_servers.py").read() + assert '"OSW_FAULT": os.environ.get("OSW_FAULT", "")' in env, "the sidecar must inherit the drill flag"