diff --git a/backend/apps/agents/core/fault_injection.py b/backend/apps/agents/core/fault_injection.py index f605512b..2c27af6e 100644 --- a/backend/apps/agents/core/fault_injection.py +++ b/backend/apps/agents/core/fault_injection.py @@ -94,6 +94,29 @@ def squeezed_context_window() -> int: P_FIRED: Set[str] = set() +P_FIRES: dict = {} + + +def fires_budget() -> int: + """OSW_FAULT_FIRES=N caps how many times an every-turn fault fires in this process (0 = unlimited). + Without it a lane can never heal inside a drill, so the heal path stays a unit test.""" + try: + return int(os.environ.get("OSW_FAULT_FIRES", "0") or 0) + except ValueError: + return 0 + + +def fire(name: str) -> bool: + """armed() plus the budget: the injection site calls THIS so a predicate read elsewhere never spends a fire.""" + if not armed(name): + return False + budget = fires_budget() + if budget and P_FIRES.get(name, 0) >= budget: + return False + P_FIRES[name] = P_FIRES.get(name, 0) + 1 + return True + + def armed_once(name: str) -> bool: """Fire a recoverable fault exactly ONCE per process. @@ -109,6 +132,7 @@ def armed_once(name: str) -> bool: def reset_fired() -> None: """Test-only: forget what has fired so a case can arm the same one-shot again.""" P_FIRED.clear() + P_FIRES.clear() def unknown_faults() -> Set[str]: diff --git a/backend/apps/agents/core/models.py b/backend/apps/agents/core/models.py index a9e41961..99758bc2 100644 --- a/backend/apps/agents/core/models.py +++ b/backend/apps/agents/core/models.py @@ -178,6 +178,7 @@ class AgentSession(BaseModel): lane_credential_dead: bool = False # The router login this chat died on (a definitive auth failure), so a reconnect of that login can pick the chat back up by itself; cleared on resume. auth_dead_provider: Optional[str] = None + auth_dead_at: Optional[float] = None # A login that flaps (probe answers, the turn 401s) would otherwise resume and die every re-probe forever. auth_resumes: int = 0 # The router login this chat dispatches through (claude, codex, gemini-cli, antigravity), written by the preflight on every turn; None on a direct API key. The error handler reads it instead of guessing from the vendor. diff --git a/backend/apps/agents/manager/run/TurnRunner.py b/backend/apps/agents/manager/run/TurnRunner.py index d20e97db..5cd39465 100644 --- a/backend/apps/agents/manager/run/TurnRunner.py +++ b/backend/apps/agents/manager/run/TurnRunner.py @@ -45,7 +45,7 @@ class TurnRunner(AgentManagerProtocol): global_settings: AppSettings, force_respawn: bool = False) -> None: from claude_agent_sdk import query, ClaudeAgentOptions, AssistantMessage, ResultMessage from claude_agent_sdk.types import StreamEvent, SystemMessage - from backend.apps.agents.core.fault_injection import armed as p_fault_armed, armed_once as p_fault_once + from backend.apps.agents.core.fault_injection import armed as p_fault_armed, armed_once as p_fault_once, fire as p_fault_fire # Deliberate faults, so the guards below get drilled instead of waited for. Inert unless # OSW_FAULT names them; a shipped build never sets it. Raised HERE because this is the same @@ -57,7 +57,7 @@ class TurnRunner(AgentManagerProtocol): "\"Output blocked as it seems to violate our Acceptable Use Policy (legal/aup): " "reverse engineering or duplicating model outputs\"}}" ) - if p_fault_armed("auth_401"): + if p_fault_fire("auth_401"): raise RuntimeError("API Error: 401 {\"error\":{\"type\":\"authentication_error\",\"message\":\"invalid x-api-key\"}}") # One-shot: a dead pipe is recoverable, so the drill needs the retry to find a clear road. # The type must be one the REAL classifier calls a lost connection, or the drill quietly diff --git a/backend/apps/agents/manager/run/handle_run_error.py b/backend/apps/agents/manager/run/handle_run_error.py index 87e1ca26..e4cb8169 100644 --- a/backend/apps/agents/manager/run/handle_run_error.py +++ b/backend/apps/agents/manager/run/handle_run_error.py @@ -5,6 +5,7 @@ the file ceiling; pure relocation, no self (operates on the passed run state)."" import asyncio import logging +import time from typing import List from typeguard import typechecked @@ -169,6 +170,7 @@ async def p_mark_login_dead(session: AgentSession) -> None: if not lane: return session.auth_dead_provider = lane + session.auth_dead_at = time.time() try: from backend.apps.nine_router.subscription_health import report_dead_now await report_dead_now(lane) diff --git a/backend/apps/agents/manager/session/SessionPersistence.py b/backend/apps/agents/manager/session/SessionPersistence.py index fdf7f033..2f8862e1 100644 --- a/backend/apps/agents/manager/session/SessionPersistence.py +++ b/backend/apps/agents/manager/session/SessionPersistence.py @@ -5,6 +5,7 @@ one concern. self.sessions resolves across the MRO as before.""" import logging import os +import time import sys from typing import Optional @@ -21,6 +22,8 @@ from backend.apps.agents.manager.session.apply_context_window import apply_conte logger = logging.getLogger(__name__) AUTH_RESUME_CAP = 2 +# A chat that died on a dead login in an EARLIER run of the app stays put: the user has moved on, and work restarting days later unasked is the backfire. +PROCESS_STARTED_AT = time.time() def auto_resume_held_because() -> Optional[str]: @@ -146,6 +149,9 @@ class SessionPersistence(AgentManagerProtocol): continue if session.status in ("running", "waiting_approval"): continue + if session.auth_dead_at is None or session.auth_dead_at < PROCESS_STARTED_AT: + logger.info(f"reconnect-resume: session {sid} died on {provider} before this app run started; leaving it to the user") + continue if session.auth_resumes >= AUTH_RESUME_CAP: logger.warning(f"reconnect-resume: session {sid} has already been resumed {session.auth_resumes} times on a dead login; leaving it to the user") continue diff --git a/backend/apps/help/changelog.py b/backend/apps/help/changelog.py index d0e84d7a..32bf7cd3 100644 --- a/backend/apps/help/changelog.py +++ b/backend/apps/help/changelog.py @@ -23,15 +23,19 @@ P_RELEASES: List[ReleaseNote] = [ # agent confidently describing something that does not exist. ReleaseNote( version="1.7.10-exp.9", - headline="A login that dies is reported, chats no longer restart the router, big tables stop weighing the board down, and the Usage page says what its number is.", + headline="A login that dies renews itself, is reported when it cannot, and its chats pick themselves up when it is back; the composer scrolls; Fable 5 and GPT-6 are in the picker; big tables stop weighing the board down.", highlights=[ - "When your ChatGPT, Claude or Gemini login stops working, the app says so within a few minutes and offers Reconnect, instead of every chat on it failing quietly.", + "A ChatGPT, Claude or Gemini login is renewed before it expires while the app runs. When it cannot be renewed, the app says so the second a chat hits it and offers Reconnect; when the login is back, by a reconnect or by itself, the chats that died on it continue where they stopped.", + "Claude Fable 5 is back in the picker on both lanes, Claude Fable 5.1 on the API-key lane, and GPT-6 Astra on both lanes. GPT-5.4 left the ChatGPT lane, where OpenAI refuses it; its API-key row stays.", + "A chat input that has grown past its height cap scrolls again; the canvas used to take the wheel instead.", "A long table in a chat loads a screenful at a time with a Show more row, and a collapsed chat shows only the first rows of its table, so a board with big tables no longer stutters on every step.", "The Usage page says its dollar figure is what the work would have cost at API prices, not a bill; subscribers were reading it as a charge. A By lane list shows which login served the requests.", "The agent answers which subscriptions are connected from the app's own router, instead of from old settings fields that told a connected user they were not.", ], fixes=[ "Chats no longer ask the app to restart its model router at every turn on a login that works; a stale error left on a healthy login was read as dead.", + "The app never restarts its model router to fix one login any more; a restart cut every chat on every lane for up to half a minute and could not revive a dead login.", + "Agent-to-agent arrows leave the facing edges of their cards and stay legible at any zoom; a collapsed chat pill keeps its label readable zoomed out; cards lose their resting drop shadow; pressing anywhere in a card selects it.", "A widget whose code had not finished loading when the chat opened used to stay a grey placeholder for good; it now draws when it arrives.", "A collapsed chat's stats card shows its numbers side by side and never clips the last one; in a narrow chat column the same card no longer stacks three numbers into a tower.", "The floating Ask me anything bar and the login pill step out of the way of a chat tiled to the bottom-left instead of covering its composer.", diff --git a/backend/tests/test_dead_login_chain.py b/backend/tests/test_dead_login_chain.py index bb344970..aa708835 100644 --- a/backend/tests/test_dead_login_chain.py +++ b/backend/tests/test_dead_login_chain.py @@ -4,6 +4,7 @@ router cannot renew. Each guard is proven to FIRE, and the innocent case for eac from datetime import datetime, timezone import pytest +import time from backend.apps.agents.core.models import AgentSession from backend.apps.nine_router import oauth_refresh as orf @@ -63,6 +64,7 @@ async def test_a_second_401_reports_the_login_dead_this_second(monkeypatch): def p_session(sid, provider, status="error", ended=False): s = AgentSession(id=sid, name=sid, prompt="x", status=status) s.auth_dead_provider = provider + s.auth_dead_at = time.time() s.ended_by_user = ended s.lane_credential_dead = True s.auth_retry_used = True @@ -220,3 +222,24 @@ def test_the_error_handler_marks_the_death_on_both_definitive_branches_and_befor final_card = src.index("absorb_repeat_card(session, error_msg)", second_mark) assert gate < second_mark < final_card, "the definitive auth card marks only on the auth-shaped reasons, before the card" + + +@pytest.mark.asyncio +async def test_a_chat_that_died_in_an_earlier_app_run_is_left_alone(): + from backend.apps.agents.manager.session.SessionPersistence import PROCESS_STARTED_AT, SessionPersistence + + class Mgr(SessionPersistence): + def __init__(self): + old = p_session("old", "codex") + old.auth_dead_at = PROCESS_STARTED_AT - 3600 + never = p_session("never-stamped", "codex") + never.auth_dead_at = None + self.sessions = {"old": old, "never-stamped": never, "fresh": p_session("fresh", "codex")} + self.sent = [] + + async def send_message(self, sid, text, hidden=False): + self.sent.append(sid) + + m = Mgr() + assert await m.resume_auth_dead_sessions("codex") == 1 + assert m.sent == ["fresh"] diff --git a/backend/tests/test_dead_login_reprobe.py b/backend/tests/test_dead_login_reprobe.py index 7a85e4c3..6b15a9d0 100644 --- a/backend/tests/test_dead_login_reprobe.py +++ b/backend/tests/test_dead_login_reprobe.py @@ -1,5 +1,6 @@ """A dead-login verdict owns its second look: the pill closes and the chats resume when the lane answers again.""" import asyncio +import time import pathlib import pytest @@ -96,6 +97,7 @@ async def test_a_flapping_login_resumes_a_chat_at_most_twice(): def __init__(self): s = AgentSession(id="flap", name="flap", prompt="x", status="error") s.auth_dead_provider = "claude" + s.auth_dead_at = time.time() self.sessions = {"flap": s} self.sent = 0 @@ -106,6 +108,7 @@ async def test_a_flapping_login_resumes_a_chat_at_most_twice(): for _ in range(AUTH_RESUME_CAP + 2): await m.resume_auth_dead_sessions("claude") m.sessions["flap"].auth_dead_provider = "claude" + m.sessions["flap"].auth_dead_at = time.time() m.sessions["flap"].status = "error" assert m.sent == AUTH_RESUME_CAP assert m.sessions["flap"].auth_dead_provider == "claude", "the marker stays so the card still says which login died" diff --git a/backend/tests/test_fault_injection.py b/backend/tests/test_fault_injection.py index 68339af9..d3a0a1e7 100644 --- a/backend/tests/test_fault_injection.py +++ b/backend/tests/test_fault_injection.py @@ -71,7 +71,7 @@ WIRED_IN = { def p_block(kind: str) -> str: """The source of the branch that fires one fault, whichever helper name arms it.""" src = open(WIRED_IN[kind]).read() - for call in (f'p_fault_armed("{kind}")', f'p_fault_once("{kind}")'): + for call in (f'p_fault_armed("{kind}")', f'p_fault_fire("{kind}")', f'p_fault_once("{kind}")'): if call in src: return src.split(call)[1].split("if p_fault_")[0] raise AssertionError(f"{kind} is armed nowhere in {WIRED_IN[kind]}") @@ -210,3 +210,15 @@ def test_the_sidecar_wedge_stops_answering_and_stops_breathing(): assert "if not P_FROZEN:" in src.split("def p_beat")[1].split("threading.Thread")[0] env = open("backend/apps/agents/manager/register_builtin_mcp_servers.py").read() assert '"OSW_FAULT": os.environ.get("OSW_FAULT", "")' in env, "the sidecar must inherit the drill flag" + + +def test_a_fire_budget_lets_an_every_turn_fault_stop_so_a_heal_drill_can_watch_the_lane_return(monkeypatch): + from backend.apps.agents.core import fault_injection as fi + monkeypatch.setenv("OSW_FAULT", "auth_401") + monkeypatch.setenv("OSW_FAULT_FIRES", "2") + fi.reset_fired() + assert [fi.fire("auth_401") for _ in range(4)] == [True, True, False, False] + assert fi.armed("auth_401"), "the predicate never spends a fire" + monkeypatch.setenv("OSW_FAULT_FIRES", "0") + fi.reset_fired() + assert all(fi.fire("auth_401") for _ in range(5)), "no budget means every turn, as before"