[eric] health: a chat resumes on a healed login only if it died in this run of the app; the fault kit takes a fire budget so a drill can watch a lane come back; the exp.9 notes carry the week

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

backend/apps/agents/core/fault_injection.py

@@ -94,6 +94,29 @@ def squeezed_context_window() -> int:
 P_FIRED: Set[str] = set()
 
 
+P_FIRES: dict = {}
+
+
+def fires_budget() -> int:
+    """OSW_FAULT_FIRES=N caps how many times an every-turn fault fires in this process (0 = unlimited).
+    Without it a lane can never heal inside a drill, so the heal path stays a unit test."""
+    try:
+        return int(os.environ.get("OSW_FAULT_FIRES", "0") or 0)
+    except ValueError:
+        return 0
+
+
+def fire(name: str) -> bool:
+    """armed() plus the budget: the injection site calls THIS so a predicate read elsewhere never spends a fire."""
+    if not armed(name):
+        return False
+    budget = fires_budget()
+    if budget and P_FIRES.get(name, 0) >= budget:
+        return False
+    P_FIRES[name] = P_FIRES.get(name, 0) + 1
+    return True
+
+
 def armed_once(name: str) -> bool:
     """Fire a recoverable fault exactly ONCE per process.
 
@@ -109,6 +132,7 @@ def armed_once(name: str) -> bool:
 def reset_fired() -> None:
     """Test-only: forget what has fired so a case can arm the same one-shot again."""
     P_FIRED.clear()
+    P_FIRES.clear()
 
 
 def unknown_faults() -> Set[str]:

backend/apps/agents/core/models.py

@@ -178,6 +178,7 @@ class AgentSession(BaseModel):
     lane_credential_dead: bool = False
     # The router login this chat died on (a definitive auth failure), so a reconnect of that login can pick the chat back up by itself; cleared on resume.
     auth_dead_provider: Optional[str] = None
+    auth_dead_at: Optional[float] = None
     # A login that flaps (probe answers, the turn 401s) would otherwise resume and die every re-probe forever.
     auth_resumes: int = 0
     # The router login this chat dispatches through (claude, codex, gemini-cli, antigravity), written by the preflight on every turn; None on a direct API key. The error handler reads it instead of guessing from the vendor.

backend/apps/agents/manager/run/TurnRunner.py

@@ -45,7 +45,7 @@ class TurnRunner(AgentManagerProtocol):
                                     global_settings: AppSettings, force_respawn: bool = False) -> None:
         from claude_agent_sdk import query, ClaudeAgentOptions, AssistantMessage, ResultMessage
         from claude_agent_sdk.types import StreamEvent, SystemMessage
-        from backend.apps.agents.core.fault_injection import armed as p_fault_armed, armed_once as p_fault_once
+        from backend.apps.agents.core.fault_injection import armed as p_fault_armed, armed_once as p_fault_once, fire as p_fault_fire
 
         # Deliberate faults, so the guards below get drilled instead of waited for. Inert unless
         # OSW_FAULT names them; a shipped build never sets it. Raised HERE because this is the same
@@ -57,7 +57,7 @@ class TurnRunner(AgentManagerProtocol):
                 "\"Output blocked as it seems to violate our Acceptable Use Policy (legal/aup): "
                 "reverse engineering or duplicating model outputs\"}}"
             )
-        if p_fault_armed("auth_401"):
+        if p_fault_fire("auth_401"):
             raise RuntimeError("API Error: 401 {\"error\":{\"type\":\"authentication_error\",\"message\":\"invalid x-api-key\"}}")
         # One-shot: a dead pipe is recoverable, so the drill needs the retry to find a clear road.
         # The type must be one the REAL classifier calls a lost connection, or the drill quietly

backend/apps/agents/manager/run/handle_run_error.py

@@ -5,6 +5,7 @@ the file ceiling; pure relocation, no self (operates on the passed run state).""
 
 import asyncio
 import logging
+import time
 from typing import List
 from typeguard import typechecked
 
@@ -169,6 +170,7 @@ async def p_mark_login_dead(session: AgentSession) -> None:
     if not lane:
         return
     session.auth_dead_provider = lane
+    session.auth_dead_at = time.time()
     try:
         from backend.apps.nine_router.subscription_health import report_dead_now
         await report_dead_now(lane)

backend/apps/agents/manager/session/SessionPersistence.py

@@ -5,6 +5,7 @@ one concern. self.sessions resolves across the MRO as before."""
 
 import logging
 import os
+import time
 import sys
 from typing import Optional
 
@@ -21,6 +22,8 @@ from backend.apps.agents.manager.session.apply_context_window import apply_conte
 logger = logging.getLogger(__name__)
 
 AUTH_RESUME_CAP = 2
+# A chat that died on a dead login in an EARLIER run of the app stays put: the user has moved on, and work restarting days later unasked is the backfire.
+PROCESS_STARTED_AT = time.time()
 
 
 def auto_resume_held_because() -> Optional[str]:
@@ -146,6 +149,9 @@ class SessionPersistence(AgentManagerProtocol):
                 continue
             if session.status in ("running", "waiting_approval"):
                 continue
+            if session.auth_dead_at is None or session.auth_dead_at < PROCESS_STARTED_AT:
+                logger.info(f"reconnect-resume: session {sid} died on {provider} before this app run started; leaving it to the user")
+                continue
             if session.auth_resumes >= AUTH_RESUME_CAP:
                 logger.warning(f"reconnect-resume: session {sid} has already been resumed {session.auth_resumes} times on a dead login; leaving it to the user")
                 continue

backend/apps/help/changelog.py

@@ -23,15 +23,19 @@ P_RELEASES: List[ReleaseNote] = [
     # agent confidently describing something that does not exist.
     ReleaseNote(
         version="1.7.10-exp.9",
-        headline="A login that dies is reported, chats no longer restart the router, big tables stop weighing the board down, and the Usage page says what its number is.",
+        headline="A login that dies renews itself, is reported when it cannot, and its chats pick themselves up when it is back; the composer scrolls; Fable 5 and GPT-6 are in the picker; big tables stop weighing the board down.",
         highlights=[
-            "When your ChatGPT, Claude or Gemini login stops working, the app says so within a few minutes and offers Reconnect, instead of every chat on it failing quietly.",
+            "A ChatGPT, Claude or Gemini login is renewed before it expires while the app runs. When it cannot be renewed, the app says so the second a chat hits it and offers Reconnect; when the login is back, by a reconnect or by itself, the chats that died on it continue where they stopped.",
+            "Claude Fable 5 is back in the picker on both lanes, Claude Fable 5.1 on the API-key lane, and GPT-6 Astra on both lanes. GPT-5.4 left the ChatGPT lane, where OpenAI refuses it; its API-key row stays.",
+            "A chat input that has grown past its height cap scrolls again; the canvas used to take the wheel instead.",
             "A long table in a chat loads a screenful at a time with a Show more row, and a collapsed chat shows only the first rows of its table, so a board with big tables no longer stutters on every step.",
             "The Usage page says its dollar figure is what the work would have cost at API prices, not a bill; subscribers were reading it as a charge. A By lane list shows which login served the requests.",
             "The agent answers which subscriptions are connected from the app's own router, instead of from old settings fields that told a connected user they were not.",
         ],
         fixes=[
             "Chats no longer ask the app to restart its model router at every turn on a login that works; a stale error left on a healthy login was read as dead.",
+            "The app never restarts its model router to fix one login any more; a restart cut every chat on every lane for up to half a minute and could not revive a dead login.",
+            "Agent-to-agent arrows leave the facing edges of their cards and stay legible at any zoom; a collapsed chat pill keeps its label readable zoomed out; cards lose their resting drop shadow; pressing anywhere in a card selects it.",
             "A widget whose code had not finished loading when the chat opened used to stay a grey placeholder for good; it now draws when it arrives.",
             "A collapsed chat's stats card shows its numbers side by side and never clips the last one; in a narrow chat column the same card no longer stacks three numbers into a tower.",
             "The floating Ask me anything bar and the login pill step out of the way of a chat tiled to the bottom-left instead of covering its composer.",

backend/tests/test_dead_login_chain.py

@@ -4,6 +4,7 @@ router cannot renew. Each guard is proven to FIRE, and the innocent case for eac
 from datetime import datetime, timezone
 
 import pytest
+import time
 
 from backend.apps.agents.core.models import AgentSession
 from backend.apps.nine_router import oauth_refresh as orf
@@ -63,6 +64,7 @@ async def test_a_second_401_reports_the_login_dead_this_second(monkeypatch):
 def p_session(sid, provider, status="error", ended=False):
     s = AgentSession(id=sid, name=sid, prompt="x", status=status)
     s.auth_dead_provider = provider
+    s.auth_dead_at = time.time()
     s.ended_by_user = ended
     s.lane_credential_dead = True
     s.auth_retry_used = True
@@ -220,3 +222,24 @@ def test_the_error_handler_marks_the_death_on_both_definitive_branches_and_befor
     final_card = src.index("absorb_repeat_card(session, error_msg)", second_mark)
     assert gate < second_mark < final_card, "the definitive auth card marks only on the auth-shaped reasons, before the card"
 
+
+
+@pytest.mark.asyncio
+async def test_a_chat_that_died_in_an_earlier_app_run_is_left_alone():
+    from backend.apps.agents.manager.session.SessionPersistence import PROCESS_STARTED_AT, SessionPersistence
+
+    class Mgr(SessionPersistence):
+        def __init__(self):
+            old = p_session("old", "codex")
+            old.auth_dead_at = PROCESS_STARTED_AT - 3600
+            never = p_session("never-stamped", "codex")
+            never.auth_dead_at = None
+            self.sessions = {"old": old, "never-stamped": never, "fresh": p_session("fresh", "codex")}
+            self.sent = []
+
+        async def send_message(self, sid, text, hidden=False):
+            self.sent.append(sid)
+
+    m = Mgr()
+    assert await m.resume_auth_dead_sessions("codex") == 1
+    assert m.sent == ["fresh"]

backend/tests/test_dead_login_reprobe.py

@@ -1,5 +1,6 @@
 """A dead-login verdict owns its second look: the pill closes and the chats resume when the lane answers again."""
 import asyncio
+import time
 import pathlib
 
 import pytest
@@ -96,6 +97,7 @@ async def test_a_flapping_login_resumes_a_chat_at_most_twice():
         def __init__(self):
             s = AgentSession(id="flap", name="flap", prompt="x", status="error")
             s.auth_dead_provider = "claude"
+            s.auth_dead_at = time.time()
             self.sessions = {"flap": s}
             self.sent = 0
 
@@ -106,6 +108,7 @@ async def test_a_flapping_login_resumes_a_chat_at_most_twice():
     for _ in range(AUTH_RESUME_CAP + 2):
         await m.resume_auth_dead_sessions("claude")
         m.sessions["flap"].auth_dead_provider = "claude"
+        m.sessions["flap"].auth_dead_at = time.time()
         m.sessions["flap"].status = "error"
     assert m.sent == AUTH_RESUME_CAP
     assert m.sessions["flap"].auth_dead_provider == "claude", "the marker stays so the card still says which login died"

backend/tests/test_fault_injection.py

@@ -71,7 +71,7 @@ WIRED_IN = {
 def p_block(kind: str) -> str:
     """The source of the branch that fires one fault, whichever helper name arms it."""
     src = open(WIRED_IN[kind]).read()
-    for call in (f'p_fault_armed("{kind}")', f'p_fault_once("{kind}")'):
+    for call in (f'p_fault_armed("{kind}")', f'p_fault_fire("{kind}")', f'p_fault_once("{kind}")'):
         if call in src:
             return src.split(call)[1].split("if p_fault_")[0]
     raise AssertionError(f"{kind} is armed nowhere in {WIRED_IN[kind]}")
@@ -210,3 +210,15 @@ def test_the_sidecar_wedge_stops_answering_and_stops_breathing():
     assert "if not P_FROZEN:" in src.split("def p_beat")[1].split("threading.Thread")[0]
     env = open("backend/apps/agents/manager/register_builtin_mcp_servers.py").read()
     assert '"OSW_FAULT": os.environ.get("OSW_FAULT", "")' in env, "the sidecar must inherit the drill flag"
+
+
+def test_a_fire_budget_lets_an_every_turn_fault_stop_so_a_heal_drill_can_watch_the_lane_return(monkeypatch):
+    from backend.apps.agents.core import fault_injection as fi
+    monkeypatch.setenv("OSW_FAULT", "auth_401")
+    monkeypatch.setenv("OSW_FAULT_FIRES", "2")
+    fi.reset_fired()
+    assert [fi.fire("auth_401") for _ in range(4)] == [True, True, False, False]
+    assert fi.armed("auth_401"), "the predicate never spends a fire"
+    monkeypatch.setenv("OSW_FAULT_FIRES", "0")
+    fi.reset_fired()
+    assert all(fi.fire("auth_401") for _ in range(5)), "no budget means every turn, as before"