From 264f2ef6de84d2d76b35bced509a7fad549c683f Mon Sep 17 00:00:00 2001 From: ciregenz Date: Mon, 24 Aug 2026 10:54:05 -0700 Subject: [PATCH] [eric] agents: the nudge ladder's last rung is bounded by construction, and can never widen a ratchet (ENG-399) Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018foyDoK19jjbYdudfzQVkZ --- backend/apps/agents/core/fault_injection.py | 10 ++ backend/apps/agents/core/models.py | 9 +- backend/apps/agents/manager/run/RunOptions.py | 24 ++++- .../apps/agents/manager/run/empty_finish.py | 39 ++++++++ backend/tests/test_fault_injection.py | 8 ++ backend/tests/test_final_rung_bounded.py | 99 +++++++++++++++++++ 6 files changed, 185 insertions(+), 4 deletions(-) create mode 100644 backend/tests/test_final_rung_bounded.py diff --git a/backend/apps/agents/core/fault_injection.py b/backend/apps/agents/core/fault_injection.py index f855aaf8..c6a53176 100644 --- a/backend/apps/agents/core/fault_injection.py +++ b/backend/apps/agents/core/fault_injection.py @@ -79,3 +79,13 @@ def announce() -> None: f"armed={live or 'NOTHING'}" + (f"; not a known fault, armed nothing: {bogus}" if bogus else "") ) + + +# A KNOWN blind spot, written down rather than discovered again: `empty_finish` cannot drive the +# nudge ladder past rung 1. Swallowing the answer also means the model does no NEW tool work, and +# the ladder's own re-nudge guard (`p_tool_calls <= empty_finish_progress_mark`) correctly refuses +# to nudge again with nothing to show for the last one. Measured live 2026-08-24: two sessions, +# tool_calls 8 == progress_mark 8, ladder stopped at nudge 1 and surfaced honestly. Drilling rung 3 +# needs a fault that swallows the answer while the model KEEPS working, which this harness cannot +# yet produce (ENG-399). +UNREACHABLE_WITH = {"empty_finish": "nudge ladder rungs 2 and 3"} diff --git a/backend/apps/agents/core/models.py b/backend/apps/agents/core/models.py index bdbab261..f94bbf6a 100644 --- a/backend/apps/agents/core/models.py +++ b/backend/apps/agents/core/models.py @@ -150,9 +150,14 @@ class AgentSession(BaseModel): # One honest "stopped without a report" line per exhausted nudge budget; resets with the budget. empty_finish_surfaced: bool = False # What a fresh CLI session may carry as history: "minimal" (the user's asks, the tool trail, a model-written summary of the dropped span; never the model's own replies verbatim) or "none". Ratchets to "none" when a provider policy filter blocks a recap-bearing turn and never back up: on the subscription lane Anthropic's anti-distillation classifier blocked 192 of our recap turns in 14 days (0 on API keys), reading replayed model text as "duplicating model outputs". - history_prefix_mode: Literal["minimal", "none"] = "minimal" + history_prefix_mode: Literal["minimal", "summary", "none"] = "minimal" + # A ONE-TURN narrowing, consumed at spawn. The nudge ladder's last rung uses "summary" so + # its request is bounded BY CONSTRUCTION (a model-written gist, no trail, no replay) rather + # than walking into the same context that already ate rungs 1 and 2. It can only ever + # narrow: a policy ratchet at "none" is never widened back by an override (ENG-399). + history_prefix_once: Optional[Literal["summary", "none"]] = None # What the LAST spawned turn actually carried, so a block can tell a recap-caused refusal from a plain one. - history_prefix_sent: Literal["minimal", "none"] = "none" + history_prefix_sent: Literal["minimal", "summary", "none"] = "none" # Consecutive dirty deaths this session was MID-TURN for; the crash auto-resume breaker (hermes #30719 pairing: auto-resume must never outrun its circuit breaker). crash_interrupt_count: int = 0 # Outage rounds spent on this ask: the in-turn ladder covers only 335s, and the work is checkpointed, so a longer drop is waited out rather than ending the task. diff --git a/backend/apps/agents/manager/run/RunOptions.py b/backend/apps/agents/manager/run/RunOptions.py index 182ab3de..4efba2ad 100644 --- a/backend/apps/agents/manager/run/RunOptions.py +++ b/backend/apps/agents/manager/run/RunOptions.py @@ -40,6 +40,24 @@ logger = logging.getLogger(__name__) from backend.apps.agents.manager.AgentManagerProtocol import AgentManagerProtocol +# Narrowing order, widest first. A one-turn override may only move RIGHT along this list: the policy +# ratchet exists because a recap-bearing turn was already refused, and an override that widened it +# back would hand the filter the exact request it just declined. +P_PREFIX_NARROWNESS = ("minimal", "summary", "none") + + +@typechecked +def p_effective_prefix_mode(session: AgentSession) -> str: + """The persisted mode, narrowed by any one-turn override, consumed on read.""" + p_mode = session.history_prefix_mode + p_once = session.history_prefix_once + session.history_prefix_once = None + if p_once is None: + return p_mode + return max(p_mode, p_once, key=P_PREFIX_NARROWNESS.index) + + + class RunOptions(AgentManagerProtocol): # No return annotation: the returned tuple carries an SDK ClaudeAgentOptions, which can't be module-imported here (mock-mode would fail to import the manager); it's lazy-imported below. @@ -275,8 +293,10 @@ class RunOptions(AgentManagerProtocol): session.needs_fork = False elif len(session.messages) > 1: # The recap never carries the model's own replies; the mode drops to "none" when a provider policy filter blocks even that (Alex's bricked-chat class). - p_mode = session.history_prefix_mode - history = "" if p_mode == "none" else build_history_prefix( + p_mode = p_effective_prefix_mode(session) + # "summary" carries the model's OWN distilled gist and nothing we authored from its + # turns, so a last-rung request is small however big the history got (ENG-399). + history = "" if p_mode in ("none", "summary") else build_history_prefix( get_branch_messages(session), cutoff_msg_id=session.compacted_through_msg_id, ) diff --git a/backend/apps/agents/manager/run/empty_finish.py b/backend/apps/agents/manager/run/empty_finish.py index 15f7b569..59f4c831 100644 --- a/backend/apps/agents/manager/run/empty_finish.py +++ b/backend/apps/agents/manager/run/empty_finish.py @@ -39,6 +39,32 @@ logger = logging.getLogger(__name__) NUDGE_HARD_CAP = 3 +# Below this a quit is not about depth, so bounding the last rung would cost a rebuild and buy +# nothing. Measured on the field install whose quits were read one by one: every one landed between +# 68K and 166K input, and the nudge-1/2/3 medians climb 105K -> 125K -> 135K, i.e. the ladder gets +# DEEPER at each rung and fails harder for the same reason (ENG-399). +FINAL_RUNG_BOUND_TOKENS = 60_000 + + +@typechecked +def p_final_rung_bound() -> int: + """The depth above which the last rung gets bounded, with a drill override. + + Otherwise this path costs a genuine 60K-token conversation to reach even once, which is exactly + how the 50KB cap and the mid-turn breaker both shipped never having fired. Unset everywhere + except a drill; junk is ignored rather than silently trusted (same contract as + OSW_COMPACT_CEILING_TOKENS).""" + import os as p_os + raw = p_os.environ.get("OSW_FINAL_RUNG_BOUND_TOKENS", "").strip() + if raw: + try: + override = int(raw) + if override > 0: + return override + except ValueError: + logger.warning(f"ignoring junk OSW_FINAL_RUNG_BOUND_TOKENS={raw!r}") + return FINAL_RUNG_BOUND_TOKENS + @typechecked def maybe_nudge_empty_finish(session: AgentSession, session_id: str) -> bool: @@ -90,6 +116,19 @@ def maybe_nudge_empty_finish(session: AgentSession, session_id: str) -> bool: session.empty_finish_nudges += 1 session.pending_continuation = True p_final = session.empty_finish_nudges >= NUDGE_HARD_CAP + # A ladder whose rungs all fail for the same reason is not a ladder. Rungs 1 and 2 just died in + # THIS context, and the compaction above is gated on reclaiming 20K, so on a big irreducible + # history the last rung walked into the same wall with tools taken away. It is the user's last + # chance, so it is bounded BY CONSTRUCTION instead: a fresh request carrying the model's own + # distilled summary and no trail at all, which cannot be too big however the history grew. + if p_final and p_input >= p_final_rung_bound(): + maybe_compact(session, force=True) + session.history_prefix_once = "summary" + session.needs_fresh_session = True + logger.warning( + f"Agent {session_id}: final nudge at {p_input} input tokens; bounding it to a " + f"summary-only request so the last rung cannot die of the same depth as the first two" + ) session.pending_continuation_prompt = FINAL_NUDGE_PROMPT if p_final else NUDGE_PROMPT # Wording alone did not hold: the same escalation shipped in 1.7.6 and the prods came back on # 1.7.7, so the last turn now runs with no tools at all rather than being asked nicely. diff --git a/backend/tests/test_fault_injection.py b/backend/tests/test_fault_injection.py index 916f488b..ee841316 100644 --- a/backend/tests/test_fault_injection.py +++ b/backend/tests/test_fault_injection.py @@ -160,3 +160,11 @@ def test_the_boot_path_actually_announces(): src = open("backend/apps/agents/agents.py").read() assert "p_announce_armed_faults()" in src, \ "an announcement nothing calls is the silence it was written to prevent" + + +def test_the_harness_declares_what_it_cannot_reach(): + # A harness that quietly cannot exercise a path is the same lie as a guard that never fires: + # the drill passes, the rung was never touched, and nobody knows which. + from backend.apps.agents.core.fault_injection import UNREACHABLE_WITH + assert "empty_finish" in UNREACHABLE_WITH + assert set(UNREACHABLE_WITH) <= KNOWN_FAULTS diff --git a/backend/tests/test_final_rung_bounded.py b/backend/tests/test_final_rung_bounded.py new file mode 100644 index 00000000..d97b58ed --- /dev/null +++ b/backend/tests/test_final_rung_bounded.py @@ -0,0 +1,99 @@ +"""The nudge ladder's last rung must not die of the same cause as the first two. + +Field data from the install whose 369 silent quits were read one by one: the nudge-1/2/3 medians +climb 181 -> 203 -> 285 tool calls and 105K -> 125K -> 135K input. The ladder gets DEEPER at each +rung, so rung 3 was walking into the very context that had just eaten rungs 1 and 2, with tools +taken away. Compaction before the nudge could not save it because it is gated on reclaiming 20K. + +These pin that the last rung is bounded BY CONSTRUCTION, and that the bound can never widen a +policy ratchet back open. +""" + +import pytest + +from backend.apps.agents.core.models import AgentSession, Message +from backend.apps.agents.manager.run import empty_finish as ef +from backend.apps.agents.manager.run.RunOptions import p_effective_prefix_mode, P_PREFIX_NARROWNESS + + +def p_deep_session(input_tokens: int, nudges: int) -> AgentSession: + s = AgentSession(name="deep", model="sonnet", dashboard_id="d") + s.tokens = {"input": input_tokens} + s.empty_finish_nudges = nudges + s.empty_finish_total = nudges + s.messages = [Message(role="user", content="do the thing")] + for i in range(30): + s.messages.append(Message(role="tool_call", content={"tool": "Bash", "input": {"command": f"c{i}"}})) + s.messages.append(Message(role="tool_result", content={"tool_name": "Bash", "text": "x" * 400})) + return s + + +def test_the_last_rung_is_bounded_when_the_context_is_deep(monkeypatch): + import backend.apps.agents.manager.context_budget as cb + monkeypatch.setattr(cb, "maybe_compact", lambda s, force=False: True, raising=True) + s = p_deep_session(135_000, ef.NUDGE_HARD_CAP - 1) + ef.maybe_nudge_empty_finish(s, s.id) + assert s.history_prefix_once == "summary", \ + "the last rung must carry a gist, not the context that already killed two rungs" + assert s.needs_fresh_session is True + assert s.pending_continuation_toolless is True, "and it still drops tools, as before" + + +def test_an_earlier_rung_is_left_alone(monkeypatch): + # A control: bounding rung 1 would pay a rebuild on every hiccup, which is the ENG-354 mistake. + import backend.apps.agents.manager.context_budget as cb + monkeypatch.setattr(cb, "maybe_compact", lambda s, force=False: True, raising=True) + s = p_deep_session(135_000, 0) + ef.maybe_nudge_empty_finish(s, s.id) + assert s.history_prefix_once is None + assert s.pending_continuation_toolless is False + + +def test_a_shallow_last_rung_pays_nothing(monkeypatch): + # Below the measured quit floor (68K) depth is not the cause, so a rebuild would buy nothing. + import backend.apps.agents.manager.context_budget as cb + monkeypatch.setattr(cb, "maybe_compact", lambda s, force=False: True, raising=True) + s = p_deep_session(20_000, ef.NUDGE_HARD_CAP - 1) + ef.maybe_nudge_empty_finish(s, s.id) + assert s.history_prefix_once is None, "a small last rung does not need bounding" + + +def test_the_bound_can_never_widen_a_policy_ratchet(): + # The ratchet is at "none" because a recap-bearing turn was REFUSED. A one-turn override that + # widened it back would hand the filter the exact request it just declined. + s = AgentSession(name="blocked", model="sonnet", dashboard_id="d") + s.history_prefix_mode = "none" + s.history_prefix_once = "summary" + assert p_effective_prefix_mode(s) == "none" + + +def test_the_override_is_consumed_so_it_cannot_leak_into_later_turns(): + s = AgentSession(name="once", model="sonnet", dashboard_id="d") + s.history_prefix_once = "summary" + assert p_effective_prefix_mode(s) == "summary" + assert s.history_prefix_once is None + assert p_effective_prefix_mode(s) == "minimal", "the turn after must be normal again" + + +def test_summary_mode_sends_no_authored_trail_at_all(): + # The whole point: bounded BY CONSTRUCTION. "summary" carries the model's own distilled gist and + # nothing we wrote from its turns, so the request size does not depend on how big history got. + src = open("backend/apps/agents/manager/run/RunOptions.py").read() + assert 'history = "" if p_mode in ("none", "summary")' in src + assert 'distilled_history_summary(session, global_settings) if p_mode != "none"' in src, \ + "summary mode must still fetch the gist, or the last rung carries nothing" + + +def test_the_narrowing_order_is_declared_not_alphabetical(): + assert P_PREFIX_NARROWNESS == ("minimal", "summary", "none") + + +def test_the_drill_seam_is_declared_and_ignores_junk(monkeypatch): + from backend.apps.agents.manager.run.empty_finish import p_final_rung_bound, FINAL_RUNG_BOUND_TOKENS + assert p_final_rung_bound() == FINAL_RUNG_BOUND_TOKENS + monkeypatch.setenv("OSW_FINAL_RUNG_BOUND_TOKENS", "1200") + assert p_final_rung_bound() == 1200 + monkeypatch.setenv("OSW_FINAL_RUNG_BOUND_TOKENS", "not-a-number") + assert p_final_rung_bound() == FINAL_RUNG_BOUND_TOKENS, "junk must never silently disarm the bound" + monkeypatch.setenv("OSW_FINAL_RUNG_BOUND_TOKENS", "-5") + assert p_final_rung_bound() == FINAL_RUNG_BOUND_TOKENS