diff --git a/backend/apps/agents/core/error_classify.py b/backend/apps/agents/core/error_classify.py index 0d05482d..069e1b06 100644 --- a/backend/apps/agents/core/error_classify.py +++ b/backend/apps/agents/core/error_classify.py @@ -34,6 +34,20 @@ P_TRANSLATION_ERROR_PATTERNS = re.compile( re.IGNORECASE, ) +# A 401/403 only counts with auth context around it: a traceback's "line 401," or a node stack's ":401:12" is not a sign-in failure, and matching them bare stalled healthy GPT runs 75s behind a false "token just rotated" notice (ENG-365). +AUTH_STATUS_RE = ( + r"(?:\b(?:http|status(?:_code)?|error(?:\s+code)?|code|response|request\s+failed|got|received|returned|responded(?:\s+with)?)\b\s*[:=]?\s*\(?\s*(?:401|403)\b" + r"|\b(?:401|403)\b[^\n]{0,24}?\b(?:unauthori[sz]ed|forbidden|client\s+error|authentication(?:_error)?|invalid|expired|token|credentials?|subscription|bearer|api[_\s-]?key|upstream|provider|api)\b" + r"|\(\s*(?:401|403)\s*\))" +) +P_AUTH_STATUS = re.compile(AUTH_STATUS_RE, re.IGNORECASE) + + +@typechecked +def has_auth_status(text: str) -> bool: + return bool(P_AUTH_STATUS.search(text)) + + # Patterns that look rate-limit-ish but are actually non-transient (user quota, auth, context-window tier gate). Must NOT retry, upgrading, reauthing, or trimming context is required. The long-context-required variant is what Anthropic returns when an OAuth Pro/Max account ships a request whose input exceeds the 200K standard tier and would need the "extra usage" tier; the user can't recover by waiting, so we surface it instead of looping. NON_TRANSIENT_PATTERNS = re.compile( r"(?:usage\s+cap\s+exceeded" @@ -46,7 +60,7 @@ NON_TRANSIENT_PATTERNS = re.compile( r"|long\s+context\s+(?:requests?\s+)?(?:requires?|not\s+(?:available|enabled))" r"|free_trial_exhausted|used\s+your\s+free" r"|blocked\s+as\s+it\s+seems\s+to\s+violate|legal/aup|acceptable\s+use\s+policy" - r"|401|403)", + r"|" + AUTH_STATUS_RE + r")", re.IGNORECASE, ) @@ -68,8 +82,8 @@ CODEX_ROTATION_RESUME_WAIT = 75 P_CODEX_ROTATION_PATTERNS = re.compile( r"(?:\[?codex/|\bcx/|\bgpt-[0-9])" r".*?" - r"(?:authentication\s+token\s+(?:is|has)\s+expired|token\s+expired|\b401\b)" - r"|(?:authentication\s+token\s+(?:is|has)\s+expired|token\s+expired|\b401\b)" + r"(?:authentication\s+token\s+(?:is|has)\s+expired|token\s+expired|" + AUTH_STATUS_RE + r")" + r"|(?:authentication\s+token\s+(?:is|has)\s+expired|token\s+expired|" + AUTH_STATUS_RE + r")" r".*?" r"(?:\[?codex/|\bcx/|\bgpt-[0-9])", re.IGNORECASE | re.DOTALL, @@ -94,8 +108,8 @@ def auth_resume_wait(exc: BaseException, attempt: int, extra_text: str = "") -> if is_translation_error(exc, extra_text): return None if not re.search( - r"\b(?:401|403)\b" - r"|unauthori[sz]ed" + AUTH_STATUS_RE + + r"|unauthori[sz]ed" r"|invalid\s+authentication" r"|invalid.*api[_\s-]?key" r"|invalid.*token" @@ -228,8 +242,8 @@ def is_auth_error(exc: BaseException, extra_text: str = "") -> bool: if re.search(r"reset\s+after|try\s+again\s+in", combined, re.IGNORECASE): return False return bool(re.search( - r"\b(401|403)\b" - r"|invalid\s+authentication\s+credentials" + AUTH_STATUS_RE + + r"|invalid\s+authentication\s+credentials" r"|invalid.*api[_\s-]?key" r"|missing\s+bearer\s+token" r"|unauthori[sz]ed" diff --git a/backend/apps/agents/manager/Messaging.py b/backend/apps/agents/manager/Messaging.py index e90d955d..17a532b2 100644 --- a/backend/apps/agents/manager/Messaging.py +++ b/backend/apps/agents/manager/Messaging.py @@ -159,6 +159,8 @@ class Messaging(AgentManagerProtocol): session.empty_finish_progress_mark = 0 session.empty_finish_surfaced = False session.auth_retry_used = False + # The repeat-quit floor and the vanishing-quit rule key on this; one false positive used to arm both for the session's life (ENG-364). + session.empty_finish_total = 0 # A human is here and driving, so an earlier outage stops counting against the next one. session.reconnect_attempts = 0 session.awaiting_reconnect = False diff --git a/backend/apps/agents/manager/run/empty_finish.py b/backend/apps/agents/manager/run/empty_finish.py index c87f124b..982106fc 100644 --- a/backend/apps/agents/manager/run/empty_finish.py +++ b/backend/apps/agents/manager/run/empty_finish.py @@ -206,6 +206,19 @@ def p_tool_name_of(msg: object) -> str: return "" +@typechecked +def p_text_of(content: object) -> str: + """The visible text of an assistant message whatever shape its writer used (a plain string, a + content-block list, a dict block); a structured final answer used to score as a silent quit.""" + if isinstance(content, str): + return content + if isinstance(content, dict): + return str(content.get("text") or "") + if isinstance(content, list): + return "".join(p_text_of(c) for c in content) + return "" + + @typechecked def turn_finished_empty(session: AgentSession) -> bool: """True when the branch's last visible message is a tool result whose call was ordinary work @@ -217,8 +230,7 @@ def turn_finished_empty(session: AgentSession) -> bool: continue role = getattr(m, "role", "") if role == "assistant": - text = m.content if isinstance(m.content, str) else "" - return not text.strip() + return not p_text_of(m.content).strip() if role == "tool_result": continue if role == "tool_call": diff --git a/backend/apps/agents/manager/run/handle_run_error.py b/backend/apps/agents/manager/run/handle_run_error.py index fdcd76c6..baf4d277 100644 --- a/backend/apps/agents/manager/run/handle_run_error.py +++ b/backend/apps/agents/manager/run/handle_run_error.py @@ -18,6 +18,7 @@ from backend.apps.agents.core.error_classify import ( is_transient_capacity_error, is_free_trial_exhausted, is_out_of_tokens, + has_auth_status, is_auth_error, is_cert_failure, is_cli_binary_missing, @@ -302,7 +303,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str, p_combined = f"{e!s}\n{p_stderr_tail}".lower() p_codex_rotation = ( ("codex/" in p_combined or "[codex/" in p_combined or p_model.startswith(("cx/", "gpt-"))) - and ("authentication token is expired" in p_combined or "authentication token has expired" in p_combined or "401" in p_combined) + and ("authentication token is expired" in p_combined or "authentication token has expired" in p_combined or has_auth_status(p_combined)) ) # Every sub lane gets ONE silent self-heal before any card; only a missing credential (config problem, retry fails identically) goes straight to the card. Codex waits out its rotation window first. # A lane the router had ALREADY given up on before this turn is a dead credential, so the @@ -343,7 +344,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str, # Codex/OpenAI subscription tokens rotate every ~2-3 minutes, the user sees the rotation window as a 401 with "reset after 1m 59s" or similar. Don't ask them to reconnect; just tell them to wait it out and retry. if ( ("codex/" in p_combined or "[codex/" in p_combined or p_model.startswith(("cx/", "gpt-"))) - and ("authentication token is expired" in p_combined or "authentication token has expired" in p_combined or "401" in p_combined) + and ("authentication token is expired" in p_combined or "authentication token has expired" in p_combined or has_auth_status(p_combined)) ): friendly_msg = ( "GPT subscription token just rotated, this is " diff --git a/backend/tests/test_auth_resume.py b/backend/tests/test_auth_resume.py index 6aff8970..268e09e2 100644 --- a/backend/tests/test_auth_resume.py +++ b/backend/tests/test_auth_resume.py @@ -116,3 +116,33 @@ def test_non_codex_auth_failures_keep_the_short_resume(): "Invalid bearer token", ): assert auth_resume_wait(Exception(text), 0) == 20, text + + +def test_a_traceback_line_number_is_not_an_auth_failure(): + """ENG-365: `line 401,` in a Python traceback and `:401:12` in a node stack read as a bare 401, + which stalled healthy GPT runs 75s behind a false "token just rotated" notice.""" + from backend.apps.agents.core.error_classify import NON_TRANSIENT_PATTERNS, has_auth_status, is_auth_error + for text in ( + 'Traceback (most recent call last):\n File "/app/x.py", line 401, in run\n raise ValueError("boom")', + "TypeError: cannot read properties of undefined\n at handler (/app/error.js:401:12)", + "processed 14010 rows, 4031 skipped, 403 bytes written", + ): + assert auth_resume_wait(Exception("cx/gpt-5.4 process exited 1"), 0, extra_text=text) is None, text + assert not is_auth_error(Exception("process exited 1"), extra_text=text), text + assert not has_auth_status(text), text + assert not NON_TRANSIENT_PATTERNS.search(text), text + + +def test_real_auth_statuses_still_count(): + from backend.apps.agents.core.error_classify import has_auth_status, is_auth_error + for text in ( + "API Error: 401 authentication token is expired", + "Request failed: 401 Unauthorized", + "HTTP 403 Forbidden", + "status_code=401", + "Error code: 401 - {'type': 'authentication_error'}", + "upstream says: invalid token (401)", + "No credentials for provider: claude (401)", + ): + assert has_auth_status(text), text + assert is_auth_error(Exception(text)), text diff --git a/backend/tests/test_empty_finish.py b/backend/tests/test_empty_finish.py index a28cf60f..bebf65a5 100644 --- a/backend/tests/test_empty_finish.py +++ b/backend/tests/test_empty_finish.py @@ -287,3 +287,4 @@ def test_the_honest_lines_survive_the_frontends_jargon_filter() -> None: "the exhausted note would be swallowed by the UI's dev-jargon filter" ) assert EXHAUSTED_NOTE.strip(), "an empty note renders as nothing at all" + diff --git a/backend/tests/test_empty_finish_scoring.py b/backend/tests/test_empty_finish_scoring.py new file mode 100644 index 00000000..44413d30 --- /dev/null +++ b/backend/tests/test_empty_finish_scoring.py @@ -0,0 +1,36 @@ +"""ENG-364 pins: a structured final answer is an answer, and a real user message forgives the +session's silent-quit history (the repeat floor and the vanishing-quit rule key on it).""" + +from backend.apps.agents.core.models import AgentSession, Message +from backend.apps.agents.manager.run.empty_finish import turn_finished_empty + + +def p_session(*msgs) -> AgentSession: + s = AgentSession(name="t", model="sonnet") + for role, content in msgs: + s.messages.append(Message(role=role, content=content, branch_id="main")) + return s + + +def test_structured_assistant_content_is_not_a_silent_quit(): + """ENG-364: a final answer written as content blocks used to score as "", i.e. as a quit.""" + s = p_session(("user", "audit"), + ("tool_call", {"tool": "Bash", "input": {}}), + ("tool_result", {"text": "ok"}), + ("assistant", [{"type": "text", "text": "Done: 3 findings."}])) + assert turn_finished_empty(s) is False + s2 = p_session(("user", "audit"), + ("tool_call", {"tool": "Bash", "input": {}}), + ("tool_result", {"text": "ok"}), + ("assistant", [{"type": "text", "text": " "}])) + assert turn_finished_empty(s2) is True + + +def test_a_real_user_message_forgives_the_quit_history(): + """ENG-364: empty_finish_total drives the 40% repeat floor and the vanishing-quit rule; it must + reset with the rest of the per-ask budget or one false positive arms both forever.""" + import inspect + from backend.apps.agents.manager import Messaging + src = inspect.getsource(Messaging) + block = src.split("session.empty_finish_nudges = 0", 1)[1].split("if not hidden and prompt", 1)[0] + assert "session.empty_finish_total = 0" in block