From 60da51f3c940c984c58c666e1fa9b68bfcd20366 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Thu, 13 Aug 2026 11:34:50 -0700 Subject: [PATCH] [eric] browser: a run reports its call counts beside the prose, so a caller can check instead of trust (ENG-297) --- backend/apps/agents/browser/browser_agent.py | 4 ++ backend/apps/agents/browser/browser_loop.py | 25 +++++++++++ .../tests/test_browser_completion_honesty.py | 42 +++++++++++++++++++ 3 files changed, 71 insertions(+) diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index a9fa5967..f34ce0c3 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -47,6 +47,7 @@ from backend.apps.agents.browser.browser_loop import ( deliverable_is_informational, interstitial_dismiss_target, is_mutation_task, + outcome_facts, is_publish_task, is_removal_task, recoverable_tool_error, @@ -3275,6 +3276,9 @@ async def run_browser_agent( # surface the honest failure to the parent so it doesn't treat a did-nothing run as a success it can build on **({} if honest else {"error": summary}), "action_log": action_log, + # Counts beside the prose, so a caller can reject a "completed" that mutated nothing + # without parsing the summary or eyeballing the action list (ENG-297). + "outcome": outcome_facts(action_log), "final_screenshot": final_screenshot, } diff --git a/backend/apps/agents/browser/browser_loop.py b/backend/apps/agents/browser/browser_loop.py index 36d1d769..c184016a 100644 --- a/backend/apps/agents/browser/browser_loop.py +++ b/backend/apps/agents/browser/browser_loop.py @@ -414,6 +414,31 @@ P_FABRICATED_CALL_RE = re.compile( r"<\s*(?:antml:)?tool_use\s+", re.I) +def outcome_facts(action_log: list[dict]) -> dict: + """What the run actually did, in counts a caller can check without reading the prose. + + The six bad dispatches in ENG-297 were only caught because a human noticed "3 read-only calls" + under a "Task completed." That is a suspicious reader applying a heuristic, and a less suspicious + pass ships the fabrication onward. These counts travel WITH the summary so a calling agent can + distrust prose on principle instead of by intuition. + + Every key is always present, including zeros: a missing key makes a caller's check pass silently, + which is the failure mode this exists to remove. + """ + log = action_log or [] + mutations = [a for a in log if a.get("tool") in P_PRODUCTIVE_TOOLS] + return { + "calls": len(log), + "mutations_attempted": len(mutations), + "mutations_succeeded": len([a for a in mutations if a.get("ok")]), + "reads_with_content": len([ + a for a in log + if a.get("tool") in P_READ_TOOLS and a.get("ok") + and str(a.get("result_summary") or "").strip() + ]), + } + + def summary_fabricates_tool_calls(summary: str) -> bool: """True when a run's summary contains tool-call MARKUP rather than a report. diff --git a/backend/tests/test_browser_completion_honesty.py b/backend/tests/test_browser_completion_honesty.py index eb831015..a307aa80 100644 --- a/backend/tests/test_browser_completion_honesty.py +++ b/backend/tests/test_browser_completion_honesty.py @@ -7,6 +7,7 @@ Run: from backend.apps.agents.browser.browser_loop import ( completion_is_honest, is_mutation_task, + outcome_facts, summary_fabricates_tool_calls, ) @@ -119,3 +120,44 @@ def test_a_fabricating_run_cannot_report_completed(): log, summary='{"script":"editor.getValue()"}') assert not honest, "a summary that faked tool calls was reported as a completion" assert "fabricat" in reason.lower(), reason + + +# --- Gate 2 (ENG-297): the caller needs facts it can check, not prose it must trust. --- +# +# The only reason any of the six bad dispatches was caught is that a human read the "Actions taken" +# list and noticed 3 read-only calls under a "Task completed." That is a manual heuristic applied by +# a suspicious reader. A calling agent should be able to distrust prose on principle, which means +# the counts have to arrive beside it in a shape nothing can narrate its way around. + + +def p_err(tool): + return {"tool": tool, "ok": False, "result_summary": ""} + + +def test_outcome_facts_counts_what_actually_happened(): + log = [ + p_ok("BrowserGetText", "function doPost() {"), + p_ok("BrowserClickIndex", "Clicked"), + p_err("BrowserType"), + p_ok("BrowserScreenshot", "captured"), + ] + f = outcome_facts(log) + assert f["calls"] == 4, f + assert f["mutations_attempted"] == 2, f # ClickIndex + Type + assert f["mutations_succeeded"] == 1, f # Type failed + assert f["reads_with_content"] == 2, f + + +def test_outcome_facts_expose_the_ghost_shape(): + """The dispatch-5 shape: reads only. A caller can reject this without reading a word.""" + log = [p_ok("BrowserGetText", "x"), p_ok("BrowserListInteractives", "y"), p_ok("BrowserScreenshot", "z")] + f = outcome_facts(log) + assert f["mutations_attempted"] == 0 and f["mutations_succeeded"] == 0, f + assert f["reads_with_content"] == 3, f + + +def test_outcome_facts_on_an_empty_run_are_all_zero_not_missing(): + """Absent keys would make a caller's check silently pass; zeros make it fail honestly.""" + f = outcome_facts([]) + for k in ("calls", "mutations_attempted", "mutations_succeeded", "reads_with_content"): + assert f[k] == 0, f"{k} missing or non-zero on an empty run: {f}"