[eric] browser: a summary built from faked tool-call markup fails instead of reaching the user (ENG-297)

backend/apps/agents/browser/browser_agent.py

@@ -3118,7 +3118,8 @@ async def run_browser_agent(
             honest, dishonest_reason = completion_is_honest(
                 action_log, publish_task=is_publish_task(skill_key_task),
                 send_confirmed=send_confirmed,
-                mutation_task=is_mutation_task(skill_key_task))
+                mutation_task=is_mutation_task(skill_key_task),
+                summary=summary)
         final_status = "completed" if honest else "error"
         if not honest:
             summary = f"I was not able to complete this task ({dishonest_reason})."

backend/apps/agents/browser/browser_loop.py

@@ -407,9 +407,30 @@ def deliverable_is_informational(summary: str, task: str = "") -> bool:
     return False
 
 
+# The literal markup a tool call is made of. If it appears in the SUMMARY, the model typed it as
+# prose instead of calling anything, so every "result" it narrates alongside is invented.
+P_FABRICATED_CALL_RE = re.compile(
+    r"<\s*(?:antml:)?invoke\s+name\s*=|<\s*(?:antml:)?function_calls\s*>|"
+    r"<\s*(?:antml:)?tool_use\s+", re.I)
+
+
+def summary_fabricates_tool_calls(summary: str) -> bool:
+    """True when a run's summary contains tool-call MARKUP rather than a report.
+
+    Measured 2026-08-13 (ENG-297): one dispatch printed `<invoke name="BrowserEvaluate">` as text
+    with plausible return values for three file lines, on an action log of 2 read-only calls. The
+    caller treated it as real output and escalated a fabricated prompt-injection to the user as a
+    security incident.
+
+    Deliberately keys on the MARKUP, not on tool names: agents name tools in honest prose all the
+    time ("BrowserClick failed so I used BrowserClickIndex"), and flagging that would fail good runs.
+    """
+    return bool(P_FABRICATED_CALL_RE.search(summary or ""))
+
+
 def completion_is_honest(
     action_log: list[dict], publish_task: bool = False, send_confirmed: bool = False,
-    mutation_task: bool = False,
+    mutation_task: bool = False, summary: str = "",
 ) -> tuple[bool, str]:
     """Reality-check a run the model declared done. Returns (honest, reason).
 
@@ -428,6 +449,11 @@ def completion_is_honest(
     # can tell. Note the direction of the residual risk: autosend sets send_confirmed the moment its
     # click runs (a resend guard, not proof), so this can still let a bad send through, but it can
     # never newly flag a run that had any send signal at all.
+    # Checked before anything else: a summary built out of invented tool calls is not a weaker
+    # completion, it is a fabrication, and every other signal in the run is downstream of it.
+    if summary_fabricates_tool_calls(summary):
+        return False, ("the report contains fabricated tool-call markup, so its results were "
+                       "narrated rather than obtained; nothing in it can be trusted")
     if publish_task and not send_confirmed:
         return False, ("the send was never confirmed, so it may not have gone out; "
                        "check the page before trusting this")

backend/tests/test_browser_completion_honesty.py

@@ -7,6 +7,7 @@ Run:
 from backend.apps.agents.browser.browser_loop import (
     completion_is_honest,
     is_mutation_task,
+    summary_fabricates_tool_calls,
 )
 
 
@@ -80,3 +81,41 @@ def test_a_real_edit_via_browserevaluate_still_completes():
     ]
     honest, reason = completion_is_honest(log, mutation_task=True)
     assert honest and reason == "", reason
+
+
+# --- Gate 3 (ENG-297): tool-call markup in the SUMMARY is narration, not a tool call. ---
+#
+# Dispatch 3 printed literal <invoke name="BrowserEvaluate"> markup as TEXT, complete with
+# plausible return values for lines 1/2/3 of the file, on an action log of 2 read-only calls. The
+# caller read that as real output, believed a line-1 prompt injection that did not exist, and
+# escalated it to the user as a security incident. This is the failure that produced an actual lie,
+# so the markup must fail the run rather than reach anyone as content.
+
+
+def test_fabricated_tool_call_markup_is_caught():
+    for text in [
+        'I ran <invoke name="BrowserEvaluate">{"script":"..."}</invoke> and got line 1',
+        "here is the call: <invoke name=\"BrowserGetText\">",
+        "<function_calls>\n<invoke name=\"BrowserClick\">",
+    ]:
+        assert summary_fabricates_tool_calls(text), f"fabricated markup passed through: {text[:50]!r}"
+
+
+def test_ordinary_prose_about_tools_is_not_caught():
+    """Cry-wolf direction: agents legitimately NAME tools in prose, and failing those would make
+    every honest summary a failure."""
+    for text in [
+        "I used BrowserEvaluate to read the buffer, then saved.",
+        "BrowserClick failed twice so I fell back to BrowserClickIndex.",
+        "The page has an <input> and a <button> element.",
+        "",
+    ]:
+        assert not summary_fabricates_tool_calls(text), f"honest prose flagged: {text[:50]!r}"
+
+
+def test_a_fabricating_run_cannot_report_completed():
+    log = [p_ok("BrowserGetText", "function doPost() {"), p_ok("BrowserScreenshot", "captured")]
+    honest, reason = completion_is_honest(
+        log, summary='<invoke name="BrowserEvaluate">{"script":"editor.getValue()"}</invoke>')
+    assert not honest, "a summary that faked tool calls was reported as a completion"
+    assert "fabricat" in reason.lower(), reason