From 68017465ce5fe27ee54b6712bfdc9815536302b8 Mon Sep 17 00:00:00 2001
From: ciregenz <ciregenz@berkeley.edu>
Date: Tue, 2 Jun 2026 18:14:57 -0700
Subject: [PATCH] [eric] browser: add a deterministic completion-honesty gate
 to catch ghost successes

---
 backend/apps/agents/browser/browser_loop.py | 44 +++++++++++++++++
 backend/tests/test_browser_stagnation.py    | 52 +++++++++++++++++++++
 2 files changed, 96 insertions(+)

diff --git a/backend/apps/agents/browser/browser_loop.py b/backend/apps/agents/browser/browser_loop.py
index dea90c9f..7568d1b0 100644
--- a/backend/apps/agents/browser/browser_loop.py
+++ b/backend/apps/agents/browser/browser_loop.py
@@ -179,3 +179,47 @@ def stagnation_exhausted(streak: int) -> bool:
     """True once deterministic nudging has been exhausted; the caller may then
     escalate to a one-shot aux-LLM adjudication (see browser_validator)."""
     return streak >= _STAGNATION_MAX
+
+
+# --- completion honesty gate ----------------------------------------------
+# A model that ends its turn is NOT proof the goal happened. The worst ghost we
+# measured: multi-minute runs where every tool errored, still reported
+# "completed". This deterministic gate reality-checks the run before we let the
+# status say "done", so a fake success is reported as the failure it actually is.
+
+# State-changing tools: a task that needed to DO something must land one of these.
+_PRODUCTIVE_TOOLS = {
+    "BrowserClick", "BrowserClickIndex", "BrowserType", "BrowserNavigate",
+    "BrowserPressKey", "BrowserScroll", "BrowserBatch",
+}
+# Read/extract tools: a look-only task's evidence is that a read returned content.
+_READ_TOOLS = {
+    "BrowserGetText", "BrowserGetElements", "BrowserListInteractives",
+    "BrowserListRoutes", "BrowserReplayRoute", "BrowserScreenshot", "BrowserEvaluate",
+}
+
+
+def completion_is_honest(action_log: list[dict]) -> tuple[bool, str]:
+    """Reality-check a run the model declared done. Returns (honest, reason).
+
+    Conservative by design (it can flip a 'completed' into an error, so it must
+    not cry wolf on a real success): it flags ONLY the unambiguous ghosts, a run
+    that took zero actions, one whose every state-changing action errored, or one
+    that only looked around (no action and no read returned content). A read-only
+    task stays honest as long as some read came back with content; a partially
+    erroring run that still landed a real action stays honest.
+    """
+    if not action_log:
+        return False, "declared done without taking a single action"
+    actions = [a for a in action_log if a.get("tool") in _PRODUCTIVE_TOOLS]
+    actions_ok = [a for a in actions if a.get("ok")]
+    reads_ok = [
+        a for a in action_log
+        if a.get("tool") in _READ_TOOLS and a.get("ok")
+        and str(a.get("result_summary") or "").strip()
+    ]
+    if actions and not actions_ok:
+        return False, "every state-changing action failed"
+    if not actions and not reads_ok:
+        return False, "only looked around: no action taken and no content read back"
+    return True, ""
diff --git a/backend/tests/test_browser_stagnation.py b/backend/tests/test_browser_stagnation.py
index 866fcd5f..b593ca93 100644
--- a/backend/tests/test_browser_stagnation.py
+++ b/backend/tests/test_browser_stagnation.py
@@ -5,6 +5,7 @@ from backend.apps.agents.browser.browser_loop import (
     _STAGNATION_MAX,
     _looks_like_failure,
     advance_stagnation,
+    completion_is_honest,
     is_unproductive,
     stagnation_exhausted,
     stagnation_nudge,
@@ -99,3 +100,54 @@ def test_advance_fires_again_at_max():
     assert streak == _STAGNATION_MAX
     assert nudge is not None and "RequestHumanIntervention" in nudge
     assert stagnation_exhausted(streak)
+
+
+# --- completion honesty gate ----------------------------------------------
+# Catches the worst measured ghost: multi-minute runs, every tool errored, still
+# reported 'completed'. Must NOT cry wolf on real successes (it overrides status).
+
+def _ok(tool, summary="done"):
+    return {"tool": tool, "ok": True, "result_summary": summary}
+
+
+def _err(tool):
+    return {"tool": tool, "ok": False, "result_summary": "Element not found: '.x'"}
+
+
+def test_completion_honest_when_an_action_succeeded():
+    log = [_ok("BrowserListInteractives", "1 button"), _ok("BrowserClickIndex", "Clicked")]
+    honest, reason = completion_is_honest(log)
+    assert honest and reason == ""
+
+
+def test_completion_ghost_when_every_action_errored():
+    # the exact LinkedIn ghost: 8 tools, all errored, model said 'completed'
+    log = [_err("BrowserClick") for _ in range(8)]
+    honest, reason = completion_is_honest(log)
+    assert not honest and "every state-changing action failed" in reason
+
+
+def test_completion_ghost_when_zero_actions_taken():
+    honest, reason = completion_is_honest([])
+    assert not honest and "without taking a single action" in reason
+
+
+def test_completion_ghost_when_only_looked_around_with_no_content():
+    # screenshot returned but no text, no action -> nothing real happened
+    log = [{"tool": "BrowserScreenshot", "ok": True, "result_summary": ""}]
+    honest, reason = completion_is_honest(log)
+    assert not honest and "only looked around" in reason
+
+
+def test_completion_honest_for_a_read_only_task_that_returned_content():
+    # a legit "tell me what's on the page" task: no action, but a read got content
+    log = [_ok("BrowserGetText", "The page says hello world")]
+    honest, reason = completion_is_honest(log)
+    assert honest and reason == ""
+
+
+def test_completion_honest_when_some_errors_but_an_action_landed():
+    # partial failure is fine as long as a real action ultimately succeeded
+    log = [_err("BrowserClick"), _err("BrowserClick"), _ok("BrowserClickIndex", "Clicked Submit")]
+    honest, reason = completion_is_honest(log)
+    assert honest