From a340f1d67df7ab3b69e78f6906b512991b6d69d8 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Wed, 26 Aug 2026 13:53:39 -0700 Subject: [PATCH] [eric] browser: a run that read nothing back says its specifics are unverified (ENG-404) --- backend/apps/agents/browser/browser_agent.py | 16 +++++- .../apps/agents/browser/effect_disposition.py | 23 ++++++++ backend/tests/test_effect_disposition.py | 53 +++++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index 410d1f5d..eac99dbf 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -34,7 +34,9 @@ from backend.apps.agents.browser.browser_history import ( refusal_shaped_summary, PAGE_STATE_MARKER, ) -from backend.apps.agents.browser.effect_disposition import disposition_line, effect_disposition +from backend.apps.agents.browser.effect_disposition import ( + disposition_line, effect_disposition, unverified_reads_line, +) from backend.apps.agents.browser.browser_loop import ( LOOP_DETECTION_EXCLUDED_TOOLS, LOOP_HARD_CAP, @@ -3275,6 +3277,18 @@ async def run_browser_agent( # A ghost run's transcript is exactly the memory the next agent must not inherit. clear_browser_history(browser_id) + else: + # The gate's "only looked around" check is skipped the moment any action succeeded, so a + # run whose clicks landed and whose every read failed came home as a clean answer with + # invented specifics in it. Label the data rather than reject the run (ENG-404). + p_unverified = unverified_reads_line(action_log) + if p_unverified: + summary = f"{summary}\n\n{p_unverified}" + logger.warning( + f"[browser-agent {session_id}] run reported success with zero successful reads; " + f"its specifics are unverified" + ) + session.status = final_status logger.info( f"[browser-batching {session_id}] run summary: turns={turn + 1} " diff --git a/backend/apps/agents/browser/effect_disposition.py b/backend/apps/agents/browser/effect_disposition.py index c499700c..9567f1d6 100644 --- a/backend/apps/agents/browser/effect_disposition.py +++ b/backend/apps/agents/browser/effect_disposition.py @@ -49,3 +49,26 @@ def disposition_line(disposition: Disposition) -> str: return ("A change was attempted and could not be confirmed, so this may or may not have gone " "through. Read the page and check before retrying; repeating it blind risks doing it " "twice.") + + +@typechecked +def unverified_reads_line(action_log: List[Dict]) -> str: + """One line when the run tried to read the page and got nothing back, else "". + + The honesty gate already refuses a run that only looked around, but that check is skipped the + moment any state-changing action succeeded. Haik's playlist run took that path: the clicks + reported ok, every read failed, and the agent handed the user "6 confirmed tracks" with titles + and artists. In his own words, "the path of least resistance was to paper over the ambiguity + with a confident-sounding answer" (ENG-404). + + It labels rather than rejects on purpose. A click that lands while the verification read fails + is a real, honest partial result, and flipping that to an error would delete work to punish a + word. What the reader was missing is the distinction between verified and assumed, so say it. + """ + p_reads = [a for a in action_log or [] if str(a.get("tool") or "") in READ_ONLY_TOOLS] + if not p_reads: + return "" + if any(a.get("ok") and str(a.get("result_summary") or "").strip() for a in p_reads): + return "" + return ("No page content was read back successfully during this run, so any specific values " + "above are unverified and must not be treated as confirmed.") diff --git a/backend/tests/test_effect_disposition.py b/backend/tests/test_effect_disposition.py index 44a3a917..56fa78e3 100644 --- a/backend/tests/test_effect_disposition.py +++ b/backend/tests/test_effect_disposition.py @@ -56,3 +56,56 @@ def test_the_failed_run_carries_the_disposition_home(): body = src[i - 400:i + 300] assert "effect_disposition(action_log" in body assert "disposition_line(p_effect)" in body, "the sentence must reach the caller's summary" + + +# ---------------------------------------------- the READ side of the same problem (ENG-404) + +def test_a_run_whose_every_read_failed_says_its_specifics_are_unverified(): + from backend.apps.agents.browser.effect_disposition import unverified_reads_line + # Haik's playlist run: the clicks reported ok, every read failed, and the agent handed the user + # "6 confirmed tracks" with titles and artists. The gate's "only looked around" check never + # fires once an action succeeds, so nothing caught it. + log = [{"tool": "BrowserClick", "ok": True}, + {"tool": "BrowserGetText", "ok": False}, + {"tool": "BrowserGetElements", "ok": False}] + line = unverified_reads_line(log) + assert "unverified" in line and "must not be treated as confirmed" in line + + +def test_a_read_that_returned_nothing_counts_as_failed(): + from backend.apps.agents.browser.effect_disposition import unverified_reads_line + assert unverified_reads_line([{"tool": "BrowserGetText", "ok": True, "result_summary": " "}]) + + +def test_a_run_that_really_read_the_page_is_left_alone(): + from backend.apps.agents.browser.effect_disposition import unverified_reads_line + assert unverified_reads_line( + [{"tool": "BrowserGetText", "ok": True, "result_summary": "Track 1 - Artist"}]) == "" + + +def test_a_pure_write_run_is_not_labelled(): + # It never tried to read, so there is nothing unverified to warn about; a note on every write + # would be noise, and noise is how a real warning stops being read. + from backend.apps.agents.browser.effect_disposition import unverified_reads_line + assert unverified_reads_line([{"tool": "BrowserType", "ok": True}]) == "" + assert unverified_reads_line([]) == "" + + +def test_the_label_rides_the_SUCCESSFUL_path_where_the_fabrication_travelled(): + src = open("backend/apps/agents/browser/browser_agent.py").read() + i_ghost = src.index("completion gate caught a ghost") + i_label = src.index("unverified_reads_line(action_log)") + assert i_label > i_ghost, "it belongs on the else branch: the run the gate let through" + assert "summary = f\"{summary}\\n\\n{p_unverified}\"" in src + + +def test_it_labels_rather_than_rejects(): + # A click that lands while the verification read fails is an honest partial. Flipping that to an + # error would delete real work to punish a word. + src = open("backend/apps/agents/browser/browser_agent.py").read() + i = src.index("unverified_reads_line(action_log)") + # Up to (not including) the status assignment that closes the branch: nothing in here may + # change the verdict, only the text. + body = src[i:src.index("session.status = final_status", i)] + assert "final_status =" not in body and "honest = False" not in body + assert "summary = " in body, "the only thing it touches is the wording"