[eric] browser: say 'I never typed it' when nothing was typed, not 'check the page'

This commit is contained in:
ciregenz
2026-08-04 21:53:58 -07:00
parent 94c30d1f9b
commit 1eaa23b9c1
3 changed files with 120 additions and 1 deletions
@@ -223,6 +223,38 @@ P_PRODUCTIVE_TOOLS = {
"BrowserClick", "BrowserClickIndex", "BrowserType", "BrowserNavigate",
"BrowserPressKey", "BrowserScroll", "BrowserBatch", "BrowserActVerified",
}
def typed_anything(action_log: list[dict]) -> bool:
"""Did this run put TEXT into the page, as opposed to merely moving around it?
Keyed on the text ARGUMENT, not the tool name, because the same tool does both jobs:
BrowserClickIndex with a `text` arg is the fill path, and without one it is navigation. A run of
bare clicks and reads is busy and has typed nothing. Reported live on a "send hi to charles zheng
on linkedin" run whose entire trace was ListInteractives/GetText/ClickIndex/ListInteractives/
GetText: two reads, two navigation clicks, no message. Telling that user their send "may not have
gone out" points them at a page where nothing was ever composed.
"""
for a in action_log:
if not a.get("ok"):
continue
inp = a.get("input") or {}
if isinstance(inp, dict) and str(inp.get("text") or "").strip():
return True
if a.get("tool") in ("BrowserType", "BrowserFindComposer", "send click"):
return True
# Some paths record the fill in the RESULT rather than the input: BrowserActVerified logs
# "filled:<payload>", and the send script's own rows describe the fill in prose. Missing
# those would call a run that really did type "never typed", which is the same class of
# wrong answer in the other direction.
if "fill" in str(a.get("result_summary") or "").lower():
return True
# A batch carries its own sub-actions; any typed sub counts.
for sub in (inp.get("actions") or []) if isinstance(inp, dict) else []:
sp = (sub or {}).get("params") or {}
if str(sp.get("text") or "").strip():
return True
return False
# Read/extract tools: a look-only task's evidence is that a read returned content.
P_READ_TOOLS = {
"BrowserGetText", "BrowserGetElements", "BrowserListInteractives",
@@ -391,6 +423,16 @@ def completion_is_honest(
if removal_task:
return False, ("the deletion was never confirmed, so the item may still be there; "
"check the page before trusting this")
# "may not have gone out" implies something WAS attempted. When the run only ever read the
# page, nothing was: reported live on a "send hi to charles zheng on linkedin" run whose
# whole trace was ListInteractives/GetText/ClickIndex/ListInteractives/GetText, two reads
# and no typing. Telling that user to "check the page" sends them looking for a message that
# was never composed, and it hides the actual reason the run stopped early. Say which of the
# two happened, because they have completely different fixes.
if not typed_anything(action_log):
return False, ("I never actually typed the message, so nothing was sent and there is "
"nothing on the page to check. Tell me the exact words to send "
"(in quotes) and I will do it in one go")
return False, ("the send was never confirmed, so it may not have gone out; "
"check the page before trusting this")
if not action_log:
+74
View File
@@ -0,0 +1,74 @@
"""A send that never typed anything must say so, not "check the page".
Reported live by Eric on 2026-08-04: "send hi to charles zheng on linkedin". The run's ENTIRE trace
was ListInteractives, GetText, ClickIndex(21), ListInteractives, GetText, ClickIndex(116),
ListInteractives, GetText. Two navigation clicks and three reads. Nothing was ever typed. It then
reported "the send was never confirmed, so it may not have gone out; check the page before trusting
this", which sends the user hunting for a message that was never composed and hides why the run
stopped early.
Both halves of that are failures, but they are DIFFERENT failures with different fixes: "we typed it
and could not prove it left" needs receipt work; "we never typed it" needs the payload. The gate now
distinguishes them, keyed on the text ARGUMENT rather than the tool name, because the same tool does
both jobs: BrowserClickIndex with a `text` arg is the fill path, without one it is navigation.
Root cause of the underlying decline, for the record: "send hi to X" carries no QUOTED payload, so
quoted_payload returns "" and the scripted send stands down by design (it will not guess which words
to send). That guard is right. What was wrong was the sentence the user got afterwards.
"""
from backend.apps.agents.browser.browser_loop import completion_is_honest, typed_anything
# Eric's trace, verbatim.
ERIC = [
{"tool": "BrowserListInteractives", "ok": True},
{"tool": "BrowserGetText", "ok": True},
{"tool": "BrowserClickIndex", "ok": True, "input": {"index": 21}},
{"tool": "BrowserListInteractives", "ok": True},
{"tool": "BrowserGetText", "ok": True},
{"tool": "BrowserClickIndex", "ok": True, "input": {"index": 116}},
{"tool": "BrowserListInteractives", "ok": True},
{"tool": "BrowserGetText", "ok": True},
]
def test_erics_run_is_told_nothing_was_typed():
honest, why = completion_is_honest(ERIC, publish_task=True, send_confirmed=False)
assert not honest
assert "never actually typed" in why
assert "check the page" not in why, "there is nothing on the page to check"
assert "in quotes" in why, "and it must say what would let it succeed next time"
def test_a_run_that_did_type_keeps_the_unconfirmed_wording():
"""The other failure is still a failure, and still worth checking the page for."""
typed = [{"tool": "BrowserClickIndex", "ok": True, "input": {"index": 5, "text": "hi"}}]
honest, why = completion_is_honest(typed, publish_task=True, send_confirmed=False)
assert not honest and "may not have gone out" in why
def test_typing_is_detected_by_the_text_argument_not_the_tool_name():
"""The distinction that makes this work: one tool, two jobs."""
assert not typed_anything([{"tool": "BrowserClickIndex", "ok": True, "input": {"index": 3}}])
assert typed_anything([{"tool": "BrowserClickIndex", "ok": True,
"input": {"index": 3, "text": "hello"}}])
assert typed_anything([{"tool": "BrowserType", "ok": True, "input": {"selector": "#a"}}])
def test_a_typed_sub_action_inside_a_batch_counts():
"""The efficient path bundles type+press_key into one BrowserBatch; the text is a level down."""
assert typed_anything([{"tool": "BrowserBatch", "ok": True, "input": {"actions": [
{"type": "click_index", "params": {"index": 2}},
{"type": "type", "params": {"selector": "#m", "text": "hello"}},
]}}])
def test_a_failed_fill_does_not_count_as_typing():
"""An attempt that errored put nothing in the box, so the user has nothing to check."""
assert not typed_anything([{"tool": "BrowserClickIndex", "ok": False,
"input": {"index": 5, "text": "hi"}}])
def test_whitespace_is_not_a_message():
assert not typed_anything([{"tool": "BrowserClickIndex", "ok": True,
"input": {"index": 5, "text": " "}}])
@@ -90,5 +90,8 @@ def test_the_completion_gate_describes_the_task_the_user_actually_gave():
assert "deletion" in why and "may still be there" in why
assert "gone out" not in why, "a delete must not be described as a send"
honest, why = completion_is_honest([], publish_task=True, send_confirmed=False)
# A run that DID type keeps the unconfirmed-send wording. (An empty log now correctly reports
# "never typed" instead, which is a different true statement; see test_never_typed_message.py.)
typed = [{"tool": "BrowserClickIndex", "ok": True, "input": {"index": 5, "text": "hello"}}]
honest, why = completion_is_honest(typed, publish_task=True, send_confirmed=False)
assert not honest and "gone out" in why, "a real send keeps its own wording"