diff --git a/backend/apps/agents/browser/browser_loop.py b/backend/apps/agents/browser/browser_loop.py index 50f4534e..726c22ba 100644 --- a/backend/apps/agents/browser/browser_loop.py +++ b/backend/apps/agents/browser/browser_loop.py @@ -223,6 +223,38 @@ P_PRODUCTIVE_TOOLS = { "BrowserClick", "BrowserClickIndex", "BrowserType", "BrowserNavigate", "BrowserPressKey", "BrowserScroll", "BrowserBatch", "BrowserActVerified", } +def typed_anything(action_log: list[dict]) -> bool: + """Did this run put TEXT into the page, as opposed to merely moving around it? + + Keyed on the text ARGUMENT, not the tool name, because the same tool does both jobs: + BrowserClickIndex with a `text` arg is the fill path, and without one it is navigation. A run of + bare clicks and reads is busy and has typed nothing. Reported live on a "send hi to charles zheng + on linkedin" run whose entire trace was ListInteractives/GetText/ClickIndex/ListInteractives/ + GetText: two reads, two navigation clicks, no message. Telling that user their send "may not have + gone out" points them at a page where nothing was ever composed. + """ + for a in action_log: + if not a.get("ok"): + continue + inp = a.get("input") or {} + if isinstance(inp, dict) and str(inp.get("text") or "").strip(): + return True + if a.get("tool") in ("BrowserType", "BrowserFindComposer", "send click"): + return True + # Some paths record the fill in the RESULT rather than the input: BrowserActVerified logs + # "filled:", and the send script's own rows describe the fill in prose. Missing + # those would call a run that really did type "never typed", which is the same class of + # wrong answer in the other direction. + if "fill" in str(a.get("result_summary") or "").lower(): + return True + # A batch carries its own sub-actions; any typed sub counts. + for sub in (inp.get("actions") or []) if isinstance(inp, dict) else []: + sp = (sub or {}).get("params") or {} + if str(sp.get("text") or "").strip(): + return True + return False + + # Read/extract tools: a look-only task's evidence is that a read returned content. P_READ_TOOLS = { "BrowserGetText", "BrowserGetElements", "BrowserListInteractives", @@ -391,6 +423,16 @@ def completion_is_honest( if removal_task: return False, ("the deletion was never confirmed, so the item may still be there; " "check the page before trusting this") + # "may not have gone out" implies something WAS attempted. When the run only ever read the + # page, nothing was: reported live on a "send hi to charles zheng on linkedin" run whose + # whole trace was ListInteractives/GetText/ClickIndex/ListInteractives/GetText, two reads + # and no typing. Telling that user to "check the page" sends them looking for a message that + # was never composed, and it hides the actual reason the run stopped early. Say which of the + # two happened, because they have completely different fixes. + if not typed_anything(action_log): + return False, ("I never actually typed the message, so nothing was sent and there is " + "nothing on the page to check. Tell me the exact words to send " + "(in quotes) and I will do it in one go") return False, ("the send was never confirmed, so it may not have gone out; " "check the page before trusting this") if not action_log: diff --git a/backend/tests/test_never_typed_message.py b/backend/tests/test_never_typed_message.py new file mode 100644 index 00000000..4b0ffb27 --- /dev/null +++ b/backend/tests/test_never_typed_message.py @@ -0,0 +1,74 @@ +"""A send that never typed anything must say so, not "check the page". + +Reported live by Eric on 2026-08-04: "send hi to charles zheng on linkedin". The run's ENTIRE trace +was ListInteractives, GetText, ClickIndex(21), ListInteractives, GetText, ClickIndex(116), +ListInteractives, GetText. Two navigation clicks and three reads. Nothing was ever typed. It then +reported "the send was never confirmed, so it may not have gone out; check the page before trusting +this", which sends the user hunting for a message that was never composed and hides why the run +stopped early. + +Both halves of that are failures, but they are DIFFERENT failures with different fixes: "we typed it +and could not prove it left" needs receipt work; "we never typed it" needs the payload. The gate now +distinguishes them, keyed on the text ARGUMENT rather than the tool name, because the same tool does +both jobs: BrowserClickIndex with a `text` arg is the fill path, without one it is navigation. + +Root cause of the underlying decline, for the record: "send hi to X" carries no QUOTED payload, so +quoted_payload returns "" and the scripted send stands down by design (it will not guess which words +to send). That guard is right. What was wrong was the sentence the user got afterwards. +""" + +from backend.apps.agents.browser.browser_loop import completion_is_honest, typed_anything + +# Eric's trace, verbatim. +ERIC = [ + {"tool": "BrowserListInteractives", "ok": True}, + {"tool": "BrowserGetText", "ok": True}, + {"tool": "BrowserClickIndex", "ok": True, "input": {"index": 21}}, + {"tool": "BrowserListInteractives", "ok": True}, + {"tool": "BrowserGetText", "ok": True}, + {"tool": "BrowserClickIndex", "ok": True, "input": {"index": 116}}, + {"tool": "BrowserListInteractives", "ok": True}, + {"tool": "BrowserGetText", "ok": True}, +] + + +def test_erics_run_is_told_nothing_was_typed(): + honest, why = completion_is_honest(ERIC, publish_task=True, send_confirmed=False) + assert not honest + assert "never actually typed" in why + assert "check the page" not in why, "there is nothing on the page to check" + assert "in quotes" in why, "and it must say what would let it succeed next time" + + +def test_a_run_that_did_type_keeps_the_unconfirmed_wording(): + """The other failure is still a failure, and still worth checking the page for.""" + typed = [{"tool": "BrowserClickIndex", "ok": True, "input": {"index": 5, "text": "hi"}}] + honest, why = completion_is_honest(typed, publish_task=True, send_confirmed=False) + assert not honest and "may not have gone out" in why + + +def test_typing_is_detected_by_the_text_argument_not_the_tool_name(): + """The distinction that makes this work: one tool, two jobs.""" + assert not typed_anything([{"tool": "BrowserClickIndex", "ok": True, "input": {"index": 3}}]) + assert typed_anything([{"tool": "BrowserClickIndex", "ok": True, + "input": {"index": 3, "text": "hello"}}]) + assert typed_anything([{"tool": "BrowserType", "ok": True, "input": {"selector": "#a"}}]) + + +def test_a_typed_sub_action_inside_a_batch_counts(): + """The efficient path bundles type+press_key into one BrowserBatch; the text is a level down.""" + assert typed_anything([{"tool": "BrowserBatch", "ok": True, "input": {"actions": [ + {"type": "click_index", "params": {"index": 2}}, + {"type": "type", "params": {"selector": "#m", "text": "hello"}}, + ]}}]) + + +def test_a_failed_fill_does_not_count_as_typing(): + """An attempt that errored put nothing in the box, so the user has nothing to check.""" + assert not typed_anything([{"tool": "BrowserClickIndex", "ok": False, + "input": {"index": 5, "text": "hi"}}]) + + +def test_whitespace_is_not_a_message(): + assert not typed_anything([{"tool": "BrowserClickIndex", "ok": True, + "input": {"index": 5, "text": " "}}]) diff --git a/backend/tests/test_removal_never_claims_a_send.py b/backend/tests/test_removal_never_claims_a_send.py index d661fdfb..c91e3211 100644 --- a/backend/tests/test_removal_never_claims_a_send.py +++ b/backend/tests/test_removal_never_claims_a_send.py @@ -90,5 +90,8 @@ def test_the_completion_gate_describes_the_task_the_user_actually_gave(): assert "deletion" in why and "may still be there" in why assert "gone out" not in why, "a delete must not be described as a send" - honest, why = completion_is_honest([], publish_task=True, send_confirmed=False) + # A run that DID type keeps the unconfirmed-send wording. (An empty log now correctly reports + # "never typed" instead, which is a different true statement; see test_never_typed_message.py.) + typed = [{"tool": "BrowserClickIndex", "ok": True, "input": {"index": 5, "text": "hello"}}] + honest, why = completion_is_honest(typed, publish_task=True, send_confirmed=False) assert not honest and "gone out" in why, "a real send keeps its own wording"