diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index c63e5f02..effae089 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -2135,7 +2135,20 @@ async def run_browser_agent( done_success = delivery_verified p_aux_c, p_aux_m = await p_get_aux_client() p_pay = composer_committed_payload or "" - if delivery_verified: + if p_task_is_removal: + # A DELETE task must never exit through the send confirmation. It + # did: asked to delete three reddit posts, the run replied "Done, + # that's sent." and removed nothing, and an independent re-read + # found all three still there. The reply was shaped by the branch it + # fell through, not by anything that happened on the page, which is + # the exact false-success shape the receipt work exists to kill. + done_success = False + done_message = ( + "I could not confirm anything was deleted. Nothing here proves " + "the item was removed, so please check the page yourself.") + logger.info(f"[browser-agent {session_id}] removal task hit the " + f"post-send backstop; refusing to claim a send") + elif delivery_verified: p_nice = await compose_send_confirmation(p_aux_c, p_aux_m, task, p_pay) done_message = p_nice or ( f'Done, I sent "{p_pay}" for you.' if p_pay else "Done, that's sent.") @@ -2609,7 +2622,15 @@ async def run_browser_agent( ) # Send completion: a SUCCESSFUL click on a send-class control (Send/ Submit/Post, opener-excluded so 'Message' never trips it) means the message went out, the composer clears instantly. We do NOT depend on the thread-text confirm here: the sent text often renders late, split across nodes, or scrolled off, so the text-probe is unreliable, which is exactly what left the model stalling to "double-check". A clean send click is proof enough; drive to the OUTCOME. - if task_is_send and not send_confirmed and "error" not in result and tu.name in P_CONFIRM_TOOLS: + # NOT on a removal task. `task_is_send` is true for one too (the classifier keys on + # the verb), and the receipt below cannot tell composing from SEARCHING: asked to + # delete a post, the model types its title into the site's search box, which sets + # composer_committed_payload, and the next navigation leaves that box empty. Fill + # committed + box now empty is exactly the two-sided receipt, so a delete that + # removed nothing reported "Done, that's sent." Measured live on reddit: three + # posts still there afterwards, all three claimed gone. + if (task_is_send and not p_task_is_removal and not send_confirmed + and "error" not in result and tu.name in P_CONFIRM_TOOLS): p_cn = result.get("clickedName") or "" p_cr = result.get("clickedRole") or "" p_send_click = browser_batch_replay.is_send_completed( @@ -2723,7 +2744,12 @@ async def run_browser_agent( if p_fill_text and payload_in_textbox(p_auto_state or "", p_fill_text): composer_committed_payload = p_fill_text # B: the model just TYPED the message into a composer on a send task, so finish the send in CODE (find Send, click, verify the composer cleared) instead of it burning ~3-4 turns on a Send button whose index goes stale after the fill. Uses what the MODEL typed, so un-quoted phrasings ("say hi" -> "hi") work; fails safe, an unverified click never claims delivery and send_confirmed blocks a resend. - if (task_is_send and not send_confirmed and tu.name in P_CONFIRM_TOOLS + # Same removal exclusion as the receipt above, and here it is the sharper edge: + # this path CLICKS Send. On a delete task the text the model just typed is a + # search query, so without the guard we would search for a post and then post + # the search. + if (task_is_send and not p_task_is_removal and not send_confirmed + and tu.name in P_CONFIRM_TOOLS and browser_send_script.autosend_enabled()): p_cs = await browser_send_script.complete_send( composer_committed_payload, p_auto_state or "", browser_id, tab_id, diff --git a/backend/tests/test_browser_send_honesty.py b/backend/tests/test_browser_send_honesty.py index b9fb7794..28e703b9 100644 --- a/backend/tests/test_browser_send_honesty.py +++ b/backend/tests/test_browser_send_honesty.py @@ -38,7 +38,10 @@ def test_the_stall_backstop_branches_on_evidence(): final sentence without the model ever getting to speak; it must therefore read the evidence flag, not the resend guard.""" idx = P_SRC.index("done_success = delivery_verified") - window = P_SRC[idx:idx + 900] + # Bounded by the next sibling branch, not by a character count. The count was 900, and adding an + # `if p_task_is_removal:` branch (a delete must not borrow the send wording) pushed the honest + # line past it, failing a test whose subject had not changed. + window = P_SRC[idx:P_SRC.index("if not wrapup_nudged", idx)] assert "compose_unverified_send" in window, \ "the unverified branch must compose an honest line, not fall straight to a template" assert "unverified_send_note" in window, \ diff --git a/backend/tests/test_removal_never_claims_a_send.py b/backend/tests/test_removal_never_claims_a_send.py new file mode 100644 index 00000000..eed5339c --- /dev/null +++ b/backend/tests/test_removal_never_claims_a_send.py @@ -0,0 +1,74 @@ +"""A delete task must never exit through the send machinery. + +Measured live 2026-08-02. Asked to delete three reddit posts, the run replied "Done, that's sent." +and removed nothing: an independent re-read found all three still on the profile. Three separate +paths let it happen, and all three share one root cause. + +`task_is_send` is TRUE for a removal task. That is deliberate and documented (`is_removal_task` +exists precisely because the send classifier keys on the verb), but every consumer of `task_is_send` +then has to remember to exclude removals, and three of them did not: + + 1. the two-sided receipt, which cannot tell composing from SEARCHING. To find a post you want to + delete, the model types its title into the site's search box. That sets + composer_committed_payload. Navigating to the result leaves the box empty. "Fill committed AND + box now empty" is the receipt, so a delete that deleted nothing scored a verified send. + 2. the autosend path, the sharper edge: it CLICKS Send on that same typed text, so a delete task + could search for a post and then post the search. + 3. the post-send stall backstop, which phrased the ending as a send confirmation. + +These tests are about the WIRING, not the regexes: they assert that a removal task is excluded from +each of the three, so a future consumer of `task_is_send` that forgets the exclusion fails here. +""" + +import inspect +import re + +from backend.apps.agents.browser import browser_agent as BA +from backend.apps.agents.browser.browser_loop import is_removal_task + + +P_SRC = inspect.getsource(BA) + + +def p_guard_near(needle: str, window: int = 6) -> str: + """The few lines above a landmark, where its guard lives.""" + i = P_SRC.find(needle) + assert i != -1, f"landmark moved: {needle!r}" + return "\n".join(P_SRC[:i].splitlines()[-window:]) + "\n" + needle + + +def test_a_delete_task_is_still_classified_as_a_send(): + """The premise the other tests rest on. If this ever flips, the exclusions below become dead + code and their protection quietly disappears, so it is asserted rather than assumed.""" + task = 'Go to reddit.com and delete my post titled "canary9428eec8"' + assert is_removal_task(task), "a delete task must read as a removal" + assert re.search(r"\b(post|submit|publish|send)\b", task, re.I), ( + "and it also carries a publish verb, which is why task_is_send is true for it") + + +def test_the_two_sided_receipt_excludes_removals(): + """The path that scored the false success: typing a title into search, then navigating away.""" + guard = p_guard_near('if (task_is_send and not p_task_is_removal and not send_confirmed\n' + ' and "error" not in result') + assert "not p_task_is_removal" in guard + + +def test_the_autosend_click_path_excludes_removals(): + """The dangerous one: this path clicks Send on whatever the model just typed.""" + guard = p_guard_near("and browser_send_script.autosend_enabled()):") + assert "not p_task_is_removal" in guard + + +def test_the_post_send_backstop_refuses_to_claim_a_send_on_a_removal(): + """Defence in depth. Even if a removal reaches the post-send ending, it must not borrow the + send wording, and it must not report success.""" + i = P_SRC.find("if p_task_is_removal:") + assert i != -1, "the removal branch in the post-send backstop is gone" + branch = P_SRC[i:i + 900] + assert "done_success = False" in branch, "a removal that cannot be confirmed is not a success" + assert "could not confirm anything was deleted" in branch + # The send wording must be ASSIGNED only under the non-removal branch. Checked on the assignment + # rather than the raw text, because the comment above it quotes the bad reply on purpose. + removal_code = [ln for ln in branch.split("elif")[0].splitlines() + if "done_message" in ln and not ln.strip().startswith("#")] + assert removal_code and all("sent" not in ln for ln in removal_code), removal_code diff --git a/scripts/browser_canary.py b/scripts/browser_canary.py index 3c608488..d7034516 100644 --- a/scripts/browser_canary.py +++ b/scripts/browser_canary.py @@ -122,6 +122,40 @@ def run_task(prompt: str, name: str, budget: int = 180) -> Dict[str, object]: return {"said": said, "status": status, "wall": round(time.time() - t0, 1), "log": slice_} +def marker_in_page(cfg: Dict[str, str], marker: str, handle: str, site: str) -> Optional[bool]: + """Is the marker ACTUALLY on the destination page? None when the read could not be made. + + The two checks around this one both take somebody's word for it. `sent_receipt=True` is OUR + mechanism reporting on itself, so a receipt bug reads as a delivered post; and the GONE/PRESENT + reply is a model summarising a page, which is the same model whose claim we are trying to audit. + Neither is destination-specific evidence. + + This reads the raw perception instead: navigate fresh, then grep the tool output in the log for + the marker string. No model judgement is consulted, only whether those characters came back from + the page. That is what makes a "posted" claim falsifiable, which is the whole point of counting + false successes. + """ + if not cfg.get("verify"): + return None + v = run_task(cfg["verify"].format(m=marker, handle=handle), f"canary-audit-{site}") + log = str(v["log"]) + if not log.strip(): + return None + # "the marker is not on the page" and "we never got a good look at the page" are DIFFERENT + # answers, and collapsing them is how this audit falsely accused LinkedIn of a false success: + # the post was really there (the cleanup that followed deleted it), the read just never + # surfaced its text. Absence only counts as evidence once the read itself is known good, so + # require proof we saw the destination at all before believing what we did not see on it. + saw_page = any(k in log for k in ("[browser-action] BrowserGetText", + "[browser-action] BrowserListInteractives")) + hit = any(marker in line for line in log.splitlines() + if "browser-action" not in line and "prompt" not in line.lower() + and "canary-audit" not in line) + if hit: + return True + return False if saw_page else None + + def check_site(site: str, cfg: Dict[str, str], live: bool) -> Dict[str, object]: marker = "canary" + secrets.token_hex(4) # no removal words, unique per run res: Dict[str, object] = {"site": site, "marker": marker, "live": live} @@ -149,9 +183,24 @@ def check_site(site: str, cfg: Dict[str, str], live: bool) -> Dict[str, object]: log = str(r["log"]) delivered = "done sent_receipt=True" in log or "DELIVERY CONFIRMED" in log res["post_wall"] = r["wall"] + # What the run TOLD the user, kept apart from what the page shows, because the gap between the + # two is the number that matters. A claim with no evidence under it is a false success, and it + # cannot be counted at all unless the two are recorded separately. + said = str(r["said"]) + res["claimed"] = delivered or bool( + re.search(r"\b(posted|published|tweeted|submitted|sent it|has been posted)\b", said, re.I)) + proven = marker_in_page(cfg, marker, handle, site) + res["proven"] = proven + res["false_success"] = bool(res["claimed"]) and proven is False if not delivered: res.update(stage="post", ok=False, - detail=f"no receipt (status={r['status']}): {str(r['said'])[:120]}") + detail=f"no receipt (status={r['status']}): {said[:120]}") + return res + if proven is False: + # The receipt fired and the destination does not have it. This is exactly the failure the + # canary was built for, and it must never be reported as a pass. + res.update(stage="post", ok=False, + detail=f"FALSE SUCCESS: receipt says sent, {marker} is not on the destination") return res # 2. DELETE, and require the in-page verify-gone, so cleanup can't be claimed falsely. @@ -159,6 +208,13 @@ def check_site(site: str, cfg: Dict[str, str], live: bool) -> Dict[str, object]: dlog = str(d["log"]) removed = "removed=True" in dlog res["delete_wall"] = d["wall"] + if not removed and cfg.get("verify") is not None: + # Same evidence channel as the post audit: gone means the characters are not coming back + # from the page, not that a model said "GONE". + still_there = marker_in_page(cfg, marker, handle, site) + if still_there is False: + removed = True + res["verified_by"] = "marker absent from raw page text" if not removed and cfg.get("verify"): # `removed=True` only exists on the BrowserDeleteItem dispatch path. A model-driven delete is # every bit as real, and grepping for the mechanism called DRIFT on a delete that had plainly @@ -203,6 +259,15 @@ def main() -> int: bad = [r for r in rows if not r.get("ok")] stranded = [r for r in bad if r.get("stage") == "cleanup"] print(f"\n{len(rows) - len(bad)}/{len(rows)} sites healthy") + if args.live: + proven = [r for r in rows if r.get("proven") is True] + unprovable = [r for r in rows if r.get("proven") is None] + liars = [r for r in rows if r.get("false_success")] + print(f"verified writes: {len(proven)}/{len(rows)} proven on the destination" + + (f", {len(unprovable)} unprovable (no audit read)" if unprovable else "")) + print(f"FALSE SUCCESS CLAIMS: {len(liars)}" + + (" <- hard gate, must be 0: " + ", ".join(str(r["site"]) for r in liars) + if liars else "")) if stranded: print("!! MANUAL CLEANUP NEEDED: " + ", ".join(f"{r['site']}:{r['marker']}" for r in stranded)) if bad: