diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index 57634785..fbf35b32 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -44,6 +44,7 @@ from backend.apps.agents.browser.browser_loop import ( completion_is_honest, deliverable_is_informational, interstitial_dismiss_target, + is_publish_task, is_removal_task, recoverable_tool_error, replay_recheck_is_safe, @@ -3069,8 +3070,17 @@ async def run_browser_agent( # turns while deleting nothing (measured live: repeat deletes all ghost-succeeded). Deletes # always run fresh through the delete-dispatch. p_is_removal = is_removal_task(skill_key_task) - logger.info(f"[browser-skills] record gate: honest={honest} informational={informational} removal={p_is_removal}") - if honest and not informational and not p_is_removal: + # The same trap as removals, one door down. A submit that never confirmed leaves an action_log + # of navigating and typing with NO send in it, so what distills is the scaffolding around a + # thing that did not happen. Measured live on reddit: the composer filled, then + # `send_button_found=False`, the agent blind-tapped a coordinate, nothing posted, and it still + # recorded a one-step "skill" (click the body textbox) for "create a text post and submit it". + # Replaying that reports done in one turn while posting nothing. A write only teaches us + # something once it actually went through. + p_unconfirmed_send = is_publish_task(skill_key_task) and not send_confirmed + logger.info(f"[browser-skills] record gate: honest={honest} informational={informational} " + f"removal={p_is_removal} unconfirmed_send={p_unconfirmed_send}") + if honest and not informational and not p_is_removal and not p_unconfirmed_send: try: rec_host = browser_skills.host_of(last_seen_url) p_distilled = browser_skills.distill_steps(action_log) diff --git a/backend/apps/agents/browser/browser_loop.py b/backend/apps/agents/browser/browser_loop.py index 5050ffd7..c34b2250 100644 --- a/backend/apps/agents/browser/browser_loop.py +++ b/backend/apps/agents/browser/browser_loop.py @@ -298,6 +298,25 @@ def is_removal_task(task: str) -> bool: return bool(P_DELETE_INTENT_RE.search(task or "")) +# Verbs that put something OUT into the world, as opposed to merely acting on a page. Deliberately +# narrower than task_is_send, which only means "not an informational ask" and so counts a plain +# "click the Search button": gating the skill store on THAT stopped the agent learning any click +# task at all, which is the whole speed mechanism. +P_PUBLISH_INTENT_RE = re.compile( + r"\b(post|submit|publish|send|tweet|comment|repl(?:y|ies)|dm|message)\b", re.I) + + +def is_publish_task(task: str) -> bool: + """A task whose deliverable LEAVES the machine (a post, a reply, a message). + + Keeps an unconfirmed write out of the skill store. Measured live on reddit: the composer filled, + `send_button_found=False`, the agent blind-tapped a coordinate, nothing posted, and a one-step + "skill" (click the body textbox) still got recorded for "create a text post and submit it". + Replaying that reports done in one turn while posting nothing, the same ghost the removal gate + already exists to stop.""" + return bool(P_PUBLISH_INTENT_RE.search(task or "")) + + def deliverable_is_informational(summary: str, task: str = "") -> bool: """True if the run's final answer is GATHERED CONTENT (a list/report the model extracted or judged), not a short action confirmation. A deterministic replay diff --git a/backend/tests/test_browser_agent_loop.py b/backend/tests/test_browser_agent_loop.py index fed80478..2ac3bf5c 100644 --- a/backend/tests/test_browser_agent_loop.py +++ b/backend/tests/test_browser_agent_loop.py @@ -1913,3 +1913,47 @@ def test_login_wall_skip_does_not_remember(monkeypatch, tmp_path): p_run_settled(task="log into acme", browser_id="b1", model="sonnet") assert not H.is_authenticated("acme.example") + + +def test_an_unconfirmed_post_teaches_us_nothing(monkeypatch): + """Measured live on reddit 2026-07-31: the composer filled, then `send_button_found=False`, the + agent blind-tapped a coordinate, NOTHING posted, and it still recorded a one-step "skill" (click + the body textbox) for "create a text post and submit it". Replaying that reports done in one turn + while posting nothing, the ghost class the removal gate already exists to stop. + + Uses the SAME tool shape as test_skill_is_recorded_then_replayed_with_zero_llm_calls, which does + record, so the only difference here is that the ask publishes and the send never confirmed.""" + import backend.apps.agents.browser.browser_skills as SK + SK.clear() + BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear() + task = "submit the post" + primary = FakeLLM([ + Resp([p_rp("find the control"), p_tu("BrowserListInteractives")]), + Resp([p_rp("click it"), p_tu("BrowserClickIndex", index=1)]), + Resp([Blk("text", "Done, I sent it for you.")], stop_reason="end_turn"), + ]) + p_install(monkeypatch, primary, FakeAux()) + asyncio.run(BA.run_browser_agent( + task=task, browser_id="b1", model="sonnet", initial_url=DOC_URL, + )) + assert SK.find_skill("docs.google.com", task) is None, ( + "an unconfirmed publish must never be distilled into a replayable skill") + + +def test_a_plain_click_task_still_learns(monkeypatch): + """The publish gate must stay narrow. `task_is_send` only means "not an informational ask", so + keying on it stopped the agent learning ANY click task, which is the whole speed mechanism.""" + import backend.apps.agents.browser.browser_skills as SK + SK.clear() + BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear() + primary = FakeLLM([ + Resp([p_rp("find it"), p_tu("BrowserListInteractives")]), + Resp([p_rp("click it"), p_tu("BrowserClickIndex", index=1)]), + Resp([Blk("text", "Done, clicked Search.")], stop_reason="end_turn"), + ]) + p_install(monkeypatch, primary, FakeAux()) + asyncio.run(BA.run_browser_agent( + task="click the Search button", browser_id="b1", model="sonnet", initial_url=DOC_URL, + )) + assert SK.find_skill("docs.google.com", "click the Search button") is not None, ( + "a non-publishing action task must still be learnable")