[eric] browser: a post that never confirmed is not a skill worth learning

This commit is contained in:
ciregenz
2026-07-31 01:27:55 -07:00
parent 955abfa69c
commit 013bf663ef
3 changed files with 75 additions and 2 deletions
+12 -2
View File
@@ -44,6 +44,7 @@ from backend.apps.agents.browser.browser_loop import (
completion_is_honest,
deliverable_is_informational,
interstitial_dismiss_target,
is_publish_task,
is_removal_task,
recoverable_tool_error,
replay_recheck_is_safe,
@@ -3069,8 +3070,17 @@ async def run_browser_agent(
# turns while deleting nothing (measured live: repeat deletes all ghost-succeeded). Deletes
# always run fresh through the delete-dispatch.
p_is_removal = is_removal_task(skill_key_task)
logger.info(f"[browser-skills] record gate: honest={honest} informational={informational} removal={p_is_removal}")
if honest and not informational and not p_is_removal:
# The same trap as removals, one door down. A submit that never confirmed leaves an action_log
# of navigating and typing with NO send in it, so what distills is the scaffolding around a
# thing that did not happen. Measured live on reddit: the composer filled, then
# `send_button_found=False`, the agent blind-tapped a coordinate, nothing posted, and it still
# recorded a one-step "skill" (click the body textbox) for "create a text post and submit it".
# Replaying that reports done in one turn while posting nothing. A write only teaches us
# something once it actually went through.
p_unconfirmed_send = is_publish_task(skill_key_task) and not send_confirmed
logger.info(f"[browser-skills] record gate: honest={honest} informational={informational} "
f"removal={p_is_removal} unconfirmed_send={p_unconfirmed_send}")
if honest and not informational and not p_is_removal and not p_unconfirmed_send:
try:
rec_host = browser_skills.host_of(last_seen_url)
p_distilled = browser_skills.distill_steps(action_log)
@@ -298,6 +298,25 @@ def is_removal_task(task: str) -> bool:
return bool(P_DELETE_INTENT_RE.search(task or ""))
# Verbs that put something OUT into the world, as opposed to merely acting on a page. Deliberately
# narrower than task_is_send, which only means "not an informational ask" and so counts a plain
# "click the Search button": gating the skill store on THAT stopped the agent learning any click
# task at all, which is the whole speed mechanism.
P_PUBLISH_INTENT_RE = re.compile(
r"\b(post|submit|publish|send|tweet|comment|repl(?:y|ies)|dm|message)\b", re.I)
def is_publish_task(task: str) -> bool:
"""A task whose deliverable LEAVES the machine (a post, a reply, a message).
Keeps an unconfirmed write out of the skill store. Measured live on reddit: the composer filled,
`send_button_found=False`, the agent blind-tapped a coordinate, nothing posted, and a one-step
"skill" (click the body textbox) still got recorded for "create a text post and submit it".
Replaying that reports done in one turn while posting nothing, the same ghost the removal gate
already exists to stop."""
return bool(P_PUBLISH_INTENT_RE.search(task or ""))
def deliverable_is_informational(summary: str, task: str = "") -> bool:
"""True if the run's final answer is GATHERED CONTENT (a list/report the model
extracted or judged), not a short action confirmation. A deterministic replay
+44
View File
@@ -1913,3 +1913,47 @@ def test_login_wall_skip_does_not_remember(monkeypatch, tmp_path):
p_run_settled(task="log into acme", browser_id="b1", model="sonnet")
assert not H.is_authenticated("acme.example")
def test_an_unconfirmed_post_teaches_us_nothing(monkeypatch):
"""Measured live on reddit 2026-07-31: the composer filled, then `send_button_found=False`, the
agent blind-tapped a coordinate, NOTHING posted, and it still recorded a one-step "skill" (click
the body textbox) for "create a text post and submit it". Replaying that reports done in one turn
while posting nothing, the ghost class the removal gate already exists to stop.
Uses the SAME tool shape as test_skill_is_recorded_then_replayed_with_zero_llm_calls, which does
record, so the only difference here is that the ask publishes and the send never confirmed."""
import backend.apps.agents.browser.browser_skills as SK
SK.clear()
BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear()
task = "submit the post"
primary = FakeLLM([
Resp([p_rp("find the control"), p_tu("BrowserListInteractives")]),
Resp([p_rp("click it"), p_tu("BrowserClickIndex", index=1)]),
Resp([Blk("text", "Done, I sent it for you.")], stop_reason="end_turn"),
])
p_install(monkeypatch, primary, FakeAux())
asyncio.run(BA.run_browser_agent(
task=task, browser_id="b1", model="sonnet", initial_url=DOC_URL,
))
assert SK.find_skill("docs.google.com", task) is None, (
"an unconfirmed publish must never be distilled into a replayable skill")
def test_a_plain_click_task_still_learns(monkeypatch):
"""The publish gate must stay narrow. `task_is_send` only means "not an informational ask", so
keying on it stopped the agent learning ANY click task, which is the whole speed mechanism."""
import backend.apps.agents.browser.browser_skills as SK
SK.clear()
BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear()
primary = FakeLLM([
Resp([p_rp("find it"), p_tu("BrowserListInteractives")]),
Resp([p_rp("click it"), p_tu("BrowserClickIndex", index=1)]),
Resp([Blk("text", "Done, clicked Search.")], stop_reason="end_turn"),
])
p_install(monkeypatch, primary, FakeAux())
asyncio.run(BA.run_browser_agent(
task="click the Search button", browser_id="b1", model="sonnet", initial_url=DOC_URL,
))
assert SK.find_skill("docs.google.com", "click the Search button") is not None, (
"a non-publishing action task must still be learnable")