mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-26 11:34:50 +02:00
[eric] browser: a post that never confirmed is not a skill worth learning
This commit is contained in:
@@ -44,6 +44,7 @@ from backend.apps.agents.browser.browser_loop import (
|
||||
completion_is_honest,
|
||||
deliverable_is_informational,
|
||||
interstitial_dismiss_target,
|
||||
is_publish_task,
|
||||
is_removal_task,
|
||||
recoverable_tool_error,
|
||||
replay_recheck_is_safe,
|
||||
@@ -3069,8 +3070,17 @@ async def run_browser_agent(
|
||||
# turns while deleting nothing (measured live: repeat deletes all ghost-succeeded). Deletes
|
||||
# always run fresh through the delete-dispatch.
|
||||
p_is_removal = is_removal_task(skill_key_task)
|
||||
logger.info(f"[browser-skills] record gate: honest={honest} informational={informational} removal={p_is_removal}")
|
||||
if honest and not informational and not p_is_removal:
|
||||
# The same trap as removals, one door down. A submit that never confirmed leaves an action_log
|
||||
# of navigating and typing with NO send in it, so what distills is the scaffolding around a
|
||||
# thing that did not happen. Measured live on reddit: the composer filled, then
|
||||
# `send_button_found=False`, the agent blind-tapped a coordinate, nothing posted, and it still
|
||||
# recorded a one-step "skill" (click the body textbox) for "create a text post and submit it".
|
||||
# Replaying that reports done in one turn while posting nothing. A write only teaches us
|
||||
# something once it actually went through.
|
||||
p_unconfirmed_send = is_publish_task(skill_key_task) and not send_confirmed
|
||||
logger.info(f"[browser-skills] record gate: honest={honest} informational={informational} "
|
||||
f"removal={p_is_removal} unconfirmed_send={p_unconfirmed_send}")
|
||||
if honest and not informational and not p_is_removal and not p_unconfirmed_send:
|
||||
try:
|
||||
rec_host = browser_skills.host_of(last_seen_url)
|
||||
p_distilled = browser_skills.distill_steps(action_log)
|
||||
|
||||
@@ -298,6 +298,25 @@ def is_removal_task(task: str) -> bool:
|
||||
return bool(P_DELETE_INTENT_RE.search(task or ""))
|
||||
|
||||
|
||||
# Verbs that put something OUT into the world, as opposed to merely acting on a page. Deliberately
|
||||
# narrower than task_is_send, which only means "not an informational ask" and so counts a plain
|
||||
# "click the Search button": gating the skill store on THAT stopped the agent learning any click
|
||||
# task at all, which is the whole speed mechanism.
|
||||
P_PUBLISH_INTENT_RE = re.compile(
|
||||
r"\b(post|submit|publish|send|tweet|comment|repl(?:y|ies)|dm|message)\b", re.I)
|
||||
|
||||
|
||||
def is_publish_task(task: str) -> bool:
|
||||
"""A task whose deliverable LEAVES the machine (a post, a reply, a message).
|
||||
|
||||
Keeps an unconfirmed write out of the skill store. Measured live on reddit: the composer filled,
|
||||
`send_button_found=False`, the agent blind-tapped a coordinate, nothing posted, and a one-step
|
||||
"skill" (click the body textbox) still got recorded for "create a text post and submit it".
|
||||
Replaying that reports done in one turn while posting nothing, the same ghost the removal gate
|
||||
already exists to stop."""
|
||||
return bool(P_PUBLISH_INTENT_RE.search(task or ""))
|
||||
|
||||
|
||||
def deliverable_is_informational(summary: str, task: str = "") -> bool:
|
||||
"""True if the run's final answer is GATHERED CONTENT (a list/report the model
|
||||
extracted or judged), not a short action confirmation. A deterministic replay
|
||||
|
||||
@@ -1913,3 +1913,47 @@ def test_login_wall_skip_does_not_remember(monkeypatch, tmp_path):
|
||||
p_run_settled(task="log into acme", browser_id="b1", model="sonnet")
|
||||
|
||||
assert not H.is_authenticated("acme.example")
|
||||
|
||||
|
||||
def test_an_unconfirmed_post_teaches_us_nothing(monkeypatch):
|
||||
"""Measured live on reddit 2026-07-31: the composer filled, then `send_button_found=False`, the
|
||||
agent blind-tapped a coordinate, NOTHING posted, and it still recorded a one-step "skill" (click
|
||||
the body textbox) for "create a text post and submit it". Replaying that reports done in one turn
|
||||
while posting nothing, the ghost class the removal gate already exists to stop.
|
||||
|
||||
Uses the SAME tool shape as test_skill_is_recorded_then_replayed_with_zero_llm_calls, which does
|
||||
record, so the only difference here is that the ask publishes and the send never confirmed."""
|
||||
import backend.apps.agents.browser.browser_skills as SK
|
||||
SK.clear()
|
||||
BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear()
|
||||
task = "submit the post"
|
||||
primary = FakeLLM([
|
||||
Resp([p_rp("find the control"), p_tu("BrowserListInteractives")]),
|
||||
Resp([p_rp("click it"), p_tu("BrowserClickIndex", index=1)]),
|
||||
Resp([Blk("text", "Done, I sent it for you.")], stop_reason="end_turn"),
|
||||
])
|
||||
p_install(monkeypatch, primary, FakeAux())
|
||||
asyncio.run(BA.run_browser_agent(
|
||||
task=task, browser_id="b1", model="sonnet", initial_url=DOC_URL,
|
||||
))
|
||||
assert SK.find_skill("docs.google.com", task) is None, (
|
||||
"an unconfirmed publish must never be distilled into a replayable skill")
|
||||
|
||||
|
||||
def test_a_plain_click_task_still_learns(monkeypatch):
|
||||
"""The publish gate must stay narrow. `task_is_send` only means "not an informational ask", so
|
||||
keying on it stopped the agent learning ANY click task, which is the whole speed mechanism."""
|
||||
import backend.apps.agents.browser.browser_skills as SK
|
||||
SK.clear()
|
||||
BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear()
|
||||
primary = FakeLLM([
|
||||
Resp([p_rp("find it"), p_tu("BrowserListInteractives")]),
|
||||
Resp([p_rp("click it"), p_tu("BrowserClickIndex", index=1)]),
|
||||
Resp([Blk("text", "Done, clicked Search.")], stop_reason="end_turn"),
|
||||
])
|
||||
p_install(monkeypatch, primary, FakeAux())
|
||||
asyncio.run(BA.run_browser_agent(
|
||||
task="click the Search button", browser_id="b1", model="sonnet", initial_url=DOC_URL,
|
||||
))
|
||||
assert SK.find_skill("docs.google.com", "click the Search button") is not None, (
|
||||
"a non-publishing action task must still be learnable")
|
||||
|
||||
Reference in New Issue
Block a user