mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-08-17 18:25:42 +02:00
[eric] browser: never click a disabled submit, and turn the structural composer finder on
This commit is contained in:
@@ -2705,6 +2705,16 @@ async def run_browser_agent(
|
||||
else:
|
||||
task = f"{task}\n\n[{p_cs.get('note')}]"
|
||||
logger.info(f"[browser-autosend {session_id}] post-fill send click ran, receipt unverified; model verifies")
|
||||
elif p_cs.get("note"):
|
||||
# Nothing was clicked, and the reason is worth more than the silence:
|
||||
# a disabled submit means the FORM is incomplete, which is the one
|
||||
# failure here the model can actually fix (fill the title, the subject,
|
||||
# the missing required field). Dropping this note left it re-clicking a
|
||||
# greyed-out button until the turn budget ran out.
|
||||
task = f"{task}\n\n[{p_cs.get('note')}]"
|
||||
action_log.extend(p_cs.get("log") or [])
|
||||
logger.info(f"[browser-autosend {session_id}] no send click ran; handed the "
|
||||
f"model the reason instead")
|
||||
|
||||
# One gentle nudge per violating turn, folded onto the action that ran, so the model self-corrects next turn without us costing it one.
|
||||
if rp_reminder_pending and tu.name in ACTION_TOOLS_REQUIRING_REPORT:
|
||||
|
||||
@@ -97,6 +97,21 @@ async def complete_send(
|
||||
{"xPercent": float(p_v["xPct"]), "yPercent": float(p_v["yPct"])}, browser_id, tab_id)
|
||||
send_name = str(p_v.get("name") or "submit")
|
||||
via = "container"
|
||||
elif isinstance(p_v, dict) and p_v.get("disabled"):
|
||||
# The submit EXISTS and the site is refusing it, so there is nothing here a better
|
||||
# click could win: guessing at the literal "Send" would tap some other widget and the
|
||||
# cleared composer would read as delivery. Measured live on reddit's r/test/submit,
|
||||
# which is the whole shape of issue #94: its "post" button sits greyed out until the
|
||||
# title field is filled, and every run blind-tapped a coordinate and claimed success.
|
||||
# Handing back untouched is what lets the model do the one thing that CAN fix this,
|
||||
# fill the rest of the form, and it costs a run that was never going to send anyway.
|
||||
logger.info(f"[browser-sendscript] submit {str(p_v.get('name'))!r} is present but DISABLED; "
|
||||
f"the form is incomplete, handing to the model without clicking anything")
|
||||
return {"clicked": False, "sent": False, "log": log,
|
||||
"note": (f"The {str(p_v.get('name')) or 'submit'} button is visible but disabled, so this "
|
||||
f"form is not ready to send: something it requires is still empty (often a "
|
||||
f"title or subject), or the editor never registered the typed text. Fill the "
|
||||
f"remaining fields, then send. Nothing was clicked and nothing was posted.")}
|
||||
else:
|
||||
p_why = p_v.get("why") if isinstance(p_v, dict) else "unreadable eval"
|
||||
logger.info(f"[browser-sendscript] container submit miss ({p_why}); by-name fallback")
|
||||
@@ -192,7 +207,13 @@ async def run_send_script(
|
||||
prompt; the composed task carries the routing brief whose own quoted strings
|
||||
made every real payload look ambiguous (r242/r243)."""
|
||||
t0 = time.monotonic()
|
||||
p_struct = os.environ.get("OSW_COMPOSER_STRUCT") == "1"
|
||||
# Default ON as of 2026-07-31, on a measured sweep over the 9 sites this profile is genuinely
|
||||
# signed into: composer reach 3/9 -> 6/9, submit resolution 2/9 -> 5/9. The name-based detector
|
||||
# it backs up only sees a composer that carries a recognisable accessible name, which is a
|
||||
# minority of the web; instagram reached its composer on this tier and nothing else. It costs
|
||||
# one page scan on a page that has no composer, and every gate downstream (quoted payload,
|
||||
# fill-seen-committed, two-sided receipt) is unchanged, so a wider search cannot loosen safety.
|
||||
p_struct = os.environ.get("OSW_COMPOSER_STRUCT", "1") != "0"
|
||||
|
||||
async def fresh_list() -> str:
|
||||
try:
|
||||
@@ -225,8 +246,10 @@ async def run_send_script(
|
||||
if browser_send_parse.is_readonly(task_sans_brief) or (payload_source and browser_send_parse.is_readonly(payload_source)):
|
||||
logger.info("[browser-sendscript] decline: read-only directive in user request")
|
||||
return None
|
||||
if browser_send_parse.looks_like_login_wall(current_url, state_text):
|
||||
logger.info(f"[browser-sendscript] decline: login/auth wall ({(current_url or '')[:60]!r})")
|
||||
p_wall = browser_send_parse.login_wall_reason(current_url, state_text)
|
||||
if p_wall:
|
||||
logger.info(f"[browser-sendscript] decline: login/auth wall ({(current_url or '')[:60]!r}) "
|
||||
f"triggered by {p_wall}")
|
||||
return None
|
||||
payload = browser_send_parse.quoted_payload(payload_source or task)
|
||||
if not payload:
|
||||
@@ -299,7 +322,11 @@ async def run_send_script(
|
||||
# OSW_COMPOSER_REVEAL: let the finder take one reversible reveal action (open the
|
||||
# compose surface: a modal trigger, the first conversation, or a scroll) when the
|
||||
# composer isn't painted yet. It never commits a send, only opens a surface.
|
||||
p_reveal = os.environ.get("OSW_COMPOSER_REVEAL") == "1"
|
||||
# Default ON with the same sweep behind it: youtube's comment box exists only after a
|
||||
# scroll and a click on its placeholder, so no amount of scanning a painted page finds
|
||||
# it, and it was the single site this tier won. Reveal never commits anything: it opens
|
||||
# a surface, and its HARDBLOCK list keeps it off send/submit/pay/delete controls.
|
||||
p_reveal = os.environ.get("OSW_COMPOSER_REVEAL", "1") != "0"
|
||||
# A reveal that OPENS the first list item (a Reddit thread, a TikTok video, a GitHub
|
||||
# issue) is a full-page NAVIGATION: it kills the finder's own JS context, so that one
|
||||
# call can't reach the composer that only exists on the destination. When the finder
|
||||
@@ -416,7 +443,22 @@ async def run_send_script(
|
||||
# found, fill committed); we stop before the irreversible click and report readiness.
|
||||
if os.environ.get("OSW_SENDSCRIPT_DRYRUN") == "1":
|
||||
send_ready = bool(send_index_in_state(state2, composer[0]))
|
||||
logger.info(f"[browser-sendscript] DRYRUN: WOULD send (fill committed, send_button_listed={send_ready}); not clicking")
|
||||
# Resolve the submit too, WITHOUT clicking it: the resolver is a page read that hands back
|
||||
# coordinates, so a dry run can measure both halves of coverage (did we reach a composer AND
|
||||
# can we find its send) on the great majority of sites we are never allowed to post to.
|
||||
# Measuring only the fill is exactly how a resolver that could not find reddit's button hid
|
||||
# behind a passing suite: every dry sweep said "ready to send" about a send that would have
|
||||
# blind-tapped a coordinate.
|
||||
r_ev = await execute_tool(
|
||||
"BrowserEvaluate",
|
||||
{"expression": browser_submit_click.container_submit_expression(payload)}, browser_id, tab_id)
|
||||
p_v = browser_submit_click.parse_eval_value(r_ev)
|
||||
p_ok = bool(isinstance(p_v, dict) and p_v.get("ok"))
|
||||
p_named = (str(p_v.get("name") or "") if p_ok else
|
||||
str(p_v.get("why") or "unreadable eval") if isinstance(p_v, dict) else "unreadable eval")
|
||||
logger.info(f"[browser-sendscript] DRYRUN: WOULD send (fill committed, "
|
||||
f"send_button_listed={send_ready}, submit_resolved={p_ok}, "
|
||||
f"submit_rank={p_v.get('rank') if p_ok else 0}, submit={p_named!r}); not clicking")
|
||||
return {"sent": False, "payload": payload, "log": log,
|
||||
"note": "DRYRUN: filled + ready to send, stopped before the irreversible click"}
|
||||
# 3+4: the irreversible click + two-sided receipt, shared with the mid-loop takeover. A click error hands back to the model (fill committed, not sent); a clicked-but-unverified send returns sent=False so the caller never claims delivery.
|
||||
|
||||
@@ -20,7 +20,8 @@ COMMITTED = f'[2]<textbox "Write a message" value="{PAYLOAD}">'
|
||||
CLEARED = '[2]<textbox "Write a message">\n[9]<button "Attach">'
|
||||
|
||||
|
||||
def make_exec(*, submit_listed: bool, container_ok: bool, visible_after: bool, cleared: bool = True):
|
||||
def make_exec(*, submit_listed: bool, container_ok: bool, visible_after: bool, cleared: bool = True,
|
||||
container_disabled: bool = False):
|
||||
"""A composer whose submit is or isn't resolvable, and a page that does or doesn't end up
|
||||
showing the payload. Records which route the tail took."""
|
||||
calls = {"clicks": [], "lists": 0, "evals": []}
|
||||
@@ -40,6 +41,9 @@ def make_exec(*, submit_listed: bool, container_ok: bool, visible_after: bool, c
|
||||
return {"text": f'{{"visible": {str(visible_after).lower()}}}'}
|
||||
if container_ok: # the container submit resolver
|
||||
return {"text": '{"ok": true, "xPct": 50.0, "yPct": 50.0, "name": "Post"}'}
|
||||
if container_disabled: # found it, the site is refusing it
|
||||
return {"text": '{"ok": false, "disabled": true, "name": "post", '
|
||||
'"why": "the submit control is present but DISABLED"}'}
|
||||
return {"text": '{"ok": false, "why": "no submit control in the composer container"}'}
|
||||
calls["clicks"].append((tool, params))
|
||||
return {"ok": True}
|
||||
@@ -103,3 +107,26 @@ async def test_a_composer_that_never_cleared_is_still_not_sent():
|
||||
composer still holds the text, nothing was sent no matter what else is on the page."""
|
||||
r, _ = await run_tail(submit_listed=False, container_ok=False, visible_after=True, cleared=False)
|
||||
assert r["sent"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_disabled_submit_clicks_absolutely_nothing():
|
||||
"""Issue #94, the reddit shape. Its submit is present and greyed out until the title is filled.
|
||||
The old code read that as "no submit control", guessed at a button named Send, and the cleared
|
||||
composer then read as a delivery for a post that never existed. A control the SITE is refusing
|
||||
cannot be won by clicking harder, so the only correct move is to touch nothing."""
|
||||
r, calls = await run_tail(submit_listed=False, container_ok=False, container_disabled=True,
|
||||
visible_after=False)
|
||||
assert calls["clicks"] == [], "a disabled submit must not produce ANY click, guessed or otherwise"
|
||||
assert r["clicked"] is False and r["sent"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_disabled_submit_says_what_is_actually_wrong():
|
||||
"""The note is the whole value of the fix: it is the one failure here a model can repair, by
|
||||
filling whatever the form still wants. It has to name that, and it must not imply a post."""
|
||||
r, _ = await run_tail(submit_listed=False, container_ok=False, container_disabled=True,
|
||||
visible_after=False)
|
||||
note = str(r["note"]).lower()
|
||||
assert "disabled" in note and "nothing was posted" in note
|
||||
assert "title" in note, "name the usual culprit so the model knows where to look"
|
||||
|
||||
@@ -67,10 +67,11 @@ async def test_structural_finder_declines_when_no_editable(monkeypatch):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_structural_off_by_default_never_calls_finder(monkeypatch):
|
||||
"""Flag off = the proven name path only; a name-less perception declines and the finder
|
||||
is never invoked (the structural path can't perturb the default)."""
|
||||
monkeypatch.delenv("OSW_COMPOSER_STRUCT", raising=False)
|
||||
async def test_the_structural_kill_switch_really_kills_it(monkeypatch):
|
||||
"""The finder went default-ON (composer reach 3/9 -> 6/9 measured live), so the thing worth
|
||||
pinning is no longer the default but the escape hatch: =0 must fall all the way back to the
|
||||
proven name path with the finder never invoked, so a site it upsets is fixable without a build."""
|
||||
monkeypatch.setenv("OSW_COMPOSER_STRUCT", "0")
|
||||
ex, calls = make_struct_exec({"found": True, "filled": True, "selector": "x", "role": "textarea"})
|
||||
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_submit_index_in_state,
|
||||
payload_in_textbox, payload_source=TASK, current_url="https://example.com/")
|
||||
@@ -131,11 +132,12 @@ async def test_cross_nav_no_retry_when_reveal_did_not_navigate(monkeypatch):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reveal_flag_off_by_default(monkeypatch):
|
||||
"""Struct on but reveal unset: the finder is still called, but reveal=False, so it only
|
||||
scans what's painted and never clicks a trigger (the safe default)."""
|
||||
async def test_the_reveal_kill_switch_leaves_the_finder_scanning_only(monkeypatch):
|
||||
"""Reveal also went default-ON (it is the only tier that reaches youtube's comment box, which
|
||||
exists only after a scroll and a click). =0 must still call the finder but forbid it from
|
||||
touching anything: scan what is painted, click no trigger."""
|
||||
monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
|
||||
monkeypatch.delenv("OSW_COMPOSER_REVEAL", raising=False)
|
||||
monkeypatch.setenv("OSW_COMPOSER_REVEAL", "0")
|
||||
ex, calls = make_struct_exec({"found": False})
|
||||
await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_submit_index_in_state,
|
||||
payload_in_textbox, payload_source=TASK, current_url="https://example.com/")
|
||||
|
||||
@@ -58,14 +58,19 @@ X_COMPOSER = '[3]<textbox "Post your reply">\n[8]<button "Reply">' # X, not Lin
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_surface_gate_declines_when_no_composer_in_perception():
|
||||
"""STRUCTURAL gate: a page whose perception has no compose-shaped textbox and no
|
||||
messaging opener declines UNTOUCHED, regardless of URL, so the script never fires
|
||||
where a fill would land nowhere useful. Stays composer-less through the poll."""
|
||||
"""STRUCTURAL gate: a page whose perception has no compose-shaped textbox and no messaging
|
||||
opener declines, regardless of URL, so the script never fires where a fill would land nowhere
|
||||
useful. Stays composer-less through the poll.
|
||||
|
||||
Since the structural finder went default-ON it gets ONE look here (that is the point of it, and
|
||||
it is how instagram reaches its composer at all). So "untouched" is now specifically: it may ASK
|
||||
the page whether it has an editable, and it may do nothing else. No index fill, no click, no
|
||||
send. The finder that answers "found nothing" must still put the run on the model path."""
|
||||
ex, calls = make_exec([NO_COMPOSER, NO_COMPOSER, NO_COMPOSER, NO_COMPOSER])
|
||||
r = await ss.run_send_script(TASK, "b1", "", NO_COMPOSER, ex, send_submit_index_in_state,
|
||||
payload_in_textbox, payload_source=TASK, current_url=FEED_URL)
|
||||
assert r is None
|
||||
assert not calls["clicks"]
|
||||
assert [t for t, _ in calls["clicks"]] == ["BrowserFindComposer"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
Reference in New Issue
Block a user