diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index 34b872fc..3eefabfd 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -1133,8 +1133,10 @@ async def run_browser_agent( else: result["text"] = ( f"{result.get('text') or ''}\nNOT confirmed: '{_expect}' did not appear within " - f"{_conf.get('waited_ms')}ms, so the action may not have worked. Check the page " - "before assuming success, and never re-fire an irreversible action " + f"{_conf.get('waited_ms')}ms. This only means that exact text was not found on " + "the page; if this result already contains direct evidence (e.g. 'Verified: the " + "box now contains ...'), TRUST THAT and do not redo the action. Otherwise check " + "the page before assuming success, and never re-fire an irreversible action " "(Send/Submit/Pay/Post) without first verifying the previous one did not go through." ) diff --git a/backend/apps/agents/browser/browser_schema.py b/backend/apps/agents/browser/browser_schema.py index 4fea9102..c42f12ed 100644 --- a/backend/apps/agents/browser/browser_schema.py +++ b/backend/apps/agents/browser/browser_schema.py @@ -18,11 +18,13 @@ MODEL_MAP = { _EXPECT_DESC = { "type": "string", "description": ( - "Optional but recommended: the specific change this action should cause, a " - "button label, text, or element you expect to see afterward (e.g. 'Write a " - "message', the recipient's name in the thread). It's confirmed right after, so " - "you learn whether it actually worked. REQUIRED for anything you can't undo " - "(Send/Submit/Pay/Post): set it to proof the action landed." + "Optional but recommended: LITERAL text that should be VISIBLE on the page " + "after this action; an exact button label, a person's name, the exact text you " + "just typed. Never a description of the change: 'message appears in box' is not " + "page text, can never match, and will always come back NOT confirmed. It's " + "checked right after, so you learn whether it actually worked. REQUIRED for " + "anything you can't undo (Send/Submit/Pay/Post): set it to proof the action " + "landed (for typing, the typed text itself)." ), } diff --git a/frontend/src/shared/browserCommandHandler.ts b/frontend/src/shared/browserCommandHandler.ts index fd6a9210..e33c2ed0 100644 --- a/frontend/src/shared/browserCommandHandler.ts +++ b/frontend/src/shared/browserCommandHandler.ts @@ -275,6 +275,7 @@ interface InteractiveElement { name: string; backendNodeId: number; sessionId?: string; + value?: string; } function extractAxValue(prop: any): string { @@ -394,7 +395,14 @@ function axNodesToCandidates(nodes: any[], sessionId?: string): RankItem[] { if (backendNodeId == null) continue; const shortName = name.slice(0, 80); if (twinOfAncestor(node, shortName)) continue; - out.push({ role, name: shortName, backendNodeId, sessionId, context: contextOf(node, name) }); + let value = ''; + if (role === 'textbox' || role === 'searchbox' || role === 'combobox') { + const isProtected = (node.properties || []).some( + (p: any) => p?.name === 'protected' && p?.value?.value === true, + ); + value = isProtected ? '' : extractAxValue(node.value).slice(0, 60); + } + out.push({ role, name: shortName, backendNodeId, sessionId, context: contextOf(node, name), value }); } return out; } @@ -518,25 +526,31 @@ async function clickBackendNode( if (typeof opts.text === 'string' && opts.text.length > 0) { // Read the text back from the node itself; "insert reported OK" is not // "the box has the text" (rich-text editors can swallow synthetic input). - const textLanded = async (): Promise => { + // Returns the box's actual content (or null on miss) so the result can + // echo the OBSERVED state; a bare "typed it" claim loses to a wrongly + // pessimistic expect-confirm and provokes a double-fill. + const readBack = async (): Promise => { try { const t = await sendCdp(wv, 'DOM.resolveNode', { backendNodeId }, sessionId); const r = await sendCdp(wv, 'Runtime.callFunctionOn', { objectId: t.object.objectId, functionDeclaration: - 'function(s) { const v = (this.value !== undefined ? this.value : this.textContent) || ""; return v.includes(s); }', + 'function(s) { const v = (this.value !== undefined ? this.value : this.textContent) || ""; return v.includes(s) ? v.slice(0, 120) : null; }', arguments: [{ value: opts.text }], returnByValue: true, }, sessionId); - return r?.result?.value === true; - } catch { return true; } // unverifiable beats a false alarm + return typeof r?.result?.value === 'string' ? r.result.value : null; + } catch { return opts.text ?? ''; } // unverifiable beats a false alarm }; + const landedMsg = (got: string, via = '') => + ({ text: `Focused ${label} and typed the text in${via}. Verified: the box now contains "${got}". Do NOT type it again.` }); try { await sendCdp(wv, 'Input.insertText', { text: opts.text }, sessionId); } catch (err: any) { return { error: `Focused ${label} but could not type into it: ${err?.message || String(err)}` }; } - if (await textLanded()) return { text: `Focused ${label} and typed the text in.` }; + let got = await readBack(); + if (got !== null) return landedMsg(got); try { const t = await sendCdp(wv, 'DOM.resolveNode', { backendNodeId }, sessionId); await sendCdp(wv, 'Runtime.callFunctionOn', { @@ -546,7 +560,8 @@ async function clickBackendNode( arguments: [{ value: opts.text }], }, sessionId); } catch { /* verified below; the honest error covers this failing too */ } - if (await textLanded()) return { text: `Focused ${label} and typed the text in (via editor command).` }; + got = await readBack(); + if (got !== null) return landedMsg(got, ' (via editor command)'); return { error: `Focused ${label} but the text did not register; the box may be a custom editor. Try BrowserPressKey per character or a different element.` }; } return { text: `Focused ${label}; the cursor is in it now (type with BrowserPressKey, or pass a text arg to fill it in one call).` }; @@ -726,6 +741,7 @@ async function handleListInteractives(wv: BrowserWebview, params: Record { const dup = (nameCounts.get(`${el.role}|${el.name}`) || 0) > 1; const ctx = dup && el.context ? ` ctx="${el.context}"` : ''; - return `[${el.index}]${el.isNew ? '*' : ''}<${el.role} "${el.name}"${ctx}>`; + const val = el.value ? ` value="${el.value}"` : ''; + return `[${el.index}]${el.isNew ? '*' : ''}<${el.role} "${el.name}"${ctx}${val}>`; }); let text: string; if (lines.length === 0) { diff --git a/frontend/src/shared/interactiveRanking.ts b/frontend/src/shared/interactiveRanking.ts index 834d9e0b..3d6eadec 100644 --- a/frontend/src/shared/interactiveRanking.ts +++ b/frontend/src/shared/interactiveRanking.ts @@ -16,6 +16,9 @@ export interface RankItem { // Nearby text that disambiguates same-named twins (the card/section this // element sits in, e.g. which "Message" button belongs to which person). context?: string; + // Current text of a textbox/searchbox/combobox. Without it a filled compose + // box still renders by its placeholder name and reads as "typing failed". + value?: string; } // Lower number = higher priority. Inputs the user types into first, then