From 5ca7308fa5d4e3ad892db861e0feeef435c09736 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Tue, 7 Jul 2026 15:05:41 -0700 Subject: [PATCH] [eric] browser: self-healing click (Phase 1) = a failed cached-index click auto-escalates to fresh by-name resolution in-code before the model ever sees the error (stale index = ~half of all runs' tool-errors, each costs a ~3s re-strategize turn); error-triggered so no double-act, OSW_SELFHEAL_CLICK=0 kill switch, 5 unit tests --- backend/apps/agents/browser/browser_agent.py | 3 +++ frontend/src/shared/browserCommandHandler.ts | 15 ++++++++++- frontend/src/shared/selfHealClick.test.ts | 26 ++++++++++++++++++++ frontend/src/shared/selfHealClick.ts | 9 +++++++ 4 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 frontend/src/shared/selfHealClick.test.ts create mode 100644 frontend/src/shared/selfHealClick.ts diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index 72e7352a..34ce049f 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -293,6 +293,9 @@ async def execute_browser_tool( return {"error": f"Unknown browser tool: {tool_name}"} params = {k: v for k, v in tool_input.items()} + # Self-healing click toggle (renderer escalates a stale-index failure to a fresh by-name click). Default on; OSW_SELFHEAL_CLICK=0 disables it for the A/B off-arm. + if action == "click_index": + params["selfheal"] = os.environ.get("OSW_SELFHEAL_CLICK", "1") != "0" request_id = uuid4().hex result = await ws_manager.send_browser_command( request_id, action, browser_id, params, tab_id=tab_id, diff --git a/frontend/src/shared/browserCommandHandler.ts b/frontend/src/shared/browserCommandHandler.ts index 3ede183d..15387d44 100644 --- a/frontend/src/shared/browserCommandHandler.ts +++ b/frontend/src/shared/browserCommandHandler.ts @@ -1,4 +1,5 @@ import { getWebview, findWebviewByDomain, hasDomReady, markDomReady, type BrowserWebview } from './browserRegistry'; +import { shouldSelfHealClick } from './selfHealClick'; import { store } from './state/store'; import { resumeBrowserCard } from './state/dashboardLayoutSlice'; import { dashboardWs } from './ws/WebSocketManager'; @@ -971,8 +972,20 @@ async function handleClickIndex(wv: BrowserWebview, params: Record) }; } + const wantsText = typeof params.text === 'string' && params.text.length > 0; const result = await clickBackendNode(wv, backendNodeId, sessionId, `index ${idx}`, - { role, text: typeof params.text === 'string' ? params.text : undefined }); + { role, text: wantsText ? params.text : undefined }); + // Self-healing escalation: a cached index goes stale the instant the page mutates, which is HALF of all runs' tool-errors. An explicit clickBackendNode error PROVES the click never landed (so re-trying the same target can't double-act), so before we hand a ~3s re-strategize turn back to the model, resolve the SAME element fresh from the full DOM by its name+role, the exact rung the model would have climbed to itself. Plain clicks only (a text-fill has its own readback path); gated so an A/B can turn it off. + if (shouldSelfHealClick(!!result.error, wantsText, name, params.selfheal)) { + const healed = await handleClickByName(wv, { name: name as string, role: role || '' }); + if (!healed.error) { + console.log(`[selfheal] index ${idx} stale -> recovered via name "${String(name).slice(0, 40)}"`); + healed.clickedRole = role || ''; + healed.clickedName = name || ''; + healed.selfHealed = 'by-name'; + return healed; + } + } // Surface what was clicked so the agent loop can record a stable, replayable click-by-name step (indices are ephemeral; names aren't). if (!result.error) { result.clickedRole = role || ''; diff --git a/frontend/src/shared/selfHealClick.test.ts b/frontend/src/shared/selfHealClick.test.ts new file mode 100644 index 00000000..d2cc86c3 --- /dev/null +++ b/frontend/src/shared/selfHealClick.test.ts @@ -0,0 +1,26 @@ +// Run: node --test frontend/src/shared/selfHealClick.test.ts +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { shouldSelfHealClick } from './selfHealClick.ts'; + +test('escalates a plain named click that errored', () => { + assert.equal(shouldSelfHealClick(true, false, 'Send', true), true); +}); + +test('never escalates a click that SUCCEEDED (no double-act)', () => { + assert.equal(shouldSelfHealClick(false, false, 'Send', true), false); +}); + +test('never escalates a text-fill (fills verify themselves)', () => { + assert.equal(shouldSelfHealClick(true, true, 'Write a message', true), false); +}); + +test('cannot escalate without a name to re-resolve by', () => { + assert.equal(shouldSelfHealClick(true, false, undefined, true), false); + assert.equal(shouldSelfHealClick(true, false, '', true), false); +}); + +test('A/B off-arm (selfheal === false) disables it; undefined/absent stays on', () => { + assert.equal(shouldSelfHealClick(true, false, 'Send', false), false); + assert.equal(shouldSelfHealClick(true, false, 'Send', undefined), true); +}); diff --git a/frontend/src/shared/selfHealClick.ts b/frontend/src/shared/selfHealClick.ts new file mode 100644 index 00000000..65e3099a --- /dev/null +++ b/frontend/src/shared/selfHealClick.ts @@ -0,0 +1,9 @@ +// Escalate a failed cached-index click to a fresh by-name resolution only when it's +// safe and possible: the click explicitly ERRORED (so it provably never landed, no +// double-act risk), it wasn't a text-fill (fills verify themselves), we know the +// element's name to re-find it, and the A/B toggle is on. Pure so the invariant is testable. +export function shouldSelfHealClick( + errored: boolean, wantsText: boolean, name: string | undefined, selfheal: unknown, +): boolean { + return errored && !wantsText && !!name && selfheal !== false; +}