[eric] browser: self-healing click (Phase 1) = a failed cached-index click auto-escalates to fresh by-name resolution in-code before the model ever sees the error (stale index = ~half of all runs' tool-errors, each costs a ~3s re-strategize turn); error-triggered so no double-act, OSW_SELFHEAL_CLICK=0 kill switch, 5 unit tests

This commit is contained in:
ciregenz
2026-07-07 15:05:41 -07:00
parent 6bdd75d978
commit 5ca7308fa5
4 changed files with 52 additions and 1 deletions
@@ -293,6 +293,9 @@ async def execute_browser_tool(
return {"error": f"Unknown browser tool: {tool_name}"}
params = {k: v for k, v in tool_input.items()}
# Self-healing click toggle (renderer escalates a stale-index failure to a fresh by-name click). Default on; OSW_SELFHEAL_CLICK=0 disables it for the A/B off-arm.
if action == "click_index":
params["selfheal"] = os.environ.get("OSW_SELFHEAL_CLICK", "1") != "0"
request_id = uuid4().hex
result = await ws_manager.send_browser_command(
request_id, action, browser_id, params, tab_id=tab_id,
+14 -1
View File
@@ -1,4 +1,5 @@
import { getWebview, findWebviewByDomain, hasDomReady, markDomReady, type BrowserWebview } from './browserRegistry';
import { shouldSelfHealClick } from './selfHealClick';
import { store } from './state/store';
import { resumeBrowserCard } from './state/dashboardLayoutSlice';
import { dashboardWs } from './ws/WebSocketManager';
@@ -971,8 +972,20 @@ async function handleClickIndex(wv: BrowserWebview, params: Record<string, any>)
};
}
const wantsText = typeof params.text === 'string' && params.text.length > 0;
const result = await clickBackendNode(wv, backendNodeId, sessionId, `index ${idx}`,
{ role, text: typeof params.text === 'string' ? params.text : undefined });
{ role, text: wantsText ? params.text : undefined });
// Self-healing escalation: a cached index goes stale the instant the page mutates, which is HALF of all runs' tool-errors. An explicit clickBackendNode error PROVES the click never landed (so re-trying the same target can't double-act), so before we hand a ~3s re-strategize turn back to the model, resolve the SAME element fresh from the full DOM by its name+role, the exact rung the model would have climbed to itself. Plain clicks only (a text-fill has its own readback path); gated so an A/B can turn it off.
if (shouldSelfHealClick(!!result.error, wantsText, name, params.selfheal)) {
const healed = await handleClickByName(wv, { name: name as string, role: role || '' });
if (!healed.error) {
console.log(`[selfheal] index ${idx} stale -> recovered via name "${String(name).slice(0, 40)}"`);
healed.clickedRole = role || '';
healed.clickedName = name || '';
healed.selfHealed = 'by-name';
return healed;
}
}
// Surface what was clicked so the agent loop can record a stable, replayable click-by-name step (indices are ephemeral; names aren't).
if (!result.error) {
result.clickedRole = role || '';
+26
View File
@@ -0,0 +1,26 @@
// Run: node --test frontend/src/shared/selfHealClick.test.ts
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { shouldSelfHealClick } from './selfHealClick.ts';
test('escalates a plain named click that errored', () => {
assert.equal(shouldSelfHealClick(true, false, 'Send', true), true);
});
test('never escalates a click that SUCCEEDED (no double-act)', () => {
assert.equal(shouldSelfHealClick(false, false, 'Send', true), false);
});
test('never escalates a text-fill (fills verify themselves)', () => {
assert.equal(shouldSelfHealClick(true, true, 'Write a message', true), false);
});
test('cannot escalate without a name to re-resolve by', () => {
assert.equal(shouldSelfHealClick(true, false, undefined, true), false);
assert.equal(shouldSelfHealClick(true, false, '', true), false);
});
test('A/B off-arm (selfheal === false) disables it; undefined/absent stays on', () => {
assert.equal(shouldSelfHealClick(true, false, 'Send', false), false);
assert.equal(shouldSelfHealClick(true, false, 'Send', undefined), true);
});
+9
View File
@@ -0,0 +1,9 @@
// Escalate a failed cached-index click to a fresh by-name resolution only when it's
// safe and possible: the click explicitly ERRORED (so it provably never landed, no
// double-act risk), it wasn't a text-fill (fills verify themselves), we know the
// element's name to re-find it, and the A/B toggle is on. Pure so the invariant is testable.
export function shouldSelfHealClick(
errored: boolean, wantsText: boolean, name: string | undefined, selfheal: unknown,
): boolean {
return errored && !wantsText && !!name && selfheal !== false;
}