mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-11 04:07:44 +02:00
[eric] browser: click-effect metric (measure before fixing) = fingerprint the page before/after a click to catch the INVISIBLE failure the tool-error counter misses (a click that succeeds but hits the wrong/dead element = nothing changes); flag-gated OSW_CLICK_EFFECT_PROBE, zero cost off, 3 unit tests
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
import { getWebview, findWebviewByDomain, hasDomReady, markDomReady, type BrowserWebview } from './browserRegistry';
|
||||
import { shouldSelfHealClick } from './selfHealClick';
|
||||
import { FP_EXPR, clickEffect } from './clickEffect';
|
||||
import { store } from './state/store';
|
||||
import { resumeBrowserCard } from './state/dashboardLayoutSlice';
|
||||
import { dashboardWs } from './ws/WebSocketManager';
|
||||
@@ -655,7 +656,7 @@ async function enumerateCandidates(wv: BrowserWebview): Promise<RankItem[]> {
|
||||
// Resolve + click a specific backend node (revalidate, frame-local box model, OS-level dispatch in the element's own frame, cosmetic top-level ripple). Shared by click_index (cache lookup) and click_by_name (fresh resolution).
|
||||
async function clickBackendNode(
|
||||
wv: BrowserWebview, backendNodeId: number, sessionId: string | undefined, label: string,
|
||||
opts: { role?: string; text?: string } = {},
|
||||
opts: { role?: string; text?: string; effectProbe?: boolean } = {},
|
||||
): Promise<Record<string, any>> {
|
||||
let resolvedObjectId: string | undefined;
|
||||
try {
|
||||
@@ -784,16 +785,24 @@ async function clickBackendNode(
|
||||
}
|
||||
}
|
||||
|
||||
// Metric only (flag-gated): fingerprint the page just before the click so we can
|
||||
// tell afterwards whether it actually did anything. Off by default = zero cost.
|
||||
let fpBefore = '';
|
||||
if (opts.effectProbe) { try { fpBefore = String(await wv.executeJavaScript(FP_EXPR)); } catch { /* ignore */ } }
|
||||
try {
|
||||
await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mousePressed', x: lx, y: ly, button: 'left', clickCount: 1 }, sessionId);
|
||||
await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x: lx, y: ly, button: 'left', clickCount: 1 }, sessionId);
|
||||
} catch (err: any) {
|
||||
return { error: `Click failed: ${err.message || String(err)}` };
|
||||
}
|
||||
return {
|
||||
text: `Clicked ${label} at (${Math.round(rx)}, ${Math.round(ry)})`,
|
||||
...ripple,
|
||||
};
|
||||
const out: Record<string, any> = { text: `Clicked ${label} at (${Math.round(rx)}, ${Math.round(ry)})`, ...ripple };
|
||||
if (opts.effectProbe) {
|
||||
await new Promise((r) => setTimeout(r, 400));
|
||||
let fpAfter = '';
|
||||
try { fpAfter = String(await wv.executeJavaScript(FP_EXPR)); } catch { /* ignore */ }
|
||||
out.clickEffect = clickEffect(fpBefore, fpAfter);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Drop list rows the user literally cannot click: zero-size nodes and ones whose center hits a DIFFERENT element (modal backdrop, sticky header, cookie banner). Ground truth via elementFromPoint, the same predicate the click path trusts. Offscreen-but-scrollable elements are kept; the page-wide list is deliberately wider than the viewport. Chunked with a hard budget so a heavy page degrades to an unfiltered list, never a stall.
|
||||
@@ -974,7 +983,7 @@ async function handleClickIndex(wv: BrowserWebview, params: Record<string, any>)
|
||||
|
||||
const wantsText = typeof params.text === 'string' && params.text.length > 0;
|
||||
const result = await clickBackendNode(wv, backendNodeId, sessionId, `index ${idx}`,
|
||||
{ role, text: wantsText ? params.text : undefined });
|
||||
{ role, text: wantsText ? params.text : undefined, effectProbe: params.effectProbe === true });
|
||||
// Self-healing escalation: a cached index goes stale the instant the page mutates, which is HALF of all runs' tool-errors. An explicit clickBackendNode error PROVES the click never landed (so re-trying the same target can't double-act), so before we hand a ~3s re-strategize turn back to the model, resolve the SAME element fresh from the full DOM by its name+role, the exact rung the model would have climbed to itself. Plain clicks only (a text-fill has its own readback path); gated so an A/B can turn it off.
|
||||
if (shouldSelfHealClick(!!result.error, wantsText, name, params.selfheal)) {
|
||||
const healed = await handleClickByName(wv, { name: name as string, role: role || '' });
|
||||
|
||||
Reference in New Issue
Block a user