diff --git a/frontend/src/shared/browserCommandHandler.ts b/frontend/src/shared/browserCommandHandler.ts index 74436048..4b76ed90 100644 --- a/frontend/src/shared/browserCommandHandler.ts +++ b/frontend/src/shared/browserCommandHandler.ts @@ -8,7 +8,7 @@ import { shouldStopWaiting, SETTLE_POLL_MS, settleProbeJs } from './browserSettl let initialized = false; -export type BrowserAction = 'screenshot' | 'get_text' | 'get_console' | 'navigate' | 'click' | 'type' | 'evaluate' | 'get_elements' | 'scroll' | 'wait' | 'press_key' | 'list_interactives' | 'click_index' | 'batch' | 'detect_webmcp' | 'list_routes' | 'replay_route' | 'click_by_name'; +export type BrowserAction = 'screenshot' | 'get_text' | 'get_console' | 'navigate' | 'click' | 'type' | 'evaluate' | 'get_elements' | 'scroll' | 'wait' | 'press_key' | 'list_interactives' | 'click_index' | 'click_point' | 'batch' | 'detect_webmcp' | 'list_routes' | 'replay_route' | 'click_by_name'; export interface BrowserActivity { action: BrowserAction; @@ -68,6 +68,7 @@ const ACTION_LABELS: Record = { press_key: 'Pressing key...', list_interactives: 'Reading page structure...', click_index: 'Clicking element...', + click_point: 'Tapping screen...', click_by_name: 'Clicking element...', batch: 'Running batch...', }; @@ -349,6 +350,45 @@ async function handlePressKey(wv: BrowserWebview, params: Record): return { text: `Pressed ${rawKey}` }; } +// Click at a viewport coordinate (percent of the view's width/height) with a +// real, trusted CDP mouse event, NO DOM element required. This is what lets the +// app agent operate a bare game the way a person taps the screen: the +// AX-tree click paths (click_index/click_by_name) can't target a canvas because +// it exposes no nodes, but a coordinate dispatch lands anywhere. Optional +// hold_ms presses and holds (platformers, charge-up mechanics). +async function handleClickPoint(wv: BrowserWebview, params: Record): Promise> { + const xPercent = Number(params.xPercent); + const yPercent = Number(params.yPercent); + if (!Number.isFinite(xPercent) || !Number.isFinite(yPercent)) { + return { error: 'xPercent and yPercent are required (0-100, percent of the view).' }; + } + const cx = Math.max(0, Math.min(100, xPercent)); + const cy = Math.max(0, Math.min(100, yPercent)); + const button = params.button === 'right' ? 'right' : params.button === 'middle' ? 'middle' : 'left'; + const holdMs = Math.max(0, Math.min(Number(params.hold_ms) || 0, 5000)); + // Read the guest's own viewport so coords are correct under zoom/DPR, not the + // host element's box. One cheap round-trip; falls back to the element box. + let vw = wv.clientWidth, vh = wv.clientHeight; + try { + const d = await wv.executeJavaScript('({w: window.innerWidth, h: window.innerHeight})'); + if (d && d.w > 0 && d.h > 0) { vw = d.w; vh = d.h; } + } catch { /* use the element box as a fallback */ } + const x = (cx / 100) * vw; + const y = (cy / 100) * vh; + try { + await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x, y }); + await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button, clickCount: 1 }); + if (holdMs > 0) await new Promise((r) => setTimeout(r, holdMs)); + await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x, y, button, clickCount: 1 }); + } catch (err: any) { + return { error: `Click point failed: ${err?.message || String(err)}` }; + } + return { + text: `Clicked at (${Math.round(x)}, ${Math.round(y)})${holdMs ? ` held ${holdMs}ms` : ''}.`, + clickX: cx, clickY: cy, url: wv.getURL(), + }; +} + // CDP Accessibility.getFullAXTree sees computed roles/names even on hostile sites with unlabeled DOMs. const INTERACTIVE_ROLES = new Set([ 'button', 'link', 'textbox', 'combobox', 'checkbox', 'menuitem', @@ -961,11 +1001,12 @@ const MAX_BATCH_ACTIONS = 5; type SubActionType = | 'click_index' | 'press_key' | 'type' | 'wait' - | 'scroll' | 'navigate' | 'click' | 'list_interactives'; + | 'scroll' | 'navigate' | 'click' | 'click_point' | 'list_interactives'; const BATCH_DISPATCH: Record) => Promise>> = { click_index: handleClickIndex, press_key: handlePressKey, + click_point: handleClickPoint, type: handleType, wait: handleWait, scroll: handleScroll, @@ -1483,6 +1524,16 @@ async function runBrowserCommand( }); } break; + case 'click_point': + result = await handleClickPoint(wv, params); + if (result.clickX != null && result.clickY != null) { + setActivity(browser_id, { + action: 'click_point', + detail, + coords: { xPercent: result.clickX, yPercent: result.clickY }, + }); + } + break; case 'batch': result = await handleBatch(wv, params); break;