[pierre] feat: handle click_point via native CDP mouse event in browser command handler

frontend/src/shared/browserCommandHandler.ts

@@ -8,7 +8,7 @@ import { shouldStopWaiting, SETTLE_POLL_MS, settleProbeJs } from './browserSettl
 
 let initialized = false;
 
-export type BrowserAction = 'screenshot' | 'get_text' | 'get_console' | 'navigate' | 'click' | 'type' | 'evaluate' | 'get_elements' | 'scroll' | 'wait' | 'press_key' | 'list_interactives' | 'click_index' | 'batch' | 'detect_webmcp' | 'list_routes' | 'replay_route' | 'click_by_name';
+export type BrowserAction = 'screenshot' | 'get_text' | 'get_console' | 'navigate' | 'click' | 'type' | 'evaluate' | 'get_elements' | 'scroll' | 'wait' | 'press_key' | 'list_interactives' | 'click_index' | 'click_point' | 'batch' | 'detect_webmcp' | 'list_routes' | 'replay_route' | 'click_by_name';
 
 export interface BrowserActivity {
   action: BrowserAction;
@@ -68,6 +68,7 @@ const ACTION_LABELS: Record<string, string> = {
   press_key: 'Pressing key...',
   list_interactives: 'Reading page structure...',
   click_index: 'Clicking element...',
+  click_point: 'Tapping screen...',
   click_by_name: 'Clicking element...',
   batch: 'Running batch...',
 };
@@ -349,6 +350,45 @@ async function handlePressKey(wv: BrowserWebview, params: Record<string, any>):
   return { text: `Pressed ${rawKey}` };
 }
 
+// Click at a viewport coordinate (percent of the view's width/height) with a
+// real, trusted CDP mouse event, NO DOM element required. This is what lets the
+// app agent operate a bare <canvas> game the way a person taps the screen: the
+// AX-tree click paths (click_index/click_by_name) can't target a canvas because
+// it exposes no nodes, but a coordinate dispatch lands anywhere. Optional
+// hold_ms presses and holds (platformers, charge-up mechanics).
+async function handleClickPoint(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
+  const xPercent = Number(params.xPercent);
+  const yPercent = Number(params.yPercent);
+  if (!Number.isFinite(xPercent) || !Number.isFinite(yPercent)) {
+    return { error: 'xPercent and yPercent are required (0-100, percent of the view).' };
+  }
+  const cx = Math.max(0, Math.min(100, xPercent));
+  const cy = Math.max(0, Math.min(100, yPercent));
+  const button = params.button === 'right' ? 'right' : params.button === 'middle' ? 'middle' : 'left';
+  const holdMs = Math.max(0, Math.min(Number(params.hold_ms) || 0, 5000));
+  // Read the guest's own viewport so coords are correct under zoom/DPR, not the
+  // host element's box. One cheap round-trip; falls back to the element box.
+  let vw = wv.clientWidth, vh = wv.clientHeight;
+  try {
+    const d = await wv.executeJavaScript('({w: window.innerWidth, h: window.innerHeight})');
+    if (d && d.w > 0 && d.h > 0) { vw = d.w; vh = d.h; }
+  } catch { /* use the element box as a fallback */ }
+  const x = (cx / 100) * vw;
+  const y = (cy / 100) * vh;
+  try {
+    await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x, y });
+    await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button, clickCount: 1 });
+    if (holdMs > 0) await new Promise((r) => setTimeout(r, holdMs));
+    await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x, y, button, clickCount: 1 });
+  } catch (err: any) {
+    return { error: `Click point failed: ${err?.message || String(err)}` };
+  }
+  return {
+    text: `Clicked at (${Math.round(x)}, ${Math.round(y)})${holdMs ? ` held ${holdMs}ms` : ''}.`,
+    clickX: cx, clickY: cy, url: wv.getURL(),
+  };
+}
+
 // CDP Accessibility.getFullAXTree sees computed roles/names even on hostile sites with unlabeled DOMs.
 const INTERACTIVE_ROLES = new Set([
   'button', 'link', 'textbox', 'combobox', 'checkbox', 'menuitem',
@@ -961,11 +1001,12 @@ const MAX_BATCH_ACTIONS = 5;
 
 type SubActionType =
   | 'click_index' | 'press_key' | 'type' | 'wait'
-  | 'scroll' | 'navigate' | 'click' | 'list_interactives';
+  | 'scroll' | 'navigate' | 'click' | 'click_point' | 'list_interactives';
 
 const BATCH_DISPATCH: Record<SubActionType, (wv: BrowserWebview, p: Record<string, any>) => Promise<Record<string, any>>> = {
   click_index: handleClickIndex,
   press_key: handlePressKey,
+  click_point: handleClickPoint,
   type: handleType,
   wait: handleWait,
   scroll: handleScroll,
@@ -1483,6 +1524,16 @@ async function runBrowserCommand(
           });
         }
         break;
+      case 'click_point':
+        result = await handleClickPoint(wv, params);
+        if (result.clickX != null && result.clickY != null) {
+          setActivity(browser_id, {
+            action: 'click_point',
+            detail,
+            coords: { xPercent: result.clickX, yPercent: result.clickY },
+          });
+        }
+        break;
       case 'batch':
         result = await handleBatch(wv, params);
         break;