From 6e1fc1dee20994ddb6feffc199a3873079239a5d Mon Sep 17 00:00:00 2001 From: haikdc Date: Tue, 17 Mar 2026 23:52:20 -0700 Subject: [PATCH] [Haik]: ckpt (Round 1 of fixing mcp server issues in prod - still has bugs) (Fixed some browser related issues like popup windows rendering out of app and weird js framework related errors when u open sites like spotify) (Added option in settings to toggle item selection to be on by default when selecting a new chat) (Browser agent actions now show up in chat output) (Browser agent actions each have their own custom icon) --- backend/apps/agents/browser_agent.py | 64 +++++++++- backend/apps/agents/browser_mcp_server.py | 54 ++++++++ frontend/src/shared/browserCommandHandler.ts | 126 ++++++++++++++++--- 3 files changed, 222 insertions(+), 22 deletions(-) diff --git a/backend/apps/agents/browser_agent.py b/backend/apps/agents/browser_agent.py index ea2064e6..09ba789d 100644 --- a/backend/apps/agents/browser_agent.py +++ b/backend/apps/agents/browser_agent.py @@ -112,6 +112,48 @@ BROWSER_TOOLS_SCHEMA = [ "required": [], }, }, + { + "name": "BrowserScroll", + "description": ( + "Scroll the page up or down. Automatically finds the correct scrollable " + "container (works on SPAs like Notion, Gmail, etc. that use nested scroll " + "containers instead of window-level scrolling). Returns scroll position info " + "including whether top/bottom has been reached." + ), + "input_schema": { + "type": "object", + "properties": { + "direction": { + "type": "string", + "enum": ["up", "down"], + "description": "Scroll direction. Defaults to 'down'.", + }, + "amount": { + "type": "number", + "description": "Pixels to scroll. Defaults to 500.", + }, + }, + "required": [], + }, + }, + { + "name": "BrowserWait", + "description": ( + "Wait for a specified duration. Useful after navigation or actions that " + "trigger page loads, animations, or async content rendering. " + "Min 100ms, max 10000ms." + ), + "input_schema": { + "type": "object", + "properties": { + "milliseconds": { + "type": "number", + "description": "Duration to wait in milliseconds. Defaults to 1000.", + }, + }, + "required": [], + }, + }, ] ACTION_MAP = { @@ -122,17 +164,29 @@ ACTION_MAP = { "BrowserType": "type", "BrowserEvaluate": "evaluate", "BrowserGetElements": "get_elements", + "BrowserScroll": "scroll", + "BrowserWait": "wait", } SYSTEM_PROMPT = ( "You are a browser automation agent. You control a single browser tab and " "execute the task you are given.\n\n" "Strategy:\n" - "1. Start by taking a screenshot or calling BrowserGetElements to understand the page.\n" - "2. Use BrowserGetElements BEFORE clicking or typing to discover valid CSS selectors.\n" - "3. After performing actions, take a screenshot to verify the result.\n" - "4. If an action fails, try alternative selectors or approaches.\n" - "5. When the task is complete, provide a clear summary of what you accomplished.\n\n" + "1. Start by taking a screenshot to understand the page.\n" + "2. After navigation, use BrowserWait (1-3 seconds) to let the page finish loading.\n" + "3. Use BrowserScroll to scroll through pages — do NOT use BrowserEvaluate with " + "window.scrollBy() as many sites use nested scroll containers that BrowserScroll " + "handles automatically.\n" + "4. Use BrowserGetElements BEFORE clicking or typing to discover valid CSS selectors.\n" + "5. After performing actions, take a screenshot to verify the result.\n" + "6. If an action fails, try alternative selectors or approaches.\n" + "7. When the task is complete, provide a clear summary of what you accomplished.\n\n" + "Important notes:\n" + "- BrowserGetText returns up to 15000 chars of visible text — use it to read page content.\n" + "- BrowserScroll returns position info including atTop/atBottom — use this to know when " + "you've reached the end of the page.\n" + "- For complex SPAs (Notion, Gmail, etc.), prefer BrowserScroll over BrowserEvaluate for scrolling.\n" + "- Avoid looping: if scrolling shows no new content (scrolled 0px), you're at the boundary.\n\n" "You have access ONLY to browser tools. Do not ask the user questions — " "complete the task autonomously to the best of your ability." ) diff --git a/backend/apps/agents/browser_mcp_server.py b/backend/apps/agents/browser_mcp_server.py index f32054c2..86f1ad5e 100644 --- a/backend/apps/agents/browser_mcp_server.py +++ b/backend/apps/agents/browser_mcp_server.py @@ -181,6 +181,58 @@ TOOLS = [ "required": ["browser_id"], }, }, + { + "name": "BrowserScroll", + "description": ( + "Scroll the page up or down. Automatically finds the correct scrollable " + "container (works on SPAs like Notion, Gmail, etc. that use nested scroll " + "containers instead of window-level scrolling). Returns scroll position info " + "including whether top/bottom has been reached." + ), + "inputSchema": { + "type": "object", + "properties": { + "browser_id": { + "type": "string", + "description": "The browser card ID.", + }, + "tab_id": TAB_ID_PROP, + "direction": { + "type": "string", + "enum": ["up", "down"], + "description": "Scroll direction. Defaults to 'down'.", + }, + "amount": { + "type": "number", + "description": "Pixels to scroll. Defaults to 500.", + }, + }, + "required": ["browser_id"], + }, + }, + { + "name": "BrowserWait", + "description": ( + "Wait for a specified duration. Useful after navigation or actions that " + "trigger page loads, animations, or async content rendering. " + "Min 100ms, max 10000ms." + ), + "inputSchema": { + "type": "object", + "properties": { + "browser_id": { + "type": "string", + "description": "The browser card ID.", + }, + "tab_id": TAB_ID_PROP, + "milliseconds": { + "type": "number", + "description": "Duration to wait in milliseconds. Defaults to 1000.", + }, + }, + "required": ["browser_id"], + }, + }, ] @@ -260,6 +312,8 @@ def handle_tool_call(tool_name: str, arguments: dict) -> dict: "BrowserType": "type", "BrowserEvaluate": "evaluate", "BrowserGetElements": "get_elements", + "BrowserScroll": "scroll", + "BrowserWait": "wait", } action = action_map.get(tool_name) if not action: diff --git a/frontend/src/shared/browserCommandHandler.ts b/frontend/src/shared/browserCommandHandler.ts index b9ddd233..fa0446fc 100644 --- a/frontend/src/shared/browserCommandHandler.ts +++ b/frontend/src/shared/browserCommandHandler.ts @@ -4,7 +4,7 @@ import { resolveInput } from './resolveUrl'; let initialized = false; -export type BrowserAction = 'screenshot' | 'get_text' | 'navigate' | 'click' | 'type' | 'evaluate' | 'get_elements'; +export type BrowserAction = 'screenshot' | 'get_text' | 'navigate' | 'click' | 'type' | 'evaluate' | 'get_elements' | 'scroll' | 'wait'; export interface BrowserActivity { action: BrowserAction; @@ -42,6 +42,8 @@ const ACTION_LABELS: Record = { type: 'Typing...', evaluate: 'Evaluating...', get_elements: 'Inspecting...', + scroll: 'Scrolling...', + wait: 'Waiting...', }; export function getActionLabel(action: string): string { @@ -128,6 +130,84 @@ async function handleType(wv: BrowserWebview, params: Record): Prom return result; } +async function handleScroll(wv: BrowserWebview, params: Record): Promise> { + const direction = (params.direction as string) || 'down'; + const amount = (params.amount as number) || 500; + const code = `(() => { + function findScrollable() { + const candidates = document.querySelectorAll( + '[class*="scroller"], [class*="scroll-container"], [class*="content"], ' + + 'main, [role="main"], article, .notion-scroller, .notion-frame' + ); + for (const el of candidates) { + const s = window.getComputedStyle(el); + const isScrollable = (s.overflow === 'auto' || s.overflow === 'scroll' + || s.overflowY === 'auto' || s.overflowY === 'scroll'); + if (isScrollable && el.scrollHeight > el.clientHeight + 10) return el; + } + const all = document.querySelectorAll('*'); + for (const el of all) { + if (el === document.body || el === document.documentElement) continue; + const s = window.getComputedStyle(el); + const isScrollable = (s.overflow === 'auto' || s.overflow === 'scroll' + || s.overflowY === 'auto' || s.overflowY === 'scroll'); + if (isScrollable && el.scrollHeight > el.clientHeight + 50 + && el.clientHeight > 200) return el; + } + return null; + } + const dy = ${JSON.stringify(direction)} === 'up' ? -${amount} : ${amount}; + const container = findScrollable(); + if (container) { + const before = container.scrollTop; + container.scrollBy({ top: dy, behavior: 'instant' }); + const after = container.scrollTop; + return { + scrolled: Math.abs(after - before), + scrollTop: after, + scrollHeight: container.scrollHeight, + clientHeight: container.clientHeight, + atTop: after <= 0, + atBottom: after + container.clientHeight >= container.scrollHeight - 5, + target: 'container', + }; + } + const before = window.scrollY; + window.scrollBy({ top: dy, behavior: 'instant' }); + const after = window.scrollY; + return { + scrolled: Math.abs(after - before), + scrollTop: after, + scrollHeight: document.documentElement.scrollHeight, + clientHeight: window.innerHeight, + atTop: after <= 0, + atBottom: after + window.innerHeight >= document.documentElement.scrollHeight - 5, + target: 'window', + }; + })()`; + try { + const result = await wv.executeJavaScript(code); + const status = result.atBottom ? ' (reached bottom)' : result.atTop ? ' (reached top)' : ''; + return { + text: `Scrolled ${direction} by ${result.scrolled}px${status}. Position: ${result.scrollTop}/${result.scrollHeight - result.clientHeight}px`, + ...result, + url: wv.getURL(), + }; + } catch (err: any) { + return { error: `Scroll failed: ${err?.message || String(err)}` }; + } +} + +async function handleWait(wv: BrowserWebview, params: Record): Promise> { + const ms = Math.min(Math.max((params.milliseconds as number) || 1000, 100), 10000); + await new Promise((resolve) => setTimeout(resolve, ms)); + return { + text: `Waited ${ms}ms. Current URL: ${wv.getURL()}`, + url: wv.getURL(), + title: wv.getTitle(), + }; +} + async function handleGetElements(wv: BrowserWebview, params: Record): Promise> { const scope = (params.selector as string) || 'body'; const safeScope = JSON.stringify(scope); @@ -135,51 +215,57 @@ async function handleGetElements(wv: BrowserWebview, params: Record const scope = document.querySelector(${safeScope}) || document.body; const interactive = scope.querySelectorAll( 'a[href], button, input, textarea, select, [role="button"], [role="link"], ' - + '[role="textbox"], [role="searchbox"], [onclick], [tabindex]:not([tabindex="-1"])' + + '[role="textbox"], [role="searchbox"], [role="menuitem"], [role="tab"], ' + + '[role="checkbox"], [role="switch"], [role="option"], ' + + '[onclick], [tabindex]:not([tabindex="-1"]), ' + + '[data-block-id], [contenteditable="true"]' ); + const seen = new Set(); const results = []; for (const el of interactive) { - if (results.length >= 60) break; + if (results.length >= 80) break; const rect = el.getBoundingClientRect(); if (rect.width === 0 && rect.height === 0) continue; - if (window.getComputedStyle(el).visibility === 'hidden') continue; + const style = window.getComputedStyle(el); + if (style.visibility === 'hidden' || style.display === 'none') continue; + if (style.opacity === '0') continue; let selector = el.tagName.toLowerCase(); if (el.id) { - selector = '#' + el.id; + selector = '#' + CSS.escape(el.id); + } else if (el.getAttribute('data-block-id')) { + selector = '[data-block-id="' + el.getAttribute('data-block-id') + '"]'; } else if (el.getAttribute('name')) { - selector = el.tagName.toLowerCase() + '[name="' + el.getAttribute('name') + '"]'; + selector = el.tagName.toLowerCase() + '[name="' + CSS.escape(el.getAttribute('name')) + '"]'; } else if (el.getAttribute('aria-label')) { - selector = el.tagName.toLowerCase() + '[aria-label="' + el.getAttribute('aria-label') + '"]'; + selector = el.tagName.toLowerCase() + '[aria-label="' + CSS.escape(el.getAttribute('aria-label')) + '"]'; } else if (el.getAttribute('type') && el.tagName === 'INPUT') { selector = 'input[type="' + el.getAttribute('type') + '"]'; if (el.getAttribute('placeholder')) - selector += '[placeholder="' + el.getAttribute('placeholder') + '"]'; + selector += '[placeholder="' + CSS.escape(el.getAttribute('placeholder')) + '"]'; } else if (el.className && typeof el.className === 'string') { const cls = el.className.trim().split(/\\s+/)[0]; - if (cls && cls.length < 40) + if (cls && cls.length < 60) selector = el.tagName.toLowerCase() + '.' + CSS.escape(cls); } - const verify = document.querySelectorAll(selector); - if (verify.length > 1) { + if (seen.has(selector)) { const parent = el.parentElement; if (parent && parent.id) { - selector = '#' + parent.id + ' > ' + selector; + selector = '#' + CSS.escape(parent.id) + ' > ' + selector; } else { - const siblings = parent ? Array.from(parent.querySelectorAll(':scope > ' + el.tagName.toLowerCase())) : []; + const siblings = parent ? Array.from(parent.children) : []; const idx = siblings.indexOf(el); - if (idx >= 0 && parent) - selector = (parent.tagName.toLowerCase() + (parent.className ? '.' + CSS.escape(parent.className.trim().split(/\\s+/)[0]) : '')) - + ' > ' + el.tagName.toLowerCase() + ':nth-child(' + (idx + 1) + ')'; + if (idx >= 0) selector += ':nth-child(' + (idx + 1) + ')'; } } + seen.add(selector); results.push({ selector, tag: el.tagName.toLowerCase(), type: el.type || null, - text: (el.textContent || '').trim().substring(0, 80) || null, + text: (el.textContent || '').trim().substring(0, 120) || null, placeholder: el.placeholder || null, ariaLabel: el.getAttribute('aria-label') || null, role: el.getAttribute('role') || null, @@ -248,6 +334,12 @@ async function handleBrowserCommand(data: Record) { case 'get_elements': result = await handleGetElements(wv, params); break; + case 'scroll': + result = await handleScroll(wv, params); + break; + case 'wait': + result = await handleWait(wv, params); + break; default: result = { error: `Unknown browser action: ${action}` }; }