mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-10 11:47:43 +02:00
[eric] browser sub-agent: loop detection, ReportProgress, CDP ax tree, batching, compaction, hide-don't-unmount Dashboard so browser cards survive route navigation
This commit is contained in:
ciregenz
@@ -27,7 +27,321 @@ MODEL_MAP = {
|
||||
"haiku": "claude-haiku-4-5-20251001",
|
||||
}
|
||||
|
||||
# Cache of conversation history per browser_id so successive BrowserAgent
|
||||
# calls on the same browser can resume rather than restart from scratch.
|
||||
# Without this every "swipe right" / "swipe left" call has to take a new
|
||||
# screenshot and re-orient itself, costing 30-60s per action.
|
||||
_browser_history: dict[str, list[dict]] = {}
|
||||
# Cap history to prevent unbounded growth on long-lived browsers.
|
||||
_MAX_HISTORY_MESSAGES = 30
|
||||
|
||||
|
||||
def clear_browser_history(browser_id: str) -> None:
|
||||
"""Drop cached conversation history for a browser (e.g. when it's closed)."""
|
||||
_browser_history.pop(browser_id, None)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Loop detection
|
||||
#
|
||||
# Tracks recent state-mutating tool calls in a sliding window. If the model
|
||||
# repeats the same (tool, input) with the same result several times, we
|
||||
# inject an is_error message in the next tool_result to force a strategy
|
||||
# change. This prevents the model from burning the entire turn budget on
|
||||
# a failing approach.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Tools that are read-only / idempotent and should NOT count toward loop
|
||||
# detection. Repeating these is normal (scrolling through a feed, taking
|
||||
# successive screenshots, polling for an element to appear).
|
||||
_LOOP_DETECTION_EXCLUDED_TOOLS = {
|
||||
"BrowserScreenshot",
|
||||
"BrowserGetText",
|
||||
"BrowserGetElements",
|
||||
"BrowserListInteractives", # Phase 3
|
||||
"BrowserWait",
|
||||
"ReportProgress", # Phase 2
|
||||
"RequestHumanIntervention",
|
||||
}
|
||||
|
||||
_LOOP_WINDOW_SIZE = 5
|
||||
_LOOP_REPEAT_THRESHOLD = 3
|
||||
_LOOP_HARD_CAP = 5
|
||||
|
||||
|
||||
def _hash_tool_call(tool_name: str, tool_input: dict, result: dict) -> tuple[str, str, str]:
|
||||
"""Build a stable hash key for a tool call, including its result.
|
||||
|
||||
Including the result hash means that legitimate progress (same input,
|
||||
different output — e.g. BrowserScroll on a long feed) does NOT count
|
||||
as a loop. Only same-input + same-output is treated as stuck.
|
||||
"""
|
||||
try:
|
||||
input_key = json.dumps(tool_input, sort_keys=True, default=str)
|
||||
except Exception:
|
||||
input_key = repr(tool_input)
|
||||
try:
|
||||
# Truncate the result hash to avoid huge image blobs in the key
|
||||
result_key = json.dumps(result, sort_keys=True, default=str)[:300]
|
||||
except Exception:
|
||||
result_key = repr(result)[:300]
|
||||
return (tool_name, input_key, result_key)
|
||||
|
||||
|
||||
def _detect_loop(
|
||||
recent_calls: list[tuple[str, str, str]],
|
||||
new_call: tuple[str, str, str],
|
||||
) -> bool:
|
||||
"""Return True if `new_call` constitutes a loop given recent history.
|
||||
|
||||
A loop is when the same (tool, input, result) has appeared at least
|
||||
`_LOOP_REPEAT_THRESHOLD` times within the last `_LOOP_WINDOW_SIZE`
|
||||
state-mutating calls (the new call counts as one of those occurrences).
|
||||
"""
|
||||
if new_call[0] in _LOOP_DETECTION_EXCLUDED_TOOLS:
|
||||
return False
|
||||
window = recent_calls[-(_LOOP_WINDOW_SIZE - 1):] + [new_call]
|
||||
matches = sum(1 for c in window if c == new_call)
|
||||
return matches >= _LOOP_REPEAT_THRESHOLD
|
||||
|
||||
|
||||
_LOOP_WARNING_TEXT = (
|
||||
"LOOP DETECTED: You have called this tool with these exact parameters and "
|
||||
"gotten the same result {count} times in a row. STOP retrying this approach "
|
||||
"— it is not working. Try a fundamentally different strategy: "
|
||||
"(1) check the page state with BrowserScreenshot or BrowserGetText, "
|
||||
"(2) try a different selector or a different tool, "
|
||||
"(3) use BrowserPressKey for keyboard shortcuts if the site supports them, "
|
||||
"or (4) call RequestHumanIntervention if you genuinely cannot proceed."
|
||||
)
|
||||
|
||||
|
||||
def _validate_message_pairing(messages: list[dict]) -> bool:
|
||||
"""Verify every tool_result references a tool_use_id from a prior assistant
|
||||
message in the same list. Returns False if there's an orphan, which means
|
||||
the cached history would 400 if sent to the API.
|
||||
|
||||
This is the last line of defense against cache corruption — if it ever
|
||||
returns False on a resume, we drop the cache and start fresh rather than
|
||||
crash on the next API call.
|
||||
"""
|
||||
declared_tool_use_ids: set[str] = set()
|
||||
for msg in messages:
|
||||
role = msg.get("role")
|
||||
content = msg.get("content")
|
||||
if role == "assistant" and isinstance(content, list):
|
||||
for block in content:
|
||||
if isinstance(block, dict) and block.get("type") == "tool_use":
|
||||
tu_id = block.get("id")
|
||||
if tu_id:
|
||||
declared_tool_use_ids.add(tu_id)
|
||||
elif role == "user" and isinstance(content, list):
|
||||
for block in content:
|
||||
if isinstance(block, dict) and block.get("type") == "tool_result":
|
||||
tr_id = block.get("tool_use_id")
|
||||
if tr_id and tr_id not in declared_tool_use_ids:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _is_fresh_user_message(msg: dict) -> bool:
|
||||
"""A 'fresh' user message starts a new turn — string content or a list
|
||||
that contains no tool_result blocks. These are the only safe cut points
|
||||
because they don't reference any prior assistant tool_use blocks."""
|
||||
if msg.get("role") != "user":
|
||||
return False
|
||||
content = msg.get("content")
|
||||
if isinstance(content, str):
|
||||
return True
|
||||
if isinstance(content, list) and not any(
|
||||
isinstance(c, dict) and c.get("type") == "tool_result" for c in content
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _summarize_messages(messages: list[dict]) -> str:
|
||||
"""Build a programmatic summary of older browser-agent messages.
|
||||
|
||||
Extracts the original user task, a count of tool calls by name with their
|
||||
key parameters, the last few ReportProgress brain states, and the most
|
||||
recent assistant text. No LLM call required — this is purely structural
|
||||
extraction from the existing message history.
|
||||
"""
|
||||
if not messages:
|
||||
return ""
|
||||
|
||||
# Find the original user task (first user-text message)
|
||||
initial_task = ""
|
||||
for msg in messages:
|
||||
if msg.get("role") == "user":
|
||||
content = msg.get("content")
|
||||
if isinstance(content, str) and content.strip():
|
||||
initial_task = content.strip()[:300]
|
||||
break
|
||||
|
||||
# Count tool calls by name with key params
|
||||
tool_call_summary: dict[str, list[str]] = {}
|
||||
brain_states: list[str] = []
|
||||
last_assistant_text = ""
|
||||
|
||||
for msg in messages:
|
||||
if msg.get("role") != "assistant":
|
||||
continue
|
||||
content = msg.get("content")
|
||||
if not isinstance(content, list):
|
||||
continue
|
||||
for block in content:
|
||||
if not isinstance(block, dict):
|
||||
continue
|
||||
btype = block.get("type")
|
||||
if btype == "tool_use":
|
||||
name = block.get("name", "unknown")
|
||||
inp = block.get("input") or {}
|
||||
if name == "ReportProgress":
|
||||
# Capture the brain state for inline summary
|
||||
brain_states.append(
|
||||
f" • {inp.get('next_goal', '')[:120]}"
|
||||
)
|
||||
continue
|
||||
# Compact one-line description with key params
|
||||
key_param = ""
|
||||
for k in ("index", "key", "url", "selector", "direction", "text"):
|
||||
if k in inp:
|
||||
v = str(inp[k])[:40]
|
||||
key_param = f"{k}={v}"
|
||||
break
|
||||
desc = f"{name}({key_param})" if key_param else name
|
||||
tool_call_summary.setdefault(name, []).append(desc)
|
||||
elif btype == "text":
|
||||
txt = block.get("text", "").strip()
|
||||
if txt:
|
||||
last_assistant_text = txt
|
||||
|
||||
# Build the summary text
|
||||
parts = ["[Summary of earlier browser-agent activity]"]
|
||||
if initial_task:
|
||||
parts.append(f'Original task: "{initial_task}"')
|
||||
if tool_call_summary:
|
||||
total = sum(len(v) for v in tool_call_summary.values())
|
||||
parts.append(f"Actions taken ({total} total):")
|
||||
# Show count + a couple of representative examples per tool
|
||||
for name in sorted(tool_call_summary.keys()):
|
||||
calls = tool_call_summary[name]
|
||||
count = len(calls)
|
||||
sample = calls[-1] # most recent example
|
||||
if count == 1:
|
||||
parts.append(f" - {sample}")
|
||||
else:
|
||||
parts.append(f" - {sample} (×{count})")
|
||||
if brain_states:
|
||||
parts.append("Recent intents:")
|
||||
parts.extend(brain_states[-5:]) # last 5 brain states
|
||||
if last_assistant_text:
|
||||
snippet = last_assistant_text[:400]
|
||||
parts.append(f"Last update from assistant: {snippet}")
|
||||
parts.append(
|
||||
"(Earlier turn-by-turn details have been compacted to keep the "
|
||||
"context window manageable. Continue from where you left off.)"
|
||||
)
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def _trim_history_by_turns(messages: list[dict], max_messages: int) -> list[dict]:
|
||||
"""Compact message history when it exceeds max_messages.
|
||||
|
||||
The Anthropic API requires every `tool_result` block to reference a
|
||||
`tool_use_id` from a previous assistant message. Naive slicing can drop
|
||||
a tool_use while keeping its tool_result, causing 400 errors. This
|
||||
function avoids that by:
|
||||
|
||||
1. Walking forward to find a clean turn boundary (a fresh user-text
|
||||
message that starts a new turn — no tool_result content).
|
||||
2. Summarizing everything BEFORE that boundary into a single user-text
|
||||
message and prepending it to the kept tail.
|
||||
3. If no clean boundary exists at all, returning the original history
|
||||
unchanged. Better to temporarily exceed the cap than to corrupt the
|
||||
conversation and 400 every subsequent request.
|
||||
|
||||
The summary is built programmatically (no LLM call) from the message
|
||||
structure: original task, tool call counts, recent ReportProgress brain
|
||||
states, and last assistant text.
|
||||
"""
|
||||
if len(messages) <= max_messages:
|
||||
return list(messages)
|
||||
|
||||
target_tail_size = max_messages - 1 # leave room for the summary message
|
||||
cut_index: int | None = None
|
||||
|
||||
# First pass: walk forward looking for the EARLIEST clean cut point that
|
||||
# gets us under the cap. This preserves the most recent detail.
|
||||
for i in range(1, len(messages)):
|
||||
if not _is_fresh_user_message(messages[i]):
|
||||
continue
|
||||
if len(messages) - i <= target_tail_size:
|
||||
cut_index = i
|
||||
break
|
||||
|
||||
# Second pass: if no cut point gets us under the cap (e.g. the current
|
||||
# turn alone is bigger than max_messages), use the LATEST clean cut point
|
||||
# available. The tail will still exceed the cap, but it's the smallest
|
||||
# safe history we can produce — and any compaction is better than none.
|
||||
if cut_index is None:
|
||||
for i in range(len(messages) - 1, 0, -1):
|
||||
if _is_fresh_user_message(messages[i]):
|
||||
cut_index = i
|
||||
break
|
||||
|
||||
if cut_index is None:
|
||||
# No clean cut anywhere in the history. Return original — better to
|
||||
# exceed the cap than to corrupt the conversation.
|
||||
return list(messages)
|
||||
|
||||
# Compact: summarize messages[0..cut_index-1], prepend as a single
|
||||
# user-text message, then keep messages[cut_index..end] verbatim.
|
||||
summary_text = _summarize_messages(messages[:cut_index])
|
||||
summary_msg = {"role": "user", "content": summary_text}
|
||||
return [summary_msg] + list(messages[cut_index:])
|
||||
|
||||
BROWSER_TOOLS_SCHEMA = [
|
||||
{
|
||||
"name": "ReportProgress",
|
||||
"description": (
|
||||
"Record your assessment of the previous action and your plan for the "
|
||||
"next one. You MUST call this BEFORE any browser action tools in every "
|
||||
"turn (after the very first turn). This is how you reflect on what just "
|
||||
"happened, track what you've learned about this site, and articulate what "
|
||||
"you're trying to do next. Skipping it is not allowed and will be rejected."
|
||||
),
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"evaluation_previous": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"What did the previous action(s) accomplish? Did they succeed? "
|
||||
"If not, why? Be specific about what changed on the page."
|
||||
),
|
||||
},
|
||||
"working_memory": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Short notes about what you've learned about this site so far — "
|
||||
"selectors that work, keyboard shortcuts, layout quirks, what "
|
||||
"you've tried that failed. Carry this forward across turns."
|
||||
),
|
||||
},
|
||||
"next_goal": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"What you're trying to achieve with the action(s) you're about "
|
||||
"to take next. Be concrete."
|
||||
),
|
||||
},
|
||||
},
|
||||
"required": ["evaluation_previous", "working_memory", "next_goal"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "BrowserScreenshot",
|
||||
"description": (
|
||||
@@ -137,6 +451,112 @@ BROWSER_TOOLS_SCHEMA = [
|
||||
"required": [],
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "BrowserListInteractives",
|
||||
"description": (
|
||||
"Get a NUMBERED LIST of interactive elements on the page using the "
|
||||
"browser's accessibility tree. Returns elements like [1]<button \"Like\">, "
|
||||
"[2]<link \"Settings\">, etc. Use this BEFORE BrowserClickIndex. This is "
|
||||
"the PREFERRED way to discover clickable elements on hostile sites "
|
||||
"(Tinder, Instagram, TikTok) where CSS selectors fail because the page "
|
||||
"uses unlabeled <div>s — the accessibility tree sees roles and names "
|
||||
"even when raw HTML doesn't expose them. Much more reliable than "
|
||||
"BrowserGetElements (which uses CSS selectors)."
|
||||
),
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
"required": [],
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "BrowserClickIndex",
|
||||
"description": (
|
||||
"Click an element by its numeric index from BrowserListInteractives. "
|
||||
"Uses native OS-level mouse events (event.isTrusted=true) so it works "
|
||||
"on sites that filter out synthetic JS events. Always call "
|
||||
"BrowserListInteractives first to get a fresh index list. If the click "
|
||||
"returns 'index no longer valid', the page changed — re-list and retry."
|
||||
),
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"index": {
|
||||
"type": "integer",
|
||||
"description": "The numeric index from BrowserListInteractives (1-based).",
|
||||
},
|
||||
},
|
||||
"required": ["index"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "BrowserBatch",
|
||||
"description": (
|
||||
"Run a sequence of browser actions in one tool call. Each sub-action "
|
||||
"is executed in order, with the URL captured before/after each one. "
|
||||
"If the URL changes mid-batch (the page navigated), the rest of the "
|
||||
"batch is aborted and you get a partial result. Use this when you "
|
||||
"have a known sequence — typing then pressing Enter, swiping multiple "
|
||||
"times, clicking through pagination. Max 5 actions per batch.\n\n"
|
||||
"Sub-action types and their params:\n"
|
||||
"- click_index: { index: int }\n"
|
||||
"- press_key: { key: str }\n"
|
||||
"- type: { selector: str, text: str }\n"
|
||||
"- click: { selector: str }\n"
|
||||
"- scroll: { direction?: 'up'|'down', amount?: int }\n"
|
||||
"- wait: { milliseconds?: int }\n"
|
||||
"- navigate: { url: str }\n\n"
|
||||
"Example: { actions: [{type: 'click_index', params: {index: 1}}, "
|
||||
"{type: 'wait', params: {milliseconds: 500}}, "
|
||||
"{type: 'press_key', params: {key: 'ArrowRight'}}] }"
|
||||
),
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"actions": {
|
||||
"type": "array",
|
||||
"maxItems": 5,
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"enum": ["click_index", "press_key", "type", "wait", "scroll", "navigate", "click"],
|
||||
},
|
||||
"params": {"type": "object"},
|
||||
},
|
||||
"required": ["type", "params"],
|
||||
},
|
||||
},
|
||||
},
|
||||
"required": ["actions"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "BrowserPressKey",
|
||||
"description": (
|
||||
"Press a keyboard key (or key combination) on the page using a real native "
|
||||
"input event. Use this for keyboard shortcuts when JS-dispatched events get "
|
||||
"ignored — sites like Tinder, Slack, Notion, Gmail listen for trusted key "
|
||||
"events. Examples: 'ArrowLeft', 'ArrowRight', 'Enter', 'Escape', 'Tab', "
|
||||
"'Space', single letters like 'a'. Prefer this over BrowserEvaluate with "
|
||||
"dispatchEvent for keyboard shortcuts."
|
||||
),
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"key": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"The key to press. Use JS KeyboardEvent.key names like "
|
||||
"'ArrowUp', 'ArrowDown', 'Enter', 'Escape', 'Tab', 'Space', "
|
||||
"'Backspace', or a single character like 'a'."
|
||||
),
|
||||
},
|
||||
},
|
||||
"required": ["key"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "BrowserWait",
|
||||
"description": (
|
||||
@@ -190,33 +610,133 @@ ACTION_MAP = {
|
||||
"BrowserGetElements": "get_elements",
|
||||
"BrowserScroll": "scroll",
|
||||
"BrowserWait": "wait",
|
||||
"BrowserPressKey": "press_key",
|
||||
"BrowserListInteractives": "list_interactives",
|
||||
"BrowserClickIndex": "click_index",
|
||||
"BrowserBatch": "batch",
|
||||
}
|
||||
|
||||
SYSTEM_PROMPT = (
|
||||
"You are a browser automation agent. You control a single browser tab and "
|
||||
"execute the task you are given.\n\n"
|
||||
"Strategy:\n"
|
||||
"1. Start by taking a screenshot to understand the page.\n"
|
||||
"2. After navigation, use BrowserWait (1-3 seconds) to let the page finish loading.\n"
|
||||
"3. Use BrowserScroll to scroll through pages — do NOT use BrowserEvaluate with "
|
||||
"window.scrollBy() as many sites use nested scroll containers that BrowserScroll "
|
||||
"handles automatically.\n"
|
||||
"4. Use BrowserGetElements BEFORE clicking or typing to discover valid CSS selectors.\n"
|
||||
"5. After performing actions, take a screenshot to verify the result.\n"
|
||||
"6. If an action fails, try alternative selectors or approaches.\n"
|
||||
"7. When the task is complete, provide a clear summary of what you accomplished.\n\n"
|
||||
"Important notes:\n"
|
||||
"- BrowserGetText returns up to 15000 chars of visible text — use it to read page content.\n"
|
||||
"- BrowserScroll returns position info including atTop/atBottom — use this to know when "
|
||||
"you've reached the end of the page.\n"
|
||||
"- For complex SPAs (Notion, Gmail, etc.), prefer BrowserScroll over BrowserEvaluate for scrolling.\n"
|
||||
"- Avoid looping: if scrolling shows no new content (scrolled 0px), you're at the boundary.\n\n"
|
||||
"You have access ONLY to browser tools. Complete the task autonomously. "
|
||||
"If you encounter a captcha, login wall, or popup you cannot bypass, "
|
||||
"use RequestHumanIntervention to ask the user for help instead of retrying endlessly."
|
||||
"You are a website-agnostic browser automation agent. You can operate on ANY "
|
||||
"website the user is signed into — social media, dating apps, email, productivity "
|
||||
"tools, dashboards, ecommerce, anything. Assume the user has already logged in.\n\n"
|
||||
|
||||
"## Required output structure: ReportProgress before every action\n"
|
||||
"Before ANY action tool (BrowserClick, BrowserType, BrowserNavigate, "
|
||||
"BrowserPressKey, BrowserScroll, BrowserEvaluate, BrowserClickIndex, "
|
||||
"BrowserBatch), you MUST call the ReportProgress tool in the SAME turn. "
|
||||
"ReportProgress takes three short fields:\n"
|
||||
"- evaluation_previous: did your last action work? what changed on the page?\n"
|
||||
"- working_memory: what have you learned about this site? what worked, what didn't?\n"
|
||||
"- next_goal: what specifically are you trying to do with the next action?\n"
|
||||
"Emit ReportProgress and your action tool(s) together in the same response. "
|
||||
"If you skip ReportProgress, your action tools will be REJECTED with an error "
|
||||
"and you will have to retry. This is not optional. Read-only tools "
|
||||
"(BrowserScreenshot, BrowserGetText, BrowserGetElements, BrowserWait) do not "
|
||||
"require ReportProgress.\n\n"
|
||||
|
||||
"## Loop awareness\n"
|
||||
"If you see a tool result containing 'LOOP DETECTED' or '⚠️', it means you "
|
||||
"have called the same tool with the same parameters and gotten the same "
|
||||
"result multiple times in a row. STOP. Do NOT retry the same approach. "
|
||||
"Switch strategy entirely: try a different tool, a different selector, "
|
||||
"keyboard shortcuts, or call RequestHumanIntervention if you genuinely "
|
||||
"cannot proceed. The loop detector will force-exit the agent if you "
|
||||
"ignore it more than 5 times.\n\n"
|
||||
|
||||
"## Use prior context\n"
|
||||
"If this is a continuation of an earlier conversation on the same browser, the "
|
||||
"messages above already contain everything you've tried, what worked, what failed, "
|
||||
"and the page state. READ THAT HISTORY before acting. Do NOT take a fresh screenshot "
|
||||
"or re-explore the DOM if you already know what's on screen — just act. Only re-orient "
|
||||
"if the page has clearly changed (after navigation, after a multi-second wait, or if "
|
||||
"your last action mutated the page in unexpected ways).\n\n"
|
||||
|
||||
"## Try multiple strategies, learn from failures\n"
|
||||
"Sites vary wildly. When one approach fails, switch tactics — don't retry the same "
|
||||
"thing. The escalation ladder, fastest to slowest:\n"
|
||||
"1. **Keyboard shortcuts via BrowserPressKey** — fastest and most reliable on sites "
|
||||
"that support them (Tinder swipes, Gmail navigation, Slack message jump, etc.). "
|
||||
"Always check if the site shows keyboard hints in the UI before falling back to clicks. "
|
||||
"BrowserPressKey sends real native events that pass the `event.isTrusted` check, so "
|
||||
"it works where dispatchEvent in BrowserEvaluate silently fails.\n"
|
||||
"2. **Accessibility tree via BrowserListInteractives + BrowserClickIndex** — the "
|
||||
"accessibility tree sees roles and names that the raw DOM doesn't, even on sites "
|
||||
"like Tinder, Instagram, and TikTok that use unlabeled <div>s with click handlers. "
|
||||
"Call BrowserListInteractives to get a numbered list (`[1]<button \"Like\">`, "
|
||||
"`[2]<link \"Settings\">`), then BrowserClickIndex with the number. The click uses "
|
||||
"native OS-level mouse events so it works where DOM .click() doesn't. THIS IS YOUR "
|
||||
"GO-TO STRATEGY for unlabeled or hostile sites — try this BEFORE BrowserGetElements.\n"
|
||||
"3. **Semantic CSS selectors** — `button[aria-label='X']`, `[role='button']`, "
|
||||
"`a[href*='...']`. Try these via BrowserGetElements + BrowserClick when the site "
|
||||
"actually has semantic HTML.\n"
|
||||
"4. **Text-based JS query** — when both of the above fail, use BrowserEvaluate to "
|
||||
"find elements by visible text: `Array.from(document.querySelectorAll('*')).find(el => el.textContent.trim() === 'Like')`.\n"
|
||||
"5. **Coordinate-based fallback** — last resort: take a screenshot, identify the "
|
||||
"button visually, then click by approximate coords.\n\n"
|
||||
|
||||
"## Batch known sequences with BrowserBatch\n"
|
||||
"When you have a known sequence of actions — typing then pressing Enter, "
|
||||
"swiping multiple times, clicking through pagination — emit them all in a "
|
||||
"single BrowserBatch call instead of one tool per turn. The batch executes "
|
||||
"sub-actions sequentially and aborts if the URL changes mid-batch (so you "
|
||||
"won't operate on stale state). Max 5 sub-actions per batch.\n"
|
||||
"Use BrowserBatch when:\n"
|
||||
"- You're doing the same action repeatedly (5 swipes, 3 scrolls)\n"
|
||||
"- You have a deterministic flow (type query → press Enter → click first result)\n"
|
||||
"Don't use BrowserBatch when:\n"
|
||||
"- You need to read the page state between actions\n"
|
||||
"- You're uncertain about what comes next\n"
|
||||
"- An action might trigger an unexpected popup or navigation\n\n"
|
||||
|
||||
"## Avoid wasted cycles\n"
|
||||
"- Do NOT screenshot after every single action. Screenshot ONLY when you genuinely "
|
||||
"don't know the page state (start of task, after navigation, after a failure).\n"
|
||||
"- Do NOT call BrowserGetElements on the entire body if you already know roughly "
|
||||
"where the target is. Scope it: `BrowserGetElements({selector: 'nav'})`.\n"
|
||||
"- Do NOT call the same failing tool twice with identical parameters. If selector "
|
||||
"X failed, try a DIFFERENT selector or a DIFFERENT strategy.\n"
|
||||
"- For repeated actions (swiping through profiles, going through inbox messages), "
|
||||
"use BrowserPressKey if available — it's an order of magnitude faster than DOM clicks.\n\n"
|
||||
|
||||
"## When you genuinely cannot proceed\n"
|
||||
"Use RequestHumanIntervention for:\n"
|
||||
"- Login walls (the user thinks they're logged in but the session expired)\n"
|
||||
"- Captchas, 2FA prompts, age verification gates\n"
|
||||
"- Anything genuinely ambiguous about user intent\n"
|
||||
"Don't use it for normal tool failures — try a different approach first.\n\n"
|
||||
|
||||
"## Tool reference\n"
|
||||
"- BrowserScreenshot: visual snapshot. Use sparingly, not after every action.\n"
|
||||
"- BrowserGetText: returns up to 15000 chars of visible text. Useful for reading "
|
||||
"content without an image.\n"
|
||||
"- BrowserScroll: handles nested scroll containers (Notion, Gmail). Returns "
|
||||
"atTop/atBottom — stop looping when scroll delta is 0.\n"
|
||||
"- BrowserGetElements: enumerate interactive elements with selectors.\n"
|
||||
"- BrowserClick / BrowserType: standard DOM interaction.\n"
|
||||
"- BrowserPressKey: native key events (preferred for shortcuts).\n"
|
||||
"- BrowserEvaluate: arbitrary JS for everything else, including text-based element "
|
||||
"search and reading state. Avoid for scrolling and keyboard events.\n"
|
||||
"- BrowserWait: 1-3s after navigation, 0.5s after most clicks.\n\n"
|
||||
|
||||
"Complete the task autonomously and report a clear, brief summary."
|
||||
)
|
||||
|
||||
MAX_TURNS = 25
|
||||
MAX_TURNS = 40
|
||||
|
||||
# Tools that count as "action tools" — calling any of these in a turn requires
|
||||
# the model to also call ReportProgress in the same turn (after the first
|
||||
# turn). Read-only tools and meta tools are exempt.
|
||||
_ACTION_TOOLS_REQUIRING_REPORT = {
|
||||
"BrowserClick",
|
||||
"BrowserType",
|
||||
"BrowserNavigate",
|
||||
"BrowserPressKey",
|
||||
"BrowserScroll",
|
||||
"BrowserEvaluate",
|
||||
"BrowserClickIndex", # Phase 3
|
||||
"BrowserBatch", # Phase 4
|
||||
}
|
||||
|
||||
|
||||
async def execute_browser_tool(
|
||||
@@ -356,10 +876,27 @@ async def run_browser_agent(
|
||||
from backend.apps.settings.credentials import get_anthropic_client
|
||||
client = get_anthropic_client(load_settings())
|
||||
|
||||
messages: list[dict] = [{"role": "user", "content": task}]
|
||||
# Resume prior conversation on this browser if we have one cached. This
|
||||
# lets the sub-agent skip the "take a screenshot to figure out where I am"
|
||||
# cycle every time the parent issues a new task. Defensively validate
|
||||
# the cache — if it's somehow corrupted (orphaned tool_use_ids), drop
|
||||
# it and start fresh rather than crash on the next API call.
|
||||
prior_messages = _browser_history.get(browser_id) or []
|
||||
if prior_messages and not _validate_message_pairing(prior_messages):
|
||||
logger.warning(
|
||||
f"[browser-agent {session_id}] cached history for {browser_id} has "
|
||||
f"orphaned tool_use_ids — dropping cache and starting fresh"
|
||||
)
|
||||
_browser_history.pop(browser_id, None)
|
||||
prior_messages = []
|
||||
messages: list[dict] = list(prior_messages) + [{"role": "user", "content": task}]
|
||||
action_log: list[dict] = []
|
||||
final_screenshot: str | None = None
|
||||
|
||||
# Loop detection state — sliding window of recent state-mutating tool calls
|
||||
recent_tool_calls: list[tuple[str, str, str]] = []
|
||||
loop_trigger_count = 0
|
||||
|
||||
user_msg = Message(role="user", content=task)
|
||||
session.messages.append(user_msg)
|
||||
await ws_manager.send_to_session(session_id, "agent:message", {
|
||||
@@ -446,11 +983,94 @@ async def run_browser_agent(
|
||||
|
||||
tool_results = []
|
||||
cancelled = False
|
||||
for tu in tool_uses:
|
||||
|
||||
# Sort tool_uses so ReportProgress is always processed first within
|
||||
# a turn, even if the model emits it after action tools. This way
|
||||
# the brain state is recorded before any actions execute.
|
||||
has_report_progress = any(tu.name == "ReportProgress" for tu in tool_uses)
|
||||
has_action_tools = any(
|
||||
tu.name in _ACTION_TOOLS_REQUIRING_REPORT for tu in tool_uses
|
||||
)
|
||||
# Violation: action tools without ReportProgress in the same turn.
|
||||
# The model MUST articulate its evaluation/memory/goal before acting.
|
||||
report_progress_violation = has_action_tools and not has_report_progress
|
||||
if report_progress_violation:
|
||||
logger.warning(
|
||||
f"[browser-agent {session_id}] ReportProgress violation: "
|
||||
f"action tools called without brain state"
|
||||
)
|
||||
# Stable sort: ReportProgress first, then everything else in order.
|
||||
tool_uses_sorted = sorted(
|
||||
tool_uses,
|
||||
key=lambda t: 0 if t.name == "ReportProgress" else 1,
|
||||
)
|
||||
|
||||
for tu in tool_uses_sorted:
|
||||
if cancel_event.is_set():
|
||||
cancelled = True
|
||||
break
|
||||
|
||||
# Handle ReportProgress — no-op execution that just records the
|
||||
# model's brain state and streams it to the dashboard.
|
||||
if tu.name == "ReportProgress":
|
||||
eval_prev = tu.input.get("evaluation_previous", "")
|
||||
working_mem = tu.input.get("working_memory", "")
|
||||
next_goal = tu.input.get("next_goal", "")
|
||||
brain_text = (
|
||||
f"📋 **Plan**\n"
|
||||
f"_Previous_: {eval_prev}\n"
|
||||
f"_Memory_: {working_mem}\n"
|
||||
f"_Next_: {next_goal}"
|
||||
)
|
||||
brain_msg = Message(role="assistant", content=brain_text)
|
||||
session.messages.append(brain_msg)
|
||||
await ws_manager.send_to_session(session_id, "agent:message", {
|
||||
"session_id": session_id,
|
||||
"message": brain_msg.model_dump(mode="json"),
|
||||
})
|
||||
tool_results.append({
|
||||
"type": "tool_result",
|
||||
"tool_use_id": tu.id,
|
||||
"content": [{"type": "text", "text": "Progress recorded."}],
|
||||
})
|
||||
continue
|
||||
|
||||
# Reject action tools when ReportProgress is missing this turn.
|
||||
# We MUST still emit a tool_result for every tool_use_id or the
|
||||
# next API request 400s.
|
||||
if (
|
||||
report_progress_violation
|
||||
and tu.name in _ACTION_TOOLS_REQUIRING_REPORT
|
||||
):
|
||||
rejection_text = (
|
||||
"REJECTED: You called an action tool without first calling "
|
||||
"ReportProgress in the same turn. ReportProgress is REQUIRED "
|
||||
"before every batch of action tools — it's how you reflect "
|
||||
"on what just happened and articulate your next goal. Try "
|
||||
"again: emit ReportProgress and your action tool(s) in the "
|
||||
"same response."
|
||||
)
|
||||
tool_results.append({
|
||||
"type": "tool_result",
|
||||
"tool_use_id": tu.id,
|
||||
"content": [{"type": "text", "text": rejection_text}],
|
||||
"is_error": True,
|
||||
})
|
||||
result_msg = Message(
|
||||
role="tool_result",
|
||||
content={
|
||||
"text": rejection_text,
|
||||
"tool_name": tu.name,
|
||||
"elapsed_ms": 0,
|
||||
},
|
||||
)
|
||||
session.messages.append(result_msg)
|
||||
await ws_manager.send_to_session(session_id, "agent:message", {
|
||||
"session_id": session_id,
|
||||
"message": result_msg.model_dump(mode="json"),
|
||||
})
|
||||
continue
|
||||
|
||||
# Handle RequestHumanIntervention — pause and wait for user
|
||||
if tu.name == "RequestHumanIntervention":
|
||||
problem = tu.input.get("problem", "")
|
||||
@@ -536,11 +1156,34 @@ async def run_browser_agent(
|
||||
if tu.name == "BrowserScreenshot" and result.get("image"):
|
||||
final_screenshot = result["image"]
|
||||
|
||||
# Loop detection: did we just repeat the same (tool, input,
|
||||
# result) for the third time in a row? If so, attach a loud
|
||||
# warning to this tool_result so the model is forced to
|
||||
# acknowledge it on its next turn.
|
||||
call_key = _hash_tool_call(tu.name, tu.input, result)
|
||||
is_loop = _detect_loop(recent_tool_calls, call_key)
|
||||
if call_key[0] not in _LOOP_DETECTION_EXCLUDED_TOOLS:
|
||||
recent_tool_calls.append(call_key)
|
||||
if len(recent_tool_calls) > _LOOP_WINDOW_SIZE * 2:
|
||||
recent_tool_calls = recent_tool_calls[-_LOOP_WINDOW_SIZE * 2:]
|
||||
|
||||
content_blocks = _format_tool_result(result, tu.name)
|
||||
if is_loop:
|
||||
loop_trigger_count += 1
|
||||
repeat_count = sum(1 for c in recent_tool_calls if c == call_key)
|
||||
warning = _LOOP_WARNING_TEXT.format(count=repeat_count)
|
||||
logger.warning(
|
||||
f"[browser-agent {session_id}] loop detected on {tu.name} "
|
||||
f"(trigger #{loop_trigger_count}): {warning}"
|
||||
)
|
||||
content_blocks = content_blocks + [
|
||||
{"type": "text", "text": f"\n\n⚠️ {warning}"}
|
||||
]
|
||||
tool_results.append({
|
||||
"type": "tool_result",
|
||||
"tool_use_id": tu.id,
|
||||
"content": content_blocks,
|
||||
**({"is_error": True} if is_loop else {}),
|
||||
})
|
||||
|
||||
result_text = result.get("text", result.get("error", ""))
|
||||
@@ -559,6 +1202,16 @@ async def run_browser_agent(
|
||||
if cancelled:
|
||||
break
|
||||
|
||||
# Hard cap on loops: if the model keeps repeating itself even
|
||||
# after we warn it, force-exit so we don't burn the entire turn
|
||||
# budget on a stuck agent.
|
||||
if loop_trigger_count >= _LOOP_HARD_CAP:
|
||||
logger.warning(
|
||||
f"[browser-agent {session_id}] hit loop hard cap "
|
||||
f"({_LOOP_HARD_CAP}) — force-exiting"
|
||||
)
|
||||
break
|
||||
|
||||
if cancel_event.is_set():
|
||||
session.status = "stopped"
|
||||
await ws_manager.send_to_session(session_id, "agent:status", {
|
||||
@@ -588,6 +1241,15 @@ async def run_browser_agent(
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Persist conversation history so the next BrowserAgent call on this
|
||||
# browser can resume rather than re-orient. Trim to the most recent
|
||||
# _MAX_HISTORY_MESSAGES turns to keep token usage bounded — but
|
||||
# never split a tool_use ↔ tool_result pair across the cut, or the
|
||||
# next API request will 400.
|
||||
_browser_history[browser_id] = _trim_history_by_turns(
|
||||
messages, _MAX_HISTORY_MESSAGES,
|
||||
)
|
||||
|
||||
session.status = "completed"
|
||||
agent_manager._fire_session_completed(session)
|
||||
await ws_manager.send_to_session(session_id, "agent:status", {
|
||||
|
||||
@@ -51,6 +51,11 @@ BUILTIN_TOOLS: list[BuiltinTool] = [
|
||||
BuiltinTool(name="BrowserGetElements", description="List interactive elements with CSS selectors", category="browser_action"),
|
||||
BuiltinTool(name="BrowserScroll", description="Scroll the page up or down", category="browser_action"),
|
||||
BuiltinTool(name="BrowserWait", description="Wait for page loads or animations", category="browser_action"),
|
||||
BuiltinTool(name="BrowserPressKey", description="Press a keyboard key (native event, works on sites that ignore JS-dispatched events)", category="browser_action"),
|
||||
BuiltinTool(name="BrowserListInteractives", description="List interactive elements via the accessibility tree (works on hostile sites with no semantic HTML)", category="browser_action"),
|
||||
BuiltinTool(name="BrowserClickIndex", description="Click an element by its index from BrowserListInteractives (uses native mouse events)", category="browser_action"),
|
||||
BuiltinTool(name="BrowserBatch", description="Run a sequence of browser actions in one tool call with URL-change abort guard", category="browser_action"),
|
||||
BuiltinTool(name="ReportProgress", description="Record evaluation of previous action, working memory, and next goal (required before action tools)", category="browser_action"),
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -425,6 +425,35 @@ app.on('web-contents-created', (_event, contents) => {
|
||||
}
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------
|
||||
// CDP debugger auto-attach for browser sub-agent accessibility tree
|
||||
// -----------------------------------------------------------------
|
||||
// The browser sub-agent uses Chrome DevTools Protocol (specifically
|
||||
// Accessibility.getFullAXTree, DOM.resolveNode, Input.dispatchMouseEvent)
|
||||
// to perceive and act on hostile sites where CSS selectors fail. CDP
|
||||
// commands require webContents.debugger.attach() which is only callable
|
||||
// from the main process. We attach lazily on first use rather than at
|
||||
// creation time — that avoids the "Another debugger is already attached"
|
||||
// race when DevTools is opened on the webview.
|
||||
try {
|
||||
contents.debugger.on('detach', (_e, reason) => {
|
||||
console.log(`[cdp] detach on wcId ${contents.id}: ${reason}`);
|
||||
cdpAxIndexCache.delete(contents.id);
|
||||
});
|
||||
} catch (e) {
|
||||
// Older Electron may not have the listener API; non-fatal.
|
||||
}
|
||||
|
||||
contents.on('destroyed', () => {
|
||||
cdpAxIndexCache.delete(contents.id);
|
||||
cdpQueueByWcId.delete(contents.id);
|
||||
});
|
||||
|
||||
contents.on('render-process-gone', () => {
|
||||
cdpAxIndexCache.delete(contents.id);
|
||||
cdpQueueByWcId.delete(contents.id);
|
||||
});
|
||||
|
||||
contents.on('dom-ready', () => {
|
||||
const url = contents.getURL();
|
||||
if (url.includes('spotify')) {
|
||||
@@ -530,6 +559,86 @@ ipcMain.handle('open-external', (_event, url) => {
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CDP debugger bridge for the browser sub-agent
|
||||
// ---------------------------------------------------------------------------
|
||||
// Maintains a per-webContents AX index cache (numeric index → backendNodeId)
|
||||
// and serializes CDP commands per target so concurrent calls don't interleave.
|
||||
// The renderer calls window.openswarm.sendCdpCommand(wcId, method, params),
|
||||
// which routes through this handler to webContents.debugger.sendCommand().
|
||||
|
||||
const cdpAxIndexCache = new Map(); // wcId -> Map<index, backendNodeId>
|
||||
const cdpQueueByWcId = new Map(); // wcId -> Promise (serialization tail)
|
||||
|
||||
function getWebContentsById(wcId) {
|
||||
// webContents is exposed as a top-level Electron API
|
||||
const { webContents } = require('electron');
|
||||
return webContents.fromId(wcId);
|
||||
}
|
||||
|
||||
async function ensureDebuggerAttached(wc) {
|
||||
if (!wc || wc.isDestroyed()) {
|
||||
throw new Error('webContents is destroyed');
|
||||
}
|
||||
if (wc.debugger.isAttached()) return;
|
||||
try {
|
||||
wc.debugger.attach('1.3');
|
||||
} catch (err) {
|
||||
// Re-raise as a clean error string for the renderer.
|
||||
throw new Error(`debugger.attach failed: ${err.message || err}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function sendCdpCommandSerialized(wcId, method, params) {
|
||||
// Chain on the per-wcId queue so concurrent renderer calls run in order.
|
||||
const prev = cdpQueueByWcId.get(wcId) || Promise.resolve();
|
||||
const next = prev
|
||||
.catch(() => {}) // never let a previous failure poison the chain
|
||||
.then(async () => {
|
||||
const wc = getWebContentsById(wcId);
|
||||
if (!wc || wc.isDestroyed()) {
|
||||
throw new Error(`webContents ${wcId} not found or destroyed`);
|
||||
}
|
||||
await ensureDebuggerAttached(wc);
|
||||
return await wc.debugger.sendCommand(method, params || {});
|
||||
});
|
||||
cdpQueueByWcId.set(wcId, next);
|
||||
try {
|
||||
return await next;
|
||||
} finally {
|
||||
// If we're still the tail of the queue, clear it so the map doesn't grow.
|
||||
if (cdpQueueByWcId.get(wcId) === next) {
|
||||
cdpQueueByWcId.delete(wcId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ipcMain.handle('send-cdp-command', async (_event, wcId, method, params) => {
|
||||
try {
|
||||
const result = await sendCdpCommandSerialized(wcId, method, params);
|
||||
return { ok: true, result };
|
||||
} catch (err) {
|
||||
return { ok: false, error: err && err.message ? err.message : String(err) };
|
||||
}
|
||||
});
|
||||
|
||||
// Renderer-side AX index cache helpers — the renderer stores its own copy
|
||||
// keyed by (browser_id, tab_id). The main process only stores per-wcId for
|
||||
// invalidation purposes.
|
||||
ipcMain.handle('cdp-cache-set', (_event, wcId, indexMap) => {
|
||||
cdpAxIndexCache.set(wcId, indexMap || {});
|
||||
return { ok: true };
|
||||
});
|
||||
|
||||
ipcMain.handle('cdp-cache-get', (_event, wcId) => {
|
||||
return cdpAxIndexCache.get(wcId) || null;
|
||||
});
|
||||
|
||||
ipcMain.handle('cdp-cache-clear', (_event, wcId) => {
|
||||
cdpAxIndexCache.delete(wcId);
|
||||
return { ok: true };
|
||||
});
|
||||
|
||||
ipcMain.handle('connect-slack', async () => {
|
||||
const win = new BrowserWindow({
|
||||
width: 900,
|
||||
|
||||
@@ -13,6 +13,10 @@ const { contextBridge, ipcRenderer } = require('electron');
|
||||
getAppVersion: () => ipcRenderer.invoke('get-app-version'),
|
||||
openExternal: (url) => ipcRenderer.invoke('open-external', url),
|
||||
connectSlack: () => ipcRenderer.invoke('connect-slack'),
|
||||
sendCdpCommand: (wcId, method, params) => ipcRenderer.invoke('send-cdp-command', wcId, method, params),
|
||||
cdpCacheSet: (wcId, indexMap) => ipcRenderer.invoke('cdp-cache-set', wcId, indexMap),
|
||||
cdpCacheGet: (wcId) => ipcRenderer.invoke('cdp-cache-get', wcId),
|
||||
cdpCacheClear: (wcId) => ipcRenderer.invoke('cdp-cache-clear', wcId),
|
||||
capturePage: (rect) => ipcRenderer.invoke('capture-page', rect),
|
||||
getUpdateStatus: () => ipcRenderer.invoke('get-update-status'),
|
||||
checkForUpdates: () => ipcRenderer.invoke('check-for-updates'),
|
||||
|
||||
@@ -15,7 +15,6 @@ import {
|
||||
setUpdateError,
|
||||
} from '@/shared/state/updateSlice';
|
||||
import AppShell from './components/Layout/AppShell';
|
||||
import Dashboard from './pages/Dashboard/Dashboard';
|
||||
import DashboardSelection from './pages/DashboardSelection/DashboardSelection';
|
||||
import Templates from './pages/Templates/Templates';
|
||||
import Skills from './pages/Skills/Skills';
|
||||
@@ -249,7 +248,10 @@ const ThemedApp: React.FC = () => {
|
||||
<Routes>
|
||||
<Route element={<AppShell />}>
|
||||
<Route path="/" element={<DashboardSelection />} />
|
||||
<Route path="/dashboard/:id" element={<Dashboard />} />
|
||||
{/* Dashboard route is a no-op stub — the actual <Dashboard /> is rendered
|
||||
persistently inside AppShell so its webviews survive navigation between
|
||||
routes. This route exists only so React Router matches the URL. */}
|
||||
<Route path="/dashboard/:id" element={null} />
|
||||
<Route path="/customization" element={<Customization />} />
|
||||
<Route path="/templates" element={<Templates />} />
|
||||
<Route path="/skills" element={<Skills />} />
|
||||
|
||||
@@ -32,6 +32,9 @@ import CloseIcon from '@mui/icons-material/Close';
|
||||
import LinearProgress from '@mui/material/LinearProgress';
|
||||
import Settings from '@/app/pages/Settings/Settings';
|
||||
import DynamicIsland from '@/app/components/DynamicIsland';
|
||||
import Dashboard from '@/app/pages/Dashboard/Dashboard';
|
||||
import DashboardHost from '@/app/components/Layout/DashboardHost';
|
||||
import { useLastDashboardId } from '@/shared/hooks/useLastDashboardId';
|
||||
import { useAppDispatch, useAppSelector } from '@/shared/hooks';
|
||||
import { fetchDashboards, createDashboard, renameDashboard } from '@/shared/state/dashboardsSlice';
|
||||
import { addBrowserCard, addBrowserTab } from '@/shared/state/dashboardLayoutSlice';
|
||||
@@ -227,11 +230,16 @@ const AppShell: React.FC = () => {
|
||||
}, []);
|
||||
|
||||
const isDashboardRoute = location.pathname === '/' || location.pathname.startsWith('/dashboard/');
|
||||
const isDashboardViewActive = location.pathname.startsWith('/dashboard/');
|
||||
const isAppsRoute = location.pathname === '/apps' || location.pathname.startsWith('/apps/');
|
||||
const isCustomizationRoute = location.pathname === '/customization' || CUSTOMIZATION_PATHS.has(location.pathname);
|
||||
const activeDashboardId = location.pathname.startsWith('/dashboard/')
|
||||
? location.pathname.split('/dashboard/')[1]
|
||||
: null;
|
||||
|
||||
// Sticky last-visited dashboard id — survives navigation away from /dashboard/:id
|
||||
// so the Dashboard component can stay mounted with stable props.
|
||||
const [lastDashboardId, setLastDashboardId] = useLastDashboardId();
|
||||
const activeAppId = location.pathname.startsWith('/apps/')
|
||||
? location.pathname.split('/apps/')[1]
|
||||
: null;
|
||||
@@ -924,8 +932,30 @@ const AppShell: React.FC = () => {
|
||||
</>
|
||||
)}
|
||||
|
||||
<Box sx={{ flex: 1, overflow: 'hidden', bgcolor: c.bg.page }}>
|
||||
<Outlet />
|
||||
<Box sx={{ flex: 1, overflow: 'hidden', bgcolor: c.bg.page, position: 'relative' }}>
|
||||
{/* Non-dashboard routes render here. Hidden when the dashboard view is active
|
||||
so the persistent Dashboard layered above can take over the visible area. */}
|
||||
<Box
|
||||
sx={{
|
||||
position: 'absolute',
|
||||
inset: 0,
|
||||
visibility: isDashboardViewActive ? 'hidden' : 'visible',
|
||||
pointerEvents: isDashboardViewActive ? 'none' : 'auto',
|
||||
}}
|
||||
>
|
||||
<Outlet />
|
||||
</Box>
|
||||
|
||||
{/* Persistent Dashboard layer — always mounted once a dashboard has been visited.
|
||||
Hidden via CSS when on other routes so webviews and dashboard state survive
|
||||
route navigation. The Dashboard component reads its dashboardId from the
|
||||
sticky lastDashboardId hook so its dashboardId useEffect doesn't re-fire on
|
||||
incidental URL changes. */}
|
||||
{lastDashboardId && (
|
||||
<DashboardHost visible={isDashboardViewActive}>
|
||||
<Dashboard dashboardId={lastDashboardId} isActive={isDashboardViewActive} />
|
||||
</DashboardHost>
|
||||
)}
|
||||
</Box>
|
||||
</Box>
|
||||
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import React, { useEffect } from 'react';
|
||||
import { DashboardActiveProvider } from '@/shared/hooks/useDashboardActive';
|
||||
|
||||
interface DashboardHostProps {
|
||||
visible: boolean;
|
||||
children: React.ReactNode;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps the Dashboard component in a stable container that toggles visibility
|
||||
* via CSS instead of unmounting. This is what keeps the embedded webviews
|
||||
* alive across non-dashboard route navigation.
|
||||
*
|
||||
* Why this approach (vs. display: none or unmount):
|
||||
* - `visibility: hidden` preserves webview state without triggering Chromium
|
||||
* to mark the page as hidden (so background sub-agents keep working).
|
||||
* - `display: none` would trigger full layout recalc on toggle and may pause
|
||||
* pages that check `document.hidden`.
|
||||
* - Unmount destroys the webview DOM element, tearing down its Chromium tab.
|
||||
*
|
||||
* Also provides DashboardActiveContext to all children so they can gate
|
||||
* expensive work (canvas rendering, screenshot capture, etc.) on visibility.
|
||||
*/
|
||||
const DashboardHost: React.FC<DashboardHostProps> = ({ visible, children }) => {
|
||||
// When transitioning from visible -> hidden, blur any focused element so
|
||||
// a focused webview doesn't keep stealing keyboard input behind the scenes.
|
||||
useEffect(() => {
|
||||
if (!visible) {
|
||||
const el = document.activeElement;
|
||||
if (el instanceof HTMLElement) {
|
||||
el.blur();
|
||||
}
|
||||
}
|
||||
}, [visible]);
|
||||
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
position: 'absolute',
|
||||
inset: 0,
|
||||
// Negative z-index when hidden so any visible Outlet content sits above
|
||||
zIndex: visible ? 10 : -1,
|
||||
visibility: visible ? 'visible' : 'hidden',
|
||||
// Belt-and-suspenders: even if z-index ordering glitches, no clicks land
|
||||
pointerEvents: visible ? 'auto' : 'none',
|
||||
}}
|
||||
>
|
||||
<DashboardActiveProvider value={visible}>
|
||||
{children}
|
||||
</DashboardActiveProvider>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default DashboardHost;
|
||||
@@ -30,6 +30,7 @@ import { QuestionForm } from '@/app/pages/AgentChat/ApprovalBar';
|
||||
import AgentChat from '@/app/pages/AgentChat/AgentChat';
|
||||
import { parseMcpToolName, getMcpShortAction } from '@/app/pages/AgentChat/ToolCallBubble';
|
||||
import { useClaudeTokens } from '@/shared/styles/ThemeContext';
|
||||
import { useDashboardActive } from '@/shared/hooks/useDashboardActive';
|
||||
import { useOverlayScrollPassthrough } from './useOverlayScrollPassthrough';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -239,14 +240,23 @@ const AgentCard: React.FC<Props> = ({
|
||||
}) => {
|
||||
const c = useClaudeTokens();
|
||||
const dispatch = useAppDispatch();
|
||||
const isDashboardActive = useDashboardActive();
|
||||
const hasApiKey = !!useAppSelector((s) => s.settings.data.anthropic_api_key);
|
||||
const scrollOverlayRef = useOverlayScrollPassthrough(isSelected);
|
||||
|
||||
const cardBoxRef = useRef<HTMLDivElement>(null);
|
||||
// Capture isDashboardActive in a ref so the ResizeObserver callback always
|
||||
// sees the latest value without forcing the observer to re-attach when the
|
||||
// active state flips.
|
||||
const isDashboardActiveRef = useRef(isDashboardActive);
|
||||
useEffect(() => { isDashboardActiveRef.current = isDashboardActive; }, [isDashboardActive]);
|
||||
useEffect(() => {
|
||||
const el = cardBoxRef.current;
|
||||
if (!el || !onMeasuredHeight) return;
|
||||
const ro = new ResizeObserver((entries) => {
|
||||
// Short-circuit when dashboard is hidden — observer stays attached so
|
||||
// the next resize after returning to the dashboard fires correctly.
|
||||
if (!isDashboardActiveRef.current) return;
|
||||
for (const entry of entries) {
|
||||
onMeasuredHeight(session.id, entry.contentRect.height);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import React, { useEffect, useCallback, useRef, useState, useMemo } from 'react';
|
||||
import { AnimatePresence, motion } from 'framer-motion';
|
||||
import { useParams } from 'react-router-dom';
|
||||
import Box from '@mui/material/Box';
|
||||
import Typography from '@mui/material/Typography';
|
||||
import DashboardHeader from './DashboardHeader';
|
||||
@@ -85,10 +84,14 @@ function isCardTarget(target: EventTarget | null, boundary: EventTarget | null):
|
||||
return false;
|
||||
}
|
||||
|
||||
const DashboardInner: React.FC = () => {
|
||||
interface DashboardProps {
|
||||
dashboardId: string;
|
||||
isActive?: boolean;
|
||||
}
|
||||
|
||||
const DashboardInner: React.FC<DashboardProps> = ({ dashboardId, isActive = true }) => {
|
||||
const c = useClaudeTokens();
|
||||
const dispatch = useAppDispatch();
|
||||
const { id: dashboardId } = useParams<{ id: string }>();
|
||||
const elementSelectionCtx = useElementSelection();
|
||||
const isElementSelectMode = elementSelectionCtx?.selectMode ?? false;
|
||||
const dashboardName = useAppSelector((state) =>
|
||||
@@ -128,7 +131,7 @@ const DashboardInner: React.FC = () => {
|
||||
return { minX, minY, maxX, maxY };
|
||||
}, [cards, viewCards, browserCards]);
|
||||
|
||||
const canvas = useCanvasControls(zoomSensitivity, contentBounds);
|
||||
const canvas = useCanvasControls(zoomSensitivity, contentBounds, isActive);
|
||||
const selection = useDashboardSelection(
|
||||
{ panX: canvas.panX, panY: canvas.panY, zoom: canvas.zoom, viewportRef: canvas.viewportRef },
|
||||
cards,
|
||||
@@ -438,7 +441,12 @@ const DashboardInner: React.FC = () => {
|
||||
dispatch(fetchOutputs());
|
||||
dashboardWs.connect();
|
||||
const cleanupBrowserHandler = initBrowserCommandHandler();
|
||||
return () => { dispatch(resetLayout()); cleanupBrowserHandler(); dashboardWs.disconnect(); };
|
||||
// Note: cleanup runs only on dashboardId change (explicit dashboard switch)
|
||||
// or full unmount. With the hide-don't-unmount pattern in AppShell, route
|
||||
// navigation no longer triggers this cleanup. We deliberately do NOT call
|
||||
// resetLayout() here — switching dashboards already calls it via the next
|
||||
// effect run, and calling it from cleanup would race with the new layout fetch.
|
||||
return () => { cleanupBrowserHandler(); dashboardWs.disconnect(); };
|
||||
}, [dispatch, dashboardId]);
|
||||
|
||||
const pendingBrowserUrl = useAppSelector((state) => state.tempState.pendingBrowserUrl);
|
||||
@@ -481,11 +489,12 @@ const DashboardInner: React.FC = () => {
|
||||
}, [canvas.viewportRef, canvas.contentRef]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isActive) return; // Skip thumbnail capture when dashboard is hidden
|
||||
if (!dashboardId || !layoutInitialized) return;
|
||||
if (captureTimerRef.current) clearTimeout(captureTimerRef.current);
|
||||
captureTimerRef.current = setTimeout(captureNow, 2000);
|
||||
return () => { if (captureTimerRef.current) clearTimeout(captureTimerRef.current); };
|
||||
}, [dashboardId, layoutInitialized, captureNow]);
|
||||
}, [isActive, dashboardId, layoutInitialized, captureNow]);
|
||||
|
||||
// On exit, save the captured thumbnail to the backend
|
||||
useEffect(() => {
|
||||
@@ -501,14 +510,16 @@ const DashboardInner: React.FC = () => {
|
||||
}, [dashboardId]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isActive) return; // Don't auto-fit while dashboard is hidden
|
||||
if (!layoutInitialized || hasFittedRef.current) return;
|
||||
if (pendingFocusAgentId) return;
|
||||
hasFittedRef.current = true;
|
||||
const timer = setTimeout(() => canvas.actions.fitToView(), 150);
|
||||
return () => clearTimeout(timer);
|
||||
}, [layoutInitialized, canvas.actions, pendingFocusAgentId]);
|
||||
}, [isActive, layoutInitialized, canvas.actions, pendingFocusAgentId]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isActive) return; // Defer focus animation until dashboard is visible
|
||||
if (!pendingFocusAgentId || !layoutInitialized) return;
|
||||
const agentId = pendingFocusAgentId;
|
||||
dispatch(clearPendingFocusAgentId());
|
||||
@@ -520,7 +531,7 @@ const DashboardInner: React.FC = () => {
|
||||
handleHighlightCard(agentId);
|
||||
}
|
||||
}, 350);
|
||||
}, [pendingFocusAgentId, layoutInitialized, dispatch, canvas.actions, handleHighlightCard]);
|
||||
}, [isActive, pendingFocusAgentId, layoutInitialized, dispatch, canvas.actions, handleHighlightCard]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!layoutInitialized || restoredExpandedRef.current) return;
|
||||
@@ -547,6 +558,7 @@ const DashboardInner: React.FC = () => {
|
||||
const prevParentStatusRef = useRef<Record<string, string>>({});
|
||||
|
||||
useEffect(() => {
|
||||
if (!isActive) return; // Heavy logic — pause when dashboard is hidden
|
||||
if (!layoutInitialized || !autoRevealSubAgents) return;
|
||||
|
||||
const subSessions = Object.values(sessions).filter(
|
||||
@@ -633,13 +645,14 @@ const DashboardInner: React.FC = () => {
|
||||
if (parent) newParentStatuses[pid] = parent.status;
|
||||
}
|
||||
prevParentStatusRef.current = newParentStatuses;
|
||||
}, [sessions, cards, layoutInitialized, autoRevealSubAgents, dispatch]);
|
||||
}, [isActive, sessions, cards, layoutInitialized, autoRevealSubAgents, dispatch]);
|
||||
|
||||
const skipInitialSave = useRef(true);
|
||||
const saveTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
const pendingSaveRef = useRef<Parameters<typeof saveLayout>[0] | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isActive) return; // Don't persist layout while dashboard is hidden — save buffers in pendingSaveRef and flushes on resume
|
||||
if (!layoutInitialized || !dashboardId) return;
|
||||
if (skipInitialSave.current) {
|
||||
skipInitialSave.current = false;
|
||||
@@ -654,7 +667,7 @@ const DashboardInner: React.FC = () => {
|
||||
saveTimerRef.current = null;
|
||||
captureNow();
|
||||
}, 500);
|
||||
}, [cards, viewCards, browserCards, expandedSessionIds, layoutInitialized, dashboardId, dispatch, captureNow]);
|
||||
}, [isActive, cards, viewCards, browserCards, expandedSessionIds, layoutInitialized, dashboardId, dispatch, captureNow]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
@@ -678,6 +691,7 @@ const DashboardInner: React.FC = () => {
|
||||
const needsAlt = parts.includes('alt');
|
||||
|
||||
const handleShortcut = (e: KeyboardEvent) => {
|
||||
if (!isActive) return; // Don't fire shortcuts when dashboard is hidden
|
||||
if (e.key.toLowerCase() !== key) return;
|
||||
if (needsMeta !== e.metaKey) return;
|
||||
if (needsCtrl !== e.ctrlKey) return;
|
||||
@@ -692,6 +706,7 @@ const DashboardInner: React.FC = () => {
|
||||
|
||||
useEffect(() => {
|
||||
const handleEnter = (e: KeyboardEvent) => {
|
||||
if (!isActive) return; // Don't fire shortcuts when dashboard is hidden
|
||||
if (e.key !== 'Enter') return;
|
||||
const tag = (e.target as HTMLElement)?.tagName;
|
||||
if (tag === 'INPUT' || tag === 'TEXTAREA' || (e.target as HTMLElement)?.isContentEditable) return;
|
||||
@@ -707,6 +722,7 @@ const DashboardInner: React.FC = () => {
|
||||
|
||||
useEffect(() => {
|
||||
const handleDelete = (e: KeyboardEvent) => {
|
||||
if (!isActive) return; // Don't fire shortcuts when dashboard is hidden
|
||||
if (e.key !== 'Backspace' && e.key !== 'Delete') return;
|
||||
const tag = (e.target as HTMLElement)?.tagName;
|
||||
if (tag === 'INPUT' || tag === 'TEXTAREA' || (e.target as HTMLElement)?.isContentEditable) return;
|
||||
@@ -730,6 +746,7 @@ const DashboardInner: React.FC = () => {
|
||||
// Cmd+F to open card search palette
|
||||
useEffect(() => {
|
||||
const handleSearch = (e: KeyboardEvent) => {
|
||||
if (!isActive) return; // Don't fire shortcuts when dashboard is hidden
|
||||
if (!(e.metaKey || e.ctrlKey) || e.key.toLowerCase() !== 'f') return;
|
||||
const tag = (e.target as HTMLElement)?.tagName;
|
||||
if (tag === 'INPUT' || tag === 'TEXTAREA' || (e.target as HTMLElement)?.isContentEditable) return;
|
||||
@@ -743,6 +760,7 @@ const DashboardInner: React.FC = () => {
|
||||
|
||||
useEffect(() => {
|
||||
const handleCopy = (e: KeyboardEvent) => {
|
||||
if (!isActive) return; // Don't fire shortcuts when dashboard is hidden
|
||||
if (!(e.metaKey || e.ctrlKey) || e.key.toLowerCase() !== 'c') return;
|
||||
const tag = (e.target as HTMLElement)?.tagName;
|
||||
if (tag === 'INPUT' || tag === 'TEXTAREA' || (e.target as HTMLElement)?.isContentEditable) return;
|
||||
@@ -796,6 +814,7 @@ const DashboardInner: React.FC = () => {
|
||||
useEffect(() => {
|
||||
const PASTE_OFFSET = 40;
|
||||
const handlePaste = async (e: KeyboardEvent) => {
|
||||
if (!isActive) return; // Don't fire shortcuts when dashboard is hidden
|
||||
if (!(e.metaKey || e.ctrlKey) || e.key.toLowerCase() !== 'v') return;
|
||||
const tag = (e.target as HTMLElement)?.tagName;
|
||||
if (tag === 'INPUT' || tag === 'TEXTAREA' || (e.target as HTMLElement)?.isContentEditable) return;
|
||||
@@ -916,6 +935,7 @@ const DashboardInner: React.FC = () => {
|
||||
|
||||
useEffect(() => {
|
||||
const handleArrowNav = (e: KeyboardEvent) => {
|
||||
if (!isActive) return; // Don't fire shortcuts when dashboard is hidden
|
||||
const currentFocused = focusedCardIdRef.current;
|
||||
if (!currentFocused || canvasZoomRef.current < 0.9) return;
|
||||
|
||||
@@ -1193,6 +1213,7 @@ const DashboardInner: React.FC = () => {
|
||||
}, [dispatch, canvas.actions]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isActive) return; // Heavy geometry recalculation — pause when dashboard is hidden
|
||||
const DRIFT_THRESHOLD = 60;
|
||||
|
||||
// Group tethered sub-agent cards by source, only including those still in the spawn column
|
||||
@@ -1230,9 +1251,10 @@ const DashboardInner: React.FC = () => {
|
||||
// measuredHeightsTick in deps ensures we re-run once ResizeObserver reports
|
||||
// the new height after a collapse (avoids stale-height no-ops)
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [expandedSessionIds, glowingAgentCards, cards, dispatch, measuredHeightsTick]);
|
||||
}, [isActive, expandedSessionIds, glowingAgentCards, cards, dispatch, measuredHeightsTick]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isActive) return; // Heavy geometry recalculation — pause when dashboard is hidden
|
||||
const DRIFT_THRESHOLD = 60;
|
||||
|
||||
const sourceToSiblings = new Map<string, string[]>();
|
||||
@@ -1262,7 +1284,7 @@ const DashboardInner: React.FC = () => {
|
||||
cursor += bc.height + GRID_GAP * 2;
|
||||
}
|
||||
}
|
||||
}, [glowingBrowserCards, browserCards, cards, dispatch]);
|
||||
}, [isActive, glowingBrowserCards, browserCards, cards, dispatch]);
|
||||
|
||||
const TETHER_FADE_MS = 2500;
|
||||
|
||||
@@ -1853,9 +1875,9 @@ const DashboardInner: React.FC = () => {
|
||||
);
|
||||
};
|
||||
|
||||
const Dashboard: React.FC = () => (
|
||||
const Dashboard: React.FC<DashboardProps> = ({ dashboardId, isActive = true }) => (
|
||||
<ElementSelectionProvider>
|
||||
<DashboardInner />
|
||||
<DashboardInner dashboardId={dashboardId} isActive={isActive} />
|
||||
</ElementSelectionProvider>
|
||||
);
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@ export interface ContentBounds {
|
||||
maxY: number;
|
||||
}
|
||||
|
||||
export function useCanvasControls(zoomSensitivity: number = 50, contentBounds?: ContentBounds) {
|
||||
export function useCanvasControls(zoomSensitivity: number = 50, contentBounds?: ContentBounds, enabled: boolean = true) {
|
||||
const viewportRef = useRef<HTMLDivElement>(null);
|
||||
const contentRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
@@ -168,7 +168,7 @@ export function useCanvasControls(zoomSensitivity: number = 50, contentBounds?:
|
||||
// Wheel zoom centered on cursor
|
||||
useEffect(() => {
|
||||
const el = viewportRef.current;
|
||||
if (!el) return;
|
||||
if (!el || !enabled) return; // Skip wheel listener when canvas is hidden
|
||||
|
||||
const onWheel = (e: WheelEvent) => {
|
||||
// Pinch-to-zoom on trackpads sets ctrlKey; plain scroll does not
|
||||
@@ -244,7 +244,7 @@ export function useCanvasControls(zoomSensitivity: number = 50, contentBounds?:
|
||||
|
||||
el.addEventListener('wheel', onWheel, { passive: false });
|
||||
return () => el.removeEventListener('wheel', onWheel);
|
||||
}, []);
|
||||
}, [enabled]);
|
||||
|
||||
const handleMouseDown = useCallback((e: React.MouseEvent) => {
|
||||
e.preventDefault();
|
||||
|
||||
@@ -4,7 +4,7 @@ import { resolveInput } from './resolveUrl';
|
||||
|
||||
let initialized = false;
|
||||
|
||||
export type BrowserAction = 'screenshot' | 'get_text' | 'navigate' | 'click' | 'type' | 'evaluate' | 'get_elements' | 'scroll' | 'wait';
|
||||
export type BrowserAction = 'screenshot' | 'get_text' | 'navigate' | 'click' | 'type' | 'evaluate' | 'get_elements' | 'scroll' | 'wait' | 'press_key' | 'list_interactives' | 'click_index' | 'batch';
|
||||
|
||||
export interface BrowserActivity {
|
||||
action: BrowserAction;
|
||||
@@ -45,6 +45,10 @@ const ACTION_LABELS: Record<string, string> = {
|
||||
get_elements: 'Inspecting...',
|
||||
scroll: 'Scrolling...',
|
||||
wait: 'Waiting...',
|
||||
press_key: 'Pressing key...',
|
||||
list_interactives: 'Reading page structure...',
|
||||
click_index: 'Clicking element...',
|
||||
batch: 'Running batch...',
|
||||
};
|
||||
|
||||
export function getActionLabel(action: string): string {
|
||||
@@ -128,11 +132,319 @@ async function handleType(wv: BrowserWebview, params: Record<string, any>): Prom
|
||||
return {
|
||||
text: 'Typed into: ' + el.tagName.toLowerCase() + (el.id ? '#' + el.id : ''),
|
||||
};
|
||||
})()`;
|
||||
})()`;
|
||||
const result = await wv.executeJavaScript(code);
|
||||
return result;
|
||||
}
|
||||
|
||||
// Map common JS KeyboardEvent.key values to Electron's accelerator keyCodes.
|
||||
// Electron's sendInputEvent expects: 'Up', 'Down', 'Left', 'Right', 'Enter',
|
||||
// 'Escape', 'Tab', 'Backspace', 'Delete', 'Space', or single char letters.
|
||||
const KEY_NAME_MAP: Record<string, string> = {
|
||||
ArrowUp: 'Up',
|
||||
ArrowDown: 'Down',
|
||||
ArrowLeft: 'Left',
|
||||
ArrowRight: 'Right',
|
||||
' ': 'Space',
|
||||
Spacebar: 'Space',
|
||||
Esc: 'Escape',
|
||||
Del: 'Delete',
|
||||
};
|
||||
|
||||
async function handlePressKey(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
||||
const rawKey = (params.key as string) || '';
|
||||
if (!rawKey) return { error: 'key parameter is required' };
|
||||
const keyCode = KEY_NAME_MAP[rawKey] || rawKey;
|
||||
// Focus the page first so the key event has a sensible target.
|
||||
await wv.executeJavaScript('document.body && document.body.focus && document.body.focus(); true');
|
||||
// Native OS-level key events — these have event.isTrusted === true so site
|
||||
// keyboard handlers (Tinder, Slack, Notion, etc.) actually respect them.
|
||||
wv.sendInputEvent({ type: 'keyDown', keyCode });
|
||||
wv.sendInputEvent({ type: 'char', keyCode });
|
||||
wv.sendInputEvent({ type: 'keyUp', keyCode });
|
||||
return { text: `Pressed ${rawKey}` };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CDP accessibility-tree element indexing
|
||||
// ---------------------------------------------------------------------------
|
||||
// list_interactives uses Chrome DevTools Protocol's Accessibility.getFullAXTree
|
||||
// to get the *computed* accessibility tree, not the raw DOM. This sees roles,
|
||||
// names, and labels even on hostile sites (Tinder, Instagram) where the raw
|
||||
// HTML is just unlabeled <div>s with click handlers — because Chromium computes
|
||||
// accessible names for screen readers from icons, surrounding text, etc.
|
||||
//
|
||||
// Each interactive element is assigned a numeric index. The index → backendNodeId
|
||||
// map is cached server-side per webContents and used by click_index. This is
|
||||
// orders of magnitude more reliable than CSS-selector-based clicking on sites
|
||||
// that don't expose semantic markup.
|
||||
|
||||
const INTERACTIVE_ROLES = new Set([
|
||||
'button', 'link', 'textbox', 'combobox', 'checkbox', 'menuitem',
|
||||
'tab', 'switch', 'searchbox', 'slider', 'listbox', 'option',
|
||||
'radio', 'menuitemcheckbox', 'menuitemradio', 'spinbutton', 'treeitem',
|
||||
]);
|
||||
|
||||
interface InteractiveElement {
|
||||
index: number;
|
||||
role: string;
|
||||
name: string;
|
||||
backendNodeId: number;
|
||||
}
|
||||
|
||||
function extractAxValue(prop: any): string {
|
||||
if (!prop) return '';
|
||||
if (typeof prop === 'string') return prop;
|
||||
if (prop.value !== undefined) {
|
||||
if (typeof prop.value === 'string') return prop.value;
|
||||
if (typeof prop.value === 'object' && prop.value && 'value' in prop.value) {
|
||||
return String(prop.value.value || '');
|
||||
}
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
interface CdpResult { ok: boolean; result?: any; error?: string }
|
||||
|
||||
async function sendCdp(wv: BrowserWebview, method: string, params?: Record<string, any>): Promise<any> {
|
||||
const wcId = wv.getWebContentsId();
|
||||
const bridge = (window as any).openswarm?.sendCdpCommand as
|
||||
| ((id: number, m: string, p?: any) => Promise<CdpResult>)
|
||||
| undefined;
|
||||
if (!bridge) throw new Error('CDP bridge not available — restart the app');
|
||||
const resp = await bridge(wcId, method, params);
|
||||
if (!resp || !resp.ok) {
|
||||
throw new Error(resp?.error || `CDP ${method} failed`);
|
||||
}
|
||||
return resp.result;
|
||||
}
|
||||
|
||||
async function handleListInteractives(wv: BrowserWebview): Promise<Record<string, any>> {
|
||||
let axResult;
|
||||
try {
|
||||
axResult = await sendCdp(wv, 'Accessibility.getFullAXTree', {});
|
||||
} catch (err: any) {
|
||||
return { error: `getFullAXTree failed: ${err.message || String(err)}` };
|
||||
}
|
||||
|
||||
const nodes: any[] = axResult?.nodes || [];
|
||||
const interactives: InteractiveElement[] = [];
|
||||
let index = 1;
|
||||
|
||||
for (const node of nodes) {
|
||||
if (node.ignored) continue;
|
||||
const role = extractAxValue(node.role);
|
||||
if (!INTERACTIVE_ROLES.has(role)) continue;
|
||||
const name = extractAxValue(node.name);
|
||||
if (!name && role !== 'textbox' && role !== 'searchbox' && role !== 'combobox') {
|
||||
// Skip nameless elements unless they're inputs (which can be empty)
|
||||
continue;
|
||||
}
|
||||
const backendNodeId = node.backendDOMNodeId;
|
||||
if (backendNodeId == null) continue;
|
||||
interactives.push({ index, role, name: name.slice(0, 80), backendNodeId });
|
||||
index++;
|
||||
}
|
||||
|
||||
// Cache the index map in main-process storage so click_index can resolve it
|
||||
// even across separate WebSocket commands.
|
||||
const indexMap: Record<number, number> = {};
|
||||
for (const el of interactives) {
|
||||
indexMap[el.index] = el.backendNodeId;
|
||||
}
|
||||
try {
|
||||
const cacheBridge = (window as any).openswarm?.cdpCacheSet;
|
||||
if (cacheBridge) await cacheBridge(wv.getWebContentsId(), indexMap);
|
||||
} catch {
|
||||
// Cache is best-effort; click_index will fall back to re-listing.
|
||||
}
|
||||
|
||||
// Build the model-friendly text representation: [1]<button "Like">
|
||||
const lines = interactives.map(
|
||||
(el) => `[${el.index}]<${el.role} "${el.name}">`,
|
||||
);
|
||||
const text = lines.length
|
||||
? `${lines.length} interactive elements:\n${lines.join('\n')}`
|
||||
: 'No interactive elements found on this page.';
|
||||
|
||||
return {
|
||||
text,
|
||||
elements: interactives.map((el) => ({ index: el.index, role: el.role, name: el.name })),
|
||||
url: wv.getURL(),
|
||||
};
|
||||
}
|
||||
|
||||
async function handleClickIndex(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
||||
const idx = Number(params.index);
|
||||
if (!Number.isFinite(idx) || idx < 1) {
|
||||
return { error: 'index parameter is required and must be a positive integer' };
|
||||
}
|
||||
|
||||
// Look up the cached index → backendNodeId mapping.
|
||||
let backendNodeId: number | undefined;
|
||||
try {
|
||||
const cacheBridge = (window as any).openswarm?.cdpCacheGet;
|
||||
if (cacheBridge) {
|
||||
const cached = await cacheBridge(wv.getWebContentsId());
|
||||
if (cached && cached[idx] != null) {
|
||||
backendNodeId = Number(cached[idx]);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// fall through to error path below
|
||||
}
|
||||
|
||||
if (backendNodeId == null) {
|
||||
return {
|
||||
error: `Index ${idx} is not in the cached element map. Call BrowserListInteractives first to refresh the index, then try again.`,
|
||||
};
|
||||
}
|
||||
|
||||
// Cheap revalidation: resolve the backend node ID to a runtime object.
|
||||
// If the page has mutated and the node is gone, this fails fast with a
|
||||
// clear error message instead of clicking the wrong element.
|
||||
try {
|
||||
await sendCdp(wv, 'DOM.resolveNode', { backendNodeId });
|
||||
} catch (err: any) {
|
||||
return {
|
||||
error: `Index ${idx} is no longer valid (${err.message || 'node not found'}). The page may have changed. Call BrowserListInteractives again.`,
|
||||
};
|
||||
}
|
||||
|
||||
// Get the element's bounding box for clicking via Input.dispatchMouseEvent
|
||||
// (more reliable than Element.click() on hostile sites — bypasses any
|
||||
// synthetic-event filtering since these are real OS-level mouse events).
|
||||
let boxModel;
|
||||
try {
|
||||
boxModel = await sendCdp(wv, 'DOM.getBoxModel', { backendNodeId });
|
||||
} catch (err: any) {
|
||||
return {
|
||||
error: `Index ${idx} has no box model (likely off-screen or hidden). Try scrolling first or call BrowserListInteractives again.`,
|
||||
};
|
||||
}
|
||||
|
||||
const content = boxModel?.model?.content;
|
||||
if (!Array.isArray(content) || content.length < 8) {
|
||||
return { error: `Index ${idx} has no valid bounding rect.` };
|
||||
}
|
||||
// content is [x1,y1, x2,y2, x3,y3, x4,y4] — compute center
|
||||
const x = (content[0] + content[4]) / 2;
|
||||
const y = (content[1] + content[5]) / 2;
|
||||
|
||||
try {
|
||||
await sendCdp(wv, 'Input.dispatchMouseEvent', {
|
||||
type: 'mousePressed',
|
||||
x, y,
|
||||
button: 'left',
|
||||
clickCount: 1,
|
||||
});
|
||||
await sendCdp(wv, 'Input.dispatchMouseEvent', {
|
||||
type: 'mouseReleased',
|
||||
x, y,
|
||||
button: 'left',
|
||||
clickCount: 1,
|
||||
});
|
||||
} catch (err: any) {
|
||||
return { error: `Click failed: ${err.message || String(err)}` };
|
||||
}
|
||||
|
||||
return {
|
||||
text: `Clicked index ${idx} at (${Math.round(x)}, ${Math.round(y)})`,
|
||||
clickX: x / wv.clientWidth * 100,
|
||||
clickY: y / wv.clientHeight * 100,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Batched actions
|
||||
// ---------------------------------------------------------------------------
|
||||
// handleBatch executes a list of sub-actions sequentially on the same webview,
|
||||
// capturing the URL before/after each one and aborting the rest of the batch
|
||||
// if the URL changes mid-batch (page navigated → indices and selectors are
|
||||
// stale). This lets the model emit "[click_index 7, wait 500, type 'eric',
|
||||
// press_key Enter]" in a single tool call instead of round-tripping for each
|
||||
// action.
|
||||
|
||||
const MAX_BATCH_ACTIONS = 5;
|
||||
|
||||
type SubActionType =
|
||||
| 'click_index' | 'press_key' | 'type' | 'wait'
|
||||
| 'scroll' | 'navigate' | 'click';
|
||||
|
||||
const BATCH_DISPATCH: Record<SubActionType, (wv: BrowserWebview, p: Record<string, any>) => Promise<Record<string, any>>> = {
|
||||
click_index: handleClickIndex,
|
||||
press_key: handlePressKey,
|
||||
type: handleType,
|
||||
wait: handleWait,
|
||||
scroll: handleScroll,
|
||||
navigate: handleNavigate,
|
||||
click: handleClick,
|
||||
};
|
||||
|
||||
async function handleBatch(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
||||
const actions: any[] = Array.isArray(params.actions) ? params.actions : [];
|
||||
if (actions.length === 0) {
|
||||
return { error: 'actions parameter must be a non-empty array' };
|
||||
}
|
||||
if (actions.length > MAX_BATCH_ACTIONS) {
|
||||
return {
|
||||
error: `Batch too large: ${actions.length} actions (max ${MAX_BATCH_ACTIONS}). Split into smaller batches.`,
|
||||
};
|
||||
}
|
||||
|
||||
const results: Array<Record<string, any>> = [];
|
||||
let aborted_at: number | null = null;
|
||||
let abort_reason: string | null = null;
|
||||
|
||||
for (let i = 0; i < actions.length; i++) {
|
||||
const action = actions[i];
|
||||
const subType = action?.type as SubActionType;
|
||||
const subParams = action?.params || {};
|
||||
|
||||
if (!subType || !(subType in BATCH_DISPATCH)) {
|
||||
results.push({ index: i, type: subType, error: `Unknown sub-action type: ${subType}` });
|
||||
// Continue with the rest — per-action failures don't abort the batch.
|
||||
continue;
|
||||
}
|
||||
|
||||
const urlBefore = wv.getURL();
|
||||
let subResult: Record<string, any>;
|
||||
try {
|
||||
subResult = await BATCH_DISPATCH[subType](wv, subParams);
|
||||
} catch (err: any) {
|
||||
subResult = { error: `Sub-action failed: ${err?.message || String(err)}` };
|
||||
}
|
||||
results.push({ index: i, type: subType, ...subResult });
|
||||
|
||||
// If the URL changed, abort the rest — selectors and indices are stale
|
||||
// and any subsequent actions would be operating on a half-loaded page.
|
||||
const urlAfter = wv.getURL();
|
||||
if (urlAfter !== urlBefore && i < actions.length - 1) {
|
||||
aborted_at = i + 1;
|
||||
abort_reason = `URL changed mid-batch from ${urlBefore} to ${urlAfter}; remaining ${actions.length - i - 1} action(s) skipped`;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const summary_lines = results.map((r, i) => {
|
||||
const status = r.error ? `FAIL (${r.error})` : 'OK';
|
||||
return ` ${i + 1}. ${r.type}: ${status}`;
|
||||
});
|
||||
const text = [
|
||||
`Batch executed ${results.length}/${actions.length} actions`,
|
||||
...summary_lines,
|
||||
aborted_at !== null ? `\nABORTED at action ${aborted_at}: ${abort_reason}` : '',
|
||||
].filter(Boolean).join('\n');
|
||||
|
||||
return {
|
||||
text,
|
||||
results,
|
||||
aborted_at,
|
||||
abort_reason,
|
||||
url: wv.getURL(),
|
||||
};
|
||||
}
|
||||
|
||||
async function handleScroll(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
||||
const direction = (params.direction as string) || 'down';
|
||||
const amount = (params.amount as number) || 500;
|
||||
@@ -350,6 +662,25 @@ async function handleBrowserCommand(data: Record<string, any>) {
|
||||
case 'wait':
|
||||
result = await handleWait(wv, params);
|
||||
break;
|
||||
case 'press_key':
|
||||
result = await handlePressKey(wv, params);
|
||||
break;
|
||||
case 'list_interactives':
|
||||
result = await handleListInteractives(wv);
|
||||
break;
|
||||
case 'click_index':
|
||||
result = await handleClickIndex(wv, params);
|
||||
if (result.clickX != null && result.clickY != null) {
|
||||
setActivity(browser_id, {
|
||||
action: 'click_index',
|
||||
detail,
|
||||
coords: { xPercent: result.clickX, yPercent: result.clickY },
|
||||
});
|
||||
}
|
||||
break;
|
||||
case 'batch':
|
||||
result = await handleBatch(wv, params);
|
||||
break;
|
||||
default:
|
||||
result = { error: `Unknown browser action: ${action}` };
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ export interface BrowserWebview extends HTMLElement {
|
||||
}>;
|
||||
executeJavaScript: (code: string) => Promise<any>;
|
||||
sendInputEvent: (event: any) => void;
|
||||
getWebContentsId: () => number;
|
||||
addEventListener: (event: string, listener: (...args: any[]) => void) => void;
|
||||
removeEventListener: (event: string, listener: (...args: any[]) => void) => void;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
import React, { createContext, useContext } from 'react';
|
||||
|
||||
/**
|
||||
* React context that signals whether the Dashboard is currently the active
|
||||
* route (i.e. visible to the user) vs hidden in the background.
|
||||
*
|
||||
* Defaults to `true` so any standalone usage of dashboard children outside
|
||||
* the DashboardHost wrapper just behaves normally.
|
||||
*
|
||||
* Heavy/expensive Dashboard children read this via `useDashboardActive()`
|
||||
* and short-circuit their work when the dashboard is hidden — that's how
|
||||
* we keep CPU usage near-zero while the user is on /actions or /settings
|
||||
* with the Dashboard mounted but invisible.
|
||||
*/
|
||||
const DashboardActiveContext = createContext<boolean>(true);
|
||||
|
||||
export const DashboardActiveProvider = DashboardActiveContext.Provider;
|
||||
|
||||
export function useDashboardActive(): boolean {
|
||||
return useContext(DashboardActiveContext);
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import { useEffect, useState, useCallback } from 'react';
|
||||
import { useLocation } from 'react-router-dom';
|
||||
|
||||
const STORAGE_KEY = 'openswarm_last_dashboard_id';
|
||||
const WINDOW_KEY = '__openswarm_last_dashboard_id';
|
||||
|
||||
/**
|
||||
* Tracks the last visited dashboard id in a "sticky" way: once a dashboard
|
||||
* has been visited, the id stays set even when the user navigates to other
|
||||
* routes. This is the foundation for keeping the Dashboard component mounted
|
||||
* across non-dashboard route navigation (hide-don't-unmount pattern).
|
||||
*
|
||||
* The Dashboard component reads its dashboardId from this hook (via a prop
|
||||
* passed by AppShell) instead of from `useParams()`, so the id never goes
|
||||
* undefined when the URL changes to /actions etc. This prevents the
|
||||
* dashboardId useEffect from re-firing on every incidental route change,
|
||||
* which would cause `resetLayout` + `fetchLayout` and visibly reload the
|
||||
* browser cards.
|
||||
*
|
||||
* Returns a tuple of `[lastDashboardId, setLastDashboardId]`. The setter
|
||||
* is exposed so explicit dashboard close/delete handlers can clear it
|
||||
* (which causes the Dashboard to fully unmount and tear down its webviews).
|
||||
*/
|
||||
export function useLastDashboardId(): [string | null, (id: string | null) => void] {
|
||||
const location = useLocation();
|
||||
const [lastId, setLastIdState] = useState<string | null>(() => {
|
||||
try {
|
||||
return localStorage.getItem(STORAGE_KEY);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
});
|
||||
|
||||
// Watch the URL — when it matches /dashboard/:id, update the sticky id.
|
||||
// Critically: do NOT clear the sticky id when the URL stops matching.
|
||||
useEffect(() => {
|
||||
const match = location.pathname.match(/^\/dashboard\/([^/]+)/);
|
||||
if (match && match[1] && match[1] !== lastId) {
|
||||
setLastIdState(match[1]);
|
||||
try {
|
||||
localStorage.setItem(STORAGE_KEY, match[1]);
|
||||
} catch {}
|
||||
(window as any)[WINDOW_KEY] = match[1];
|
||||
}
|
||||
}, [location.pathname, lastId]);
|
||||
|
||||
const setLastId = useCallback((id: string | null) => {
|
||||
setLastIdState(id);
|
||||
try {
|
||||
if (id) {
|
||||
localStorage.setItem(STORAGE_KEY, id);
|
||||
} else {
|
||||
localStorage.removeItem(STORAGE_KEY);
|
||||
}
|
||||
} catch {}
|
||||
if (id) {
|
||||
(window as any)[WINDOW_KEY] = id;
|
||||
} else {
|
||||
delete (window as any)[WINDOW_KEY];
|
||||
}
|
||||
}, []);
|
||||
|
||||
return [lastId, setLastId];
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user