diff --git a/backend/apps/agents/browser_agent.py b/backend/apps/agents/browser_agent.py index 11d38eec..dbbcbe17 100644 --- a/backend/apps/agents/browser_agent.py +++ b/backend/apps/agents/browser_agent.py @@ -27,7 +27,321 @@ MODEL_MAP = { "haiku": "claude-haiku-4-5-20251001", } +# Cache of conversation history per browser_id so successive BrowserAgent +# calls on the same browser can resume rather than restart from scratch. +# Without this every "swipe right" / "swipe left" call has to take a new +# screenshot and re-orient itself, costing 30-60s per action. +_browser_history: dict[str, list[dict]] = {} +# Cap history to prevent unbounded growth on long-lived browsers. +_MAX_HISTORY_MESSAGES = 30 + + +def clear_browser_history(browser_id: str) -> None: + """Drop cached conversation history for a browser (e.g. when it's closed).""" + _browser_history.pop(browser_id, None) + + +# --------------------------------------------------------------------------- +# Loop detection +# +# Tracks recent state-mutating tool calls in a sliding window. If the model +# repeats the same (tool, input) with the same result several times, we +# inject an is_error message in the next tool_result to force a strategy +# change. This prevents the model from burning the entire turn budget on +# a failing approach. +# --------------------------------------------------------------------------- + +# Tools that are read-only / idempotent and should NOT count toward loop +# detection. Repeating these is normal (scrolling through a feed, taking +# successive screenshots, polling for an element to appear). +_LOOP_DETECTION_EXCLUDED_TOOLS = { + "BrowserScreenshot", + "BrowserGetText", + "BrowserGetElements", + "BrowserListInteractives", # Phase 3 + "BrowserWait", + "ReportProgress", # Phase 2 + "RequestHumanIntervention", +} + +_LOOP_WINDOW_SIZE = 5 +_LOOP_REPEAT_THRESHOLD = 3 +_LOOP_HARD_CAP = 5 + + +def _hash_tool_call(tool_name: str, tool_input: dict, result: dict) -> tuple[str, str, str]: + """Build a stable hash key for a tool call, including its result. + + Including the result hash means that legitimate progress (same input, + different output — e.g. BrowserScroll on a long feed) does NOT count + as a loop. Only same-input + same-output is treated as stuck. + """ + try: + input_key = json.dumps(tool_input, sort_keys=True, default=str) + except Exception: + input_key = repr(tool_input) + try: + # Truncate the result hash to avoid huge image blobs in the key + result_key = json.dumps(result, sort_keys=True, default=str)[:300] + except Exception: + result_key = repr(result)[:300] + return (tool_name, input_key, result_key) + + +def _detect_loop( + recent_calls: list[tuple[str, str, str]], + new_call: tuple[str, str, str], +) -> bool: + """Return True if `new_call` constitutes a loop given recent history. + + A loop is when the same (tool, input, result) has appeared at least + `_LOOP_REPEAT_THRESHOLD` times within the last `_LOOP_WINDOW_SIZE` + state-mutating calls (the new call counts as one of those occurrences). + """ + if new_call[0] in _LOOP_DETECTION_EXCLUDED_TOOLS: + return False + window = recent_calls[-(_LOOP_WINDOW_SIZE - 1):] + [new_call] + matches = sum(1 for c in window if c == new_call) + return matches >= _LOOP_REPEAT_THRESHOLD + + +_LOOP_WARNING_TEXT = ( + "LOOP DETECTED: You have called this tool with these exact parameters and " + "gotten the same result {count} times in a row. STOP retrying this approach " + "— it is not working. Try a fundamentally different strategy: " + "(1) check the page state with BrowserScreenshot or BrowserGetText, " + "(2) try a different selector or a different tool, " + "(3) use BrowserPressKey for keyboard shortcuts if the site supports them, " + "or (4) call RequestHumanIntervention if you genuinely cannot proceed." +) + + +def _validate_message_pairing(messages: list[dict]) -> bool: + """Verify every tool_result references a tool_use_id from a prior assistant + message in the same list. Returns False if there's an orphan, which means + the cached history would 400 if sent to the API. + + This is the last line of defense against cache corruption — if it ever + returns False on a resume, we drop the cache and start fresh rather than + crash on the next API call. + """ + declared_tool_use_ids: set[str] = set() + for msg in messages: + role = msg.get("role") + content = msg.get("content") + if role == "assistant" and isinstance(content, list): + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + tu_id = block.get("id") + if tu_id: + declared_tool_use_ids.add(tu_id) + elif role == "user" and isinstance(content, list): + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_result": + tr_id = block.get("tool_use_id") + if tr_id and tr_id not in declared_tool_use_ids: + return False + return True + + +def _is_fresh_user_message(msg: dict) -> bool: + """A 'fresh' user message starts a new turn — string content or a list + that contains no tool_result blocks. These are the only safe cut points + because they don't reference any prior assistant tool_use blocks.""" + if msg.get("role") != "user": + return False + content = msg.get("content") + if isinstance(content, str): + return True + if isinstance(content, list) and not any( + isinstance(c, dict) and c.get("type") == "tool_result" for c in content + ): + return True + return False + + +def _summarize_messages(messages: list[dict]) -> str: + """Build a programmatic summary of older browser-agent messages. + + Extracts the original user task, a count of tool calls by name with their + key parameters, the last few ReportProgress brain states, and the most + recent assistant text. No LLM call required — this is purely structural + extraction from the existing message history. + """ + if not messages: + return "" + + # Find the original user task (first user-text message) + initial_task = "" + for msg in messages: + if msg.get("role") == "user": + content = msg.get("content") + if isinstance(content, str) and content.strip(): + initial_task = content.strip()[:300] + break + + # Count tool calls by name with key params + tool_call_summary: dict[str, list[str]] = {} + brain_states: list[str] = [] + last_assistant_text = "" + + for msg in messages: + if msg.get("role") != "assistant": + continue + content = msg.get("content") + if not isinstance(content, list): + continue + for block in content: + if not isinstance(block, dict): + continue + btype = block.get("type") + if btype == "tool_use": + name = block.get("name", "unknown") + inp = block.get("input") or {} + if name == "ReportProgress": + # Capture the brain state for inline summary + brain_states.append( + f" • {inp.get('next_goal', '')[:120]}" + ) + continue + # Compact one-line description with key params + key_param = "" + for k in ("index", "key", "url", "selector", "direction", "text"): + if k in inp: + v = str(inp[k])[:40] + key_param = f"{k}={v}" + break + desc = f"{name}({key_param})" if key_param else name + tool_call_summary.setdefault(name, []).append(desc) + elif btype == "text": + txt = block.get("text", "").strip() + if txt: + last_assistant_text = txt + + # Build the summary text + parts = ["[Summary of earlier browser-agent activity]"] + if initial_task: + parts.append(f'Original task: "{initial_task}"') + if tool_call_summary: + total = sum(len(v) for v in tool_call_summary.values()) + parts.append(f"Actions taken ({total} total):") + # Show count + a couple of representative examples per tool + for name in sorted(tool_call_summary.keys()): + calls = tool_call_summary[name] + count = len(calls) + sample = calls[-1] # most recent example + if count == 1: + parts.append(f" - {sample}") + else: + parts.append(f" - {sample} (×{count})") + if brain_states: + parts.append("Recent intents:") + parts.extend(brain_states[-5:]) # last 5 brain states + if last_assistant_text: + snippet = last_assistant_text[:400] + parts.append(f"Last update from assistant: {snippet}") + parts.append( + "(Earlier turn-by-turn details have been compacted to keep the " + "context window manageable. Continue from where you left off.)" + ) + return "\n".join(parts) + + +def _trim_history_by_turns(messages: list[dict], max_messages: int) -> list[dict]: + """Compact message history when it exceeds max_messages. + + The Anthropic API requires every `tool_result` block to reference a + `tool_use_id` from a previous assistant message. Naive slicing can drop + a tool_use while keeping its tool_result, causing 400 errors. This + function avoids that by: + + 1. Walking forward to find a clean turn boundary (a fresh user-text + message that starts a new turn — no tool_result content). + 2. Summarizing everything BEFORE that boundary into a single user-text + message and prepending it to the kept tail. + 3. If no clean boundary exists at all, returning the original history + unchanged. Better to temporarily exceed the cap than to corrupt the + conversation and 400 every subsequent request. + + The summary is built programmatically (no LLM call) from the message + structure: original task, tool call counts, recent ReportProgress brain + states, and last assistant text. + """ + if len(messages) <= max_messages: + return list(messages) + + target_tail_size = max_messages - 1 # leave room for the summary message + cut_index: int | None = None + + # First pass: walk forward looking for the EARLIEST clean cut point that + # gets us under the cap. This preserves the most recent detail. + for i in range(1, len(messages)): + if not _is_fresh_user_message(messages[i]): + continue + if len(messages) - i <= target_tail_size: + cut_index = i + break + + # Second pass: if no cut point gets us under the cap (e.g. the current + # turn alone is bigger than max_messages), use the LATEST clean cut point + # available. The tail will still exceed the cap, but it's the smallest + # safe history we can produce — and any compaction is better than none. + if cut_index is None: + for i in range(len(messages) - 1, 0, -1): + if _is_fresh_user_message(messages[i]): + cut_index = i + break + + if cut_index is None: + # No clean cut anywhere in the history. Return original — better to + # exceed the cap than to corrupt the conversation. + return list(messages) + + # Compact: summarize messages[0..cut_index-1], prepend as a single + # user-text message, then keep messages[cut_index..end] verbatim. + summary_text = _summarize_messages(messages[:cut_index]) + summary_msg = {"role": "user", "content": summary_text} + return [summary_msg] + list(messages[cut_index:]) + BROWSER_TOOLS_SCHEMA = [ + { + "name": "ReportProgress", + "description": ( + "Record your assessment of the previous action and your plan for the " + "next one. You MUST call this BEFORE any browser action tools in every " + "turn (after the very first turn). This is how you reflect on what just " + "happened, track what you've learned about this site, and articulate what " + "you're trying to do next. Skipping it is not allowed and will be rejected." + ), + "input_schema": { + "type": "object", + "properties": { + "evaluation_previous": { + "type": "string", + "description": ( + "What did the previous action(s) accomplish? Did they succeed? " + "If not, why? Be specific about what changed on the page." + ), + }, + "working_memory": { + "type": "string", + "description": ( + "Short notes about what you've learned about this site so far — " + "selectors that work, keyboard shortcuts, layout quirks, what " + "you've tried that failed. Carry this forward across turns." + ), + }, + "next_goal": { + "type": "string", + "description": ( + "What you're trying to achieve with the action(s) you're about " + "to take next. Be concrete." + ), + }, + }, + "required": ["evaluation_previous", "working_memory", "next_goal"], + }, + }, { "name": "BrowserScreenshot", "description": ( @@ -137,6 +451,112 @@ BROWSER_TOOLS_SCHEMA = [ "required": [], }, }, + { + "name": "BrowserListInteractives", + "description": ( + "Get a NUMBERED LIST of interactive elements on the page using the " + "browser's accessibility tree. Returns elements like [1]