[eric] browser: per-domain hints, deterministic stagnation, and aux-LLM fallback

backend/apps/agents/browser/browser_agent.py

@@ -29,7 +29,10 @@ from backend.apps.agents.browser.browser_loop import (
     _LOOP_WINDOW_SIZE,
     _detect_loop,
     _hash_tool_call,
+    advance_stagnation,
+    stagnation_exhausted,
 )
+from backend.apps.agents.browser.browser_validator import adjudicate_stuck
 from backend.apps.agents.browser.browser_schema import (
     _ACTION_TOOLS_REQUIRING_REPORT,
     ACTION_MAP,
@@ -271,6 +274,49 @@ async def run_browser_agent(
     recent_tool_calls: list[tuple[str, str, str]] = []
     loop_trigger_count = 0
 
+    # Stagnation state: busy-but-stuck detection (no URL change + failures
+    # across a run of actions), distinct from the exact-repeat loop above.
+    stagnation_streak = 0
+    stagnation_prev_url = ""
+    stagnation_prev_text = ""
+    aux_adjudicated = False  # the one-shot stuck-adjudication fires at most once per run
+
+    # Lazily-resolved cheap aux client, used only for the rare stuck-adjudication
+    # call once deterministic nudging is exhausted. Provider-agnostic.
+    _aux_state = {"resolved": False, "client": None, "model": None}
+
+    async def _get_aux_client():
+        if not _aux_state["resolved"]:
+            _aux_state["resolved"] = True
+            try:
+                aux_model, _ = await resolve_aux_model(browser_settings, preferred_tier="haiku")
+                _aux_state["model"] = aux_model
+                _aux_state["client"] = get_anthropic_client_for_model(browser_settings, aux_model)
+            except Exception as e:
+                logger.warning(f"[browser-agent {session_id}] no aux model for adjudication: {e}")
+        return _aux_state["client"], _aux_state["model"]
+
+    # Latest goal from ReportProgress; threaded into BrowserListInteractives so
+    # the frontend floats goal-matching elements to the top of the list. Seeded
+    # with the task so the first listing (before any ReportProgress) is boosted.
+    current_next_goal = task
+
+    # Advisory per-domain hints: seed the system prompt with what a prior agent
+    # learned about this domain (if we know the domain at start), and keep the
+    # store fresh from each ReportProgress. Re-verify, never blindly trust.
+    start_domain = _extract_domain(initial_url) if initial_url else None
+    run_system_prompt = SYSTEM_PROMPT
+    if start_domain:
+        prior_note = browser_history.get_domain_note(start_domain)
+        if prior_note:
+            run_system_prompt = (
+                SYSTEM_PROMPT
+                + f"\n\n## Notes from a previous visit to {start_domain}\n"
+                + "Learned last time on this site. Use it as a head start, but "
+                + "re-verify since the page may have changed:\n"
+                + prior_note
+            )
+
     user_msg = Message(role="user", content=task)
     session.messages.append(user_msg)
     await ws_manager.send_to_session(session_id, "agent:message", {
@@ -309,7 +355,7 @@ async def run_browser_agent(
             response = await _cancellable(client.messages.create(
                 model=api_model,
                 max_tokens=4096,
-                system=SYSTEM_PROMPT,
+                system=run_system_prompt,
                 tools=BROWSER_TOOLS_SCHEMA,
                 messages=messages,
             ))
@@ -434,6 +480,21 @@ async def run_browser_agent(
                     eval_prev = tu.input.get("evaluation_previous", "")
                     working_mem = tu.input.get("working_memory", "")
                     next_goal = tu.input.get("next_goal", "")
+                    if next_goal:
+                        current_next_goal = next_goal
+                    # Distill the agent's own working memory into a per-domain
+                    # hint for the next visit. Only persist when the run stayed
+                    # on a SINGLE apex domain: working_memory is cumulative, so
+                    # on a multi-domain run it would describe one site but get
+                    # filed under whichever domain happens to be current.
+                    note_domain = (
+                        session.browser_domains[-1]
+                        if session.browser_domains
+                        else start_domain
+                    )
+                    single_domain = len(set(session.browser_domains)) <= 1
+                    if note_domain and working_mem and single_domain:
+                        browser_history.set_domain_note(note_domain, working_mem)
                     brain_text = (
                         f"📋 **Plan**\n"
                         f"_Previous_: {eval_prev}\n"
@@ -563,8 +624,11 @@ async def run_browser_agent(
                         continue
 
                 start = time.time()
+                tool_input = tu.input
+                if tu.name == "BrowserListInteractives" and current_next_goal:
+                    tool_input = {**tu.input, "goal": current_next_goal}
                 result = await _cancellable(execute_browser_tool(
-                    tu.name, tu.input, browser_id, tab_id,
+                    tu.name, tool_input, browser_id, tab_id,
                 ))
                 if result is None:
                     cancelled = True
@@ -612,6 +676,43 @@ async def run_browser_agent(
                     content_blocks = content_blocks + [
                         {"type": "text", "text": f"\n\n⚠️ {warning}"}
                     ]
+
+                # Stagnation: busy-but-stuck (no URL change + failures across a
+                # run of actions), distinct from the exact-repeat loop above.
+                stagnation_streak, stagnation_prev_url, stagnation_prev_text, stag_nudge = advance_stagnation(
+                    stagnation_streak, stagnation_prev_url, stagnation_prev_text, tu.name, result,
+                )
+                # Skip the nudge when the loud loop warning already fired this
+                # turn (avoid double-messaging), but the aux adjudication below
+                # is NOT gated on is_loop: repeated identical failures trip BOTH
+                # detectors, and that's exactly when the escape hatch is needed.
+                if stag_nudge and not is_loop:
+                    logger.warning(
+                        f"[browser-agent {session_id}] stagnation streak "
+                        f"{stagnation_streak} on {tu.name}"
+                    )
+                    content_blocks = content_blocks + [
+                        {"type": "text", "text": f"\n\n⚠️ {stag_nudge}"}
+                    ]
+                # Deterministic nudging exhausted: ONE cheap aux adjudication
+                # to suggest a concrete next step before we keep failing.
+                if stagnation_exhausted(stagnation_streak) and not aux_adjudicated:
+                    aux_adjudicated = True
+                    aux_client, aux_model = await _get_aux_client()
+                    if aux_client and aux_model:
+                        recent = "\n".join(
+                            f"- {a['tool']} -> {str(a.get('result_summary', ''))[:120]}"
+                            for a in action_log[-3:]
+                        )
+                        page_text = str(result.get("text") or result.get("error") or "")
+                        guidance = await _cancellable(adjudicate_stuck(
+                            aux_client, aux_model, current_next_goal, recent, page_text,
+                        ))
+                        if guidance:
+                            content_blocks = content_blocks + [
+                                {"type": "text", "text": f"\n\n💡 Suggested next step: {guidance}"}
+                                ]
+
                 tool_results.append({
                     "type": "tool_result",
                     "tool_use_id": tu.id,

backend/apps/agents/browser/browser_history.py

@@ -15,6 +15,25 @@ _browser_history: dict[str, list[dict]] = {}
 # Cap history to prevent unbounded growth on long-lived browsers.
 _MAX_HISTORY_MESSAGES = 30
 
+# Per-apex-domain advisory notes, distilled from the agent's own ReportProgress
+# working_memory. Process-lifetime only (never written to disk); seeds a later
+# agent on the same domain so it skips re-learning the same quirks. Advisory
+# text only, never auto-executed.
+_domain_notes: dict[str, str] = {}
+_MAX_DOMAIN_NOTE_CHARS = 600
+
+
+def get_domain_note(domain: str) -> str:
+    """Return the advisory note for a domain, or empty string if none."""
+    return _domain_notes.get(domain, "")
+
+
+def set_domain_note(domain: str, note: str) -> None:
+    """Store/overwrite the advisory note for a domain (trimmed + capped)."""
+    if not domain or not note or not note.strip():
+        return
+    _domain_notes[domain] = note.strip()[:_MAX_DOMAIN_NOTE_CHARS]
+
 
 def clear_browser_history(browser_id: str) -> None:
     """Drop cached conversation history for a browser (e.g. when it's closed)."""

backend/apps/agents/browser/browser_loop.py

@@ -72,3 +72,108 @@ _LOOP_WARNING_TEXT = (
     "(3) use BrowserPressKey for keyboard shortcuts if the site supports them, "
     "or (4) call RequestHumanIntervention if you genuinely cannot proceed."
 )
+
+
+# --- Stagnation detection -------------------------------------------------
+# Distinct from the exact-repeat loop above. The agent can be "busy but stuck":
+# trying selector A, then B, then C, all failing. The inputs differ so the
+# exact-repeat detector never fires, yet the page never changes. We watch for a
+# run of state-mutating actions that produced no URL change AND looked like
+# failures (or just repeated the same observation), and nudge the model down
+# the strategy ladder before it burns the whole turn budget.
+
+# Read-only / meta tools don't count toward stagnation (same exemption set as
+# the loop detector): re-orienting is not "being stuck".
+_STAGNATION_NEUTRAL_TOOLS = _LOOP_DETECTION_EXCLUDED_TOOLS
+_STAGNATION_ESCALATION_AT = 3
+_STAGNATION_MAX = 5
+
+_FAILURE_MARKERS = (
+    "error", "not found", "no longer valid", "no box model",
+    "no valid bounding rect", "failed", "rejected", "timed out",
+    "could not", "unable to", "denied",
+)
+
+
+def _looks_like_failure(text: str) -> bool:
+    low = text.lower()
+    return any(m in low for m in _FAILURE_MARKERS)
+
+
+def is_unproductive(
+    tool_name: str, result: dict, prev_url: str, prev_text: str,
+) -> bool:
+    """True if a state-mutating action changed nothing observable.
+
+    Productive (returns False): a URL change, or a success-shaped result, gets
+    the benefit of the doubt (a click that opens a dropdown changes no URL but
+    is real progress). Unproductive (returns True): an error result, a
+    failure-shaped message, or the exact same observation as the previous
+    action, all with no URL change. Neutral tools (screenshot, get_text, etc.)
+    never count.
+    """
+    if tool_name in _STAGNATION_NEUTRAL_TOOLS:
+        return False
+    new_url = str(result.get("url") or "")
+    if new_url and prev_url and new_url != prev_url:
+        return False
+    if "error" in result:
+        return True
+    text = str(result.get("text") or result.get("error") or "")
+    if _looks_like_failure(text):
+        return True
+    if prev_text and text[:200] == prev_text[:200]:
+        return True
+    return False
+
+
+_STAGNATION_NUDGE = (
+    "NO PROGRESS: your last {streak} actions changed nothing on the page and "
+    "looked like failures. STOP repeating this approach. Walk DOWN the strategy "
+    "ladder: switch from CSS clicks to BrowserListInteractives + "
+    "BrowserClickIndex; if that already failed, try BrowserPressKey (Tab/Enter) "
+    "or use BrowserEvaluate to find the element by its visible text; take ONE "
+    "BrowserScreenshot to re-orient if you are unsure what's on screen."
+)
+
+
+def stagnation_nudge(streak: int) -> str:
+    base = _STAGNATION_NUDGE.format(streak=streak)
+    if streak >= _STAGNATION_MAX:
+        base += (
+            " If nothing here works, call RequestHumanIntervention instead of "
+            "continuing to fail."
+        )
+    return base
+
+
+def advance_stagnation(
+    streak: int, prev_url: str, prev_text: str, tool_name: str, result: dict,
+) -> tuple[int, str, str, str | None]:
+    """Advance the stagnation streak for one executed tool.
+
+    Neutral read/meta tools pass through unchanged (no bump, no reset). For a
+    state-mutating action, bump the streak when unproductive else reset it, and
+    return a nudge string when the streak crosses an escalation threshold.
+    Returns (new_streak, new_prev_url, new_prev_text, nudge_or_None).
+    """
+    if tool_name in _STAGNATION_NEUTRAL_TOOLS:
+        return streak, prev_url, prev_text, None
+    if is_unproductive(tool_name, result, prev_url, prev_text):
+        streak += 1
+    else:
+        streak = 0
+    new_url = str(result.get("url") or "") or prev_url
+    new_text = str(result.get("text") or result.get("error") or "")[:200]
+    nudge = (
+        stagnation_nudge(streak)
+        if streak in (_STAGNATION_ESCALATION_AT, _STAGNATION_MAX)
+        else None
+    )
+    return streak, new_url, new_text, nudge
+
+
+def stagnation_exhausted(streak: int) -> bool:
+    """True once deterministic nudging has been exhausted; the caller may then
+    escalate to a one-shot aux-LLM adjudication (see browser_validator)."""
+    return streak >= _STAGNATION_MAX

backend/apps/agents/browser/browser_validator.py

@@ -0,0 +1,57 @@
+"""
+Last-resort adjudication for a stuck browser sub-agent.
+
+Deterministic stagnation detection (browser_loop) handles the common cases for
+free. When it's exhausted (a run of actions with no progress despite escalating
+nudges), we make ONE cheap aux-tier LLM call to suggest a concrete next step.
+It is rare by construction, so the cost stays near zero. The model + client are
+resolved provider-agnostically by the caller (cheap tier of whatever provider
+the user has connected), so nothing here hardcodes Anthropic/Haiku.
+"""
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+_ADJUDICATION_PROMPT = (
+    "A browser automation agent is stuck: its recent actions produced no "
+    "progress on the page.\n\n"
+    "GOAL: {goal}\n\n"
+    "RECENT ACTIONS (most recent last):\n{recent}\n\n"
+    "CURRENT PAGE (truncated):\n{page}\n\n"
+    "In 2 to 3 sentences, give the single most promising concrete next step. "
+    "Prefer, in order: a different element or selector, a keyboard shortcut "
+    "(Tab then Enter), scrolling to reveal a hidden control, or calling "
+    "RequestHumanIntervention if this is a login / captcha / 2FA wall. Be "
+    "specific and brief; do not restate the goal."
+)
+
+
+def _extract_text(response) -> str:
+    """Pull the text out of an Anthropic-style response object."""
+    parts = []
+    for block in getattr(response, "content", None) or []:
+        if getattr(block, "type", None) == "text" and getattr(block, "text", ""):
+            parts.append(block.text.strip())
+    return " ".join(p for p in parts if p).strip()
+
+
+async def adjudicate_stuck(
+    client, model: str, goal: str, recent_actions: str, page_text: str,
+) -> str:
+    """One cheap aux call returning concrete guidance, or "" on any failure."""
+    prompt = _ADJUDICATION_PROMPT.format(
+        goal=(goal or "(unknown)")[:400],
+        recent=(recent_actions or "(none)")[:1200],
+        page=(page_text or "(empty)")[:1500],
+    )
+    try:
+        response = await client.messages.create(
+            model=model,
+            max_tokens=300,
+            messages=[{"role": "user", "content": prompt}],
+        )
+    except Exception as e:
+        logger.warning(f"[browser-validator] adjudication call failed: {e}")
+        return ""
+    return _extract_text(response)