[eric] browser: unit and end-to-end loop tests for the new browser behaviors

backend/tests/test_browser_agent_loop.py

@@ -0,0 +1,184 @@
+"""End-to-end integration test of the real browser agent loop.
+
+Drives run_browser_agent() with only the two external boundaries faked: the LLM
+client (scripted tool calls) and the browser executor (scripted results). Proves
+the four ported behaviors fire together in the actual loop, not just in isolation:
+  - goal threading into BrowserListInteractives,
+  - deterministic stagnation nudges,
+  - exactly-once aux-LLM adjudication at exhaustion,
+  - per-domain hints written, then seeded into the system prompt next run.
+"""
+
+import asyncio
+import json
+import uuid
+
+from backend.apps.agents.browser import browser_agent as BA
+from backend.apps.agents.browser import browser_history as BH
+
+
+# --- fake Anthropic-shaped objects -----------------------------------------
+class Blk:
+    def __init__(self, type, text=None, id=None, name=None, input=None):
+        self.type = type; self.text = text; self.id = id; self.name = name; self.input = input
+
+
+class Resp:
+    def __init__(self, content, stop_reason="tool_use"):
+        self.content = content
+        self.stop_reason = stop_reason
+        self.usage = type("U", (), {"input_tokens": 1, "output_tokens": 1})()
+
+
+class FakeLLM:
+    def __init__(self, scripted):
+        self.scripted = scripted; self.turn = 0; self.calls = []
+        self.messages = self
+
+    async def create(self, **kw):
+        self.calls.append(kw)
+        i = min(self.turn, len(self.scripted) - 1)
+        self.turn += 1
+        return self.scripted[i]
+
+
+class FakeAux:
+    def __init__(self):
+        self.calls = []
+        self.messages = self
+
+    async def create(self, **kw):
+        self.calls.append(kw)
+        return Resp([Blk("text", "Try BrowserListInteractives then BrowserClickIndex.")], stop_reason="end_turn")
+
+
+def _tu(name, **inp):
+    return Blk("tool_use", id="t" + uuid.uuid4().hex[:8], name=name, input=inp)
+
+
+def _rp(goal, mem="Share dialog is a cross-origin iframe; use the index list."):
+    return _tu("ReportProgress", evaluation_previous="prev", working_memory=mem, next_goal=goal)
+
+
+DOC_URL = "https://docs.google.com/document/d/abc/edit"
+
+
+def _install(monkeypatch, primary, aux):
+    # local imports inside run_browser_agent resolve from these source modules
+    import backend.apps.settings.settings as settings_mod
+    import backend.apps.settings.credentials as cred_mod
+    import backend.apps.agents.providers.registry as reg_mod
+    import backend.apps.agents.agent_manager as am_mod
+
+    monkeypatch.setattr(settings_mod, "load_settings", lambda: {"fake": True}, raising=True)
+    monkeypatch.setattr(reg_mod, "_find_builtin_model", lambda m: object(), raising=True)
+    monkeypatch.setattr(reg_mod, "resolve_model_id_for_sdk", lambda m, s: "primary-x", raising=True)
+
+    async def _aux_resolve(s, preferred_tier="haiku"):
+        return ("aux-x", None)
+    monkeypatch.setattr(reg_mod, "resolve_aux_model", _aux_resolve, raising=True)
+
+    def _client_for(s, model):
+        return aux if model == "aux-x" else primary
+    monkeypatch.setattr(cred_mod, "get_anthropic_client_for_model", _client_for, raising=True)
+
+    monkeypatch.setattr(BA, "load_builtin_permissions", lambda: {}, raising=True)
+    monkeypatch.setattr(am_mod.agent_manager, "_sync_session_close", lambda *a, **k: None, raising=False)
+
+    # fake WS: record browser commands, script results by action
+    sent = []
+
+    async def _send_browser_command(request_id, action, browser_id, params, tab_id=""):
+        sent.append({"action": action, "params": params})
+        if action == "list_interactives":
+            return {"text": '1 interactive elements:\n[1]<button "Submit">', "url": DOC_URL}
+        if action == "click":
+            return {"error": "Element not found: '.submit'"}
+        if action == "navigate":
+            return {"text": "Navigated", "url": params.get("url", DOC_URL)}
+        if action == "screenshot":
+            return {"text": "shot"}
+        return {"text": "ok", "url": DOC_URL}
+
+    async def _noop(*a, **k):
+        return None
+
+    monkeypatch.setattr(BA.ws_manager, "send_browser_command", _send_browser_command, raising=False)
+    monkeypatch.setattr(BA.ws_manager, "send_to_session", _noop, raising=False)
+    return sent
+
+
+def test_full_loop_goal_stagnation_adjudication_and_hint_write(monkeypatch):
+    BH._browser_history.clear(); BH._domain_notes.clear()
+    primary = FakeLLM([
+        Resp([_rp("click the Submit button"), _tu("BrowserListInteractives")]),
+        Resp([_rp("click submit"), _tu("BrowserClick", selector=".s1")]),
+        Resp([_rp("retry"), _tu("BrowserClick", selector=".s2")]),
+        Resp([_rp("retry"), _tu("BrowserClick", selector=".s3")]),
+        Resp([_rp("retry"), _tu("BrowserClick", selector=".s4")]),
+        Resp([_rp("retry"), _tu("BrowserClick", selector=".s5")]),
+        Resp([Blk("text", "Giving up cleanly.")], stop_reason="end_turn"),
+    ])
+    aux = FakeAux()
+    sent = _install(monkeypatch, primary, aux)
+
+    result = asyncio.run(BA.run_browser_agent(
+        task="Share the doc with someone", browser_id="b1", model="sonnet",
+    ))
+    assert result["browser_id"] == "b1"
+
+    # 1) goal threaded into the real list_interactives call
+    list_calls = [c for c in sent if c["action"] == "list_interactives"]
+    assert list_calls and list_calls[0]["params"].get("goal") == "click the Submit button"
+
+    # 2) stagnation nudge injected into a tool_result (seen by a later LLM turn)
+    all_msgs = json.dumps([c["messages"] for c in primary.calls])
+    assert "NO PROGRESS" in all_msgs
+
+    # 3) aux adjudication fired EXACTLY once, at exhaustion, and was injected
+    assert len(aux.calls) == 1
+    assert "Suggested next step" in all_msgs
+
+    # 4) per-domain hint written from working_memory
+    assert "cross-origin iframe" in BH.get_domain_note("google.com")
+
+
+def test_aux_adjudication_fires_even_when_loop_detector_trips(monkeypatch):
+    # Repeated IDENTICAL failing clicks trip the exact-repeat loop detector AND
+    # reach stagnation exhaustion on the same turn. The aux escape hatch must
+    # still fire (it was previously suppressed by the `not is_loop` guard).
+    BH._browser_history.clear(); BH._domain_notes.clear()
+    primary = FakeLLM([
+        Resp([_rp("click submit"), _tu("BrowserListInteractives")]),
+        *[Resp([_rp("retry same"), _tu("BrowserClick", selector=".same")]) for _ in range(6)],
+        Resp([Blk("text", "done")], stop_reason="end_turn"),
+    ])
+    aux = FakeAux()
+    sent = _install(monkeypatch, primary, aux)
+
+    asyncio.run(BA.run_browser_agent(
+        task="Share the doc", browser_id="b3", model="sonnet",
+    ))
+    all_msgs = json.dumps([c["messages"] for c in primary.calls])
+    # the loop detector definitely tripped (identical tool+input+result)
+    assert "LOOP DETECTED" in all_msgs
+    # ...and the aux adjudication STILL fired exactly once despite that
+    assert len(aux.calls) == 1
+    assert "Suggested next step" in all_msgs
+
+
+def test_prior_domain_hint_is_seeded_into_system_prompt(monkeypatch):
+    BH._browser_history.clear(); BH._domain_notes.clear()
+    BH.set_domain_note("google.com", "REMEMBERED: Share button is index 43; Tab into the dialog.")
+    primary = FakeLLM([Resp([Blk("text", "done")], stop_reason="end_turn")])
+    aux = FakeAux()
+    _install(monkeypatch, primary, aux)
+
+    asyncio.run(BA.run_browser_agent(
+        task="open the doc", browser_id="b2", model="sonnet", initial_url=DOC_URL,
+    ))
+    assert primary.calls, "LLM should have been called"
+    system = primary.calls[0]["system"]
+    assert "Notes from a previous visit" in system
+    assert "REMEMBERED: Share button is index 43" in system
+    assert len(aux.calls) == 0  # no exhaustion, no adjudication on a clean run

backend/tests/test_browser_hints.py

@@ -0,0 +1,36 @@
+"""Per-domain advisory hint store for the browser sub-agent."""
+
+from backend.apps.agents.browser import browser_history as bh
+
+
+def setup_function(_):
+    bh._domain_notes.clear()
+
+
+def test_set_get_roundtrip():
+    bh.set_domain_note("notion.so", "Share button is top-right; Tab into the dialog.")
+    assert "Share button" in bh.get_domain_note("notion.so")
+
+
+def test_caps_length():
+    bh.set_domain_note("x.com", "a" * 5000)
+    assert len(bh.get_domain_note("x.com")) == bh._MAX_DOMAIN_NOTE_CHARS
+
+
+def test_ignores_empty_domain_or_note():
+    bh.set_domain_note("", "note")
+    bh.set_domain_note("y.com", "")
+    bh.set_domain_note("z.com", "   ")
+    assert bh.get_domain_note("") == ""
+    assert bh.get_domain_note("y.com") == ""
+    assert bh.get_domain_note("z.com") == ""
+
+
+def test_unknown_domain_returns_empty():
+    assert bh.get_domain_note("never.seen") == ""
+
+
+def test_overwrite_keeps_latest():
+    bh.set_domain_note("docs.google.com", "first note")
+    bh.set_domain_note("docs.google.com", "second note")
+    assert bh.get_domain_note("docs.google.com") == "second note"

backend/tests/test_browser_stagnation.py

@@ -0,0 +1,101 @@
+"""Deterministic stagnation detection for the browser sub-agent."""
+
+from backend.apps.agents.browser.browser_loop import (
+    _STAGNATION_ESCALATION_AT,
+    _STAGNATION_MAX,
+    _looks_like_failure,
+    advance_stagnation,
+    is_unproductive,
+    stagnation_exhausted,
+    stagnation_nudge,
+)
+
+
+def _fail(url="https://a.com"):
+    return {"text": "Element not found: '.x'", "url": url}
+
+
+def test_looks_like_failure_positive():
+    assert _looks_like_failure("Element not found: '.foo'")
+    assert _looks_like_failure("Index 4 is no longer valid")
+    assert _looks_like_failure("Error: something broke")
+
+
+def test_looks_like_failure_negative():
+    assert not _looks_like_failure("Clicked element: button#submit")
+    assert not _looks_like_failure("Typed into: input#email")
+
+
+def test_error_result_is_unproductive():
+    assert is_unproductive("BrowserClick", {"error": "boom"}, "", "")
+
+
+def test_failure_text_is_unproductive():
+    r = {"text": "Element not found: '.x'", "url": "https://a.com"}
+    assert is_unproductive("BrowserClick", r, "https://a.com", "prev")
+
+
+def test_url_change_is_productive():
+    r = {"text": "Element not found", "url": "https://b.com"}
+    # even a failure-shaped message counts as progress if the URL moved
+    assert not is_unproductive("BrowserClick", r, "https://a.com", "prev")
+
+
+def test_success_without_url_change_gets_benefit_of_doubt():
+    r = {"text": "Clicked element: button#menu", "url": "https://a.com"}
+    assert not is_unproductive("BrowserClickIndex", r, "https://a.com", "prev")
+
+
+def test_identical_observation_is_unproductive():
+    r = {"text": "same observation", "url": "https://a.com"}
+    assert is_unproductive("BrowserScroll", r, "https://a.com", "same observation")
+
+
+def test_neutral_read_tools_never_count():
+    r = {"error": "whatever"}
+    assert not is_unproductive("BrowserScreenshot", r, "", "")
+    assert not is_unproductive("BrowserGetText", r, "", "")
+    assert not is_unproductive("BrowserListInteractives", r, "", "")
+
+
+def test_nudge_mentions_human_intervention_only_at_max():
+    assert "RequestHumanIntervention" not in stagnation_nudge(3)
+    assert "RequestHumanIntervention" in stagnation_nudge(_STAGNATION_MAX)
+    assert "ladder" in stagnation_nudge(3)
+
+
+def test_advance_increments_on_failures_and_nudges_at_threshold():
+    streak, url, text, nudge = 0, "", "", None
+    nudges = []
+    for _ in range(_STAGNATION_ESCALATION_AT):
+        streak, url, text, nudge = advance_stagnation(streak, url, text, "BrowserClick", _fail())
+        nudges.append(nudge)
+    assert streak == _STAGNATION_ESCALATION_AT
+    assert nudges[-1] is not None  # nudge fires exactly when the threshold is hit
+    assert nudges[0] is None and nudges[1] is None
+
+
+def test_advance_resets_on_progress():
+    # two failures, then a navigation (URL change) clears the streak
+    streak, url, text, _ = advance_stagnation(0, "", "", "BrowserClick", _fail("https://a.com"))
+    streak, url, text, _ = advance_stagnation(streak, url, text, "BrowserClick", _fail("https://a.com"))
+    assert streak == 2
+    streak, url, text, _ = advance_stagnation(
+        streak, url, text, "BrowserNavigate", {"text": "Navigated", "url": "https://b.com"},
+    )
+    assert streak == 0
+
+
+def test_advance_neutral_tools_pass_through_unchanged():
+    streak, url, text, nudge = advance_stagnation(
+        2, "https://a.com", "prev", "BrowserScreenshot", {"image": "..."},
+    )
+    assert (streak, url, text, nudge) == (2, "https://a.com", "prev", None)
+
+
+def test_advance_fires_again_at_max():
+    streak, url, text = _STAGNATION_MAX - 1, "https://a.com", "prev different"
+    streak, url, text, nudge = advance_stagnation(streak, url, text, "BrowserClick", _fail())
+    assert streak == _STAGNATION_MAX
+    assert nudge is not None and "RequestHumanIntervention" in nudge
+    assert stagnation_exhausted(streak)

backend/tests/test_browser_validator.py

@@ -0,0 +1,64 @@
+"""Aux-LLM stuck-adjudication for the browser sub-agent."""
+
+import asyncio
+
+from backend.apps.agents.browser.browser_validator import adjudicate_stuck, _extract_text
+
+
+class _Block:
+    def __init__(self, type_, text=""):
+        self.type = type_
+        self.text = text
+
+
+class _Resp:
+    def __init__(self, blocks):
+        self.content = blocks
+
+
+class _FakeClient:
+    """Minimal Anthropic-shaped client: client.messages.create(...)."""
+
+    def __init__(self, resp=None, raise_exc=None):
+        self._resp = resp
+        self._raise = raise_exc
+        self.calls = []
+        self.messages = self
+
+    async def create(self, **kwargs):
+        self.calls.append(kwargs)
+        if self._raise:
+            raise self._raise
+        return self._resp
+
+
+def test_returns_extracted_guidance_and_assembles_prompt():
+    fc = _FakeClient(resp=_Resp([_Block("text", "Press Tab then Enter to focus the field.")]))
+    out = asyncio.run(adjudicate_stuck(fc, "cheap-model", "share the doc", "- click -> not found", "the page"))
+    assert out == "Press Tab then Enter to focus the field."
+    call = fc.calls[0]
+    assert call["model"] == "cheap-model"
+    assert call["max_tokens"] == 300
+    prompt = call["messages"][0]["content"]
+    assert "share the doc" in prompt
+    assert "not found" in prompt
+
+
+def test_swallows_provider_error_and_returns_empty():
+    fc = _FakeClient(raise_exc=RuntimeError("429 rate limited"))
+    out = asyncio.run(adjudicate_stuck(fc, "m", "g", "r", "p"))
+    assert out == ""
+
+
+def test_extract_text_joins_text_blocks_and_ignores_others():
+    resp = _Resp([_Block("text", "First."), _Block("tool_use"), _Block("text", "Second.")])
+    assert _extract_text(resp) == "First. Second."
+
+
+def test_handles_empty_inputs_without_crashing():
+    fc = _FakeClient(resp=_Resp([_Block("text", "ok")]))
+    out = asyncio.run(adjudicate_stuck(fc, "m", "", "", ""))
+    assert out == "ok"
+    # placeholders keep the prompt well-formed
+    prompt = fc.calls[0]["messages"][0]["content"]
+    assert "(unknown)" in prompt and "(none)" in prompt and "(empty)" in prompt