From 9e11f128d3d0e19d0a03cb70dc4f89bfec1d9563 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Thu, 23 Jul 2026 17:10:13 -0700 Subject: [PATCH] [eric] browser: login-once handoff (pause on wall, sign in once, remember the site) --- backend/apps/agents/browser/browser_agent.py | 29 ++++++ .../agents/browser/browser_login_handoff.py | 93 +++++++++++++++++++ backend/tests/test_browser_agent_loop.py | 44 +++++++++ backend/tests/test_browser_login_handoff.py | 67 +++++++++++++ 4 files changed, 233 insertions(+) create mode 100644 backend/apps/agents/browser/browser_login_handoff.py create mode 100644 backend/tests/test_browser_login_handoff.py diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index 3f1b4f0f..59bf7ac0 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -64,6 +64,7 @@ from backend.apps.agents.browser import browser_extract from backend.apps.agents.browser import browser_metrics from backend.apps.agents.browser import browser_send_script from backend.apps.agents.browser import browser_send_parse +from backend.apps.agents.browser import browser_login_handoff from backend.apps.agents.browser import browser_delivery_check from backend.apps.agents.browser import browser_submit_click from backend.apps.agents.browser import browser_playbook @@ -1115,6 +1116,7 @@ async def run_browser_agent( loop_trigger_count = 0 card_gone_streak = 0 # consecutive "card is gone" results -> fail fast, don't spin route_hinted_hosts: set[str] = set() # surface the fast network tier once per host + p_login_prompted: set[str] = set() # login-once handoff: at most one sign-in pause per domain per run # Stagnation state: busy-but-stuck detection (no URL change + failures across a run of actions), distinct from the exact-repeat loop above. stagnation_streak = 0 @@ -1652,6 +1654,33 @@ async def run_browser_agent( if done_called or cancel_event.is_set(): break + # Login-once handoff: if we've landed on a login wall, pause so the user can sign in ONCE + # in this card (the persistent partition keeps the session, so future runs won't ask + # again), then continue. At most one pause per domain per run; the model's own + # RequestHumanIntervention stays as the fallback for walls this detector misses. + p_wall_dom = browser_login_handoff.login_wall_domain( + last_seen_url, "\n".join(attached_state_seen)) + if p_wall_dom and p_wall_dom not in p_login_prompted: + p_login_prompted.add(p_wall_dom) + p_login_problem, p_login_instruction = browser_login_handoff.prompt_copy(p_wall_dom) + p_login_decision = await p_request_browser_approval( + session, "RequestHumanIntervention", + {"problem": p_login_problem, "instruction": p_login_instruction}) + if cancel_event.is_set(): + break + if p_login_decision.get("behavior") != "deny": + browser_login_handoff.record_login(p_wall_dom) + p_signed_note = (f"You are now signed in to {p_wall_dom}. The page has changed; " + "look at it fresh and continue the task.") + if messages and messages[-1].get("role") == "user": + p_prev = messages[-1]["content"] + if isinstance(p_prev, list): + p_prev.append({"type": "text", "text": p_signed_note}) + else: + messages[-1]["content"] = f"{p_prev}\n\n{p_signed_note}" + else: + messages.append({"role": "user", "content": p_signed_note}) + # Drop stale screenshots before each call: keep first + previous + current, stub the rest. Images are ~1.3-2k tokens each and get re-read every turn, so this is the biggest per-turn context win on any visual task (measured ~2.9x fewer image tokens, ~5x less upload). browser_history.prune_old_screenshots(messages) browser_history.prune_stale_page_state(messages) diff --git a/backend/apps/agents/browser/browser_login_handoff.py b/backend/apps/agents/browser/browser_login_handoff.py new file mode 100644 index 00000000..3217cfcb --- /dev/null +++ b/backend/apps/agents/browser/browser_login_handoff.py @@ -0,0 +1,93 @@ +"""Login-once handoff: when the browser agent lands on a login wall, it pauses for the user to +sign in ONCE in the app's browser card, then continues, and we REMEMBER which sites the user has +authenticated so future runs skip the prompt and only re-ask on a genuine expiry or a different +account. The session itself lives in Electron's persist:openswarm-browser partition (which keeps +it across quits, so "sign in once, never again" is really the partition's doing); this module is +the durable memory of it plus the detection and the wording, keyed by registrable domain. + +Detection reuses the one structural login-wall definition in browser_send_parse, so the pause and +the send-script's decline can never disagree about what a login wall is. +""" + +import datetime +import os +from typing import Dict, List, Optional, Tuple +from urllib.parse import urlparse + +from typeguard import typechecked + +from backend.apps.agents.browser import browser_send_parse +from backend.config.json_store import atomic_write_json, read_json_or_none +from backend.config.paths import SETTINGS_DIR + +P_STORE_PATH = os.path.join(SETTINGS_DIR, "authenticated_domains.json") + + +@typechecked +def registrable_domain(url_or_host: str) -> str: + s = (url_or_host or "").strip() + host = urlparse(s).hostname if "://" in s else s.split("/")[0] + host = (host or "").lower().strip().lstrip(".").split(":")[0] + if host.startswith("www."): + host = host[4:] + return host + + +@typechecked +def p_load() -> Dict[str, Dict[str, str]]: + data = read_json_or_none(P_STORE_PATH) + return data if isinstance(data, dict) else {} + + +@typechecked +def is_authenticated(url_or_host: str) -> bool: + return registrable_domain(url_or_host) in p_load() + + +@typechecked +def authenticated_domains() -> List[str]: + return sorted(p_load().keys()) + + +@typechecked +def login_record(url_or_host: str) -> Optional[Dict[str, str]]: + """The stored {first_seen, last_login} for a site, or None. For a future 'signed-in sites' view.""" + return p_load().get(registrable_domain(url_or_host)) + + +@typechecked +def record_login(url_or_host: str) -> None: + """Remember that the user signed into this site, so future walls read as re-auth not first-run. + Fail-open: a write error just means the next run treats it as a fresh sign-in (harmless).""" + d = registrable_domain(url_or_host) + if not d: + return + store = p_load() + now = datetime.datetime.now(datetime.timezone.utc).isoformat() + prior = store.get(d) or {} + store[d] = {"first_seen": prior.get("first_seen") or now, "last_login": now} + try: + atomic_write_json(P_STORE_PATH, store) + except OSError: + pass + + +@typechecked +def login_wall_domain(current_url: str, state_text: str) -> Optional[str]: + """The registrable domain of a login wall the agent is stuck on, or None. One definition of + 'login wall', shared with the send-script's decline gate.""" + if not browser_send_parse.looks_like_login_wall(current_url or "", state_text or ""): + return None + return registrable_domain(current_url) or None + + +@typechecked +def prompt_copy(domain: str) -> Tuple[str, str]: + """(problem, instruction) for the pause overlay, worded by whether the user has signed into + this site before (re-auth) or it's a first sign-in.""" + if is_authenticated(domain): + problem = f"Your {domain} sign-in looks signed out, it may have expired or be a different account." + else: + problem = f"{domain} needs you to sign in before I can keep going." + instruction = "Log in to the site in the browser above, then click Done and I'll pick up right where I left off." + return problem, instruction diff --git a/backend/tests/test_browser_agent_loop.py b/backend/tests/test_browser_agent_loop.py index 39d65906..4e83f391 100644 --- a/backend/tests/test_browser_agent_loop.py +++ b/backend/tests/test_browser_agent_loop.py @@ -1765,3 +1765,47 @@ def test_autosend_finishes_the_send_after_the_model_fills(monkeypatch): assert "sent" in str(result.get("summary", "")).lower() # the model was called ONCE (the fill turn); autosend ended the run, no second send turn assert len(primary.calls) == 1, f"model called {len(primary.calls)}x; the send should cost zero model turns" + + +def test_login_wall_pauses_and_remembers_the_site(monkeypatch, tmp_path): + """Landing on a login wall auto-fires the RequestHumanIntervention pause with sign-in wording, + and once the user resolves it (Done), the domain is remembered so future runs skip re-prompting.""" + from backend.apps.agents.browser import browser_login_handoff as H + monkeypatch.setattr(H, "P_STORE_PATH", str(tmp_path / "auth.json")) + + approvals = [] + + async def p_fake_approval(session, tool_name, tool_input): + approvals.append((tool_name, tool_input)) + return {"behavior": "allow"} + monkeypatch.setattr(BA, "p_request_browser_approval", p_fake_approval) + + primary = FakeLLM([ + Resp([p_rp("open the login page"), p_tu("BrowserNavigate", url="https://acme.example/login")]), + Resp([p_tu("Done", message="all set")]), + ]) + p_install(monkeypatch, primary, FakeAux()) + p_run_settled(task="log into acme and open my dashboard", browser_id="b1", model="sonnet") + + assert any(t == "RequestHumanIntervention" and "sign in" in ti["problem"].lower() + for t, ti in approvals), approvals + assert H.is_authenticated("acme.example") + + +def test_login_wall_skip_does_not_remember(monkeypatch, tmp_path): + """Skipping the sign-in (deny) leaves the site UNremembered and lets the run continue.""" + from backend.apps.agents.browser import browser_login_handoff as H + monkeypatch.setattr(H, "P_STORE_PATH", str(tmp_path / "auth.json")) + + async def p_deny(session, tool_name, tool_input): + return {"behavior": "deny", "message": "Skipped by user"} + monkeypatch.setattr(BA, "p_request_browser_approval", p_deny) + + primary = FakeLLM([ + Resp([p_rp("open the login page"), p_tu("BrowserNavigate", url="https://acme.example/login")]), + Resp([p_tu("Done", message="ok")]), + ]) + p_install(monkeypatch, primary, FakeAux()) + p_run_settled(task="log into acme", browser_id="b1", model="sonnet") + + assert not H.is_authenticated("acme.example") diff --git a/backend/tests/test_browser_login_handoff.py b/backend/tests/test_browser_login_handoff.py new file mode 100644 index 00000000..c531e8dd --- /dev/null +++ b/backend/tests/test_browser_login_handoff.py @@ -0,0 +1,67 @@ +"""Login-once handoff: registrable-domain keying, login-wall detection reuse, and the durable +authenticated-domains memory that keeps future runs from re-prompting.""" +import os + +import pytest + +from backend.apps.agents.browser import browser_login_handoff as h + + +@pytest.fixture(autouse=True) +def temp_store(tmp_path, monkeypatch): + monkeypatch.setattr(h, "P_STORE_PATH", os.path.join(str(tmp_path), "authenticated_domains.json")) + + +def test_registrable_domain_normalizes(): + assert h.registrable_domain("https://www.x.com/i/flow/login") == "x.com" + assert h.registrable_domain("https://mail.google.com/mail/u/0") == "mail.google.com" + assert h.registrable_domain("reddit.com") == "reddit.com" + assert h.registrable_domain("https://X.COM:443/home") == "x.com" + assert h.registrable_domain("") == "" + + +def test_login_wall_domain_reuses_the_one_detector(): + # a login URL is a wall + assert h.login_wall_domain("https://x.com/i/flow/login", "") == "x.com" + # a password field in the perception is a wall even off a login URL + assert h.login_wall_domain("https://acme.example/app", '[3]') == "acme.example" + # a normal page is not a wall + assert h.login_wall_domain("https://x.com/home", '[1]