WEBDIENSTE/openswarm
1
0
Fork 0
mirror of https://github.com/openswarm-ai/openswarm.git synced 2026-09-08 02:37:45 +02:00
Code Issues Packages Projects Releases Wiki Activity

[eric] browser: leading-_ -> p_/P_/public for module functions+constants+state across browser subsystem; promote cross-file symbols public (detect_loop, BROWSER_HISTORY, TRUSTED, etc.), align analyze-browser-metrics.py + tests

Browse Source
This commit is contained in:
ciregenz
2026-06-23 19:37:40 -07:00
parent 394dcd7492
commit 1767389cbb
35 changed files with 775 additions and 775 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/apps/agents/browser/browser_agent.py
+94 -94
View File
@@ -18,20 +18,20 @@ import anthropic
from backend.apps.agents.browser import browser_history
from backend.apps.agents.browser.browser_history import (
_MAX_HISTORY_MESSAGES,
_trim_history_by_turns,
_validate_message_pairing,
MAX_HISTORY_MESSAGES,
trim_history_by_turns,
validate_message_pairing,
clear_browser_history,
PAGE_STATE_MARKER,
)
from backend.apps.agents.browser.browser_loop import (
_LOOP_DETECTION_EXCLUDED_TOOLS,
_LOOP_HARD_CAP,
_LOOP_WARNING_TEXT,
_LOOP_WINDOW_SIZE,
_detect_loop,
_hash_tool_call,
_CARD_GONE_LIMIT,
LOOP_DETECTION_EXCLUDED_TOOLS,
LOOP_HARD_CAP,
LOOP_WARNING_TEXT,
LOOP_WINDOW_SIZE,
detect_loop,
hash_tool_call,
CARD_GONE_LIMIT,
advance_stagnation,
card_is_unavailable,
completion_is_honest,
@@ -45,14 +45,14 @@ from backend.apps.agents.browser.browser_validator import adjudicate_stuck
# Single actions the model could have folded into one BrowserBatch turn;
# reads, waits, and the batch tools themselves don't count toward the streak.
_BATCHABLE_ACTION_TOOLS = {
P_BATCHABLE_ACTION_TOOLS = {
"BrowserNavigate", "BrowserClick", "BrowserClickIndex", "BrowserClickByName",
"BrowserType", "BrowserPressKey", "BrowserScroll",
}
# Injected when the spin backstop trips: one chance to land a real answer from
# what's already gathered, instead of the loop cutting it off mid-thought.
_WRAPUP_NUDGE = (
P_WRAPUP_NUDGE = (
"You've spent several turns looking without finishing. Wrap up NOW: call Done with the "
"best answer you can give from what you've ALREADY gathered. For a find/list ask, put the "
"actual items in the message and, if the site exposes fewer than asked, say so plainly and "
@@ -69,7 +69,7 @@ from backend.apps.agents.browser import browser_skills
from backend.apps.agents.browser import browser_wait
from backend.apps.agents.browser import browser_schema
from backend.apps.agents.browser.browser_schema import (
_ACTION_TOOLS_REQUIRING_REPORT,
ACTION_TOOLS_REQUIRING_REPORT,
ACTION_MAP,
BROWSER_TOOLS_SCHEMA,
MAX_TURNS,
@@ -84,7 +84,7 @@ logger = logging.getLogger(__name__)
# Mutating actions that can carry an `expect` (the change they should cause) and be
# confirmed after running. Reads/waits aren't here, there's nothing to confirm.
_CONFIRM_TOOLS = {
P_CONFIRM_TOOLS = {
"BrowserClick", "BrowserClickIndex", "BrowserClickByName",
"BrowserType", "BrowserNavigate", "BrowserPressKey", "BrowserBatch",
}
@@ -106,7 +106,7 @@ async def execute_browser_tool(
return result
def _extract_domain(url: str) -> str | None:
def p_extract_domain(url: str) -> str | None:
"""Extract the apex domain from a URL (acme-corp.notion.so → notion.so).
Returns None for non-http URLs."""
try:
@@ -123,7 +123,7 @@ def _extract_domain(url: str) -> str | None:
return None
def _strip_lone_surrogates(s: str) -> str:
def strip_lone_surrogates(s: str) -> str:
# The JS/webview hands us page text as UTF-16, so an emoji can arrive as half
# of its surrogate pair; Python carries the orphan but .encode('utf-8') later
# (the SDK serializing the request to the LLM) detonates with "surrogates not
@@ -131,10 +131,10 @@ def _strip_lone_surrogates(s: str) -> str:
return re.sub(r"[\ud800-\udfff]", "�", s) if s else s
def _format_tool_result(result: dict, tool_name: str) -> list[dict]:
def format_tool_result(result: dict, tool_name: str) -> list[dict]:
"""Convert a browser command result dict into Anthropic API content blocks."""
if "error" in result:
return [{"type": "text", "text": _strip_lone_surrogates(f"Error: {result['error']}")}]
return [{"type": "text", "text": strip_lone_surrogates(f"Error: {result['error']}")}]
if tool_name == "BrowserScreenshot" and result.get("image"):
blocks = [
@@ -151,31 +151,31 @@ def _format_tool_result(result: dict, tool_name: str) -> list[dict]:
return blocks
text = result.get("text", json.dumps(result))
return [{"type": "text", "text": _strip_lone_surrogates(str(text))}]
return [{"type": "text", "text": strip_lone_surrogates(str(text))}]
# Mutating tools whose results get fresh page state attached (the browser-use
# loop shape: act, settle, see), so acting and seeing are one turn, not two.
_AUTO_STATE_TOOLS = {
P_AUTO_STATE_TOOLS = {
"BrowserNavigate", "BrowserClick", "BrowserClickIndex", "BrowserClickByName",
"BrowserType", "BrowserPressKey", "BrowserScroll", "BrowserBatch",
}
_AUTO_STATE_MAX_LINES = 35
_AUTO_SETTLE_CAPS_MS = {"BrowserNavigate": 2500, "BrowserBatch": 1500}
P_AUTO_STATE_MAX_LINES = 35
P_AUTO_SETTLE_CAPS_MS = {"BrowserNavigate": 2500, "BrowserBatch": 1500}
# URL shapes that mean "a list of candidates to pick from" (auto candidate scan)
_RESULTS_URL_RE = re.compile(
RESULTS_URL_RE = re.compile(
r"[?&](q|query|keywords|search|search_query|find|term)=|/search\b|/results\b", re.I,
)
_AUTO_SCAN_MAX_PER_RUN = 2
P_AUTO_SCAN_MAX_PER_RUN = 2
def _batch_ends_with_read(tool_input: dict) -> bool:
def p_batch_ends_with_read(tool_input: dict) -> bool:
actions = (tool_input or {}).get("actions") or []
return bool(actions) and (actions[-1] or {}).get("type") == "list_interactives"
def _truncate_state(text: str, max_lines: int = _AUTO_STATE_MAX_LINES) -> str:
def p_truncate_state(text: str, max_lines: int = P_AUTO_STATE_MAX_LINES) -> str:
lines = str(text).splitlines()
if len(lines) <= max_lines:
return str(text)
@@ -184,7 +184,7 @@ def _truncate_state(text: str, max_lines: int = _AUTO_STATE_MAX_LINES) -> str:
)
def _delta_state(text: str, seen_lines: set[str]) -> str:
def delta_state(text: str, seen_lines: set[str]) -> str:
"""Shrink an attached element list to the rows that changed since the last
attach; stable indices make a line's identity meaningful, so re-sending 30
unchanged rows every action is pure token burn. Mutates `seen_lines` to the
@@ -210,20 +210,20 @@ def _delta_state(text: str, seen_lines: set[str]) -> str:
# A button row whose name is exactly a Send control (not "Send InMail credit" or
# "Send a message to X"); used to hand the model the Send button after it types,
# so it never burns turns hunting a button that's right there.
_SEND_ROW_RE = re.compile(r'\[(\d+)\]\*?<\s*button\s+"([^"]*)"', re.I)
P_SEND_ROW_RE = re.compile(r'\[(\d+)\]\*?<\s*button\s+"([^"]*)"', re.I)
def _send_index_in_state(state_text: str):
def send_index_in_state(state_text: str):
"""(index, name) of a real Send button in an interactives list, or None.
Strict exact match so it never grabs an upsell or a profile 'Send a message' link."""
for line in (state_text or "").splitlines():
m = _SEND_ROW_RE.search(line)
m = P_SEND_ROW_RE.search(line)
if m and m.group(2).strip().lower() in ("send", "send now", "send message"):
return int(m.group(1)), m.group(2)
return None
def _is_composer_fill(tool_name: str, tool_input: dict) -> bool:
def is_composer_fill(tool_name: str, tool_input: dict) -> bool:
"""True if this action typed a message into a composer (the moment the Send
button is about to matter). Covers the solo fill, BrowserType, and a batched
fill, the three ways the model composes."""
@@ -238,25 +238,25 @@ def _is_composer_fill(tool_name: str, tool_input: dict) -> bool:
return False
async def _post_action_state(
async def post_action_state(
tool_name: str, tool_input: dict, result: dict,
browser_id: str, tab_id: str, wait_exec, goal: str,
seen_lines: set[str] | None = None,
) -> str:
"""Settle the page after a mutating action, then return a compact fresh
interactives list to append to its result. Empty string = attach nothing."""
if tool_name not in _AUTO_STATE_TOOLS or not isinstance(result, dict) or "error" in result:
if tool_name not in P_AUTO_STATE_TOOLS or not isinstance(result, dict) or "error" in result:
return ""
if tool_name == "BrowserBatch" and _batch_ends_with_read(tool_input):
if tool_name == "BrowserBatch" and p_batch_ends_with_read(tool_input):
return ""
# an `expect` confirm already ran its own smart_wait; don't settle twice
if not str((tool_input or {}).get("expect") or "").strip():
settle = await browser_wait.smart_wait(
wait_exec, browser_id, tab_id, _AUTO_SETTLE_CAPS_MS.get(tool_name, 1200),
wait_exec, browser_id, tab_id, P_AUTO_SETTLE_CAPS_MS.get(tool_name, 1200),
)
if settle.get("hung"):
return ""
_composer_fill = _is_composer_fill(tool_name, tool_input)
_composer_fill = is_composer_fill(tool_name, tool_input)
params = {"goal": goal} if goal else {}
lst = None
_send_si = None
@@ -275,7 +275,7 @@ async def _post_action_state(
break
if isinstance(_l, dict) and "error" not in _l and _l.get("text"):
lst = _l
_send_si = _send_index_in_state(_l["text"])
_send_si = send_index_in_state(_l["text"])
if _send_si:
break
if time.monotonic() >= _deadline:
@@ -290,8 +290,8 @@ async def _post_action_state(
return ""
if not isinstance(lst, dict) or "error" in lst or not lst.get("text"):
return ""
state = lst["text"] if seen_lines is None else _delta_state(lst["text"], seen_lines)
out = f"\n\n{PAGE_STATE_MARKER}\n{_truncate_state(state)}"
state = lst["text"] if seen_lines is None else delta_state(lst["text"], seen_lines)
out = f"\n\n{PAGE_STATE_MARKER}\n{p_truncate_state(state)}"
# Hand the Send button's index over so the model clicks it directly instead of
# scanning the list or hunting via CSS/JS/screenshots (the polled list above is what
# makes Send actually present to point at, the two work together).
@@ -303,7 +303,7 @@ async def _post_action_state(
return out
async def _request_browser_approval(
async def p_request_browser_approval(
session: AgentSession, tool_name: str, tool_input: dict,
) -> dict:
"""Send an approval request for a browser sub-agent tool and wait for the decision."""
@@ -430,7 +430,7 @@ async def run_browser_agent(
preloaded_perception = ""
current_url = ""
preloaded_reads: list[dict] = [] # real front-loaded reads, seeded into action_log
_resumed = bool(browser_history._browser_history.get(browser_id))
_resumed = bool(browser_history.BROWSER_HISTORY.get(browser_id))
if initial_url:
nav_result = await execute_browser_tool(
"BrowserNavigate", {"url": initial_url}, browser_id, tab_id,
@@ -503,8 +503,8 @@ async def run_browser_agent(
# cycle every time the parent issues a new task. Defensively validate
# the cache; if it's somehow corrupted (orphaned tool_use_ids), drop
# it and start fresh rather than crash on the next API call.
prior_messages = browser_history._browser_history.get(browser_id) or []
if prior_messages and not _validate_message_pairing(prior_messages):
prior_messages = browser_history.BROWSER_HISTORY.get(browser_id) or []
if prior_messages and not validate_message_pairing(prior_messages):
logger.warning(
f"[browser-agent {session_id}] cached history for {browser_id} has "
f"orphaned tool_use_ids; dropping cache and starting fresh"
@@ -594,7 +594,7 @@ async def run_browser_agent(
# Advisory per-domain hints: seed the system prompt with what a prior agent
# learned about this domain (if we know the domain at start), and keep the
# store fresh from each ReportProgress. Re-verify, never blindly trust.
start_domain = _extract_domain(initial_url) if initial_url else None
start_domain = p_extract_domain(initial_url) if initial_url else None
run_system_prompt = SYSTEM_PROMPT
if start_domain:
prior_note = browser_history.get_domain_note(start_domain)
@@ -791,7 +791,7 @@ async def run_browser_agent(
try:
lst = await execute_browser_tool("BrowserListInteractives", {}, browser_id, tab_id)
if isinstance(lst, dict) and lst.get("text") and "error" not in lst:
_fresh = f"\nCurrent page state after the replayed prefix:\n{_truncate_state(lst['text'])}"
_fresh = f"\nCurrent page state after the replayed prefix:\n{p_truncate_state(lst['text'])}"
except Exception:
pass
remaining = "; ".join(f"{s['tool']}({str(s.get('params', {}))[:80]})" for s in steps[unsafe_i:])
@@ -838,7 +838,7 @@ async def run_browser_agent(
summary = browser_metrics.record_task(
session_id, browser_id, task, "completed", metrics_started_at,
turns_spent, rlog, session.tokens,
path="replay", task_sig=browser_skills._sig(skill_key_task),
path="replay", task_sig=browser_skills.compute_sig(skill_key_task),
)
logger.info(f"[browser-skills] REPLAY SUCCEEDED in {summary['total_ms']}ms ({turns_spent} LLM turn(s))")
try:
@@ -909,7 +909,7 @@ async def run_browser_agent(
# Pre-nav landed on a results page (the cold entry case): scan it NOW so the
# model's very first turn can pick a candidate instead of read-then-decide.
_start_url = (current_url or initial_url or "").split("#")[0]
if _start_url and _RESULTS_URL_RE.search(_start_url):
if _start_url and RESULTS_URL_RE.search(_start_url):
auto_scanned_urls.add(_start_url)
_scan_json, _sc_ms = await _scan_results(task)
if _scan_json:
@@ -1048,7 +1048,7 @@ async def run_browser_agent(
# tool is the redundant narration the prompt now forbids; count it so
# the bench can verify the prose actually went away.
if any(t.strip() for t in text_parts) and any(
tu.name in _ACTION_TOOLS_REQUIRING_REPORT for tu in tool_uses
tu.name in ACTION_TOOLS_REQUIRING_REPORT for tu in tool_uses
):
narration_turns += 1
@@ -1088,7 +1088,7 @@ async def run_browser_agent(
# the brain state is recorded before any actions execute.
has_report_progress = any(tu.name == "ReportProgress" for tu in tool_uses)
has_action_tools = any(
tu.name in _ACTION_TOOLS_REQUIRING_REPORT for tu in tool_uses
tu.name in ACTION_TOOLS_REQUIRING_REPORT for tu in tool_uses
)
# Violation: action tools without ReportProgress in the same turn.
# The model MUST articulate its evaluation/memory/goal before acting.
@@ -1105,7 +1105,7 @@ async def run_browser_agent(
rp_violations += 1
rp_reminder_pending = True
if not current_next_goal or current_next_goal == task:
_synth = next((t.name for t in tool_uses if t.name in _ACTION_TOOLS_REQUIRING_REPORT), "act")
_synth = next((t.name for t in tool_uses if t.name in ACTION_TOOLS_REQUIRING_REPORT), "act")
current_next_goal = f"(continuing) {_synth.replace('Browser', '').lower()}"
logger.info(
f"[browser-agent {session_id}] ReportProgress omitted; running the action "
@@ -1119,7 +1119,7 @@ async def run_browser_agent(
# Under-batching detector: the model ignores prompt-level batching
# invitations, so measure each turn and nudge mechanically below.
_turn_actions = sum(1 for t in tool_uses_sorted if t.name in _BATCHABLE_ACTION_TOOLS)
_turn_actions = sum(1 for t in tool_uses_sorted if t.name in P_BATCHABLE_ACTION_TOOLS)
_turn_has_batch = any(t.name in ("BrowserBatch", "BrowserRepeatFlow") for t in tool_uses_sorted)
if _turn_actions >= 2 or _turn_has_batch:
multi_action_turns += 1
@@ -1139,7 +1139,7 @@ async def run_browser_agent(
# it off with partial results. Re-reading the same page yields no new sig.
_novel_read = False
for _a in action_log:
if (_a.get("ok") and _a.get("tool") not in _BATCHABLE_ACTION_TOOLS
if (_a.get("ok") and _a.get("tool") not in P_BATCHABLE_ACTION_TOOLS
and _a.get("tool") not in ("ReportProgress", "Done")):
_sig = f"{_a.get('tool')}:{_a.get('result_summary') or ''}"
if _sig not in seen_read_sigs:
@@ -1166,7 +1166,7 @@ async def run_browser_agent(
# The general backstop only applies AFTER the agent has actually
# done something; early pure-perception is legitimate orienting on a
# cold/slow page, which we must never cut short.
_acted = any(a.get("ok") and a.get("tool") in (_BATCHABLE_ACTION_TOOLS | {"BrowserBatch"})
_acted = any(a.get("ok") and a.get("tool") in (P_BATCHABLE_ACTION_TOOLS | {"BrowserBatch"})
for a in action_log)
_stall_limit = (_POST_SEND_STALL_LIMIT if send_confirmed
else (_PERCEPTION_STALL_LIMIT if _acted else 10 ** 9))
@@ -1442,7 +1442,7 @@ async def run_browser_agent(
if tu.name == "RequestHumanIntervention":
problem = tu.input.get("problem", "")
instruction = tu.input.get("instruction", "")
decision = await _request_browser_approval(
decision = await p_request_browser_approval(
session, tu.name, {"problem": problem, "instruction": instruction},
)
if decision.get("behavior") != "deny":
@@ -1490,7 +1490,7 @@ async def run_browser_agent(
continue
if policy == "ask":
decision = await _request_browser_approval(
decision = await p_request_browser_approval(
session, tu.name, tu.input,
)
if decision.get("behavior") == "deny":
@@ -1562,7 +1562,7 @@ async def run_browser_agent(
# so the agent never claims a success it didn't see or re-fires blindly.
_expect = (str(tu.input.get("expect") or "").strip()
if isinstance(tu.input, dict) else "")
if _expect and "error" not in result and tu.name in _CONFIRM_TOOLS:
if _expect and "error" not in result and tu.name in P_CONFIRM_TOOLS:
# target_only: wait for the expected text to actually appear, don't
# call it 'not confirmed' just because the page settled first (a sent
# message lands in the thread a beat after settle, esp. under load)
@@ -1589,7 +1589,7 @@ async def run_browser_agent(
# across nodes, or scrolled off, so the text-probe is unreliable, which
# is exactly what left the model stalling to "double-check". A clean
# send click is proof enough; drive to the OUTCOME.
if task_is_send and not send_confirmed and "error" not in result and tu.name in _CONFIRM_TOOLS:
if task_is_send and not send_confirmed and "error" not in result and tu.name in P_CONFIRM_TOOLS:
_cn = result.get("clickedName") or ""
_cr = result.get("clickedRole") or ""
_send_click = browser_batch_replay.is_send_completed(
@@ -1643,7 +1643,7 @@ async def run_browser_agent(
result["text"] = (f"{result.get('error')}\n\n[recovery] That action did not "
f"take effect, but the page is live. Current elements (re-act "
f"from HERE, do not just retry the old index):\n"
f"{_truncate_state(str(_rl['text']))}")
f"{p_truncate_state(str(_rl['text']))}")
recovery_attaches += 1
logger.info(f"[browser-recovery {session_id}] attached fresh state after "
f"recoverable error at turn {turn}: {str(result.get('error'))[:60]}")
@@ -1656,7 +1656,7 @@ async def run_browser_agent(
attached_state_seen.update(
l for l in str(result.get("text") or "").splitlines() if l.startswith("[")
)
_auto_state = await _post_action_state(
_auto_state = await post_action_state(
tu.name, tu.input, result, browser_id, tab_id, _wait_exec, current_next_goal,
seen_lines=attached_state_seen,
)
@@ -1669,7 +1669,7 @@ async def run_browser_agent(
# One gentle nudge per violating turn, folded onto the action that
# ran, so the model self-corrects next turn without us costing it one.
if rp_reminder_pending and tu.name in _ACTION_TOOLS_REQUIRING_REPORT:
if rp_reminder_pending and tu.name in ACTION_TOOLS_REQUIRING_REPORT:
rp_reminder_pending = False
result["text"] = (f"{result.get('text') or ''}\n\n[note] Action ran. Next turn, "
"include ReportProgress (working_memory + next_goal) alongside "
@@ -1680,7 +1680,7 @@ async def run_browser_agent(
# per URL, only on the tight throwaway-dismiss vocabulary that
# never sits on a task-needed control, so it can't close anything
# required. After closing, re-list so the model sees the page beneath.
if tu.name in _AUTO_STATE_TOOLS and "error" not in result:
if tu.name in P_AUTO_STATE_TOOLS and "error" not in result:
_pop_url = (result.get("url") or last_seen_url or "").split("#")[0]
if _pop_url and _pop_url not in dismissed_popup_urls:
_close = interstitial_dismiss_target("\n".join(attached_state_seen))
@@ -1692,7 +1692,7 @@ async def run_browser_agent(
logger.info(f"[browser-popup {session_id}] auto-dismissed '{_close}' "
f"ok={_dok} on {_pop_url[:80]}")
if _dok:
_fresh = await _post_action_state(
_fresh = await post_action_state(
"BrowserClickByName", {}, _dres or {}, browser_id, tab_id,
_wait_exec, current_next_goal, seen_lines=attached_state_seen)
result["text"] = (f"{result.get('text') or ''}\n\n[auto] Closed a blocking "
@@ -1702,10 +1702,10 @@ async def run_browser_agent(
# costs a read-then-decide turn pair; the cheap aux model reads it
# now so the pick happens on this same turn. Capped, per-URL,
# fail-silent (a miss just means the old two-turn dance).
if (tu.name in _AUTO_STATE_TOOLS and "error" not in result
and auto_scan_count < _AUTO_SCAN_MAX_PER_RUN):
if (tu.name in P_AUTO_STATE_TOOLS and "error" not in result
and auto_scan_count < P_AUTO_SCAN_MAX_PER_RUN):
_scan_url = (result.get("url") or last_seen_url or "").split("#")[0]
if _scan_url and _scan_url not in auto_scanned_urls and _RESULTS_URL_RE.search(_scan_url):
if _scan_url and _scan_url not in auto_scanned_urls and RESULTS_URL_RE.search(_scan_url):
auto_scanned_urls.add(_scan_url)
_scan_json, _sc_ms = await _scan_results(task)
if _scan_json:
@@ -1771,22 +1771,22 @@ async def run_browser_agent(
# acknowledge it on its next turn.
# Loop detection only covers the non-excluded tools, so skip the
# hash entirely for the excluded ones; otherwise a screenshot/read
# serializes its full ~1MB result here just for _detect_loop to
# serializes its full ~1MB result here just for detect_loop to
# discard it (it short-circuits excluded tools to False anyway).
if tu.name in _LOOP_DETECTION_EXCLUDED_TOOLS:
if tu.name in LOOP_DETECTION_EXCLUDED_TOOLS:
is_loop = False
else:
call_key = _hash_tool_call(tu.name, tu.input, result)
is_loop = _detect_loop(recent_tool_calls, call_key)
call_key = hash_tool_call(tu.name, tu.input, result)
is_loop = detect_loop(recent_tool_calls, call_key)
recent_tool_calls.append(call_key)
if len(recent_tool_calls) > _LOOP_WINDOW_SIZE * 2:
recent_tool_calls = recent_tool_calls[-_LOOP_WINDOW_SIZE * 2:]
if len(recent_tool_calls) > LOOP_WINDOW_SIZE * 2:
recent_tool_calls = recent_tool_calls[-LOOP_WINDOW_SIZE * 2:]
content_blocks = _format_tool_result(result, tu.name)
content_blocks = format_tool_result(result, tu.name)
try:
url = result.get("url") or (tu.input or {}).get("url")
if url:
domain = _extract_domain(str(url))
domain = p_extract_domain(str(url))
if domain and domain not in session.browser_domains:
session.browser_domains.append(domain)
except Exception:
@@ -1809,7 +1809,7 @@ async def run_browser_agent(
if is_loop:
loop_trigger_count += 1
repeat_count = sum(1 for c in recent_tool_calls if c == call_key)
warning = _LOOP_WARNING_TEXT.format(count=repeat_count)
warning = LOOP_WARNING_TEXT.format(count=repeat_count)
logger.warning(
f"[browser-agent {session_id}] loop detected on {tu.name} "
f"(trigger #{loop_trigger_count}): {warning}"
@@ -1936,7 +1936,7 @@ async def run_browser_agent(
# tool_results (a text block alongside them) so the model's next turn is
# a clean Done instead of more looking.
if wrapup_pending:
tool_results.append({"type": "text", "text": _WRAPUP_NUDGE})
tool_results.append({"type": "text", "text": P_WRAPUP_NUDGE})
messages.append({"role": "user", "content": tool_results})
if done_called:
@@ -1948,10 +1948,10 @@ async def run_browser_agent(
# Hard cap on loops: if the model keeps repeating itself even
# after we warn it, force-exit so we don't burn the entire turn
# budget on a stuck agent.
if loop_trigger_count >= _LOOP_HARD_CAP:
if loop_trigger_count >= LOOP_HARD_CAP:
logger.warning(
f"[browser-agent {session_id}] hit loop hard cap "
f"({_LOOP_HARD_CAP}); force-exiting"
f"({LOOP_HARD_CAP}); force-exiting"
)
break
@@ -1959,7 +1959,7 @@ async def run_browser_agent(
# out / the page never responds). Either way the agent can't make
# progress, so stop retrying after a short streak and report honestly,
# instead of the 20-minute spin on a wedged tab.
if card_gone_streak >= _CARD_GONE_LIMIT:
if card_gone_streak >= CARD_GONE_LIMIT:
logger.warning(
f"[browser-agent {session_id}] browser card {browser_id} is unusable "
f"({card_gone_streak} consecutive gone/hung results); aborting fast"
@@ -2005,18 +2005,18 @@ async def run_browser_agent(
# Persist conversation history so the next BrowserAgent call on this
# browser can resume rather than re-orient. Trim to the most recent
# _MAX_HISTORY_MESSAGES turns to keep token usage bounded; but
# MAX_HISTORY_MESSAGES turns to keep token usage bounded; but
# never split a tool_use ↔ tool_result pair across the cut, or the
# next API request will 400.
browser_history._browser_history[browser_id] = _trim_history_by_turns(
messages, _MAX_HISTORY_MESSAGES,
browser_history.BROWSER_HISTORY[browser_id] = trim_history_by_turns(
messages, MAX_HISTORY_MESSAGES,
)
# Honesty gate: the model declaring done is not proof the goal happened.
# If the run did no real work (zero actions, all actions errored, or only
# looked around), report the truth instead of a ghost "completed". A gone
# card gets its own precise reason instead of the generic verdict.
if card_gone_streak >= _CARD_GONE_LIMIT:
if card_gone_streak >= CARD_GONE_LIMIT:
honest, dishonest_reason = False, "the browser became unresponsive (the tab hung or was closed); it needs a fresh browser to continue"
else:
honest, dishonest_reason = completion_is_honest(action_log)
@@ -2073,7 +2073,7 @@ async def run_browser_agent(
browser_metrics.record_task(session_id, browser_id, task, final_status,
metrics_started_at, turn + 1, action_log, session.tokens,
path="llm_fallback" if replay_attempted else "llm",
task_sig=browser_skills._sig(skill_key_task),
task_sig=browser_skills.compute_sig(skill_key_task),
playbook_seeded=pb_seeded)
# Learn this task ONLY from a genuinely successful run whose deliverable a
# deterministic replay can actually reproduce. We skip recording when the
@@ -2202,13 +2202,13 @@ async def run_browser_agent(
# Cards a sub-agent is actively driving in this process. Reuse must never hand
# two agents one webview (their commands would interleave into chaos).
_active_agent_cards: set[str] = set()
ACTIVE_AGENT_CARDS: set[str] = set()
# find+claim+create must be one critical section or two parallel dispatches
# race to claim the same idle card (or both miss and double-create).
_card_pick_lock = asyncio.Lock()
p_card_pick_lock = asyncio.Lock()
def _find_reusable_card(dashboard_id: str, url: str, parent_session_id: str | None) -> str:
def find_reusable_card(dashboard_id: str, url: str, parent_session_id: str | None) -> str:
"""An existing same-host spawned card to drive instead of stacking another
webview: concurrent same-site webviews wedge each other (shared-partition
lock contention), so a retry must REUSE, not multiply. The parent's own
@@ -2226,7 +2226,7 @@ def _find_reusable_card(dashboard_id: str, url: str, parent_session_id: str | No
own, orphan = "", ""
for bid, card in cards.items():
spawned = getattr(card, "spawned_by", None)
if not spawned or bid in _active_agent_cards:
if not spawned or bid in ACTIVE_AGENT_CARDS:
continue
if browser_skills.host_of(getattr(card, "url", "") or "") != want:
continue
@@ -2239,7 +2239,7 @@ def _find_reusable_card(dashboard_id: str, url: str, parent_session_id: str | No
return own or orphan
async def _create_browser_card(dashboard_id: str, url: str, parent_session_id: str | None = None) -> str:
async def p_create_browser_card(dashboard_id: str, url: str, parent_session_id: str | None = None) -> str:
"""Create a new browser card on the dashboard and return its browser_id."""
from backend.apps.dashboards.dashboards import _load, _save
from backend.apps.dashboards.models import BrowserCardPosition, BrowserTab
@@ -2326,15 +2326,15 @@ async def run_browser_agents(
# the url param is often empty with the target buried in the task
# text; a url there still names the host we must not duplicate
host_src = url or entry_url or next(iter(re.findall(r"https?://[^\s)\"'<>]+", task_text)), "")
async with _card_pick_lock:
browser_id = _find_reusable_card(dashboard_id, host_src, parent_session_id)
async with p_card_pick_lock:
browser_id = find_reusable_card(dashboard_id, host_src, parent_session_id)
if browser_id:
reused = True
else:
browser_id = await _create_browser_card(dashboard_id, url or entry_url, parent_session_id)
browser_id = await p_create_browser_card(dashboard_id, url or entry_url, parent_session_id)
if entry_url and not url:
logger.info(f"[browser-cold] new card {browser_id} opens at brief entry {entry_url}")
_active_agent_cards.add(browser_id)
ACTIVE_AGENT_CARDS.add(browser_id)
if reused:
logger.info(f"[browser-agent] reusing same-host card {browser_id} instead of stacking another webview")
if url:
@@ -2346,7 +2346,7 @@ async def run_browser_agents(
else:
await asyncio.sleep(2.0)
elif browser_id:
_active_agent_cards.add(browser_id)
ACTIVE_AGENT_CARDS.add(browser_id)
is_pre_selected = browser_id in pre_selected
_nav_url = url or ("" if reused else entry_url)
@@ -2362,7 +2362,7 @@ async def run_browser_agents(
parent_session_id=parent_session_id,
)
finally:
_active_agent_cards.discard(browser_id)
ACTIVE_AGENT_CARDS.discard(browser_id)
results = await asyncio.gather(*[_run_one(t) for t in tasks], return_exceptions=True)
backend/apps/agents/browser/browser_batch_replay.py
+30 -30
View File
@@ -37,7 +37,7 @@ import re
PLACEHOLDER = "{{value}}"
# Agent-facing step action -> (tool_name, the param keys it carries).
_STEP_TOOLS: dict[str, tuple[str, tuple[str, ...]]] = {
P_STEP_TOOLS: dict[str, tuple[str, tuple[str, ...]]] = {
"navigate": ("BrowserNavigate", ("url",)),
"get_text": ("BrowserGetText", ()),
"evaluate": ("BrowserEvaluate", ("expression",)),
@@ -49,27 +49,27 @@ _STEP_TOOLS: dict[str, tuple[str, tuple[str, ...]]] = {
}
# Reads/navigation don't mutate anything irreversible; safe to loop freely.
_READONLY_ACTIONS = {"navigate", "get_text", "evaluate", "scroll", "replay_route"}
P_READONLY_ACTIONS = {"navigate", "get_text", "evaluate", "scroll", "replay_route"}
# Irreversible / outward-facing words on a clicked control. Conservative on
# purpose: we'd rather refuse a borderline loop than auto-send 10 messages.
_SEND_NAME_RE = re.compile(
P_SEND_NAME_RE = re.compile(
r"\b(send|submit|post|publish|connect|invite|follow|like|react|comment|reply|"
r"share|message|dm|pay|buy|order|checkout|purchase|place\s*order|book|"
r"confirm|apply|accept|decline|delete|remove|unsend|withdraw|endorse)\b",
re.I,
)
# A field that reads like a message/comment composer; typing here is part of a send.
_COMPOSE_SEL_RE = re.compile(r"message|compose|comment|msg|reply|editor|body|tweet|post", re.I)
P_COMPOSE_SEL_RE = re.compile(r"message|compose|comment|msg|reply|editor|body|tweet|post", re.I)
def is_send_step(step: dict) -> bool:
"""True if this step is irreversible / outward-facing, so the whole loop must
be gated rather than auto-replayed."""
action = step.get("action")
if action == "click" and _SEND_NAME_RE.search(str(step.get("name") or "")):
if action == "click" and P_SEND_NAME_RE.search(str(step.get("name") or "")):
return True
if action == "type" and _COMPOSE_SEL_RE.search(str(step.get("selector") or "")):
if action == "type" and P_COMPOSE_SEL_RE.search(str(step.get("selector") or "")):
return True
return False
@@ -83,9 +83,9 @@ def validate_template(steps) -> tuple[bool, str]:
if not isinstance(step, dict):
return False, f"step {i+1} is not an object"
action = step.get("action")
spec = _STEP_TOOLS.get(action)
spec = P_STEP_TOOLS.get(action)
if not spec:
return False, f"step {i+1}: unknown action {action!r} (allowed: {', '.join(_STEP_TOOLS)})"
return False, f"step {i+1}: unknown action {action!r} (allowed: {', '.join(P_STEP_TOOLS)})"
_, required = spec
for key in required:
if step.get(key) in (None, ""):
@@ -105,9 +105,9 @@ def template_safety(steps) -> tuple[bool, str]:
return True, ""
# Like _SEND_NAME_RE minus composer-openers ("Message"/"DM" buttons open a
# Like P_SEND_NAME_RE minus composer-openers ("Message"/"DM" buttons open a
# compose box, they don't send), so routine flows still batch freely.
_LIVE_IRREVERSIBLE_RE = re.compile(
P_LIVE_IRREVERSIBLE_RE = re.compile(
r"\b(send|submit|post|publish|connect|invite|follow|like|react|comment|reply|"
r"share|pay|buy|order|checkout|purchase|place\s*order|book|"
r"confirm|apply|accept|decline|delete|remove|unsend|withdraw|endorse)\b",
@@ -123,18 +123,18 @@ def is_replay_boundary(step: dict) -> bool:
crosses to the live model. Uses the same opener-excluded wordlist the live
send-guard already trusts, so a recorded Send still stops the prefix."""
action = step.get("action")
if action == "click" and _LIVE_IRREVERSIBLE_RE.search(str(step.get("name") or "")):
if action == "click" and P_LIVE_IRREVERSIBLE_RE.search(str(step.get("name") or "")):
return True
if action == "type" and _COMPOSE_SEL_RE.search(str(step.get("selector") or "")):
if action == "type" and P_COMPOSE_SEL_RE.search(str(step.get("selector") or "")):
return True
return False
_SEND_COMPLETED_RE = re.compile(
P_SEND_COMPLETED_RE = re.compile(
r"\b(send|submit|pay|place\s*order|complete\s*(order|purchase|checkout|payment))\b",
re.I,
)
_OPENER_ROLES = frozenset({"menuitem", "menuitemcheckbox", "menuitemradio", "link", "tab"})
P_OPENER_ROLES = frozenset({"menuitem", "menuitemcheckbox", "menuitemradio", "link", "tab"})
def is_send_completed(step: dict) -> bool:
@@ -145,9 +145,9 @@ def is_send_completed(step: dict) -> bool:
if step.get("action") != "click":
return False
role = str(step.get("role") or "").lower()
if role in _OPENER_ROLES:
if role in P_OPENER_ROLES:
return False
return bool(_SEND_COMPLETED_RE.search(str(step.get("name") or "")))
return bool(P_SEND_COMPLETED_RE.search(str(step.get("name") or "")))
def live_batch_guard(actions, seen_lines, composer_pending: bool = False) -> str:
@@ -172,7 +172,7 @@ def live_batch_guard(actions, seen_lines, composer_pending: bool = False) -> str
elif typ == "click":
label = str(params.get("selector") or "")
elif typ == "type":
if _COMPOSE_SEL_RE.search(str(params.get("selector") or "")):
if P_COMPOSE_SEL_RE.search(str(params.get("selector") or "")):
typed_composer = True
continue
elif typ == "press_key":
@@ -184,7 +184,7 @@ def live_batch_guard(actions, seen_lines, composer_pending: bool = False) -> str
continue
# selectors hide words behind underscores/dashes (msg-form__send-button),
# which defeat \b; flatten separators so the word check still sees them
if label and _LIVE_IRREVERSIBLE_RE.search(re.sub(r"[_\-./#\[\]]+", " ", label)):
if label and P_LIVE_IRREVERSIBLE_RE.search(re.sub(r"[_\-./#\[\]]+", " ", label)):
return (f"sub-action {i+1} ({typ}) targets {label.strip()!r}, "
"which is irreversible/outward-facing")
return ""
@@ -209,12 +209,12 @@ def send_payload_from_log(action_log, prompt: str = "") -> str:
summ = str(a.get("result_summary") or "")
# focus+type results carry no clicked fields (r47's live miss); the
# executor's own "typed the text" wording is the surviving signal
if _COMPOSE_SEL_RE.search(name) or (len(text) >= 20 and (
if P_COMPOSE_SEL_RE.search(name) or (len(text) >= 20 and (
role == "textbox" or "typed the text" in summ.lower())):
typed.append(text)
elif tool == "BrowserType":
sel = str(inp.get("selector") or "")
if _COMPOSE_SEL_RE.search(sel) or (not sel and len(text) >= 20):
if P_COMPOSE_SEL_RE.search(sel) or (not sel and len(text) >= 20):
typed.append(text)
elif tool == "BrowserBatch":
for sub in (inp.get("actions") or []):
@@ -224,7 +224,7 @@ def send_payload_from_log(action_log, prompt: str = "") -> str:
sub_text = str(p.get("text") or "").strip()
sub_sel = str(p.get("selector") or "")
if sub.get("type") == "type" and sub_text and (
_COMPOSE_SEL_RE.search(sub_sel)
P_COMPOSE_SEL_RE.search(sub_sel)
or (not sub_sel and len(sub_text) >= 20)):
typed.append(sub_text)
if not typed:
@@ -237,7 +237,7 @@ def send_payload_from_log(action_log, prompt: str = "") -> str:
return typed[-1]
def _sub(val, value: str):
def p_sub(val, value: str):
return value if val == PLACEHOLDER else (
val.replace(PLACEHOLDER, value) if isinstance(val, str) else val
)
@@ -247,14 +247,14 @@ def fill_step(step: dict, value: str) -> tuple[str, dict]:
"""Turn one template step + one value into (tool_name, params) ready for
execute_browser_tool. Substitutes {{value}} anywhere it appears."""
action = step["action"]
tool_name, keys = _STEP_TOOLS[action]
tool_name, keys = P_STEP_TOOLS[action]
params = {}
for k in keys:
if k in step:
params[k] = _sub(step[k], value)
params[k] = p_sub(step[k], value)
# carry an optional role default for clicks
if action == "click" and "role" not in params:
params["role"] = _sub(step.get("role", ""), value)
params["role"] = p_sub(step.get("role", ""), value)
return tool_name, params
@@ -265,19 +265,19 @@ def fill_template(steps, value: str) -> list[tuple[str, dict]]:
def is_readonly_template(steps) -> bool:
"""True if every step is a pure read/navigation (no clicks/types at all), the
safest class of loop."""
return all(s.get("action") in _READONLY_ACTIONS for s in steps)
return all(s.get("action") in P_READONLY_ACTIONS for s in steps)
# A batch READ is useless if it doesn't hand the data back. We return each item's
# read output, capped so a 20-item batch stays cheap, and stay honest about
# failures (named, with the error) and truncation (named, never silently dropped).
_MAX_ITEM_CHARS = 500
_MAX_TOTAL_CHARS = 6000
P_MAX_ITEM_CHARS = 500
P_MAX_TOTAL_CHARS = 6000
def summarize_batch(records: list[dict], readonly: bool,
max_item_chars: int = _MAX_ITEM_CHARS,
max_total_chars: int = _MAX_TOTAL_CHARS) -> str:
max_item_chars: int = P_MAX_ITEM_CHARS,
max_total_chars: int = P_MAX_TOTAL_CHARS) -> str:
"""Turn per-item batch results into the text the agent gets back.
`records`: [{value, ok, text}]. For a successful item `text` is its read
backend/apps/agents/browser/browser_extract.py
+8 -8
View File
@@ -13,12 +13,12 @@ import re
logger = logging.getLogger(__name__)
_MAX_PAGE_CHARS = 12000
_MAX_OUT_TOKENS = 1200
_MAX_SCHEMA_CHARS = 2000
P_MAX_PAGE_CHARS = 12000
P_MAX_OUT_TOKENS = 1200
P_MAX_SCHEMA_CHARS = 2000
def _first_json(text: str) -> str:
def first_json(text: str) -> str:
"""The model's output minus any prose/fences around the JSON, or ''."""
cleaned = re.sub(r"```(?:json)?|```", "", text or "")
m = re.search(r"\{.*\}|\[.*\]", cleaned, re.DOTALL)
@@ -38,7 +38,7 @@ async def extract_structured(
if not aux_client or not aux_model or not page_text:
return ""
shape = (
f"Return JSON matching this schema exactly:\n{json.dumps(schema)[:_MAX_SCHEMA_CHARS]}"
f"Return JSON matching this schema exactly:\n{json.dumps(schema)[:P_MAX_SCHEMA_CHARS]}"
if isinstance(schema, dict) and schema else "Return one compact JSON object."
)
prompt = (
@@ -47,15 +47,15 @@ async def extract_structured(
"Output ONLY the JSON, no prose, no code fences. Use only what is on the "
'page, never guess. If the requested data is not on the page, output '
'{"not_found": true, "reason": "<one short line>"}.\n\n'
f"PAGE TEXT:\n{page_text[:_MAX_PAGE_CHARS]}"
f"PAGE TEXT:\n{page_text[:P_MAX_PAGE_CHARS]}"
)
try:
resp = await aux_client.messages.create(
model=aux_model, max_tokens=_MAX_OUT_TOKENS,
model=aux_model, max_tokens=P_MAX_OUT_TOKENS,
messages=[{"role": "user", "content": prompt}],
)
text = "".join(getattr(b, "text", "") for b in (resp.content or []))
return _first_json(text)
return first_json(text)
except Exception as e:
logger.debug(f"[browser-extract] aux extraction failed: {e}")
return ""
backend/apps/agents/browser/browser_fast_path.py
+10 -10
View File
@@ -25,7 +25,7 @@ logger = logging.getLogger(__name__)
# Zero-cost smell test: only prompts that mention the web at all are worth a
# classifier call. False negatives just take the normal path.
_BROWSY_RE = re.compile(
P_BROWSY_RE = re.compile(
r"https?://|www\.|\b[a-z0-9-]+\.(com|org|net|io|co|ai|dev|app)\b"
r"|\b(browse|browser|website|web ?page|webpage|site|url|tab)\b"
r"|\b(go to|open|visit|navigate|log ?in|sign ?in|search on|look up on|check on)\b"
@@ -34,7 +34,7 @@ _BROWSY_RE = re.compile(
re.I,
)
_CLASSIFIER_SYSTEM = (
P_CLASSIFIER_SYSTEM = (
"You route requests to a web-browsing agent. It drives a real signed-in browser: "
"navigating sites, reading or extracting or counting what is on pages, clicking, "
"typing, and acting inside web apps (sending messages on LinkedIn or any site, "
@@ -87,10 +87,10 @@ def fast_path_eligible(
return False
if not prompt or not prompt.strip():
return False
return bool(_BROWSY_RE.search(prompt))
return bool(P_BROWSY_RE.search(prompt))
def _parse_verdict_and_brief(text: str) -> tuple[str, str]:
def parse_verdict_and_brief(text: str) -> tuple[str, str]:
"""Line 1 carries READ/ACT/NO; the rest is the routing brief. Anything
unparseable is 'no' (normal path)."""
lines = (text or "").strip().splitlines()
@@ -105,14 +105,14 @@ def _parse_verdict_and_brief(text: str) -> tuple[str, str]:
return verdict, brief[:700]
_ENTRY_RE = re.compile(r"^\s*ENTRY:\s*(https?://\S+)", re.I | re.M)
P_ENTRY_RE = re.compile(r"^\s*ENTRY:\s*(https?://\S+)", re.I | re.M)
def entry_url_from_brief(brief: str) -> str:
"""The brief's ENTRY deep URL, or ''. Powers dispatch pre-navigation: a NEW
card opens directly on it instead of google, killing the orient+navigate
turns; a REUSED card is never moved (its deeper live state wins)."""
m = _ENTRY_RE.search(brief or "")
m = P_ENTRY_RE.search(brief or "")
return m.group(1).rstrip(".,;)") if m else ""
@@ -211,7 +211,7 @@ def unverifiable_reply(payload: str, first_report: str) -> str:
)
def _normalize_for_classifier(prompt: str) -> str:
def normalize_for_classifier(prompt: str) -> str:
"""Haiku reads bare 'text him' as SMS even with a site as context. In the
browsy-prefiltered pool, text-with-no-phone-number is in-site messaging,
so spell it out for the small model. Only the classifier sees this."""
@@ -237,13 +237,13 @@ async def classify_and_brief(prompt: str, settings, primary_api: str | None) ->
model=aux_model,
max_tokens=250,
temperature=0,
system=_CLASSIFIER_SYSTEM,
messages=[{"role": "user", "content": _normalize_for_classifier(prompt[:2000])}],
system=P_CLASSIFIER_SYSTEM,
messages=[{"role": "user", "content": normalize_for_classifier(prompt[:2000])}],
),
timeout=8.0,
)
from backend.apps.agents.core.aux_llm import safe_resp_text
verdict, brief = _parse_verdict_and_brief(safe_resp_text(resp))
verdict, brief = parse_verdict_and_brief(safe_resp_text(resp))
logger.info(
f"[browser-fast-path] classifier: {verdict.upper()} brief={len(brief)}ch "
f"model={aux_model} in {int((time.monotonic() - t0) * 1000)}ms"
backend/apps/agents/browser/browser_fast_read.py
+10 -10
View File
@@ -13,12 +13,12 @@ import time
logger = logging.getLogger(__name__)
_ENTRY_RE = re.compile(r"^ENTRY:\s*(https?://\S+)", re.I | re.M)
_MIN_PAGE_CHARS = 500
_MAX_PAGE_CHARS = 24000
_FETCH_ERROR_PREFIXES = ("HTTP error", "Error fetching", "Refused to fetch")
P_ENTRY_RE = re.compile(r"^ENTRY:\s*(https?://\S+)", re.I | re.M)
P_MIN_PAGE_CHARS = 500
P_MAX_PAGE_CHARS = 24000
P_FETCH_ERROR_PREFIXES = ("HTTP error", "Error fetching", "Refused to fetch")
_ANSWER_SYSTEM = (
P_ANSWER_SYSTEM = (
"Answer the user's request using ONLY the page text provided. Be direct and "
"complete in a few sentences; quote exact titles/values from the page. End "
"with nothing else.\n"
@@ -28,16 +28,16 @@ _ANSWER_SYSTEM = (
def extract_entry_url(brief: str) -> str:
m = _ENTRY_RE.search(brief or "")
m = P_ENTRY_RE.search(brief or "")
return m.group(1).rstrip(").,") if m else ""
def page_is_thin(text: str) -> bool:
t = (text or "").strip()
if not t or t.startswith(_FETCH_ERROR_PREFIXES):
if not t or t.startswith(P_FETCH_ERROR_PREFIXES):
return True
body = t.split("\n\n", 1)[-1] if "\n\n" in t else t
return len(body.strip()) < _MIN_PAGE_CHARS
return len(body.strip()) < P_MIN_PAGE_CHARS
async def try_fast_read(prompt: str, brief: str, settings, primary_api: str | None) -> str | None:
@@ -75,10 +75,10 @@ async def try_fast_read(prompt: str, brief: str, settings, primary_api: str | No
model=aux_model,
max_tokens=500,
temperature=0,
system=_ANSWER_SYSTEM,
system=P_ANSWER_SYSTEM,
messages=[{
"role": "user",
"content": f"Request: {prompt}\n\nPage text from {entry}:\n{text[:_MAX_PAGE_CHARS]}",
"content": f"Request: {prompt}\n\nPage text from {entry}:\n{text[:P_MAX_PAGE_CHARS]}",
}],
),
timeout=15.0,
backend/apps/agents/browser/browser_history.py
+24 -24
View File
@@ -6,44 +6,44 @@ the same browser can resume rather than restart from scratch. Without this every
"swipe right" / "swipe left" call has to take a new screenshot and re-orient
itself, costing 30-60s per action.
The `_browser_history` mutable cache lives in EXACTLY this module; all reads and
The `BROWSER_HISTORY` mutable cache lives in EXACTLY this module; all reads and
writes route through here so there's a single source of truth.
"""
# browser_id -> cached Anthropic message list for resume.
_browser_history: dict[str, list[dict]] = {}
BROWSER_HISTORY: dict[str, list[dict]] = {}
# Cap history to prevent unbounded growth on long-lived browsers.
_MAX_HISTORY_MESSAGES = 30
MAX_HISTORY_MESSAGES = 30
# Per-apex-domain advisory notes, distilled from the agent's own ReportProgress
# working_memory. Process-lifetime only (never written to disk); seeds a later
# agent on the same domain so it skips re-learning the same quirks. Advisory
# text only, never auto-executed.
_domain_notes: dict[str, str] = {}
_MAX_DOMAIN_NOTE_CHARS = 600
DOMAIN_NOTES: dict[str, str] = {}
MAX_DOMAIN_NOTE_CHARS = 600
def get_domain_note(domain: str) -> str:
"""Return the advisory note for a domain, or empty string if none."""
return _domain_notes.get(domain, "")
return DOMAIN_NOTES.get(domain, "")
def set_domain_note(domain: str, note: str) -> None:
"""Store/overwrite the advisory note for a domain (trimmed + capped)."""
if not domain or not note or not note.strip():
return
_domain_notes[domain] = note.strip()[:_MAX_DOMAIN_NOTE_CHARS]
DOMAIN_NOTES[domain] = note.strip()[:MAX_DOMAIN_NOTE_CHARS]
def clear_browser_history(browser_id: str) -> None:
"""Drop cached conversation history for a browser (e.g. when it's closed)."""
_browser_history.pop(browser_id, None)
BROWSER_HISTORY.pop(browser_id, None)
_OMITTED_SCREENSHOT_STUB = "[earlier screenshot omitted to save context]"
OMITTED_SCREENSHOT_STUB = "[earlier screenshot omitted to save context]"
def _iter_image_block_refs(messages: list[dict]):
def p_iter_image_block_refs(messages: list[dict]):
"""Yield (container_list, index) for every image block, in document order.
Screenshots live either directly in a message's content list or nested inside
@@ -76,7 +76,7 @@ def prune_old_screenshots(messages: list[dict], keep_first: bool = True, keep_re
are dropped, not the memory. If the agent must re-see, it just re-screenshots.
Returns how many images were collapsed.
"""
refs = list(_iter_image_block_refs(messages))
refs = list(p_iter_image_block_refs(messages))
keep_count = keep_recent + (1 if keep_first else 0)
if len(refs) <= keep_count:
return 0
@@ -89,7 +89,7 @@ def prune_old_screenshots(messages: list[dict], keep_first: bool = True, keep_re
for idx, (container, i) in enumerate(refs):
if idx in keep:
continue
container[i] = {"type": "text", "text": _OMITTED_SCREENSHOT_STUB}
container[i] = {"type": "text", "text": OMITTED_SCREENSHOT_STUB}
collapsed += 1
return collapsed
@@ -97,9 +97,9 @@ def prune_old_screenshots(messages: list[dict], keep_first: bool = True, keep_re
# Sentinel prefixing the auto-attached element list on mutating action results.
# Lives here so the attacher (browser_agent) and the pruner share one spelling.
PAGE_STATE_MARKER = "[page state after action]"
_STATE_STUB = "[stale page state pruned; see the latest action result for current state]"
_HEAVY_READ_TOOLS = {"BrowserListInteractives", "BrowserGetText"}
_HEAVY_READ_MIN_CHARS = 600
P_STATE_STUB = "[stale page state pruned; see the latest action result for current state]"
P_HEAVY_READ_TOOLS = {"BrowserListInteractives", "BrowserGetText"}
P_HEAVY_READ_MIN_CHARS = 600
def prune_stale_page_state(messages: list[dict], keep_recent: int = 2) -> int:
@@ -136,12 +136,12 @@ def prune_stale_page_state(messages: list[dict], keep_recent: int = 2) -> int:
txt = ib.get("text") or ""
if PAGE_STATE_MARKER in txt:
attached.append(ib)
elif tool in _HEAVY_READ_TOOLS and len(txt) >= _HEAVY_READ_MIN_CHARS:
elif tool in P_HEAVY_READ_TOOLS and len(txt) >= P_HEAVY_READ_MIN_CHARS:
heavy.append(ib)
pruned = 0
for ib in attached[:-keep_recent] if keep_recent else attached:
txt = ib["text"]
ib["text"] = txt[: txt.index(PAGE_STATE_MARKER)] + _STATE_STUB
ib["text"] = txt[: txt.index(PAGE_STATE_MARKER)] + P_STATE_STUB
pruned += 1
for ib in heavy[:-keep_recent] if keep_recent else heavy:
head = (ib["text"] or "").splitlines()[0][:100]
@@ -175,7 +175,7 @@ def place_cache_marker(messages: list[dict], depth: int = 8) -> None:
return
def _validate_message_pairing(messages: list[dict]) -> bool:
def validate_message_pairing(messages: list[dict]) -> bool:
"""Verify tool_use and tool_result blocks pair up BOTH ways, or the cached
history 400s if sent to the API. Two failure shapes, both checked:
- an orphan tool_result (references a tool_use_id that was never declared), and
@@ -209,7 +209,7 @@ def _validate_message_pairing(messages: list[dict]) -> bool:
return declared_tool_use_ids.issubset(answered_tool_use_ids)
def _is_fresh_user_message(msg: dict) -> bool:
def p_is_fresh_user_message(msg: dict) -> bool:
"""A 'fresh' user message starts a new turn; string content or a list
that contains no tool_result blocks. These are the only safe cut points
because they don't reference any prior assistant tool_use blocks."""
@@ -225,7 +225,7 @@ def _is_fresh_user_message(msg: dict) -> bool:
return False
def _summarize_messages(messages: list[dict]) -> str:
def p_summarize_messages(messages: list[dict]) -> str:
"""Build a programmatic summary of older browser-agent messages.
Extracts the original user task, a count of tool calls by name with their
@@ -312,7 +312,7 @@ def _summarize_messages(messages: list[dict]) -> str:
return "\n".join(parts)
def _trim_history_by_turns(messages: list[dict], max_messages: int) -> list[dict]:
def trim_history_by_turns(messages: list[dict], max_messages: int) -> list[dict]:
"""Compact message history when it exceeds max_messages.
The Anthropic API requires every `tool_result` block to reference a
@@ -341,7 +341,7 @@ def _trim_history_by_turns(messages: list[dict], max_messages: int) -> list[dict
# First pass: walk forward looking for the EARLIEST clean cut point that
# gets us under the cap. This preserves the most recent detail.
for i in range(1, len(messages)):
if not _is_fresh_user_message(messages[i]):
if not p_is_fresh_user_message(messages[i]):
continue
if len(messages) - i <= target_tail_size:
cut_index = i
@@ -353,7 +353,7 @@ def _trim_history_by_turns(messages: list[dict], max_messages: int) -> list[dict
# safe history we can produce; and any compaction is better than none.
if cut_index is None:
for i in range(len(messages) - 1, 0, -1):
if _is_fresh_user_message(messages[i]):
if p_is_fresh_user_message(messages[i]):
cut_index = i
break
@@ -364,6 +364,6 @@ def _trim_history_by_turns(messages: list[dict], max_messages: int) -> list[dict
# Compact: summarize messages[0..cut_index-1], prepend as a single
# user-text message, then keep messages[cut_index..end] verbatim.
summary_text = _summarize_messages(messages[:cut_index])
summary_text = p_summarize_messages(messages[:cut_index])
summary_msg = {"role": "user", "content": summary_text}
return [summary_msg] + list(messages[cut_index:])
backend/apps/agents/browser/browser_loop.py
+46 -46
View File
@@ -13,7 +13,7 @@ import re
# Tools that are read-only / idempotent and should NOT count toward loop
# detection. Repeating these is normal (scrolling through a feed, taking
# successive screenshots, polling for an element to appear).
_LOOP_DETECTION_EXCLUDED_TOOLS = {
LOOP_DETECTION_EXCLUDED_TOOLS = {
"BrowserScreenshot",
"BrowserGetText",
"BrowserGetConsole", # read-only diagnostic; reading it repeatedly is fine
@@ -27,9 +27,9 @@ _LOOP_DETECTION_EXCLUDED_TOOLS = {
"BrowserRepeatFlow", # batch: drives its own verified per-item loop
}
_LOOP_WINDOW_SIZE = 5
_LOOP_REPEAT_THRESHOLD = 2 # the SECOND identical (tool,input,result) is already a wall
_LOOP_HARD_CAP = 5
LOOP_WINDOW_SIZE = 5
P_LOOP_REPEAT_THRESHOLD = 2 # the SECOND identical (tool,input,result) is already a wall
LOOP_HARD_CAP = 5
# Universal close-affordance vocabulary for blocking popups (cookie walls,
@@ -39,13 +39,13 @@ _LOOP_HARD_CAP = 5
# task required. Deliberately omits generic "Close"/"Dismiss"/"Skip", which DO
# appear on needed dialogs (e.g. "Close your conversation"). Keys on the pattern,
# not any one site, so it generalizes.
_DISMISS_NAMES = frozenset({
P_DISMISS_NAMES = frozenset({
"no thanks", "no, thanks", "maybe later", "not now", "skip for now",
"remind me later", "got it", "decline", "no, maybe later", "not interested",
})
# never dismiss anything that smells like security or a real decision
_DANGER_NAME_RE = re.compile(r"verif|confirm|2fa|password|sign|pay|delete|send|post|submit", re.I)
_ROW_RE = re.compile(r'<\s*([a-z]+)\s+"([^"]*)"', re.I) # matches a [i]<role "name"> row
P_DANGER_NAME_RE = re.compile(r"verif|confirm|2fa|password|sign|pay|delete|send|post|submit", re.I)
P_ROW_RE = re.compile(r'<\s*([a-z]+)\s+"([^"]*)"', re.I) # matches a [i]<role "name"> row
def interstitial_dismiss_target(interactives_text: str) -> str | None:
@@ -55,19 +55,19 @@ def interstitial_dismiss_target(interactives_text: str) -> str | None:
never anything with security/confirm/commit wording, so a mechanical dismiss
can never close a dialog the task actually required."""
for line in (interactives_text or "").splitlines():
m = _ROW_RE.search(line)
m = P_ROW_RE.search(line)
if not m:
continue
role, name = m.group(1).lower(), m.group(2).strip()
if role not in ("button", "link"):
continue
norm = re.sub(r"[^a-z, ]", "", name.lower()).strip()
if norm in _DISMISS_NAMES and not _DANGER_NAME_RE.search(name):
if norm in P_DISMISS_NAMES and not P_DANGER_NAME_RE.search(name):
return name
return None
def _hash_tool_call(tool_name: str, tool_input: dict, result: dict) -> tuple[str, str, str]:
def hash_tool_call(tool_name: str, tool_input: dict, result: dict) -> tuple[str, str, str]:
"""Build a stable hash key for a tool call, including its result.
Including the result hash means that legitimate progress (same input,
@@ -86,24 +86,24 @@ def _hash_tool_call(tool_name: str, tool_input: dict, result: dict) -> tuple[str
return (tool_name, input_key, result_key)
def _detect_loop(
def detect_loop(
recent_calls: list[tuple[str, str, str]],
new_call: tuple[str, str, str],
) -> bool:
"""Return True if `new_call` constitutes a loop given recent history.
A loop is when the same (tool, input, result) has appeared at least
`_LOOP_REPEAT_THRESHOLD` times within the last `_LOOP_WINDOW_SIZE`
`P_LOOP_REPEAT_THRESHOLD` times within the last `LOOP_WINDOW_SIZE`
state-mutating calls (the new call counts as one of those occurrences).
"""
if new_call[0] in _LOOP_DETECTION_EXCLUDED_TOOLS:
if new_call[0] in LOOP_DETECTION_EXCLUDED_TOOLS:
return False
window = recent_calls[-(_LOOP_WINDOW_SIZE - 1):] + [new_call]
window = recent_calls[-(LOOP_WINDOW_SIZE - 1):] + [new_call]
matches = sum(1 for c in window if c == new_call)
return matches >= _LOOP_REPEAT_THRESHOLD
return matches >= P_LOOP_REPEAT_THRESHOLD
_LOOP_WARNING_TEXT = (
LOOP_WARNING_TEXT = (
"LOOP DETECTED: the same action got the same result {count} times, so repeating "
"it will NOT help. Diagnose the REAL cause before anything else, do not assume: "
"read the exact error in the result; call BrowserGetConsole to see the page's own "
@@ -128,20 +128,20 @@ _LOOP_WARNING_TEXT = (
# Read-only / meta tools don't count toward stagnation (same exemption set as
# the loop detector): re-orienting is not "being stuck".
_STAGNATION_NEUTRAL_TOOLS = _LOOP_DETECTION_EXCLUDED_TOOLS
_STAGNATION_ESCALATION_AT = 3
_STAGNATION_MAX = 5
P_STAGNATION_NEUTRAL_TOOLS = LOOP_DETECTION_EXCLUDED_TOOLS
STAGNATION_ESCALATION_AT = 3
STAGNATION_MAX = 5
_FAILURE_MARKERS = (
P_FAILURE_MARKERS = (
"error", "not found", "no longer valid", "no box model",
"no valid bounding rect", "failed", "rejected", "timed out",
"could not", "unable to", "denied",
)
def _looks_like_failure(text: str) -> bool:
def looks_like_failure(text: str) -> bool:
low = text.lower()
return any(m in low for m in _FAILURE_MARKERS)
return any(m in low for m in P_FAILURE_MARKERS)
def is_unproductive(
@@ -156,7 +156,7 @@ def is_unproductive(
action, all with no URL change. Neutral tools (screenshot, get_text, etc.)
never count.
"""
if tool_name in _STAGNATION_NEUTRAL_TOOLS:
if tool_name in P_STAGNATION_NEUTRAL_TOOLS:
return False
new_url = str(result.get("url") or "")
if new_url and prev_url and new_url != prev_url:
@@ -164,14 +164,14 @@ def is_unproductive(
if "error" in result:
return True
text = str(result.get("text") or result.get("error") or "")
if _looks_like_failure(text):
if looks_like_failure(text):
return True
if prev_text and text[:200] == prev_text[:200]:
return True
return False
_STAGNATION_NUDGE = (
P_STAGNATION_NUDGE = (
"NO PROGRESS: your last {streak} actions changed nothing and looked like "
"failures. Before trying yet another variation, find out WHY: read the exact "
"errors; call BrowserGetConsole for the page's own JS/network errors; use "
@@ -185,8 +185,8 @@ _STAGNATION_NUDGE = (
def stagnation_nudge(streak: int) -> str:
base = _STAGNATION_NUDGE.format(streak=streak)
if streak >= _STAGNATION_MAX:
base = P_STAGNATION_NUDGE.format(streak=streak)
if streak >= STAGNATION_MAX:
base += (
" Switching selectors hasn't worked, so the PLAN itself is likely "
"wrong: step back and revise your overall approach (a different page, "
@@ -207,7 +207,7 @@ def advance_stagnation(
return a nudge string when the streak crosses an escalation threshold.
Returns (new_streak, new_prev_url, new_prev_text, nudge_or_None).
"""
if tool_name in _STAGNATION_NEUTRAL_TOOLS:
if tool_name in P_STAGNATION_NEUTRAL_TOOLS:
return streak, prev_url, prev_text, None
if is_unproductive(tool_name, result, prev_url, prev_text):
streak += 1
@@ -217,7 +217,7 @@ def advance_stagnation(
new_text = str(result.get("text") or result.get("error") or "")[:200]
nudge = (
stagnation_nudge(streak)
if streak in (_STAGNATION_ESCALATION_AT, _STAGNATION_MAX)
if streak in (STAGNATION_ESCALATION_AT, STAGNATION_MAX)
else None
)
return streak, new_url, new_text, nudge
@@ -226,7 +226,7 @@ def advance_stagnation(
def stagnation_exhausted(streak: int) -> bool:
"""True once deterministic nudging has been exhausted; the caller may then
escalate to a one-shot aux-LLM adjudication (see browser_validator)."""
return streak >= _STAGNATION_MAX
return streak >= STAGNATION_MAX
# --- completion honesty gate ----------------------------------------------
@@ -236,12 +236,12 @@ def stagnation_exhausted(streak: int) -> bool:
# status say "done", so a fake success is reported as the failure it actually is.
# State-changing tools: a task that needed to DO something must land one of these.
_PRODUCTIVE_TOOLS = {
P_PRODUCTIVE_TOOLS = {
"BrowserClick", "BrowserClickIndex", "BrowserType", "BrowserNavigate",
"BrowserPressKey", "BrowserScroll", "BrowserBatch",
}
# Read/extract tools: a look-only task's evidence is that a read returned content.
_READ_TOOLS = {
P_READ_TOOLS = {
"BrowserGetText", "BrowserGetElements", "BrowserListInteractives",
"BrowserListRoutes", "BrowserReplayRoute", "BrowserScreenshot", "BrowserEvaluate",
}
@@ -253,18 +253,18 @@ _READ_TOOLS = {
# 20-minute LinkedIn spin), so we fail fast. The streak (reset on any good result)
# absorbs a one-off transient; only a SUSTAINED pattern trips it, so a merely-busy
# page that recovers is never mistaken for dead.
_CARD_GONE_MARKERS = (
P_CARD_GONE_MARKERS = (
"not an electron webview", # card closed / destroyed
"no dashboard is connected", # dashboard view not mounted
"command timed out", # hung: the command never came back
"page unresponsive", # hung: smart-wait gave up probing the tab
)
_CARD_GONE_LIMIT = 2 # consecutive misses before we give up (absorbs a transient)
CARD_GONE_LIMIT = 2 # consecutive misses before we give up (absorbs a transient)
def card_is_unavailable(result: dict) -> bool:
err = str(result.get("error") or "").lower()
return any(m in err for m in _CARD_GONE_MARKERS)
return any(m in err for m in P_CARD_GONE_MARKERS)
# Errors where the action MISSED but the page is alive (stale index after a
@@ -272,7 +272,7 @@ def card_is_unavailable(result: dict) -> bool:
# itself is fine, so re-attaching the CURRENT element list to the error lets the
# model re-act next turn instead of burning a turn re-listing. This NEVER retries
# the action (no double-send risk); it only enriches the error with fresh state.
_RECOVERABLE_ERR_MARKERS = (
P_RECOVERABLE_ERR_MARKERS = (
"no longer valid", "no node with given id", "page may have changed",
"covered it", "obscured", "intercepted", "not clickable",
"box model", "try scrolling", "not visible",
@@ -283,15 +283,15 @@ def recoverable_tool_error(err: str) -> bool:
"""True for a 'the action missed but the page is alive' error worth showing
fresh state for. False for a dead card (handled separately) or no error."""
e = (err or "").lower()
if not e or any(m in e for m in _CARD_GONE_MARKERS):
if not e or any(m in e for m in P_CARD_GONE_MARKERS):
return False
return any(m in e for m in _RECOVERABLE_ERR_MARKERS)
return any(m in e for m in P_RECOVERABLE_ERR_MARKERS)
# Actions that DIRTY the page so replay-from-here is no longer equivalent to a
# clean dispatch. Navigation and reads don't dirty anything (they just get us to
# the page), so the deferred replay re-check is allowed after only those.
_REPLAY_DIRTYING_TOOLS = {
P_REPLAY_DIRTYING_TOOLS = {
"BrowserType", "BrowserClick", "BrowserClickIndex",
"BrowserPressKey", "BrowserScroll", "BrowserBatch",
}
@@ -301,18 +301,18 @@ def replay_recheck_is_safe(action_log: list[dict]) -> bool:
"""True if nothing in the run so far has mutated page state, so switching to
a learned-skill replay now is equivalent to replaying from a clean dispatch
(the agent only navigated / looked around to get to the right host)."""
return not any(a.get("tool") in _REPLAY_DIRTYING_TOOLS for a in action_log)
return not any(a.get("tool") in P_REPLAY_DIRTYING_TOOLS for a in action_log)
# What the user ASKED FOR outranks how the sub narrated it: an info ask can
# never replay (the answer must be fresh), an action ask can.
_INFO_ASK_RE = re.compile(
P_INFO_ASK_RE = re.compile(
r"\b(tell me|what(?:'s| is| are)|how (?:many|much)|count|list|summari[sz]e|"
r"extract|find (?:me|out)|show me|look up|read (?:me|the)|get the|give me|which|"
r"who (?:is|are)|report back|most (?:viewed|popular|liked|recent|rated|watched)|top \d+)\b",
re.I,
)
_ACTION_ASK_RE = re.compile(
P_ACTION_ASK_RE = re.compile(
r"\b(open|go to|navigate|click|send|post|submit|fill|type|search for|log ?in|"
r"sign ?in|upload|download|book|order|buy|add|create|delete|message|dm|text)\b",
re.I,
@@ -333,9 +333,9 @@ def deliverable_is_informational(summary: str, task: str = "") -> bool:
(re-run via the LLM), never a ghost completion."""
t = (task or "").strip()
if t:
if _INFO_ASK_RE.search(t):
if P_INFO_ASK_RE.search(t):
return True
if _ACTION_ASK_RE.search(t):
if P_ACTION_ASK_RE.search(t):
return False
s = (summary or "").strip()
s = re.sub(r"OUTCOME:.*$", "", s, flags=re.S).strip()
@@ -358,11 +358,11 @@ def completion_is_honest(action_log: list[dict]) -> tuple[bool, str]:
"""
if not action_log:
return False, "declared done without taking a single action"
actions = [a for a in action_log if a.get("tool") in _PRODUCTIVE_TOOLS]
actions = [a for a in action_log if a.get("tool") in P_PRODUCTIVE_TOOLS]
actions_ok = [a for a in actions if a.get("ok")]
reads_ok = [
a for a in action_log
if a.get("tool") in _READ_TOOLS and a.get("ok")
if a.get("tool") in P_READ_TOOLS and a.get("ok")
and str(a.get("result_summary") or "").strip()
]
if actions and not actions_ok:
backend/apps/agents/browser/browser_meta_playbook.py
+29 -29
View File
@@ -21,18 +21,18 @@ import os
import tempfile
import time
from backend.apps.agents.browser.browser_playbook import _clean_bullet
from backend.apps.agents.browser.browser_playbook import clean_bullet
logger = logging.getLogger(__name__)
_VERSION = 1
_MAX_BULLETS = 10 # a touch larger than per-site: these earn their keep everywhere
_FILE = "meta_playbook.json"
P_VERSION = 1
MAX_BULLETS = 10 # a touch larger than per-site: these earn their keep everywhere
P_FILE = "meta_playbook.json"
_cache: list[str] | None = None
CACHE: list[str] | None = None
def _dir() -> str | None:
def p_dir() -> str | None:
base = os.environ.get("OPENSWARM_BROWSER_META_DIR")
if not base:
try:
@@ -47,34 +47,34 @@ def _dir() -> str | None:
return base
def _path() -> str | None:
d = _dir()
return os.path.join(d, _FILE) if d else None
def p_path() -> str | None:
d = p_dir()
return os.path.join(d, P_FILE) if d else None
def _load() -> list[str]:
path = _path()
def load() -> list[str]:
path = p_path()
if not path or not os.path.exists(path):
return []
try:
with open(path, encoding="utf-8") as f:
data = json.load(f)
if data.get("version") != _VERSION:
if data.get("version") != P_VERSION:
return []
return [b for b in (data.get("bullets") or []) if isinstance(b, str)]
except Exception:
return []
def _persist(bullets: list[str]) -> None:
path = _path()
def persist(bullets: list[str]) -> None:
path = p_path()
if not path:
return
try:
d = os.path.dirname(path)
fd, tmp = tempfile.mkstemp(dir=d, suffix=".tmp")
with os.fdopen(fd, "w", encoding="utf-8") as f:
json.dump({"version": _VERSION, "bullets": bullets, "updated_at": time.time()}, f)
json.dump({"version": P_VERSION, "bullets": bullets, "updated_at": time.time()}, f)
os.replace(tmp, path) # atomic
except Exception as e:
logger.debug(f"[browser-meta] persist failed: {e}")
@@ -82,10 +82,10 @@ def _persist(bullets: list[str]) -> None:
def get_meta() -> list[str]:
"""The cross-site bullets (cheap, no LLM). Cached after first read."""
global _cache
if _cache is None:
_cache = _load() or list(_SEED)
return _cache
global CACHE
if CACHE is None:
CACHE = load() or list(P_SEED)
return CACHE
def format_for_prompt() -> str:
@@ -94,7 +94,7 @@ def format_for_prompt() -> str:
bullets = get_meta()
if not bullets:
return ""
lines = "\n".join(f"- {b}" for b in bullets[:_MAX_BULLETS])
lines = "\n".join(f"- {b}" for b in bullets[:MAX_BULLETS])
return (
"\n\n## General web priors (learned across many sites, verify against THIS page)\n"
+ lines
@@ -113,26 +113,26 @@ def absorb(universal_bullets: list[str]) -> bool:
truly_new: list[str] = []
seen: set[str] = set()
for b in universal_bullets:
cb = _clean_bullet(b)
cb = clean_bullet(b)
if cb and cb.lower() not in existing_lower and cb.lower() not in seen:
seen.add(cb.lower())
truly_new.append(cb)
if not truly_new:
return False
merged = (truly_new + existing)[:_MAX_BULLETS]
global _cache
_cache = merged
_persist(merged)
merged = (truly_new + existing)[:MAX_BULLETS]
global CACHE
CACHE = merged
persist(merged)
logger.info(f"[browser-meta] {len(merged)} cross-site prior(s) (was {len(existing)})")
return True
def clear(wipe_disk: bool = False) -> None:
"""Test/maintenance reset of the in-memory cache (and optionally disk)."""
global _cache
_cache = None
global CACHE
CACHE = None
if wipe_disk:
path = _path()
path = p_path()
if path and os.path.exists(path):
try:
os.remove(path)
@@ -142,7 +142,7 @@ def clear(wipe_disk: bool = False) -> None:
# Shipped starting priors: the hard-won universal lessons from this codebase's own
# browser work, so tier 3 is useful on day one and accrues more as sites confirm them.
_SEED = (
P_SEED = (
"A message composer CLEARS when the send goes through; the empty box IS your "
"confirmation, do not hunt the thread for the sent text to 'verify'.",
"An opener (Message/DM/Compose) only OPENS the box and is reversible; only the "
backend/apps/agents/browser/browser_metrics.py
+26 -26
View File
@@ -30,7 +30,7 @@ logger = logging.getLogger(__name__)
# Map each tool to the waterfall tier it represents, so per-tier speed/cost
# rolls up cleanly. Control/meta tools are their own bucket.
_TIER = {
P_TIER = {
"BrowserDetectWebMCP": "t1_webmcp",
"BrowserListRoutes": "t2_route_list",
"BrowserReplayRoute": "t2_route_replay",
@@ -54,17 +54,17 @@ _TIER = {
def tier_for(tool_name: str) -> str:
return _TIER.get(tool_name, "other")
return P_TIER.get(tool_name, "other")
_metrics_dir_cache: str | None = None
p_metrics_dir_cache: str | None = None
def _metrics_dir() -> str:
def metrics_dir() -> str:
# Resolved + mkdir'd once, not on every tool call (this runs in the hot path).
global _metrics_dir_cache
if _metrics_dir_cache is not None:
return _metrics_dir_cache
global p_metrics_dir_cache
if p_metrics_dir_cache is not None:
return p_metrics_dir_cache
override = os.environ.get("OPENSWARM_BROWSER_METRICS_DIR")
if override:
base = override
@@ -79,13 +79,13 @@ def _metrics_dir() -> str:
os.makedirs(base, mode=0o700, exist_ok=True)
except Exception:
pass
_metrics_dir_cache = base
p_metrics_dir_cache = base
return base
def _append(filename: str, obj: dict) -> None:
def p_append(filename: str, obj: dict) -> None:
try:
path = os.path.join(_metrics_dir(), filename)
path = os.path.join(metrics_dir(), filename)
# owner-only: these lines can carry task text and error snippets
fd = os.open(path, os.O_APPEND | os.O_CREAT | os.O_WRONLY, 0o600)
with os.fdopen(fd, "a", encoding="utf-8") as f:
@@ -97,22 +97,22 @@ def _append(filename: str, obj: dict) -> None:
# A task prompt can carry a literal secret ("log in with password hunter2");
# scrub the value before it lands in tasks.jsonl. Keyword+value and known
# token prefixes only; the task's normal words stay greppable.
_TASK_SECRET_RE = re.compile(
P_TASK_SECRET_RE = re.compile(
r"\b(password|passcode|passphrase|pin|otp|token|secret|api[_-]?key)\b\s*(?:is|[:=])?\s*\S+",
re.I,
)
_TASK_TOKEN_RE = re.compile(r"\b(sk-|ghp_|gho_|pk_|xox[bap]-|AIza|eyJ)[A-Za-z0-9_\-.]{8,}")
P_TASK_TOKEN_RE = re.compile(r"\b(sk-|ghp_|gho_|pk_|xox[bap]-|AIza|eyJ)[A-Za-z0-9_\-.]{8,}")
def _scrub_task(task: str) -> str:
t = _TASK_SECRET_RE.sub(lambda m: f"{m.group(1)} [redacted]", task or "")
return _TASK_TOKEN_RE.sub("[redacted]", t)
def p_scrub_task(task: str) -> str:
t = P_TASK_SECRET_RE.sub(lambda m: f"{m.group(1)} [redacted]", task or "")
return P_TASK_TOKEN_RE.sub("[redacted]", t)
def record_tool(session_id, browser_id, turn, tool, elapsed_ms, ok, error,
is_loop, stagnation_streak, result_len) -> None:
"""One line per executed tool call. Best-effort."""
_append("events.jsonl", {
p_append("events.jsonl", {
"ts": time.time(),
"session_id": session_id,
"browser_id": browser_id,
@@ -140,7 +140,7 @@ def record_skill_event(kind, host, task_sig, rev=0, state="", extra=None) -> Non
learn / edit / promote / quarantine / demote / compose / invalidate. This is
what lets the analyzer prove the skill layer is helping (promotes accumulate,
repeats replay) vs. silently thrashing (re-learn loops, never promotes)."""
_append("skill_events.jsonl", {
p_append("skill_events.jsonl", {
"ts": time.time(), "kind": kind, "host": host, "task_sig": task_sig,
"rev": rev, "state": state, "extra": extra or {},
})
@@ -172,7 +172,7 @@ def record_task(session_id, browser_id, task, status, started_at, turns,
"ts": time.time(),
"session_id": session_id,
"browser_id": browser_id,
"task": _scrub_task(task)[:200],
"task": p_scrub_task(task)[:200],
"task_sig": task_sig,
"path": path,
"playbook_seeded": bool(playbook_seeded),
@@ -186,27 +186,27 @@ def record_task(session_id, browser_id, task, status, started_at, turns,
"by_tier": by_tier,
"recurring_errors": err_counter.most_common(5),
}
_append("tasks.jsonl", summary)
p_append("tasks.jsonl", summary)
logger.info(
f"[browser-metrics] TASK {status} path={path} total={total_ms}ms turns={turns} "
f"tools={len(action_log)} tok_in={summary['tokens_in']} tok_out={summary['tokens_out']} "
f"recurring_errs={summary['recurring_errors'][:2]}"
)
_maybe_self_audit()
p_maybe_self_audit()
return summary
_AUDIT_EVERY_N = 25 # refresh the learning self-audit roughly this often
_task_count = 0
P_AUDIT_EVERY_N = 25 # refresh the learning self-audit roughly this often
p_task_count = 0
def _maybe_self_audit() -> None:
def p_maybe_self_audit() -> None:
"""Every N finished tasks, refresh the self-audit report in a daemon thread so
it never adds latency to a run (the audit is ~3ms but stays off the hot path).
Proposal-only: it writes a report a human reads, it changes nothing."""
global _task_count
_task_count += 1
if _task_count % _AUDIT_EVERY_N != 0:
global p_task_count
p_task_count += 1
if p_task_count % P_AUDIT_EVERY_N != 0:
return
def _run():
backend/apps/agents/browser/browser_playbook.py
+49 -49
View File
@@ -38,18 +38,18 @@ from backend.apps.agents.browser.seed_playbooks import seed_for
logger = logging.getLogger(__name__)
_PLAYBOOK_FORMAT_VERSION = 1
_MAX_BULLETS = 8 # cap per site; reconcile keeps the most useful
_MAX_BULLET_CHARS = 160
_MAX_DISK_PLAYBOOKS = 500
_MIN_TURNS_TO_LEARN = 4 # a 1-3 turn run taught nothing worth a durable bullet
P_PLAYBOOK_FORMAT_VERSION = 1
MAX_BULLETS = 8 # cap per site; reconcile keeps the most useful
P_MAX_BULLET_CHARS = 160
P_MAX_DISK_PLAYBOOKS = 500
P_MIN_TURNS_TO_LEARN = 4 # a 1-3 turn run taught nothing worth a durable bullet
# In-memory hot cache: host -> list[str] bullets.
_cache: dict[str, list[str]] = {}
CACHE: dict[str, list[str]] = {}
# Same sensitivity guard the skill layer uses: a strategy bullet must never carry
# a secret (email/token/etc.). We scrub bullets through this before persisting.
_SECRET_RE = re.compile(
P_SECRET_RE = re.compile(
r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}" # email
r"|\b(sk-|ghp_|gho_|pk_|xox[bap]-|AIza|eyJ)[A-Za-z0-9._-]+" # token prefixes
r"|\b(?:\d[ -]?){13,19}\b" # card-ish
@@ -64,19 +64,19 @@ def host_of(url: str) -> str:
return ""
def _has_secret(text: str) -> bool:
return bool(_SECRET_RE.search(text or ""))
def p_has_secret(text: str) -> bool:
return bool(P_SECRET_RE.search(text or ""))
def _clean_bullet(b: str) -> str | None:
def clean_bullet(b: str) -> str | None:
b = re.sub(r"\s+", " ", str(b or "")).strip().lstrip("-*• ").strip()
if not b or _has_secret(b):
if not b or p_has_secret(b):
return None
return b[:_MAX_BULLET_CHARS]
return b[:P_MAX_BULLET_CHARS]
# --- persistence (mirrors browser_skills, separate dir) -------------------
def _dir() -> str | None:
def p_dir() -> str | None:
base = os.environ.get("OPENSWARM_BROWSER_PLAYBOOK_DIR")
if not base:
try:
@@ -91,20 +91,20 @@ def _dir() -> str | None:
return base
def _path(host: str) -> str | None:
def p_path(host: str) -> str | None:
import hashlib
d = _dir()
d = p_dir()
if not d:
return None
h = hashlib.sha256(host.encode("utf-8")).hexdigest()[:32]
return os.path.join(d, f"{h}.json")
def _persist(host: str, bullets: list[str]) -> None:
path = _path(host)
def persist(host: str, bullets: list[str]) -> None:
path = p_path(host)
if not path:
return
payload = {"version": _PLAYBOOK_FORMAT_VERSION, "host": host,
payload = {"version": P_PLAYBOOK_FORMAT_VERSION, "host": host,
"bullets": bullets, "updated_at": time.time()}
try:
d = os.path.dirname(path)
@@ -112,18 +112,18 @@ def _persist(host: str, bullets: list[str]) -> None:
with os.fdopen(fd, "w", encoding="utf-8") as f:
json.dump(payload, f)
os.replace(tmp, path) # atomic; a reader never sees a half-written file
_evict_if_over_cap(d)
p_evict_if_over_cap(d)
except Exception as e:
logger.debug(f"[browser-playbook] persist failed: {e}")
def _evict_if_over_cap(d: str) -> None:
def p_evict_if_over_cap(d: str) -> None:
try:
files = [os.path.join(d, f) for f in os.listdir(d) if f.endswith(".json")]
if len(files) <= _MAX_DISK_PLAYBOOKS:
if len(files) <= P_MAX_DISK_PLAYBOOKS:
return
files.sort(key=lambda p: os.path.getmtime(p))
for p in files[: len(files) - _MAX_DISK_PLAYBOOKS]:
for p in files[: len(files) - P_MAX_DISK_PLAYBOOKS]:
try:
os.remove(p)
except Exception:
@@ -132,14 +132,14 @@ def _evict_if_over_cap(d: str) -> None:
pass
def _load(host: str) -> list[str]:
path = _path(host)
def load(host: str) -> list[str]:
path = p_path(host)
if not path or not os.path.exists(path):
return []
try:
with open(path, encoding="utf-8") as f:
data = json.load(f)
if data.get("version") != _PLAYBOOK_FORMAT_VERSION:
if data.get("version") != P_PLAYBOOK_FORMAT_VERSION:
return []
return [b for b in (data.get("bullets") or []) if isinstance(b, str)]
except Exception:
@@ -155,10 +155,10 @@ def get_playbook(host: str) -> list[str]:
writes a learned playbook that supersedes it."""
if not host:
return []
if host in _cache:
return _cache[host]
bullets = _load(host) or seed_for(host)
_cache[host] = bullets
if host in CACHE:
return CACHE[host]
bullets = load(host) or seed_for(host)
CACHE[host] = bullets
return bullets
@@ -167,7 +167,7 @@ def format_for_prompt(host: str) -> str:
bullets = get_playbook(host)
if not bullets:
return ""
lines = "\n".join(f"- {b}" for b in bullets[:_MAX_BULLETS])
lines = "\n".join(f"- {b}" for b in bullets[:MAX_BULLETS])
return (
f"\n\n## What you learned about {host} on past visits\n"
"Use these as a head start to skip re-discovery, but re-verify since the "
@@ -175,19 +175,19 @@ def format_for_prompt(host: str) -> str:
)
def _store(host: str, bullets: list[str]) -> list[str]:
def p_store(host: str, bullets: list[str]) -> list[str]:
"""Clean, cap, persist, warm cache. Returns the stored list."""
cleaned: list[str] = []
seen = set()
for b in bullets:
cb = _clean_bullet(b)
cb = clean_bullet(b)
if cb and cb.lower() not in seen:
seen.add(cb.lower())
cleaned.append(cb)
if len(cleaned) >= _MAX_BULLETS:
if len(cleaned) >= MAX_BULLETS:
break
_cache[host] = cleaned
_persist(host, cleaned)
CACHE[host] = cleaned
persist(host, cleaned)
return cleaned
@@ -195,10 +195,10 @@ def _store(host: str, bullets: list[str]) -> list[str]:
def should_learn(honest: bool, turns: int) -> bool:
"""Only learn from a verified, substantive success: a ghost teaches nothing,
and a 1-3 turn run has no durable site strategy worth a bullet."""
return bool(honest) and turns >= _MIN_TURNS_TO_LEARN
return bool(honest) and turns >= P_MIN_TURNS_TO_LEARN
def _build_prompt(host: str, task: str, working_memory: str, summary: str,
def p_build_prompt(host: str, task: str, working_memory: str, summary: str,
existing: list[str]) -> str:
ex = "\n".join(f"{i+1}. {b}" for i, b in enumerate(existing)) or "(empty)"
return (
@@ -214,7 +214,7 @@ def _build_prompt(host: str, task: str, working_memory: str, summary: str,
"lessons that are SITE-AGNOSTIC (true on ANY website, e.g. how composers/Send "
"buttons behave in general), so other sites can reuse them. `universal` may be "
"empty; never put site-specific URLs, selectors, or names in it. Rules:\n"
f"- At most {_MAX_BULLETS} bullets, each under {_MAX_BULLET_CHARS} chars, "
f"- At most {MAX_BULLETS} bullets, each under {P_MAX_BULLET_CHARS} chars, "
"atomic and REUSABLE for ANY task on this site.\n"
"- Keep only durable site strategy: which queries/filters/URLs work, what "
"to avoid, where things live, walls that are safe to ignore.\n"
@@ -234,7 +234,7 @@ def _build_prompt(host: str, task: str, working_memory: str, summary: str,
)
def _parse(text: str) -> list[str] | None:
def p_parse(text: str) -> list[str] | None:
"""Pull the bullet list out of the aux reply. Tolerant of code fences/prose."""
if not text:
return None
@@ -251,7 +251,7 @@ def _parse(text: str) -> list[str] | None:
return [str(x) for x in pb if isinstance(x, (str, int, float))]
def _parse_universal(text: str) -> list[str]:
def p_parse_universal(text: str) -> list[str]:
"""The site-agnostic subset the distill flagged, for the cross-site meta-playbook.
Tolerant: missing/garbled `universal` just yields nothing (the site distill still runs)."""
if not text:
@@ -279,22 +279,22 @@ async def distill_and_store(host, task, working_memory, summary,
if not host or not aux_client or not aux_model:
return False
existing = get_playbook(host)
prompt = _build_prompt(host, task or "", working_memory or "", summary or "", existing)
prompt = p_build_prompt(host, task or "", working_memory or "", summary or "", existing)
resp = await aux_client.messages.create(
model=aux_model, max_tokens=600,
messages=[{"role": "user", "content": prompt}],
)
text = "".join(getattr(b, "text", "") for b in (resp.content or []))
new_bullets = _parse(text)
new_bullets = p_parse(text)
if new_bullets is None:
return False
stored = _store(host, new_bullets)
stored = p_store(host, new_bullets)
changed = stored != existing
# Fold any site-agnostic lessons into the cross-site meta-playbook (no extra
# LLM call, they rode along in this same reply). Best-effort, never fatal.
try:
from backend.apps.agents.browser import browser_meta_playbook
browser_meta_playbook.absorb(_parse_universal(text))
browser_meta_playbook.absorb(p_parse_universal(text))
except Exception:
pass
if changed:
@@ -310,7 +310,7 @@ async def distill_and_store(host, task, working_memory, summary,
def list_hosts() -> list[dict]:
"""Every site we have a playbook for, for a 'what has it learned' view."""
out = []
d = _dir()
d = p_dir()
if d:
try:
for f in os.listdir(d):
@@ -333,8 +333,8 @@ def forget(host: str) -> bool:
"""User-facing: drop a site's learned strategy (it re-learns next success)."""
if not host:
return False
_cache.pop(host, None)
path = _path(host)
CACHE.pop(host, None)
path = p_path(host)
if path and os.path.exists(path):
try:
os.remove(path)
@@ -347,9 +347,9 @@ def forget(host: str) -> bool:
def clear(wipe_disk: bool = False) -> None:
"""Clear the in-memory cache (tests). With wipe_disk, also remove files."""
_cache.clear()
CACHE.clear()
if wipe_disk:
d = _dir()
d = p_dir()
if d:
try:
for f in os.listdir(d):
backend/apps/agents/browser/browser_save.py
+10 -10
View File
@@ -10,15 +10,15 @@ workspace subdir. Returns a short receipt for the model, never the data itself.
import json
import os
_MAX_BYTES = 25 * 1024 * 1024 # a page can't realistically hold more scraped data
_ALLOWED_EXT = {".json", ".ndjson", ".csv", ".tsv", ".txt", ".md"}
_SUBDIR = "browser-data" # never the workspace root, so we can't clobber project files
MAX_BYTES = 25 * 1024 * 1024 # a page can't realistically hold more scraped data
ALLOWED_EXT = {".json", ".ndjson", ".csv", ".tsv", ".txt", ".md"}
SUBDIR = "browser-data" # never the workspace root, so we can't clobber project files
def _dest_dir(cwd: str | None, session_id: str) -> str:
def p_dest_dir(cwd: str | None, session_id: str) -> str:
base = cwd if (cwd and os.path.isdir(cwd)) else os.path.join(
os.path.expanduser("~"), ".openswarm", "workspaces", session_id or "browser")
dest = os.path.join(base, _SUBDIR)
dest = os.path.join(base, SUBDIR)
os.makedirs(dest, exist_ok=True)
return dest
@@ -31,15 +31,15 @@ def save_page_data(cwd: str | None, session_id: str, filename: str, content: str
if not name:
return "Save failed: give a plain filename like results.json."
ext = os.path.splitext(name)[1].lower()
if ext not in _ALLOWED_EXT:
if ext not in ALLOWED_EXT:
return (f"Save failed: '{ext or 'no extension'}' isn't allowed; this tool is for data, "
f"not code. Use one of: {', '.join(sorted(_ALLOWED_EXT))}.")
f"not code. Use one of: {', '.join(sorted(ALLOWED_EXT))}.")
body = content or ""
if len(body.encode("utf-8", "ignore")) > _MAX_BYTES:
return f"Save failed: that's over the {_MAX_BYTES // (1024 * 1024)}MB cap; save fewer fields or rows."
if len(body.encode("utf-8", "ignore")) > MAX_BYTES:
return f"Save failed: that's over the {MAX_BYTES // (1024 * 1024)}MB cap; save fewer fields or rows."
try:
dest_dir = _dest_dir(cwd, session_id)
dest_dir = p_dest_dir(cwd, session_id)
dest_real = os.path.realpath(dest_dir)
full = os.path.realpath(os.path.join(dest_dir, name))
# realpath + os.sep guard: defeats traversal, absolute paths, symlinks, AND a
backend/apps/agents/browser/browser_schema.py
+10 -10
View File
@@ -10,7 +10,7 @@ purpose because it is one cohesive data blob, not multiple responsibilities.
# (no prose beside action tools; ReportProgress IS the thinking) cut per-turn output
# ~28% and roughly halved narration turns. MERGE_VERIFY (a confirmed `expect` is the
# proof, skip the re-check) drops a wasted round-trip at the end.
_THINK_SHORTER = (
P_THINK_SHORTER = (
"Do NOT write a free-text sentence next to your action tools: your ReportProgress "
"fields ARE your thinking, and a separate prose explanation just repeats them and slows "
"the turn. Don't narrate to the user as you go either. When the task is done you finish by "
@@ -18,7 +18,7 @@ _THINK_SHORTER = (
"tools, no prose.\n"
)
_MERGE_VERIFY = (
P_MERGE_VERIFY = (
"When that `expect` CONFIRMS (the result says 'Confirmed: ...'), that IS your "
"verification: go STRAIGHT to calling Done. Do NOT spend an "
"extra screenshot or read turn to re-check what the confirmation already proved, that "
@@ -35,7 +35,7 @@ MODEL_MAP = {
# The change an action should cause, declared by the agent and CONFIRMED after the
# action runs (success is observed, never assumed). A hit returns fast; a miss tells
# the agent it may not have worked instead of letting it claim a false success.
_EXPECT_DESC = {
P_EXPECT_DESC = {
"type": "string",
"description": (
"Optional but recommended: LITERAL text that should be VISIBLE on the page "
@@ -264,7 +264,7 @@ BROWSER_TOOLS_SCHEMA = [
"type": "object",
"properties": {
"selector": {"type": "string", "description": "CSS selector of the element to click."},
"expect": _EXPECT_DESC,
"expect": P_EXPECT_DESC,
},
"required": ["selector"],
},
@@ -387,7 +387,7 @@ BROWSER_TOOLS_SCHEMA = [
"to fill a compose/message box reliably."
),
},
"expect": _EXPECT_DESC,
"expect": P_EXPECT_DESC,
},
"required": ["index"],
},
@@ -653,8 +653,8 @@ BROWSER_TOOLS_SCHEMA = [
# tools are not offered to it at all; acting means a BrowserBatch array, and
# the one deliberate solo path is BrowserClickIndex (irreversible step with
# expect, or a text-box fill). Executors and replay still support everything.
_SOLO_MUTATORS_HIDDEN = {"BrowserNavigate", "BrowserClick", "BrowserType", "BrowserScroll", "BrowserPressKey"}
MODEL_VISIBLE_TOOLS = [t for t in BROWSER_TOOLS_SCHEMA if t["name"] not in _SOLO_MUTATORS_HIDDEN]
P_SOLO_MUTATORS_HIDDEN = {"BrowserNavigate", "BrowserClick", "BrowserType", "BrowserScroll", "BrowserPressKey"}
MODEL_VISIBLE_TOOLS = [t for t in BROWSER_TOOLS_SCHEMA if t["name"] not in P_SOLO_MUTATORS_HIDDEN]
ACTION_MAP = {
"BrowserScreenshot": "screenshot",
@@ -720,7 +720,7 @@ SYSTEM_PROMPT = (
"needs (the exact selector, index, or value). Each token you write is generated one at a "
"time and is the main thing that slows a turn, so write the fewest that still carry the "
"plan forward. Only write working_memory when you learn something NEW this turn; else 'none'.\n"
+ _THINK_SHORTER +
+ P_THINK_SHORTER +
"Emit ReportProgress and your action tool(s) together in the same response. "
"If you skip ReportProgress, your action tools will be REJECTED with an error "
"and you will have to retry. This is not optional. Read-only tools "
@@ -751,7 +751,7 @@ SYSTEM_PROMPT = (
"inspections, or screenshots just because the post-type state only re-lists the rows "
"that CHANGED, the unchanged Send is still there at its number. If clicking it comes "
"back 'NOT confirmed', only THEN re-list to find where it moved.\n"
+ _MERGE_VERIFY + "\n"
+ P_MERGE_VERIFY + "\n"
"## Loop awareness\n"
"If you see a tool result containing 'LOOP DETECTED' or '⚠️', it means you "
@@ -908,7 +908,7 @@ MAX_TURNS = 40
# Tools that count as "action tools"; calling any of these in a turn requires
# the model to also call ReportProgress in the same turn (after the first
# turn). Read-only tools and meta tools are exempt.
_ACTION_TOOLS_REQUIRING_REPORT = {
ACTION_TOOLS_REQUIRING_REPORT = {
"BrowserClick",
"BrowserType",
"BrowserNavigate",
backend/apps/agents/browser/browser_self_audit.py
+18 -18
View File
@@ -21,13 +21,13 @@ from collections import defaultdict
logger = logging.getLogger(__name__)
# Thresholds for flagging; conservative so the report stays signal, not noise.
_THRASH_MIN_RELEARNS = 3 # a skill re-versioned this many times with 0 replays = stuck
_STALL_TURN_FACTOR = 2.0 # a run 2x the host's median turns is a stall worth noting
_MIN_RUNS_FOR_MEDIAN = 4 # don't call a "norm" from too few runs
_ERROR_RATE_FLAG = 0.25 # >25% of a host's tool calls erroring = something systemic
P_THRASH_MIN_RELEARNS = 3 # a skill re-versioned this many times with 0 replays = stuck
P_STALL_TURN_FACTOR = 2.0 # a run 2x the host's median turns is a stall worth noting
P_MIN_RUNS_FOR_MEDIAN = 4 # don't call a "norm" from too few runs
P_ERROR_RATE_FLAG = 0.25 # >25% of a host's tool calls erroring = something systemic
def _read_jsonl(path: str, cap: int = 20000) -> list[dict]:
def p_read_jsonl(path: str, cap: int = 20000) -> list[dict]:
out: list[dict] = []
if not path or not os.path.exists(path):
return out
@@ -48,7 +48,7 @@ def _read_jsonl(path: str, cap: int = 20000) -> list[dict]:
return out
def _median(xs: list[float]) -> float:
def p_median(xs: list[float]) -> float:
s = sorted(xs)
n = len(s)
if not n:
@@ -59,8 +59,8 @@ def _median(xs: list[float]) -> float:
def audit(metrics_dir: str) -> dict:
"""Read the metrics + skill events and return a structured findings dict.
Pure read; safe to call anytime. The caller renders/persists it."""
tasks = _read_jsonl(os.path.join(metrics_dir, "tasks.jsonl"))
skill_events = _read_jsonl(os.path.join(metrics_dir, "skill_events.jsonl"))
tasks = p_read_jsonl(os.path.join(metrics_dir, "tasks.jsonl"))
skill_events = p_read_jsonl(os.path.join(metrics_dir, "skill_events.jsonl"))
findings: list[dict] = []
# 1) THRASH: a skill re-versioned (edit) or sent to quarantine many times but
@@ -76,7 +76,7 @@ def audit(metrics_dir: str) -> dict:
elif kind == "promote":
promotes[key] += 1
for key, n in churn.items():
if n >= _THRASH_MIN_RELEARNS and promotes.get(key, 0) == 0:
if n >= P_THRASH_MIN_RELEARNS and promotes.get(key, 0) == 0:
findings.append({
"kind": "thrash",
"host": key[0],
@@ -89,19 +89,19 @@ def audit(metrics_dir: str) -> dict:
# 2) STALL: runs far above the host's median turn count.
by_host_turns: dict[str, list[int]] = defaultdict(list)
for t in tasks:
h = _host_of_task(t)
h = p_host_of_task(t)
if t.get("turns"):
by_host_turns[h].append(int(t["turns"]))
for h, turns in by_host_turns.items():
if len(turns) < _MIN_RUNS_FOR_MEDIAN:
if len(turns) < P_MIN_RUNS_FOR_MEDIAN:
continue
med = _median([float(x) for x in turns])
stalls = [x for x in turns if med and x >= med * _STALL_TURN_FACTOR]
med = p_median([float(x) for x in turns])
stalls = [x for x in turns if med and x >= med * P_STALL_TURN_FACTOR]
if stalls:
findings.append({
"kind": "stall",
"host": h,
"detail": f"{len(stalls)} run(s) at >= {_STALL_TURN_FACTOR}x the median "
"detail": f"{len(stalls)} run(s) at >= {P_STALL_TURN_FACTOR}x the median "
f"{med:.0f} turns (worst {max(stalls)})",
"suggestion": "a few runs spike well above normal, likely a perception/verify "
"loop or an env hang; check whether a prompt prior or mechanical "
@@ -112,13 +112,13 @@ def audit(metrics_dir: str) -> dict:
err = defaultdict(int)
tot = defaultdict(int)
for t in tasks:
h = _host_of_task(t)
h = p_host_of_task(t)
rc = t.get("recurring_errors") or {}
tot[h] += int(t.get("tool_calls") or 0)
if isinstance(rc, dict):
err[h] += sum(int(v) for v in rc.values() if isinstance(v, (int, float)))
for h in tot:
if tot[h] >= 20 and err[h] / max(1, tot[h]) >= _ERROR_RATE_FLAG:
if tot[h] >= 20 and err[h] / max(1, tot[h]) >= P_ERROR_RATE_FLAG:
findings.append({
"kind": "error_rate",
"host": h,
@@ -135,7 +135,7 @@ def audit(metrics_dir: str) -> dict:
}
def _host_of_task(task: dict) -> str:
def p_host_of_task(task: dict) -> str:
# tasks.jsonl doesn't store host directly; task_sig is host-agnostic, so fall
# back to a coarse bucket. browser_id groups a card's runs well enough for norms.
return task.get("browser_id") or task.get("task_sig") or "unknown"
@@ -171,7 +171,7 @@ def run_and_write(metrics_dir: str | None = None) -> str | None:
try:
if metrics_dir is None:
from backend.apps.agents.browser import browser_metrics
metrics_dir = browser_metrics._metrics_dir()
metrics_dir = browser_metrics.metrics_dir()
result = audit(metrics_dir)
report = render_report(result)
path = os.path.join(metrics_dir, "self_audit_report.md")
backend/apps/agents/browser/browser_skills.py
+147 -147
View File
@@ -55,7 +55,7 @@ from urllib.parse import urlparse, urlunparse
logger = logging.getLogger(__name__)
def _event(kind: str, host: str, sig: str, rev: int = 0, state: str = "", **extra) -> None:
def p_event(kind: str, host: str, sig: str, rev: int = 0, state: str = "", **extra) -> None:
"""Mirror a lifecycle transition into the metrics sink so the analyzer can
prove the skill layer helps vs. silently thrashes. Lazy + best-effort: this
module never hard-depends on metrics, and a metrics failure never propagates."""
@@ -67,61 +67,61 @@ def _event(kind: str, host: str, sig: str, rev: int = 0, state: str = "", **extr
# In-memory hot cache: key "host::task_sig" -> skill dict. Bounded.
_skills: dict[str, dict] = {}
_MAX_MEM_SKILLS = 200
_MAX_DISK_SKILLS = 1000 # bound the on-disk library; evict oldest by mtime
_SKILL_FORMAT_VERSION = 1
SKILLS: dict[str, dict] = {}
P_MAX_MEM_SKILLS = 200
P_MAX_DISK_SKILLS = 1000 # bound the on-disk library; evict oldest by mtime
P_SKILL_FORMAT_VERSION = 1
# Trust state (the verify gate). A skill moves PROBATION -> TRUSTED only by a
# successful end-to-end replay; an unproven (probation) skill that fails a replay
# goes to QUARANTINE and is never replayed again (task falls back to pure LLM).
_PROBATION = "probation"
_TRUSTED = "trusted"
_QUARANTINE = "quarantine"
PROBATION = "probation"
TRUSTED = "trusted"
QUARANTINE = "quarantine"
# A proven skill tolerates this many consecutive transient replay misses before
# it's demoted back to probation (forced to re-earn trust).
_FAIL_DEMOTE_THRESHOLD = 2
P_FAIL_DEMOTE_THRESHOLD = 2
# Tools that change page state (worth replaying). Reads/meta are never recorded.
_PRODUCTIVE = {"BrowserType", "BrowserClickIndex", "BrowserClick", "BrowserPressKey", "BrowserScroll"}
PRODUCTIVE = {"BrowserType", "BrowserClickIndex", "BrowserClick", "BrowserPressKey", "BrowserScroll"}
_URL_RE = re.compile(r"https?://\S+")
_WS_RE = re.compile(r"\s+")
_PUNCT_RE = re.compile(r"[^a-z0-9 ]+")
_STOP = {
P_URL_RE = re.compile(r"https?://\S+")
P_WS_RE = re.compile(r"\s+")
P_PUNCT_RE = re.compile(r"[^a-z0-9 ]+")
P_STOP = {
"the", "a", "an", "to", "into", "on", "this", "that", "page", "please",
"then", "and", "go", "open", "browser", "tell", "me", "whether", "it",
"of", "in", "for", "with", "your", "after", "if", "you", "can",
}
# --- sensitivity detection (gate for what may touch disk) ------------------
_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
_SSN_RE = re.compile(r"\b\d{3}-\d{2}-\d{4}\b")
_CARD_RE = re.compile(r"\b(?:\d[ -]?){13,19}\b")
_PHONE_RE = re.compile(r"\b(?:\+?\d[ -]?){10,15}\b")
_TOKEN_PREFIX_RE = re.compile(r"\b(sk-|ghp_|gho_|pk_|xox[bap]-|AIza|eyJ)")
_SENSITIVE_FIELD_RE = re.compile(
P_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
P_SSN_RE = re.compile(r"\b\d{3}-\d{2}-\d{4}\b")
P_CARD_RE = re.compile(r"\b(?:\d[ -]?){13,19}\b")
P_PHONE_RE = re.compile(r"\b(?:\+?\d[ -]?){10,15}\b")
P_TOKEN_PREFIX_RE = re.compile(r"\b(sk-|ghp_|gho_|pk_|xox[bap]-|AIza|eyJ)")
P_SENSITIVE_FIELD_RE = re.compile(
r"pass|pwd|secret|otp|cvv|cvc|ssn|card|token|api[_-]?key|security"
r"|user|login|sign[-_]?in|email|auth|seed|recovery|phrase|\bpin\b|2fa|verif|code",
re.I,
)
def _looks_sensitive(text: str, selector: str = "") -> bool:
def looks_sensitive(text: str, selector: str = "") -> bool:
"""Conservative: err toward 'sensitive' so secrets never persist. Catches
emails, SSNs, card/phone-shaped digit runs, known key prefixes, long
high-entropy tokens, bare one-time-code digit runs, and anything typed into
a credential-shaped field (a wrongly-blocked persist just keeps the skill
in-memory, so false positives are cheap; a leak is not)."""
if selector and _SENSITIVE_FIELD_RE.search(selector):
if selector and P_SENSITIVE_FIELD_RE.search(selector):
return True
if not text:
return False
if _EMAIL_RE.search(text) or _SSN_RE.search(text) or _CARD_RE.search(text):
if P_EMAIL_RE.search(text) or P_SSN_RE.search(text) or P_CARD_RE.search(text):
return True
if _TOKEN_PREFIX_RE.search(text):
if P_TOKEN_PREFIX_RE.search(text):
return True
if _PHONE_RE.search(text):
if P_PHONE_RE.search(text):
return True
stripped = text.strip()
# bare 6-8 digit run: the shape of every 2FA/SMS code; never worth persisting
@@ -133,7 +133,7 @@ def _looks_sensitive(text: str, selector: str = "") -> bool:
return False
def _sanitize_url(url: str) -> str:
def p_sanitize_url(url: str) -> str:
"""Strip userinfo (user:pass@) and fragment from a URL before it persists."""
try:
p = urlparse(url)
@@ -148,9 +148,9 @@ def _sanitize_url(url: str) -> str:
def normalize_task(task: str) -> str:
"""Stable task signature: lowercase, drop urls/punct/filler, collapse ws."""
t = (task or "").lower()
t = _URL_RE.sub(" ", t)
t = _PUNCT_RE.sub(" ", t)
toks = [w for w in _WS_RE.sub(" ", t).strip().split(" ") if w and w not in _STOP]
t = P_URL_RE.sub(" ", t)
t = P_PUNCT_RE.sub(" ", t)
toks = [w for w in P_WS_RE.sub(" ", t).strip().split(" ") if w and w not in P_STOP]
return " ".join(toks)
@@ -163,8 +163,8 @@ def normalize_task(task: str) -> str:
# Lookarounds keep word-internal apostrophes (chen's, don't) from opening a
# span; without them every possessive made each task wording a unique sig and
# silently disabled skill matching for those tasks.
_QUOTE_RE = re.compile(r'(?<!\w)["“”‘’\']([^"“”‘’\']{1,200})["“”‘’\'](?!\w)')
_SLOT_TOKEN = " slotvalue "
P_QUOTE_RE = re.compile(r'(?<!\w)["“”‘’\']([^"“”‘’\']{1,200})["“”‘’\'](?!\w)')
P_SLOT_TOKEN = " slotvalue "
def template_task(task: str) -> tuple[str, list[str]]:
@@ -173,19 +173,19 @@ def template_task(task: str) -> tuple[str, list[str]]:
def _repl(m):
values.append(m.group(1))
return _SLOT_TOKEN
return P_SLOT_TOKEN
return _QUOTE_RE.sub(_repl, task or ""), values
return P_QUOTE_RE.sub(_repl, task or ""), values
def _sig(task: str) -> str:
def compute_sig(task: str) -> str:
"""Skill key signature: template out quoted values, then normalize, so the
same task with different quoted inputs maps to the same key."""
templated, _ = template_task(task)
return normalize_task(templated)
def _parameterize(steps: list[dict], task: str) -> list[dict]:
def p_parameterize(steps: list[dict], task: str) -> list[dict]:
"""Convert any BrowserType whose text is a quoted task value into a slot
step (value_slot index), so the value is sourced live at replay, not stored."""
_, values = template_task(task)
@@ -309,10 +309,10 @@ def distill_steps(action_log: list[dict]) -> list[dict]:
productive_count += 1
if productive_count == 0:
return []
return _prune_detours(steps)
return p_prune_detours(steps)
def _prune_detours(steps: list[dict]) -> list[dict]:
def p_prune_detours(steps: list[dict]) -> list[dict]:
"""Drop an abandoned-page detour: a BrowserNavigate whose page was never
acted on because the very next step navigates somewhere else. Conservative
on purpose, only consecutive navigates qualify (if a page had been used,
@@ -381,17 +381,17 @@ def steps_are_persistable(steps: list[dict]) -> bool:
for s in steps:
p = s.get("params", {})
if s["tool"] == "BrowserType":
if _looks_sensitive(p.get("text", ""), p.get("selector", "")):
if looks_sensitive(p.get("text", ""), p.get("selector", "")):
return False
elif s["tool"] == "BrowserNavigate":
url = p.get("url", "")
# a tokenized/credentialed URL is both sensitive and non-reproducible
if "@" in (urlparse(url).netloc or "") or _looks_sensitive(url):
if "@" in (urlparse(url).netloc or "") or looks_sensitive(url):
return False
return True
def _step_key(s: dict) -> tuple:
def p_step_key(s: dict) -> tuple:
"""Canonical identity of a step, ignoring volatile detail, so we can tell a
real EDIT (page changed -> different steps) from a transient re-derivation
(same steps, the miss was just a timing blip). A slot and a literal are
@@ -407,7 +407,7 @@ def _step_key(s: dict) -> tuple:
if tool == "BrowserClick":
return (tool, p.get("selector"))
if tool == "BrowserNavigate":
return (tool, _sanitize_url(p.get("url", "")))
return (tool, p_sanitize_url(p.get("url", "")))
if tool == "BrowserPressKey":
return (tool, p.get("key"))
if tool == "BrowserScroll":
@@ -415,23 +415,23 @@ def _step_key(s: dict) -> tuple:
return (tool, json.dumps(p, sort_keys=True, default=str))
def _steps_equal(a: list[dict], b: list[dict]) -> bool:
return [_step_key(s) for s in a] == [_step_key(s) for s in b]
def steps_equal(a: list[dict], b: list[dict]) -> bool:
return [p_step_key(s) for s in a] == [p_step_key(s) for s in b]
def _sanitized_steps_for_disk(steps: list[dict]) -> list[dict]:
def p_sanitized_steps_for_disk(steps: list[dict]) -> list[dict]:
"""Copy of steps safe to persist: navigate URLs stripped of userinfo+fragment."""
out = []
for s in steps:
if s["tool"] == "BrowserNavigate":
out.append({"tool": "BrowserNavigate", "params": {"url": _sanitize_url(s["params"].get("url", ""))}})
out.append({"tool": "BrowserNavigate", "params": {"url": p_sanitize_url(s["params"].get("url", ""))}})
else:
out.append({"tool": s["tool"], "params": dict(s.get("params", {}))})
return out
# --- persistence ----------------------------------------------------------
def _skills_dir() -> str | None:
def p_skills_dir() -> str | None:
override = os.environ.get("OPENSWARM_BROWSER_SKILLS_DIR")
base = override
if not base:
@@ -447,31 +447,31 @@ def _skills_dir() -> str | None:
return base
def _key(host: str, sig: str) -> str:
def p_key(host: str, sig: str) -> str:
return f"{host}::{sig}"
def _skill_path(host: str, sig: str) -> str | None:
d = _skills_dir()
def skill_path(host: str, sig: str) -> str | None:
d = p_skills_dir()
if not d:
return None
h = hashlib.sha256(_key(host, sig).encode("utf-8")).hexdigest()[:32]
h = hashlib.sha256(p_key(host, sig).encode("utf-8")).hexdigest()[:32]
return os.path.join(d, f"{h}.json")
def _persist(host: str, sig: str, skill: dict) -> None:
def persist(host: str, sig: str, skill: dict) -> None:
"""Atomic per-skill write. Best-effort; never raises. Evicts oldest on cap."""
path = _skill_path(host, sig)
path = skill_path(host, sig)
if not path:
return
payload = {
"version": _SKILL_FORMAT_VERSION,
"version": P_SKILL_FORMAT_VERSION,
"host": host, "task_sig": sig,
"steps": _sanitized_steps_for_disk(skill["steps"]),
"steps": p_sanitized_steps_for_disk(skill["steps"]),
"recorded_at": skill.get("recorded_at", time.time()),
"replays": skill.get("replays", 0),
"rev": skill.get("rev", 1),
"state": skill.get("state", _PROBATION),
"state": skill.get("state", PROBATION),
"fails": skill.get("fails", 0),
"composed_of": skill.get("composed_of", []),
}
@@ -481,18 +481,18 @@ def _persist(host: str, sig: str, skill: dict) -> None:
with os.fdopen(fd, "w", encoding="utf-8") as f:
json.dump(payload, f)
os.replace(tmp, path) # atomic; a reader never sees a half-written file
_evict_disk_if_over_cap(d)
p_evict_disk_if_over_cap(d)
except Exception as e:
logger.debug(f"[browser-skills] persist failed: {e}")
def _evict_disk_if_over_cap(d: str) -> None:
def p_evict_disk_if_over_cap(d: str) -> None:
try:
files = [os.path.join(d, f) for f in os.listdir(d) if f.endswith(".json")]
if len(files) <= _MAX_DISK_SKILLS:
if len(files) <= P_MAX_DISK_SKILLS:
return
files.sort(key=lambda p: os.path.getmtime(p)) # oldest first
for p in files[: len(files) - _MAX_DISK_SKILLS]:
for p in files[: len(files) - P_MAX_DISK_SKILLS]:
try:
os.remove(p)
except Exception:
@@ -501,14 +501,14 @@ def _evict_disk_if_over_cap(d: str) -> None:
pass
def _load_from_disk(host: str, sig: str) -> dict | None:
path = _skill_path(host, sig)
def p_load_from_disk(host: str, sig: str) -> dict | None:
path = skill_path(host, sig)
if not path or not os.path.exists(path):
return None
try:
with open(path, encoding="utf-8") as f:
data = json.load(f)
if data.get("version") != _SKILL_FORMAT_VERSION:
if data.get("version") != P_SKILL_FORMAT_VERSION:
return None # format changed -> ignore stale file
if not data.get("steps"):
return None
@@ -516,7 +516,7 @@ def _load_from_disk(host: str, sig: str) -> dict | None:
"host": data.get("host", host), "task_sig": data.get("task_sig", sig),
"steps": data["steps"], "recorded_at": data.get("recorded_at", 0),
"replays": data.get("replays", 0), "persisted": True,
"rev": data.get("rev", 1), "state": data.get("state", _PROBATION),
"rev": data.get("rev", 1), "state": data.get("state", PROBATION),
"fails": data.get("fails", 0), "composed_of": data.get("composed_of", []),
}
except Exception as e:
@@ -524,12 +524,12 @@ def _load_from_disk(host: str, sig: str) -> dict | None:
return None
def _host_skills(host: str) -> dict[str, dict]:
def p_host_skills(host: str) -> dict[str, dict]:
"""Every skill for one host, keyed by task_sig, in-memory authoritative over
disk. One flat scan of the library dir (same cost list_skills always paid);
callers that run per-record gate on cheap pre-checks before calling."""
out: dict[str, dict] = {}
d = _skills_dir()
d = p_skills_dir()
if d:
try:
for f in os.listdir(d):
@@ -544,7 +544,7 @@ def _host_skills(host: str) -> dict[str, dict]:
out[data["task_sig"]] = {**data, "persisted": True}
except Exception:
pass
for s in _skills.values():
for s in SKILLS.values():
if s.get("host") == host and s.get("task_sig"):
out[s["task_sig"]] = s
return out
@@ -557,41 +557,41 @@ def _host_skills(host: str) -> dict[str, dict]:
# its own); the link is provenance + a safety wire: if that foundation is later
# deprecated or goes stale, every skill built on it is knocked back to probation
# so it must re-prove instead of silently riding a now-broken sub-sequence.
_COMPOSE_MIN_SUB_STEPS = 2
P_COMPOSE_MIN_SUB_STEPS = 2
def _detect_composition(host: str, sig: str, steps: list[dict]) -> list[str]:
def p_detect_composition(host: str, sig: str, steps: list[dict]) -> list[str]:
"""Sigs of TRUSTED host skills whose full step list is a strict opening
prefix of `steps`. Gated: needs a tail, so only runs for >=3-step skills."""
if len(steps) < _COMPOSE_MIN_SUB_STEPS + 1:
if len(steps) < P_COMPOSE_MIN_SUB_STEPS + 1:
return []
keys = [_step_key(s) for s in steps]
keys = [p_step_key(s) for s in steps]
found: list[str] = []
for other_sig, other in _host_skills(host).items():
if other_sig == sig or other.get("state") != _TRUSTED:
for other_sig, other in p_host_skills(host).items():
if other_sig == sig or other.get("state") != TRUSTED:
continue
osteps = other.get("steps", [])
if len(osteps) < _COMPOSE_MIN_SUB_STEPS or len(osteps) >= len(steps):
if len(osteps) < P_COMPOSE_MIN_SUB_STEPS or len(osteps) >= len(steps):
continue
if [_step_key(s) for s in osteps] == keys[: len(osteps)]:
if [p_step_key(s) for s in osteps] == keys[: len(osteps)]:
found.append(other_sig)
return found
def _invalidate_dependents(host: str, sub_sig: str) -> None:
def p_invalidate_dependents(host: str, sub_sig: str) -> None:
"""Knock every skill that builds on `sub_sig` back to probation: its proven
foundation just moved (edited/deprecated/demoted), so it must re-earn trust
rather than ghost-ride a sub-sequence that may no longer hold."""
for dep_sig, dep in _host_skills(host).items():
if sub_sig in dep.get("composed_of", []) and dep.get("state") == _TRUSTED:
k = _key(host, dep_sig)
live = _skills.get(k) or dep
live["state"] = _PROBATION
for dep_sig, dep in p_host_skills(host).items():
if sub_sig in dep.get("composed_of", []) and dep.get("state") == TRUSTED:
k = p_key(host, dep_sig)
live = SKILLS.get(k) or dep
live["state"] = PROBATION
live["fails"] = 0
_skills[k] = live
SKILLS[k] = live
if live.get("persisted"):
_persist(host, dep_sig, live)
_event("invalidate", host, dep_sig, rev=live.get("rev", 1), state=_PROBATION, foundation=sub_sig)
persist(host, dep_sig, live)
p_event("invalidate", host, dep_sig, rev=live.get("rev", 1), state=PROBATION, foundation=sub_sig)
logger.info(f"[browser-skills] {host}::{dep_sig} knocked to probation "
f"(its foundation {sub_sig} changed)")
@@ -610,15 +610,15 @@ def record_skill(host: str, task: str, action_log: list[dict]) -> bool:
steps = distill_steps(action_log)
if not steps:
return False
sig = _sig(task)
sig = compute_sig(task)
if not sig:
return False
steps = _parameterize(steps, task) # quoted values -> slots (not stored)
steps = p_parameterize(steps, task) # quoted values -> slots (not stored)
persistable = steps_are_persistable(steps)
k = _key(host, sig)
existing = _skills.get(k) or _load_from_disk(host, sig)
k = p_key(host, sig)
existing = SKILLS.get(k) or p_load_from_disk(host, sig)
if existing and _steps_equal(existing.get("steps", []), steps):
if existing and steps_equal(existing.get("steps", []), steps):
# Same skill re-derived: the replay that triggered this was a transient
# miss, not a stale skill. Keep rev + trust; just clear the fail streak.
# If it was quarantined (a known-bad distillation), leave it quarantined
@@ -626,9 +626,9 @@ def record_skill(host: str, task: str, action_log: list[dict]) -> bool:
existing["fails"] = 0
existing["recorded_at"] = time.time()
existing["persisted"] = persistable
_skills[k] = existing
SKILLS[k] = existing
if persistable:
_persist(host, sig, existing)
persist(host, sig, existing)
logger.info(f"[browser-skills] re-derived identical {len(steps)}-step skill for {host} "
f"(rev {existing.get('rev', 1)}, state={existing.get('state')}, transient miss)")
return True
@@ -637,25 +637,25 @@ def record_skill(host: str, task: str, action_log: list[dict]) -> bool:
skill = {
"host": host, "task_sig": sig, "steps": steps,
"recorded_at": time.time(), "replays": 0, "persisted": persistable,
"rev": rev, "state": _PROBATION, "fails": 0,
"composed_of": _detect_composition(host, sig, steps),
"rev": rev, "state": PROBATION, "fails": 0,
"composed_of": p_detect_composition(host, sig, steps),
}
_skills[k] = skill
if len(_skills) > _MAX_MEM_SKILLS:
oldest = min(_skills, key=lambda kk: _skills[kk]["recorded_at"])
_skills.pop(oldest, None)
SKILLS[k] = skill
if len(SKILLS) > P_MAX_MEM_SKILLS:
oldest = min(SKILLS, key=lambda kk: SKILLS[kk]["recorded_at"])
SKILLS.pop(oldest, None)
if persistable:
_persist(host, sig, skill)
persist(host, sig, skill)
verb = "EDITED" if existing else "learned"
comp = f", builds on {skill['composed_of']}" if skill["composed_of"] else ""
logger.info(f"[browser-skills] {verb} {len(steps)}-step skill for {host} "
f"(rev {rev}, probationary{', persisted' if persistable else ', in-memory only: sensitive'}{comp})")
_event("edit" if existing else "learn", host, sig, rev=rev, state=_PROBATION,
p_event("edit" if existing else "learn", host, sig, rev=rev, state=PROBATION,
steps=len(steps), composed_of=skill["composed_of"], persisted=persistable)
if skill["composed_of"]:
_event("compose", host, sig, rev=rev, state=_PROBATION, builds_on=skill["composed_of"])
p_event("compose", host, sig, rev=rev, state=PROBATION, builds_on=skill["composed_of"])
if existing:
_invalidate_dependents(host, sig) # anything built on the OLD version must re-prove
p_invalidate_dependents(host, sig) # anything built on the OLD version must re-prove
return True
except Exception as e:
logger.debug(f"[browser-skills] record failed: {e}")
@@ -669,17 +669,17 @@ def find_skill(host: str, task: str) -> dict | None:
re-attempting a known-bad replay. Cheap + flat as the library grows."""
if not host:
return None
sig = _sig(task)
sig = compute_sig(task)
if not sig:
return None
k = _key(host, sig)
hit = _skills.get(k)
k = p_key(host, sig)
hit = SKILLS.get(k)
if not hit:
loaded = _load_from_disk(host, sig)
loaded = p_load_from_disk(host, sig)
if loaded:
_skills[k] = loaded # warm the hot cache (even if quarantined)
SKILLS[k] = loaded # warm the hot cache (even if quarantined)
hit = loaded
if not hit or hit.get("state") == _QUARANTINE:
if not hit or hit.get("state") == QUARANTINE:
return None
return hit
@@ -690,8 +690,8 @@ def find_skill(host: str, task: str) -> dict | None:
# similar skill is rendered as advisory text the live agent adapts and verifies,
# so it generalizes across wordings and stays send-safe (the agent still
# confirms everything; a stale hint just wastes one glance).
_HINT_MIN_OVERLAP = 0.5
_HINT_MAX_STEPS = 10
P_HINT_MIN_OVERLAP = 0.5
P_HINT_MAX_STEPS = 10
def find_similar_skill(host: str, task: str) -> tuple[dict | None, float]:
@@ -700,13 +700,13 @@ def find_similar_skill(host: str, task: str) -> tuple[dict | None, float]:
stays exact-key; this feeds route hints, never mechanical execution."""
if not host:
return None, 0.0
sig = _sig(task)
sig = compute_sig(task)
stoks = set(sig.split())
if not stoks:
return None, 0.0
best, best_score = None, 0.0
for other_sig, s in _host_skills(host).items():
if s.get("state") == _QUARANTINE or not s.get("steps"):
for other_sig, s in p_host_skills(host).items():
if s.get("state") == QUARANTINE or not s.get("steps"):
continue
otoks = set(other_sig.split())
if not otoks:
@@ -714,14 +714,14 @@ def find_similar_skill(host: str, task: str) -> tuple[dict | None, float]:
score = len(stoks & otoks) / len(stoks | otoks)
# a proven skill wins ties against an unproven one
if score > best_score or (score == best_score and best is not None
and s.get("state") == _TRUSTED and best.get("state") != _TRUSTED):
and s.get("state") == TRUSTED and best.get("state") != TRUSTED):
best, best_score = s, score
if best and best_score >= _HINT_MIN_OVERLAP:
if best and best_score >= P_HINT_MIN_OVERLAP:
return best, best_score
return None, 0.0
def _hint_step_line(step: dict, values: list[str]) -> str:
def p_hint_step_line(step: dict, values: list[str]) -> str:
tool = step.get("tool", "")
p = step.get("params", {}) or {}
if tool == "BrowserNavigate":
@@ -750,7 +750,7 @@ def render_route_hint(skill: dict, task: str, score: float) -> tuple[str, list[t
"""Compact advisory route block from a skill's steps, plus the step keys for
adoption measurement. Slots are filled from the LIVE task's quoted values
(never from disk); the first irreversible step is flagged solo-only."""
steps = (skill.get("steps") or [])[:_HINT_MAX_STEPS]
steps = (skill.get("steps") or [])[:P_HINT_MAX_STEPS]
if not steps:
return "", []
from backend.apps.agents.browser import browser_batch_replay
@@ -766,8 +766,8 @@ def render_route_hint(skill: dict, task: str, score: float) -> tuple[str, list[t
name = p.get("name") or p.get("selector") or ""
if len(name) <= 40 and browser_batch_replay.is_replay_boundary({"action": "click", "name": name}):
mark = " [IRREVERSIBLE: do this SOLO with `expect` proof, never in a batch]"
lines.append(f"{i + 1}. {_hint_step_line(s, values)}{mark}")
trust = "proven by a verified rerun" if skill.get("state") == _TRUSTED else "from one verified success"
lines.append(f"{i + 1}. {p_hint_step_line(s, values)}{mark}")
trust = "proven by a verified rerun" if skill.get("state") == TRUSTED else "from one verified success"
safe_until = unsafe_i if unsafe_i >= 0 else len(steps)
batch_line = (
f"Steps 1-{safe_until} are routine; combine them into ONE BrowserBatch where the page allows."
@@ -778,7 +778,7 @@ def render_route_hint(skill: dict, task: str, score: float) -> tuple[str, list[t
"Adapt where the live page differs and verify each step as usual:\n"
+ "\n".join(lines) + (f"\n{batch_line}" if batch_line else "")
)
return hint, [_step_key(s) for s in steps]
return hint, [p_step_key(s) for s in steps]
def hint_step_adopted(step_key: tuple, action_log: list[dict]) -> bool:
@@ -817,13 +817,13 @@ def mark_replay_succeeded(host: str, task: str) -> None:
return
s["replays"] = s.get("replays", 0) + 1
s["fails"] = 0
promoted = s.get("state") != _TRUSTED
s["state"] = _TRUSTED
promoted = s.get("state") != TRUSTED
s["state"] = TRUSTED
if s.get("persisted"):
_persist(host, s["task_sig"], s) # keep the on-disk count + state fresh
persist(host, s["task_sig"], s) # keep the on-disk count + state fresh
if promoted:
logger.info(f"[browser-skills] {host}::{s['task_sig']} PROVEN by replay (rev {s.get('rev', 1)}) -> trusted")
_event("promote", host, s["task_sig"], rev=s.get("rev", 1), state=_TRUSTED, replays=s["replays"])
p_event("promote", host, s["task_sig"], rev=s.get("rev", 1), state=TRUSTED, replays=s["replays"])
def mark_replay_failed(host: str, task: str) -> str:
@@ -840,27 +840,27 @@ def mark_replay_failed(host: str, task: str) -> str:
if not s:
return "none"
sig = s["task_sig"]
if s.get("state") != _TRUSTED:
s["state"] = _QUARANTINE
if s.get("state") != TRUSTED:
s["state"] = QUARANTINE
s["fails"] = s.get("fails", 0) + 1
if s.get("persisted"):
_persist(host, sig, s)
_event("quarantine", host, sig, rev=s.get("rev", 1), state=_QUARANTINE)
_invalidate_dependents(host, sig)
persist(host, sig, s)
p_event("quarantine", host, sig, rev=s.get("rev", 1), state=QUARANTINE)
p_invalidate_dependents(host, sig)
logger.info(f"[browser-skills] {host}::{sig} (unproven) failed replay -> quarantined (baseline from here)")
return "quarantined"
s["fails"] = s.get("fails", 0) + 1
if s["fails"] >= _FAIL_DEMOTE_THRESHOLD:
s["state"] = _PROBATION
if s["fails"] >= P_FAIL_DEMOTE_THRESHOLD:
s["state"] = PROBATION
if s.get("persisted"):
_persist(host, sig, s)
_event("demote", host, sig, rev=s.get("rev", 1), state=_PROBATION, fails=s["fails"])
_invalidate_dependents(host, sig)
persist(host, sig, s)
p_event("demote", host, sig, rev=s.get("rev", 1), state=PROBATION, fails=s["fails"])
p_invalidate_dependents(host, sig)
logger.info(f"[browser-skills] {host}::{sig} failed {s['fails']}x -> demoted to probation")
return "demoted"
if s.get("persisted"):
_persist(host, sig, s)
logger.info(f"[browser-skills] {host}::{sig} transient replay miss ({s['fails']}/{_FAIL_DEMOTE_THRESHOLD}), trust kept")
persist(host, sig, s)
logger.info(f"[browser-skills] {host}::{sig} transient replay miss ({s['fails']}/{P_FAIL_DEMOTE_THRESHOLD}), trust kept")
return "kept"
@@ -870,28 +870,28 @@ def list_skills(host: str) -> list[dict]:
agent can ask "what shortcuts do I have here?" without pulling a wall of
detail into context. Reads in-memory + the on-disk library for this host."""
out = []
for sig, s in _host_skills(host).items():
for sig, s in p_host_skills(host).items():
out.append({
"task": sig, "steps": len(s.get("steps", [])),
"replays": s.get("replays", 0), "persisted": s.get("persisted", False),
"state": s.get("state", _PROBATION), "rev": s.get("rev", 1),
"state": s.get("state", PROBATION), "rev": s.get("rev", 1),
"builds_on": list(s.get("composed_of", [])),
})
# trusted first, then most-reused
return sorted(out, key=lambda x: (x["state"] != _TRUSTED, -x["replays"]))
return sorted(out, key=lambda x: (x["state"] != TRUSTED, -x["replays"]))
def deprecate_skill(host: str, task: str) -> bool:
"""Remove a skill (in-memory + disk) so it stops being replayed, and knock
any skill that was built on it back to probation. The agent calls this when
it judges a saved shortcut is stale / wrong (page changed). Accepts either the
raw task or the task_sig from list_skills (sig is idempotent under _sig).
raw task or the task_sig from list_skills (sig is idempotent under sig).
Returns True if something was removed."""
if not host:
return False
sig = _sig(task)
removed = _skills.pop(_key(host, sig), None) is not None
path = _skill_path(host, sig)
sig = compute_sig(task)
removed = SKILLS.pop(p_key(host, sig), None) is not None
path = skill_path(host, sig)
if path and os.path.exists(path):
try:
os.remove(path)
@@ -899,7 +899,7 @@ def deprecate_skill(host: str, task: str) -> bool:
except Exception:
pass
if removed:
_invalidate_dependents(host, sig)
p_invalidate_dependents(host, sig)
logger.info(f"[browser-skills] deprecated skill {host}::{sig}")
return removed
@@ -910,9 +910,9 @@ def forget_host(host: str) -> int:
if not host:
return 0
n = 0
for sig in list(_host_skills(host).keys()):
removed = _skills.pop(_key(host, sig), None) is not None
path = _skill_path(host, sig)
for sig in list(p_host_skills(host).keys()):
removed = SKILLS.pop(p_key(host, sig), None) is not None
path = skill_path(host, sig)
if path and os.path.exists(path):
try:
os.remove(path)
@@ -929,9 +929,9 @@ def forget_host(host: str) -> int:
def clear(wipe_disk: bool = False) -> None:
"""Clear the in-memory cache. With wipe_disk, also remove persisted files
in the current skills dir (used by tests for isolation)."""
_skills.clear()
SKILLS.clear()
if wipe_disk:
d = _skills_dir()
d = p_skills_dir()
if d:
try:
for f in os.listdir(d):
backend/apps/agents/browser/browser_validator.py
+4 -4
View File
@@ -13,7 +13,7 @@ import logging
logger = logging.getLogger(__name__)
_ADJUDICATION_PROMPT = (
P_ADJUDICATION_PROMPT = (
"A browser automation agent is stuck: its recent actions produced no "
"progress on the page.\n\n"
"GOAL: {goal}\n\n"
@@ -27,7 +27,7 @@ _ADJUDICATION_PROMPT = (
)
def _extract_text(response) -> str:
def extract_text(response) -> str:
"""Pull the text out of an Anthropic-style response object."""
parts = []
for block in getattr(response, "content", None) or []:
@@ -40,7 +40,7 @@ async def adjudicate_stuck(
client, model: str, goal: str, recent_actions: str, page_text: str,
) -> str:
"""One cheap aux call returning concrete guidance, or "" on any failure."""
prompt = _ADJUDICATION_PROMPT.format(
prompt = P_ADJUDICATION_PROMPT.format(
goal=(goal or "(unknown)")[:400],
recent=(recent_actions or "(none)")[:1200],
page=(page_text or "(empty)")[:1500],
@@ -54,4 +54,4 @@ async def adjudicate_stuck(
except Exception as e:
logger.warning(f"[browser-validator] adjudication call failed: {e}")
return ""
return _extract_text(response)
return extract_text(response)
backend/apps/agents/browser/browser_wait.py
+12 -12
View File
@@ -34,7 +34,7 @@ logger = logging.getLogger(__name__)
# elems - element count; the loop watches it stop changing = DOM/visual settle
# found - the agent's `until` target is present + visible (visible text or selector)
# `until` is JSON-encoded into a string literal, so it's data, never executable.
def _probe_js(until: str) -> str:
def p_probe_js(until: str) -> str:
spec = json.dumps(until or "")
return (
"(()=>{const n=performance.now();"
@@ -50,21 +50,21 @@ def _probe_js(until: str) -> str:
"quiet:Math.round(n-last),elems:document.getElementsByTagName('*').length,found});})()"
)
_QUIET_WINDOW_MS = 400 # network must be silent this long to count as settled
_FLOOR_MS = 250 # never return before this (a momentary gap isn't 'settled')
_POLL_MS = 150
P_QUIET_WINDOW_MS = 400 # network must be silent this long to count as settled
P_FLOOR_MS = 250 # never return before this (a momentary gap isn't 'settled')
P_POLL_MS = 150
# A healthy probe is tens of ms. A busy-but-fine SPA (heavy main-thread work mid-
# hydration) can occasionally block longer, so a slow probe is NOT proof of death,
# it's just a reason to stop THIS wait early instead of inheriting the 30s command
# timeout. We bound each probe at this, and after a few consecutive non-responses
# we surface hung=True as a SIGNAL (the loop folds it into a cross-command streak
# and only then acts), never as a unilateral abort from a single wait.
_PROBE_TIMEOUT_S = 2.5
_MAX_PROBE_TIMEOUTS = 3
P_PROBE_TIMEOUT_S = 2.5
MAX_PROBE_TIMEOUTS = 3
def decide_stop(ready, quiet_ms, dom_stable_ms, found, elapsed_ms,
floor_ms=_FLOOR_MS, settle_window_ms=_QUIET_WINDOW_MS) -> bool:
floor_ms=P_FLOOR_MS, settle_window_ms=P_QUIET_WINDOW_MS) -> bool:
"""Pure decision. Stop the INSTANT the agent's target is present (no floor, it's
exactly what we were waiting for). Otherwise, past the floor and once the document
is complete, stop as soon as it has gone quiet by EITHER the network OR the DOM
@@ -81,9 +81,9 @@ def decide_stop(ready, quiet_ms, dom_stable_ms, found, elapsed_ms,
async def smart_wait(execute_fn, browser_id, tab_id, max_ms, *, until="",
poll_ms=_POLL_MS, floor_ms=_FLOOR_MS,
quiet_window_ms=_QUIET_WINDOW_MS,
probe_timeout_s=_PROBE_TIMEOUT_S, target_only=False) -> dict:
poll_ms=P_POLL_MS, floor_ms=P_FLOOR_MS,
quiet_window_ms=P_QUIET_WINDOW_MS,
probe_timeout_s=P_PROBE_TIMEOUT_S, target_only=False) -> dict:
"""Wait up to `max_ms`, returning early once the page is ready. `until` (optional)
is a label / visible text / selector the agent expects to appear; the wait ends the
INSTANT it's present, so the agent isn't waiting blind. `execute_fn` is an async
@@ -96,7 +96,7 @@ async def smart_wait(execute_fn, browser_id, tab_id, max_ms, *, until="",
a result that renders a beat AFTER settle (a sent message landing in a thread under
load), which the settle-early path otherwise misses, reporting a false 'not confirmed'."""
max_ms = max(100, min(int(max_ms or 1000), 10000))
probe_js = _probe_js(until)
probe_js = p_probe_js(until)
start = time.monotonic()
settled = False
found = False
@@ -124,7 +124,7 @@ async def smart_wait(execute_fn, browser_id, tab_id, max_ms, *, until="",
)
except asyncio.TimeoutError:
probe_timeouts += 1
if probe_timeouts >= _MAX_PROBE_TIMEOUTS:
if probe_timeouts >= MAX_PROBE_TIMEOUTS:
hung = True
break
continue
backend/tests/conftest.py
+1 -1
View File
@@ -32,7 +32,7 @@ def _isolate_browser_state(monkeypatch):
# where ITS env var points, not where the first test's pointed
try:
from backend.apps.agents.browser import browser_metrics as _bm
_bm._metrics_dir_cache = None
_bm.p_metrics_dir_cache = None
except Exception:
pass
_reset()
backend/tests/test_browser_agent_loop.py
+98 -98
View File
@@ -135,7 +135,7 @@ def _install(monkeypatch, primary, aux):
def test_full_loop_goal_stagnation_adjudication_and_hint_write(monkeypatch):
BH._browser_history.clear(); BH._domain_notes.clear()
BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear()
primary = FakeLLM([
Resp([_rp("click the Search button"), _tu("BrowserListInteractives")]),
Resp([_rp("click submit"), _tu("BrowserClick", selector=".s1")]),
@@ -174,7 +174,7 @@ def test_action_with_expect_is_confirmed(monkeypatch):
# An action that declares `expect` is CONFIRMED after it runs: the loop issues a
# target-aware confirm probe and feeds the next turn a tool_result stating the
# expected change is present (observed success, never assumed).
BH._browser_history.clear(); BH._domain_notes.clear()
BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear()
primary = FakeLLM([
Resp([_rp("click submit and confirm"),
_tu("BrowserClickIndex", index=1, expect="Submitted")]),
@@ -196,7 +196,7 @@ def test_action_with_expect_is_confirmed(monkeypatch):
def test_missing_report_progress_runs_the_action_and_reminds_not_rejects(monkeypatch):
# The model acts WITHOUT ReportProgress. Old behavior rejected the turn (wasted
# a round-trip); new behavior runs the action and folds in a one-line reminder.
BH._browser_history.clear(); BH._domain_notes.clear()
BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear()
primary = FakeLLM([
Resp([_tu("BrowserClickIndex", index=2)]), # NO ReportProgress this turn
Resp([Blk("text", "done")], stop_reason="end_turn"),
@@ -218,7 +218,7 @@ def test_confirmed_send_ends_the_run_instead_of_stalling(monkeypatch):
# After an irreversible send CONFIRMS, the model must not burn turns re-verifying.
# Here it sends (index 99 = "Send", expect confirms) then tries to stall forever
# with pure-perception turns; the loop must END within a turn or two, not spin.
BH._browser_history.clear(); BH._domain_notes.clear()
BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear()
primary = FakeLLM([
Resp([_rp("send the message"), _tu("BrowserClickIndex", index=99, expect="Sent")]),
# the model now STALLS, re-looking instead of finishing (the bug)
@@ -243,7 +243,7 @@ def test_confirmed_send_ends_the_run_instead_of_stalling(monkeypatch):
def test_done_tool_delivers_a_clean_human_summary(monkeypatch):
# Canonical finish: the model calls Done(message); that message is the user's
# reply verbatim (no OUTCOME tag, no UI mechanics) and `done` is True.
BH._browser_history.clear(); BH._domain_notes.clear()
BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear()
primary = FakeLLM([
Resp([_rp("open profile + send"), _tu("BrowserClickIndex", index=5, expect="Sent")]),
Resp([_tu("Done", message="Sent your message to Tyler, it's in the thread now.")]),
@@ -259,7 +259,7 @@ def test_done_tool_delivers_a_clean_human_summary(monkeypatch):
def test_done_tool_success_false_marks_not_done(monkeypatch):
# Done(success=false) is the honest "couldn't finish": done is False so the
# fast path knows to recover, and the message still reads like a person wrote it.
BH._browser_history.clear(); BH._domain_notes.clear()
BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear()
primary = FakeLLM([
Resp([_rp("look for thread"), _tu("BrowserClickIndex", index=3)]),
Resp([_tu("Done", message="I hit a login wall, so I couldn't open the chat.", success=False)]),
@@ -275,7 +275,7 @@ def test_run_that_never_calls_done_is_not_a_clean_success(monkeypatch):
# A run that does real work but stops with plain text (never calls Done) is a
# half-finish, not a clean success: done must be False so the fast path recovers
# instead of shipping a silent stop (the 'Task completed.' that wasn't).
BH._browser_history.clear(); BH._domain_notes.clear()
BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear()
primary = FakeLLM([
Resp([_rp("click it"), _tu("BrowserClickIndex", index=3)]),
Resp([Blk("text", "I clicked the thing.")], stop_reason="end_turn"),
@@ -292,7 +292,7 @@ def test_send_shortcut_does_not_arm_on_a_gather_task(monkeypatch):
# arm the send-completion shortcut, there is no send to confirm. If it did, the
# run cuts at the 2-turn post-send limit and leaks the canned "message went
# through" line. On a gather task it should run the full perception budget.
BH._browser_history.clear(); BH._domain_notes.clear()
BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear()
primary = FakeLLM([
Resp([_rp("dismiss the cookie banner"), _tu("BrowserClickIndex", index=99)]),
*[Resp([_rp("keep reading the list"), _tu("BrowserScreenshot")]) for _ in range(8)],
@@ -313,7 +313,7 @@ def test_browser_save_data_writes_a_file_and_returns_a_receipt(monkeypatch, tmp_
# of a dozen reply-chunks. The mock's evaluate echoes its expression as the data.
import os as _os
monkeypatch.setattr(_os.path, "expanduser", lambda p: str(tmp_path)) # fallback workspace -> tmp
BH._browser_history.clear(); BH._domain_notes.clear()
BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear()
primary = FakeLLM([
Resp([_rp("save the rows"), _tu("BrowserSaveData", expression="JSON.stringify(window.__rows)", filename="rows.json")]),
Resp([_tu("Done", message="Saved the full set to rows.json.")]),
@@ -329,7 +329,7 @@ def test_browser_save_data_writes_a_file_and_returns_a_receipt(monkeypatch, tmp_
# listings every turn) must NOT trip the spin backstop, gathering is the work,
# not spinning. Here 9 straight Extract turns each return distinct data; the run
# should keep going (no early wrap-up nudge) and finish on the model's own Done.
BH._browser_history.clear(); BH._domain_notes.clear()
BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear()
primary = FakeLLM([
# each turn reads a DIFFERENT page (distinct expression -> distinct result)
*[Resp([_rp(f"page {i}"), _tu("BrowserEvaluate", expression=f"parsePage({i})")]) for i in range(9)],
@@ -349,7 +349,7 @@ def test_spin_backstop_nudges_a_clean_wrapup_instead_of_a_midthought(monkeypatch
# The Airbnb mid-thought bug: a read-heavy run that trips the spin backstop must
# get ONE wrap-up nudge to summarize via Done, not be cut off mid-sentence. The
# final reply is the model's clean Done answer, and the nudge actually reached it.
BH._browser_history.clear(); BH._domain_notes.clear()
BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear()
primary = FakeLLM([
Resp([_rp("open the list"), _tu("BrowserClickIndex", index=3)]), # an action arms the backstop
# repeated identical screenshots (same result, no new data) = genuine spinning
@@ -368,7 +368,7 @@ def test_early_perception_is_not_cut_short_before_any_action(monkeypatch):
# Orienting on a cold/slow page can take several look-only turns; the stall
# backstop must NOT fire before the agent has done anything (it only bounds a
# POST-action spin). Here 7 perception turns precede the finish; all must run.
BH._browser_history.clear(); BH._domain_notes.clear()
BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear()
# varied read tools so the (separate) identical-repeat loop detector doesn't trip;
# this isolates the stall backstop, which must NOT fire pre-action
_reads = ["BrowserListInteractives", "BrowserGetText", "BrowserScreenshot"]
@@ -387,7 +387,7 @@ def test_aux_adjudication_fires_even_when_loop_detector_trips(monkeypatch):
# Repeated IDENTICAL failing clicks trip the exact-repeat loop detector AND
# reach stagnation exhaustion on the same turn. The aux escape hatch must
# still fire (it was previously suppressed by the `not is_loop` guard).
BH._browser_history.clear(); BH._domain_notes.clear()
BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear()
primary = FakeLLM([
Resp([_rp("click submit"), _tu("BrowserListInteractives")]),
*[Resp([_rp("retry same"), _tu("BrowserClick", selector=".same")]) for _ in range(6)],
@@ -410,7 +410,7 @@ def test_aux_adjudication_fires_even_when_loop_detector_trips(monkeypatch):
def test_tier1_and_tier2_tools_drive_through_the_real_loop(monkeypatch):
# The agent can call the new tier-1 (WebMCP detect) and tier-2 (list/replay)
# tools through the actual run_browser_agent loop, and replay threads its url.
BH._browser_history.clear(); BH._domain_notes.clear()
BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear()
primary = FakeLLM([
Resp([_rp("check for a faster path"), _tu("BrowserDetectWebMCP")]),
Resp([_rp("list captured routes"), _tu("BrowserListRoutes")]),
@@ -438,7 +438,7 @@ def test_skill_is_recorded_then_replayed_with_zero_llm_calls(monkeypatch):
# Run 2: same task/host -> replays via the no-LLM fast path (the speed win).
import backend.apps.agents.browser.browser_skills as SK
SK.clear()
BH._browser_history.clear(); BH._domain_notes.clear()
BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear()
primary = FakeLLM([
Resp([_rp("click submit"), _tu("BrowserListInteractives")]),
Resp([_rp("click it"), _tu("BrowserClickIndex", index=1)]),
@@ -471,7 +471,7 @@ def test_replay_falls_back_to_full_agent_when_a_step_fails(monkeypatch):
# the full LLM agent instead (never ghost-succeed on a stale skill).
import backend.apps.agents.browser.browser_skills as SK
SK.clear()
BH._browser_history.clear()
BH.BROWSER_HISTORY.clear()
# Pre-seed a skill whose click target no longer exists on the page.
SK.record_skill("docs.google.com", "click the Save button", [
{"tool": "BrowserClickIndex", "input": {"index": 1}, "ok": True,
@@ -504,7 +504,7 @@ def test_deferred_replay_fires_after_navigating_to_the_right_host(monkeypatch):
# deferred re-check must switch to replay instead of grinding the LLM loop.
import backend.apps.agents.browser.browser_skills as SK
SK.clear()
BH._browser_history.clear()
BH.BROWSER_HISTORY.clear()
SK.record_skill("docs.google.com", "click the Search button", [
{"tool": "BrowserClickIndex", "input": {}, "ok": True,
"clicked_role": "button", "clicked_name": "Search"},
@@ -535,7 +535,7 @@ def test_deferred_replay_fires_after_navigating_to_the_right_host(monkeypatch):
assert any(c["action"] == "click_by_name" for c in sent), "replay re-resolved by name"
assert len(primary.calls) == 1, "only the navigate turn ran; the re-check preempted the rest"
# and the deferred replay still promotes the skill through the trust gate
assert SK.find_skill("docs.google.com", "click the Search button")["state"] == SK._TRUSTED
assert SK.find_skill("docs.google.com", "click the Search button")["state"] == SK.TRUSTED
def test_deferred_replay_does_not_fire_after_the_page_was_dirtied(monkeypatch):
@@ -544,7 +544,7 @@ def test_deferred_replay_does_not_fire_after_the_page_was_dirtied(monkeypatch):
# state is dirty), so the re-check must stay disabled and the LLM finishes.
import backend.apps.agents.browser.browser_skills as SK
SK.clear()
BH._browser_history.clear()
BH.BROWSER_HISTORY.clear()
SK.record_skill("docs.google.com", "click the Search button", [
{"tool": "BrowserClickIndex", "input": {}, "ok": True,
"clicked_role": "button", "clicked_name": "Search"},
@@ -582,7 +582,7 @@ def test_replay_resolves_host_from_live_page_when_no_initial_url(monkeypatch):
# orchestrated flow (records skills it can never look up again).
import backend.apps.agents.browser.browser_skills as SK
SK.clear()
BH._browser_history.clear()
BH.BROWSER_HISTORY.clear()
# a skill exists for the host the live page will report (DOC_URL -> docs.google.com)
SK.record_skill("docs.google.com", "click the Search button", [
{"tool": "BrowserClickIndex", "input": {}, "ok": True,
@@ -608,7 +608,7 @@ def test_skill_keys_on_parent_user_message_so_reformulations_share_a_skill(monke
import backend.apps.agents.browser.browser_skills as SK
import backend.apps.agents.agent_manager as am_mod
SK.clear()
BH._browser_history.clear()
BH.BROWSER_HISTORY.clear()
class _Msg:
def __init__(self, role, content):
@@ -651,7 +651,7 @@ def test_skill_key_falls_back_to_delegated_task_on_multi_quote_message(monkeypat
import backend.apps.agents.browser.browser_skills as SK
import backend.apps.agents.agent_manager as am_mod
SK.clear()
BH._browser_history.clear()
BH.BROWSER_HISTORY.clear()
class _Msg:
def __init__(self, role, content):
@@ -680,7 +680,7 @@ def test_replay_success_promotes_skill_to_trusted_through_the_loop(monkeypatch):
# it successfully, which must PROMOTE it to trusted (proven by a real replay).
import backend.apps.agents.browser.browser_skills as SK
SK.clear()
BH._browser_history.clear(); BH._domain_notes.clear()
BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear()
primary = FakeLLM([
Resp([_rp("click submit"), _tu("BrowserListInteractives")]),
Resp([_rp("click it"), _tu("BrowserClickIndex", index=1)]),
@@ -691,12 +691,12 @@ def test_replay_success_promotes_skill_to_trusted_through_the_loop(monkeypatch):
asyncio.run(BA.run_browser_agent(
task="click the Search button", browser_id="b1", model="sonnet", initial_url=DOC_URL,
))
assert SK.find_skill("docs.google.com", "click the Search button")["state"] == SK._PROBATION
assert SK.find_skill("docs.google.com", "click the Search button")["state"] == SK.PROBATION
r2 = asyncio.run(BA.run_browser_agent(
task="click the Search button", browser_id="b1", model="sonnet", initial_url=DOC_URL,
))
assert r2.get("replayed") is True
assert SK.find_skill("docs.google.com", "click the Search button")["state"] == SK._TRUSTED
assert SK.find_skill("docs.google.com", "click the Search button")["state"] == SK.TRUSTED
def test_skill_with_send_step_never_replays_silently(monkeypatch):
@@ -705,7 +705,7 @@ def test_skill_with_send_step_never_replays_silently(monkeypatch):
# confirms before anything outward) runs instead, and trust is untouched.
import backend.apps.agents.browser.browser_skills as SK
SK.clear()
BH._browser_history.clear()
BH.BROWSER_HISTORY.clear()
SK.record_skill("docs.google.com", "message tyler saying hi", [
{"tool": "BrowserClickIndex", "input": {}, "ok": True,
"clicked_role": "button", "clicked_name": "Send"},
@@ -718,7 +718,7 @@ def test_skill_with_send_step_never_replays_silently(monkeypatch):
assert not r.get("replayed"), "a send-step skill must never auto-replay"
assert not any(c["action"] == "click_by_name" for c in sent), "the recorded Send was not re-fired"
assert len(primary.calls) > 0, "the live agent ran instead"
assert SK.find_skill("docs.google.com", "message tyler saying hi")["state"] == SK._PROBATION, \
assert SK.find_skill("docs.google.com", "message tyler saying hi")["state"] == SK.PROBATION, \
"skipping replay is not a replay failure; trust stays untouched"
@@ -728,7 +728,7 @@ def test_unproven_skill_that_fails_is_quarantined_and_never_retried(monkeypatch)
# it goes straight to the pure-LLM baseline. A silent re-fail would be a ghost.
import backend.apps.agents.browser.browser_skills as SK
SK.clear()
BH._browser_history.clear()
BH.BROWSER_HISTORY.clear()
SK.record_skill("docs.google.com", "click the Save button", [
{"tool": "BrowserClickIndex", "input": {"index": 1}, "ok": True,
"clicked_role": "button", "clicked_name": "Save"},
@@ -750,7 +750,7 @@ def test_unproven_skill_that_fails_is_quarantined_and_never_retried(monkeypatch)
task="click the Save button", browser_id="b1", model="sonnet", initial_url=DOC_URL,
))
assert any(c["action"] == "click_by_name" for c in sent), "run 1 DID attempt the replay"
assert SK.list_skills("docs.google.com")[0]["state"] == SK._QUARANTINE
assert SK.list_skills("docs.google.com")[0]["state"] == SK.QUARANTINE
# Run 2: the quarantined skill must NOT be replayed again.
sent.clear()
@@ -769,7 +769,7 @@ def test_informational_run_records_no_skill_to_avoid_thin_ghost(monkeypatch):
# falsely claim the whole task done without regenerating the judged list.
import backend.apps.agents.browser.browser_skills as SK
SK.clear()
BH._browser_history.clear()
BH.BROWSER_HISTORY.clear()
ten = "\n".join(f"{i}. Engineer {i}, very cracked, at Startup{i}" for i in range(1, 11))
primary = FakeLLM([
Resp([_rp("search"), _tu("BrowserClickIndex", index=1)]), # a real productive action
@@ -790,7 +790,7 @@ def test_read_answered_from_frontloaded_perception_is_not_a_ghost(monkeypatch):
# a read task straight from that (zero further tools), the honesty gate must
# NOT flag it as 'declared done without taking a single action'. The front-
# loaded reads are real and seed action_log. (This bug caused retry loops.)
BH._browser_history.clear()
BH.BROWSER_HISTORY.clear()
primary = FakeLLM([
# the model answers immediately from the front-loaded page text, no tools
Resp([Blk("text", "The first sentence is: Alan Turing was a mathematician.")], stop_reason="end_turn"),
@@ -819,7 +819,7 @@ def test_ghost_completion_is_reported_as_error_not_completed(monkeypatch):
# and must NOT record a skill from a run that accomplished nothing.
import backend.apps.agents.browser.browser_skills as SK
SK.clear()
BH._browser_history.clear()
BH.BROWSER_HISTORY.clear()
primary = FakeLLM([
Resp([_rp("click submit"), _tu("BrowserClick", selector=".s1")]),
Resp([_rp("retry"), _tu("BrowserClick", selector=".s2")]),
@@ -854,7 +854,7 @@ def test_dead_browser_card_aborts_fast_without_spinning(monkeypatch):
# turns, not the whole budget) and report the precise reason.
import backend.apps.agents.browser.browser_skills as SK
SK.clear()
BH._browser_history.clear()
BH.BROWSER_HISTORY.clear()
# the model would happily keep clicking for 8 turns if we let it
primary = FakeLLM(
[Resp([_rp("click"), _tu("BrowserClick", selector=f".s{i}")]) for i in range(8)]
@@ -890,7 +890,7 @@ def test_hung_browser_card_aborts_fast_not_a_20_minute_loop(monkeypatch):
# feeds the same fast-fail streak and aborts in a couple of turns.
import backend.apps.agents.browser.browser_skills as SK
SK.clear()
BH._browser_history.clear()
BH.BROWSER_HISTORY.clear()
primary = FakeLLM(
[Resp([_rp("read"), _tu("BrowserGetText")]) for _ in range(8)]
+ [Resp([Blk("text", "done")], stop_reason="end_turn")]
@@ -921,7 +921,7 @@ def test_perception_is_frontloaded_into_first_turn(monkeypatch):
# With a known start URL, the agent should prefetch the element list + page
# text and put them in the FIRST user message, so the model can act on turn 1
# instead of spending early turns orienting.
BH._browser_history.clear(); BH._domain_notes.clear()
BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear()
primary = FakeLLM([Resp([Blk("text", "done")], stop_reason="end_turn")])
aux = FakeAux()
_install(monkeypatch, primary, aux)
@@ -939,7 +939,7 @@ def test_prompt_caching_markers_present(monkeypatch):
# The fixed system+tools prefix must carry cache_control so it's cached
# across turns (the first-run speed/cost win). Without the marker the
# ~4k-token prefix is reprocessed every turn.
BH._browser_history.clear(); BH._domain_notes.clear()
BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear()
primary = FakeLLM([Resp([Blk("text", "done")], stop_reason="end_turn")])
aux = FakeAux()
_install(monkeypatch, primary, aux)
@@ -958,7 +958,7 @@ def test_agent_can_list_and_deprecate_its_own_skills(monkeypatch):
# handled, never sent to the webview), giving it agency over its own memory.
import backend.apps.agents.browser.browser_skills as SK
SK.clear()
BH._browser_history.clear()
BH.BROWSER_HISTORY.clear()
# pre-seed a skill on this host
SK.record_skill("docs.google.com", "share the doc now", [
{"tool": "BrowserClickIndex", "input": {}, "ok": True, "clicked_role": "button", "clicked_name": "Share"},
@@ -992,7 +992,7 @@ def test_playbook_distills_on_success_survives_restart_and_seeds_next_run(monkey
import backend.apps.agents.browser.browser_skills as SK
import json as _json
SK.clear(); PB.clear(wipe_disk=True)
BH._browser_history.clear()
BH.BROWSER_HISTORY.clear()
# aux returns a strategy playbook as JSON (the distill+reconcile reply)
class PBAux:
@@ -1025,7 +1025,7 @@ def test_playbook_distills_on_success_survives_restart_and_seeds_next_run(monkey
# Restart: drop in-memory, keep disk.
PB.clear(wipe_disk=False)
assert not PB._cache
assert not PB.CACHE
# Run 2: fresh task, same host -> playbook must be seeded into the system prompt.
primary2 = FakeLLM([Resp([Blk("text", "done")], stop_reason="end_turn")])
@@ -1047,7 +1047,7 @@ def test_ambient_memory_signals_fire_calmly(monkeypatch):
import backend.apps.agents.browser.browser_skills as SK
import json as _json
SK.clear(); PB.clear(wipe_disk=True)
BH._browser_history.clear()
BH.BROWSER_HISTORY.clear()
class PBAux:
def __init__(self): self.messages = self
@@ -1095,7 +1095,7 @@ def test_playbook_not_learned_from_a_ghost_completion(monkeypatch):
import backend.apps.agents.browser.browser_playbook as PB
import backend.apps.agents.browser.browser_skills as SK
SK.clear(); PB.clear(wipe_disk=True)
BH._browser_history.clear()
BH.BROWSER_HISTORY.clear()
class CountingAux:
def __init__(self):
@@ -1126,7 +1126,7 @@ def test_playbook_not_learned_from_a_ghost_completion(monkeypatch):
def test_batch_replay_runs_a_read_loop_for_all_values(monkeypatch):
# The win: do one item the slow way, then BrowserRepeatFlow runs the same
# read flow for the rest at machine speed, one tool turn, no screenshots.
BH._browser_history.clear()
BH.BROWSER_HISTORY.clear()
steps = [{"action": "navigate", "url": "https://docs.google.com/in/{{value}}"},
{"action": "evaluate", "expression": "read('{{value}}')"}]
primary = FakeLLM([
@@ -1159,7 +1159,7 @@ def test_batch_replay_is_ghost_proof_when_an_item_does_not_match(monkeypatch):
# THE anti-ghost test: per-item pages vary. Value 'grace' errors mid-flow ->
# it must be reported as needs-manual, the others still succeed, and the tally
# is HONEST ('2 of 3'), never a silent 'did them all'.
BH._browser_history.clear()
BH.BROWSER_HISTORY.clear()
steps = [{"action": "navigate", "url": "https://docs.google.com/in/{{value}}"},
{"action": "evaluate", "expression": "read('{{value}}')"}]
primary = FakeLLM([
@@ -1191,7 +1191,7 @@ def test_batch_replay_is_ghost_proof_when_an_item_does_not_match(monkeypatch):
def test_batch_replay_refuses_a_send_loop_and_executes_nothing(monkeypatch):
# The send gate: a flow that clicks 'Send message' must be REFUSED outright,
# nothing is clicked, so we can never auto-message N people.
BH._browser_history.clear()
BH.BROWSER_HISTORY.clear()
steps = [{"action": "navigate", "url": "https://docs.google.com/in/{{value}}"},
{"action": "click", "role": "button", "name": "Message"},
{"action": "type", "selector": "#msg", "text": "hi {{value}}"},
@@ -1212,7 +1212,7 @@ def test_batch_replay_refuses_a_send_loop_and_executes_nothing(monkeypatch):
def test_batch_replay_uses_the_fast_network_route_per_value(monkeypatch):
# Folds in the audit finding: a read-loop can hit a captured API endpoint
# (replay_route) per value instead of clicking the UI, the fast tier.
BH._browser_history.clear()
BH.BROWSER_HISTORY.clear()
steps = [{"action": "replay_route", "url": "https://docs.google.com/api/p?u={{value}}"}]
primary = FakeLLM([
Resp([_rp("fetch via api"), _tu("BrowserRepeatFlow", steps=steps, values=["ada", "grace"])]),
@@ -1228,7 +1228,7 @@ def test_captured_routes_are_surfaced_once_per_host(monkeypatch):
# Drives the dead network tier: when a READ shows safe GET routes were captured
# (sampled on get_text, after the SPA's XHRs fired, not on navigate), the agent
# gets a ONE-TIME nudge per host toward BrowserReplayRoute, not on every read.
BH._browser_history.clear()
BH.BROWSER_HISTORY.clear()
primary = FakeLLM([
Resp([_rp("read 1"), _tu("BrowserEvaluate", expression="document.title")]),
Resp([_rp("read 2"), _tu("BrowserEvaluate", expression="document.title")]),
@@ -1253,7 +1253,7 @@ def test_captured_routes_are_surfaced_once_per_host(monkeypatch):
def test_browser_wait_routes_through_smart_wait_and_returns_early(monkeypatch):
# BrowserWait must no longer be a blind sleep: it probes the page (evaluate)
# and returns as soon as it's settled, well under the requested cap.
BH._browser_history.clear()
BH.BROWSER_HISTORY.clear()
primary = FakeLLM([
Resp([_rp("let it settle"), _tu("BrowserWait", milliseconds=8000)]),
Resp([Blk("text", "Settled, moving on.")], stop_reason="end_turn"),
@@ -1271,7 +1271,7 @@ def test_browser_wait_routes_through_smart_wait_and_returns_early(monkeypatch):
def test_prior_domain_hint_is_seeded_into_system_prompt(monkeypatch):
BH._browser_history.clear(); BH._domain_notes.clear()
BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear()
BH.set_domain_note("google.com", "REMEMBERED: Share button is index 43; Tab into the dialog.")
primary = FakeLLM([Resp([Blk("text", "done")], stop_reason="end_turn")])
aux = FakeAux()
@@ -1323,22 +1323,22 @@ def test_find_reusable_card_reuses_own_then_orphan_never_user(monkeypatch):
target = "https://www.linkedin.com/search/results/people/?keywords=t"
# the parent's own same-host card wins
assert BA._find_reusable_card("d1", target, "p1") == "b-own"
assert BA.find_reusable_card("d1", target, "p1") == "b-own"
# a different parent skips p1's... unless that parent finished (orphan); first orphan wins
assert BA._find_reusable_card("d1", target, "p2") == "b-orphan"
assert BA.find_reusable_card("d1", target, "p2") == "b-orphan"
# never a different host
assert BA._find_reusable_card("d1", "https://example.com/", "p1") == ""
assert BA.find_reusable_card("d1", "https://example.com/", "p1") == ""
# an actively-driven card is never grabbed
BA._active_agent_cards.update({"b-own", "b-orphan"})
BA.ACTIVE_AGENT_CARDS.update({"b-own", "b-orphan"})
try:
assert BA._find_reusable_card("d1", target, "p1") == ""
assert BA.find_reusable_card("d1", target, "p1") == ""
finally:
BA._active_agent_cards.clear()
BA.ACTIVE_AGENT_CARDS.clear()
# cards of a still-RUNNING other parent are off limits
class _Running:
status = "running"
monkeypatch.setattr(am_mod.agent_manager, "get_session", lambda sid: _Running(), raising=False)
assert BA._find_reusable_card("d1", target, "p2") == ""
assert BA.find_reusable_card("d1", target, "p2") == ""
def _fake_settle(calls):
@@ -1360,7 +1360,7 @@ def test_post_action_state_settles_then_attaches(monkeypatch):
from backend.apps.agents.browser import browser_agent as ba
calls = []
monkeypatch.setattr(ba.browser_wait, "smart_wait", _fake_settle(calls))
out = asyncio.run(ba._post_action_state(
out = asyncio.run(ba.post_action_state(
"BrowserClickIndex", {"index": 1}, {"text": "Clicked"},
"b1", "", _fake_exec(calls), "find tyler",
))
@@ -1374,7 +1374,7 @@ def test_post_action_state_navigate_gets_longer_settle(monkeypatch):
from backend.apps.agents.browser import browser_agent as ba
calls = []
monkeypatch.setattr(ba.browser_wait, "smart_wait", _fake_settle(calls))
asyncio.run(ba._post_action_state(
asyncio.run(ba.post_action_state(
"BrowserNavigate", {"url": "https://x.com"}, {"text": "Navigated"},
"b1", "", _fake_exec(calls), "",
))
@@ -1386,7 +1386,7 @@ def test_post_action_state_expect_skips_double_settle(monkeypatch):
from backend.apps.agents.browser import browser_agent as ba
calls = []
monkeypatch.setattr(ba.browser_wait, "smart_wait", _fake_settle(calls))
out = asyncio.run(ba._post_action_state(
out = asyncio.run(ba.post_action_state(
"BrowserClickIndex", {"index": 2, "expect": "Sent"}, {"text": "Clicked"},
"b1", "", _fake_exec(calls), "",
))
@@ -1400,13 +1400,13 @@ def test_post_action_state_skips_errors_reads_and_batch_reads(monkeypatch):
calls = []
monkeypatch.setattr(ba.browser_wait, "smart_wait", _fake_settle(calls))
exec_fn = _fake_exec(calls)
assert asyncio.run(ba._post_action_state(
assert asyncio.run(ba.post_action_state(
"BrowserClickIndex", {"index": 1}, {"error": "nope"}, "b", "", exec_fn, "")) == ""
assert asyncio.run(ba._post_action_state(
assert asyncio.run(ba.post_action_state(
"BrowserGetText", {}, {"text": "page text"}, "b", "", exec_fn, "")) == ""
batch_in = {"actions": [{"type": "click_index", "params": {"index": 1}},
{"type": "list_interactives", "params": {}}]}
assert asyncio.run(ba._post_action_state(
assert asyncio.run(ba.post_action_state(
"BrowserBatch", batch_in, {"text": "ran 2"}, "b", "", exec_fn, "")) == ""
assert calls == []
@@ -1417,7 +1417,7 @@ def test_post_action_state_truncates_long_lists(monkeypatch):
calls = []
monkeypatch.setattr(ba.browser_wait, "smart_wait", _fake_settle(calls))
long_list = "\n".join(f'[{i}]<button "b{i}">' for i in range(60))
out = asyncio.run(ba._post_action_state(
out = asyncio.run(ba.post_action_state(
"BrowserType", {"selector": "#q", "text": "hi"}, {"text": "Typed"},
"b1", "", _fake_exec(calls, long_list), "",
))
@@ -1431,7 +1431,7 @@ def test_post_action_state_hung_settle_attaches_nothing(monkeypatch):
async def hung_wait(execute_fn, browser_id, tab_id, max_ms, **kw):
return {"settled": False, "hung": True}
monkeypatch.setattr(ba.browser_wait, "smart_wait", hung_wait)
out = asyncio.run(ba._post_action_state(
out = asyncio.run(ba.post_action_state(
"BrowserClick", {"selector": "a"}, {"text": "Clicked"},
"b1", "", _fake_exec(calls), "",
))
@@ -1439,20 +1439,20 @@ def test_post_action_state_hung_settle_attaches_nothing(monkeypatch):
def test_delta_state_first_attach_sends_full_list():
from backend.apps.agents.browser.browser_agent import _delta_state
from backend.apps.agents.browser.browser_agent import delta_state
seen = set()
full = "8 interactive elements:\n" + "\n".join(f'[{i}]<button "b{i}">' for i in range(1, 9))
assert _delta_state(full, seen) == full
assert delta_state(full, seen) == full
assert len(seen) == 8
def test_delta_state_shrinks_to_changed_rows():
from backend.apps.agents.browser.browser_agent import _delta_state
from backend.apps.agents.browser.browser_agent import delta_state
rows = [f'[{i}]<button "b{i}">' for i in range(1, 11)]
seen = set()
_delta_state("\n".join(rows), seen)
delta_state("\n".join(rows), seen)
nxt = rows[:9] + ['[10]<button "b10" value="typed">', '[11]*<button "new">']
out = _delta_state("\n".join(nxt), seen)
out = delta_state("\n".join(nxt), seen)
assert '[11]*<button "new">' in out and 'value="typed"' in out
assert '[3]<button "b3">' not in out
assert "+9 rows unchanged" in out
@@ -1460,20 +1460,20 @@ def test_delta_state_shrinks_to_changed_rows():
def test_delta_state_no_changes_collapses_to_one_line():
from backend.apps.agents.browser.browser_agent import _delta_state
from backend.apps.agents.browser.browser_agent import delta_state
rows = "\n".join(f'[{i}]<link "l{i}">' for i in range(1, 13))
seen = set()
_delta_state(rows, seen)
out = _delta_state(rows, seen)
delta_state(rows, seen)
out = delta_state(rows, seen)
assert out.startswith("(all 12 element rows unchanged")
def test_delta_state_reshuffle_resends_full():
from backend.apps.agents.browser.browser_agent import _delta_state
from backend.apps.agents.browser.browser_agent import delta_state
seen = set()
_delta_state("\n".join(f'[{i}]<button "a{i}">' for i in range(1, 11)), seen)
delta_state("\n".join(f'[{i}]<button "a{i}">' for i in range(1, 11)), seen)
new_page = "10 interactive elements:\n" + "\n".join(f'[{i}]<button "z{i}">' for i in range(1, 11))
assert _delta_state(new_page, seen) == new_page
assert delta_state(new_page, seen) == new_page
def test_informational_gate_judges_the_task_ask_first():
@@ -1553,58 +1553,58 @@ def test_recoverable_tool_error_classifier():
def test_message_pairing_validator_catches_both_orphan_and_dangling():
from backend.apps.agents.browser.browser_history import _validate_message_pairing
from backend.apps.agents.browser.browser_history import validate_message_pairing
au = lambda i: {"role": "assistant", "content": [{"type": "tool_use", "id": i, "name": "X", "input": {}}]}
tr = lambda i: {"role": "user", "content": [{"type": "tool_result", "tool_use_id": i, "content": []}]}
# well-formed: every tool_use answered
assert _validate_message_pairing([au("t1"), tr("t1")]) is True
assert validate_message_pairing([au("t1"), tr("t1")]) is True
# DANGLING tool_use (the exact 400: a call with no result) -> invalid
assert _validate_message_pairing([au("t1")]) is False
assert _validate_message_pairing([au("t1"), tr("t1"), au("t2")]) is False
assert validate_message_pairing([au("t1")]) is False
assert validate_message_pairing([au("t1"), tr("t1"), au("t2")]) is False
# ORPHAN tool_result (result for a never-declared id) -> invalid
assert _validate_message_pairing([tr("ghost")]) is False
assert validate_message_pairing([tr("ghost")]) is False
# plain text turns are fine
assert _validate_message_pairing([{"role": "user", "content": "hi"},
assert validate_message_pairing([{"role": "user", "content": "hi"},
{"role": "assistant", "content": "done"}]) is True
def test_composer_fill_detection():
# detecting a composer fill is what arms the post-type wait for the Send button
# to render before we re-list (so the model sees it instead of hunting)
from backend.apps.agents.browser.browser_agent import _is_composer_fill
assert _is_composer_fill("BrowserClickIndex", {"index": 4, "text": "hello world"})
assert _is_composer_fill("BrowserType", {"selector": "#m", "text": "hi"})
assert _is_composer_fill("BrowserBatch", {"actions": [
from backend.apps.agents.browser.browser_agent import is_composer_fill
assert is_composer_fill("BrowserClickIndex", {"index": 4, "text": "hello world"})
assert is_composer_fill("BrowserType", {"selector": "#m", "text": "hi"})
assert is_composer_fill("BrowserBatch", {"actions": [
{"type": "click_index", "params": {"index": 4, "text": "hi there"}}]})
# a plain click (no text) is NOT a fill
assert not _is_composer_fill("BrowserClickIndex", {"index": 4})
assert not _is_composer_fill("BrowserScroll", {})
assert not is_composer_fill("BrowserClickIndex", {"index": 4})
assert not is_composer_fill("BrowserScroll", {})
def test_send_index_handoff_points_only_at_a_real_send_button():
# after a composer fill we hand the model the Send button's index so it clicks
# it directly instead of hunting; must never mistake an upsell/profile link for it
from backend.apps.agents.browser.browser_agent import _send_index_in_state
from backend.apps.agents.browser.browser_agent import send_index_in_state
page = '[1]<link "Tyler Chen">\n[33]<textbox "Write a message">\n[44]<button "Send">'
assert _send_index_in_state(page) == (44, "Send")
assert _send_index_in_state('[12]<button "Send InMail credit">') is None
assert _send_index_in_state('[5]<button "Send a message to Maya">') is None
assert _send_index_in_state("") is None
assert send_index_in_state(page) == (44, "Send")
assert send_index_in_state('[12]<button "Send InMail credit">') is None
assert send_index_in_state('[5]<button "Send a message to Maya">') is None
assert send_index_in_state("") is None
def test_strip_lone_surrogates():
from backend.apps.agents.browser.browser_agent import _strip_lone_surrogates, _format_tool_result
from backend.apps.agents.browser.browser_agent import strip_lone_surrogates, format_tool_result
# an orphan UTF-16 surrogate (half an emoji from the webview) is what crashes
# the turn at .encode('utf-8'); it must be swapped, not carried through
out = _strip_lone_surrogates("Twitch \ud83e live")
out = strip_lone_surrogates("Twitch \ud83e live")
assert "\ud83e" not in out and "�" in out
out.encode("utf-8") # the operation that used to raise "surrogates not allowed"
# valid emoji (a real code point) and plain text are left alone
assert _strip_lone_surrogates("cheese \U0001f9c0 ok") == "cheese \U0001f9c0 ok"
assert _strip_lone_surrogates("Search Amazon") == "Search Amazon"
assert _strip_lone_surrogates("") == ""
assert strip_lone_surrogates("cheese \U0001f9c0 ok") == "cheese \U0001f9c0 ok"
assert strip_lone_surrogates("Search Amazon") == "Search Amazon"
assert strip_lone_surrogates("") == ""
# the boundary that feeds the model is sanitized for both result and error text
blocks = _format_tool_result({"text": "name \ud83e here"}, "BrowserListInteractives")
blocks = format_tool_result({"text": "name \ud83e here"}, "BrowserListInteractives")
blocks[0]["text"].encode("utf-8")
err = _format_tool_result({"error": "bad \ud83e node"}, "BrowserClickIndex")
err = format_tool_result({"error": "bad \ud83e node"}, "BrowserClickIndex")
err[0]["text"].encode("utf-8")
backend/tests/test_browser_fast_path.py
+16 -16
View File
@@ -1,6 +1,6 @@
from backend.apps.agents.browser.browser_fast_path import (
_normalize_for_classifier,
_parse_verdict_and_brief,
normalize_for_classifier,
parse_verdict_and_brief,
compose_task,
dispatch_failed,
fast_path_eligible,
@@ -30,15 +30,15 @@ def test_non_browsy_or_gated_messages_fall_through():
def test_verdict_parsing_is_strict():
v, brief = _parse_verdict_and_brief("READ\nENTRY: https://news.ycombinator.com\n1. read top story")
v, brief = parse_verdict_and_brief("READ\nENTRY: https://news.ycombinator.com\n1. read top story")
assert v == "read" and brief.startswith("ENTRY:") and "top story" in brief
assert _parse_verdict_and_brief("ACT\nENTRY: https://x.com")[0] == "act"
assert _parse_verdict_and_brief("yes") == ("act", "")
assert _parse_verdict_and_brief("NO") == ("no", "")
assert _parse_verdict_and_brief("Maybe\nENTRY: x") == ("no", "")
assert _parse_verdict_and_brief("") == ("no", "")
assert parse_verdict_and_brief("ACT\nENTRY: https://x.com")[0] == "act"
assert parse_verdict_and_brief("yes") == ("act", "")
assert parse_verdict_and_brief("NO") == ("no", "")
assert parse_verdict_and_brief("Maybe\nENTRY: x") == ("no", "")
assert parse_verdict_and_brief("") == ("no", "")
long_brief = "ACT\n" + "x" * 2000
assert len(_parse_verdict_and_brief(long_brief)[1]) == 700
assert len(parse_verdict_and_brief(long_brief)[1]) == 700
def test_fast_read_entry_extraction_and_thin_detection():
@@ -84,14 +84,14 @@ def test_recovery_task_verifies_before_repeating():
def test_text_normalizes_to_message_without_phone_number():
assert (
_normalize_for_classifier("go to maya's linkedin and text her thanks")
normalize_for_classifier("go to maya's linkedin and text her thanks")
== "go to maya's linkedin and message her thanks"
)
assert _normalize_for_classifier("keep texting until he replies").startswith("keep message")
assert normalize_for_classifier("keep texting until he replies").startswith("keep message")
sms = "text 4085551234 saying im running late"
assert _normalize_for_classifier(sms) == sms
assert normalize_for_classifier(sms) == sms
count = "count messages containing the exact text r10-os"
assert "message r10-os" in _normalize_for_classifier(count)
assert "message r10-os" in normalize_for_classifier(count)
def test_dispatch_refused_when_no_dashboard_connected(monkeypatch):
@@ -163,7 +163,7 @@ def test_entry_url_extracted_from_brief():
def test_results_url_shapes():
from backend.apps.agents.browser.browser_agent import _RESULTS_URL_RE
from backend.apps.agents.browser.browser_agent import RESULTS_URL_RE
hits = [
"https://www.linkedin.com/search/results/people/?keywords=tyler+chen",
"https://www.google.com/search?q=anything",
@@ -176,6 +176,6 @@ def test_results_url_shapes():
"https://www.linkedin.com/messaging/thread/abc123/",
]
for u in hits:
assert _RESULTS_URL_RE.search(u), u
assert RESULTS_URL_RE.search(u), u
for u in misses:
assert not _RESULTS_URL_RE.search(u), u
assert not RESULTS_URL_RE.search(u), u
backend/tests/test_browser_hints.py
+2 -2
View File
@@ -4,7 +4,7 @@ from backend.apps.agents.browser import browser_history as bh
def setup_function(_):
bh._domain_notes.clear()
bh.DOMAIN_NOTES.clear()
def test_set_get_roundtrip():
@@ -14,7 +14,7 @@ def test_set_get_roundtrip():
def test_caps_length():
bh.set_domain_note("x.com", "a" * 5000)
assert len(bh.get_domain_note("x.com")) == bh._MAX_DOMAIN_NOTE_CHARS
assert len(bh.get_domain_note("x.com")) == bh.MAX_DOMAIN_NOTE_CHARS
def test_ignores_empty_domain_or_note():
backend/tests/test_browser_hotpath_waste.py
+13 -13
View File
@@ -1,23 +1,23 @@
"""Hot-path waste removals in the browser sub-agent loop.
Two per-action costs that were pure waste:
1. browser_metrics._metrics_dir() ran os.makedirs() on EVERY tool call.
1. browser_metrics.metrics_dir() ran os.makedirs() on EVERY tool call.
2. The loop-detection hash serialized a tool's full result (a ~1MB screenshot
or 15KB read) even for tools that are excluded from loop detection, where
_detect_loop ignores the hash entirely. These pin both fixes.
detect_loop ignores the hash entirely. These pin both fixes.
"""
import os
import backend.apps.agents.browser.browser_metrics as M
from backend.apps.agents.browser.browser_loop import (
_detect_loop,
_LOOP_DETECTION_EXCLUDED_TOOLS,
detect_loop,
LOOP_DETECTION_EXCLUDED_TOOLS,
)
def test_metrics_dir_is_cached_makedirs_runs_once(monkeypatch):
M._metrics_dir_cache = None
M.p_metrics_dir_cache = None
calls = {"n": 0}
real = os.makedirs
@@ -26,9 +26,9 @@ def test_metrics_dir_is_cached_makedirs_runs_once(monkeypatch):
return real(*a, **k)
monkeypatch.setattr(os, "makedirs", counting)
d1 = M._metrics_dir()
d2 = M._metrics_dir()
d3 = M._metrics_dir()
d1 = M.metrics_dir()
d2 = M.metrics_dir()
d3 = M.metrics_dir()
assert d1 == d2 == d3
assert calls["n"] == 1, f"makedirs must run once, ran {calls['n']}x"
@@ -37,9 +37,9 @@ def test_excluded_tools_never_register_a_loop():
# The invariant the hash-skip relies on: for every excluded tool, even ten
# identical calls in a row are NOT a loop, so computing/storing the hash for
# them was dead work. Setting is_loop=False directly is therefore equivalent.
for tool in _LOOP_DETECTION_EXCLUDED_TOOLS:
for tool in LOOP_DETECTION_EXCLUDED_TOOLS:
key = (tool, "in", "out")
assert _detect_loop([key] * 10, key) is False, f"{tool} wrongly looped"
assert detect_loop([key] * 10, key) is False, f"{tool} wrongly looped"
def test_non_excluded_tool_still_loops_after_threshold():
@@ -47,6 +47,6 @@ def test_non_excluded_tool_still_loops_after_threshold():
# that need it (clicks/types/etc.).
key = ("BrowserClick", '{"selector":"#x"}', "clicked")
# below threshold -> not a loop; at/over threshold within the window -> loop
assert _detect_loop([], key) is False # 1st occurrence: not yet a wall
assert _detect_loop([key], key) is True # 2nd identical (threshold=2): a wall
assert _detect_loop([key] * 5, key) is True
assert detect_loop([], key) is False # 1st occurrence: not yet a wall
assert detect_loop([key], key) is True # 2nd identical (threshold=2): a wall
assert detect_loop([key] * 5, key) is True
backend/tests/test_browser_meta_playbook.py
+2 -2
View File
@@ -33,13 +33,13 @@ def test_absorb_adds_dedups_and_caps():
assert meta.absorb([]) is False
# capped: flooding never exceeds the cap
meta.absorb([f"unique universal lesson number {i}" for i in range(50)])
assert len(meta.get_meta()) <= meta._MAX_BULLETS
assert len(meta.get_meta()) <= meta.MAX_BULLETS
def test_survives_a_restart():
meta.absorb(["a durable cross-site lesson worth keeping"])
meta.clear(wipe_disk=False) # in-memory gone, disk intact (== restart)
assert meta._cache is None
assert meta.CACHE is None
assert any("durable cross-site lesson" in x for x in meta.get_meta())
backend/tests/test_browser_metrics.py
+3 -3
View File
@@ -14,7 +14,7 @@ def metrics(monkeypatch):
from backend.apps.agents.browser import browser_metrics as bm
# The dir is memoized once for the prod hot path; drop the cache so each test
# re-resolves to its own temp dir instead of inheriting a prior test's.
bm._metrics_dir_cache = None
bm.p_metrics_dir_cache = None
return bm, d
@@ -83,7 +83,7 @@ def test_metrics_never_raises_on_bad_dir(monkeypatch):
# An unwritable dir must not throw into the agent loop.
monkeypatch.setenv("OPENSWARM_BROWSER_METRICS_DIR", "/proc/cannot/write/here")
from backend.apps.agents.browser import browser_metrics as bm
bm._metrics_dir_cache = None # re-resolve so we actually hit the bad dir
bm.p_metrics_dir_cache = None # re-resolve so we actually hit the bad dir
bm.record_tool("s", "b", 1, "BrowserScreenshot", 5, ok=True, error="",
is_loop=False, stagnation_streak=0, result_len=1) # must not raise
bm.record_task("s", "b", "t", "error", __import__("time").time(), 1, [], {})
@@ -94,7 +94,7 @@ def test_task_secrets_are_scrubbed_from_tasks_jsonl(tmp_path, monkeypatch):
import os as _os
import time as _time
monkeypatch.setenv("OPENSWARM_BROWSER_METRICS_DIR", str(tmp_path))
bm._metrics_dir_cache = None
bm.p_metrics_dir_cache = None
bm.record_task("s1", "b1", "log into acme with password hunter2 then post sk-abc12345678901234567",
"completed", _time.time() - 1, 2, [], {})
line = open(_os.path.join(str(tmp_path), "tasks.jsonl")).read()
backend/tests/test_browser_playbook.py
+4 -4
View File
@@ -103,7 +103,7 @@ async def test_playbook_is_capped():
pb.clear(wipe_disk=True)
many = [f"strategy bullet number {i}" for i in range(20)]
await _distill("big.com", "t", "m", "s", FakeAux(_pb(*many)))
assert len(pb.get_playbook("big.com")) <= pb._MAX_BULLETS
assert len(pb.get_playbook("big.com")) <= pb.MAX_BULLETS
@pytest.mark.asyncio
@@ -119,7 +119,7 @@ async def test_playbook_survives_a_restart():
pb.clear(wipe_disk=True)
await _distill("persist.com", "t", "m", "s", FakeAux(_pb("durable lesson one", "durable lesson two")))
pb.clear(wipe_disk=False) # restart: memory gone, disk intact
assert not pb._cache
assert not pb.CACHE
bullets = pb.get_playbook("persist.com")
assert len(bullets) == 2 and "durable lesson one" in bullets
@@ -184,6 +184,6 @@ def test_seed_playbook_fallback_and_supersede():
assert seeded and any("github.com/search" in b for b in seeded)
# a learned playbook supersedes the seed (real usage wins)
pb.clear(wipe_disk=True)
pb._persist("github.com", ["learned: use the org filter"])
pb._cache.clear()
pb.persist("github.com", ["learned: use the org filter"])
pb.CACHE.clear()
assert pb.get_playbook("github.com") == ["learned: use the org filter"]
backend/tests/test_browser_save.py
+9 -9
View File
@@ -4,7 +4,7 @@ and extension allowlist are tested hard, including hostile filenames."""
import json
import os
from backend.apps.agents.browser.browser_save import save_page_data, _ALLOWED_EXT, _MAX_BYTES, _SUBDIR
from backend.apps.agents.browser.browser_save import save_page_data, ALLOWED_EXT, MAX_BYTES, SUBDIR
def test_happy_path_writes_into_browser_data_subdir(tmp_path):
@@ -12,7 +12,7 @@ def test_happy_path_writes_into_browser_data_subdir(tmp_path):
msg = save_page_data(str(tmp_path), "sid", "rows.json", payload)
assert msg.startswith("Saved")
assert "3 items" in msg
out = tmp_path / _SUBDIR / "rows.json"
out = tmp_path / SUBDIR / "rows.json"
assert out.is_file()
assert json.loads(out.read_text()) == json.loads(payload)
@@ -26,7 +26,7 @@ def test_traversal_filename_is_confined_not_escaped(tmp_path):
# a '../../evil.json' must NOT land outside the sandbox; basename flattens it
msg = save_page_data(str(tmp_path), "sid", "../../evil.json", "[]")
assert msg.startswith("Saved")
assert (tmp_path / _SUBDIR / "evil.json").is_file()
assert (tmp_path / SUBDIR / "evil.json").is_file()
# nothing was written two levels up
assert not (tmp_path.parent.parent / "evil.json").exists()
@@ -34,7 +34,7 @@ def test_traversal_filename_is_confined_not_escaped(tmp_path):
def test_absolute_path_filename_is_confined(tmp_path):
msg = save_page_data(str(tmp_path), "sid", "/etc/evil.json", "[]")
assert msg.startswith("Saved")
assert (tmp_path / _SUBDIR / "evil.json").is_file()
assert (tmp_path / SUBDIR / "evil.json").is_file()
assert not os.path.exists("/etc/evil.json")
@@ -43,7 +43,7 @@ def test_disallowed_extension_is_rejected(tmp_path):
msg = save_page_data(str(tmp_path), "sid", bad, "data")
assert msg.startswith("Save failed"), bad
# the allowed ones all pass
for good in sorted(_ALLOWED_EXT):
for good in sorted(ALLOWED_EXT):
msg = save_page_data(str(tmp_path), "sid", f"file{good}", "x")
assert msg.startswith("Saved"), good
@@ -54,21 +54,21 @@ def test_empty_filename_is_rejected(tmp_path):
def test_oversize_payload_is_rejected(tmp_path):
big = "x" * (_MAX_BYTES + 1)
big = "x" * (MAX_BYTES + 1)
msg = save_page_data(str(tmp_path), "sid", "big.txt", big)
assert msg.startswith("Save failed")
assert not (tmp_path / _SUBDIR / "big.txt").exists()
assert not (tmp_path / SUBDIR / "big.txt").exists()
def test_falls_back_to_home_workspace_when_no_cwd(tmp_path, monkeypatch):
monkeypatch.setattr(os.path, "expanduser", lambda p: str(tmp_path))
msg = save_page_data(None, "sess-xyz", "f.json", "[]")
assert msg.startswith("Saved")
assert (tmp_path / ".openswarm" / "workspaces" / "sess-xyz" / _SUBDIR / "f.json").is_file()
assert (tmp_path / ".openswarm" / "workspaces" / "sess-xyz" / SUBDIR / "f.json").is_file()
def test_non_json_content_still_saves_without_count(tmp_path):
msg = save_page_data(str(tmp_path), "sid", "notes.txt", "just some text")
assert msg.startswith("Saved")
assert "items" not in msg and "keys" not in msg
assert (tmp_path / _SUBDIR / "notes.txt").read_text() == "just some text"
assert (tmp_path / SUBDIR / "notes.txt").read_text() == "just some text"
backend/tests/test_browser_screenshot_prune.py
+3 -3
View File
@@ -8,7 +8,7 @@ text stay). These pin the keep-set, the in-place mutation, and tool_result safet
from backend.apps.agents.browser.browser_history import (
prune_old_screenshots,
_OMITTED_SCREENSHOT_STUB,
OMITTED_SCREENSHOT_STUB,
)
@@ -58,7 +58,7 @@ def test_stub_preserves_the_url_text_block():
prune_old_screenshots(msgs)
# the collapsed shot (#1) keeps its "URL:" text, only the image became a stub
collapsed_tr = msgs[1]["content"][0]["content"]
assert any(b.get("text") == _OMITTED_SCREENSHOT_STUB for b in collapsed_tr)
assert any(b.get("text") == OMITTED_SCREENSHOT_STUB for b in collapsed_tr)
assert any("URL: https://site/1" in b.get("text", "") for b in collapsed_tr)
@@ -71,7 +71,7 @@ def test_handles_direct_image_blocks_too():
]
collapsed = prune_old_screenshots(msgs)
assert collapsed == 1 # keep a (first), c+d (last two); stub b
assert msgs[1]["content"][0] == {"type": "text", "text": _OMITTED_SCREENSHOT_STUB}
assert msgs[1]["content"][0] == {"type": "text", "text": OMITTED_SCREENSHOT_STUB}
def test_keep_recent_is_tunable():
backend/tests/test_browser_self_audit.py
+3 -3
View File
@@ -65,9 +65,9 @@ def test_audit_fires_every_n_finished_tasks(monkeypatch, tmp_path):
# threads synchronous so the test is deterministic, and use a small N.
from backend.apps.agents.browser import browser_metrics as m
monkeypatch.setenv("OPENSWARM_BROWSER_METRICS_DIR", str(tmp_path))
m._metrics_dir_cache = None
m._task_count = 0
monkeypatch.setattr(m, "_AUDIT_EVERY_N", 5)
m.p_metrics_dir_cache = None
m.p_task_count = 0
monkeypatch.setattr(m, "P_AUDIT_EVERY_N", 5)
class _SyncThread:
def __init__(self, target=None, **kw):
backend/tests/test_browser_skills.py
+55 -55
View File
@@ -124,7 +124,7 @@ def test_skill_persists_across_restart(_isolated_skills):
# find must re-load it from disk.
assert sk.record_skill("localhost:8901", "type hello and click Send", _log()) is True
sk.clear(wipe_disk=False) # in-memory gone, disk intact (== restart)
assert not sk._skills # cache truly empty
assert not sk.SKILLS # cache truly empty
found = sk.find_skill("localhost:8901", "type hello and click Send")
assert found is not None and found.get("persisted") is True
assert [s["tool"] for s in found["steps"]] == ["BrowserNavigate", "BrowserType", "BrowserClickByName"]
@@ -138,7 +138,7 @@ def test_sensitive_text_is_NOT_persisted(_isolated_skills):
]
assert sk.record_skill("site.com", "enter email and submit", log) is True # stored in memory
# nothing on disk for this skill
path = sk._skill_path("site.com", sk.normalize_task("enter email and submit"))
path = sk.skill_path("site.com", sk.normalize_task("enter email and submit"))
assert path is not None and not os.path.exists(path)
# and after a "restart" it's gone (was never persisted)
sk.clear(wipe_disk=False)
@@ -151,18 +151,18 @@ def test_password_field_selector_blocks_persistence(_isolated_skills):
{"tool": "BrowserClickIndex", "input": {}, "ok": True, "clicked_role": "button", "clicked_name": "Log in"},
]
sk.record_skill("site.com", "log in", log)
assert not os.path.exists(sk._skill_path("site.com", sk.normalize_task("log in")))
assert not os.path.exists(sk.skill_path("site.com", sk.normalize_task("log in")))
def test_sensitivity_detector():
assert sk._looks_sensitive("eric@example.com")
assert sk._looks_sensitive("4111 1111 1111 1111") # card-shaped
assert sk._looks_sensitive("123-45-6789") # ssn
assert sk._looks_sensitive("sk-ant-api03-abc123") # token prefix
assert sk._looks_sensitive("anything", selector="#pwd") # password field
assert sk._looks_sensitive("aB3xK9mQ2pL7wR4tY8nZ") # long high-entropy
assert not sk._looks_sensitive("hello world")
assert not sk._looks_sensitive("openswarm", selector="#search")
assert sk.looks_sensitive("eric@example.com")
assert sk.looks_sensitive("4111 1111 1111 1111") # card-shaped
assert sk.looks_sensitive("123-45-6789") # ssn
assert sk.looks_sensitive("sk-ant-api03-abc123") # token prefix
assert sk.looks_sensitive("anything", selector="#pwd") # password field
assert sk.looks_sensitive("aB3xK9mQ2pL7wR4tY8nZ") # long high-entropy
assert not sk.looks_sensitive("hello world")
assert not sk.looks_sensitive("openswarm", selector="#search")
def test_navigate_url_userinfo_and_fragment_stripped_on_disk(_isolated_skills):
@@ -172,7 +172,7 @@ def test_navigate_url_userinfo_and_fragment_stripped_on_disk(_isolated_skills):
]
# userinfo in the URL makes the whole skill non-persistable (credentialed URL)
sk.record_skill("site.com", "search shoes", log)
assert not os.path.exists(sk._skill_path("site.com", sk.normalize_task("search shoes")))
assert not os.path.exists(sk.skill_path("site.com", sk.normalize_task("search shoes")))
# but a clean URL with a fragment persists with the fragment stripped
log2 = [
{"tool": "BrowserNavigate", "input": {"url": "https://site.com/app#section"}, "ok": True},
@@ -189,7 +189,7 @@ def test_navigate_url_userinfo_and_fragment_stripped_on_disk(_isolated_skills):
def test_format_version_mismatch_is_ignored(_isolated_skills, monkeypatch):
sk.record_skill("v.com", "do a thing now", _log())
sk.clear(wipe_disk=False)
monkeypatch.setattr(sk, "_SKILL_FORMAT_VERSION", 999) # pretend the format moved on
monkeypatch.setattr(sk, "P_SKILL_FORMAT_VERSION", 999) # pretend the format moved on
assert sk.find_skill("v.com", "do a thing now") is None
@@ -216,7 +216,7 @@ def test_parameterized_value_is_not_persisted(_isolated_skills):
{"tool": "BrowserClickIndex", "input": {}, "ok": True, "clicked_role": "button", "clicked_name": "Search"},
]
sk.record_skill("shop.com", 'search for "running shoes"', log)
path = sk._skill_path("shop.com", sk._sig('search for "running shoes"'))
path = sk.skill_path("shop.com", sk.compute_sig('search for "running shoes"'))
blob = open(path).read()
assert "running shoes" not in blob # the quoted value never hits disk
assert '"value_slot": 0' in blob or '"value_slot":0' in blob
@@ -263,11 +263,11 @@ def test_list_skills_reads_disk_after_restart(_isolated_skills):
def test_deprecate_removes_skill_from_memory_and_disk(_isolated_skills):
sk.record_skill("shop.com", "search for shoes now", _log())
sig = sk._sig("search for shoes now")
assert os.path.exists(sk._skill_path("shop.com", sig))
sig = sk.compute_sig("search for shoes now")
assert os.path.exists(sk.skill_path("shop.com", sig))
# deprecate using the task_sig as list_skills would surface it
assert sk.deprecate_skill("shop.com", sig) is True
assert not os.path.exists(sk._skill_path("shop.com", sig))
assert not os.path.exists(sk.skill_path("shop.com", sig))
assert sk.find_skill("shop.com", "search for shoes now") is None
@@ -283,14 +283,14 @@ def test_deprecate_unknown_is_false(_isolated_skills):
def test_new_skill_starts_on_probation(_isolated_skills):
sk.record_skill("shop.com", "do a thing now", _log())
s = sk.find_skill("shop.com", "do a thing now")
assert s["state"] == sk._PROBATION and s["rev"] == 1 and s["replays"] == 0
assert s["state"] == sk.PROBATION and s["rev"] == 1 and s["replays"] == 0
def test_replay_success_promotes_probation_to_trusted(_isolated_skills):
sk.record_skill("shop.com", "do a thing now", _log())
sk.mark_replay_succeeded("shop.com", "do a thing now")
s = sk.find_skill("shop.com", "do a thing now")
assert s["state"] == sk._TRUSTED and s["replays"] == 1 and s["fails"] == 0
assert s["state"] == sk.TRUSTED and s["replays"] == 1 and s["fails"] == 0
def test_probation_failure_quarantines_and_blocks_future_replay(_isolated_skills):
@@ -301,7 +301,7 @@ def test_probation_failure_quarantines_and_blocks_future_replay(_isolated_skills
assert sk.find_skill("shop.com", "do a thing now") is None
# ...but the record still exists (visible + deprecatable), it just won't run
listed = sk.list_skills("shop.com")
assert len(listed) == 1 and listed[0]["state"] == sk._QUARANTINE
assert len(listed) == 1 and listed[0]["state"] == sk.QUARANTINE
def test_quarantined_skill_re_recorded_identical_stays_quarantined(_isolated_skills):
@@ -311,7 +311,7 @@ def test_quarantined_skill_re_recorded_identical_stays_quarantined(_isolated_ski
sk.record_skill("shop.com", "do a thing now", _log())
# it must stay quarantined -> pure-LLM baseline, never a wasted replay again
assert sk.find_skill("shop.com", "do a thing now") is None
assert sk.list_skills("shop.com")[0]["state"] == sk._QUARANTINE
assert sk.list_skills("shop.com")[0]["state"] == sk.QUARANTINE
def test_quarantined_skill_unquarantines_on_a_real_edit(_isolated_skills):
@@ -323,7 +323,7 @@ def test_quarantined_skill_unquarantines_on_a_real_edit(_isolated_skills):
"clicked_role": "button", "clicked_name": "Submit"}]
sk.record_skill("shop.com", "do a thing now", edited)
s = sk.find_skill("shop.com", "do a thing now")
assert s is not None and s["state"] == sk._PROBATION and s["rev"] == 2
assert s is not None and s["state"] == sk.PROBATION and s["rev"] == 2
def test_trusted_skill_tolerates_one_transient_miss_then_demotes(_isolated_skills):
@@ -331,9 +331,9 @@ def test_trusted_skill_tolerates_one_transient_miss_then_demotes(_isolated_skill
sk.mark_replay_succeeded("shop.com", "do a thing now") # trusted
assert sk.mark_replay_failed("shop.com", "do a thing now") == "kept"
s = sk.find_skill("shop.com", "do a thing now")
assert s["state"] == sk._TRUSTED and s["fails"] == 1 # still usable
assert s["state"] == sk.TRUSTED and s["fails"] == 1 # still usable
assert sk.mark_replay_failed("shop.com", "do a thing now") == "demoted"
assert sk.find_skill("shop.com", "do a thing now")["state"] == sk._PROBATION
assert sk.find_skill("shop.com", "do a thing now")["state"] == sk.PROBATION
def test_re_record_identical_keeps_trust_and_rev(_isolated_skills):
@@ -342,7 +342,7 @@ def test_re_record_identical_keeps_trust_and_rev(_isolated_skills):
sk.find_skill("shop.com", "do a thing now")["replays"] = 5 # pretend reused a lot
sk.record_skill("shop.com", "do a thing now", _log()) # identical re-derive
s = sk.find_skill("shop.com", "do a thing now")
assert s["state"] == sk._TRUSTED and s["rev"] == 1 and s["replays"] == 5
assert s["state"] == sk.TRUSTED and s["rev"] == 1 and s["replays"] == 5
def test_re_record_different_is_an_edit_that_reversions_to_probation(_isolated_skills):
@@ -352,7 +352,7 @@ def test_re_record_different_is_an_edit_that_reversions_to_probation(_isolated_s
"clicked_role": "button", "clicked_name": "Submit"}]
sk.record_skill("shop.com", "do a thing now", edited) # different -> EDIT
s = sk.find_skill("shop.com", "do a thing now")
assert s["rev"] == 2 and s["state"] == sk._PROBATION and s["replays"] == 0
assert s["rev"] == 2 and s["state"] == sk.PROBATION and s["replays"] == 0
cbn = next(x for x in s["steps"] if x["tool"] == "BrowserClickByName")
assert cbn["params"]["name"] == "Submit" # the new step stuck
@@ -365,7 +365,7 @@ def test_rev_and_state_persist_across_restart(_isolated_skills):
sk.record_skill("shop.com", "do a thing now", edited) # rev 2, probation
sk.clear(wipe_disk=False) # restart
s = sk.find_skill("shop.com", "do a thing now")
assert s["rev"] == 2 and s["state"] == sk._PROBATION
assert s["rev"] == 2 and s["state"] == sk.PROBATION
def test_steps_equal_distinguishes_slot_from_literal_and_changed_click():
@@ -375,9 +375,9 @@ def test_steps_equal_distinguishes_slot_from_literal_and_changed_click():
slot = {"tool": "BrowserType", "params": {"selector": "#q", "value_slot": 0}}
send = {"tool": "BrowserClickByName", "params": {"role": "button", "name": "Send"}}
submit = {"tool": "BrowserClickByName", "params": {"role": "button", "name": "Submit"}}
assert sk._steps_equal([nav], [nav2]) # fragment-only diff is NOT an edit
assert not sk._steps_equal([lit], [slot]) # literal vs parameterized IS an edit
assert not sk._steps_equal([send], [submit]) # renamed button IS an edit
assert sk.steps_equal([nav], [nav2]) # fragment-only diff is NOT an edit
assert not sk.steps_equal([lit], [slot]) # literal vs parameterized IS an edit
assert not sk.steps_equal([send], [submit]) # renamed button IS an edit
def test_mark_replay_helpers_on_unknown_are_safe(_isolated_skills):
@@ -390,9 +390,9 @@ def test_demoted_skill_can_be_re_proven(_isolated_skills):
sk.mark_replay_succeeded("shop.com", "do a thing now") # trusted
sk.mark_replay_failed("shop.com", "do a thing now")
sk.mark_replay_failed("shop.com", "do a thing now") # demoted to probation
assert sk.find_skill("shop.com", "do a thing now")["state"] == sk._PROBATION
assert sk.find_skill("shop.com", "do a thing now")["state"] == sk.PROBATION
sk.mark_replay_succeeded("shop.com", "do a thing now") # earns trust back
assert sk.find_skill("shop.com", "do a thing now")["state"] == sk._TRUSTED
assert sk.find_skill("shop.com", "do a thing now")["state"] == sk.TRUSTED
# --- composition: build on what's already proven, propagate staleness -------
@@ -412,7 +412,7 @@ def test_composition_links_to_trusted_sub_skill(_isolated_skills):
_trust("shop.com", "search shoes now", _log()) # trusted foundation
sk.record_skill("shop.com", "search shoes and checkout now", _log_plus())
c = sk.find_skill("shop.com", "search shoes and checkout now")
assert c["composed_of"] == [sk._sig("search shoes now")]
assert c["composed_of"] == [sk.compute_sig("search shoes now")]
def test_composition_ignores_untrusted_foundation(_isolated_skills):
@@ -425,11 +425,11 @@ def test_composition_ignores_untrusted_foundation(_isolated_skills):
def test_deprecating_a_foundation_demotes_everything_built_on_it(_isolated_skills):
_trust("shop.com", "search shoes now", _log())
_trust("shop.com", "search shoes and checkout now", _log_plus()) # composed + trusted
assert sk.find_skill("shop.com", "search shoes and checkout now")["state"] == sk._TRUSTED
assert sk.find_skill("shop.com", "search shoes and checkout now")["state"] == sk.TRUSTED
sk.deprecate_skill("shop.com", "search shoes now") # foundation pulled
# the ghost guard for composition: the dependent must NOT stay trusted on a
# foundation that no longer exists; it's knocked back to re-prove
assert sk.find_skill("shop.com", "search shoes and checkout now")["state"] == sk._PROBATION
assert sk.find_skill("shop.com", "search shoes and checkout now")["state"] == sk.PROBATION
def test_demoting_a_foundation_demotes_its_dependents(_isolated_skills):
@@ -437,7 +437,7 @@ def test_demoting_a_foundation_demotes_its_dependents(_isolated_skills):
_trust("shop.com", "search shoes and checkout now", _log_plus())
sk.mark_replay_failed("shop.com", "search shoes now")
sk.mark_replay_failed("shop.com", "search shoes now") # foundation demoted
assert sk.find_skill("shop.com", "search shoes and checkout now")["state"] == sk._PROBATION
assert sk.find_skill("shop.com", "search shoes and checkout now")["state"] == sk.PROBATION
def test_editing_a_foundation_demotes_its_dependents(_isolated_skills):
@@ -446,17 +446,17 @@ def test_editing_a_foundation_demotes_its_dependents(_isolated_skills):
edited = _log()[:-1] + [{"tool": "BrowserClickIndex", "input": {}, "ok": True,
"clicked_role": "button", "clicked_name": "Find"}]
sk.record_skill("shop.com", "search shoes now", edited) # foundation changed
assert sk.find_skill("shop.com", "search shoes and checkout now")["state"] == sk._PROBATION
assert sk.find_skill("shop.com", "search shoes and checkout now")["state"] == sk.PROBATION
def test_list_skills_surfaces_state_rev_and_builds_on(_isolated_skills):
_trust("shop.com", "search shoes now", _log())
sk.record_skill("shop.com", "search shoes and checkout now", _log_plus())
listed = {x["task"]: x for x in sk.list_skills("shop.com")}
foundation = listed[sk._sig("search shoes now")]
composed = listed[sk._sig("search shoes and checkout now")]
assert foundation["state"] == sk._TRUSTED and foundation["builds_on"] == []
assert composed["builds_on"] == [sk._sig("search shoes now")]
foundation = listed[sk.compute_sig("search shoes now")]
composed = listed[sk.compute_sig("search shoes and checkout now")]
assert foundation["state"] == sk.TRUSTED and foundation["builds_on"] == []
assert composed["builds_on"] == [sk.compute_sig("search shoes now")]
assert "rev" in composed and "steps" in composed
@@ -478,24 +478,24 @@ def test_replay_safety_refuses_send_steps_and_passes_reads():
def test_extract_first_json_strips_fences_and_prose():
from backend.apps.agents.browser.browser_extract import _first_json
assert _first_json('```json\n{"a": 1}\n```') == '{"a": 1}'
assert _first_json('Here you go: [{"n": "x"}] hope that helps') == '[{"n": "x"}]'
assert _first_json("no json here") == ""
assert _first_json('{"broken": ') == ""
from backend.apps.agents.browser.browser_extract import first_json
assert first_json('```json\n{"a": 1}\n```') == '{"a": 1}'
assert first_json('Here you go: [{"n": "x"}] hope that helps') == '[{"n": "x"}]'
assert first_json("no json here") == ""
assert first_json('{"broken": ') == ""
def test_widened_redaction_catches_audit_bypasses():
# the audit's three named bypasses: bare 2FA digits, credential-shaped
# fields the old regex missed, and seed/recovery phrase boxes
assert sk._looks_sensitive("481922", "")
assert sk._looks_sensitive("hunter2", "#user")
assert sk._looks_sensitive("me@corp.com", "#login-email")
assert sk._looks_sensitive("correct horse battery staple", "#seed-phrase")
assert sk._looks_sensitive("123456", "input[name='verification-code']")
assert sk.looks_sensitive("481922", "")
assert sk.looks_sensitive("hunter2", "#user")
assert sk.looks_sensitive("me@corp.com", "#login-email")
assert sk.looks_sensitive("correct horse battery staple", "#seed-phrase")
assert sk.looks_sensitive("123456", "input[name='verification-code']")
# the bread-and-butter skill (a search query) still persists
assert not sk._looks_sensitive("shoes", "#search-input")
assert not sk._looks_sensitive("Ada Lovelace", ".search-global-typeahead input")
assert not sk.looks_sensitive("shoes", "#search-input")
assert not sk.looks_sensitive("Ada Lovelace", ".search-global-typeahead input")
def test_first_unsafe_step_splits_send_skills():
@@ -517,13 +517,13 @@ def test_first_unsafe_step_splits_send_skills():
def test_template_task_ignores_possessive_apostrophes():
from backend.apps.agents.browser.browser_skills import template_task, _sig
from backend.apps.agents.browser.browser_skills import template_task, compute_sig
r14 = "go to tyler chen's linkedin hes in entrepreneurs first and text him '[test] hello world r14-os'"
r15 = "go to tyler chen's linkedin hes in entrepreneurs first and text him '[test] hello world r15-os'"
t14, v14 = template_task(r14)
assert v14 == ["[test] hello world r14-os"]
assert "chen's linkedin" in t14
assert _sig(r14) == _sig(r15)
assert compute_sig(r14) == compute_sig(r15)
assert template_task("no quotes here at all") == ("no quotes here at all", [])
backend/tests/test_browser_stagnation.py
+13 -13
View File
@@ -1,9 +1,9 @@
"""Deterministic stagnation detection for the browser sub-agent."""
from backend.apps.agents.browser.browser_loop import (
_STAGNATION_ESCALATION_AT,
_STAGNATION_MAX,
_looks_like_failure,
STAGNATION_ESCALATION_AT,
STAGNATION_MAX,
looks_like_failure,
advance_stagnation,
card_is_unavailable,
completion_is_honest,
@@ -19,14 +19,14 @@ def _fail(url="https://a.com"):
def test_looks_like_failure_positive():
assert _looks_like_failure("Element not found: '.foo'")
assert _looks_like_failure("Index 4 is no longer valid")
assert _looks_like_failure("Error: something broke")
assert looks_like_failure("Element not found: '.foo'")
assert looks_like_failure("Index 4 is no longer valid")
assert looks_like_failure("Error: something broke")
def test_looks_like_failure_negative():
assert not _looks_like_failure("Clicked element: button#submit")
assert not _looks_like_failure("Typed into: input#email")
assert not looks_like_failure("Clicked element: button#submit")
assert not looks_like_failure("Typed into: input#email")
def test_error_result_is_unproductive():
@@ -63,17 +63,17 @@ def test_neutral_read_tools_never_count():
def test_nudge_mentions_human_intervention_only_at_max():
assert "RequestHumanIntervention" not in stagnation_nudge(3)
assert "RequestHumanIntervention" in stagnation_nudge(_STAGNATION_MAX)
assert "RequestHumanIntervention" in stagnation_nudge(STAGNATION_MAX)
assert "ladder" in stagnation_nudge(3)
def test_advance_increments_on_failures_and_nudges_at_threshold():
streak, url, text, nudge = 0, "", "", None
nudges = []
for _ in range(_STAGNATION_ESCALATION_AT):
for _ in range(STAGNATION_ESCALATION_AT):
streak, url, text, nudge = advance_stagnation(streak, url, text, "BrowserClick", _fail())
nudges.append(nudge)
assert streak == _STAGNATION_ESCALATION_AT
assert streak == STAGNATION_ESCALATION_AT
assert nudges[-1] is not None # nudge fires exactly when the threshold is hit
assert nudges[0] is None and nudges[1] is None
@@ -97,9 +97,9 @@ def test_advance_neutral_tools_pass_through_unchanged():
def test_advance_fires_again_at_max():
streak, url, text = _STAGNATION_MAX - 1, "https://a.com", "prev different"
streak, url, text = STAGNATION_MAX - 1, "https://a.com", "prev different"
streak, url, text, nudge = advance_stagnation(streak, url, text, "BrowserClick", _fail())
assert streak == _STAGNATION_MAX
assert streak == STAGNATION_MAX
assert nudge is not None and "RequestHumanIntervention" in nudge
assert stagnation_exhausted(streak)
backend/tests/test_browser_validator.py
+2 -2
View File
@@ -2,7 +2,7 @@
import asyncio
from backend.apps.agents.browser.browser_validator import adjudicate_stuck, _extract_text
from backend.apps.agents.browser.browser_validator import adjudicate_stuck, extract_text
class _Block:
@@ -52,7 +52,7 @@ def test_swallows_provider_error_and_returns_empty():
def test_extract_text_joins_text_blocks_and_ignores_others():
resp = _Resp([_Block("text", "First."), _Block("tool_use"), _Block("text", "Second.")])
assert _extract_text(resp) == "First. Second."
assert extract_text(resp) == "First. Second."
def test_handles_empty_inputs_without_crashing():
backend/tests/test_browser_wait.py
+1 -1
View File
@@ -168,7 +168,7 @@ async def test_hung_tab_returns_fast_not_after_the_full_command_timeout():
assert out.get("error") == "page unresponsive"
assert elapsed < 3.0, f"hung wait must return fast, took {elapsed:.1f}s"
# it bailed after the timeout threshold, not after burning the whole cap
assert ex.calls <= bw._MAX_PROBE_TIMEOUTS
assert ex.calls <= bw.MAX_PROBE_TIMEOUTS
@pytest.mark.asyncio
backend/tests/test_ghost_detector.py
+1 -1
View File
@@ -68,4 +68,4 @@ def test_tier_mapping_present():
# the analyzer's _PRODUCTIVE set must include the real mutation tools
m = _load()
for t in ("BrowserClick", "BrowserClickIndex", "BrowserType", "BrowserNavigate", "BrowserReplayRoute"):
assert t in m._PRODUCTIVE
assert t in m.PRODUCTIVE
backend/tests/test_skill_metrics_and_analyzer.py
+14 -14
View File
@@ -57,14 +57,14 @@ def test_analyzer_measures_replay_speedup_when_the_layer_helps(_metrics_dir, cap
sk.clear(wipe_disk=True)
# A repeated task: 1 slow LLM run, then 2 fast replays -> measurable speedup.
sk.record_skill("shop.com", "search now", _log())
_task_row(sk._sig("search now"), "llm", 4.0)
_task_row(sk.compute_sig("search now"), "llm", 4.0)
sk.mark_replay_succeeded("shop.com", "search now")
_task_row(sk._sig("search now"), "replay", 0.04)
_task_row(sk._sig("search now"), "replay", 0.05)
_task_row(sk.compute_sig("search now"), "replay", 0.04)
_task_row(sk.compute_sig("search now"), "replay", 0.05)
mod = _load_analyzer()
tasks = mod._load(os.path.join(_metrics_dir, "tasks.jsonl"))
sevs = mod._load(os.path.join(_metrics_dir, "skill_events.jsonl"))
tasks = mod.load(os.path.join(_metrics_dir, "tasks.jsonl"))
sevs = mod.load(os.path.join(_metrics_dir, "skill_events.jsonl"))
mod.skill_layer_report(tasks, sevs)
out = capsys.readouterr().out
assert "REPLAY SPEEDUP" in out
@@ -81,12 +81,12 @@ def test_analyzer_flags_silent_non_help_thrash(_metrics_dir, capsys):
"clicked_role": "button", "clicked_name": "Other"}]
sk.record_skill("bad.com", "do thing now", edited) # edit (un-quarantine)
sk.mark_replay_failed("bad.com", "do thing now") # quarantine again
_task_row(sk._sig("do thing now"), "llm", 3.0)
_task_row(sk._sig("do thing now"), "llm_fallback", 3.2)
_task_row(sk.compute_sig("do thing now"), "llm", 3.0)
_task_row(sk.compute_sig("do thing now"), "llm_fallback", 3.2)
mod = _load_analyzer()
tasks = mod._load(os.path.join(_metrics_dir, "tasks.jsonl"))
sevs = mod._load(os.path.join(_metrics_dir, "skill_events.jsonl"))
tasks = mod.load(os.path.join(_metrics_dir, "tasks.jsonl"))
sevs = mod.load(os.path.join(_metrics_dir, "skill_events.jsonl"))
mod.skill_layer_report(tasks, sevs)
out = capsys.readouterr().out
assert "SILENT NON-HELP" in out # repeated but never replayed
@@ -104,7 +104,7 @@ def test_analyzer_reports_composition(_metrics_dir, capsys):
sk.deprecate_skill("shop.com", "search now") # must invalidate the TRUSTED dependent
mod = _load_analyzer()
sevs = mod._load(os.path.join(_metrics_dir, "skill_events.jsonl"))
sevs = mod.load(os.path.join(_metrics_dir, "skill_events.jsonl"))
# the invalidate EVENT must actually fire (end-to-end), not just the state flip
assert any(e["kind"] == "invalidate" for e in sevs)
mod.skill_layer_report([], sevs)
@@ -117,11 +117,11 @@ def test_analyzer_reports_composition(_metrics_dir, capsys):
def test_analyzer_reports_playbook_cutting_exploration_turns(_metrics_dir, capsys):
# tier-2 win: a cold run on a host takes many turns; once strategy is seeded,
# the same kind of task takes fewer. The analyzer must report HELPS.
sig = sk._sig("find people")
sig = sk.compute_sig("find people")
_task_row(sig, "llm", 60.0, turns=14, playbook_seeded=False) # cold
_task_row(sig, "llm", 40.0, turns=8, playbook_seeded=True) # seeded -> fewer turns
mod = _load_analyzer()
tasks = mod._load(os.path.join(_metrics_dir, "tasks.jsonl"))
tasks = mod.load(os.path.join(_metrics_dir, "tasks.jsonl"))
mod.playbook_report(tasks)
out = capsys.readouterr().out
assert "STRATEGIC PLAYBOOK" in out and "HELPS" in out and "NOT HELPING" not in out
@@ -129,11 +129,11 @@ def test_analyzer_reports_playbook_cutting_exploration_turns(_metrics_dir, capsy
def test_analyzer_flags_playbook_that_does_not_help(_metrics_dir, capsys):
# anti-ghost: memory is active (seeded) but seeded runs are NOT cheaper -> flag.
sig = sk._sig("stubborn task")
sig = sk.compute_sig("stubborn task")
_task_row(sig, "llm", 60.0, turns=10, playbook_seeded=False)
_task_row(sig, "llm", 60.0, turns=12, playbook_seeded=True) # seeded but MORE turns
mod = _load_analyzer()
tasks = mod._load(os.path.join(_metrics_dir, "tasks.jsonl"))
tasks = mod.load(os.path.join(_metrics_dir, "tasks.jsonl"))
mod.playbook_report(tasks)
out = capsys.readouterr().out
assert "NOT HELPING" in out
backend/tests/test_skill_registry_seed.py
+1 -1
View File
@@ -30,7 +30,7 @@ def test_seed_makes_catalog_non_empty_offline(monkeypatch, tmp_path):
seeded = sr._load_seed_cache()
assert len(seeded) >= 10
sr._cache = seeded
sr.CACHE = seeded
res = asyncio.run(sr.registry_search(q="", limit=100, offset=0, sort="name", category=""))
assert res["total"] >= 10 and len(res["skills"]) >= 10
scripts/analyze-browser-metrics.py
+7 -7
View File
@@ -18,7 +18,7 @@ from collections import Counter, defaultdict
# Tools that actually change page state (vs. read/meta). A "completed" task that
# never ran one of these did nothing but look around, suspicious.
_PRODUCTIVE = {
PRODUCTIVE = {
"BrowserClick", "BrowserClickIndex", "BrowserType", "BrowserNavigate",
"BrowserPressKey", "BrowserBatch", "BrowserReplayRoute",
}
@@ -34,7 +34,7 @@ def _default_dir():
return os.path.join(os.path.dirname(__file__), "..", "backend", "data", "browser_metrics")
def _load(path):
def load(path):
if not os.path.exists(path):
return []
out = []
@@ -56,7 +56,7 @@ def ghost_verdict(task, events_for_task):
return False, []
reasons = []
tools = [e["tool"] for e in events_for_task]
productive = [t for t in tools if t in _PRODUCTIVE]
productive = [t for t in tools if t in PRODUCTIVE]
errs = sum(1 for e in events_for_task if not e.get("ok"))
total = len(events_for_task)
# A read/extract task legitimately has no state-changing action; its evidence
@@ -76,7 +76,7 @@ def ghost_verdict(task, events_for_task):
reasons.append(f"{errs}/{total} tool calls errored but still marked completed")
if any(e.get("is_loop") for e in events_for_task):
reasons.append("loop detector fired during a 'completed' task")
prod_ok = [e for e in events_for_task if e["tool"] in _PRODUCTIVE and e.get("ok")]
prod_ok = [e for e in events_for_task if e["tool"] in PRODUCTIVE and e.get("ok")]
if productive and not prod_ok:
reasons.append("every state-changing action errored, yet marked completed")
return (len(reasons) > 0), reasons
@@ -172,9 +172,9 @@ def playbook_report(tasks):
def main():
d = sys.argv[1] if len(sys.argv) > 1 else _default_dir()
events = _load(os.path.join(d, "events.jsonl"))
tasks = _load(os.path.join(d, "tasks.jsonl"))
skill_events = _load(os.path.join(d, "skill_events.jsonl"))
events = load(os.path.join(d, "events.jsonl"))
tasks = load(os.path.join(d, "tasks.jsonl"))
skill_events = load(os.path.join(d, "skill_events.jsonl"))
print(f"metrics dir: {d}")
print(f"events: {len(events)} tasks: {len(tasks)} skill_events: {len(skill_events)}\n")
if not tasks and not events: