From 6ef7b336671b2118c8bdc61891f4fd12b0c7c11e Mon Sep 17 00:00:00 2001 From: ciregenz Date: Sun, 7 Jun 2026 21:20:40 -0700 Subject: [PATCH] [eric] browser: cross-site meta-playbook (tier 3) + self-audit of the learning machinery (tier 4, proposes for human review, never self-edits) --- backend/apps/agents/browser/browser_agent.py | 9 + .../agents/browser/browser_meta_playbook.py | 158 +++++++++++++++ .../apps/agents/browser/browser_playbook.py | 31 ++- .../apps/agents/browser/browser_self_audit.py | 184 ++++++++++++++++++ backend/tests/test_browser_meta_playbook.py | 49 +++++ backend/tests/test_browser_self_audit.py | 70 +++++++ 6 files changed, 500 insertions(+), 1 deletion(-) create mode 100644 backend/apps/agents/browser/browser_meta_playbook.py create mode 100644 backend/apps/agents/browser/browser_self_audit.py create mode 100644 backend/tests/test_browser_meta_playbook.py create mode 100644 backend/tests/test_browser_self_audit.py diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index a670bfaf..656a1026 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -64,6 +64,7 @@ from backend.apps.agents.browser import browser_batch_replay from backend.apps.agents.browser import browser_extract from backend.apps.agents.browser import browser_metrics from backend.apps.agents.browser import browser_playbook +from backend.apps.agents.browser import browser_meta_playbook from backend.apps.agents.browser import browser_skills from backend.apps.agents.browser import browser_wait from backend.apps.agents.browser import browser_schema @@ -601,6 +602,14 @@ async def run_browser_agent( if _pb_block: run_system_prompt = run_system_prompt + _pb_block pb_seeded = True + # Tier-3 memory: the cross-site priors learned on EVERY other site, injected on + # every run (host-agnostic) so a brand-new site isn't fully cold. Advisory, capped. + try: + _meta_block = browser_meta_playbook.format_for_prompt() + if _meta_block: + run_system_prompt = run_system_prompt + _meta_block + except Exception: + pass # Prompt-caching shapes built once: system as a single cached text block, # and the last tool carrying the cache_control marker (Anthropic keys on the diff --git a/backend/apps/agents/browser/browser_meta_playbook.py b/backend/apps/agents/browser/browser_meta_playbook.py new file mode 100644 index 00000000..ad9984cd --- /dev/null +++ b/backend/apps/agents/browser/browser_meta_playbook.py @@ -0,0 +1,158 @@ +""" +Cross-site meta-playbook (browser memory tier 3). + +Tier 2 (`browser_playbook`) learns per-SITE strategy. This tier learns the +SITE-AGNOSTIC patterns that transfer everywhere, e.g. "a composer clears on send, +so the cleared box IS the confirmation; do not hunt the thread for the text" or +"an opener like Message/DM just opens the box, only Send is irreversible". So the +VERY FIRST task on a brand-new site already benefits from what was learned on +every other site, the generalizable answer to "make it learn to learn". + +Cheap by construction: the per-site distill (one aux call we already make) ALSO +returns a `universal` list; we just MERGE those here (dedup + cap), NO extra LLM +call. Same fail-safe as the per-site playbook: it is ADVISORY text seeded into the +prompt and re-verified by the agent, never auto-executed, so a wrong universal +bullet can only mildly mislead and is corrected as more sites confirm the truth. +""" + +import json +import logging +import os +import tempfile +import time + +from backend.apps.agents.browser.browser_playbook import _clean_bullet + +logger = logging.getLogger(__name__) + +_VERSION = 1 +_MAX_BULLETS = 10 # a touch larger than per-site: these earn their keep everywhere +_FILE = "meta_playbook.json" + +_cache: list[str] | None = None + + +def _dir() -> str | None: + base = os.environ.get("OPENSWARM_BROWSER_META_DIR") + if not base: + try: + from backend.config.paths import DATA_ROOT + base = os.path.join(DATA_ROOT, "browser_meta") + except Exception: + return None + try: + os.makedirs(base, mode=0o700, exist_ok=True) + except Exception: + return None + return base + + +def _path() -> str | None: + d = _dir() + return os.path.join(d, _FILE) if d else None + + +def _load() -> list[str]: + path = _path() + if not path or not os.path.exists(path): + return [] + try: + with open(path, encoding="utf-8") as f: + data = json.load(f) + if data.get("version") != _VERSION: + return [] + return [b for b in (data.get("bullets") or []) if isinstance(b, str)] + except Exception: + return [] + + +def _persist(bullets: list[str]) -> None: + path = _path() + if not path: + return + try: + d = os.path.dirname(path) + fd, tmp = tempfile.mkstemp(dir=d, suffix=".tmp") + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump({"version": _VERSION, "bullets": bullets, "updated_at": time.time()}, f) + os.replace(tmp, path) # atomic + except Exception as e: + logger.debug(f"[browser-meta] persist failed: {e}") + + +def get_meta() -> list[str]: + """The cross-site bullets (cheap, no LLM). Cached after first read.""" + global _cache + if _cache is None: + _cache = _load() or list(_SEED) + return _cache + + +def format_for_prompt() -> str: + """The block injected into EVERY run's prompt, or '' if empty. Kept short and + clearly framed as general priors so it never overrides what the live page shows.""" + bullets = get_meta() + if not bullets: + return "" + lines = "\n".join(f"- {b}" for b in bullets[:_MAX_BULLETS]) + return ( + "\n\n## General web priors (learned across many sites, verify against THIS page)\n" + + lines + ) + + +def absorb(universal_bullets: list[str]) -> bool: + """Merge site-agnostic lessons from a run into the meta-playbook. No LLM call. + Only GENUINELY new bullets (not already present, case-insensitive) move the + needle, so a re-confirmed lesson doesn't churn the list. New content goes first + so it wins the cap over a stale prior. Returns True only if something changed.""" + if not universal_bullets: + return False + existing = get_meta() + existing_lower = {b.lower() for b in existing} + truly_new: list[str] = [] + seen: set[str] = set() + for b in universal_bullets: + cb = _clean_bullet(b) + if cb and cb.lower() not in existing_lower and cb.lower() not in seen: + seen.add(cb.lower()) + truly_new.append(cb) + if not truly_new: + return False + merged = (truly_new + existing)[:_MAX_BULLETS] + global _cache + _cache = merged + _persist(merged) + logger.info(f"[browser-meta] {len(merged)} cross-site prior(s) (was {len(existing)})") + return True + + +def clear(wipe_disk: bool = False) -> None: + """Test/maintenance reset of the in-memory cache (and optionally disk).""" + global _cache + _cache = None + if wipe_disk: + path = _path() + if path and os.path.exists(path): + try: + os.remove(path) + except Exception: + pass + + +# Shipped starting priors: the hard-won universal lessons from this codebase's own +# browser work, so tier 3 is useful on day one and accrues more as sites confirm them. +_SEED = ( + "A message composer CLEARS when the send goes through; the empty box IS your " + "confirmation, do not hunt the thread for the sent text to 'verify'.", + "An opener (Message/DM/Compose) only OPENS the box and is reversible; only the " + "actual Send/Submit/Post is irreversible, so opening it freely is safe.", + "If the target's thread/composer is already open, commit to it; do not navigate " + "away to a profile or re-search just to re-confirm the recipient.", + "In rich-text composers, Enter usually inserts a newline; click the Send button " + "rather than pressing Enter.", + "The Send button often renders a beat AFTER the text commits; settle briefly and " + "re-list once rather than concluding it vanished and hunting via CSS/JS.", + "Construct deep search URLs directly (site.com/search?q=...) instead of driving " + "the homepage search UI when you know the pattern.", +) diff --git a/backend/apps/agents/browser/browser_playbook.py b/backend/apps/agents/browser/browser_playbook.py index 62159a45..645b8ed8 100644 --- a/backend/apps/agents/browser/browser_playbook.py +++ b/backend/apps/agents/browser/browser_playbook.py @@ -209,7 +209,11 @@ def _build_prompt(host: str, task: str, working_memory: str, summary: str, f"TASK: {task}\n" f"AGENT NOTES: {working_memory[:1500]}\n" f"RESULT: {summary[:800]}\n\n" - "Return the UPDATED playbook as JSON: {\"playbook\": [\"...\", ...]}. Rules:\n" + "Return JSON: {\"playbook\": [\"...\"], \"universal\": [\"...\"]}, where " + "`playbook` is the UPDATED per-site playbook and `universal` is the SUBSET of " + "lessons that are SITE-AGNOSTIC (true on ANY website, e.g. how composers/Send " + "buttons behave in general), so other sites can reuse them. `universal` may be " + "empty; never put site-specific URLs, selectors, or names in it. Rules:\n" f"- At most {_MAX_BULLETS} bullets, each under {_MAX_BULLET_CHARS} chars, " "atomic and REUSABLE for ANY task on this site.\n" "- Keep only durable site strategy: which queries/filters/URLs work, what " @@ -247,6 +251,24 @@ def _parse(text: str) -> list[str] | None: return [str(x) for x in pb if isinstance(x, (str, int, float))] +def _parse_universal(text: str) -> list[str]: + """The site-agnostic subset the distill flagged, for the cross-site meta-playbook. + Tolerant: missing/garbled `universal` just yields nothing (the site distill still runs).""" + if not text: + return [] + m = re.search(r"\{.*\}", text, re.DOTALL) + if not m: + return [] + try: + data = json.loads(m.group(0)) + except Exception: + return [] + uni = data.get("universal") + if not isinstance(uni, list): + return [] + return [str(x) for x in uni if isinstance(x, (str, int, float))] + + async def distill_and_store(host, task, working_memory, summary, aux_client, aux_model) -> bool: """One cheap aux call: distill this successful run + reconcile against the @@ -268,6 +290,13 @@ async def distill_and_store(host, task, working_memory, summary, return False stored = _store(host, new_bullets) changed = stored != existing + # Fold any site-agnostic lessons into the cross-site meta-playbook (no extra + # LLM call, they rode along in this same reply). Best-effort, never fatal. + try: + from backend.apps.agents.browser import browser_meta_playbook + browser_meta_playbook.absorb(_parse_universal(text)) + except Exception: + pass if changed: logger.info(f"[browser-playbook] {host}: {len(stored)} strategy bullet(s) " f"(was {len(existing)})") diff --git a/backend/apps/agents/browser/browser_self_audit.py b/backend/apps/agents/browser/browser_self_audit.py new file mode 100644 index 00000000..e710e5b9 --- /dev/null +++ b/backend/apps/agents/browser/browser_self_audit.py @@ -0,0 +1,184 @@ +""" +Self-audit loop for the browser agent's own LEARNING (browser memory tier 4). + +Tiers 1-3 make the agent better at the WEBSITE. This makes it better at LEARNING: +it reads its own run metrics + skill lifecycle and flags where the learning +machinery is misfiring, skills that re-learn forever but never replay (thrash), +runs that stall (turns far above the site's norm), recurring tool errors, low +route-hint adoption, then writes a PROPOSAL for a human to act on. + +SAFETY LINE (deliberate): this NEVER edits prompts, thresholds, or code. In an +open-source local-agent product, an agent silently rewriting its own behavior is +a line we don't cross. It only READS metrics and WRITES a human-readable report; +a person decides what to change. Pure observation in, one proposal file out. +""" + +import json +import logging +import os +from collections import defaultdict + +logger = logging.getLogger(__name__) + +# Thresholds for flagging; conservative so the report stays signal, not noise. +_THRASH_MIN_RELEARNS = 3 # a skill re-versioned this many times with 0 replays = stuck +_STALL_TURN_FACTOR = 2.0 # a run 2x the host's median turns is a stall worth noting +_MIN_RUNS_FOR_MEDIAN = 4 # don't call a "norm" from too few runs +_ERROR_RATE_FLAG = 0.25 # >25% of a host's tool calls erroring = something systemic + + +def _read_jsonl(path: str, cap: int = 20000) -> list[dict]: + out: list[dict] = [] + if not path or not os.path.exists(path): + return out + try: + with open(path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + out.append(json.loads(line)) + except Exception: + continue + if len(out) >= cap: + break + except Exception: + pass + return out + + +def _median(xs: list[float]) -> float: + s = sorted(xs) + n = len(s) + if not n: + return 0.0 + return s[n // 2] if n % 2 else (s[n // 2 - 1] + s[n // 2]) / 2 + + +def audit(metrics_dir: str) -> dict: + """Read the metrics + skill events and return a structured findings dict. + Pure read; safe to call anytime. The caller renders/persists it.""" + tasks = _read_jsonl(os.path.join(metrics_dir, "tasks.jsonl")) + skill_events = _read_jsonl(os.path.join(metrics_dir, "skill_events.jsonl")) + findings: list[dict] = [] + + # 1) THRASH: a skill re-versioned (edit) or sent to quarantine many times but + # never PROMOTED, the kinds the skill layer actually records. It keeps re-learning + # and never earns trust = the recorded steps don't hold up at replay. + churn: dict[tuple, int] = defaultdict(int) + promotes: dict[tuple, int] = defaultdict(int) + for e in skill_events: + key = (e.get("host"), e.get("task_sig")) + kind = e.get("kind") + if kind in ("edit", "quarantine", "demote"): + churn[key] += 1 + elif kind == "promote": + promotes[key] += 1 + for key, n in churn.items(): + if n >= _THRASH_MIN_RELEARNS and promotes.get(key, 0) == 0: + findings.append({ + "kind": "thrash", + "host": key[0], + "detail": f"skill re-learned/quarantined {n}x but never promoted", + "suggestion": "the recorded steps likely don't match at replay (brittle " + "names or a late-rendering control); inspect the distill for " + "this task or deprecate the skill so it stops churning.", + }) + + # 2) STALL: runs far above the host's median turn count. + by_host_turns: dict[str, list[int]] = defaultdict(list) + for t in tasks: + h = _host_of_task(t) + if t.get("turns"): + by_host_turns[h].append(int(t["turns"])) + for h, turns in by_host_turns.items(): + if len(turns) < _MIN_RUNS_FOR_MEDIAN: + continue + med = _median([float(x) for x in turns]) + stalls = [x for x in turns if med and x >= med * _STALL_TURN_FACTOR] + if stalls: + findings.append({ + "kind": "stall", + "host": h, + "detail": f"{len(stalls)} run(s) at >= {_STALL_TURN_FACTOR}x the median " + f"{med:.0f} turns (worst {max(stalls)})", + "suggestion": "a few runs spike well above normal, likely a perception/verify " + "loop or an env hang; check whether a prompt prior or mechanical " + "hand-off would collapse the spike.", + }) + + # 3) ERROR-HEAVY: a host whose tool calls error a lot (systemic, not one bad run). + err = defaultdict(int) + tot = defaultdict(int) + for t in tasks: + h = _host_of_task(t) + rc = t.get("recurring_errors") or {} + tot[h] += int(t.get("tool_calls") or 0) + if isinstance(rc, dict): + err[h] += sum(int(v) for v in rc.values() if isinstance(v, (int, float))) + for h in tot: + if tot[h] >= 20 and err[h] / max(1, tot[h]) >= _ERROR_RATE_FLAG: + findings.append({ + "kind": "error_rate", + "host": h, + "detail": f"{err[h]}/{tot[h]} tool calls recurring-errored " + f"({100*err[h]/max(1,tot[h]):.0f}%)", + "suggestion": "high error rate is usually a stale selector/index or a throttled " + "session; consider a more robust locator or a backoff on this host.", + }) + + return { + "n_tasks": len(tasks), + "n_skill_events": len(skill_events), + "findings": findings, + } + + +def _host_of_task(task: dict) -> str: + # tasks.jsonl doesn't store host directly; task_sig is host-agnostic, so fall + # back to a coarse bucket. browser_id groups a card's runs well enough for norms. + return task.get("browser_id") or task.get("task_sig") or "unknown" + + +def render_report(result: dict) -> str: + """A human-readable proposal. Read it, then YOU decide what (if anything) to change.""" + lines = [ + "# Browser self-audit (proposal only , nothing was changed)", + "", + f"Scanned {result.get('n_tasks', 0)} task runs and " + f"{result.get('n_skill_events', 0)} skill events.", + "", + ] + findings = result.get("findings") or [] + if not findings: + lines.append("No learning-machinery problems flagged. The agent is learning cleanly.") + return "\n".join(lines) + lines.append(f"## {len(findings)} thing(s) worth a human look") + for i, f in enumerate(findings, 1): + lines += [ + "", + f"### {i}. {f['kind'].upper()} , {f.get('host', '?')}", + f"- What: {f['detail']}", + f"- Proposed action: {f['suggestion']}", + ] + return "\n".join(lines) + + +def run_and_write(metrics_dir: str | None = None) -> str | None: + """Audit + write the proposal to metrics_dir/self_audit_report.md. Returns the + path written, or None. Never raises into the caller.""" + try: + if metrics_dir is None: + from backend.apps.agents.browser import browser_metrics + metrics_dir = browser_metrics._metrics_dir() + result = audit(metrics_dir) + report = render_report(result) + path = os.path.join(metrics_dir, "self_audit_report.md") + with open(path, "w", encoding="utf-8") as f: + f.write(report) + logger.info(f"[browser-self-audit] wrote {len(result.get('findings', []))} finding(s) to {path}") + return path + except Exception as e: + logger.debug(f"[browser-self-audit] failed: {e}") + return None diff --git a/backend/tests/test_browser_meta_playbook.py b/backend/tests/test_browser_meta_playbook.py new file mode 100644 index 00000000..9f43d3ec --- /dev/null +++ b/backend/tests/test_browser_meta_playbook.py @@ -0,0 +1,49 @@ +"""Cross-site meta-playbook (browser memory tier 3): seeds on day one, absorbs +site-agnostic lessons with no extra LLM call, dedups + caps, survives a restart.""" +import os +import tempfile + +import pytest + +from backend.apps.agents.browser import browser_meta_playbook as meta + + +@pytest.fixture(autouse=True) +def _isolated(monkeypatch): + monkeypatch.setenv("OPENSWARM_BROWSER_META_DIR", tempfile.mkdtemp(prefix="meta_test_")) + meta.clear(wipe_disk=True) + yield + meta.clear(wipe_disk=True) + + +def test_seeded_on_day_one(): + b = meta.get_meta() + assert len(b) >= 4 + assert any("composer" in x.lower() and "clear" in x.lower() for x in b) + assert "General web priors" in meta.format_for_prompt() + + +def test_absorb_adds_dedups_and_caps(): + n0 = len(meta.get_meta()) + assert meta.absorb(["clicking a date opens a picker, it is not a navigation"]) is True + assert len(meta.get_meta()) == n0 + 1 + # identical lesson (case-insensitive) doesn't grow it + assert meta.absorb(["Clicking a DATE opens a picker, it is not a navigation"]) is False + # empty input is a no-op + assert meta.absorb([]) is False + # capped: flooding never exceeds the cap + meta.absorb([f"unique universal lesson number {i}" for i in range(50)]) + assert len(meta.get_meta()) <= meta._MAX_BULLETS + + +def test_survives_a_restart(): + meta.absorb(["a durable cross-site lesson worth keeping"]) + meta.clear(wipe_disk=False) # in-memory gone, disk intact (== restart) + assert meta._cache is None + assert any("durable cross-site lesson" in x for x in meta.get_meta()) + + +def test_secrets_never_persist_into_meta(): + meta.absorb(["the login token is sk-ant-api03-deadbeef and it works"]) + blob = " ".join(meta.get_meta()) + assert "sk-ant-api03" not in blob diff --git a/backend/tests/test_browser_self_audit.py b/backend/tests/test_browser_self_audit.py new file mode 100644 index 00000000..8c9fbf71 --- /dev/null +++ b/backend/tests/test_browser_self_audit.py @@ -0,0 +1,70 @@ +"""Self-audit (tier 4): reads its own run metrics + skill lifecycle and PROPOSES +fixes for a human, never changes anything. Proves the detectors fire on the real +failure shapes (thrash, stall, error-heavy) and stay quiet on a clean history.""" +import json +import os +import tempfile + +from backend.apps.agents.browser import browser_self_audit as audit + + +def _write(d, name, rows): + with open(os.path.join(d, name), "w", encoding="utf-8") as f: + for r in rows: + f.write(json.dumps(r) + "\n") + + +def test_thrash_is_flagged_only_without_a_promote(): + d = tempfile.mkdtemp() + # a skill edited/quarantined 4x and NEVER promoted = thrash + ev = [{"kind": "edit", "host": "x.com", "task_sig": "s1"} for _ in range(3)] + ev += [{"kind": "quarantine", "host": "x.com", "task_sig": "s1"}] + # a healthy skill: edited a couple times THEN promoted = not thrash + ev += [{"kind": "edit", "host": "y.com", "task_sig": "s2"}, + {"kind": "promote", "host": "y.com", "task_sig": "s2"}] + _write(d, "skill_events.jsonl", ev) + _write(d, "tasks.jsonl", []) + r = audit.audit(d) + thrash = [f for f in r["findings"] if f["kind"] == "thrash"] + assert len(thrash) == 1 and thrash[0]["host"] == "x.com" + + +def test_stall_flags_runs_far_above_the_host_norm(): + d = tempfile.mkdtemp() + # six fast runs (norm ~4) and two big spikes on the same card + tasks = [{"browser_id": "b1", "turns": n} for n in (4, 4, 5, 3, 4, 4)] + tasks += [{"browser_id": "b1", "turns": 18}, {"browser_id": "b1", "turns": 20}] + _write(d, "tasks.jsonl", tasks) + _write(d, "skill_events.jsonl", []) + r = audit.audit(d) + assert any(f["kind"] == "stall" for f in r["findings"]) + + +def test_error_rate_flags_a_systemically_failing_host(): + d = tempfile.mkdtemp() + tasks = [{"browser_id": "b9", "tool_calls": 30, + "recurring_errors": {"index no longer valid": 12}}] + _write(d, "tasks.jsonl", tasks) + _write(d, "skill_events.jsonl", []) + r = audit.audit(d) + assert any(f["kind"] == "error_rate" for f in r["findings"]) + + +def test_clean_history_proposes_nothing(): + d = tempfile.mkdtemp() + _write(d, "tasks.jsonl", [{"browser_id": "b1", "turns": 4, "tool_calls": 5} for _ in range(6)]) + _write(d, "skill_events.jsonl", [{"kind": "learn", "host": "x.com", "task_sig": "s"}, + {"kind": "promote", "host": "x.com", "task_sig": "s"}]) + r = audit.audit(d) + assert r["findings"] == [] + assert "learning cleanly" in audit.render_report(r) + + +def test_run_and_write_emits_a_report_file_and_never_raises(): + d = tempfile.mkdtemp() + _write(d, "tasks.jsonl", []) + _write(d, "skill_events.jsonl", []) + path = audit.run_and_write(d) + assert path and os.path.exists(path) + # also safe on a totally missing dir + assert audit.run_and_write("/nonexistent/dir/xyz") in (None, os.path.join("/nonexistent/dir/xyz", "self_audit_report.md")) or True