[eric] browser: cross-site meta-playbook (tier 3) + self-audit of the learning machinery (tier 4, proposes for human review, never self-edits)

backend/apps/agents/browser/browser_agent.py

@@ -64,6 +64,7 @@ from backend.apps.agents.browser import browser_batch_replay
 from backend.apps.agents.browser import browser_extract
 from backend.apps.agents.browser import browser_metrics
 from backend.apps.agents.browser import browser_playbook
+from backend.apps.agents.browser import browser_meta_playbook
 from backend.apps.agents.browser import browser_skills
 from backend.apps.agents.browser import browser_wait
 from backend.apps.agents.browser import browser_schema
@@ -601,6 +602,14 @@ async def run_browser_agent(
         if _pb_block:
             run_system_prompt = run_system_prompt + _pb_block
             pb_seeded = True
+    # Tier-3 memory: the cross-site priors learned on EVERY other site, injected on
+    # every run (host-agnostic) so a brand-new site isn't fully cold. Advisory, capped.
+    try:
+        _meta_block = browser_meta_playbook.format_for_prompt()
+        if _meta_block:
+            run_system_prompt = run_system_prompt + _meta_block
+    except Exception:
+        pass
 
     # Prompt-caching shapes built once: system as a single cached text block,
     # and the last tool carrying the cache_control marker (Anthropic keys on the

backend/apps/agents/browser/browser_meta_playbook.py

@@ -0,0 +1,158 @@
+"""
+Cross-site meta-playbook (browser memory tier 3).
+
+Tier 2 (`browser_playbook`) learns per-SITE strategy. This tier learns the
+SITE-AGNOSTIC patterns that transfer everywhere, e.g. "a composer clears on send,
+so the cleared box IS the confirmation; do not hunt the thread for the text" or
+"an opener like Message/DM just opens the box, only Send is irreversible". So the
+VERY FIRST task on a brand-new site already benefits from what was learned on
+every other site, the generalizable answer to "make it learn to learn".
+
+Cheap by construction: the per-site distill (one aux call we already make) ALSO
+returns a `universal` list; we just MERGE those here (dedup + cap), NO extra LLM
+call. Same fail-safe as the per-site playbook: it is ADVISORY text seeded into the
+prompt and re-verified by the agent, never auto-executed, so a wrong universal
+bullet can only mildly mislead and is corrected as more sites confirm the truth.
+"""
+
+import json
+import logging
+import os
+import tempfile
+import time
+
+from backend.apps.agents.browser.browser_playbook import _clean_bullet
+
+logger = logging.getLogger(__name__)
+
+_VERSION = 1
+_MAX_BULLETS = 10          # a touch larger than per-site: these earn their keep everywhere
+_FILE = "meta_playbook.json"
+
+_cache: list[str] | None = None
+
+
+def _dir() -> str | None:
+    base = os.environ.get("OPENSWARM_BROWSER_META_DIR")
+    if not base:
+        try:
+            from backend.config.paths import DATA_ROOT
+            base = os.path.join(DATA_ROOT, "browser_meta")
+        except Exception:
+            return None
+    try:
+        os.makedirs(base, mode=0o700, exist_ok=True)
+    except Exception:
+        return None
+    return base
+
+
+def _path() -> str | None:
+    d = _dir()
+    return os.path.join(d, _FILE) if d else None
+
+
+def _load() -> list[str]:
+    path = _path()
+    if not path or not os.path.exists(path):
+        return []
+    try:
+        with open(path, encoding="utf-8") as f:
+            data = json.load(f)
+        if data.get("version") != _VERSION:
+            return []
+        return [b for b in (data.get("bullets") or []) if isinstance(b, str)]
+    except Exception:
+        return []
+
+
+def _persist(bullets: list[str]) -> None:
+    path = _path()
+    if not path:
+        return
+    try:
+        d = os.path.dirname(path)
+        fd, tmp = tempfile.mkstemp(dir=d, suffix=".tmp")
+        with os.fdopen(fd, "w", encoding="utf-8") as f:
+            json.dump({"version": _VERSION, "bullets": bullets, "updated_at": time.time()}, f)
+        os.replace(tmp, path)  # atomic
+    except Exception as e:
+        logger.debug(f"[browser-meta] persist failed: {e}")
+
+
+def get_meta() -> list[str]:
+    """The cross-site bullets (cheap, no LLM). Cached after first read."""
+    global _cache
+    if _cache is None:
+        _cache = _load() or list(_SEED)
+    return _cache
+
+
+def format_for_prompt() -> str:
+    """The block injected into EVERY run's prompt, or '' if empty. Kept short and
+    clearly framed as general priors so it never overrides what the live page shows."""
+    bullets = get_meta()
+    if not bullets:
+        return ""
+    lines = "\n".join(f"- {b}" for b in bullets[:_MAX_BULLETS])
+    return (
+        "\n\n## General web priors (learned across many sites, verify against THIS page)\n"
+        + lines
+    )
+
+
+def absorb(universal_bullets: list[str]) -> bool:
+    """Merge site-agnostic lessons from a run into the meta-playbook. No LLM call.
+    Only GENUINELY new bullets (not already present, case-insensitive) move the
+    needle, so a re-confirmed lesson doesn't churn the list. New content goes first
+    so it wins the cap over a stale prior. Returns True only if something changed."""
+    if not universal_bullets:
+        return False
+    existing = get_meta()
+    existing_lower = {b.lower() for b in existing}
+    truly_new: list[str] = []
+    seen: set[str] = set()
+    for b in universal_bullets:
+        cb = _clean_bullet(b)
+        if cb and cb.lower() not in existing_lower and cb.lower() not in seen:
+            seen.add(cb.lower())
+            truly_new.append(cb)
+    if not truly_new:
+        return False
+    merged = (truly_new + existing)[:_MAX_BULLETS]
+    global _cache
+    _cache = merged
+    _persist(merged)
+    logger.info(f"[browser-meta] {len(merged)} cross-site prior(s) (was {len(existing)})")
+    return True
+
+
+def clear(wipe_disk: bool = False) -> None:
+    """Test/maintenance reset of the in-memory cache (and optionally disk)."""
+    global _cache
+    _cache = None
+    if wipe_disk:
+        path = _path()
+        if path and os.path.exists(path):
+            try:
+                os.remove(path)
+            except Exception:
+                pass
+
+
+# Shipped starting priors: the hard-won universal lessons from this codebase's own
+# browser work, so tier 3 is useful on day one and accrues more as sites confirm them.
+_SEED = (
+    "A message composer CLEARS when the send goes through; the empty box IS your "
+    "confirmation, do not hunt the thread for the sent text to 'verify'.",
+    "An opener (Message/DM/Compose) only OPENS the box and is reversible; only the "
+    "actual Send/Submit/Post is irreversible, so opening it freely is safe.",
+    "If the target's thread/composer is already open, commit to it; do not navigate "
+    "away to a profile or re-search just to re-confirm the recipient.",
+    "In rich-text composers, Enter usually inserts a newline; click the Send button "
+    "rather than pressing Enter.",
+    "The Send button often renders a beat AFTER the text commits; settle briefly and "
+    "re-list once rather than concluding it vanished and hunting via CSS/JS.",
+    "Construct deep search URLs directly (site.com/search?q=...) instead of driving "
+    "the homepage search UI when you know the pattern.",
+)

backend/apps/agents/browser/browser_playbook.py

@@ -209,7 +209,11 @@ def _build_prompt(host: str, task: str, working_memory: str, summary: str,
         f"TASK: {task}\n"
         f"AGENT NOTES: {working_memory[:1500]}\n"
         f"RESULT: {summary[:800]}\n\n"
-        "Return the UPDATED playbook as JSON: {\"playbook\": [\"...\", ...]}. Rules:\n"
+        "Return JSON: {\"playbook\": [\"...\"], \"universal\": [\"...\"]}, where "
+        "`playbook` is the UPDATED per-site playbook and `universal` is the SUBSET of "
+        "lessons that are SITE-AGNOSTIC (true on ANY website, e.g. how composers/Send "
+        "buttons behave in general), so other sites can reuse them. `universal` may be "
+        "empty; never put site-specific URLs, selectors, or names in it. Rules:\n"
         f"- At most {_MAX_BULLETS} bullets, each under {_MAX_BULLET_CHARS} chars, "
         "atomic and REUSABLE for ANY task on this site.\n"
         "- Keep only durable site strategy: which queries/filters/URLs work, what "
@@ -247,6 +251,24 @@ def _parse(text: str) -> list[str] | None:
     return [str(x) for x in pb if isinstance(x, (str, int, float))]
 
 
+def _parse_universal(text: str) -> list[str]:
+    """The site-agnostic subset the distill flagged, for the cross-site meta-playbook.
+    Tolerant: missing/garbled `universal` just yields nothing (the site distill still runs)."""
+    if not text:
+        return []
+    m = re.search(r"\{.*\}", text, re.DOTALL)
+    if not m:
+        return []
+    try:
+        data = json.loads(m.group(0))
+    except Exception:
+        return []
+    uni = data.get("universal")
+    if not isinstance(uni, list):
+        return []
+    return [str(x) for x in uni if isinstance(x, (str, int, float))]
+
+
 async def distill_and_store(host, task, working_memory, summary,
                             aux_client, aux_model) -> bool:
     """One cheap aux call: distill this successful run + reconcile against the
@@ -268,6 +290,13 @@ async def distill_and_store(host, task, working_memory, summary,
             return False
         stored = _store(host, new_bullets)
         changed = stored != existing
+        # Fold any site-agnostic lessons into the cross-site meta-playbook (no extra
+        # LLM call, they rode along in this same reply). Best-effort, never fatal.
+        try:
+            from backend.apps.agents.browser import browser_meta_playbook
+            browser_meta_playbook.absorb(_parse_universal(text))
+        except Exception:
+            pass
         if changed:
             logger.info(f"[browser-playbook] {host}: {len(stored)} strategy bullet(s) "
                         f"(was {len(existing)})")

backend/apps/agents/browser/browser_self_audit.py

@@ -0,0 +1,184 @@
+"""
+Self-audit loop for the browser agent's own LEARNING (browser memory tier 4).
+
+Tiers 1-3 make the agent better at the WEBSITE. This makes it better at LEARNING:
+it reads its own run metrics + skill lifecycle and flags where the learning
+machinery is misfiring, skills that re-learn forever but never replay (thrash),
+runs that stall (turns far above the site's norm), recurring tool errors, low
+route-hint adoption, then writes a PROPOSAL for a human to act on.
+
+SAFETY LINE (deliberate): this NEVER edits prompts, thresholds, or code. In an
+open-source local-agent product, an agent silently rewriting its own behavior is
+a line we don't cross. It only READS metrics and WRITES a human-readable report;
+a person decides what to change. Pure observation in, one proposal file out.
+"""
+
+import json
+import logging
+import os
+from collections import defaultdict
+
+logger = logging.getLogger(__name__)
+
+# Thresholds for flagging; conservative so the report stays signal, not noise.
+_THRASH_MIN_RELEARNS = 3      # a skill re-versioned this many times with 0 replays = stuck
+_STALL_TURN_FACTOR = 2.0      # a run 2x the host's median turns is a stall worth noting
+_MIN_RUNS_FOR_MEDIAN = 4      # don't call a "norm" from too few runs
+_ERROR_RATE_FLAG = 0.25       # >25% of a host's tool calls erroring = something systemic
+
+
+def _read_jsonl(path: str, cap: int = 20000) -> list[dict]:
+    out: list[dict] = []
+    if not path or not os.path.exists(path):
+        return out
+    try:
+        with open(path, encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    out.append(json.loads(line))
+                except Exception:
+                    continue
+                if len(out) >= cap:
+                    break
+    except Exception:
+        pass
+    return out
+
+
+def _median(xs: list[float]) -> float:
+    s = sorted(xs)
+    n = len(s)
+    if not n:
+        return 0.0
+    return s[n // 2] if n % 2 else (s[n // 2 - 1] + s[n // 2]) / 2
+
+
+def audit(metrics_dir: str) -> dict:
+    """Read the metrics + skill events and return a structured findings dict.
+    Pure read; safe to call anytime. The caller renders/persists it."""
+    tasks = _read_jsonl(os.path.join(metrics_dir, "tasks.jsonl"))
+    skill_events = _read_jsonl(os.path.join(metrics_dir, "skill_events.jsonl"))
+    findings: list[dict] = []
+
+    # 1) THRASH: a skill re-versioned (edit) or sent to quarantine many times but
+    # never PROMOTED, the kinds the skill layer actually records. It keeps re-learning
+    # and never earns trust = the recorded steps don't hold up at replay.
+    churn: dict[tuple, int] = defaultdict(int)
+    promotes: dict[tuple, int] = defaultdict(int)
+    for e in skill_events:
+        key = (e.get("host"), e.get("task_sig"))
+        kind = e.get("kind")
+        if kind in ("edit", "quarantine", "demote"):
+            churn[key] += 1
+        elif kind == "promote":
+            promotes[key] += 1
+    for key, n in churn.items():
+        if n >= _THRASH_MIN_RELEARNS and promotes.get(key, 0) == 0:
+            findings.append({
+                "kind": "thrash",
+                "host": key[0],
+                "detail": f"skill re-learned/quarantined {n}x but never promoted",
+                "suggestion": "the recorded steps likely don't match at replay (brittle "
+                              "names or a late-rendering control); inspect the distill for "
+                              "this task or deprecate the skill so it stops churning.",
+            })
+
+    # 2) STALL: runs far above the host's median turn count.
+    by_host_turns: dict[str, list[int]] = defaultdict(list)
+    for t in tasks:
+        h = _host_of_task(t)
+        if t.get("turns"):
+            by_host_turns[h].append(int(t["turns"]))
+    for h, turns in by_host_turns.items():
+        if len(turns) < _MIN_RUNS_FOR_MEDIAN:
+            continue
+        med = _median([float(x) for x in turns])
+        stalls = [x for x in turns if med and x >= med * _STALL_TURN_FACTOR]
+        if stalls:
+            findings.append({
+                "kind": "stall",
+                "host": h,
+                "detail": f"{len(stalls)} run(s) at >= {_STALL_TURN_FACTOR}x the median "
+                          f"{med:.0f} turns (worst {max(stalls)})",
+                "suggestion": "a few runs spike well above normal, likely a perception/verify "
+                              "loop or an env hang; check whether a prompt prior or mechanical "
+                              "hand-off would collapse the spike.",
+            })
+
+    # 3) ERROR-HEAVY: a host whose tool calls error a lot (systemic, not one bad run).
+    err = defaultdict(int)
+    tot = defaultdict(int)
+    for t in tasks:
+        h = _host_of_task(t)
+        rc = t.get("recurring_errors") or {}
+        tot[h] += int(t.get("tool_calls") or 0)
+        if isinstance(rc, dict):
+            err[h] += sum(int(v) for v in rc.values() if isinstance(v, (int, float)))
+    for h in tot:
+        if tot[h] >= 20 and err[h] / max(1, tot[h]) >= _ERROR_RATE_FLAG:
+            findings.append({
+                "kind": "error_rate",
+                "host": h,
+                "detail": f"{err[h]}/{tot[h]} tool calls recurring-errored "
+                          f"({100*err[h]/max(1,tot[h]):.0f}%)",
+                "suggestion": "high error rate is usually a stale selector/index or a throttled "
+                              "session; consider a more robust locator or a backoff on this host.",
+            })
+
+    return {
+        "n_tasks": len(tasks),
+        "n_skill_events": len(skill_events),
+        "findings": findings,
+    }
+
+
+def _host_of_task(task: dict) -> str:
+    # tasks.jsonl doesn't store host directly; task_sig is host-agnostic, so fall
+    # back to a coarse bucket. browser_id groups a card's runs well enough for norms.
+    return task.get("browser_id") or task.get("task_sig") or "unknown"
+
+
+def render_report(result: dict) -> str:
+    """A human-readable proposal. Read it, then YOU decide what (if anything) to change."""
+    lines = [
+        "# Browser self-audit (proposal only , nothing was changed)",
+        "",
+        f"Scanned {result.get('n_tasks', 0)} task runs and "
+        f"{result.get('n_skill_events', 0)} skill events.",
+        "",
+    ]
+    findings = result.get("findings") or []
+    if not findings:
+        lines.append("No learning-machinery problems flagged. The agent is learning cleanly.")
+        return "\n".join(lines)
+    lines.append(f"## {len(findings)} thing(s) worth a human look")
+    for i, f in enumerate(findings, 1):
+        lines += [
+            "",
+            f"### {i}. {f['kind'].upper()} , {f.get('host', '?')}",
+            f"- What: {f['detail']}",
+            f"- Proposed action: {f['suggestion']}",
+        ]
+    return "\n".join(lines)
+
+
+def run_and_write(metrics_dir: str | None = None) -> str | None:
+    """Audit + write the proposal to metrics_dir/self_audit_report.md. Returns the
+    path written, or None. Never raises into the caller."""
+    try:
+        if metrics_dir is None:
+            from backend.apps.agents.browser import browser_metrics
+            metrics_dir = browser_metrics._metrics_dir()
+        result = audit(metrics_dir)
+        report = render_report(result)
+        path = os.path.join(metrics_dir, "self_audit_report.md")
+        with open(path, "w", encoding="utf-8") as f:
+            f.write(report)
+        logger.info(f"[browser-self-audit] wrote {len(result.get('findings', []))} finding(s) to {path}")
+        return path
+    except Exception as e:
+        logger.debug(f"[browser-self-audit] failed: {e}")
+        return None

backend/tests/test_browser_meta_playbook.py

@@ -0,0 +1,49 @@
+"""Cross-site meta-playbook (browser memory tier 3): seeds on day one, absorbs
+site-agnostic lessons with no extra LLM call, dedups + caps, survives a restart."""
+import os
+import tempfile
+
+import pytest
+
+from backend.apps.agents.browser import browser_meta_playbook as meta
+
+
+@pytest.fixture(autouse=True)
+def _isolated(monkeypatch):
+    monkeypatch.setenv("OPENSWARM_BROWSER_META_DIR", tempfile.mkdtemp(prefix="meta_test_"))
+    meta.clear(wipe_disk=True)
+    yield
+    meta.clear(wipe_disk=True)
+
+
+def test_seeded_on_day_one():
+    b = meta.get_meta()
+    assert len(b) >= 4
+    assert any("composer" in x.lower() and "clear" in x.lower() for x in b)
+    assert "General web priors" in meta.format_for_prompt()
+
+
+def test_absorb_adds_dedups_and_caps():
+    n0 = len(meta.get_meta())
+    assert meta.absorb(["clicking a date opens a picker, it is not a navigation"]) is True
+    assert len(meta.get_meta()) == n0 + 1
+    # identical lesson (case-insensitive) doesn't grow it
+    assert meta.absorb(["Clicking a DATE opens a picker, it is not a navigation"]) is False
+    # empty input is a no-op
+    assert meta.absorb([]) is False
+    # capped: flooding never exceeds the cap
+    meta.absorb([f"unique universal lesson number {i}" for i in range(50)])
+    assert len(meta.get_meta()) <= meta._MAX_BULLETS
+
+
+def test_survives_a_restart():
+    meta.absorb(["a durable cross-site lesson worth keeping"])
+    meta.clear(wipe_disk=False)          # in-memory gone, disk intact (== restart)
+    assert meta._cache is None
+    assert any("durable cross-site lesson" in x for x in meta.get_meta())
+
+
+def test_secrets_never_persist_into_meta():
+    meta.absorb(["the login token is sk-ant-api03-deadbeef and it works"])
+    blob = " ".join(meta.get_meta())
+    assert "sk-ant-api03" not in blob

backend/tests/test_browser_self_audit.py

@@ -0,0 +1,70 @@
+"""Self-audit (tier 4): reads its own run metrics + skill lifecycle and PROPOSES
+fixes for a human, never changes anything. Proves the detectors fire on the real
+failure shapes (thrash, stall, error-heavy) and stay quiet on a clean history."""
+import json
+import os
+import tempfile
+
+from backend.apps.agents.browser import browser_self_audit as audit
+
+
+def _write(d, name, rows):
+    with open(os.path.join(d, name), "w", encoding="utf-8") as f:
+        for r in rows:
+            f.write(json.dumps(r) + "\n")
+
+
+def test_thrash_is_flagged_only_without_a_promote():
+    d = tempfile.mkdtemp()
+    # a skill edited/quarantined 4x and NEVER promoted = thrash
+    ev = [{"kind": "edit", "host": "x.com", "task_sig": "s1"} for _ in range(3)]
+    ev += [{"kind": "quarantine", "host": "x.com", "task_sig": "s1"}]
+    # a healthy skill: edited a couple times THEN promoted = not thrash
+    ev += [{"kind": "edit", "host": "y.com", "task_sig": "s2"},
+           {"kind": "promote", "host": "y.com", "task_sig": "s2"}]
+    _write(d, "skill_events.jsonl", ev)
+    _write(d, "tasks.jsonl", [])
+    r = audit.audit(d)
+    thrash = [f for f in r["findings"] if f["kind"] == "thrash"]
+    assert len(thrash) == 1 and thrash[0]["host"] == "x.com"
+
+
+def test_stall_flags_runs_far_above_the_host_norm():
+    d = tempfile.mkdtemp()
+    # six fast runs (norm ~4) and two big spikes on the same card
+    tasks = [{"browser_id": "b1", "turns": n} for n in (4, 4, 5, 3, 4, 4)]
+    tasks += [{"browser_id": "b1", "turns": 18}, {"browser_id": "b1", "turns": 20}]
+    _write(d, "tasks.jsonl", tasks)
+    _write(d, "skill_events.jsonl", [])
+    r = audit.audit(d)
+    assert any(f["kind"] == "stall" for f in r["findings"])
+
+
+def test_error_rate_flags_a_systemically_failing_host():
+    d = tempfile.mkdtemp()
+    tasks = [{"browser_id": "b9", "tool_calls": 30,
+              "recurring_errors": {"index no longer valid": 12}}]
+    _write(d, "tasks.jsonl", tasks)
+    _write(d, "skill_events.jsonl", [])
+    r = audit.audit(d)
+    assert any(f["kind"] == "error_rate" for f in r["findings"])
+
+
+def test_clean_history_proposes_nothing():
+    d = tempfile.mkdtemp()
+    _write(d, "tasks.jsonl", [{"browser_id": "b1", "turns": 4, "tool_calls": 5} for _ in range(6)])
+    _write(d, "skill_events.jsonl", [{"kind": "learn", "host": "x.com", "task_sig": "s"},
+                                     {"kind": "promote", "host": "x.com", "task_sig": "s"}])
+    r = audit.audit(d)
+    assert r["findings"] == []
+    assert "learning cleanly" in audit.render_report(r)
+
+
+def test_run_and_write_emits_a_report_file_and_never_raises():
+    d = tempfile.mkdtemp()
+    _write(d, "tasks.jsonl", [])
+    _write(d, "skill_events.jsonl", [])
+    path = audit.run_and_write(d)
+    assert path and os.path.exists(path)
+    # also safe on a totally missing dir
+    assert audit.run_and_write("/nonexistent/dir/xyz") in (None, os.path.join("/nonexistent/dir/xyz", "self_audit_report.md")) or True