[eric] browser: measure whether the playbook cuts exploration turns, flag when it doesn't

backend/apps/agents/browser/browser_agent.py

@@ -375,11 +375,13 @@ async def run_browser_agent(
     # Tier-2 memory: seed the DURABLE strategy playbook for this host (distilled
     # from past successful runs) so the model skips re-discovery. Advisory text,
     # re-verified by the agent, never auto-run. Keyed by full host like skills.
+    pb_seeded = False  # whether tier-2 strategy was injected, for measuring its effect
     _pb_host = browser_skills.host_of(initial_url or current_url or "")
     if _pb_host:
         _pb_block = browser_playbook.format_for_prompt(_pb_host)
         if _pb_block:
             run_system_prompt = run_system_prompt + _pb_block
+            pb_seeded = True
 
     # Prompt-caching shapes built once: system as a single cached text block,
     # and the last tool carrying the cache_control marker (Anthropic keys on the
@@ -1093,7 +1095,8 @@ async def run_browser_agent(
         browser_metrics.record_task(session_id, browser_id, task, final_status,
                                     metrics_started_at, turn + 1, action_log, session.tokens,
                                     path="llm_fallback" if replay_attempted else "llm",
-                                    task_sig=browser_skills._sig(skill_key_task))
+                                    task_sig=browser_skills._sig(skill_key_task),
+                                    playbook_seeded=pb_seeded)
         # Learn this task ONLY from a genuinely successful run whose deliverable a
         # deterministic replay can actually reproduce. We skip recording when the
         # run was dishonest (ghost) OR when its answer was gathered/judged content

backend/apps/agents/browser/browser_metrics.py

@@ -119,7 +119,7 @@ def record_skill_event(kind, host, task_sig, rev=0, state="", extra=None) -> Non
 
 
 def record_task(session_id, browser_id, task, status, started_at, turns,
-                action_log, tokens, path="llm", task_sig="") -> dict:
+                action_log, tokens, path="llm", task_sig="", playbook_seeded=False) -> dict:
     """One summary line per finished task: completion, total time, per-tier
     latency, token cost, and the recurring-error rollup. `path` records HOW the
     task finished (replay = no-LLM fast path, llm = full agent, llm_fallback =
@@ -147,6 +147,7 @@ def record_task(session_id, browser_id, task, status, started_at, turns,
         "task": (task or "")[:200],
         "task_sig": task_sig,
         "path": path,
+        "playbook_seeded": bool(playbook_seeded),
         "status": status,
         "completed": status == "completed",
         "total_ms": total_ms,

backend/tests/test_skill_metrics_and_analyzer.py

@@ -32,11 +32,12 @@ def _log():
     ]
 
 
-def _task_row(sig, path, dur_s):
+def _task_row(sig, path, dur_s, turns=None, playbook_seeded=False):
     # started_at in the past makes record_task compute a realistic total_ms.
-    bm.record_task("s-" + sig + path, "b", sig, "completed",
-                   time.time() - dur_s, 0 if path == "replay" else 3,
-                   _log(), {"input": 10, "output": 5}, path=path, task_sig=sig)
+    bm.record_task("s-" + sig + path + str(turns) + str(playbook_seeded), "b", sig, "completed",
+                   time.time() - dur_s, turns if turns is not None else (0 if path == "replay" else 3),
+                   _log(), {"input": 10, "output": 5}, path=path, task_sig=sig,
+                   playbook_seeded=playbook_seeded)
 
 
 def test_skill_events_are_emitted_for_each_transition(_metrics_dir):
@@ -113,6 +114,31 @@ def test_analyzer_reports_composition(_metrics_dir, capsys):
     assert "1 dependent(s) re-proofed" in out
 
 
+def test_analyzer_reports_playbook_cutting_exploration_turns(_metrics_dir, capsys):
+    # tier-2 win: a cold run on a host takes many turns; once strategy is seeded,
+    # the same kind of task takes fewer. The analyzer must report HELPS.
+    sig = sk._sig("find people")
+    _task_row(sig, "llm", 60.0, turns=14, playbook_seeded=False)   # cold
+    _task_row(sig, "llm", 40.0, turns=8, playbook_seeded=True)     # seeded -> fewer turns
+    mod = _load_analyzer()
+    tasks = mod._load(os.path.join(_metrics_dir, "tasks.jsonl"))
+    mod.playbook_report(tasks)
+    out = capsys.readouterr().out
+    assert "STRATEGIC PLAYBOOK" in out and "HELPS" in out and "NOT HELPING" not in out
+
+
+def test_analyzer_flags_playbook_that_does_not_help(_metrics_dir, capsys):
+    # anti-ghost: memory is active (seeded) but seeded runs are NOT cheaper -> flag.
+    sig = sk._sig("stubborn task")
+    _task_row(sig, "llm", 60.0, turns=10, playbook_seeded=False)
+    _task_row(sig, "llm", 60.0, turns=12, playbook_seeded=True)    # seeded but MORE turns
+    mod = _load_analyzer()
+    tasks = mod._load(os.path.join(_metrics_dir, "tasks.jsonl"))
+    mod.playbook_report(tasks)
+    out = capsys.readouterr().out
+    assert "NOT HELPING" in out
+
+
 # --- helpers ---------------------------------------------------------------
 def _read(path):
     import json

scripts/analyze-browser-metrics.py

@@ -141,6 +141,35 @@ def skill_layer_report(tasks, skill_events):
               f"{kinds.get('invalidate', 0)} dependent(s) re-proofed after a foundation changed")
 
 
+def playbook_report(tasks):
+    """Does the tier-2 strategy playbook actually make judgment tasks cheaper over
+    time? Compare LLM-path runs on a host BEFORE a playbook existed (cold) vs once
+    it was seeded. The win is fewer exploration turns; flag a host where seeded
+    runs are NOT cheaper (the 'memory looks active but doesn't help' ghost)."""
+    from collections import defaultdict
+    by_host = defaultdict(lambda: {"cold": [], "seeded": []})
+    for t in tasks:
+        if t.get("path") not in ("llm", "llm_fallback") or not t.get("completed"):
+            continue
+        host = (t.get("task_sig") or "").split(" ")[0] or t.get("browser_id", "?")
+        bucket = "seeded" if t.get("playbook_seeded") else "cold"
+        by_host[host][bucket].append(t.get("turns", 0) or 0)
+    rows = {h: v for h, v in by_host.items() if v["cold"] and v["seeded"]}
+    if not any(t.get("playbook_seeded") for t in tasks):
+        return  # nothing seeded yet, no measurement to make
+    print("\n=== STRATEGIC PLAYBOOK (does learned site-strategy cut exploration?) ===")
+    seeded_total = sum(1 for t in tasks if t.get("playbook_seeded"))
+    print(f"  runs seeded with a playbook: {seeded_total}")
+    if not rows:
+        print("  (no host yet has BOTH a cold and a seeded run to compare)")
+        return
+    for h, v in rows.items():
+        cold = sum(v["cold"]) / len(v["cold"])
+        seeded = sum(v["seeded"]) / len(v["seeded"])
+        verdict = "HELPS" if seeded < cold else "⚠️ NOT HELPING"
+        print(f"    {h[:40]:40} cold avg {cold:.1f} turns -> seeded avg {seeded:.1f} turns  {verdict}")
+
+
 def main():
     d = sys.argv[1] if len(sys.argv) > 1 else _default_dir()
     events = _load(os.path.join(d, "events.jsonl"))
@@ -207,6 +236,7 @@ def main():
               f"(completed minus ghosts)")
 
     skill_layer_report(tasks, skill_events)
+    playbook_report(tasks)
 
 
 if __name__ == "__main__":