diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index dc37316a..ac7539e6 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -375,11 +375,13 @@ async def run_browser_agent( # Tier-2 memory: seed the DURABLE strategy playbook for this host (distilled # from past successful runs) so the model skips re-discovery. Advisory text, # re-verified by the agent, never auto-run. Keyed by full host like skills. + pb_seeded = False # whether tier-2 strategy was injected, for measuring its effect _pb_host = browser_skills.host_of(initial_url or current_url or "") if _pb_host: _pb_block = browser_playbook.format_for_prompt(_pb_host) if _pb_block: run_system_prompt = run_system_prompt + _pb_block + pb_seeded = True # Prompt-caching shapes built once: system as a single cached text block, # and the last tool carrying the cache_control marker (Anthropic keys on the @@ -1093,7 +1095,8 @@ async def run_browser_agent( browser_metrics.record_task(session_id, browser_id, task, final_status, metrics_started_at, turn + 1, action_log, session.tokens, path="llm_fallback" if replay_attempted else "llm", - task_sig=browser_skills._sig(skill_key_task)) + task_sig=browser_skills._sig(skill_key_task), + playbook_seeded=pb_seeded) # Learn this task ONLY from a genuinely successful run whose deliverable a # deterministic replay can actually reproduce. We skip recording when the # run was dishonest (ghost) OR when its answer was gathered/judged content diff --git a/backend/apps/agents/browser/browser_metrics.py b/backend/apps/agents/browser/browser_metrics.py index aa3508b0..5c0ab342 100644 --- a/backend/apps/agents/browser/browser_metrics.py +++ b/backend/apps/agents/browser/browser_metrics.py @@ -119,7 +119,7 @@ def record_skill_event(kind, host, task_sig, rev=0, state="", extra=None) -> Non def record_task(session_id, browser_id, task, status, started_at, turns, - action_log, tokens, path="llm", task_sig="") -> dict: + action_log, tokens, path="llm", task_sig="", playbook_seeded=False) -> dict: """One summary line per finished task: completion, total time, per-tier latency, token cost, and the recurring-error rollup. `path` records HOW the task finished (replay = no-LLM fast path, llm = full agent, llm_fallback = @@ -147,6 +147,7 @@ def record_task(session_id, browser_id, task, status, started_at, turns, "task": (task or "")[:200], "task_sig": task_sig, "path": path, + "playbook_seeded": bool(playbook_seeded), "status": status, "completed": status == "completed", "total_ms": total_ms, diff --git a/backend/tests/test_skill_metrics_and_analyzer.py b/backend/tests/test_skill_metrics_and_analyzer.py index cd3d51c8..f2132b83 100644 --- a/backend/tests/test_skill_metrics_and_analyzer.py +++ b/backend/tests/test_skill_metrics_and_analyzer.py @@ -32,11 +32,12 @@ def _log(): ] -def _task_row(sig, path, dur_s): +def _task_row(sig, path, dur_s, turns=None, playbook_seeded=False): # started_at in the past makes record_task compute a realistic total_ms. - bm.record_task("s-" + sig + path, "b", sig, "completed", - time.time() - dur_s, 0 if path == "replay" else 3, - _log(), {"input": 10, "output": 5}, path=path, task_sig=sig) + bm.record_task("s-" + sig + path + str(turns) + str(playbook_seeded), "b", sig, "completed", + time.time() - dur_s, turns if turns is not None else (0 if path == "replay" else 3), + _log(), {"input": 10, "output": 5}, path=path, task_sig=sig, + playbook_seeded=playbook_seeded) def test_skill_events_are_emitted_for_each_transition(_metrics_dir): @@ -113,6 +114,31 @@ def test_analyzer_reports_composition(_metrics_dir, capsys): assert "1 dependent(s) re-proofed" in out +def test_analyzer_reports_playbook_cutting_exploration_turns(_metrics_dir, capsys): + # tier-2 win: a cold run on a host takes many turns; once strategy is seeded, + # the same kind of task takes fewer. The analyzer must report HELPS. + sig = sk._sig("find people") + _task_row(sig, "llm", 60.0, turns=14, playbook_seeded=False) # cold + _task_row(sig, "llm", 40.0, turns=8, playbook_seeded=True) # seeded -> fewer turns + mod = _load_analyzer() + tasks = mod._load(os.path.join(_metrics_dir, "tasks.jsonl")) + mod.playbook_report(tasks) + out = capsys.readouterr().out + assert "STRATEGIC PLAYBOOK" in out and "HELPS" in out and "NOT HELPING" not in out + + +def test_analyzer_flags_playbook_that_does_not_help(_metrics_dir, capsys): + # anti-ghost: memory is active (seeded) but seeded runs are NOT cheaper -> flag. + sig = sk._sig("stubborn task") + _task_row(sig, "llm", 60.0, turns=10, playbook_seeded=False) + _task_row(sig, "llm", 60.0, turns=12, playbook_seeded=True) # seeded but MORE turns + mod = _load_analyzer() + tasks = mod._load(os.path.join(_metrics_dir, "tasks.jsonl")) + mod.playbook_report(tasks) + out = capsys.readouterr().out + assert "NOT HELPING" in out + + # --- helpers --------------------------------------------------------------- def _read(path): import json diff --git a/scripts/analyze-browser-metrics.py b/scripts/analyze-browser-metrics.py index e374c13e..4e7ea740 100755 --- a/scripts/analyze-browser-metrics.py +++ b/scripts/analyze-browser-metrics.py @@ -141,6 +141,35 @@ def skill_layer_report(tasks, skill_events): f"{kinds.get('invalidate', 0)} dependent(s) re-proofed after a foundation changed") +def playbook_report(tasks): + """Does the tier-2 strategy playbook actually make judgment tasks cheaper over + time? Compare LLM-path runs on a host BEFORE a playbook existed (cold) vs once + it was seeded. The win is fewer exploration turns; flag a host where seeded + runs are NOT cheaper (the 'memory looks active but doesn't help' ghost).""" + from collections import defaultdict + by_host = defaultdict(lambda: {"cold": [], "seeded": []}) + for t in tasks: + if t.get("path") not in ("llm", "llm_fallback") or not t.get("completed"): + continue + host = (t.get("task_sig") or "").split(" ")[0] or t.get("browser_id", "?") + bucket = "seeded" if t.get("playbook_seeded") else "cold" + by_host[host][bucket].append(t.get("turns", 0) or 0) + rows = {h: v for h, v in by_host.items() if v["cold"] and v["seeded"]} + if not any(t.get("playbook_seeded") for t in tasks): + return # nothing seeded yet, no measurement to make + print("\n=== STRATEGIC PLAYBOOK (does learned site-strategy cut exploration?) ===") + seeded_total = sum(1 for t in tasks if t.get("playbook_seeded")) + print(f" runs seeded with a playbook: {seeded_total}") + if not rows: + print(" (no host yet has BOTH a cold and a seeded run to compare)") + return + for h, v in rows.items(): + cold = sum(v["cold"]) / len(v["cold"]) + seeded = sum(v["seeded"]) / len(v["seeded"]) + verdict = "HELPS" if seeded < cold else "⚠️ NOT HELPING" + print(f" {h[:40]:40} cold avg {cold:.1f} turns -> seeded avg {seeded:.1f} turns {verdict}") + + def main(): d = sys.argv[1] if len(sys.argv) > 1 else _default_dir() events = _load(os.path.join(d, "events.jsonl")) @@ -207,6 +236,7 @@ def main(): f"(completed minus ghosts)") skill_layer_report(tasks, skill_events) + playbook_report(tasks) if __name__ == "__main__":