[eric] browser: don't record a skill when the deliverable was gathered content, replay can't reproduce judgment

backend/apps/agents/browser/browser_agent.py

@@ -34,6 +34,7 @@ from backend.apps.agents.browser.browser_loop import (
     advance_stagnation,
     card_is_unavailable,
     completion_is_honest,
+    deliverable_is_informational,
     replay_recheck_is_safe,
     stagnation_exhausted,
 )
@@ -1077,10 +1078,13 @@ async def run_browser_agent(
                                     metrics_started_at, turn + 1, action_log, session.tokens,
                                     path="llm_fallback" if replay_attempted else "llm",
                                     task_sig=browser_skills._sig(skill_key_task))
-        # Learn this task ONLY from a genuinely successful run: distill the action
-        # sequence into a replayable skill. A dishonest "completion" must never be
-        # recorded, or we'd persist a broken skill that ghosts on every replay.
-        if honest:
+        # Learn this task ONLY from a genuinely successful run whose deliverable a
+        # deterministic replay can actually reproduce. We skip recording when the
+        # run was dishonest (ghost) OR when its answer was gathered/judged content
+        # (a list/report): replay can redo the clicks but not regenerate the
+        # judgment, so recording it would create a thin shortcut that later ghosts.
+        informational = deliverable_is_informational(summary)
+        if honest and not informational:
             try:
                 rec_host = browser_skills.host_of(last_seen_url)
                 _distilled = browser_skills.distill_steps(action_log)
@@ -1095,6 +1099,9 @@ async def run_browser_agent(
                     logger.info(f"[browser-skills] NOT recorded (host empty or no robust steps)")
             except Exception as e:
                 logger.warning(f"[browser-skills] record raised: {e}")
+        elif honest and informational:
+            logger.info("[browser-skills] NOT recorded (deliverable was gathered/judged content; "
+                        "replay can't reproduce it, so no thin-shortcut ghost)")
         agent_manager._sync_session_close(session)
         await ws_manager.send_to_session(session_id, "agent:status", {
             "session_id": session_id,

backend/apps/agents/browser/browser_loop.py

@@ -229,6 +229,24 @@ def replay_recheck_is_safe(action_log: list[dict]) -> bool:
     return not any(a.get("tool") in _REPLAY_DIRTYING_TOOLS for a in action_log)
 
 
+def deliverable_is_informational(summary: str) -> bool:
+    """True if the run's final answer is GATHERED CONTENT (a list/report the model
+    extracted or judged), not a short action confirmation. A deterministic replay
+    reproduces clicks and navigations but CANNOT regenerate judged/collected
+    information, so recording a skill for such a run would make a thin shortcut
+    that replays the mechanical scaffolding and then falsely claims the whole task
+    is done (the 'find me 10 X' ghost). Tool counts can't separate this from a
+    legit search (measured: both look read-heavy), but the deliverable shape can.
+    Conservative + FAIL-SAFE: when in doubt we DON'T record, so the worst case is
+    a lost speedup (re-run via the LLM), never a ghost completion."""
+    s = (summary or "").strip()
+    if len(s) > 300:
+        return True
+    if s.count("\n") >= 2:  # 3+ lines reads as a list/report, not a one-liner
+        return True
+    return False
+
+
 def completion_is_honest(action_log: list[dict]) -> tuple[bool, str]:
     """Reality-check a run the model declared done. Returns (honest, reason).
 

backend/tests/test_browser_agent_loop.py

@@ -511,6 +511,29 @@ def test_unproven_skill_that_fails_is_quarantined_and_never_retried(monkeypatch)
         "a quarantined skill must never be replayed again (would be a ghost re-fail)"
 
 
+def test_informational_run_records_no_skill_to_avoid_thin_ghost(monkeypatch):
+    # The 'find me 10 X' guard: a run that did real productive actions AND
+    # succeeded, but whose deliverable is gathered/judged content (a list), must
+    # NOT record a replayable skill, because replay would redo the clicks and
+    # falsely claim the whole task done without regenerating the judged list.
+    import backend.apps.agents.browser.browser_skills as SK
+    SK.clear()
+    BH._browser_history.clear()
+    ten = "\n".join(f"{i}. Engineer {i}, very cracked, at Startup{i}" for i in range(1, 11))
+    primary = FakeLLM([
+        Resp([_rp("search"), _tu("BrowserClickIndex", index=1)]),  # a real productive action
+        Resp([Blk("text", ten)], stop_reason="end_turn"),          # ...but the answer is a gathered list
+    ])
+    _install(monkeypatch, primary, FakeAux())
+    r = asyncio.run(BA.run_browser_agent(
+        task="find me 10 cracked design engineers", browser_id="b1", model="sonnet", initial_url=DOC_URL,
+    ))
+    # the run itself completes honestly (it did real work + returned content)...
+    assert not r.get("error")
+    # ...but NO skill is recorded, so a later run can't ghost-replay a thin shortcut
+    assert SK.find_skill("docs.google.com", "find me 10 cracked design engineers") is None
+
+
 def test_ghost_completion_is_reported_as_error_not_completed(monkeypatch):
     # The measured ghost, end to end: the model does a bunch of failing clicks
     # then declares done. The honesty gate must report 'error' (not 'completed')

backend/tests/test_browser_stagnation.py

@@ -7,6 +7,7 @@ from backend.apps.agents.browser.browser_loop import (
     advance_stagnation,
     card_is_unavailable,
     completion_is_honest,
+    deliverable_is_informational,
     is_unproductive,
     stagnation_exhausted,
     stagnation_nudge,
@@ -160,3 +161,21 @@ def test_card_is_unavailable_only_for_unrecoverable_errors():
     assert card_is_unavailable({"error": "No dashboard is connected. Open the dashboard to use browser tools."})
     assert not card_is_unavailable({"error": "Element not found: '.submit'"})
     assert not card_is_unavailable({"text": "ok", "url": "http://x"})
+
+
+# --- informational-deliverable gate (don't record a thin shortcut for a run
+# whose answer was gathered/judged content that replay can't reproduce) ---------
+
+def test_deliverable_informational_blocks_gathered_content_records_confirmations():
+    # a short action confirmation (the PROVEN Wikipedia case) -> safe to record
+    assert not deliverable_is_informational(
+        "Done. The search landed on the Alan Turing article: "
+        "https://en.wikipedia.org/wiki/Alan_Turing")
+    assert not deliverable_is_informational("Done, clicked Submit.")
+    # a gathered list/report (the 'find me 10 X' case) -> NOT safe to record
+    ten = "\n".join(f"{i}. Person {i} - Design Engineer at Co{i}" for i in range(1, 11))
+    assert deliverable_is_informational(ten)
+    # long single blob of extracted info also counts
+    assert deliverable_is_informational("Here is what I found: " + "x" * 400)
+    # empty / trivial -> not informational
+    assert not deliverable_is_informational("")