diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index bdf43c0e..9a5ce06c 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -34,6 +34,7 @@ from backend.apps.agents.browser.browser_loop import ( advance_stagnation, card_is_unavailable, completion_is_honest, + deliverable_is_informational, replay_recheck_is_safe, stagnation_exhausted, ) @@ -1077,10 +1078,13 @@ async def run_browser_agent( metrics_started_at, turn + 1, action_log, session.tokens, path="llm_fallback" if replay_attempted else "llm", task_sig=browser_skills._sig(skill_key_task)) - # Learn this task ONLY from a genuinely successful run: distill the action - # sequence into a replayable skill. A dishonest "completion" must never be - # recorded, or we'd persist a broken skill that ghosts on every replay. - if honest: + # Learn this task ONLY from a genuinely successful run whose deliverable a + # deterministic replay can actually reproduce. We skip recording when the + # run was dishonest (ghost) OR when its answer was gathered/judged content + # (a list/report): replay can redo the clicks but not regenerate the + # judgment, so recording it would create a thin shortcut that later ghosts. + informational = deliverable_is_informational(summary) + if honest and not informational: try: rec_host = browser_skills.host_of(last_seen_url) _distilled = browser_skills.distill_steps(action_log) @@ -1095,6 +1099,9 @@ async def run_browser_agent( logger.info(f"[browser-skills] NOT recorded (host empty or no robust steps)") except Exception as e: logger.warning(f"[browser-skills] record raised: {e}") + elif honest and informational: + logger.info("[browser-skills] NOT recorded (deliverable was gathered/judged content; " + "replay can't reproduce it, so no thin-shortcut ghost)") agent_manager._sync_session_close(session) await ws_manager.send_to_session(session_id, "agent:status", { "session_id": session_id, diff --git a/backend/apps/agents/browser/browser_loop.py b/backend/apps/agents/browser/browser_loop.py index 360ec7ca..8eff3faa 100644 --- a/backend/apps/agents/browser/browser_loop.py +++ b/backend/apps/agents/browser/browser_loop.py @@ -229,6 +229,24 @@ def replay_recheck_is_safe(action_log: list[dict]) -> bool: return not any(a.get("tool") in _REPLAY_DIRTYING_TOOLS for a in action_log) +def deliverable_is_informational(summary: str) -> bool: + """True if the run's final answer is GATHERED CONTENT (a list/report the model + extracted or judged), not a short action confirmation. A deterministic replay + reproduces clicks and navigations but CANNOT regenerate judged/collected + information, so recording a skill for such a run would make a thin shortcut + that replays the mechanical scaffolding and then falsely claims the whole task + is done (the 'find me 10 X' ghost). Tool counts can't separate this from a + legit search (measured: both look read-heavy), but the deliverable shape can. + Conservative + FAIL-SAFE: when in doubt we DON'T record, so the worst case is + a lost speedup (re-run via the LLM), never a ghost completion.""" + s = (summary or "").strip() + if len(s) > 300: + return True + if s.count("\n") >= 2: # 3+ lines reads as a list/report, not a one-liner + return True + return False + + def completion_is_honest(action_log: list[dict]) -> tuple[bool, str]: """Reality-check a run the model declared done. Returns (honest, reason). diff --git a/backend/tests/test_browser_agent_loop.py b/backend/tests/test_browser_agent_loop.py index 419c103f..909b956a 100644 --- a/backend/tests/test_browser_agent_loop.py +++ b/backend/tests/test_browser_agent_loop.py @@ -511,6 +511,29 @@ def test_unproven_skill_that_fails_is_quarantined_and_never_retried(monkeypatch) "a quarantined skill must never be replayed again (would be a ghost re-fail)" +def test_informational_run_records_no_skill_to_avoid_thin_ghost(monkeypatch): + # The 'find me 10 X' guard: a run that did real productive actions AND + # succeeded, but whose deliverable is gathered/judged content (a list), must + # NOT record a replayable skill, because replay would redo the clicks and + # falsely claim the whole task done without regenerating the judged list. + import backend.apps.agents.browser.browser_skills as SK + SK.clear() + BH._browser_history.clear() + ten = "\n".join(f"{i}. Engineer {i}, very cracked, at Startup{i}" for i in range(1, 11)) + primary = FakeLLM([ + Resp([_rp("search"), _tu("BrowserClickIndex", index=1)]), # a real productive action + Resp([Blk("text", ten)], stop_reason="end_turn"), # ...but the answer is a gathered list + ]) + _install(monkeypatch, primary, FakeAux()) + r = asyncio.run(BA.run_browser_agent( + task="find me 10 cracked design engineers", browser_id="b1", model="sonnet", initial_url=DOC_URL, + )) + # the run itself completes honestly (it did real work + returned content)... + assert not r.get("error") + # ...but NO skill is recorded, so a later run can't ghost-replay a thin shortcut + assert SK.find_skill("docs.google.com", "find me 10 cracked design engineers") is None + + def test_ghost_completion_is_reported_as_error_not_completed(monkeypatch): # The measured ghost, end to end: the model does a bunch of failing clicks # then declares done. The honesty gate must report 'error' (not 'completed') diff --git a/backend/tests/test_browser_stagnation.py b/backend/tests/test_browser_stagnation.py index a1996b17..e5b6470f 100644 --- a/backend/tests/test_browser_stagnation.py +++ b/backend/tests/test_browser_stagnation.py @@ -7,6 +7,7 @@ from backend.apps.agents.browser.browser_loop import ( advance_stagnation, card_is_unavailable, completion_is_honest, + deliverable_is_informational, is_unproductive, stagnation_exhausted, stagnation_nudge, @@ -160,3 +161,21 @@ def test_card_is_unavailable_only_for_unrecoverable_errors(): assert card_is_unavailable({"error": "No dashboard is connected. Open the dashboard to use browser tools."}) assert not card_is_unavailable({"error": "Element not found: '.submit'"}) assert not card_is_unavailable({"text": "ok", "url": "http://x"}) + + +# --- informational-deliverable gate (don't record a thin shortcut for a run +# whose answer was gathered/judged content that replay can't reproduce) --------- + +def test_deliverable_informational_blocks_gathered_content_records_confirmations(): + # a short action confirmation (the PROVEN Wikipedia case) -> safe to record + assert not deliverable_is_informational( + "Done. The search landed on the Alan Turing article: " + "https://en.wikipedia.org/wiki/Alan_Turing") + assert not deliverable_is_informational("Done, clicked Submit.") + # a gathered list/report (the 'find me 10 X' case) -> NOT safe to record + ten = "\n".join(f"{i}. Person {i} - Design Engineer at Co{i}" for i in range(1, 11)) + assert deliverable_is_informational(ten) + # long single blob of extracted info also counts + assert deliverable_is_informational("Here is what I found: " + "x" * 400) + # empty / trivial -> not informational + assert not deliverable_is_informational("")