From bde92de0620bb8d2349d65989c14daf8f8beff9c Mon Sep 17 00:00:00 2001 From: ciregenz Date: Fri, 7 Aug 2026 17:18:30 -0700 Subject: [PATCH] [eric] prompt: reading the web starts with WebSearch/WebFetch and only escalates to the browser when they come back thin --- backend/apps/settings/models.py | 6 ++- backend/apps/settings/store.py | 11 ++++ .../tests/test_default_prompt_migration.py | 52 +++++++++++++++++++ 3 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 backend/tests/test_default_prompt_migration.py diff --git a/backend/apps/settings/models.py b/backend/apps/settings/models.py index 320b9a01..c8c1d5e0 100644 --- a/backend/apps/settings/models.py +++ b/backend/apps/settings/models.py @@ -33,8 +33,10 @@ DEFAULT_SYSTEM_PROMPT = ( "4. **Unsure which server.** `MCPList` for a cheap survey, or " '`MCPSearch("")` to rank servers by relevance. Do this before ' "MCPActivate, never via ToolSearch.\n" - "5. **No tool fits.** WebSearch / WebFetch for information. BrowserAgent only for " - "visual interaction, form filling, or sites with no API path.\n\n" + "5. **Reading the web.** WebSearch / WebFetch first, always: they are far faster than " + "driving a browser and they cover ordinary pages. Escalate to BrowserAgent only once " + "they have actually come back thin or blocked (login wall, paywall, JS-only page), or " + "when the task needs visual interaction or form filling.\n\n" "### Choosing among similar names\n" "A matching name is a hypothesis, not an answer. Before calling, read the description " "and the required parameters, and confirm three things: it performs the action you " diff --git a/backend/apps/settings/store.py b/backend/apps/settings/store.py index 00ddcacd..12fc6afb 100644 --- a/backend/apps/settings/store.py +++ b/backend/apps/settings/store.py @@ -50,8 +50,19 @@ P_LEGACY_DEFAULT_SYSTEM_PROMPT = ( "If you genuinely need clarification on something ambiguous, use the " "AskUserQuestion tool. Never ask questions inline in plain text.\n" ) +# The 820cf578-era revision, which differs from the current default only in ladder step 5 (it framed +# the web tools as "No tool fits" so agents reached for the browser first). Derived, not duplicated. +P_LEGACY_LADDER_V1 = DEFAULT_SYSTEM_PROMPT.replace( + "5. **Reading the web.** WebSearch / WebFetch first, always: they are far faster than " + "driving a browser and they cover ordinary pages. Escalate to BrowserAgent only once " + "they have actually come back thin or blocked (login wall, paywall, JS-only page), or " + "when the task needs visual interaction or form filling.\n\n", + "5. **No tool fits.** WebSearch / WebFetch for information. BrowserAgent only for " + "visual interaction, form filling, or sites with no API path.\n\n", +) P_LEGACY_DEFAULT_SYSTEM_PROMPTS = ( P_LEGACY_DEFAULT_SYSTEM_PROMPT, + P_LEGACY_LADDER_V1, P_LEGACY_DEFAULT_SYSTEM_PROMPT.replace( "1. Connected MCP tools; fastest and most reliable. To reach an integration you " "don't already see, use MCPSearch then MCPActivate; never ToolSearch for it.\n", diff --git a/backend/tests/test_default_prompt_migration.py b/backend/tests/test_default_prompt_migration.py new file mode 100644 index 00000000..dd9963d5 --- /dev/null +++ b/backend/tests/test_default_prompt_migration.py @@ -0,0 +1,52 @@ +"""Shipping a new default system prompt has to carry the old one with it. + +The default persists into settings.json, so bumping the constant alone leaves every existing install +on the old text forever: they never see the change, and "Reset to default" compares against something +that no longer exists. Each legacy revision is derived from the current default with a `replace`, and +a `replace` whose anchor has drifted silently returns the string unchanged, which turns the whole +migration into a no-op that nothing would notice. These pin both halves. +""" + +from backend.apps.settings.models import DEFAULT_SYSTEM_PROMPT +from backend.apps.settings.store import ( + P_LEGACY_DEFAULT_SYSTEM_PROMPT, + P_LEGACY_DEFAULT_SYSTEM_PROMPTS, + P_LEGACY_LADDER_V1, +) + + +def test_every_legacy_revision_actually_differs_from_the_current_default(): + """A derived revision equal to the default means its anchor text drifted and the replace did + nothing, so users on that revision would never be migrated off it.""" + for i, legacy in enumerate(P_LEGACY_DEFAULT_SYSTEM_PROMPTS): + assert legacy != DEFAULT_SYSTEM_PROMPT, ( + f"legacy revision {i} is byte-identical to the current default, so its derivation " + "silently no-opped, most likely because the anchor string it replaces was edited" + ) + + +def test_the_shipped_revisions_are_all_tracked(): + assert P_LEGACY_DEFAULT_SYSTEM_PROMPT in P_LEGACY_DEFAULT_SYSTEM_PROMPTS + assert P_LEGACY_LADDER_V1 in P_LEGACY_DEFAULT_SYSTEM_PROMPTS + assert len(set(P_LEGACY_DEFAULT_SYSTEM_PROMPTS)) == len(P_LEGACY_DEFAULT_SYSTEM_PROMPTS), ( + "duplicate legacy revisions mean one of the derivations collapsed onto another" + ) + + +def test_the_web_ladder_tells_the_agent_to_try_the_cheap_tools_first(): + """Eric's ask, and the reason the ladder step was rewritten: an agent given a plain reading task + was driving a browser instead of searching, which is far slower. The old wording filed the web + tools under 'No tool fits', which reads as a last resort.""" + assert "WebSearch / WebFetch first" in DEFAULT_SYSTEM_PROMPT + assert "No tool fits." not in DEFAULT_SYSTEM_PROMPT, "the last-resort framing is back" + ladder = DEFAULT_SYSTEM_PROMPT[DEFAULT_SYSTEM_PROMPT.index("5. **Reading the web.**"):] + web_at = ladder.index("WebSearch") + browser_at = ladder.index("BrowserAgent") + assert web_at < browser_at, "the browser must not be named before the cheap tools" + assert "Escalate to BrowserAgent only" in DEFAULT_SYSTEM_PROMPT, "the fallback must stay conditional" + + +def test_a_user_customized_prompt_is_never_mistaken_for_a_default(): + """The migration is a verbatim match, so an edited prompt must never collide with a shipped one.""" + customized = DEFAULT_SYSTEM_PROMPT + "\nAlways answer in French.\n" + assert customized not in P_LEGACY_DEFAULT_SYSTEM_PROMPTS