diff --git a/backend/apps/agents/agent_manager.py b/backend/apps/agents/agent_manager.py
index 9cdf2778..b0aea358 100644
--- a/backend/apps/agents/agent_manager.py
+++ b/backend/apps/agents/agent_manager.py
@@ -1209,29 +1209,6 @@ class AgentManager:
# with no Anthropic path; Anthropic's hosted search is
# higher-quality so we prefer it whenever it's reachable.
_m = _router_model_id if isinstance(_router_model_id, str) else ""
- _has_anthropic_path = (
- getattr(global_settings, "connection_mode", "own_key") == "openswarm-pro"
- or bool(getattr(global_settings, "anthropic_api_key", None))
- )
- # Collect the active 9Router anthropic-family provider ids so the
- # web-search reliability check (below) can distinguish a STABLE
- # credential (direct `anthropic`) from subscription OAuth
- # (`claude`/`claude-code`), whose hosted-WebSearch delegation 401s on
- # token rotation. Subscription OAuth must NOT suppress the DDG fallback.
- _9r_provider_ids: list[str] = []
- try:
- from backend.apps.nine_router import get_providers as _9r_providers
- _conns = await _9r_providers()
- _9r_provider_ids = [
- c.get("provider")
- for c in _conns
- if isinstance(c, dict)
- and c.get("provider") in ("claude", "claude-code", "anthropic")
- and c.get("isActive")
- ]
- except Exception:
- pass
-
# When the primary is non-Claude we deliberately don't count
# OpenSwarm Pro as an Anthropic path, using the Pro pool for
# WebSearch on a GPT/Gemini session would drain it for the
@@ -1251,25 +1228,31 @@ class AgentManager:
# MCP to register so WebSearch always cascades through our own
# /api/web/search (Gemini → OpenAI → DuckDuckGo).
_is_custom_session = _api_type_for_session == "custom"
- # Only consider the user's own Anthropic API key sufficient
- # if the conversation primary IS Claude. Pre-fix: any user
- # with an Anthropic key set OR on OpenSwarm Pro skipped the
- # openswarm-web MCP registration and the CLI's built-in
- # WebSearch routed to Anthropic Haiku, which on a Codex
- # /Gemini session drained the Pro pool's Haiku quota for
- # WebSearch calls, even though the conversation primary
- # (Codex/Gemini) supports native search via its own credits.
- # Post-fix: non-Claude primaries always register openswarm-web,
- # which cascades Gemini-native → OpenAI-native → subscriptions
- # → DDG, only falling to Anthropic if everything else missing.
+ # The built-in WebSearch's aux haiku call only authenticates when it
+ # reaches an ENTITLED Anthropic endpoint. That's true in exactly two
+ # cases, mirroring the direct-Anthropic env-branch built further down:
+ # a direct Anthropic api-route model (base_url = api.anthropic.com
+ # with the user's key), or OpenSwarm Pro (entitled to the managed pool
+ # 9Router's anthropic/* resolves to). A SUBSCRIPTION-route Claude
+ # model (opus-4-8, route=None) routes the haiku call through 9Router
+ # to the managed pool and 401s for non-Pro users, so a bare key in
+ # settings is NOT enough; it must be a *-api route model. Everyone
+ # else registers openswarm-web and cascades through /api/web/search.
from backend.apps.agents.tools.web import anthropic_web_search_is_reliable
+ from backend.apps.agents.providers.registry import _find_builtin_model as _fbm_web
+ _web_model_entry = _fbm_web(session.model)
+ _uses_direct_anthropic_api = (
+ _web_model_entry is not None
+ and _web_model_entry.get("route") == "api"
+ and _web_model_entry.get("api") == "anthropic"
+ and bool(getattr(global_settings, "anthropic_api_key", None))
+ )
_has_anthropic_path = (
not _is_custom_session
and _primary_is_claude
and anthropic_web_search_is_reliable(
- has_direct_anthropic_key=bool(getattr(global_settings, "anthropic_api_key", None)),
+ uses_direct_anthropic_api=_uses_direct_anthropic_api,
is_pro=(getattr(global_settings, "connection_mode", "own_key") == "openswarm-pro"),
- provider_ids=_9r_provider_ids,
)
)
diff --git a/backend/apps/agents/tools/web.py b/backend/apps/agents/tools/web.py
index f50755a4..82e46fcf 100644
--- a/backend/apps/agents/tools/web.py
+++ b/backend/apps/agents/tools/web.py
@@ -18,23 +18,34 @@ _USER_AGENT = (
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
)
-# 9router provider ids that authenticate with a STABLE token. Subscription OAuth
-# ('claude'/'claude-code') is deliberately excluded: the CLI's built-in WebSearch
-# delegates to Haiku via the subscription's rotating OAuth token, which 401s
-# intermittently ("Invalid bearer token, reset after 2m"). Only stable creds are
-# reliable enough to suppress our free DuckDuckGo fallback.
-_STABLE_ANTHROPIC_PROVIDERS = ("anthropic",)
+class DDGRateLimited(Exception):
+ """DuckDuckGo answered with its throttle challenge (HTTP 202), not results.
-def anthropic_web_search_is_reliable(*, has_direct_anthropic_key: bool,
- is_pro: bool, provider_ids) -> bool:
- """Whether the Anthropic-hosted WebSearch path is reliable enough to suppress
- the DuckDuckGo fallback. A subscription-OAuth-only user is NOT reliable (its
- web-search delegation 401s on token rotation), so those users keep the free,
- always-working DDG path instead of a flaky hosted one."""
- if has_direct_anthropic_key or is_pro:
- return True
- return any(p in _STABLE_ANTHROPIC_PROVIDERS for p in (provider_ids or []))
+ Distinct from 'genuinely zero hits' so the caller can fail over to another
+ backend instead of reporting an empty search to the user. The throttle is
+ per-IP and burst-triggered; a quick retry on the same or the `lite` endpoint
+ does NOT clear it (both share the limiter), so the only cure is a different
+ backend or waiting it out."""
+
+def anthropic_web_search_is_reliable(*, uses_direct_anthropic_api: bool,
+ is_pro: bool) -> bool:
+ """Whether the CLI's built-in WebSearch is reliable enough to suppress the
+ DuckDuckGo fallback. The built-in tool fires an aux `claude-haiku` call, and
+ that call only authenticates when it reaches an ENTITLED Anthropic endpoint:
+
+ - `uses_direct_anthropic_api`: the session is pinned to a direct Anthropic
+ api-route model (base_url = api.anthropic.com with the user's own key),
+ so the haiku call hits Anthropic directly and works.
+ - `is_pro`: OpenSwarm Pro, entitled to the managed `anthropic` pool that
+ 9Router's `anthropic/*` route resolves to.
+
+ A bare `anthropic_api_key` in settings is NOT sufficient: a SUBSCRIPTION-route
+ Claude model (e.g. `opus-4-8`, route=None) still sends the haiku call through
+ 9Router to the managed pool, which 401s for non-Pro users ('Invalid bearer
+ token, reset after ~2m'). Only a `*-api` route model talks to Anthropic
+ directly. Everyone else keeps the free, always-working DDG path."""
+ return bool(uses_direct_anthropic_api or is_pro)
def _truncate(text: str, limit: int = _MAX_OUTPUT_BYTES) -> str:
@@ -87,6 +98,11 @@ class WebSearchTool(BaseTool):
if not results:
return [{"type": "text", "text": f"No search results found for: {query}"}]
return [{"type": "text", "text": results}]
+ except DDGRateLimited:
+ return [{"type": "text", "text": (
+ "DuckDuckGo is rate-limiting this network right now (HTTP 202). "
+ "Wait a bit and retry, or use a different search source."
+ )}]
except Exception as exc:
return [{"type": "text", "text": f"Web search error: {exc}"}]
@@ -102,6 +118,11 @@ class WebSearchTool(BaseTool):
"https://html.duckduckgo.com/html/",
data={"q": query},
)
+ # DDG serves its throttle challenge as 202 (a ~14KB no-results page),
+ # which is a 2xx so raise_for_status() sails right past it. Catch it
+ # explicitly so we report "rate-limited" instead of a bogus "no hits".
+ if resp.status_code == 202:
+ raise DDGRateLimited(query)
resp.raise_for_status()
body = resp.text
@@ -133,6 +154,13 @@ class WebSearchTool(BaseTool):
continue
raw_url = html.unescape(link_match.group(1))
+
+ # Drop sponsored rows: DDG ads point at its own y.js click-tracker
+ # (ad_domain/ad_provider) instead of a real uddg= redirect, so they'd
+ # otherwise show up as junk "duckduckgo.com/y.js?ad_..." results.
+ if "/y.js?" in raw_url or "ad_provider=" in raw_url or "ad_domain=" in raw_url:
+ continue
+
title = _strip_html(link_match.group(2)).strip()
snippet_match = re.search(
diff --git a/backend/tests/test_web_search_ddg.py b/backend/tests/test_web_search_ddg.py
new file mode 100644
index 00000000..1756ba1b
--- /dev/null
+++ b/backend/tests/test_web_search_ddg.py
@@ -0,0 +1,93 @@
+"""DuckDuckGo parsing robustness: rate-limit (202) and ad-row stripping.
+
+These pin the two bugs that turned DDG into a flaky 'No results found' source:
+ 1. DDG serves its throttle challenge as HTTP 202 (a 2xx), so raise_for_status()
+ missed it and we parsed an empty page as a real empty result set.
+ 2. Sponsored rows point at DDG's own y.js click-tracker (ad_domain/ad_provider)
+ and were emitted as junk 'duckduckgo.com/y.js?...' results.
+
+We mock the network so the test is deterministic and offline.
+"""
+
+import httpx
+import pytest
+
+from backend.apps.agents.tools.web import WebSearchTool, DDGRateLimited
+
+
+class _FakeResp:
+ def __init__(self, status_code: int, text: str):
+ self.status_code = status_code
+ self.text = text
+
+ def raise_for_status(self):
+ if self.status_code >= 400:
+ raise httpx.HTTPStatusError("err", request=None, response=None)
+
+
+class _FakeClient:
+ """Stands in for httpx.AsyncClient; returns a canned response."""
+ def __init__(self, resp: _FakeResp):
+ self._resp = resp
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, *a):
+ return False
+
+ async def post(self, *a, **k):
+ return self._resp
+
+
+def _patch_client(monkeypatch, resp: _FakeResp):
+ monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: _FakeClient(resp))
+
+
+# One real organic result + one sponsored (ad) row in DDG's html markup.
+_HTML_WITH_AD = """
+
+
+"""
+
+
+@pytest.mark.asyncio
+async def test_202_raises_rate_limited_not_empty(monkeypatch):
+ _patch_client(monkeypatch, _FakeResp(202, "throttle challenge, no results"))
+ with pytest.raises(DDGRateLimited):
+ await WebSearchTool._search_ddg("anything", 5)
+
+
+@pytest.mark.asyncio
+async def test_execute_reports_rate_limit_clearly(monkeypatch):
+ _patch_client(monkeypatch, _FakeResp(202, "throttle"))
+ parts = await WebSearchTool().execute({"query": "x", "num_results": 5}, None)
+ msg = parts[0]["text"].lower()
+ assert "rate-limit" in msg
+ assert "no search results" not in msg # the old bogus message must be gone
+
+
+@pytest.mark.asyncio
+async def test_ads_are_stripped_real_results_kept(monkeypatch):
+ _patch_client(monkeypatch, _FakeResp(200, _HTML_WITH_AD))
+ out = await WebSearchTool._search_ddg("topic", 5)
+ assert "example.com/real" in out
+ assert "Real Result Title" in out
+ # the sponsored row and its tracker URL must not appear
+ assert "y.js" not in out
+ assert "advertiser.com" not in out
+ assert "Sponsored Junk" not in out
+
+
+@pytest.mark.asyncio
+async def test_genuinely_empty_is_not_a_rate_limit(monkeypatch):
+ # 200 with no result blocks is a real empty result set, not a throttle.
+ _patch_client(monkeypatch, _FakeResp(200, "nothing here"))
+ out = await WebSearchTool._search_ddg("zxcvqwer no hits", 5)
+ assert out == ""
diff --git a/backend/tests/test_web_search_reliability.py b/backend/tests/test_web_search_reliability.py
index 08e4b698..25585c51 100644
--- a/backend/tests/test_web_search_reliability.py
+++ b/backend/tests/test_web_search_reliability.py
@@ -1,39 +1,34 @@
-"""WebSearch path reliability: subscription OAuth must fall back to DuckDuckGo.
+"""Built-in WebSearch reliability: only an ENTITLED Anthropic endpoint suppresses DDG.
-The 401 'Invalid bearer token (reset after 2m)' the user hit comes from the CLI's
-built-in WebSearch delegating to Haiku via a subscription's rotating OAuth token.
-So a subscription-OAuth-only user must NOT be treated as having a reliable hosted
-search path, they keep the free, always-working DDG fallback instead.
+The 401 'Invalid bearer token (reset after ~2m)' comes from the CLI's built-in
+WebSearch firing an aux `claude-haiku` call. That call only authenticates when it
+reaches an entitled Anthropic endpoint:
+ - a DIRECT anthropic api-route model (base_url = api.anthropic.com, user's key), or
+ - OpenSwarm Pro (entitled to the managed pool 9Router's anthropic/* resolves to).
+
+A SUBSCRIPTION-route Claude model (opus-4-8, route=None) sends the haiku call
+through 9Router to the managed pool, which 401s for non-Pro users, so a bare
+anthropic_api_key in settings is NOT enough. Those sessions must keep the free,
+always-working DDG fallback instead of a 401-ing hosted path.
"""
from backend.apps.agents.tools.web import anthropic_web_search_is_reliable as ok
-def test_direct_api_key_is_reliable():
- assert ok(has_direct_anthropic_key=True, is_pro=False, provider_ids=[]) is True
+def test_direct_anthropic_api_route_is_reliable():
+ # opus-4-8-api (route='api', api='anthropic') + key in settings -> direct, works
+ assert ok(uses_direct_anthropic_api=True, is_pro=False) is True
-def test_pro_is_reliable():
- assert ok(has_direct_anthropic_key=False, is_pro=True, provider_ids=[]) is True
+def test_openswarm_pro_is_reliable():
+ assert ok(uses_direct_anthropic_api=False, is_pro=True) is True
-def test_direct_anthropic_9router_provider_is_reliable():
- assert ok(has_direct_anthropic_key=False, is_pro=False, provider_ids=["anthropic"]) is True
+def test_both_is_reliable():
+ assert ok(uses_direct_anthropic_api=True, is_pro=True) is True
-def test_subscription_oauth_only_is_NOT_reliable():
- # the bug: this used to count as reliable -> suppressed DDG -> 401s on rotation
- assert ok(has_direct_anthropic_key=False, is_pro=False, provider_ids=["claude"]) is False
- assert ok(has_direct_anthropic_key=False, is_pro=False, provider_ids=["claude-code"]) is False
- assert ok(has_direct_anthropic_key=False, is_pro=False, provider_ids=["claude", "claude-code"]) is False
-
-
-def test_nothing_is_not_reliable():
- assert ok(has_direct_anthropic_key=False, is_pro=False, provider_ids=[]) is False
- assert ok(has_direct_anthropic_key=False, is_pro=False, provider_ids=None) is False
-
-
-def test_mixed_subscription_plus_direct_is_reliable():
- # if the user ALSO has a stable direct anthropic connection, hosted search is fine
- assert ok(has_direct_anthropic_key=False, is_pro=False,
- provider_ids=["claude", "anthropic"]) is True
+def test_subscription_route_claude_is_NOT_reliable():
+ # The exact bug: opus-4-8 (subscription route) + key in settings. The haiku
+ # call still 401s via the managed pool, so this must stay unreliable -> DDG.
+ assert ok(uses_direct_anthropic_api=False, is_pro=False) is False