From 784b33c15a26a4b6dd4c0565e2d40bbb8f792f72 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Mon, 10 Aug 2026 14:02:42 -0700 Subject: [PATCH] [eric] web: bing + brave join the keyless race behind ddg; bing went 50/50 on the burst that tripped both ddg and brave --- .../apps/agents/tools/search/engine_answer.py | 16 +++ .../apps/agents/tools/search/search_bing.py | 88 +++++++++++++++++ .../apps/agents/tools/search/search_brave.py | 78 +++++++++++++++ .../agents/tools/search/search_startpage.py | 43 +++----- .../apps/agents/tools/search/strip_tags.py | 15 +++ backend/apps/web/web.py | 24 ++++- backend/tests/test_web_search_bing.py | 97 +++++++++++++++++++ backend/tests/test_web_search_brave.py | 85 ++++++++++++++++ backend/tests/test_web_search_cascade.py | 31 ++++++ backend/tests/web_cascade_fixtures.py | 40 ++++++-- 10 files changed, 480 insertions(+), 37 deletions(-) create mode 100644 backend/apps/agents/tools/search/engine_answer.py create mode 100644 backend/apps/agents/tools/search/search_bing.py create mode 100644 backend/apps/agents/tools/search/search_brave.py create mode 100644 backend/apps/agents/tools/search/strip_tags.py create mode 100644 backend/tests/test_web_search_bing.py create mode 100644 backend/tests/test_web_search_brave.py diff --git a/backend/apps/agents/tools/search/engine_answer.py b/backend/apps/agents/tools/search/engine_answer.py new file mode 100644 index 00000000..54afb766 --- /dev/null +++ b/backend/apps/agents/tools/search/engine_answer.py @@ -0,0 +1,16 @@ +"""What a keyless search engine actually said: results, an honest nothing, or a refusal. + +"Closed" and "genuinely no hits" look identical on the wire (both are a 200 +with no result anchors) and the caller has to act on them differently: a +refusal is a failed tier that should count against the engine's breaker, an +empty result set is the honest answer to a nonsense query.""" + +from pydantic import BaseModel, ConfigDict + + +class EngineAnswer(BaseModel): + model_config = ConfigDict(validate_assignment=True) + + results: str = "" + # Challenge, HTTP error, or markup we no longer recognise: the engine did not answer the question. + refused: bool = False diff --git a/backend/apps/agents/tools/search/search_bing.py b/backend/apps/agents/tools/search/search_bing.py new file mode 100644 index 00000000..c0999060 --- /dev/null +++ b/backend/apps/agents/tools/search/search_bing.py @@ -0,0 +1,88 @@ +"""Bing search: the third keyless engine, straight from Microsoft's own index. + +DuckDuckGo largely serves Bing's index through DuckDuckGo's frontend, so when +DDG's anti-bot wall is up the index itself is usually still reachable here. +Probed live 2026-08-10 through the Chrome-impersonating client: 200 with +server-rendered results on 3/3 queries at 0.2-0.3s, the fastest of every +engine probed. + +Organic results are `
  • ` blocks (ads live in `b_ad`, so this +match skips them by construction): title inside an `

    `, snippet in the +`b_caption` paragraph. The href is a click-tracking redirect whose `u=a1` +parameter carries the real URL base64url-encoded; a result pointing at +bing.com/ck/a would be junk to a model, so decoding it is load-bearing. + +"There are no results for" is Bing's honest empty page; zero parsed blocks +without that marker means a challenge or markup drift and reads as a refusal.""" + +import base64 +import html +import re +from typing import List + +from typeguard import typechecked + +from backend.apps.agents.tools.browser_http import browser_request +from backend.apps.agents.tools.search.engine_answer import EngineAnswer +from backend.apps.agents.tools.search.strip_tags import strip_tags + +P_SEARCH_URL = "https://www.bing.com/search" +P_TIMEOUT = 12.0 + +P_BLOCK_RE = re.compile(r'
  • ]*>(.*?)

    • ", flags=re.DOTALL) +P_HREF_RE = re.compile(r']*href="([^"]+)"') +P_SNIPPET_RE = re.compile(r'class="b_caption[^"]*".*?]*>(.*?)

    ', flags=re.DOTALL) +P_REDIRECT_RE = re.compile(r"[?&]u=a1([A-Za-z0-9_\-]+)") +P_NO_RESULTS_MARKER = "There are no results for" + + +@typechecked +def p_real_url(raw: str) -> str: + raw = html.unescape(raw) + m = P_REDIRECT_RE.search(raw) + if not m: + return raw + token = m.group(1) + try: + return base64.urlsafe_b64decode(token + "=" * (-len(token) % 4)).decode("utf-8", "replace") + except Exception: + return raw + + +@typechecked +def parse_bing_results(body: str, num_results: int) -> str: + """Format Bing's b_algo rows with their redirect hrefs decoded to real URLs.""" + entries: List[str] = [] + for block in P_BLOCK_RE.findall(body): + if len(entries) >= num_results: + break + h2 = P_H2_RE.search(block) + if not h2: + continue + href = P_HREF_RE.search(h2.group(1)) + title = strip_tags(h2.group(1)) + if not href or not title: + continue + entry = f"[{len(entries) + 1}] {title}\n {p_real_url(href.group(1))}" + snippet_match = P_SNIPPET_RE.search(block) + if snippet_match: + snippet = strip_tags(snippet_match.group(1)) + if snippet: + entry += f"\n {snippet}" + entries.append(entry) + return "\n\n".join(entries) + + +@typechecked +async def search_bing(query: str, num_results: int) -> EngineAnswer: + """Bing's answer: results, an honest nothing, or a refusal.""" + reply = await browser_request(P_SEARCH_URL, params={"q": query}, timeout=P_TIMEOUT) + if reply.status != 200: + return EngineAnswer(refused=True) + results = parse_bing_results(reply.text, num_results) + if results: + return EngineAnswer(results=results) + if P_NO_RESULTS_MARKER in reply.text: + return EngineAnswer() + return EngineAnswer(refused=True) diff --git a/backend/apps/agents/tools/search/search_brave.py b/backend/apps/agents/tools/search/search_brave.py new file mode 100644 index 00000000..1b4a47be --- /dev/null +++ b/backend/apps/agents/tools/search/search_brave.py @@ -0,0 +1,78 @@ +"""Brave search: a keyless engine with its OWN index behind DuckDuckGo. + +Brave runs its own crawler, so it fails independently of the Bing-fed engines +(DuckDuckGo, Bing itself) and of Google (Startpage). Probed live 2026-08-10 +through the Chrome-impersonating client: 200 with server-rendered results on +3/3 queries at 0.7-0.9s, while Mojeek, Ecosia and Yep 403'd the same client +on the same machine and Qwant served a JS shell. + +The SERP is server-rendered Svelte: each organic result is a +`
    ` block whose first anchor carries +the REAL target URL (no redirect wrapper), whose title div carries the clean +text in its `title` attribute, and whose description sits in a +`
    `. Ads carry a different data-type, so matching +`data-type="web"` skips them by construction. + +A gibberish query still returns fuzzy matches (measured: 19 blocks plus a +"Not many great matches" banner), so zero parsed blocks on a 200 is markup +drift or a challenge page, never an honest no-hits; both read as a refusal. + +Hammered with 50 back-to-back queries it answered the first 11 then throttled, +and recovered within about a minute, so it belongs BEHIND an unthrottled rung +in the race (Bing went 50/50 on the same burst) where it only sees the +occasional rescue query, not the firehose.""" + +import html +import re +from typing import List + +from typeguard import typechecked + +from backend.apps.agents.tools.browser_http import browser_request +from backend.apps.agents.tools.search.engine_answer import EngineAnswer +from backend.apps.agents.tools.search.strip_tags import strip_tags + +P_SEARCH_URL = "https://search.brave.com/search" +P_TIMEOUT = 12.0 + +P_BLOCK_SPLIT_RE = re.compile(r'data-type="web"') +P_HREF_RE = re.compile(r']*\btitle="([^"]*)"') +P_DESC_RE = re.compile(r'
    ]*>(.*?)
    ', flags=re.DOTALL) + + +@typechecked +def parse_brave_results(body: str, num_results: int) -> str: + """Format Brave's organic rows; each split chunk starts with one result's own markup.""" + chunks = P_BLOCK_SPLIT_RE.split(body)[1:] + entries: List[str] = [] + for chunk in chunks: + if len(entries) >= num_results: + break + href = P_HREF_RE.search(chunk) + title = P_TITLE_RE.search(chunk) + if not href or not title: + continue + title_text = html.unescape(title.group(1)).strip() + if not title_text: + continue + entry = f"[{len(entries) + 1}] {title_text}\n {html.unescape(href.group(1))}" + desc = P_DESC_RE.search(chunk) + if desc: + snippet = strip_tags(desc.group(1)) + if snippet: + entry += f"\n {snippet}" + entries.append(entry) + return "\n\n".join(entries) + + +@typechecked +async def search_brave(query: str, num_results: int) -> EngineAnswer: + """Brave's answer: results, or a refusal (it fuzzy-matches, so empty means blocked or drifted).""" + reply = await browser_request(P_SEARCH_URL, params={"q": query}, timeout=P_TIMEOUT) + if reply.status != 200: + return EngineAnswer(refused=True) + results = parse_brave_results(reply.text, num_results) + if results: + return EngineAnswer(results=results) + return EngineAnswer(refused=True) diff --git a/backend/apps/agents/tools/search/search_startpage.py b/backend/apps/agents/tools/search/search_startpage.py index d593f292..f79d3688 100644 --- a/backend/apps/agents/tools/search/search_startpage.py +++ b/backend/apps/agents/tools/search/search_startpage.py @@ -11,21 +11,20 @@ returns the real result page. Parsing is anchored on `result-link` / `gl-title-link` and the `description` paragraph, never on the emotion CSS hashes in the same class attributes, which change build to build. -Answers a `StartpageAnswer` rather than a string because "closed" and -"genuinely no hits" look identical on the wire (both are a 200 with no result -anchors) and the caller has to act on them differently: a refusal is a failed -tier that should count against the engine, an empty result set is the honest -answer to a nonsense query. Startpage names the second case itself, in an +Answers an `EngineAnswer` rather than a string because "closed" and +"genuinely no hits" look identical on the wire and the caller has to act on +them differently. Startpage names the honest-empty case itself, in an "Uh-oh, there are no results for this search" page.""" import html import re from typing import List -from pydantic import BaseModel, ConfigDict from typeguard import typechecked from backend.apps.agents.tools.browser_http import browser_request +from backend.apps.agents.tools.search.engine_answer import EngineAnswer +from backend.apps.agents.tools.search.strip_tags import strip_tags P_SEARCH_URL = "https://www.startpage.com/sp/search" P_TIMEOUT = 12.0 @@ -38,25 +37,9 @@ P_TITLE_RE = re.compile(r"]*>(.*?)", flags=re.DOTALL) P_DESC_RE = re.compile( r']*class="[^"]*\bdescription\b[^"]*"[^>]*>(.*?)

    ', flags=re.DOTALL, ) -# Startpage inlines a