[eric] web: add Startpage as a second independent search engine so one bot challenge cannot close free search

2026-08-17 18:25:42 +02:00 · 2026-07-30 14:34:29 -07:00
parent 356f7a4ea1
commit d84f2d5301
12 changed files with 409 additions and 11 deletions
@@ -0,0 +1,118 @@
+"""DuckDuckGo web search: html endpoint primary, lite endpoint fallback.
+
+The html endpoint is the richer parse; lite (see search_ddg_lite) covers the two
+ways html dies: the 202 bot challenge and silent markup drift. Only both
+endpoints challenging raises DDGRateLimited, so free search no longer has a
+single point of failure (the outage class that stranded subscription-only users
+on "No search backend is configured").
+
+Both rungs go out through `browser_http`, whose Chrome TLS fingerprint is what
+actually decides whether DuckDuckGo answers; a plain httpx client scored 4/8 on
+the same queries this one scored 8/8 on."""
+
+import html
+import re
+
+from backend.apps.agents.tools.browser_http import CHROME_UA
+from backend.apps.agents.tools.browser_http import browser_request
+from backend.apps.agents.tools.search.search_ddg_lite import search_ddg_lite
+
+HTTP_TIMEOUT = 30
+USER_AGENT = CHROME_UA
+
+
+class DDGRateLimited(Exception):
+    """Every DuckDuckGo frontend answered with the bot challenge (HTTP 202).
+
+    Named for history; this is an anti-automation challenge keyed on the
+    client's fingerprint, NOT a per-IP rate limit. Distinct from 'genuinely
+    zero hits' so the caller can fail over to another backend instead of
+    reporting an empty search to the user."""
+
+
+def strip_html(raw_html: str) -> str:
+    """Naive but effective HTML to plain-text conversion."""
+    text = re.sub(r"<(script|style)[^>]*>.*?</\1>", "", raw_html, flags=re.DOTALL | re.IGNORECASE)
+    text = re.sub(r"<[^>]+>", " ", text)
+    text = html.unescape(text)
+    text = re.sub(r"[ \t]+", " ", text)
+    text = re.sub(r"\n{3,}", "\n\n", text)
+    return text.strip()
+
+
+async def search_ddg(query: str, num_results: int) -> str:
+    """Query DuckDuckGo's html endpoint and parse results; lite is the free fallback."""
+    reply = await browser_request(
+        "https://html.duckduckgo.com/html/", params={"q": query}, timeout=HTTP_TIMEOUT,
+    )
+    # DDG serves its bot challenge as 202 (a ~14KB no-results page), which is a 2xx so a status check sails right past it. Before giving up, try the lite frontend; only when BOTH challenge is free DDG actually dead.
+    if reply.status == 202:
+        lite = await search_ddg_lite(query, num_results)
+        if lite is None:
+            raise DDGRateLimited(query)
+        return lite
+    if reply.status >= 400:
+        raise RuntimeError(f"DuckDuckGo html returned HTTP {reply.status}")
+
+    body = reply.text
+
+    result_blocks = re.findall(
+        r'<div[^>]*class="[^"]*result[^"]*"[^>]*>(.*?)</div>\s*(?=<div[^>]*class="[^"]*result|$)',
+        body,
+        flags=re.DOTALL,
+    )
+
+    entries: list[str] = []
+    for block in result_blocks:
+        if len(entries) >= num_results:
+            break
+
+        # Handle both class-before-href and href-before-class attribute orders.
+        link_match = re.search(
+            r'<a[^>]*class="[^"]*result__a[^"]*"[^>]*href="([^"]*)"[^>]*>(.*?)</a>',
+            block,
+            flags=re.DOTALL,
+        )
+        if not link_match:
+            link_match = re.search(
+                r'<a[^>]*href="([^"]*)"[^>]*class="[^"]*result__a[^"]*"[^>]*>(.*?)</a>',
+                block,
+                flags=re.DOTALL,
+            )
+        if not link_match:
+            continue
+
+        raw_url = html.unescape(link_match.group(1))
+
+        # Drop sponsored rows: DDG ads point at its own y.js click-tracker (ad_domain/ad_provider) instead of a real uddg= redirect, so they'd otherwise show up as junk "duckduckgo.com/y.js?ad_..." results.
+        if "/y.js?" in raw_url or "ad_provider=" in raw_url or "ad_domain=" in raw_url:
+            continue
+
+        title = strip_html(link_match.group(2)).strip()
+
+        snippet_match = re.search(
+            r'<a[^>]*class="[^"]*result__snippet[^"]*"[^>]*>(.*?)</a>',
+            block,
+            flags=re.DOTALL,
+        )
+        snippet = strip_html(snippet_match.group(1)).strip() if snippet_match else ""
+
+        # DDG wraps URLs in a redirect; extract the real one.
+        real_url_match = re.search(r"uddg=([^&]+)", raw_url)
+        if real_url_match:
+            from urllib.parse import unquote
+            url = unquote(real_url_match.group(1))
+        else:
+            url = raw_url
+
+        entry = f"[{len(entries) + 1}] {title}\n    {url}"
+        if snippet:
+            entry += f"\n    {snippet}"
+        entries.append(entry)
+
+    # 200 with zero parsed entries usually means DDG changed its markup out from under the regexes (it has before), not a genuine no-hits; lite's simpler shape is the safety net.
+    if not entries:
+        lite = await search_ddg_lite(query, num_results)
+        if lite:
+            return lite
+    return "\n\n".join(entries)
@@ -0,0 +1,60 @@
+"""DuckDuckGo lite-endpoint search: the fallback when html.duckduckgo.com
+serves its bot challenge (HTTP 202) or its markup drifts. lite.duckduckgo.com
+is a separate frontend with simpler, stabler HTML and direct result URLs (no
+uddg redirect).
+
+Returns None on a challenge (caller decides whether that means every DDG
+frontend is closed) and a formatted results string (possibly empty) on
+success."""
+
+import html
+import re
+from typing import List, Optional
+
+from typeguard import typechecked
+
+from backend.apps.agents.tools.browser_http import browser_request
+
+P_LITE_URL = "https://lite.duckduckgo.com/lite/"
+P_TIMEOUT = 12.0
+P_TAG_RE = re.compile(r"<[^>]+>")
+# Lite uses single-quoted class attrs today; accept either quote style so a cosmetic flip doesn't kill the parser.
+P_LINK_RE = re.compile(
+    r"""<a[^>]*href="([^"]+)"[^>]*class=['"]result-link['"][^>]*>(.*?)</a>""",
+    flags=re.DOTALL,
+)
+P_SNIPPET_RE = re.compile(
+    r"""<td[^>]*class=['"]result-snippet['"][^>]*>(.*?)</td>""",
+    flags=re.DOTALL,
+)
+
+
+@typechecked
+def p_strip(text: str) -> str:
+    return html.unescape(P_TAG_RE.sub("", text)).strip()
+
+
+@typechecked
+def parse_lite_results(body: str, num_results: int) -> str:
+    """Format lite's result rows; links and snippets appear in document order and pair up positionally."""
+    links = P_LINK_RE.findall(body)
+    snippets = [p_strip(s) for s in P_SNIPPET_RE.findall(body)]
+    entries: List[str] = []
+    for i, (url, raw_title) in enumerate(links[:num_results]):
+        title = p_strip(raw_title)
+        entry = f"[{i + 1}] {title}\n    {html.unescape(url)}"
+        if i < len(snippets) and snippets[i]:
+            entry += f"\n    {snippets[i]}"
+        entries.append(entry)
+    return "\n\n".join(entries)
+
+
+@typechecked
+async def search_ddg_lite(query: str, num_results: int) -> Optional[str]:
+    """None = bot challenge (202), string = parsed results (may be empty on no hits)."""
+    reply = await browser_request(P_LITE_URL, params={"q": query}, timeout=P_TIMEOUT)
+    if reply.status == 202:
+        return None
+    if reply.status >= 400:
+        raise RuntimeError(f"DuckDuckGo lite returned HTTP {reply.status}")
+    return parse_lite_results(reply.text, num_results)
@@ -0,0 +1,80 @@
+"""Startpage search: the second independent engine behind DuckDuckGo.
+
+DuckDuckGo is one operator, so its bot challenge is one point of failure for
+every keyless user. Startpage serves Google's index and answered 8/8 on the
+same machine and rounds where DuckDuckGo's shipped client shape answered 4/8,
+so it is a genuine second opinion rather than a retry.
+
+It must be a POST: a GET to /sp/search is answered with an Anubis
+proof-of-work interstitial (measured, ~10KB and zero results), while the POST
+returns the real result page. Parsing is anchored on `result-link` /
+`gl-title-link` and the `description` paragraph, never on the emotion CSS
+hashes in the same class attributes, which change build to build.
+
+Returns None when Startpage served a challenge instead of results, so the
+caller can tell "closed" apart from "genuinely no hits"."""
+
+import html
+import re
+from typing import List, Optional
+
+from typeguard import typechecked
+
+from backend.apps.agents.tools.browser_http import browser_request
+
+P_SEARCH_URL = "https://www.startpage.com/sp/search"
+P_TIMEOUT = 12.0
+
+P_ANCHOR_RE = re.compile(
+    r"<a\b([^>]*(?:result-link|gl-title-link)[^>]*)>(.*?)</a>", flags=re.DOTALL,
+)
+P_HREF_RE = re.compile(r'href="([^"]+)"')
+P_TITLE_RE = re.compile(r"<h2[^>]*>(.*?)</h2>", flags=re.DOTALL)
+P_DESC_RE = re.compile(
+    r'<p[^>]*class="[^"]*\bdescription\b[^"]*"[^>]*>(.*?)</p>', flags=re.DOTALL,
+)
+# Startpage inlines a <style> block inside each result anchor, so tag-stripping alone would emit CSS as the title.
+P_NOISE_RE = re.compile(r"<(script|style)\b[^>]*>.*?</\1>", flags=re.DOTALL | re.IGNORECASE)
+P_TAG_RE = re.compile(r"<[^>]+>")
+
+
+@typechecked
+def p_strip(raw: str) -> str:
+    return re.sub(r"\s+", " ", html.unescape(P_TAG_RE.sub("", P_NOISE_RE.sub("", raw)))).strip()
+
+
+@typechecked
+def parse_startpage_results(body: str, num_results: int) -> str:
+    """Format Startpage's result rows; a snippet is only paired when it sits INSIDE its own result block."""
+    anchors = list(P_ANCHOR_RE.finditer(body))
+    entries: List[str] = []
+    for i, match in enumerate(anchors[:num_results]):
+        href = P_HREF_RE.search(match.group(1))
+        if not href:
+            continue
+        title_match = P_TITLE_RE.search(match.group(2))
+        title = p_strip(title_match.group(1)) if title_match else p_strip(match.group(2))
+        if not title:
+            continue
+        entry = f"[{len(entries) + 1}] {title}\n    {html.unescape(href.group(1))}"
+        block_end = anchors[i + 1].start() if i + 1 < len(anchors) else len(body)
+        desc = P_DESC_RE.search(body, match.end(), block_end)
+        if desc:
+            snippet = p_strip(desc.group(1))
+            if snippet:
+                entry += f"\n    {snippet}"
+        entries.append(entry)
+    return "\n\n".join(entries)
+
+
+@typechecked
+async def search_startpage(query: str, num_results: int) -> Optional[str]:
+    """None = Startpage refused (challenge or error), string = parsed results."""
+    reply = await browser_request(
+        P_SEARCH_URL, method="POST", params={"query": query, "cat": "web"}, timeout=P_TIMEOUT,
+    )
+    if reply.status != 200:
+        return None
+    results = parse_startpage_results(reply.text, num_results)
+    # The challenge page is a normal 200 that simply carries no results, so "parsed nothing" IS the refusal signal.
+    return results or None
@@ -9,13 +9,13 @@ import httpx
 from typeguard import typechecked

 from backend.apps.agents.tools.base import BaseTool, ToolContext
-from backend.apps.agents.tools.search_ddg import (
+from backend.apps.agents.tools.search.search_ddg import (
    DDGRateLimited,
    HTTP_TIMEOUT,
    USER_AGENT,
    strip_html,
 )
-from backend.apps.agents.tools.search_ddg import search_ddg as run_ddg_search
+from backend.apps.agents.tools.search.search_ddg import search_ddg as run_ddg_search
 from backend.apps.agents.tools.ssrf_guard import SSRFBlocked, safe_fetch

 P_MAX_OUTPUT_BYTES = 250 * 1024  # ~250 KB covers ~95% of articles/wikis/docs.
@@ -144,6 +144,14 @@ async def search(body: SearchBody) -> Dict:
            return None
        return {"query": body.query, "results": text, "backend": "ddg"}

+    async def try_startpage() -> Optional[Dict]:
+        # Second independent engine (Google's index), so DuckDuckGo's bot challenge is no longer a single point of failure for keyless users.
+        from backend.apps.agents.tools.search.search_startpage import search_startpage
+        text = await search_startpage(body.query, body.num_results)
+        if not text:
+            return None
+        return {"query": body.query, "results": text, "backend": "startpage"}
+
    async def try_browser_search() -> Optional[Dict]:
        # Packaged-app tier: a real Chromium's fingerprint isn't subject to the headless-client challenge, and it can scrape Google/Bing directly. Skipped (None) when no Electron main bridge is connected.
        res = await p_browser_bridge("browser_search", {"query": body.query, "num_results": body.num_results})
@@ -188,6 +196,7 @@ async def search(body: SearchBody) -> Dict:

    tiers = [
        CascadeTier(name="ddg", run=try_keyless, budget=KEYLESS_TIER_SECONDS),
+        CascadeTier(name="startpage", run=try_startpage, budget=KEYLESS_TIER_SECONDS),
        CascadeTier(name="browser_search", run=try_browser_search, budget=BROWSER_TIER_SECONDS),
    ] + p_grounded_tiers("search", body.primary, {
        "gemini_native": try_gemini,
@@ -9,8 +9,8 @@ queries. These pin the seam and the degrade path.
 import pytest

 import backend.apps.agents.tools.browser_http as BH
-import backend.apps.agents.tools.search_ddg as SD
-import backend.apps.agents.tools.search_ddg_lite as SDL
+import backend.apps.agents.tools.search.search_ddg as SD
+import backend.apps.agents.tools.search.search_ddg_lite as SDL
 from backend.apps.agents.tools.browser_http import BROWSER_HEADERS, HttpReply, browser_request


@@ -125,7 +125,7 @@ def test_search_tier_budgets_leave_room_for_the_grounded_tier():

    # A grounded native call needs ~30-42s, so the free rungs must leave it a usable slice.
    grounded_floor = 20.0
-    search_cheap = W.KEYLESS_TIER_SECONDS + W.BROWSER_TIER_SECONDS
+    search_cheap = 2 * W.KEYLESS_TIER_SECONDS + W.BROWSER_TIER_SECONDS
    assert W.SEARCH_BUDGET_SECONDS - search_cheap >= grounded_floor
    fetch_cheap = W.LOCAL_FETCH_TIER_SECONDS + W.BROWSER_TIER_SECONDS
    assert W.FETCH_BUDGET_SECONDS - fetch_cheap >= grounded_floor
@@ -15,6 +15,7 @@ import time

 import pytest

+import backend.apps.agents.tools.search.search_startpage as SP
 import backend.apps.web.web as W
 from backend.apps.web.web import search, SearchBody
 from backend.apps.agents.tools.web import WebSearchTool, DDGRateLimited
@@ -36,6 +37,10 @@ def p_no_network(monkeypatch):
    monkeypatch.setattr(W, "gemini_grounded_via_9router", p_empty)
    monkeypatch.setattr(W, "openai_websearch_via_9router", p_empty)

+    async def p_startpage_closed(query, num):
+        return None
+    monkeypatch.setattr(SP, "search_startpage", p_startpage_closed)
+

 def p_ddg_returns(monkeypatch, text):
    async def p_f(query, num):
@@ -81,6 +86,28 @@ async def test_ddg_throttled_falls_over_to_openai(monkeypatch):
    assert any("ddg" in e for e in res.get("cascade_errors", []))


+def p_startpage_returns(monkeypatch, text):
+    async def p_f(query, num):
+        return text
+    monkeypatch.setattr(SP, "search_startpage", p_f)
+
+
+@pytest.mark.asyncio
+async def test_startpage_rescues_a_ddg_challenge(monkeypatch):
+    """Two independent engines: one operator's bot challenge must not close free search."""
+    p_ddg_throttled(monkeypatch)
+    p_startpage_returns(monkeypatch, "[1] Rescued\n    https://sp.example")
+
+    async def p_boom(*a, **k):
+        raise AssertionError("a paid backend must not run while a free engine still answers")
+    monkeypatch.setattr(W, "gemini_grounded_call", p_boom)
+
+    res = await search(SearchBody(query="x"))
+    assert res["backend"] == "startpage"
+    assert "sp.example" in res["results"]
+    assert any("ddg" in e for e in res["cascade_errors"])
+
+
@pytest.mark.asyncio
 async def test_a_hung_grounded_attempt_is_bounded(monkeypatch):
    p_ddg_throttled(monkeypatch)
@@ -11,8 +11,8 @@ We mock the network so the test is deterministic and offline.

 import pytest

-import backend.apps.agents.tools.search_ddg as SD
-import backend.apps.agents.tools.search_ddg_lite as SDL
+import backend.apps.agents.tools.search.search_ddg as SD
+import backend.apps.agents.tools.search.search_ddg_lite as SDL
 from backend.apps.agents.tools.browser_http import HttpReply
 from backend.apps.agents.tools.web import WebSearchTool, DDGRateLimited

@@ -14,11 +14,11 @@ import asyncio

 import pytest

-import backend.apps.agents.tools.search_ddg as SD
-import backend.apps.agents.tools.search_ddg_lite as SDL
+import backend.apps.agents.tools.search.search_ddg as SD
+import backend.apps.agents.tools.search.search_ddg_lite as SDL
 from backend.apps.agents.tools.browser_http import HttpReply
 from backend.apps.agents.tools.web import WebSearchTool, DDGRateLimited
-from backend.apps.agents.tools.search_ddg_lite import parse_lite_results
+from backend.apps.agents.tools.search.search_ddg_lite import parse_lite_results

 P_LITE_BODY = """
 <table>
@@ -0,0 +1,103 @@
+"""Startpage rung: the second independent engine behind DuckDuckGo.
+
+The fixture is the real markup shape captured live 2026-07-30: an inline
+<style> block INSIDE each result anchor, the title in an <h2>, volatile
+emotion CSS hashes in every class attribute, and a stray description
+paragraph that does not belong to any result block.
+"""
+
+import pytest
+
+import backend.apps.agents.tools.search.search_startpage as SP
+from backend.apps.agents.tools.browser_http import HttpReply
+from backend.apps.agents.tools.search.search_startpage import (
+    parse_startpage_results,
+    search_startpage,
+)
+
+P_BODY = """
+<div class="w-gl__result">
+<style data-emotion="css 1bggj8v">.css-1bggj8v{color:#2E39B3;}</style>
+<a class="result-title result-link css-1bggj8v" href="https://docs.python.org/3/library/asyncio.html"
+   rel="noopener nofollow" data-testid="gl-title-link">
+  <style data-emotion="css i3irj7">.css-i3irj7{font-size:18px;}</style>
+  <h2 class="wgl-title css-i3irj7">asyncio &mdash; Asynchronous I/O</h2>
+</a>
+<p class="description css-1507v2l">The <b>asyncio</b> library, explained.</p>
+</div>
+<div class="w-gl__result">
+<a class="result-title result-link css-zzz" href="https://realpython.com/async-io-python/"
+   data-testid="gl-title-link"><h2 class="wgl-title css-qqq">Real Python walkthrough</h2></a>
+<p class="description css-x">A hands-on tour.</p>
+</div>
+<p class="description css-orphan">Unrelated footer blurb, belongs to no result.</p>
+"""
+
+P_CHALLENGE_BODY = """
+<html><head><script id="anubis_challenge" type="application/json">{"difficulty":2}</script></head>
+<body><div class="sp-wrap">Making sure you are not a bot...</div></body></html>
+"""
+
+
+def p_reply(status: int, text: str) -> HttpReply:
+    return HttpReply(status=status, text=text, content=text.encode(),
+                     content_type="text/html", url="https://www.startpage.com/sp/search")
+
+
+def p_answer(monkeypatch, reply: HttpReply, seen=None):
+    async def p_req(url, **kw):
+        if seen is not None:
+            seen.append((url, kw.get("method"), kw.get("params")))
+        return reply
+    monkeypatch.setattr(SP, "browser_request", p_req)
+
+
+def test_parses_title_url_and_the_snippet_from_its_own_block():
+    out = parse_startpage_results(P_BODY, 5)
+    assert "[1] asyncio — Asynchronous I/O" in out
+    assert "https://docs.python.org/3/library/asyncio.html" in out
+    assert "The asyncio library, explained." in out
+    assert "[2] Real Python walkthrough" in out
+    assert "A hands-on tour." in out
+
+
+def test_inline_style_never_becomes_the_title():
+    """Tag-stripping alone would emit the anchor's own CSS as the result title."""
+    out = parse_startpage_results(P_BODY, 5)
+    assert "css-" not in out
+    assert "font-size" not in out
+
+
+def test_an_orphan_snippet_is_not_attached_to_a_result():
+    out = parse_startpage_results(P_BODY, 5)
+    assert "Unrelated footer blurb" not in out
+
+
+def test_respects_num_results():
+    out = parse_startpage_results(P_BODY, 1)
+    assert "[1]" in out and "[2]" not in out
+
+
+@pytest.mark.asyncio
+async def test_challenge_page_reads_as_refused_not_as_zero_hits(monkeypatch):
+    # Startpage answers its proof-of-work interstitial with a normal 200, so "parsed nothing" is the only honest signal.
+    p_answer(monkeypatch, p_reply(200, P_CHALLENGE_BODY))
+    assert await search_startpage("q", 5) is None
+
+
+@pytest.mark.asyncio
+async def test_http_error_reads_as_refused(monkeypatch):
+    p_answer(monkeypatch, p_reply(503, "nope"))
+    assert await search_startpage("q", 5) is None
+
+
+@pytest.mark.asyncio
+async def test_must_post_or_startpage_serves_the_proof_of_work_wall(monkeypatch):
+    seen = []
+    p_answer(monkeypatch, p_reply(200, P_BODY), seen)
+    out = await search_startpage("some query", 5)
+    assert out is not None
+    url, method, params = seen[0]
+    assert url == "https://www.startpage.com/sp/search"
+    assert method == "POST"
+    assert params == {"query": "some query", "cat": "web"}
@@ -123,7 +123,8 @@ async function hiddenEvalWithCookies(url, cookieRecords, js) {
 // Google first (direct result URLs, best quality); DuckDuckGo in a real browser
 // second (immune to the httpx 202 throttle); Bing last (results are redirect-wrapped).
 const ENGINES = [
-  { name: 'google', url: (q) => `https://www.google.com/search?q=${encodeURIComponent(q)}&num=10&hl=en`,
+  // udm=14 is Google's plain "Web" tab: ten blue links, no AI Overview and no answer widgets, so the a-h3 scrape gets real result URLs instead of whatever the SERP decided to render today.
+  { name: 'google', url: (q) => `https://www.google.com/search?q=${encodeURIComponent(q)}&udm=14&num=10&hl=en`,
    scrape: `Array.from(document.querySelectorAll('a h3')).map(function(h){var a=h.closest('a');return a&&a.href?{t:h.innerText,u:a.href}:null;}).filter(function(x){return x&&x.u.indexOf('http')===0&&x.u.indexOf('google.')===-1;})` },
  { name: 'ddg', url: (q) => `https://html.duckduckgo.com/html/?q=${encodeURIComponent(q)}`,
    scrape: `Array.from(document.querySelectorAll('a.result__a')).map(function(a){var m=a.href.match(/uddg=([^&]+)/);return {t:a.innerText,u:m?decodeURIComponent(m[1]):a.href};})` },