[eric] web: let Startpage say 'nothing matched' and 'I refuse' differently so a closed engine stops costing its budget

backend/apps/agents/tools/search/search_startpage.py

@@ -11,13 +11,18 @@ returns the real result page. Parsing is anchored on `result-link` /
 `gl-title-link` and the `description` paragraph, never on the emotion CSS
 hashes in the same class attributes, which change build to build.
 
-Returns None when Startpage served a challenge instead of results, so the
-caller can tell "closed" apart from "genuinely no hits"."""
+Answers a `StartpageAnswer` rather than a string because "closed" and
+"genuinely no hits" look identical on the wire (both are a 200 with no result
+anchors) and the caller has to act on them differently: a refusal is a failed
+tier that should count against the engine, an empty result set is the honest
+answer to a nonsense query. Startpage names the second case itself, in an
+"Uh-oh, there are no results for this search" page."""
 
 import html
 import re
-from typing import List, Optional
+from typing import List
 
+from pydantic import BaseModel, ConfigDict
 from typeguard import typechecked
 
 from backend.apps.agents.tools.browser_http import browser_request
@@ -36,6 +41,15 @@ P_DESC_RE = re.compile(
 # Startpage inlines a <style> block inside each result anchor, so tag-stripping alone would emit CSS as the title.
 P_NOISE_RE = re.compile(r"<(script|style)\b[^>]*>.*?</\1>", flags=re.DOTALL | re.IGNORECASE)
 P_TAG_RE = re.compile(r"<[^>]+>")
+P_NO_RESULTS_MARKER = "there are no results for this search"
+
+
+class StartpageAnswer(BaseModel):
+    model_config = ConfigDict(validate_assignment=True)
+
+    results: str = ""
+    # Challenge, error, or markup we no longer recognise: Startpage did not answer the question.
+    refused: bool = False
 
 
 @typechecked
@@ -68,13 +82,17 @@ def parse_startpage_results(body: str, num_results: int) -> str:
 
 
 @typechecked
-async def search_startpage(query: str, num_results: int) -> Optional[str]:
-    """None = Startpage refused (challenge or error), string = parsed results."""
+async def search_startpage(query: str, num_results: int) -> StartpageAnswer:
+    """Startpage's answer: results, an honest nothing, or a refusal."""
     reply = await browser_request(
         P_SEARCH_URL, method="POST", params={"query": query, "cat": "web"}, timeout=P_TIMEOUT,
     )
     if reply.status != 200:
-        return None
+        return StartpageAnswer(refused=True)
     results = parse_startpage_results(reply.text, num_results)
-    # The challenge page is a normal 200 that simply carries no results, so "parsed nothing" IS the refusal signal.
-    return results or None
+    if results:
+        return StartpageAnswer(results=results)
+    # Measured once in 14 tight-loop requests: Startpage serves its own no-results page for a query that answered 10 results a second later, so an empty page is only trustworthy when it says so.
+    if P_NO_RESULTS_MARKER in reply.text:
+        return StartpageAnswer()
+    return StartpageAnswer(refused=True)

backend/apps/web/web.py

@@ -139,10 +139,13 @@ async def search(body: SearchBody) -> Dict:
     async def try_startpage() -> Optional[Dict]:
         # Second independent engine (Google's index), so DuckDuckGo's bot challenge is no longer a single point of failure for keyless users.
         from backend.apps.agents.tools.search.search_startpage import search_startpage
-        text = await search_startpage(body.query, body.num_results)
-        if not text:
+        answer = await search_startpage(body.query, body.num_results)
+        # Raise rather than return None: a refusal is the engine failing, and the breaker must count it so a closed Startpage stops costing its budget too.
+        if answer.refused:
+            raise RuntimeError("Startpage answered with a challenge instead of results")
+        if not answer.results:
             return None
-        return {"query": body.query, "results": text, "backend": "startpage"}
+        return {"query": body.query, "results": answer.results, "backend": "startpage"}
 
     async def try_browser_search() -> Optional[Dict]:
         # Packaged-app tier: a real Chromium's fingerprint isn't subject to the headless-client challenge, and it can scrape Google/Bing directly. Skipped (None) when no Electron main bridge is connected.

backend/tests/test_wayback_snapshot.py

@@ -69,11 +69,19 @@ def test_snapshot_date_parsing():
     assert snapshot_date("https://web.archive.org/nope") is None
 
 
-def test_raw_snapshot_form_is_requested(monkeypatch):
+@pytest.mark.asyncio
+async def test_raw_snapshot_form_is_requested(monkeypatch):
     """Without `id_` the archive injects its calendar toolbar, and on a Reddit snapshot
     that toolbar WAS the whole extracted text."""
-    from backend.apps.agents.tools.fetch.wayback import P_WAYBACK_LATEST
-    assert P_WAYBACK_LATEST.endswith("id_/")
+    seen = []
+
+    async def p_req(u, **kw):
+        seen.append(u)
+        return HttpReply(status=200, text=P_REAL_ARTICLE, content=P_REAL_ARTICLE.encode(),
+                         content_type="text/html", url=P_ARCHIVED_URL)
+    monkeypatch.setattr(WB, "browser_request", p_req)
+    await fetch_wayback("https://example.com/story")
+    assert seen == ["https://web.archive.org/web/2id_/https://example.com/story"]
 
 
 def test_snapshot_date_parsing_survives_the_raw_form():

backend/tests/test_web_search_cascade.py

@@ -18,10 +18,12 @@ import backend.apps.web.web as W
 from backend.apps.web.web import search, SearchBody
 from backend.tests.web_cascade_fixtures import (  # noqa: F401
     allow_urls,
+    fresh_breaker,
     patch_browser_bridge,
     ddg_returns,
     ddg_throttled,
     no_network,
+    startpage_refuses,
     startpage_returns,
 )
 
@@ -186,3 +188,28 @@ async def test_browser_search_skipped_when_no_bridge(monkeypatch):
 
     res = await search(SearchBody(query="q"))
     assert res["backend"] == "openai_native"
+
+
+@pytest.mark.asyncio
+async def test_a_refusing_startpage_counts_against_it(monkeypatch):
+    """A challenge and a genuinely empty web look identical on the wire, and treating a
+    refusal as 'no hits' meant a closed Startpage kept costing its full budget forever."""
+    from backend.apps.web.tier_breaker import FAILURES_TO_OPEN, tier_cooldown_left
+    ddg_throttled(monkeypatch)
+    startpage_refuses(monkeypatch)
+    for _ in range(FAILURES_TO_OPEN):
+        out = await search(SearchBody(query="anything", num_results=5))
+        assert out["backend"] == "none"
+    assert tier_cooldown_left("startpage") > 0
+    assert any("challenge" in e for e in out["cascade_errors"])
+
+
+@pytest.mark.asyncio
+async def test_an_honestly_empty_startpage_does_not_count_against_it(monkeypatch):
+    """Nonsense queries must not slowly cool down a perfectly healthy engine."""
+    from backend.apps.web.tier_breaker import FAILURES_TO_OPEN, tier_cooldown_left
+    ddg_throttled(monkeypatch)
+    for _ in range(FAILURES_TO_OPEN + 2):
+        out = await search(SearchBody(query="zxqvbnmklwertyuiopasdfg", num_results=5))
+        assert out["backend"] == "none"
+    assert tier_cooldown_left("startpage") == 0.0

backend/tests/test_web_search_startpage.py

@@ -82,22 +82,46 @@ def test_respects_num_results():
 async def test_challenge_page_reads_as_refused_not_as_zero_hits(monkeypatch):
     # Startpage answers its proof-of-work interstitial with a normal 200, so "parsed nothing" is the only honest signal.
     p_answer(monkeypatch, p_reply(200, P_CHALLENGE_BODY))
-    assert await search_startpage("q", 5) is None
+    answer = await search_startpage("q", 5)
+    assert answer.refused and not answer.results
 
 
 @pytest.mark.asyncio
 async def test_http_error_reads_as_refused(monkeypatch):
     p_answer(monkeypatch, p_reply(503, "nope"))
-    assert await search_startpage("q", 5) is None
+    assert (await search_startpage("q", 5)).refused
 
 
 @pytest.mark.asyncio
 async def test_must_post_or_startpage_serves_the_proof_of_work_wall(monkeypatch):
     seen = []
     p_answer(monkeypatch, p_reply(200, P_BODY), seen)
-    out = await search_startpage("some query", 5)
-    assert out is not None
+    out = (await search_startpage("some query", 5)).results
+    assert out
     url, method, params = seen[0]
     assert url == "https://www.startpage.com/sp/search"
     assert method == "POST"
     assert params == {"query": "some query", "cat": "web"}
+
+
+P_NO_RESULTS_BODY = """
+<html><body><div><img src="https://cdn.startpage.com/sp/cdn/images/dislike-face.svg"/></div>
+<h2>Uh-oh, there are no results for this search.</h2>
+<p>Let&#x27;s see, it could be due to:</p></body></html>
+"""
+
+
+@pytest.mark.asyncio
+async def test_startpages_own_no_results_page_is_an_answer_not_a_refusal(monkeypatch):
+    """Nonsense queries must read as 'nothing matched', not as an engine that shut us out."""
+    p_answer(monkeypatch, p_reply(200, P_NO_RESULTS_BODY))
+    answer = await search_startpage("zxqvbnmklwertyuiopasdfg", 5)
+    assert not answer.refused
+    assert answer.results == ""
+
+
+@pytest.mark.asyncio
+async def test_unrecognised_markup_is_a_refusal_not_zero_hits(monkeypatch):
+    """If Startpage redesigns, we must fail loudly rather than report the web as empty."""
+    p_answer(monkeypatch, p_reply(200, "<html><body><div class=whatever>redesigned</div></body></html>"))
+    assert (await search_startpage("python", 5)).refused

backend/tests/web_cascade_fixtures.py

@@ -4,6 +4,7 @@ import pytest
 
 import backend.apps.agents.tools.fetch.wayback as WB
 import backend.apps.agents.tools.search.search_startpage as SP
+from backend.apps.agents.tools.search.search_startpage import StartpageAnswer
 import backend.apps.web.web as W
 from backend.apps.agents.tools.web import DDGRateLimited, WebSearchTool
 import backend.apps.agents.tools.ssrf_guard as p_ssrf
@@ -35,7 +36,7 @@ def no_network(monkeypatch):
     monkeypatch.setattr(W, "openai_websearch_via_9router", p_empty)
 
     async def p_startpage_closed(query, num):
-        return None
+        return StartpageAnswer()
     monkeypatch.setattr(SP, "search_startpage", p_startpage_closed)
 
     async def p_no_snapshot(url):
@@ -64,7 +65,13 @@ def ddg_throttled(monkeypatch):
 
 def startpage_returns(monkeypatch, text):
     async def p_f(query, num):
-        return text
+        return StartpageAnswer(results=text)
+    monkeypatch.setattr(SP, "search_startpage", p_f)
+
+
+def startpage_refuses(monkeypatch):
+    async def p_f(query, num):
+        return StartpageAnswer(refused=True)
     monkeypatch.setattr(SP, "search_startpage", p_f)