diff --git a/backend/apps/agents/tools/search/search_ddg.py b/backend/apps/agents/tools/search/search_ddg.py new file mode 100644 index 00000000..00d8c472 --- /dev/null +++ b/backend/apps/agents/tools/search/search_ddg.py @@ -0,0 +1,118 @@ +"""DuckDuckGo web search: html endpoint primary, lite endpoint fallback. + +The html endpoint is the richer parse; lite (see search_ddg_lite) covers the two +ways html dies: the 202 bot challenge and silent markup drift. Only both +endpoints challenging raises DDGRateLimited, so free search no longer has a +single point of failure (the outage class that stranded subscription-only users +on "No search backend is configured"). + +Both rungs go out through `browser_http`, whose Chrome TLS fingerprint is what +actually decides whether DuckDuckGo answers; a plain httpx client scored 4/8 on +the same queries this one scored 8/8 on.""" + +import html +import re + +from backend.apps.agents.tools.browser_http import CHROME_UA +from backend.apps.agents.tools.browser_http import browser_request +from backend.apps.agents.tools.search.search_ddg_lite import search_ddg_lite + +HTTP_TIMEOUT = 30 +USER_AGENT = CHROME_UA + + +class DDGRateLimited(Exception): + """Every DuckDuckGo frontend answered with the bot challenge (HTTP 202). + + Named for history; this is an anti-automation challenge keyed on the + client's fingerprint, NOT a per-IP rate limit. Distinct from 'genuinely + zero hits' so the caller can fail over to another backend instead of + reporting an empty search to the user.""" + + +def strip_html(raw_html: str) -> str: + """Naive but effective HTML to plain-text conversion.""" + text = re.sub(r"<(script|style)[^>]*>.*?", "", raw_html, flags=re.DOTALL | re.IGNORECASE) + text = re.sub(r"<[^>]+>", " ", text) + text = html.unescape(text) + text = re.sub(r"[ \t]+", " ", text) + text = re.sub(r"\n{3,}", "\n\n", text) + return text.strip() + + +async def search_ddg(query: str, num_results: int) -> str: + """Query DuckDuckGo's html endpoint and parse results; lite is the free fallback.""" + reply = await browser_request( + "https://html.duckduckgo.com/html/", params={"q": query}, timeout=HTTP_TIMEOUT, + ) + # DDG serves its bot challenge as 202 (a ~14KB no-results page), which is a 2xx so a status check sails right past it. Before giving up, try the lite frontend; only when BOTH challenge is free DDG actually dead. + if reply.status == 202: + lite = await search_ddg_lite(query, num_results) + if lite is None: + raise DDGRateLimited(query) + return lite + if reply.status >= 400: + raise RuntimeError(f"DuckDuckGo html returned HTTP {reply.status}") + + body = reply.text + + result_blocks = re.findall( + r']*class="[^"]*result[^"]*"[^>]*>(.*?)\s*(?=]*class="[^"]*result|$)', + body, + flags=re.DOTALL, + ) + + entries: list[str] = [] + for block in result_blocks: + if len(entries) >= num_results: + break + + # Handle both class-before-href and href-before-class attribute orders. + link_match = re.search( + r']*class="[^"]*result__a[^"]*"[^>]*href="([^"]*)"[^>]*>(.*?)', + block, + flags=re.DOTALL, + ) + if not link_match: + link_match = re.search( + r']*href="([^"]*)"[^>]*class="[^"]*result__a[^"]*"[^>]*>(.*?)', + block, + flags=re.DOTALL, + ) + if not link_match: + continue + + raw_url = html.unescape(link_match.group(1)) + + # Drop sponsored rows: DDG ads point at its own y.js click-tracker (ad_domain/ad_provider) instead of a real uddg= redirect, so they'd otherwise show up as junk "duckduckgo.com/y.js?ad_..." results. + if "/y.js?" in raw_url or "ad_provider=" in raw_url or "ad_domain=" in raw_url: + continue + + title = strip_html(link_match.group(2)).strip() + + snippet_match = re.search( + r']*class="[^"]*result__snippet[^"]*"[^>]*>(.*?)', + block, + flags=re.DOTALL, + ) + snippet = strip_html(snippet_match.group(1)).strip() if snippet_match else "" + + # DDG wraps URLs in a redirect; extract the real one. + real_url_match = re.search(r"uddg=([^&]+)", raw_url) + if real_url_match: + from urllib.parse import unquote + url = unquote(real_url_match.group(1)) + else: + url = raw_url + + entry = f"[{len(entries) + 1}] {title}\n {url}" + if snippet: + entry += f"\n {snippet}" + entries.append(entry) + + # 200 with zero parsed entries usually means DDG changed its markup out from under the regexes (it has before), not a genuine no-hits; lite's simpler shape is the safety net. + if not entries: + lite = await search_ddg_lite(query, num_results) + if lite: + return lite + return "\n\n".join(entries) diff --git a/backend/apps/agents/tools/search/search_ddg_lite.py b/backend/apps/agents/tools/search/search_ddg_lite.py new file mode 100644 index 00000000..77eae343 --- /dev/null +++ b/backend/apps/agents/tools/search/search_ddg_lite.py @@ -0,0 +1,60 @@ +"""DuckDuckGo lite-endpoint search: the fallback when html.duckduckgo.com +serves its bot challenge (HTTP 202) or its markup drifts. lite.duckduckgo.com +is a separate frontend with simpler, stabler HTML and direct result URLs (no +uddg redirect). + +Returns None on a challenge (caller decides whether that means every DDG +frontend is closed) and a formatted results string (possibly empty) on +success.""" + +import html +import re +from typing import List, Optional + +from typeguard import typechecked + +from backend.apps.agents.tools.browser_http import browser_request + +P_LITE_URL = "https://lite.duckduckgo.com/lite/" +P_TIMEOUT = 12.0 +P_TAG_RE = re.compile(r"<[^>]+>") +# Lite uses single-quoted class attrs today; accept either quote style so a cosmetic flip doesn't kill the parser. +P_LINK_RE = re.compile( + r"""]*href="([^"]+)"[^>]*class=['"]result-link['"][^>]*>(.*?)""", + flags=re.DOTALL, +) +P_SNIPPET_RE = re.compile( + r"""]*class=['"]result-snippet['"][^>]*>(.*?)""", + flags=re.DOTALL, +) + + +@typechecked +def p_strip(text: str) -> str: + return html.unescape(P_TAG_RE.sub("", text)).strip() + + +@typechecked +def parse_lite_results(body: str, num_results: int) -> str: + """Format lite's result rows; links and snippets appear in document order and pair up positionally.""" + links = P_LINK_RE.findall(body) + snippets = [p_strip(s) for s in P_SNIPPET_RE.findall(body)] + entries: List[str] = [] + for i, (url, raw_title) in enumerate(links[:num_results]): + title = p_strip(raw_title) + entry = f"[{i + 1}] {title}\n {html.unescape(url)}" + if i < len(snippets) and snippets[i]: + entry += f"\n {snippets[i]}" + entries.append(entry) + return "\n\n".join(entries) + + +@typechecked +async def search_ddg_lite(query: str, num_results: int) -> Optional[str]: + """None = bot challenge (202), string = parsed results (may be empty on no hits).""" + reply = await browser_request(P_LITE_URL, params={"q": query}, timeout=P_TIMEOUT) + if reply.status == 202: + return None + if reply.status >= 400: + raise RuntimeError(f"DuckDuckGo lite returned HTTP {reply.status}") + return parse_lite_results(reply.text, num_results) diff --git a/backend/apps/agents/tools/search/search_startpage.py b/backend/apps/agents/tools/search/search_startpage.py new file mode 100644 index 00000000..692cb578 --- /dev/null +++ b/backend/apps/agents/tools/search/search_startpage.py @@ -0,0 +1,80 @@ +"""Startpage search: the second independent engine behind DuckDuckGo. + +DuckDuckGo is one operator, so its bot challenge is one point of failure for +every keyless user. Startpage serves Google's index and answered 8/8 on the +same machine and rounds where DuckDuckGo's shipped client shape answered 4/8, +so it is a genuine second opinion rather than a retry. + +It must be a POST: a GET to /sp/search is answered with an Anubis +proof-of-work interstitial (measured, ~10KB and zero results), while the POST +returns the real result page. Parsing is anchored on `result-link` / +`gl-title-link` and the `description` paragraph, never on the emotion CSS +hashes in the same class attributes, which change build to build. + +Returns None when Startpage served a challenge instead of results, so the +caller can tell "closed" apart from "genuinely no hits".""" + +import html +import re +from typing import List, Optional + +from typeguard import typechecked + +from backend.apps.agents.tools.browser_http import browser_request + +P_SEARCH_URL = "https://www.startpage.com/sp/search" +P_TIMEOUT = 12.0 + +P_ANCHOR_RE = re.compile( + r"]*(?:result-link|gl-title-link)[^>]*)>(.*?)", flags=re.DOTALL, +) +P_HREF_RE = re.compile(r'href="([^"]+)"') +P_TITLE_RE = re.compile(r"]*>(.*?)", flags=re.DOTALL) +P_DESC_RE = re.compile( + r']*class="[^"]*\bdescription\b[^"]*"[^>]*>(.*?)

', flags=re.DOTALL, +) +# Startpage inlines a + + +

asyncio — Asynchronous I/O

+ +

The asyncio library, explained.

+ +
+

Real Python walkthrough

 +

A hands-on tour.

+
+

Unrelated footer blurb, belongs to no result.

+""" + +P_CHALLENGE_BODY = """ + +
Making sure you are not a bot...
+""" + + +def p_reply(status: int, text: str) -> HttpReply: + return HttpReply(status=status, text=text, content=text.encode(), + content_type="text/html", url="https://www.startpage.com/sp/search") + + +def p_answer(monkeypatch, reply: HttpReply, seen=None): + async def p_req(url, **kw): + if seen is not None: + seen.append((url, kw.get("method"), kw.get("params"))) + return reply + monkeypatch.setattr(SP, "browser_request", p_req) + + +def test_parses_title_url_and_the_snippet_from_its_own_block(): + out = parse_startpage_results(P_BODY, 5) + assert "[1] asyncio — Asynchronous I/O" in out + assert "https://docs.python.org/3/library/asyncio.html" in out + assert "The asyncio library, explained." in out + assert "[2] Real Python walkthrough" in out + assert "A hands-on tour." in out + + +def test_inline_style_never_becomes_the_title(): + """Tag-stripping alone would emit the anchor's own CSS as the result title.""" + out = parse_startpage_results(P_BODY, 5) + assert "css-" not in out + assert "font-size" not in out + + +def test_an_orphan_snippet_is_not_attached_to_a_result(): + out = parse_startpage_results(P_BODY, 5) + assert "Unrelated footer blurb" not in out + + +def test_respects_num_results(): + out = parse_startpage_results(P_BODY, 1) + assert "[1]" in out and "[2]" not in out + + +@pytest.mark.asyncio +async def test_challenge_page_reads_as_refused_not_as_zero_hits(monkeypatch): + # Startpage answers its proof-of-work interstitial with a normal 200, so "parsed nothing" is the only honest signal. + p_answer(monkeypatch, p_reply(200, P_CHALLENGE_BODY)) + assert await search_startpage("q", 5) is None + + +@pytest.mark.asyncio +async def test_http_error_reads_as_refused(monkeypatch): + p_answer(monkeypatch, p_reply(503, "nope")) + assert await search_startpage("q", 5) is None + + +@pytest.mark.asyncio +async def test_must_post_or_startpage_serves_the_proof_of_work_wall(monkeypatch): + seen = [] + p_answer(monkeypatch, p_reply(200, P_BODY), seen) + out = await search_startpage("some query", 5) + assert out is not None + url, method, params = seen[0] + assert url == "https://www.startpage.com/sp/search" + assert method == "POST" + assert params == {"query": "some query", "cat": "web"} diff --git a/electron/hiddenBrowser.js b/electron/hiddenBrowser.js index df85186c..9cb5756f 100644 --- a/electron/hiddenBrowser.js +++ b/electron/hiddenBrowser.js @@ -123,7 +123,8 @@ async function hiddenEvalWithCookies(url, cookieRecords, js) { // Google first (direct result URLs, best quality); DuckDuckGo in a real browser // second (immune to the httpx 202 throttle); Bing last (results are redirect-wrapped). const ENGINES = [ - { name: 'google', url: (q) => `https://www.google.com/search?q=${encodeURIComponent(q)}&num=10&hl=en`, + // udm=14 is Google's plain "Web" tab: ten blue links, no AI Overview and no answer widgets, so the a-h3 scrape gets real result URLs instead of whatever the SERP decided to render today. + { name: 'google', url: (q) => `https://www.google.com/search?q=${encodeURIComponent(q)}&udm=14&num=10&hl=en`, scrape: `Array.from(document.querySelectorAll('a h3')).map(function(h){var a=h.closest('a');return a&&a.href?{t:h.innerText,u:a.href}:null;}).filter(function(x){return x&&x.u.indexOf('http')===0&&x.u.indexOf('google.')===-1;})` }, { name: 'ddg', url: (q) => `https://html.duckduckgo.com/html/?q=${encodeURIComponent(q)}`, scrape: `Array.from(document.querySelectorAll('a.result__a')).map(function(a){var m=a.href.match(/uddg=([^&]+)/);return {t:a.innerText,u:m?decodeURIComponent(m[1]):a.href};})` },