diff --git a/backend/apps/agents/tools/browser_http.py b/backend/apps/agents/tools/browser_http.py new file mode 100644 index 00000000..e201dc2d --- /dev/null +++ b/backend/apps/agents/tools/browser_http.py @@ -0,0 +1,116 @@ +"""One browser-shaped HTTP request, shared by every keyless web rung. + +Search frontends gate on the TLS/JA3 fingerprint of the CLIENT, not on the +headers or the verb. Measured over 8 interleaved randomised rounds against +DuckDuckGo from one machine: plain httpx POST to the html endpoint 4/8, plain +httpx GET to the lite endpoint 4/8 (so switching verb or endpoint changes +nothing), and curl_cffi's Chrome impersonation 8/8 with the same headers, verb +and URL. That is the whole difference between "search sometimes works" and +"search works", so we impersonate whenever curl_cffi imports. + +If it doesn't import (a packaging regression), we degrade to httpx with a full +Chrome header set rather than failing: half a search beats no search. + +This does NOT validate the target host, so it is only for the FIXED hosts we +choose ourselves. User- or model-supplied URLs must go through +`ssrf_guard.safe_fetch`, which re-checks every redirect hop. +""" + +from typing import Dict, Optional + +import httpx +from pydantic import BaseModel, ConfigDict +from typeguard import typechecked + +CHROME_UA = ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +) + +# A real navigation sends all of these; httpx sends almost none of them by default. +BROWSER_HEADERS: Dict[str, str] = { + "User-Agent": CHROME_UA, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + "Sec-Ch-Ua": '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"', + "Sec-Ch-Ua-Mobile": "?0", + "Sec-Ch-Ua-Platform": '"macOS"', + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Sec-Fetch-User": "?1", + "Upgrade-Insecure-Requests": "1", +} + +P_IMPERSONATE_PROFILE = "chrome" + + +class HttpReply(BaseModel): + model_config = ConfigDict(validate_assignment=True) + + status: int + text: str + content: bytes + content_type: str + url: str + + +@typechecked +def impersonation_available() -> bool: + """Whether the TLS-impersonating client is installed in this environment.""" + try: + import curl_cffi.requests # noqa: F401 + except Exception: + return False + return True + + +@typechecked +async def p_impersonated( + url: str, method: str, params: Optional[Dict], headers: Dict[str, str], + timeout: float, follow_redirects: bool, +) -> HttpReply: + from curl_cffi.requests import AsyncSession + async with AsyncSession() as session: + resp = await session.request( + method, url, params=params, headers=headers, timeout=timeout, + impersonate=P_IMPERSONATE_PROFILE, allow_redirects=follow_redirects, + ) + return HttpReply( + status=resp.status_code, text=resp.text, content=resp.content, + content_type=resp.headers.get("content-type", ""), url=str(resp.url), + ) + + +@typechecked +async def p_plain( + url: str, method: str, params: Optional[Dict], headers: Dict[str, str], + timeout: float, follow_redirects: bool, +) -> HttpReply: + async with httpx.AsyncClient( + timeout=timeout, follow_redirects=follow_redirects, headers=headers, + ) as client: + resp = await client.request(method, url, params=params) + return HttpReply( + status=resp.status_code, text=resp.text, content=resp.content, + content_type=resp.headers.get("content-type", ""), url=str(resp.url), + ) + + +@typechecked +async def browser_request( + url: str, + *, + method: str = "GET", + params: Optional[Dict] = None, + headers: Optional[Dict[str, str]] = None, + timeout: float = 10.0, + follow_redirects: bool = True, +) -> HttpReply: + """Fetch `url` looking like Chrome. Fixed hosts only; see the module docstring.""" + merged = dict(BROWSER_HEADERS) + if headers: + merged.update(headers) + if impersonation_available(): + return await p_impersonated(url, method, params, merged, timeout, follow_redirects) + return await p_plain(url, method, params, merged, timeout, follow_redirects) diff --git a/backend/apps/agents/tools/search_ddg.py b/backend/apps/agents/tools/search_ddg.py index ba478251..77084c2e 100644 --- a/backend/apps/agents/tools/search_ddg.py +++ b/backend/apps/agents/tools/search_ddg.py @@ -1,32 +1,33 @@ """DuckDuckGo web search: html endpoint primary, lite endpoint fallback. The html endpoint is the richer parse; lite (see search_ddg_lite) covers the two -ways html dies: a 202 throttle and silent markup drift. Only both endpoints -throttling raises DDGRateLimited, so free search no longer has a single point -of failure (the outage class that stranded subscription-only users on -"No search backend is configured").""" +ways html dies: the 202 bot challenge and silent markup drift. Only both +endpoints challenging raises DDGRateLimited, so free search no longer has a +single point of failure (the outage class that stranded subscription-only users +on "No search backend is configured"). + +Both rungs go out through `browser_http`, whose Chrome TLS fingerprint is what +actually decides whether DuckDuckGo answers; a plain httpx client scored 4/8 on +the same queries this one scored 8/8 on.""" import html import re -import httpx - +from backend.apps.agents.tools.browser_http import CHROME_UA +from backend.apps.agents.tools.browser_http import browser_request from backend.apps.agents.tools.search_ddg_lite import search_ddg_lite HTTP_TIMEOUT = 30 -USER_AGENT = ( - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " - "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" -) +USER_AGENT = CHROME_UA class DDGRateLimited(Exception): - """Both DuckDuckGo endpoints answered with the throttle challenge (HTTP 202). + """Every DuckDuckGo frontend answered with the bot challenge (HTTP 202). - Distinct from 'genuinely zero hits' so the caller can fail over to another - backend instead of reporting an empty search to the user. The throttle is - per-IP and burst-triggered; once BOTH html and lite serve it, the only cure - is a different backend or waiting it out.""" + Named for history; this is an anti-automation challenge keyed on the + client's fingerprint, NOT a per-IP rate limit. Distinct from 'genuinely + zero hits' so the caller can fail over to another backend instead of + reporting an empty search to the user.""" def strip_html(raw_html: str) -> str: @@ -41,24 +42,19 @@ def strip_html(raw_html: str) -> str: async def search_ddg(query: str, num_results: int) -> str: """Query DuckDuckGo's html endpoint and parse results; lite is the free fallback.""" - async with httpx.AsyncClient( - timeout=HTTP_TIMEOUT, - follow_redirects=True, - headers={"User-Agent": USER_AGENT}, - ) as client: - resp = await client.post( - "https://html.duckduckgo.com/html/", - data={"q": query}, - ) - # DDG serves its throttle challenge as 202 (a ~14KB no-results page), which is a 2xx so raise_for_status() sails right past it. Before declaring rate-limited, try the lite frontend; only when BOTH throttle is free search actually dead. - if resp.status_code == 202: - lite = await search_ddg_lite(query, num_results) - if lite is None: - raise DDGRateLimited(query) - return lite - resp.raise_for_status() + reply = await browser_request( + "https://html.duckduckgo.com/html/", params={"q": query}, timeout=HTTP_TIMEOUT, + ) + # DDG serves its bot challenge as 202 (a ~14KB no-results page), which is a 2xx so a status check sails right past it. Before giving up, try the lite frontend; only when BOTH challenge is free DDG actually dead. + if reply.status == 202: + lite = await search_ddg_lite(query, num_results) + if lite is None: + raise DDGRateLimited(query) + return lite + if reply.status >= 400: + raise RuntimeError(f"DuckDuckGo html returned HTTP {reply.status}") - body = resp.text + body = reply.text result_blocks = re.findall( r']*class="[^"]*result[^"]*"[^>]*>(.*?)\s*(?=]*class="[^"]*result|$)', diff --git a/backend/apps/agents/tools/search_ddg_lite.py b/backend/apps/agents/tools/search_ddg_lite.py index fe5ac608..77eae343 100644 --- a/backend/apps/agents/tools/search_ddg_lite.py +++ b/backend/apps/agents/tools/search_ddg_lite.py @@ -1,23 +1,22 @@ -"""DuckDuckGo lite-endpoint search: the free fallback when html.duckduckgo.com -throttles (HTTP 202) or its markup drifts. lite.duckduckgo.com is a separate -frontend with simpler, stabler HTML and direct result URLs (no uddg redirect). +"""DuckDuckGo lite-endpoint search: the fallback when html.duckduckgo.com +serves its bot challenge (HTTP 202) or its markup drifts. lite.duckduckgo.com +is a separate frontend with simpler, stabler HTML and direct result URLs (no +uddg redirect). -Returns None on a throttle (caller decides whether that means rate-limited -overall) and a formatted results string (possibly empty) on success.""" +Returns None on a challenge (caller decides whether that means every DDG +frontend is closed) and a formatted results string (possibly empty) on +success.""" import html import re from typing import List, Optional -import httpx from typeguard import typechecked +from backend.apps.agents.tools.browser_http import browser_request + P_LITE_URL = "https://lite.duckduckgo.com/lite/" P_TIMEOUT = 12.0 -P_USER_AGENT = ( - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " - "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36" -) P_TAG_RE = re.compile(r"<[^>]+>") # Lite uses single-quoted class attrs today; accept either quote style so a cosmetic flip doesn't kill the parser. P_LINK_RE = re.compile( @@ -52,14 +51,10 @@ def parse_lite_results(body: str, num_results: int) -> str: @typechecked async def search_ddg_lite(query: str, num_results: int) -> Optional[str]: - """None = throttled (202), string = parsed results (may be empty on no hits).""" - async with httpx.AsyncClient( - timeout=P_TIMEOUT, - follow_redirects=True, - headers={"User-Agent": P_USER_AGENT}, - ) as client: - resp = await client.post(P_LITE_URL, data={"q": query}) - if resp.status_code == 202: - return None - resp.raise_for_status() - return parse_lite_results(resp.text, num_results) + """None = bot challenge (202), string = parsed results (may be empty on no hits).""" + reply = await browser_request(P_LITE_URL, params={"q": query}, timeout=P_TIMEOUT) + if reply.status == 202: + return None + if reply.status >= 400: + raise RuntimeError(f"DuckDuckGo lite returned HTTP {reply.status}") + return parse_lite_results(reply.text, num_results) diff --git a/backend/requirements.lock b/backend/requirements.lock index f4755a19..fc8d4c07 100644 --- a/backend/requirements.lock +++ b/backend/requirements.lock @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile backend/requirements.txt --universal --python-version 3.13 --generate-hashes --output-file backend/requirements.lock +# uv pip compile /Users/ericzeng/Downloads/openswarm/backend/requirements.txt --universal --python-version 3.13 --generate-hashes --python /Users/ericzeng/Downloads/openswarm/backend/.venv/bin/python --output-file /Users/ericzeng/Downloads/openswarm/backend/requirements.lock annotated-doc==0.0.4 \ --hash=sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320 \ --hash=sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4 @@ -13,7 +13,7 @@ annotated-types==0.7.0 \ anthropic==0.97.0 \ --hash=sha256:021e79fd8e21e90ad94dc5ba2bbbd8b1599f424f5b1fab6c06204009cab764be \ --hash=sha256:8a1a472dfabcfc0c52ff6a3eecf724ac7e07107a2f6e2367be55ceb42f5d5613 - # via -r backend/requirements.txt + # via -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt anyio==4.13.0 \ --hash=sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708 \ --hash=sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc @@ -39,10 +39,11 @@ certifi==2026.5.20 \ --hash=sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897 \ --hash=sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d # via + # curl-cffi # httpcore # httpx # trafilatura -cffi==2.0.0 ; platform_python_implementation != 'PyPy' \ +cffi==2.0.0 \ --hash=sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb \ --hash=sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b \ --hash=sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f \ @@ -127,7 +128,9 @@ cffi==2.0.0 ; platform_python_implementation != 'PyPy' \ --hash=sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5 \ --hash=sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453 \ --hash=sha256:fe562eb1a64e67dd297ccc4f5addea2501664954f2692b69a76449ec7913ecbf - # via cryptography + # via + # cryptography + # curl-cffi charset-normalizer==3.4.7 \ --hash=sha256:007d05ec7321d12a40227aae9e2bc6dca73f3cb21058999a1df9e193555a9dcc \ --hash=sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c \ @@ -268,14 +271,14 @@ claude-agent-sdk==0.1.70 \ --hash=sha256:955b8d57cc06247f6894bc65d1441ae66b4c7bda3b3fcc0cb7f140e0d48757f8 \ --hash=sha256:c69019de2559650b2e8ae1d93f907f27f623748fe25b6f02b7f5dcf05e956f70 \ --hash=sha256:e3c3ab7a0cfd64d40fa8d9b1cf3aac9f0c4b9b910cff92dd07154f75889d63f8 - # via -r backend/requirements.txt + # via -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt click==8.4.1 \ --hash=sha256:482be17c6991b8c19c5429a1e995d9b0efdbb63172824c41f99965dc0ade8ec2 \ --hash=sha256:918b5633eddf6b41c32d4f454bf0de810065c74e3f7dbf8ee5452f8be88d3e96 # via # rich-toolkit # uvicorn -colorama==0.4.6 ; sys_platform == 'win32' \ +colorama==0.4.6 ; sys_platform == 'win32' or platform_system == 'Windows' \ --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 # via @@ -337,6 +340,29 @@ cryptography==48.0.0 \ --hash=sha256:f5333311663ea94f75dd408665686aaf426563556bb5283554a3539177e03b8c \ --hash=sha256:fdfef35d751d510fcef5252703621574364fec16418c4a1e5e1055248401054b # via pyjwt +curl-cffi==0.15.0 \ + --hash=sha256:08c799b89740b9bc49c09fbc3d5907f13ac1f845ca52620507ef9466d4639dd5 \ + --hash=sha256:0b6c0543b993996670e9e4b78e305a2d60809d5681903ffb5568e21a387434d3 \ + --hash=sha256:1977e1e12cfb5c11352cbb74acef1bed24eb7d226dab61ca57c168c21acd4d61 \ + --hash=sha256:2b6c847d86283b07ae69bb72c82eb8a59242277142aa35b89850f89e792a02fc \ + --hash=sha256:408d6f14e346841cd889c2e0962832bb235ba3b6749ebf609f347f747da5e60f \ + --hash=sha256:41f80170ba844009273b2660da1964ec31e99e5719d16b3422ada87177e32e13 \ + --hash=sha256:4682dc38d4336e0eb0b185374db90a760efde63cbea994b4e63f3521d44c4c92 \ + --hash=sha256:582e570aa2586b96ed47cf4a17586b9a3c462cbe43f780487c3dc245c6ef1527 \ + --hash=sha256:5a0c1896a0d5a5ac1eb89cd24b008d2b718dd1df6fd2f75451b59ca66e49e572 \ + --hash=sha256:7b7a92767a888ee90147e18964b396d8435ff42737030d6fb00824ffd6094805 \ + --hash=sha256:7e63539d0d839d0a8c5eacf86229bc68c57803547f35e0db7ee0986328b478c3 \ + --hash=sha256:829cc357061ecb99cc2d406301f609a039e05665322f5c025ec67c38b0dc49ce \ + --hash=sha256:838e48212447d9c81364b04707a5c861daf08f8320f9ecb3406a8919d1d5c3b3 \ + --hash=sha256:967ad7355bd8e9586f8c2d02eaa99953747549e7ea4a9b25cd53353e6b67fe6d \ + --hash=sha256:9e5e69eee735f659287e2c84444319d68a1fa68dd37abf228943a4074864283a \ + --hash=sha256:a25620d9bf989c9c029a7d1642999c4c265abb0bad811deb2f77b0b5b2b12e5b \ + --hash=sha256:a6d57f8389273a3a1f94370473c74897467bcc36af0a17336989780c507fa43d \ + --hash=sha256:aa1323950224db24f4c510d010b3affa02196ca853fb424191fa917a513d3f4b \ + --hash=sha256:b624c7ce087bfda967a013ed0a64702a525444e5b6e97d23534d567ccc6525aa \ + --hash=sha256:bda66404010e9ed743b1b83c20c86f24fe21a9a6873e17479d6e67e29d8ded28 \ + --hash=sha256:ea0c67652bf6893d34ee0f82c944f37e488f6147e9421bef1771cc6545b02ded + # via -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt dateparser==1.4.0 \ --hash=sha256:7902b8e85d603494bf70a5a0b1decdddb2270b9c6e6b2bc8a57b93476c0df378 \ --hash=sha256:97a21840d5ecdf7630c584f673338a5afac5dfe84f647baf4d7e8df98f9354a4 @@ -360,7 +386,7 @@ email-validator==2.3.0 \ fastapi==0.136.3 \ --hash=sha256:3d2a69bdf04b7e9f3afa292c3bc7a98816bbfafa10bc9b45f3f3700d2f761620 \ --hash=sha256:e487fae93ad408e6f47641ee4dfe389864fd7bec92e547ea8498fc13f43e83ab - # via -r backend/requirements.txt + # via -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt fastapi-cli==0.0.24 \ --hash=sha256:1afc9c9e21d7ebc8a3ca5e31790cd8d837742be7e4f8b9236e99cb3451f0de00 \ --hash=sha256:4a1f78ed798f106b4fee85ca93b85d8fe33c0a3570f775964d37edb80b8f0edc @@ -435,10 +461,11 @@ httpx==0.28.1 \ --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad # via - # -r backend/requirements.txt + # -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt # anthropic # fastapi # mcp + # swarm-analytics httpx-sse==0.4.3 \ --hash=sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc \ --hash=sha256:9b1ed0127459a66014aec3c56bebd93da3c1bc8bb6618c8082039a44889a755d @@ -569,7 +596,7 @@ jsonschema==4.26.0 \ --hash=sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326 \ --hash=sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce # via - # -r backend/requirements.txt + # -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt # mcp jsonschema-specifications==2025.9.1 \ --hash=sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe \ @@ -918,8 +945,8 @@ pillow==12.2.0 \ --hash=sha256:f490f9368b6fc026f021db16d7ec2fbf7d89e2edb42e8ec09d2c60505f5729c7 \ --hash=sha256:fb043ee2f06b41473269765c2feae53fc2e2fbf96e5e22ca94fb5ad677856f06 \ --hash=sha256:fc3d34d4a8fbec3e88a79b92e5465e0f9b842b628675850d860b8bd300b159f5 - # via -r backend/requirements.txt -pycparser==3.0 ; implementation_name != 'PyPy' and platform_python_implementation != 'PyPy' \ + # via -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt +pycparser==3.0 ; implementation_name != 'PyPy' \ --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 # via cffi @@ -927,12 +954,13 @@ pydantic==2.13.3 \ --hash=sha256:6db14ac8dfc9a1e57f87ea2c0de670c251240f43cb0c30a5130e9720dc612927 \ --hash=sha256:af09e9d1d09f4e7fe37145c1f577e1d61ceb9a41924bf0094a36506285d0a84d # via - # -r backend/requirements.txt + # -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt # anthropic # fastapi # mcp # pydantic-extra-types # pydantic-settings + # swarm-analytics pydantic-core==2.46.3 \ --hash=sha256:0087084960f209a9a4af50ecd1fb063d9ad3658c07bb81a7a53f452dacbfb2ba \ --hash=sha256:031bb17f4885a43773c8c763089499f242aee2ea85cf17154168775dccdecf35 \ @@ -1083,7 +1111,7 @@ python-dotenv==1.1.1 \ --hash=sha256:31f23644fe2602f88ff55e1f5c79ba497e01224ee7737937930c448e4d0e24dc \ --hash=sha256:a8a6399716257f45be6a007360200409fce5cda2661e3dec71d23dc15f6189ab # via - # -r backend/requirements.txt + # -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt # pydantic-settings # uvicorn python-multipart==0.0.29 \ @@ -1319,6 +1347,7 @@ rich==15.0.0 \ --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb \ --hash=sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36 # via + # curl-cffi # rich-toolkit # typer rich-toolkit==0.19.10 \ @@ -1472,7 +1501,7 @@ starlette==1.1.0 \ swarm-analytics==0.1.1 \ --hash=sha256:49255c5a0962ba1eca3c14471b7df8746f4258d8cd83c8c73931e62381b2be8e \ --hash=sha256:c1d368905a8b53a555bb53bd60c6707323fba4beff3ea2e79a8f83fb91b11cab - # via -r backend/requirements.txt + # via -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt tld==0.13.2 \ --hash=sha256:9b8fdbdb880e7ba65b216a4937f2c94c49a7226723783d5838fc958ac76f4e0c \ --hash=sha256:d983fa92b9d717400742fca844e29d5e18271079c7bcfabf66d01b39b4a14345 @@ -1480,11 +1509,11 @@ tld==0.13.2 \ trafilatura==2.0.0 \ --hash=sha256:77eb5d1e993747f6f20938e1de2d840020719735690c840b9a1024803a4cd51d \ --hash=sha256:ceb7094a6ecc97e72fea73c7dba36714c5c5b577b6470e4520dca893706d6247 - # via -r backend/requirements.txt + # via -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt typeguard==4.4.2 \ --hash=sha256:77a78f11f09777aeae7fa08585f33b5f4ef0e7335af40005b0c422ed398ff48c \ --hash=sha256:a6f1065813e32ef365bc3b3f503af8a96f9dd4e0033a02c28c4a4983de8c6c49 - # via -r backend/requirements.txt + # via -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt typer==0.26.1 \ --hash=sha256:537d27ae686d82967f6383382a952cb32ba4768898541effccb69ca75bbd5d23 \ --hash=sha256:933e4f0083521f3c57d6a5aedf3b073271b2f95a19761b171b494dd6fdb21ff6 @@ -1510,7 +1539,7 @@ typing-inspection==0.4.2 \ # mcp # pydantic # pydantic-settings -tzdata==2026.2 ; sys_platform == 'win32' \ +tzdata==2026.2 ; platform_system == 'Windows' \ --hash=sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10 \ --hash=sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7 # via tzlocal @@ -1518,7 +1547,7 @@ tzlocal==5.3.1 \ --hash=sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd \ --hash=sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d # via - # -r backend/requirements.txt + # -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt # dateparser urllib3==2.7.0 \ --hash=sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c \ diff --git a/backend/requirements.txt b/backend/requirements.txt index a9b7fea0..8096c282 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -16,6 +16,8 @@ python-dotenv==1.1.1 Pillow==12.2.0 httpx==0.28.1 trafilatura==2.0.0 +# curl_cffi: Chrome TLS-fingerprint impersonation for the keyless search rungs. Measured 8/8 against DuckDuckGo where plain httpx scored 4/8 on the same queries; abi3 wheels for macos arm64/x64 and win_amd64. Guarded import, so a missing wheel degrades to httpx instead of breaking the backend. +curl_cffi==0.15.0 # swarm-analytics: typed client for the product-analytics ingest; fire-and-forget so it never breaks the app. swarm-analytics==0.1.1 # tzlocal: dev-mode fallback for resolving the user's IANA timezone when diff --git a/backend/tests/test_browser_http.py b/backend/tests/test_browser_http.py new file mode 100644 index 00000000..30686064 --- /dev/null +++ b/backend/tests/test_browser_http.py @@ -0,0 +1,107 @@ +"""The keyless rungs go out through ONE browser-shaped client. + +Why it matters: DuckDuckGo's 202 challenge keys on the client's TLS +fingerprint, not on headers or verb. Measured over 8 interleaved randomised +rounds, plain httpx scored 4/8 and Chrome impersonation 8/8 on the same +queries. These pin the seam and the degrade path. +""" + +import pytest + +import backend.apps.agents.tools.browser_http as BH +import backend.apps.agents.tools.search_ddg as SD +import backend.apps.agents.tools.search_ddg_lite as SDL +from backend.apps.agents.tools.browser_http import BROWSER_HEADERS, HttpReply, browser_request + + +def p_reply(status=200, text="ok"): + return HttpReply(status=status, text=text, content=text.encode(), + content_type="text/html", url="https://x.example") + + +def p_record_transports(monkeypatch): + used = [] + + async def p_imp(*a, **k): + used.append("impersonated") + return p_reply() + + async def p_pl(*a, **k): + used.append("plain") + return p_reply() + + monkeypatch.setattr(BH, "p_impersonated", p_imp) + monkeypatch.setattr(BH, "p_plain", p_pl) + return used + + +@pytest.mark.asyncio +async def test_impersonates_when_the_client_is_installed(monkeypatch): + used = p_record_transports(monkeypatch) + monkeypatch.setattr(BH, "impersonation_available", lambda: True) + await browser_request("https://x.example") + assert used == ["impersonated"] + + +@pytest.mark.asyncio +async def test_degrades_to_httpx_instead_of_failing(monkeypatch): + """A missing wheel must cost us reliability, never the whole backend.""" + used = p_record_transports(monkeypatch) + monkeypatch.setattr(BH, "impersonation_available", lambda: False) + reply = await browser_request("https://x.example") + assert used == ["plain"] + assert reply.status == 200 + + +@pytest.mark.asyncio +async def test_caller_headers_win_over_the_defaults(monkeypatch): + seen = {} + + async def p_pl(url, method, params, headers, timeout, follow_redirects): + seen.update(headers) + return p_reply() + + monkeypatch.setattr(BH, "p_plain", p_pl) + monkeypatch.setattr(BH, "impersonation_available", lambda: False) + await browser_request("https://x.example", headers={"Accept-Language": "de-DE"}) + assert seen["Accept-Language"] == "de-DE" + assert seen["User-Agent"] == BROWSER_HEADERS["User-Agent"] + + +def test_default_headers_look_like_a_real_navigation(): + # A bare User-Agent is the tell that gets a scraper challenged. + for key in ("User-Agent", "Accept", "Accept-Language", "Sec-Fetch-Mode", "Upgrade-Insecure-Requests"): + assert BROWSER_HEADERS.get(key) + assert "Chrome/" in BROWSER_HEADERS["User-Agent"] + + +@pytest.mark.asyncio +async def test_ddg_rungs_send_the_query_as_a_get_param(monkeypatch): + """Pins the shape: a GET with params, through the shared seam, on both frontends.""" + calls = [] + + async def p_req(url, **kw): + calls.append((url, kw.get("method", "GET"), kw.get("params"))) + return p_reply(202, "challenge") + + monkeypatch.setattr(SD, "browser_request", p_req) + monkeypatch.setattr(SDL, "browser_request", p_req) + from backend.apps.agents.tools.web import DDGRateLimited + with pytest.raises(DDGRateLimited): + await SD.search_ddg("some query", 5) + + assert [c[0] for c in calls] == [ + "https://html.duckduckgo.com/html/", + "https://lite.duckduckgo.com/lite/", + ] + for _, method, params in calls: + assert method == "GET" + assert params == {"q": "some query"} + + +def test_curl_cffi_is_a_declared_dependency(): + """It must be in BOTH files: the packaged python-env installs from the LOCK.""" + from pathlib import Path + root = Path(__file__).resolve().parents[1] + assert "curl_cffi==" in (root / "requirements.txt").read_text() + assert "curl-cffi==" in (root / "requirements.lock").read_text() diff --git a/backend/tests/test_web_search_ddg.py b/backend/tests/test_web_search_ddg.py index 05c8435e..a1f37681 100644 --- a/backend/tests/test_web_search_ddg.py +++ b/backend/tests/test_web_search_ddg.py @@ -9,39 +9,25 @@ These pin the two bugs that turned DDG into a flaky 'No results found' source: We mock the network so the test is deterministic and offline. """ -import httpx import pytest +import backend.apps.agents.tools.search_ddg as SD +import backend.apps.agents.tools.search_ddg_lite as SDL +from backend.apps.agents.tools.browser_http import HttpReply from backend.apps.agents.tools.web import WebSearchTool, DDGRateLimited -class p_FakeResp: - def __init__(self, status_code: int, text: str): - self.status_code = status_code - self.text = text - - def raise_for_status(self): - if self.status_code >= 400: - raise httpx.HTTPStatusError("err", request=None, response=None) +def p_reply(status: int, text: str) -> HttpReply: + return HttpReply(status=status, text=text, content=text.encode(), + content_type="text/html", url="https://duckduckgo.example") -class p_FakeClient: - """Stands in for httpx.AsyncClient; returns a canned response.""" - def __init__(self, resp: p_FakeResp): - self.p_resp = resp - - async def __aenter__(self): - return self - - async def __aexit__(self, *a): - return False - - async def post(self, *a, **k): - return self.p_resp - - -def p_patch_client(monkeypatch, resp: p_FakeResp): - monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: p_FakeClient(resp)) +def p_patch_client(monkeypatch, reply: HttpReply): + """Mock the ONE seam every keyless rung goes through, whatever transport it picks.""" + async def p_req(url, **kw): + return reply + monkeypatch.setattr(SD, "browser_request", p_req) + monkeypatch.setattr(SDL, "browser_request", p_req) # One real organic result + one sponsored (ad) row in DDG's html markup. @@ -59,14 +45,14 @@ P_HTML_WITH_AD = """ @pytest.mark.asyncio async def test_202_raises_rate_limited_not_empty(monkeypatch): - p_patch_client(monkeypatch, p_FakeResp(202, "throttle challenge, no results")) + p_patch_client(monkeypatch, p_reply(202, "throttle challenge, no results")) with pytest.raises(DDGRateLimited): await WebSearchTool.search_ddg("anything", 5) @pytest.mark.asyncio async def test_execute_reports_rate_limit_clearly(monkeypatch): - p_patch_client(monkeypatch, p_FakeResp(202, "throttle")) + p_patch_client(monkeypatch, p_reply(202, "throttle")) parts = await WebSearchTool().execute({"query": "x", "num_results": 5}, None) msg = parts[0]["text"].lower() assert "rate-limit" in msg @@ -75,7 +61,7 @@ async def test_execute_reports_rate_limit_clearly(monkeypatch): @pytest.mark.asyncio async def test_ads_are_stripped_real_results_kept(monkeypatch): - p_patch_client(monkeypatch, p_FakeResp(200, P_HTML_WITH_AD)) + p_patch_client(monkeypatch, p_reply(200, P_HTML_WITH_AD)) out = await WebSearchTool.search_ddg("topic", 5) assert "example.com/real" in out assert "Real Result Title" in out @@ -88,6 +74,6 @@ async def test_ads_are_stripped_real_results_kept(monkeypatch): @pytest.mark.asyncio async def test_genuinely_empty_is_not_a_rate_limit(monkeypatch): # 200 with no result blocks is a real empty result set, not a throttle. - p_patch_client(monkeypatch, p_FakeResp(200, "nothing here")) + p_patch_client(monkeypatch, p_reply(200, "nothing here")) out = await WebSearchTool.search_ddg("zxcvqwer no hits", 5) assert out == "" diff --git a/backend/tests/test_web_search_ddg_lite.py b/backend/tests/test_web_search_ddg_lite.py index 41fc7485..af8b3805 100644 --- a/backend/tests/test_web_search_ddg_lite.py +++ b/backend/tests/test_web_search_ddg_lite.py @@ -12,9 +12,11 @@ Network is mocked; the lite fixture is the real markup shape captured live import asyncio -import httpx import pytest +import backend.apps.agents.tools.search_ddg as SD +import backend.apps.agents.tools.search_ddg_lite as SDL +from backend.apps.agents.tools.browser_http import HttpReply from backend.apps.agents.tools.web import WebSearchTool, DDGRateLimited from backend.apps.agents.tools.search_ddg_lite import parse_lite_results @@ -34,36 +36,20 @@ P_LITE_BODY = """ P_HTML_202_BODY = "anomaly detected, challenge page" -class p_FakeResp: - def __init__(self, status_code: int, text: str): - self.status_code = status_code - self.text = text - - def raise_for_status(self): - if self.status_code >= 400: - raise httpx.HTTPStatusError("err", request=None, response=None) - - -class p_RoutedClient: - """Fake AsyncClient that answers per-URL, so the html and lite endpoints can behave differently in one test.""" - def __init__(self, routes: dict): - self.routes = routes - - async def __aenter__(self): - return self - - async def __aexit__(self, *a): - return False - - async def post(self, url, *a, **k): - for key, resp in self.routes.items(): - if key in url: - return resp - raise AssertionError(f"unexpected URL {url}") +def p_FakeResp(status: int, text: str) -> HttpReply: + return HttpReply(status=status, text=text, content=text.encode(), + content_type="text/html", url="https://duckduckgo.example") def p_route(monkeypatch, routes: dict): - monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: p_RoutedClient(routes)) + """Answer per-URL so the html and lite endpoints can behave differently in one test.""" + async def p_req(url, **kw): + for key, reply in routes.items(): + if key in url: + return reply + raise AssertionError(f"unexpected URL {url}") + monkeypatch.setattr(SD, "browser_request", p_req) + monkeypatch.setattr(SDL, "browser_request", p_req) def test_lite_parser_on_real_shape():