mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-08-17 18:25:42 +02:00
[eric] web: send the keyless search rungs through a Chrome-fingerprint client (measured 4/8 -> 8/8 on DuckDuckGo)
This commit is contained in:
ciregenz
@@ -0,0 +1,116 @@
|
||||
"""One browser-shaped HTTP request, shared by every keyless web rung.
|
||||
|
||||
Search frontends gate on the TLS/JA3 fingerprint of the CLIENT, not on the
|
||||
headers or the verb. Measured over 8 interleaved randomised rounds against
|
||||
DuckDuckGo from one machine: plain httpx POST to the html endpoint 4/8, plain
|
||||
httpx GET to the lite endpoint 4/8 (so switching verb or endpoint changes
|
||||
nothing), and curl_cffi's Chrome impersonation 8/8 with the same headers, verb
|
||||
and URL. That is the whole difference between "search sometimes works" and
|
||||
"search works", so we impersonate whenever curl_cffi imports.
|
||||
|
||||
If it doesn't import (a packaging regression), we degrade to httpx with a full
|
||||
Chrome header set rather than failing: half a search beats no search.
|
||||
|
||||
This does NOT validate the target host, so it is only for the FIXED hosts we
|
||||
choose ourselves. User- or model-supplied URLs must go through
|
||||
`ssrf_guard.safe_fetch`, which re-checks every redirect hop.
|
||||
"""
|
||||
|
||||
from typing import Dict, Optional
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
from typeguard import typechecked
|
||||
|
||||
CHROME_UA = (
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
# A real navigation sends all of these; httpx sends almost none of them by default.
|
||||
BROWSER_HEADERS: Dict[str, str] = {
|
||||
"User-Agent": CHROME_UA,
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Sec-Ch-Ua": '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
|
||||
"Sec-Ch-Ua-Mobile": "?0",
|
||||
"Sec-Ch-Ua-Platform": '"macOS"',
|
||||
"Sec-Fetch-Dest": "document",
|
||||
"Sec-Fetch-Mode": "navigate",
|
||||
"Sec-Fetch-Site": "none",
|
||||
"Sec-Fetch-User": "?1",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
}
|
||||
|
||||
P_IMPERSONATE_PROFILE = "chrome"
|
||||
|
||||
|
||||
class HttpReply(BaseModel):
|
||||
model_config = ConfigDict(validate_assignment=True)
|
||||
|
||||
status: int
|
||||
text: str
|
||||
content: bytes
|
||||
content_type: str
|
||||
url: str
|
||||
|
||||
|
||||
@typechecked
|
||||
def impersonation_available() -> bool:
|
||||
"""Whether the TLS-impersonating client is installed in this environment."""
|
||||
try:
|
||||
import curl_cffi.requests # noqa: F401
|
||||
except Exception:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
@typechecked
|
||||
async def p_impersonated(
|
||||
url: str, method: str, params: Optional[Dict], headers: Dict[str, str],
|
||||
timeout: float, follow_redirects: bool,
|
||||
) -> HttpReply:
|
||||
from curl_cffi.requests import AsyncSession
|
||||
async with AsyncSession() as session:
|
||||
resp = await session.request(
|
||||
method, url, params=params, headers=headers, timeout=timeout,
|
||||
impersonate=P_IMPERSONATE_PROFILE, allow_redirects=follow_redirects,
|
||||
)
|
||||
return HttpReply(
|
||||
status=resp.status_code, text=resp.text, content=resp.content,
|
||||
content_type=resp.headers.get("content-type", ""), url=str(resp.url),
|
||||
)
|
||||
|
||||
|
||||
@typechecked
|
||||
async def p_plain(
|
||||
url: str, method: str, params: Optional[Dict], headers: Dict[str, str],
|
||||
timeout: float, follow_redirects: bool,
|
||||
) -> HttpReply:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=timeout, follow_redirects=follow_redirects, headers=headers,
|
||||
) as client:
|
||||
resp = await client.request(method, url, params=params)
|
||||
return HttpReply(
|
||||
status=resp.status_code, text=resp.text, content=resp.content,
|
||||
content_type=resp.headers.get("content-type", ""), url=str(resp.url),
|
||||
)
|
||||
|
||||
|
||||
@typechecked
|
||||
async def browser_request(
|
||||
url: str,
|
||||
*,
|
||||
method: str = "GET",
|
||||
params: Optional[Dict] = None,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
timeout: float = 10.0,
|
||||
follow_redirects: bool = True,
|
||||
) -> HttpReply:
|
||||
"""Fetch `url` looking like Chrome. Fixed hosts only; see the module docstring."""
|
||||
merged = dict(BROWSER_HEADERS)
|
||||
if headers:
|
||||
merged.update(headers)
|
||||
if impersonation_available():
|
||||
return await p_impersonated(url, method, params, merged, timeout, follow_redirects)
|
||||
return await p_plain(url, method, params, merged, timeout, follow_redirects)
|
||||
@@ -1,32 +1,33 @@
|
||||
"""DuckDuckGo web search: html endpoint primary, lite endpoint fallback.
|
||||
|
||||
The html endpoint is the richer parse; lite (see search_ddg_lite) covers the two
|
||||
ways html dies: a 202 throttle and silent markup drift. Only both endpoints
|
||||
throttling raises DDGRateLimited, so free search no longer has a single point
|
||||
of failure (the outage class that stranded subscription-only users on
|
||||
"No search backend is configured")."""
|
||||
ways html dies: the 202 bot challenge and silent markup drift. Only both
|
||||
endpoints challenging raises DDGRateLimited, so free search no longer has a
|
||||
single point of failure (the outage class that stranded subscription-only users
|
||||
on "No search backend is configured").
|
||||
|
||||
Both rungs go out through `browser_http`, whose Chrome TLS fingerprint is what
|
||||
actually decides whether DuckDuckGo answers; a plain httpx client scored 4/8 on
|
||||
the same queries this one scored 8/8 on."""
|
||||
|
||||
import html
|
||||
import re
|
||||
|
||||
import httpx
|
||||
|
||||
from backend.apps.agents.tools.browser_http import CHROME_UA
|
||||
from backend.apps.agents.tools.browser_http import browser_request
|
||||
from backend.apps.agents.tools.search_ddg_lite import search_ddg_lite
|
||||
|
||||
HTTP_TIMEOUT = 30
|
||||
USER_AGENT = (
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
)
|
||||
USER_AGENT = CHROME_UA
|
||||
|
||||
|
||||
class DDGRateLimited(Exception):
|
||||
"""Both DuckDuckGo endpoints answered with the throttle challenge (HTTP 202).
|
||||
"""Every DuckDuckGo frontend answered with the bot challenge (HTTP 202).
|
||||
|
||||
Distinct from 'genuinely zero hits' so the caller can fail over to another
|
||||
backend instead of reporting an empty search to the user. The throttle is
|
||||
per-IP and burst-triggered; once BOTH html and lite serve it, the only cure
|
||||
is a different backend or waiting it out."""
|
||||
Named for history; this is an anti-automation challenge keyed on the
|
||||
client's fingerprint, NOT a per-IP rate limit. Distinct from 'genuinely
|
||||
zero hits' so the caller can fail over to another backend instead of
|
||||
reporting an empty search to the user."""
|
||||
|
||||
|
||||
def strip_html(raw_html: str) -> str:
|
||||
@@ -41,24 +42,19 @@ def strip_html(raw_html: str) -> str:
|
||||
|
||||
async def search_ddg(query: str, num_results: int) -> str:
|
||||
"""Query DuckDuckGo's html endpoint and parse results; lite is the free fallback."""
|
||||
async with httpx.AsyncClient(
|
||||
timeout=HTTP_TIMEOUT,
|
||||
follow_redirects=True,
|
||||
headers={"User-Agent": USER_AGENT},
|
||||
) as client:
|
||||
resp = await client.post(
|
||||
"https://html.duckduckgo.com/html/",
|
||||
data={"q": query},
|
||||
)
|
||||
# DDG serves its throttle challenge as 202 (a ~14KB no-results page), which is a 2xx so raise_for_status() sails right past it. Before declaring rate-limited, try the lite frontend; only when BOTH throttle is free search actually dead.
|
||||
if resp.status_code == 202:
|
||||
lite = await search_ddg_lite(query, num_results)
|
||||
if lite is None:
|
||||
raise DDGRateLimited(query)
|
||||
return lite
|
||||
resp.raise_for_status()
|
||||
reply = await browser_request(
|
||||
"https://html.duckduckgo.com/html/", params={"q": query}, timeout=HTTP_TIMEOUT,
|
||||
)
|
||||
# DDG serves its bot challenge as 202 (a ~14KB no-results page), which is a 2xx so a status check sails right past it. Before giving up, try the lite frontend; only when BOTH challenge is free DDG actually dead.
|
||||
if reply.status == 202:
|
||||
lite = await search_ddg_lite(query, num_results)
|
||||
if lite is None:
|
||||
raise DDGRateLimited(query)
|
||||
return lite
|
||||
if reply.status >= 400:
|
||||
raise RuntimeError(f"DuckDuckGo html returned HTTP {reply.status}")
|
||||
|
||||
body = resp.text
|
||||
body = reply.text
|
||||
|
||||
result_blocks = re.findall(
|
||||
r'<div[^>]*class="[^"]*result[^"]*"[^>]*>(.*?)</div>\s*(?=<div[^>]*class="[^"]*result|$)',
|
||||
|
||||
@@ -1,23 +1,22 @@
|
||||
"""DuckDuckGo lite-endpoint search: the free fallback when html.duckduckgo.com
|
||||
throttles (HTTP 202) or its markup drifts. lite.duckduckgo.com is a separate
|
||||
frontend with simpler, stabler HTML and direct result URLs (no uddg redirect).
|
||||
"""DuckDuckGo lite-endpoint search: the fallback when html.duckduckgo.com
|
||||
serves its bot challenge (HTTP 202) or its markup drifts. lite.duckduckgo.com
|
||||
is a separate frontend with simpler, stabler HTML and direct result URLs (no
|
||||
uddg redirect).
|
||||
|
||||
Returns None on a throttle (caller decides whether that means rate-limited
|
||||
overall) and a formatted results string (possibly empty) on success."""
|
||||
Returns None on a challenge (caller decides whether that means every DDG
|
||||
frontend is closed) and a formatted results string (possibly empty) on
|
||||
success."""
|
||||
|
||||
import html
|
||||
import re
|
||||
from typing import List, Optional
|
||||
|
||||
import httpx
|
||||
from typeguard import typechecked
|
||||
|
||||
from backend.apps.agents.tools.browser_http import browser_request
|
||||
|
||||
P_LITE_URL = "https://lite.duckduckgo.com/lite/"
|
||||
P_TIMEOUT = 12.0
|
||||
P_USER_AGENT = (
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
|
||||
)
|
||||
P_TAG_RE = re.compile(r"<[^>]+>")
|
||||
# Lite uses single-quoted class attrs today; accept either quote style so a cosmetic flip doesn't kill the parser.
|
||||
P_LINK_RE = re.compile(
|
||||
@@ -52,14 +51,10 @@ def parse_lite_results(body: str, num_results: int) -> str:
|
||||
|
||||
@typechecked
|
||||
async def search_ddg_lite(query: str, num_results: int) -> Optional[str]:
|
||||
"""None = throttled (202), string = parsed results (may be empty on no hits)."""
|
||||
async with httpx.AsyncClient(
|
||||
timeout=P_TIMEOUT,
|
||||
follow_redirects=True,
|
||||
headers={"User-Agent": P_USER_AGENT},
|
||||
) as client:
|
||||
resp = await client.post(P_LITE_URL, data={"q": query})
|
||||
if resp.status_code == 202:
|
||||
return None
|
||||
resp.raise_for_status()
|
||||
return parse_lite_results(resp.text, num_results)
|
||||
"""None = bot challenge (202), string = parsed results (may be empty on no hits)."""
|
||||
reply = await browser_request(P_LITE_URL, params={"q": query}, timeout=P_TIMEOUT)
|
||||
if reply.status == 202:
|
||||
return None
|
||||
if reply.status >= 400:
|
||||
raise RuntimeError(f"DuckDuckGo lite returned HTTP {reply.status}")
|
||||
return parse_lite_results(reply.text, num_results)
|
||||
|
||||
+47
-18
@@ -1,5 +1,5 @@
|
||||
# This file was autogenerated by uv via the following command:
|
||||
# uv pip compile backend/requirements.txt --universal --python-version 3.13 --generate-hashes --output-file backend/requirements.lock
|
||||
# uv pip compile /Users/ericzeng/Downloads/openswarm/backend/requirements.txt --universal --python-version 3.13 --generate-hashes --python /Users/ericzeng/Downloads/openswarm/backend/.venv/bin/python --output-file /Users/ericzeng/Downloads/openswarm/backend/requirements.lock
|
||||
annotated-doc==0.0.4 \
|
||||
--hash=sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320 \
|
||||
--hash=sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4
|
||||
@@ -13,7 +13,7 @@ annotated-types==0.7.0 \
|
||||
anthropic==0.97.0 \
|
||||
--hash=sha256:021e79fd8e21e90ad94dc5ba2bbbd8b1599f424f5b1fab6c06204009cab764be \
|
||||
--hash=sha256:8a1a472dfabcfc0c52ff6a3eecf724ac7e07107a2f6e2367be55ceb42f5d5613
|
||||
# via -r backend/requirements.txt
|
||||
# via -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt
|
||||
anyio==4.13.0 \
|
||||
--hash=sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708 \
|
||||
--hash=sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc
|
||||
@@ -39,10 +39,11 @@ certifi==2026.5.20 \
|
||||
--hash=sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897 \
|
||||
--hash=sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d
|
||||
# via
|
||||
# curl-cffi
|
||||
# httpcore
|
||||
# httpx
|
||||
# trafilatura
|
||||
cffi==2.0.0 ; platform_python_implementation != 'PyPy' \
|
||||
cffi==2.0.0 \
|
||||
--hash=sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb \
|
||||
--hash=sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b \
|
||||
--hash=sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f \
|
||||
@@ -127,7 +128,9 @@ cffi==2.0.0 ; platform_python_implementation != 'PyPy' \
|
||||
--hash=sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5 \
|
||||
--hash=sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453 \
|
||||
--hash=sha256:fe562eb1a64e67dd297ccc4f5addea2501664954f2692b69a76449ec7913ecbf
|
||||
# via cryptography
|
||||
# via
|
||||
# cryptography
|
||||
# curl-cffi
|
||||
charset-normalizer==3.4.7 \
|
||||
--hash=sha256:007d05ec7321d12a40227aae9e2bc6dca73f3cb21058999a1df9e193555a9dcc \
|
||||
--hash=sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c \
|
||||
@@ -268,14 +271,14 @@ claude-agent-sdk==0.1.70 \
|
||||
--hash=sha256:955b8d57cc06247f6894bc65d1441ae66b4c7bda3b3fcc0cb7f140e0d48757f8 \
|
||||
--hash=sha256:c69019de2559650b2e8ae1d93f907f27f623748fe25b6f02b7f5dcf05e956f70 \
|
||||
--hash=sha256:e3c3ab7a0cfd64d40fa8d9b1cf3aac9f0c4b9b910cff92dd07154f75889d63f8
|
||||
# via -r backend/requirements.txt
|
||||
# via -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt
|
||||
click==8.4.1 \
|
||||
--hash=sha256:482be17c6991b8c19c5429a1e995d9b0efdbb63172824c41f99965dc0ade8ec2 \
|
||||
--hash=sha256:918b5633eddf6b41c32d4f454bf0de810065c74e3f7dbf8ee5452f8be88d3e96
|
||||
# via
|
||||
# rich-toolkit
|
||||
# uvicorn
|
||||
colorama==0.4.6 ; sys_platform == 'win32' \
|
||||
colorama==0.4.6 ; sys_platform == 'win32' or platform_system == 'Windows' \
|
||||
--hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \
|
||||
--hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6
|
||||
# via
|
||||
@@ -337,6 +340,29 @@ cryptography==48.0.0 \
|
||||
--hash=sha256:f5333311663ea94f75dd408665686aaf426563556bb5283554a3539177e03b8c \
|
||||
--hash=sha256:fdfef35d751d510fcef5252703621574364fec16418c4a1e5e1055248401054b
|
||||
# via pyjwt
|
||||
curl-cffi==0.15.0 \
|
||||
--hash=sha256:08c799b89740b9bc49c09fbc3d5907f13ac1f845ca52620507ef9466d4639dd5 \
|
||||
--hash=sha256:0b6c0543b993996670e9e4b78e305a2d60809d5681903ffb5568e21a387434d3 \
|
||||
--hash=sha256:1977e1e12cfb5c11352cbb74acef1bed24eb7d226dab61ca57c168c21acd4d61 \
|
||||
--hash=sha256:2b6c847d86283b07ae69bb72c82eb8a59242277142aa35b89850f89e792a02fc \
|
||||
--hash=sha256:408d6f14e346841cd889c2e0962832bb235ba3b6749ebf609f347f747da5e60f \
|
||||
--hash=sha256:41f80170ba844009273b2660da1964ec31e99e5719d16b3422ada87177e32e13 \
|
||||
--hash=sha256:4682dc38d4336e0eb0b185374db90a760efde63cbea994b4e63f3521d44c4c92 \
|
||||
--hash=sha256:582e570aa2586b96ed47cf4a17586b9a3c462cbe43f780487c3dc245c6ef1527 \
|
||||
--hash=sha256:5a0c1896a0d5a5ac1eb89cd24b008d2b718dd1df6fd2f75451b59ca66e49e572 \
|
||||
--hash=sha256:7b7a92767a888ee90147e18964b396d8435ff42737030d6fb00824ffd6094805 \
|
||||
--hash=sha256:7e63539d0d839d0a8c5eacf86229bc68c57803547f35e0db7ee0986328b478c3 \
|
||||
--hash=sha256:829cc357061ecb99cc2d406301f609a039e05665322f5c025ec67c38b0dc49ce \
|
||||
--hash=sha256:838e48212447d9c81364b04707a5c861daf08f8320f9ecb3406a8919d1d5c3b3 \
|
||||
--hash=sha256:967ad7355bd8e9586f8c2d02eaa99953747549e7ea4a9b25cd53353e6b67fe6d \
|
||||
--hash=sha256:9e5e69eee735f659287e2c84444319d68a1fa68dd37abf228943a4074864283a \
|
||||
--hash=sha256:a25620d9bf989c9c029a7d1642999c4c265abb0bad811deb2f77b0b5b2b12e5b \
|
||||
--hash=sha256:a6d57f8389273a3a1f94370473c74897467bcc36af0a17336989780c507fa43d \
|
||||
--hash=sha256:aa1323950224db24f4c510d010b3affa02196ca853fb424191fa917a513d3f4b \
|
||||
--hash=sha256:b624c7ce087bfda967a013ed0a64702a525444e5b6e97d23534d567ccc6525aa \
|
||||
--hash=sha256:bda66404010e9ed743b1b83c20c86f24fe21a9a6873e17479d6e67e29d8ded28 \
|
||||
--hash=sha256:ea0c67652bf6893d34ee0f82c944f37e488f6147e9421bef1771cc6545b02ded
|
||||
# via -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt
|
||||
dateparser==1.4.0 \
|
||||
--hash=sha256:7902b8e85d603494bf70a5a0b1decdddb2270b9c6e6b2bc8a57b93476c0df378 \
|
||||
--hash=sha256:97a21840d5ecdf7630c584f673338a5afac5dfe84f647baf4d7e8df98f9354a4
|
||||
@@ -360,7 +386,7 @@ email-validator==2.3.0 \
|
||||
fastapi==0.136.3 \
|
||||
--hash=sha256:3d2a69bdf04b7e9f3afa292c3bc7a98816bbfafa10bc9b45f3f3700d2f761620 \
|
||||
--hash=sha256:e487fae93ad408e6f47641ee4dfe389864fd7bec92e547ea8498fc13f43e83ab
|
||||
# via -r backend/requirements.txt
|
||||
# via -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt
|
||||
fastapi-cli==0.0.24 \
|
||||
--hash=sha256:1afc9c9e21d7ebc8a3ca5e31790cd8d837742be7e4f8b9236e99cb3451f0de00 \
|
||||
--hash=sha256:4a1f78ed798f106b4fee85ca93b85d8fe33c0a3570f775964d37edb80b8f0edc
|
||||
@@ -435,10 +461,11 @@ httpx==0.28.1 \
|
||||
--hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \
|
||||
--hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad
|
||||
# via
|
||||
# -r backend/requirements.txt
|
||||
# -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt
|
||||
# anthropic
|
||||
# fastapi
|
||||
# mcp
|
||||
# swarm-analytics
|
||||
httpx-sse==0.4.3 \
|
||||
--hash=sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc \
|
||||
--hash=sha256:9b1ed0127459a66014aec3c56bebd93da3c1bc8bb6618c8082039a44889a755d
|
||||
@@ -569,7 +596,7 @@ jsonschema==4.26.0 \
|
||||
--hash=sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326 \
|
||||
--hash=sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce
|
||||
# via
|
||||
# -r backend/requirements.txt
|
||||
# -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt
|
||||
# mcp
|
||||
jsonschema-specifications==2025.9.1 \
|
||||
--hash=sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe \
|
||||
@@ -918,8 +945,8 @@ pillow==12.2.0 \
|
||||
--hash=sha256:f490f9368b6fc026f021db16d7ec2fbf7d89e2edb42e8ec09d2c60505f5729c7 \
|
||||
--hash=sha256:fb043ee2f06b41473269765c2feae53fc2e2fbf96e5e22ca94fb5ad677856f06 \
|
||||
--hash=sha256:fc3d34d4a8fbec3e88a79b92e5465e0f9b842b628675850d860b8bd300b159f5
|
||||
# via -r backend/requirements.txt
|
||||
pycparser==3.0 ; implementation_name != 'PyPy' and platform_python_implementation != 'PyPy' \
|
||||
# via -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt
|
||||
pycparser==3.0 ; implementation_name != 'PyPy' \
|
||||
--hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \
|
||||
--hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992
|
||||
# via cffi
|
||||
@@ -927,12 +954,13 @@ pydantic==2.13.3 \
|
||||
--hash=sha256:6db14ac8dfc9a1e57f87ea2c0de670c251240f43cb0c30a5130e9720dc612927 \
|
||||
--hash=sha256:af09e9d1d09f4e7fe37145c1f577e1d61ceb9a41924bf0094a36506285d0a84d
|
||||
# via
|
||||
# -r backend/requirements.txt
|
||||
# -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt
|
||||
# anthropic
|
||||
# fastapi
|
||||
# mcp
|
||||
# pydantic-extra-types
|
||||
# pydantic-settings
|
||||
# swarm-analytics
|
||||
pydantic-core==2.46.3 \
|
||||
--hash=sha256:0087084960f209a9a4af50ecd1fb063d9ad3658c07bb81a7a53f452dacbfb2ba \
|
||||
--hash=sha256:031bb17f4885a43773c8c763089499f242aee2ea85cf17154168775dccdecf35 \
|
||||
@@ -1083,7 +1111,7 @@ python-dotenv==1.1.1 \
|
||||
--hash=sha256:31f23644fe2602f88ff55e1f5c79ba497e01224ee7737937930c448e4d0e24dc \
|
||||
--hash=sha256:a8a6399716257f45be6a007360200409fce5cda2661e3dec71d23dc15f6189ab
|
||||
# via
|
||||
# -r backend/requirements.txt
|
||||
# -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt
|
||||
# pydantic-settings
|
||||
# uvicorn
|
||||
python-multipart==0.0.29 \
|
||||
@@ -1319,6 +1347,7 @@ rich==15.0.0 \
|
||||
--hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb \
|
||||
--hash=sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36
|
||||
# via
|
||||
# curl-cffi
|
||||
# rich-toolkit
|
||||
# typer
|
||||
rich-toolkit==0.19.10 \
|
||||
@@ -1472,7 +1501,7 @@ starlette==1.1.0 \
|
||||
swarm-analytics==0.1.1 \
|
||||
--hash=sha256:49255c5a0962ba1eca3c14471b7df8746f4258d8cd83c8c73931e62381b2be8e \
|
||||
--hash=sha256:c1d368905a8b53a555bb53bd60c6707323fba4beff3ea2e79a8f83fb91b11cab
|
||||
# via -r backend/requirements.txt
|
||||
# via -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt
|
||||
tld==0.13.2 \
|
||||
--hash=sha256:9b8fdbdb880e7ba65b216a4937f2c94c49a7226723783d5838fc958ac76f4e0c \
|
||||
--hash=sha256:d983fa92b9d717400742fca844e29d5e18271079c7bcfabf66d01b39b4a14345
|
||||
@@ -1480,11 +1509,11 @@ tld==0.13.2 \
|
||||
trafilatura==2.0.0 \
|
||||
--hash=sha256:77eb5d1e993747f6f20938e1de2d840020719735690c840b9a1024803a4cd51d \
|
||||
--hash=sha256:ceb7094a6ecc97e72fea73c7dba36714c5c5b577b6470e4520dca893706d6247
|
||||
# via -r backend/requirements.txt
|
||||
# via -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt
|
||||
typeguard==4.4.2 \
|
||||
--hash=sha256:77a78f11f09777aeae7fa08585f33b5f4ef0e7335af40005b0c422ed398ff48c \
|
||||
--hash=sha256:a6f1065813e32ef365bc3b3f503af8a96f9dd4e0033a02c28c4a4983de8c6c49
|
||||
# via -r backend/requirements.txt
|
||||
# via -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt
|
||||
typer==0.26.1 \
|
||||
--hash=sha256:537d27ae686d82967f6383382a952cb32ba4768898541effccb69ca75bbd5d23 \
|
||||
--hash=sha256:933e4f0083521f3c57d6a5aedf3b073271b2f95a19761b171b494dd6fdb21ff6
|
||||
@@ -1510,7 +1539,7 @@ typing-inspection==0.4.2 \
|
||||
# mcp
|
||||
# pydantic
|
||||
# pydantic-settings
|
||||
tzdata==2026.2 ; sys_platform == 'win32' \
|
||||
tzdata==2026.2 ; platform_system == 'Windows' \
|
||||
--hash=sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10 \
|
||||
--hash=sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7
|
||||
# via tzlocal
|
||||
@@ -1518,7 +1547,7 @@ tzlocal==5.3.1 \
|
||||
--hash=sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd \
|
||||
--hash=sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d
|
||||
# via
|
||||
# -r backend/requirements.txt
|
||||
# -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt
|
||||
# dateparser
|
||||
urllib3==2.7.0 \
|
||||
--hash=sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c \
|
||||
|
||||
@@ -16,6 +16,8 @@ python-dotenv==1.1.1
|
||||
Pillow==12.2.0
|
||||
httpx==0.28.1
|
||||
trafilatura==2.0.0
|
||||
# curl_cffi: Chrome TLS-fingerprint impersonation for the keyless search rungs. Measured 8/8 against DuckDuckGo where plain httpx scored 4/8 on the same queries; abi3 wheels for macos arm64/x64 and win_amd64. Guarded import, so a missing wheel degrades to httpx instead of breaking the backend.
|
||||
curl_cffi==0.15.0
|
||||
# swarm-analytics: typed client for the product-analytics ingest; fire-and-forget so it never breaks the app.
|
||||
swarm-analytics==0.1.1
|
||||
# tzlocal: dev-mode fallback for resolving the user's IANA timezone when
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
"""The keyless rungs go out through ONE browser-shaped client.
|
||||
|
||||
Why it matters: DuckDuckGo's 202 challenge keys on the client's TLS
|
||||
fingerprint, not on headers or verb. Measured over 8 interleaved randomised
|
||||
rounds, plain httpx scored 4/8 and Chrome impersonation 8/8 on the same
|
||||
queries. These pin the seam and the degrade path.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
import backend.apps.agents.tools.browser_http as BH
|
||||
import backend.apps.agents.tools.search_ddg as SD
|
||||
import backend.apps.agents.tools.search_ddg_lite as SDL
|
||||
from backend.apps.agents.tools.browser_http import BROWSER_HEADERS, HttpReply, browser_request
|
||||
|
||||
|
||||
def p_reply(status=200, text="ok"):
|
||||
return HttpReply(status=status, text=text, content=text.encode(),
|
||||
content_type="text/html", url="https://x.example")
|
||||
|
||||
|
||||
def p_record_transports(monkeypatch):
|
||||
used = []
|
||||
|
||||
async def p_imp(*a, **k):
|
||||
used.append("impersonated")
|
||||
return p_reply()
|
||||
|
||||
async def p_pl(*a, **k):
|
||||
used.append("plain")
|
||||
return p_reply()
|
||||
|
||||
monkeypatch.setattr(BH, "p_impersonated", p_imp)
|
||||
monkeypatch.setattr(BH, "p_plain", p_pl)
|
||||
return used
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_impersonates_when_the_client_is_installed(monkeypatch):
|
||||
used = p_record_transports(monkeypatch)
|
||||
monkeypatch.setattr(BH, "impersonation_available", lambda: True)
|
||||
await browser_request("https://x.example")
|
||||
assert used == ["impersonated"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_degrades_to_httpx_instead_of_failing(monkeypatch):
|
||||
"""A missing wheel must cost us reliability, never the whole backend."""
|
||||
used = p_record_transports(monkeypatch)
|
||||
monkeypatch.setattr(BH, "impersonation_available", lambda: False)
|
||||
reply = await browser_request("https://x.example")
|
||||
assert used == ["plain"]
|
||||
assert reply.status == 200
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_caller_headers_win_over_the_defaults(monkeypatch):
|
||||
seen = {}
|
||||
|
||||
async def p_pl(url, method, params, headers, timeout, follow_redirects):
|
||||
seen.update(headers)
|
||||
return p_reply()
|
||||
|
||||
monkeypatch.setattr(BH, "p_plain", p_pl)
|
||||
monkeypatch.setattr(BH, "impersonation_available", lambda: False)
|
||||
await browser_request("https://x.example", headers={"Accept-Language": "de-DE"})
|
||||
assert seen["Accept-Language"] == "de-DE"
|
||||
assert seen["User-Agent"] == BROWSER_HEADERS["User-Agent"]
|
||||
|
||||
|
||||
def test_default_headers_look_like_a_real_navigation():
|
||||
# A bare User-Agent is the tell that gets a scraper challenged.
|
||||
for key in ("User-Agent", "Accept", "Accept-Language", "Sec-Fetch-Mode", "Upgrade-Insecure-Requests"):
|
||||
assert BROWSER_HEADERS.get(key)
|
||||
assert "Chrome/" in BROWSER_HEADERS["User-Agent"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ddg_rungs_send_the_query_as_a_get_param(monkeypatch):
|
||||
"""Pins the shape: a GET with params, through the shared seam, on both frontends."""
|
||||
calls = []
|
||||
|
||||
async def p_req(url, **kw):
|
||||
calls.append((url, kw.get("method", "GET"), kw.get("params")))
|
||||
return p_reply(202, "challenge")
|
||||
|
||||
monkeypatch.setattr(SD, "browser_request", p_req)
|
||||
monkeypatch.setattr(SDL, "browser_request", p_req)
|
||||
from backend.apps.agents.tools.web import DDGRateLimited
|
||||
with pytest.raises(DDGRateLimited):
|
||||
await SD.search_ddg("some query", 5)
|
||||
|
||||
assert [c[0] for c in calls] == [
|
||||
"https://html.duckduckgo.com/html/",
|
||||
"https://lite.duckduckgo.com/lite/",
|
||||
]
|
||||
for _, method, params in calls:
|
||||
assert method == "GET"
|
||||
assert params == {"q": "some query"}
|
||||
|
||||
|
||||
def test_curl_cffi_is_a_declared_dependency():
|
||||
"""It must be in BOTH files: the packaged python-env installs from the LOCK."""
|
||||
from pathlib import Path
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
assert "curl_cffi==" in (root / "requirements.txt").read_text()
|
||||
assert "curl-cffi==" in (root / "requirements.lock").read_text()
|
||||
@@ -9,39 +9,25 @@ These pin the two bugs that turned DDG into a flaky 'No results found' source:
|
||||
We mock the network so the test is deterministic and offline.
|
||||
"""
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
import backend.apps.agents.tools.search_ddg as SD
|
||||
import backend.apps.agents.tools.search_ddg_lite as SDL
|
||||
from backend.apps.agents.tools.browser_http import HttpReply
|
||||
from backend.apps.agents.tools.web import WebSearchTool, DDGRateLimited
|
||||
|
||||
|
||||
class p_FakeResp:
|
||||
def __init__(self, status_code: int, text: str):
|
||||
self.status_code = status_code
|
||||
self.text = text
|
||||
|
||||
def raise_for_status(self):
|
||||
if self.status_code >= 400:
|
||||
raise httpx.HTTPStatusError("err", request=None, response=None)
|
||||
def p_reply(status: int, text: str) -> HttpReply:
|
||||
return HttpReply(status=status, text=text, content=text.encode(),
|
||||
content_type="text/html", url="https://duckduckgo.example")
|
||||
|
||||
|
||||
class p_FakeClient:
|
||||
"""Stands in for httpx.AsyncClient; returns a canned response."""
|
||||
def __init__(self, resp: p_FakeResp):
|
||||
self.p_resp = resp
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *a):
|
||||
return False
|
||||
|
||||
async def post(self, *a, **k):
|
||||
return self.p_resp
|
||||
|
||||
|
||||
def p_patch_client(monkeypatch, resp: p_FakeResp):
|
||||
monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: p_FakeClient(resp))
|
||||
def p_patch_client(monkeypatch, reply: HttpReply):
|
||||
"""Mock the ONE seam every keyless rung goes through, whatever transport it picks."""
|
||||
async def p_req(url, **kw):
|
||||
return reply
|
||||
monkeypatch.setattr(SD, "browser_request", p_req)
|
||||
monkeypatch.setattr(SDL, "browser_request", p_req)
|
||||
|
||||
|
||||
# One real organic result + one sponsored (ad) row in DDG's html markup.
|
||||
@@ -59,14 +45,14 @@ P_HTML_WITH_AD = """
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_202_raises_rate_limited_not_empty(monkeypatch):
|
||||
p_patch_client(monkeypatch, p_FakeResp(202, "<html>throttle challenge, no results</html>"))
|
||||
p_patch_client(monkeypatch, p_reply(202, "<html>throttle challenge, no results</html>"))
|
||||
with pytest.raises(DDGRateLimited):
|
||||
await WebSearchTool.search_ddg("anything", 5)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execute_reports_rate_limit_clearly(monkeypatch):
|
||||
p_patch_client(monkeypatch, p_FakeResp(202, "throttle"))
|
||||
p_patch_client(monkeypatch, p_reply(202, "throttle"))
|
||||
parts = await WebSearchTool().execute({"query": "x", "num_results": 5}, None)
|
||||
msg = parts[0]["text"].lower()
|
||||
assert "rate-limit" in msg
|
||||
@@ -75,7 +61,7 @@ async def test_execute_reports_rate_limit_clearly(monkeypatch):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ads_are_stripped_real_results_kept(monkeypatch):
|
||||
p_patch_client(monkeypatch, p_FakeResp(200, P_HTML_WITH_AD))
|
||||
p_patch_client(monkeypatch, p_reply(200, P_HTML_WITH_AD))
|
||||
out = await WebSearchTool.search_ddg("topic", 5)
|
||||
assert "example.com/real" in out
|
||||
assert "Real Result Title" in out
|
||||
@@ -88,6 +74,6 @@ async def test_ads_are_stripped_real_results_kept(monkeypatch):
|
||||
@pytest.mark.asyncio
|
||||
async def test_genuinely_empty_is_not_a_rate_limit(monkeypatch):
|
||||
# 200 with no result blocks is a real empty result set, not a throttle.
|
||||
p_patch_client(monkeypatch, p_FakeResp(200, "<html><body>nothing here</body></html>"))
|
||||
p_patch_client(monkeypatch, p_reply(200, "<html><body>nothing here</body></html>"))
|
||||
out = await WebSearchTool.search_ddg("zxcvqwer no hits", 5)
|
||||
assert out == ""
|
||||
|
||||
@@ -12,9 +12,11 @@ Network is mocked; the lite fixture is the real markup shape captured live
|
||||
|
||||
import asyncio
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
import backend.apps.agents.tools.search_ddg as SD
|
||||
import backend.apps.agents.tools.search_ddg_lite as SDL
|
||||
from backend.apps.agents.tools.browser_http import HttpReply
|
||||
from backend.apps.agents.tools.web import WebSearchTool, DDGRateLimited
|
||||
from backend.apps.agents.tools.search_ddg_lite import parse_lite_results
|
||||
|
||||
@@ -34,36 +36,20 @@ P_LITE_BODY = """
|
||||
P_HTML_202_BODY = "anomaly detected, challenge page"
|
||||
|
||||
|
||||
class p_FakeResp:
|
||||
def __init__(self, status_code: int, text: str):
|
||||
self.status_code = status_code
|
||||
self.text = text
|
||||
|
||||
def raise_for_status(self):
|
||||
if self.status_code >= 400:
|
||||
raise httpx.HTTPStatusError("err", request=None, response=None)
|
||||
|
||||
|
||||
class p_RoutedClient:
|
||||
"""Fake AsyncClient that answers per-URL, so the html and lite endpoints can behave differently in one test."""
|
||||
def __init__(self, routes: dict):
|
||||
self.routes = routes
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *a):
|
||||
return False
|
||||
|
||||
async def post(self, url, *a, **k):
|
||||
for key, resp in self.routes.items():
|
||||
if key in url:
|
||||
return resp
|
||||
raise AssertionError(f"unexpected URL {url}")
|
||||
def p_FakeResp(status: int, text: str) -> HttpReply:
|
||||
return HttpReply(status=status, text=text, content=text.encode(),
|
||||
content_type="text/html", url="https://duckduckgo.example")
|
||||
|
||||
|
||||
def p_route(monkeypatch, routes: dict):
|
||||
monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: p_RoutedClient(routes))
|
||||
"""Answer per-URL so the html and lite endpoints can behave differently in one test."""
|
||||
async def p_req(url, **kw):
|
||||
for key, reply in routes.items():
|
||||
if key in url:
|
||||
return reply
|
||||
raise AssertionError(f"unexpected URL {url}")
|
||||
monkeypatch.setattr(SD, "browser_request", p_req)
|
||||
monkeypatch.setattr(SDL, "browser_request", p_req)
|
||||
|
||||
|
||||
def test_lite_parser_on_real_shape():
|
||||
|
||||
Reference in New Issue
Block a user