WEBDIENSTE/openswarm
1
0
Fork 0
mirror of https://github.com/openswarm-ai/openswarm.git synced 2026-08-17 18:25:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

[eric] web: extract PDF text and refuse other binaries instead of dumping raw bytes into the model

Browse Source
This commit is contained in:
ciregenz
2026-07-30 14:40:48 -07:00
parent f30677e255
commit 88abab0c1c
5 changed files with 294 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/apps/agents/tools/fetch/page_text.py
+110
View File
@@ -0,0 +1,110 @@
"""Turn a fetched response body into text a model can actually read.
WebFetch used to hand anything non-HTML straight to the model as `resp.text`,
so fetching a PDF posted ~173KB-2MB of `%PDF-1.5` binary into the context
window: pure cost, zero information, and it pushed real content out. PDFs now
get their text layer extracted, and anything else that isn't textual is refused
with a message that says what it was instead of dumping its bytes."""
import io
from typing import Optional
from pydantic import BaseModel, ConfigDict
from typeguard import typechecked
# A 2.2MB, 15-page paper extracts to ~40K chars in 0.8s; this bounds a pathological book-sized PDF.
MAX_PDF_PAGES = 100
P_PDF_MAGIC = b"%PDF"
P_TEXTUAL_HINTS = ("text/", "json", "xml", "javascript", "csv", "yaml", "markdown")
class PageText(BaseModel):
model_config = ConfigDict(validate_assignment=True)
text: str
kind: str
@typechecked
def looks_like_pdf(content_type: str, content: bytes) -> bool:
"""Servers mislabel PDFs constantly, so the magic bytes get the final say."""
return "pdf" in content_type.lower() or content[:4].startswith(P_PDF_MAGIC)
@typechecked
def p_is_textual(content_type: str, content: bytes) -> bool:
if any(hint in content_type.lower() for hint in P_TEXTUAL_HINTS):
return True
sample = content[:4096]
if not sample:
return True
if b"\x00" in sample:
return False
printable = sum(1 for byte in sample if byte >= 32 or byte in (9, 10, 13))
return printable / len(sample) > 0.9
@typechecked
def p_describe_size(content: bytes) -> str:
kb = len(content) / 1024
return f"{kb:.0f} KB" if kb < 1024 else f"{kb / 1024:.1f} MB"
@typechecked
def extract_pdf_text(content: bytes) -> Optional[str]:
"""The PDF's text layer, or None when there isn't one we can read."""
try:
from pypdf import PdfReader
except Exception:
return None
try:
reader = PdfReader(io.BytesIO(content))
pages = reader.pages[:MAX_PDF_PAGES]
chunks = [(page.extract_text() or "").strip() for page in pages]
except Exception:
return None
body = "\n\n".join(chunk for chunk in chunks if chunk).strip()
if not body:
return None
if len(reader.pages) > MAX_PDF_PAGES:
body += f"\n\n... (first {MAX_PDF_PAGES} of {len(reader.pages)} pages)"
return body
@typechecked
def html_to_text(raw_html: str) -> str:
"""Main-content extraction, with a regex strip as the floor for login walls and JS-heavy pages."""
from backend.apps.agents.tools.search.search_ddg import strip_html
try:
import trafilatura # type: ignore
extracted = trafilatura.extract(
raw_html, include_comments=False, include_tables=True, favor_precision=True,
)
except Exception:
extracted = None
return extracted or strip_html(raw_html)
@typechecked
def body_to_text(content_type: str, content: bytes, raw_text: str) -> PageText:
"""Readable text plus what it came from; never raw binary."""
if looks_like_pdf(content_type, content):
extracted = extract_pdf_text(content)
if extracted:
return PageText(text=extracted, kind="pdf")
return PageText(
text=(
f"This URL is a PDF ({p_describe_size(content)}) with no extractable text layer; "
"it is probably a scan or is encrypted. Nothing was read from it."
),
kind="pdf_unreadable",
)
if p_is_textual(content_type, content):
return PageText(text=raw_text, kind="text")
return PageText(
text=(
f"This URL is not a readable document: {content_type or 'unknown type'}, "
f"{p_describe_size(content)} of binary data. Nothing was read from it."
),
kind="binary",
)
backend/apps/agents/tools/web.py
+10 -19
View File
@@ -13,8 +13,8 @@ from backend.apps.agents.tools.search.search_ddg import (
DDGRateLimited,
HTTP_TIMEOUT,
USER_AGENT,
strip_html,
)
from backend.apps.agents.tools.fetch.page_text import body_to_text, html_to_text, looks_like_pdf
from backend.apps.agents.tools.search.search_ddg import search_ddg as run_ddg_search
from backend.apps.agents.tools.ssrf_guard import SSRFBlocked, safe_fetch
@@ -126,9 +126,11 @@ class WebSearchTool(BaseTool):
return [{"type": "text", "text": f"No search results found for: {query}"}]
return [{"type": "text", "text": results}]
except DDGRateLimited:
# "Wait and retry" was a dead end: the 202 is an anti-automation challenge on the client, not a cooldown, so an immediate retry gets the same answer.
return [{"type": "text", "text": (
"DuckDuckGo is rate-limiting this network right now (HTTP 202). "
"Wait a bit and retry, or use a different search source."
"DuckDuckGo answered its bot challenge (HTTP 202) instead of results, on both "
"its html and lite frontends. Retrying the same query will not clear it; use "
"another search source."
)}]
except Exception as exc:
return [{"type": "text", "text": f"Web search error: {exc}"}]
@@ -182,25 +184,14 @@ class WebFetchTool(BaseTool):
return [{"type": "text", "text": f"Error fetching {url}: {exc}"}]
content_type = resp.headers.get("content-type", "")
is_html = "html" in content_type or resp.text.strip().startswith("<!")
# A PDF's content-type often says html, so check the magic bytes before trusting the header.
is_pdf = looks_like_pdf(content_type, resp.content)
is_html = not is_pdf and ("html" in content_type or resp.text.strip().startswith("<!"))
if is_html:
# Prefer trafilatura for main-content extraction; fall back to regex strip on apps/login walls/JS-heavy pages.
text: str | None = None
try:
import trafilatura # type: ignore
text = trafilatura.extract(
resp.text,
include_comments=False,
include_tables=True,
favor_precision=True,
)
except Exception:
text = None
if not text:
text = strip_html(resp.text)
text = html_to_text(resp.text)
else:
text = resp.text
text = body_to_text(content_type, resp.content, resp.text).text
text = p_truncate(text)
backend/requirements.lock
+18 -14
View File
@@ -1,5 +1,5 @@
# This file was autogenerated by uv via the following command:
# uv pip compile /Users/ericzeng/Downloads/openswarm/backend/requirements.txt --universal --python-version 3.13 --generate-hashes --python /Users/ericzeng/Downloads/openswarm/backend/.venv/bin/python --output-file /Users/ericzeng/Downloads/openswarm/backend/requirements.lock
# uv pip compile backend/requirements.txt --universal --python-version 3.13 --generate-hashes --output-file backend/requirements.lock
annotated-doc==0.0.4 \
--hash=sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320 \
--hash=sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4
@@ -13,7 +13,7 @@ annotated-types==0.7.0 \
anthropic==0.97.0 \
--hash=sha256:021e79fd8e21e90ad94dc5ba2bbbd8b1599f424f5b1fab6c06204009cab764be \
--hash=sha256:8a1a472dfabcfc0c52ff6a3eecf724ac7e07107a2f6e2367be55ceb42f5d5613
# via -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt
# via -r backend/requirements.txt
anyio==4.13.0 \
--hash=sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708 \
--hash=sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc
@@ -271,7 +271,7 @@ claude-agent-sdk==0.1.70 \
--hash=sha256:955b8d57cc06247f6894bc65d1441ae66b4c7bda3b3fcc0cb7f140e0d48757f8 \
--hash=sha256:c69019de2559650b2e8ae1d93f907f27f623748fe25b6f02b7f5dcf05e956f70 \
--hash=sha256:e3c3ab7a0cfd64d40fa8d9b1cf3aac9f0c4b9b910cff92dd07154f75889d63f8
# via -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt
# via -r backend/requirements.txt
click==8.4.1 \
--hash=sha256:482be17c6991b8c19c5429a1e995d9b0efdbb63172824c41f99965dc0ade8ec2 \
--hash=sha256:918b5633eddf6b41c32d4f454bf0de810065c74e3f7dbf8ee5452f8be88d3e96
@@ -362,7 +362,7 @@ curl-cffi==0.15.0 \
--hash=sha256:b624c7ce087bfda967a013ed0a64702a525444e5b6e97d23534d567ccc6525aa \
--hash=sha256:bda66404010e9ed743b1b83c20c86f24fe21a9a6873e17479d6e67e29d8ded28 \
--hash=sha256:ea0c67652bf6893d34ee0f82c944f37e488f6147e9421bef1771cc6545b02ded
# via -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt
# via -r backend/requirements.txt
dateparser==1.4.0 \
--hash=sha256:7902b8e85d603494bf70a5a0b1decdddb2270b9c6e6b2bc8a57b93476c0df378 \
--hash=sha256:97a21840d5ecdf7630c584f673338a5afac5dfe84f647baf4d7e8df98f9354a4
@@ -386,7 +386,7 @@ email-validator==2.3.0 \
fastapi==0.136.3 \
--hash=sha256:3d2a69bdf04b7e9f3afa292c3bc7a98816bbfafa10bc9b45f3f3700d2f761620 \
--hash=sha256:e487fae93ad408e6f47641ee4dfe389864fd7bec92e547ea8498fc13f43e83ab
# via -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt
# via -r backend/requirements.txt
fastapi-cli==0.0.24 \
--hash=sha256:1afc9c9e21d7ebc8a3ca5e31790cd8d837742be7e4f8b9236e99cb3451f0de00 \
--hash=sha256:4a1f78ed798f106b4fee85ca93b85d8fe33c0a3570f775964d37edb80b8f0edc
@@ -461,7 +461,7 @@ httpx==0.28.1 \
--hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \
--hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad
# via
# -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt
# -r backend/requirements.txt
# anthropic
# fastapi
# mcp
@@ -596,7 +596,7 @@ jsonschema==4.26.0 \
--hash=sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326 \
--hash=sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce
# via
# -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt
# -r backend/requirements.txt
# mcp
jsonschema-specifications==2025.9.1 \
--hash=sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe \
@@ -945,7 +945,7 @@ pillow==12.2.0 \
--hash=sha256:f490f9368b6fc026f021db16d7ec2fbf7d89e2edb42e8ec09d2c60505f5729c7 \
--hash=sha256:fb043ee2f06b41473269765c2feae53fc2e2fbf96e5e22ca94fb5ad677856f06 \
--hash=sha256:fc3d34d4a8fbec3e88a79b92e5465e0f9b842b628675850d860b8bd300b159f5
# via -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt
# via -r backend/requirements.txt
pycparser==3.0 ; implementation_name != 'PyPy' \
--hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \
--hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992
@@ -954,7 +954,7 @@ pydantic==2.13.3 \
--hash=sha256:6db14ac8dfc9a1e57f87ea2c0de670c251240f43cb0c30a5130e9720dc612927 \
--hash=sha256:af09e9d1d09f4e7fe37145c1f577e1d61ceb9a41924bf0094a36506285d0a84d
# via
# -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt
# -r backend/requirements.txt
# anthropic
# fastapi
# mcp
@@ -1101,6 +1101,10 @@ pyjwt==2.13.0 \
--hash=sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423 \
--hash=sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728
# via mcp
pypdf==6.14.2 \
--hash=sha256:3f07891af76dc002657e04993ab9b4de81de29f9013b9761d0b7968bff12e946 \
--hash=sha256:7873f502fe4385e79539b21d872392dc0c4e3714327c15881cbc7fbfd1f95b25
# via -r backend/requirements.txt
python-dateutil==2.9.0.post0 \
--hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \
--hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427
@@ -1111,7 +1115,7 @@ python-dotenv==1.1.1 \
--hash=sha256:31f23644fe2602f88ff55e1f5c79ba497e01224ee7737937930c448e4d0e24dc \
--hash=sha256:a8a6399716257f45be6a007360200409fce5cda2661e3dec71d23dc15f6189ab
# via
# -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt
# -r backend/requirements.txt
# pydantic-settings
# uvicorn
python-multipart==0.0.29 \
@@ -1501,7 +1505,7 @@ starlette==1.1.0 \
swarm-analytics==0.1.1 \
--hash=sha256:49255c5a0962ba1eca3c14471b7df8746f4258d8cd83c8c73931e62381b2be8e \
--hash=sha256:c1d368905a8b53a555bb53bd60c6707323fba4beff3ea2e79a8f83fb91b11cab
# via -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt
# via -r backend/requirements.txt
tld==0.13.2 \
--hash=sha256:9b8fdbdb880e7ba65b216a4937f2c94c49a7226723783d5838fc958ac76f4e0c \
--hash=sha256:d983fa92b9d717400742fca844e29d5e18271079c7bcfabf66d01b39b4a14345
@@ -1509,11 +1513,11 @@ tld==0.13.2 \
trafilatura==2.0.0 \
--hash=sha256:77eb5d1e993747f6f20938e1de2d840020719735690c840b9a1024803a4cd51d \
--hash=sha256:ceb7094a6ecc97e72fea73c7dba36714c5c5b577b6470e4520dca893706d6247
# via -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt
# via -r backend/requirements.txt
typeguard==4.4.2 \
--hash=sha256:77a78f11f09777aeae7fa08585f33b5f4ef0e7335af40005b0c422ed398ff48c \
--hash=sha256:a6f1065813e32ef365bc3b3f503af8a96f9dd4e0033a02c28c4a4983de8c6c49
# via -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt
# via -r backend/requirements.txt
typer==0.26.1 \
--hash=sha256:537d27ae686d82967f6383382a952cb32ba4768898541effccb69ca75bbd5d23 \
--hash=sha256:933e4f0083521f3c57d6a5aedf3b073271b2f95a19761b171b494dd6fdb21ff6
@@ -1547,7 +1551,7 @@ tzlocal==5.3.1 \
--hash=sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd \
--hash=sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d
# via
# -r /Users/ericzeng/Downloads/openswarm/backend/requirements.txt
# -r backend/requirements.txt
# dateparser
urllib3==2.7.0 \
--hash=sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c \
backend/requirements.txt
+2
View File
@@ -18,6 +18,8 @@ httpx==0.28.1
trafilatura==2.0.0
# curl_cffi: Chrome TLS-fingerprint impersonation for the keyless search rungs. Measured 8/8 against DuckDuckGo where plain httpx scored 4/8 on the same queries; abi3 wheels for macos arm64/x64 and win_amd64. Guarded import, so a missing wheel degrades to httpx instead of breaking the backend.
curl_cffi==0.15.0
# pypdf: pull the text layer out of a fetched PDF. Pure python, no transitive deps; without it WebFetch posts raw %PDF binary into the model's context.
pypdf==6.14.2
# swarm-analytics: typed client for the product-analytics ingest; fire-and-forget so it never breaks the app.
swarm-analytics==0.1.1
# tzlocal: dev-mode fallback for resolving the user's IANA timezone when
backend/tests/test_web_fetch_body.py
+154
View File
@@ -0,0 +1,154 @@
"""WebFetch must never post raw binary into the model's context.
The field bug: fetching a PDF handed the model megabytes of `%PDF-1.5`
gibberish, which cost real money, told it nothing, and pushed real content out
of the window. Same class for images and any other binary body.
"""
import pytest
import backend.apps.agents.tools.fetch.wayback as WB
from backend.apps.agents.tools.fetch.page_text import (
MAX_PDF_PAGES,
body_to_text,
extract_pdf_text,
looks_like_pdf,
)
from backend.apps.agents.tools.browser_http import HttpReply
from backend.apps.agents.tools.fetch.wayback import fetch_wayback, snapshot_date
def p_minimal_pdf(text: str = "Hello from a real PDF") -> bytes:
"""A genuine 600-byte one-page PDF with a real font resource and xref table."""
stream = f"BT /F1 18 Tf 20 100 Td ({text}) Tj ET".encode()
objects = [
b"<< /Type /Catalog /Pages 2 0 R >>",
b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R "
b"/Resources << /Font << /F1 5 0 R >> >> >>",
b"<< /Length " + str(len(stream)).encode() + b" >>\nstream\n" + stream + b"\nendstream",
b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
]
out = bytearray(b"%PDF-1.4\n")
offsets = []
for i, body in enumerate(objects, start=1):
offsets.append(len(out))
out += f"{i} 0 obj\n".encode() + body + b"\nendobj\n"
xref = len(out)
out += f"xref\n0 {len(objects) + 1}\n".encode() + b"0000000000 65535 f \n"
for offset in offsets:
out += f"{offset:010d} 00000 n \n".encode()
out += f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n".encode()
return bytes(out)
# ------------------------------------------------------------------ PDF
def test_pdf_is_detected_by_magic_bytes_not_just_the_header():
# Servers mislabel PDFs as text/html constantly.
assert looks_like_pdf("text/html", b"%PDF-1.7\n...") is True
assert looks_like_pdf("application/pdf", b"") is True
assert looks_like_pdf("text/html", b"<!doctype html>") is False
def test_a_real_pdf_yields_its_text_not_its_bytes():
raw = p_minimal_pdf("Attention Is All You Need")
out = body_to_text("application/pdf", raw, raw.decode("latin-1"))
assert out.kind == "pdf"
assert "Attention Is All You Need" in out.text
assert "%PDF" not in out.text
def test_a_pdf_with_no_text_layer_says_so_instead_of_dumping_it():
fake = b"%PDF-1.4\n" + b"\x00\x01\x02" * 500
out = body_to_text("application/pdf", fake, "ignored")
assert out.kind == "pdf_unreadable"
assert "no extractable text layer" in out.text
assert "\x00" not in out.text
def test_extract_returns_none_rather_than_raising_on_garbage():
assert extract_pdf_text(b"not a pdf at all") is None
def test_page_cap_is_bounded():
assert 0 < MAX_PDF_PAGES <= 500
# ------------------------------------------------------------------ other binaries
def test_an_image_is_refused_with_a_description_not_its_bytes():
png = b"\x89PNG\r\n\x1a\n" + bytes(range(256)) * 40
out = body_to_text("image/png", png, png.decode("latin-1"))
assert out.kind == "binary"
assert "image/png" in out.text
assert "\x89PNG" not in out.text
assert len(out.text) < 400
def test_json_and_plain_text_still_pass_through_verbatim():
payload = '{"stars": 68000, "name": "cpython"}'
out = body_to_text("application/json", payload.encode(), payload)
assert out.kind == "text"
assert out.text == payload
def test_mislabelled_text_is_still_treated_as_text():
# application/octet-stream on a plain-text file is common; the bytes get the final say.
payload = "name,value\nalpha,1\nbeta,2\n"
out = body_to_text("application/octet-stream", payload.encode(), payload)
assert out.kind == "text"
assert out.text == payload
# ------------------------------------------------------------------ wayback
def p_wayback_reply(monkeypatch, status: int, text: str, url: str):
async def p_req(target, **kw):
return HttpReply(status=status, text=text, content=text.encode(),
content_type="text/html", url=url)
monkeypatch.setattr(WB, "browser_request", p_req)
P_ARCHIVED = "<html><body><article>" + ("The original article text. " * 40) + "</article></body></html>"
def test_snapshot_date_is_read_from_the_archive_url():
assert snapshot_date("https://web.archive.org/web/20260728200922/https://x.example/") == "2026-07-28"
assert snapshot_date("https://web.archive.org/web/2/https://x.example/") is None
@pytest.mark.asyncio
async def test_a_dead_link_is_answered_from_the_archive(monkeypatch):
p_wayback_reply(monkeypatch, 200, P_ARCHIVED,
"https://web.archive.org/web/20260508082837/https://gone.example/post")
out = await fetch_wayback("https://gone.example/post")
assert out is not None
assert "The original article text." in out
# the model must know it is reading a snapshot, and from when
assert "2026-05-08" in out
assert "Archived copy" in out
@pytest.mark.asyncio
async def test_no_snapshot_reads_as_no_answer(monkeypatch):
p_wayback_reply(monkeypatch, 404, "not archived",
"https://web.archive.org/web/2/https://gone.example/post")
assert await fetch_wayback("https://gone.example/post") is None
@pytest.mark.asyncio
async def test_a_stub_snapshot_is_not_passed_off_as_the_page(monkeypatch):
p_wayback_reply(monkeypatch, 200, "<html><body>tiny</body></html>",
"https://web.archive.org/web/20260101000000/https://x.example/")
assert await fetch_wayback("https://x.example/") is None
@pytest.mark.asyncio
async def test_a_redirect_off_the_archive_is_refused(monkeypatch):
"""We hand the archive a caller-supplied URL, so we confirm where we landed."""
p_wayback_reply(monkeypatch, 200, P_ARCHIVED, "http://127.0.0.1:8324/api/settings")
assert await fetch_wayback("https://x.example/") is None