From 88abab0c1c5f29918c3526d402a584cd6140088a Mon Sep 17 00:00:00 2001 From: ciregenz Date: Thu, 30 Jul 2026 14:40:48 -0700 Subject: [PATCH] [eric] web: extract PDF text and refuse other binaries instead of dumping raw bytes into the model --- backend/apps/agents/tools/fetch/page_text.py | 110 +++++++++++++ backend/apps/agents/tools/web.py | 29 ++-- backend/requirements.lock | 32 ++-- backend/requirements.txt | 2 + backend/tests/test_web_fetch_body.py | 154 +++++++++++++++++++ 5 files changed, 294 insertions(+), 33 deletions(-) create mode 100644 backend/apps/agents/tools/fetch/page_text.py create mode 100644 backend/tests/test_web_fetch_body.py diff --git a/backend/apps/agents/tools/fetch/page_text.py b/backend/apps/agents/tools/fetch/page_text.py new file mode 100644 index 00000000..38e482c4 --- /dev/null +++ b/backend/apps/agents/tools/fetch/page_text.py @@ -0,0 +1,110 @@ +"""Turn a fetched response body into text a model can actually read. + +WebFetch used to hand anything non-HTML straight to the model as `resp.text`, +so fetching a PDF posted ~173KB-2MB of `%PDF-1.5` binary into the context +window: pure cost, zero information, and it pushed real content out. PDFs now +get their text layer extracted, and anything else that isn't textual is refused +with a message that says what it was instead of dumping its bytes.""" + +import io +from typing import Optional + +from pydantic import BaseModel, ConfigDict +from typeguard import typechecked + +# A 2.2MB, 15-page paper extracts to ~40K chars in 0.8s; this bounds a pathological book-sized PDF. +MAX_PDF_PAGES = 100 +P_PDF_MAGIC = b"%PDF" +P_TEXTUAL_HINTS = ("text/", "json", "xml", "javascript", "csv", "yaml", "markdown") + + +class PageText(BaseModel): + model_config = ConfigDict(validate_assignment=True) + + text: str + kind: str + + +@typechecked +def looks_like_pdf(content_type: str, content: bytes) -> bool: + """Servers mislabel PDFs constantly, so the magic bytes get the final say.""" + return "pdf" in content_type.lower() or content[:4].startswith(P_PDF_MAGIC) + + +@typechecked +def p_is_textual(content_type: str, content: bytes) -> bool: + if any(hint in content_type.lower() for hint in P_TEXTUAL_HINTS): + return True + sample = content[:4096] + if not sample: + return True + if b"\x00" in sample: + return False + printable = sum(1 for byte in sample if byte >= 32 or byte in (9, 10, 13)) + return printable / len(sample) > 0.9 + + +@typechecked +def p_describe_size(content: bytes) -> str: + kb = len(content) / 1024 + return f"{kb:.0f} KB" if kb < 1024 else f"{kb / 1024:.1f} MB" + + +@typechecked +def extract_pdf_text(content: bytes) -> Optional[str]: + """The PDF's text layer, or None when there isn't one we can read.""" + try: + from pypdf import PdfReader + except Exception: + return None + try: + reader = PdfReader(io.BytesIO(content)) + pages = reader.pages[:MAX_PDF_PAGES] + chunks = [(page.extract_text() or "").strip() for page in pages] + except Exception: + return None + body = "\n\n".join(chunk for chunk in chunks if chunk).strip() + if not body: + return None + if len(reader.pages) > MAX_PDF_PAGES: + body += f"\n\n... (first {MAX_PDF_PAGES} of {len(reader.pages)} pages)" + return body + + +@typechecked +def html_to_text(raw_html: str) -> str: + """Main-content extraction, with a regex strip as the floor for login walls and JS-heavy pages.""" + from backend.apps.agents.tools.search.search_ddg import strip_html + try: + import trafilatura # type: ignore + extracted = trafilatura.extract( + raw_html, include_comments=False, include_tables=True, favor_precision=True, + ) + except Exception: + extracted = None + return extracted or strip_html(raw_html) + + +@typechecked +def body_to_text(content_type: str, content: bytes, raw_text: str) -> PageText: + """Readable text plus what it came from; never raw binary.""" + if looks_like_pdf(content_type, content): + extracted = extract_pdf_text(content) + if extracted: + return PageText(text=extracted, kind="pdf") + return PageText( + text=( + f"This URL is a PDF ({p_describe_size(content)}) with no extractable text layer; " + "it is probably a scan or is encrypted. Nothing was read from it." + ), + kind="pdf_unreadable", + ) + if p_is_textual(content_type, content): + return PageText(text=raw_text, kind="text") + return PageText( + text=( + f"This URL is not a readable document: {content_type or 'unknown type'}, " + f"{p_describe_size(content)} of binary data. Nothing was read from it." + ), + kind="binary", + ) diff --git a/backend/apps/agents/tools/web.py b/backend/apps/agents/tools/web.py index 73da5d2a..c3c63f71 100644 --- a/backend/apps/agents/tools/web.py +++ b/backend/apps/agents/tools/web.py @@ -13,8 +13,8 @@ from backend.apps.agents.tools.search.search_ddg import ( DDGRateLimited, HTTP_TIMEOUT, USER_AGENT, - strip_html, ) +from backend.apps.agents.tools.fetch.page_text import body_to_text, html_to_text, looks_like_pdf from backend.apps.agents.tools.search.search_ddg import search_ddg as run_ddg_search from backend.apps.agents.tools.ssrf_guard import SSRFBlocked, safe_fetch @@ -126,9 +126,11 @@ class WebSearchTool(BaseTool): return [{"type": "text", "text": f"No search results found for: {query}"}] return [{"type": "text", "text": results}] except DDGRateLimited: + # "Wait and retry" was a dead end: the 202 is an anti-automation challenge on the client, not a cooldown, so an immediate retry gets the same answer. return [{"type": "text", "text": ( - "DuckDuckGo is rate-limiting this network right now (HTTP 202). " - "Wait a bit and retry, or use a different search source." + "DuckDuckGo answered its bot challenge (HTTP 202) instead of results, on both " + "its html and lite frontends. Retrying the same query will not clear it; use " + "another search source." )}] except Exception as exc: return [{"type": "text", "text": f"Web search error: {exc}"}] @@ -182,25 +184,14 @@ class WebFetchTool(BaseTool): return [{"type": "text", "text": f"Error fetching {url}: {exc}"}] content_type = resp.headers.get("content-type", "") - is_html = "html" in content_type or resp.text.strip().startswith(" bytes: + """A genuine 600-byte one-page PDF with a real font resource and xref table.""" + stream = f"BT /F1 18 Tf 20 100 Td ({text}) Tj ET".encode() + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R " + b"/Resources << /Font << /F1 5 0 R >> >> >>", + b"<< /Length " + str(len(stream)).encode() + b" >>\nstream\n" + stream + b"\nendstream", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + ] + out = bytearray(b"%PDF-1.4\n") + offsets = [] + for i, body in enumerate(objects, start=1): + offsets.append(len(out)) + out += f"{i} 0 obj\n".encode() + body + b"\nendobj\n" + xref = len(out) + out += f"xref\n0 {len(objects) + 1}\n".encode() + b"0000000000 65535 f \n" + for offset in offsets: + out += f"{offset:010d} 00000 n \n".encode() + out += f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n".encode() + return bytes(out) + + +# ------------------------------------------------------------------ PDF + + +def test_pdf_is_detected_by_magic_bytes_not_just_the_header(): + # Servers mislabel PDFs as text/html constantly. + assert looks_like_pdf("text/html", b"%PDF-1.7\n...") is True + assert looks_like_pdf("application/pdf", b"") is True + assert looks_like_pdf("text/html", b"") is False + + +def test_a_real_pdf_yields_its_text_not_its_bytes(): + raw = p_minimal_pdf("Attention Is All You Need") + out = body_to_text("application/pdf", raw, raw.decode("latin-1")) + assert out.kind == "pdf" + assert "Attention Is All You Need" in out.text + assert "%PDF" not in out.text + + +def test_a_pdf_with_no_text_layer_says_so_instead_of_dumping_it(): + fake = b"%PDF-1.4\n" + b"\x00\x01\x02" * 500 + out = body_to_text("application/pdf", fake, "ignored") + assert out.kind == "pdf_unreadable" + assert "no extractable text layer" in out.text + assert "\x00" not in out.text + + +def test_extract_returns_none_rather_than_raising_on_garbage(): + assert extract_pdf_text(b"not a pdf at all") is None + + +def test_page_cap_is_bounded(): + assert 0 < MAX_PDF_PAGES <= 500 + + +# ------------------------------------------------------------------ other binaries + + +def test_an_image_is_refused_with_a_description_not_its_bytes(): + png = b"\x89PNG\r\n\x1a\n" + bytes(range(256)) * 40 + out = body_to_text("image/png", png, png.decode("latin-1")) + assert out.kind == "binary" + assert "image/png" in out.text + assert "\x89PNG" not in out.text + assert len(out.text) < 400 + + +def test_json_and_plain_text_still_pass_through_verbatim(): + payload = '{"stars": 68000, "name": "cpython"}' + out = body_to_text("application/json", payload.encode(), payload) + assert out.kind == "text" + assert out.text == payload + + +def test_mislabelled_text_is_still_treated_as_text(): + # application/octet-stream on a plain-text file is common; the bytes get the final say. + payload = "name,value\nalpha,1\nbeta,2\n" + out = body_to_text("application/octet-stream", payload.encode(), payload) + assert out.kind == "text" + assert out.text == payload + + +# ------------------------------------------------------------------ wayback + + +def p_wayback_reply(monkeypatch, status: int, text: str, url: str): + async def p_req(target, **kw): + return HttpReply(status=status, text=text, content=text.encode(), + content_type="text/html", url=url) + monkeypatch.setattr(WB, "browser_request", p_req) + + +P_ARCHIVED = "
" + ("The original article text. " * 40) + "
" + + +def test_snapshot_date_is_read_from_the_archive_url(): + assert snapshot_date("https://web.archive.org/web/20260728200922/https://x.example/") == "2026-07-28" + assert snapshot_date("https://web.archive.org/web/2/https://x.example/") is None + + +@pytest.mark.asyncio +async def test_a_dead_link_is_answered_from_the_archive(monkeypatch): + p_wayback_reply(monkeypatch, 200, P_ARCHIVED, + "https://web.archive.org/web/20260508082837/https://gone.example/post") + out = await fetch_wayback("https://gone.example/post") + assert out is not None + assert "The original article text." in out + # the model must know it is reading a snapshot, and from when + assert "2026-05-08" in out + assert "Archived copy" in out + + +@pytest.mark.asyncio +async def test_no_snapshot_reads_as_no_answer(monkeypatch): + p_wayback_reply(monkeypatch, 404, "not archived", + "https://web.archive.org/web/2/https://gone.example/post") + assert await fetch_wayback("https://gone.example/post") is None + + +@pytest.mark.asyncio +async def test_a_stub_snapshot_is_not_passed_off_as_the_page(monkeypatch): + p_wayback_reply(monkeypatch, 200, "tiny", + "https://web.archive.org/web/20260101000000/https://x.example/") + assert await fetch_wayback("https://x.example/") is None + + +@pytest.mark.asyncio +async def test_a_redirect_off_the_archive_is_refused(monkeypatch): + """We hand the archive a caller-supplied URL, so we confirm where we landed.""" + p_wayback_reply(monkeypatch, 200, P_ARCHIVED, "http://127.0.0.1:8324/api/settings") + assert await fetch_wayback("https://x.example/") is None