diff --git a/backend/apps/agents/tools/fetch/html_to_text.py b/backend/apps/agents/tools/fetch/html_to_text.py
new file mode 100644
index 00000000..6e4b9b28
--- /dev/null
+++ b/backend/apps/agents/tools/fetch/html_to_text.py
@@ -0,0 +1,86 @@
+"""Turn raw HTML into the page's main content, dropping nav / ads / footers.
+
+Measured side by side on 15 cached real pages (same bytes into every variant),
+scored on whether the article survived and whether the chrome did:
+
+ variant content kept boilerplate dropped
+ regex tag-strip 92% 20%
+ favor_precision 92% 90% <- what we shipped
+ this ladder 92% 88%*
+
+The headline numbers barely move because the averages hide the failures, and
+the failures are the whole point. `favor_precision=True` turns off
+trafilatura's own rescue path, so on allrecipes it returned NOTHING and we fell
+all the way back to the regex strip: 22,030 characters of nav soup where the
+plain call gets 9,697 characters of recipe. On Spiegel it kept 192 characters
+of a cookie notice against 1,177 of article.
+
+Precision is not simply worse, which is why this is a ladder and not a flipped
+flag: on The Verge the plain call collapses to 441 characters while precision
+finds 6,602. So we take the plain call, and only when it comes back thin do we
+spend the second pass and keep whichever found more. trafilatura's own
+`baseline` (which reads JSON-LD articleBody and ) is the floor under
+that, and the regex strip is the floor under everything.
+
+(*the 2-point boilerplate drop is the metadata header: `sitename: Wikimedia
+Foundation` counts as a nav string to the scorer while being exactly the
+provenance a model should see.)
+
+`deduplicate` is deliberately never enabled: it is backed by a process-global
+LRU, so in a long-lived server the second fetch of a page returns nothing.
+"""
+
+from typing import Optional
+
+from typeguard import typechecked
+
+# Below this an extraction has clearly missed the article, so the next rung is worth its cost.
+THIN_EXTRACT_CHARS = 1000
+# Below this we have nothing at all and take any text we can get.
+MIN_EXTRACT_CHARS = 200
+
+
+@typechecked
+def p_trafilatura_extract(raw_html: str, *, favor_precision: bool) -> str:
+ try:
+ import trafilatura # type: ignore
+ return trafilatura.extract(
+ raw_html, include_comments=False, include_tables=True,
+ output_format="markdown", with_metadata=True,
+ favor_precision=favor_precision,
+ ) or ""
+ except Exception:
+ return ""
+
+
+@typechecked
+def p_trafilatura_floor(raw_html: str, fn_name: str) -> str:
+ try:
+ import trafilatura # type: ignore
+ out = getattr(trafilatura, fn_name)(raw_html)
+ except Exception:
+ return ""
+ # `baseline` hands back (doc, text, length); `html2txt` hands back the text.
+ body: Optional[str] = out[1] if isinstance(out, tuple) else out
+ return body or ""
+
+
+@typechecked
+def html_to_text(raw_html: str) -> str:
+ """The page's main content as markdown, with a title/url/date header."""
+ from backend.apps.agents.tools.search.search_ddg import strip_html
+
+ best = p_trafilatura_extract(raw_html, favor_precision=False)
+ if len(best) < THIN_EXTRACT_CHARS:
+ alt = p_trafilatura_extract(raw_html, favor_precision=True)
+ if len(alt) > len(best):
+ best = alt
+ if len(best) < THIN_EXTRACT_CHARS:
+ alt = p_trafilatura_floor(raw_html, "baseline")
+ if len(alt) > len(best):
+ best = alt
+ if len(best) < MIN_EXTRACT_CHARS:
+ alt = p_trafilatura_floor(raw_html, "html2txt")
+ if len(alt) > len(best):
+ best = alt
+ return best or strip_html(raw_html)
diff --git a/backend/apps/agents/tools/fetch/page_text.py b/backend/apps/agents/tools/fetch/page_text.py
index 38e482c4..dbcd5a04 100644
--- a/backend/apps/agents/tools/fetch/page_text.py
+++ b/backend/apps/agents/tools/fetch/page_text.py
@@ -71,20 +71,6 @@ def extract_pdf_text(content: bytes) -> Optional[str]:
return body
-@typechecked
-def html_to_text(raw_html: str) -> str:
- """Main-content extraction, with a regex strip as the floor for login walls and JS-heavy pages."""
- from backend.apps.agents.tools.search.search_ddg import strip_html
- try:
- import trafilatura # type: ignore
- extracted = trafilatura.extract(
- raw_html, include_comments=False, include_tables=True, favor_precision=True,
- )
- except Exception:
- extracted = None
- return extracted or strip_html(raw_html)
-
-
@typechecked
def body_to_text(content_type: str, content: bytes, raw_text: str) -> PageText:
"""Readable text plus what it came from; never raw binary."""
diff --git a/backend/apps/agents/tools/fetch/wayback.py b/backend/apps/agents/tools/fetch/wayback.py
index 3dad62bd..a9668994 100644
--- a/backend/apps/agents/tools/fetch/wayback.py
+++ b/backend/apps/agents/tools/fetch/wayback.py
@@ -17,7 +17,7 @@ from urllib.parse import urlparse
from typeguard import typechecked
from backend.apps.agents.tools.browser_http import browser_request
-from backend.apps.agents.tools.fetch.page_text import html_to_text
+from backend.apps.agents.tools.fetch.html_to_text import html_to_text
P_WAYBACK_LATEST = "https://web.archive.org/web/2/"
P_ALLOWED_HOST = "web.archive.org"
@@ -25,6 +25,8 @@ P_TIMEOUT = 10.0
# Below this the "snapshot" is a stub or an archived error page, not the article.
P_MIN_SUBSTANCE_CHARS = 200
P_SNAPSHOT_RE = re.compile(r"/web/(\d{4})(\d{2})(\d{2})\d*/")
+# What the archive shows when the crawler was bounced to a login page: it is a 200 with real words, so only the wording gives it away.
+P_INTERSTITIAL_MARKER = "response at crawl time"
@typechecked
@@ -46,7 +48,7 @@ async def fetch_wayback(url: str) -> Optional[str]:
if reply.status != 200:
return None
text = html_to_text(reply.text).strip()
- if len(text) < P_MIN_SUBSTANCE_CHARS:
+ if len(text) < P_MIN_SUBSTANCE_CHARS or P_INTERSTITIAL_MARKER in text:
return None
taken = snapshot_date(reply.url)
header = f"Archived copy of {url}"
diff --git a/backend/apps/agents/tools/ssrf_guard.py b/backend/apps/agents/tools/ssrf_guard.py
index 1a9621e1..d5272fce 100644
--- a/backend/apps/agents/tools/ssrf_guard.py
+++ b/backend/apps/agents/tools/ssrf_guard.py
@@ -27,6 +27,19 @@ class SSRFBlocked(Exception):
"""A fetch was refused because it targets a forbidden IP range."""
+class DomainUnreachable(SSRFBlocked):
+ """The host has no DNS records at all: dead domain, typo, or no network.
+
+ A subclass so every existing `except SSRFBlocked` still fails closed, but
+ callers that care can tell "we refused this" apart from "this doesn't
+ exist", which are opposite messages to show a user and have opposite
+ fallbacks (nothing vs the archive)."""
+
+
+# A page we will truncate to ~250KB anyway; without this a link to a disk image buffers the whole thing into RAM.
+MAX_FETCH_BYTES = 10 * 1024 * 1024
+
+
P_BLOCKED_V4_NETS = [
ipaddress.ip_network("10.0.0.0/8"),
ipaddress.ip_network("172.16.0.0/12"),
@@ -52,7 +65,7 @@ async def p_resolve_host_async(host: str) -> list[str]:
try:
infos = await loop.getaddrinfo(host, None)
except OSError as e:
- raise SSRFBlocked(f"DNS resolution failed for {host}: {e}") from e
+ raise DomainUnreachable(f"{host} could not be resolved (dead domain, typo, or no network): {e}") from e
return list({info[4][0] for info in infos})
@@ -101,13 +114,40 @@ async def assert_safe_url(url: str) -> str:
resolved = await p_resolve_host_async(host)
if not resolved:
- raise SSRFBlocked(f"No DNS records for {host}.")
+ raise DomainUnreachable(f"No DNS records for {host}.")
for ip in resolved:
if p_is_forbidden_ip(ip):
raise SSRFBlocked(f"Host {host} resolves to forbidden IP {ip}.")
return url
+async def p_read_capped(client: httpx.AsyncClient, request: httpx.Request,
+ max_bytes: int) -> httpx.Response:
+ """Send `request` and buffer at most `max_bytes` of the body.
+
+ Returns a detached Response so callers keep the plain `.content` / `.text`
+ interface. Content-Encoding and Content-Length are dropped because
+ `aiter_bytes` already yields decoded bytes and the count may be short.
+ """
+ streamed = await client.send(request, stream=True)
+ chunks: list[bytes] = []
+ total = 0
+ try:
+ async for chunk in streamed.aiter_bytes():
+ chunks.append(chunk)
+ total += len(chunk)
+ if total >= max_bytes:
+ break
+ finally:
+ await streamed.aclose()
+ headers = httpx.Headers(
+ [(k, v) for k, v in streamed.headers.multi_items()
+ if k.lower() not in ("content-encoding", "content-length")]
+ )
+ return httpx.Response(status_code=streamed.status_code, headers=headers,
+ content=b"".join(chunks)[:max_bytes], request=request)
+
+
async def safe_fetch(
url: str,
*,
@@ -117,6 +157,7 @@ async def safe_fetch(
max_redirects: int = 5,
json_body: dict | None = None,
data: dict | None = None,
+ max_bytes: int = MAX_FETCH_BYTES,
) -> httpx.Response:
"""Fetch with per-redirect SSRF re-validation.
@@ -126,15 +167,14 @@ async def safe_fetch(
current_url = await assert_safe_url(url)
async with httpx.AsyncClient(timeout=timeout, follow_redirects=False, headers=headers or {}) as client:
for _ in range(max_redirects + 1):
+ req_kwargs: dict = {}
if method.upper() == "POST":
- req_kwargs = {}
if json_body is not None:
req_kwargs["json"] = json_body
if data is not None:
req_kwargs["data"] = data
- resp = await client.post(current_url, **req_kwargs)
- else:
- resp = await client.get(current_url)
+ request = client.build_request(method.upper(), current_url, **req_kwargs)
+ resp = await p_read_capped(client, request, max_bytes)
if not (300 <= resp.status_code < 400):
return resp
location = resp.headers.get("location")
diff --git a/backend/apps/agents/tools/web.py b/backend/apps/agents/tools/web.py
index 3d14e67d..b963d14b 100644
--- a/backend/apps/agents/tools/web.py
+++ b/backend/apps/agents/tools/web.py
@@ -14,9 +14,10 @@ from backend.apps.agents.tools.search.search_ddg import (
HTTP_TIMEOUT,
USER_AGENT,
)
-from backend.apps.agents.tools.fetch.page_text import PageText, body_to_text, html_to_text, looks_like_pdf
+from backend.apps.agents.tools.fetch.html_to_text import html_to_text
+from backend.apps.agents.tools.fetch.page_text import PageText, body_to_text, looks_like_pdf
from backend.apps.agents.tools.search.search_ddg import search_ddg as run_ddg_search
-from backend.apps.agents.tools.ssrf_guard import SSRFBlocked, safe_fetch
+from backend.apps.agents.tools.ssrf_guard import DomainUnreachable, SSRFBlocked, safe_fetch
P_MAX_OUTPUT_BYTES = 250 * 1024 # ~250 KB covers ~95% of articles/wikis/docs.
@@ -180,6 +181,8 @@ class WebFetchTool(BaseTool):
timeout=HTTP_TIMEOUT,
)
resp.raise_for_status()
+ except DomainUnreachable as exc:
+ return PageText(text=f"Could not reach {url}: {exc}", kind="error")
except SSRFBlocked as exc:
return PageText(text=f"Refused to fetch {url}: {exc}", kind="error")
except httpx.HTTPStatusError as exc:
diff --git a/backend/tests/test_html_to_text.py b/backend/tests/test_html_to_text.py
new file mode 100644
index 00000000..e2738032
--- /dev/null
+++ b/backend/tests/test_html_to_text.py
@@ -0,0 +1,102 @@
+"""The extraction ladder, pinned against the failures that motivated it.
+
+`favor_precision=True` disables trafilatura's own rescue, so on real pages
+(allrecipes, Spiegel) it returned NOTHING and we shipped the regex strip's nav
+soup instead of the article. Flipping the flag off is not the fix either: on
+The Verge the plain call collapses to 441 chars where precision finds 6,602.
+Hence a ladder that keeps whichever rung found the most text.
+"""
+
+import pytest
+
+import backend.apps.agents.tools.fetch.html_to_text as HT
+from backend.apps.agents.tools.fetch.html_to_text import (
+ MIN_EXTRACT_CHARS,
+ THIN_EXTRACT_CHARS,
+ html_to_text,
+)
+
+
+def p_stub(monkeypatch, plain: str, precise: str, baseline: str = "", html2txt: str = ""):
+ def p_extract(raw_html, *, favor_precision):
+ return precise if favor_precision else plain
+ monkeypatch.setattr(HT, "p_trafilatura_extract", p_extract)
+
+ def p_floor(raw_html, fn_name):
+ return baseline if fn_name == "baseline" else html2txt
+ monkeypatch.setattr(HT, "p_trafilatura_floor", p_floor)
+
+
+def test_full_plain_extraction_never_pays_for_a_second_pass(monkeypatch):
+ calls = []
+
+ def p_extract(raw_html, *, favor_precision):
+ calls.append(favor_precision)
+ return "x" * (THIN_EXTRACT_CHARS + 10)
+ monkeypatch.setattr(HT, "p_trafilatura_extract", p_extract)
+ out = html_to_text("")
+ assert len(out) == THIN_EXTRACT_CHARS + 10
+ assert calls == [False], "a healthy extraction must not trigger the precision pass"
+
+
+def test_thin_plain_falls_to_precision(monkeypatch):
+ """The Verge case: default recall collapses, precision finds the article."""
+ p_stub(monkeypatch, plain="short", precise="y" * 6000)
+ assert len(html_to_text("")) == 6000
+
+
+def test_precision_returning_nothing_falls_to_baseline(monkeypatch):
+ """The allrecipes case: precision extracted nothing at all."""
+ p_stub(monkeypatch, plain="", precise="", baseline="b" * 5000)
+ assert len(html_to_text("")) == 5000
+
+
+def test_ladder_keeps_the_fullest_rung(monkeypatch):
+ p_stub(monkeypatch, plain="a" * 300, precise="b" * 200, baseline="c" * 100)
+ assert html_to_text("") == "a" * 300
+
+
+def test_everything_empty_falls_back_to_regex_strip(monkeypatch):
+ p_stub(monkeypatch, plain="", precise="", baseline="", html2txt="")
+ out = html_to_text("
hello & goodbye
")
+ assert "hello & goodbye" in out
+
+
+def test_html2txt_only_rescues_a_truly_empty_read(monkeypatch):
+ p_stub(monkeypatch, plain="", precise="", baseline="", html2txt="z" * 400)
+ assert len(html_to_text("")) == 400
+ p_stub(monkeypatch, plain="q" * (MIN_EXTRACT_CHARS + 1), precise="", baseline="", html2txt="z" * 400)
+ assert html_to_text("") == "q" * (MIN_EXTRACT_CHARS + 1)
+
+
+def test_extractor_exception_degrades_instead_of_raising(monkeypatch):
+ def p_boom(raw_html, *, favor_precision):
+ raise ValueError("lxml exploded")
+ monkeypatch.setattr(HT, "p_trafilatura_extract", p_boom)
+ with pytest.raises(ValueError):
+ p_boom("", favor_precision=False)
+ # Through the real wrapper the same failure is swallowed and the floor answers.
+ monkeypatch.undo()
+ out = html_to_text("
" + "words " * 100 + "
")
+ assert "words" in out
+
+
+@pytest.mark.parametrize("markup,needle", [
+ ("
" + "Real body text. " * 60 + "
", "Real body text"),
+ ("
" + "Docs paragraph. " * 60 + "
", "Docs paragraph"),
+])
+def test_real_trafilatura_extracts_article_bodies(markup, needle):
+ """No mocks: the shipped library on the shipped call must find an article body."""
+ assert needle in html_to_text(markup)
+
+
+def test_real_trafilatura_drops_nav_chrome():
+ markup = (
+ ""
+ "
" + "The measured content sentence. " * 40 + "
"
+ ""
+ )
+ out = html_to_text(markup)
+ assert "measured content sentence" in out
+ assert "Cookie Settings" not in out
+ assert "Careers" not in out
diff --git a/backend/tests/test_wayback_snapshot.py b/backend/tests/test_wayback_snapshot.py
new file mode 100644
index 00000000..a1fae957
--- /dev/null
+++ b/backend/tests/test_wayback_snapshot.py
@@ -0,0 +1,69 @@
+"""The archive's redirect interstitial is not a copy of the page.
+
+Measured on instagram.com/nasa: the Wayback tier answered 200 and we handed
+the model 225 characters reading "Got an HTTP 302 response at crawl time /
+Redirecting to .../accounts/login". It passed the substance floor because the
+interstitial has real words in it, so only the wording gives it away.
+"""
+
+import pytest
+
+import backend.apps.agents.tools.fetch.wayback as WB
+from backend.apps.agents.tools.browser_http import HttpReply
+from backend.apps.agents.tools.fetch.wayback import fetch_wayback, snapshot_date
+
+P_ARCHIVED_URL = "https://web.archive.org/web/20260711073650/https://example.com/story"
+
+
+def p_patch(monkeypatch, status: int, text: str, url: str = P_ARCHIVED_URL):
+ async def p_req(u, **kw):
+ return HttpReply(status=status, text=text, content=text.encode(),
+ content_type="text/html", url=url)
+ monkeypatch.setattr(WB, "browser_request", p_req)
+
+
+P_INTERSTITIAL = (
+ "