[eric] web: extract the article instead of the nav bar, and stop calling a dead domain a security refusal

backend/apps/agents/tools/fetch/html_to_text.py

@@ -0,0 +1,86 @@
+"""Turn raw HTML into the page's main content, dropping nav / ads / footers.
+
+Measured side by side on 15 cached real pages (same bytes into every variant),
+scored on whether the article survived and whether the chrome did:
+
+    variant            content kept   boilerplate dropped
+    regex tag-strip        92%              20%
+    favor_precision        92%              90%   <- what we shipped
+    this ladder            92%              88%*
+
+The headline numbers barely move because the averages hide the failures, and
+the failures are the whole point. `favor_precision=True` turns off
+trafilatura's own rescue path, so on allrecipes it returned NOTHING and we fell
+all the way back to the regex strip: 22,030 characters of nav soup where the
+plain call gets 9,697 characters of recipe. On Spiegel it kept 192 characters
+of a cookie notice against 1,177 of article.
+
+Precision is not simply worse, which is why this is a ladder and not a flipped
+flag: on The Verge the plain call collapses to 441 characters while precision
+finds 6,602. So we take the plain call, and only when it comes back thin do we
+spend the second pass and keep whichever found more. trafilatura's own
+`baseline` (which reads JSON-LD articleBody and <article>) is the floor under
+that, and the regex strip is the floor under everything.
+
+(*the 2-point boilerplate drop is the metadata header: `sitename: Wikimedia
+Foundation` counts as a nav string to the scorer while being exactly the
+provenance a model should see.)
+
+`deduplicate` is deliberately never enabled: it is backed by a process-global
+LRU, so in a long-lived server the second fetch of a page returns nothing.
+"""
+
+from typing import Optional
+
+from typeguard import typechecked
+
+# Below this an extraction has clearly missed the article, so the next rung is worth its cost.
+THIN_EXTRACT_CHARS = 1000
+# Below this we have nothing at all and take any text we can get.
+MIN_EXTRACT_CHARS = 200
+
+
+@typechecked
+def p_trafilatura_extract(raw_html: str, *, favor_precision: bool) -> str:
+    try:
+        import trafilatura  # type: ignore
+        return trafilatura.extract(
+            raw_html, include_comments=False, include_tables=True,
+            output_format="markdown", with_metadata=True,
+            favor_precision=favor_precision,
+        ) or ""
+    except Exception:
+        return ""
+
+
+@typechecked
+def p_trafilatura_floor(raw_html: str, fn_name: str) -> str:
+    try:
+        import trafilatura  # type: ignore
+        out = getattr(trafilatura, fn_name)(raw_html)
+    except Exception:
+        return ""
+    # `baseline` hands back (doc, text, length); `html2txt` hands back the text.
+    body: Optional[str] = out[1] if isinstance(out, tuple) else out
+    return body or ""
+
+
+@typechecked
+def html_to_text(raw_html: str) -> str:
+    """The page's main content as markdown, with a title/url/date header."""
+    from backend.apps.agents.tools.search.search_ddg import strip_html
+
+    best = p_trafilatura_extract(raw_html, favor_precision=False)
+    if len(best) < THIN_EXTRACT_CHARS:
+        alt = p_trafilatura_extract(raw_html, favor_precision=True)
+        if len(alt) > len(best):
+            best = alt
+    if len(best) < THIN_EXTRACT_CHARS:
+        alt = p_trafilatura_floor(raw_html, "baseline")
+        if len(alt) > len(best):
+            best = alt
+    if len(best) < MIN_EXTRACT_CHARS:
+        alt = p_trafilatura_floor(raw_html, "html2txt")
+        if len(alt) > len(best):
+            best = alt
+    return best or strip_html(raw_html)

backend/apps/agents/tools/fetch/page_text.py

@@ -71,20 +71,6 @@ def extract_pdf_text(content: bytes) -> Optional[str]:
     return body
 
 
-@typechecked
-def html_to_text(raw_html: str) -> str:
-    """Main-content extraction, with a regex strip as the floor for login walls and JS-heavy pages."""
-    from backend.apps.agents.tools.search.search_ddg import strip_html
-    try:
-        import trafilatura  # type: ignore
-        extracted = trafilatura.extract(
-            raw_html, include_comments=False, include_tables=True, favor_precision=True,
-        )
-    except Exception:
-        extracted = None
-    return extracted or strip_html(raw_html)
-
-
 @typechecked
 def body_to_text(content_type: str, content: bytes, raw_text: str) -> PageText:
     """Readable text plus what it came from; never raw binary."""

backend/apps/agents/tools/fetch/wayback.py

@@ -17,7 +17,7 @@ from urllib.parse import urlparse
 from typeguard import typechecked
 
 from backend.apps.agents.tools.browser_http import browser_request
-from backend.apps.agents.tools.fetch.page_text import html_to_text
+from backend.apps.agents.tools.fetch.html_to_text import html_to_text
 
 P_WAYBACK_LATEST = "https://web.archive.org/web/2/"
 P_ALLOWED_HOST = "web.archive.org"
@@ -25,6 +25,8 @@ P_TIMEOUT = 10.0
 # Below this the "snapshot" is a stub or an archived error page, not the article.
 P_MIN_SUBSTANCE_CHARS = 200
 P_SNAPSHOT_RE = re.compile(r"/web/(\d{4})(\d{2})(\d{2})\d*/")
+# What the archive shows when the crawler was bounced to a login page: it is a 200 with real words, so only the wording gives it away.
+P_INTERSTITIAL_MARKER = "response at crawl time"
 
 
 @typechecked
@@ -46,7 +48,7 @@ async def fetch_wayback(url: str) -> Optional[str]:
     if reply.status != 200:
         return None
     text = html_to_text(reply.text).strip()
-    if len(text) < P_MIN_SUBSTANCE_CHARS:
+    if len(text) < P_MIN_SUBSTANCE_CHARS or P_INTERSTITIAL_MARKER in text:
         return None
     taken = snapshot_date(reply.url)
     header = f"Archived copy of {url}"

backend/apps/agents/tools/ssrf_guard.py

@@ -27,6 +27,19 @@ class SSRFBlocked(Exception):
     """A fetch was refused because it targets a forbidden IP range."""
 
 
+class DomainUnreachable(SSRFBlocked):
+    """The host has no DNS records at all: dead domain, typo, or no network.
+
+    A subclass so every existing `except SSRFBlocked` still fails closed, but
+    callers that care can tell "we refused this" apart from "this doesn't
+    exist", which are opposite messages to show a user and have opposite
+    fallbacks (nothing vs the archive)."""
+
+
+# A page we will truncate to ~250KB anyway; without this a link to a disk image buffers the whole thing into RAM.
+MAX_FETCH_BYTES = 10 * 1024 * 1024
+
+
 P_BLOCKED_V4_NETS = [
     ipaddress.ip_network("10.0.0.0/8"),
     ipaddress.ip_network("172.16.0.0/12"),
@@ -52,7 +65,7 @@ async def p_resolve_host_async(host: str) -> list[str]:
     try:
         infos = await loop.getaddrinfo(host, None)
     except OSError as e:
-        raise SSRFBlocked(f"DNS resolution failed for {host}: {e}") from e
+        raise DomainUnreachable(f"{host} could not be resolved (dead domain, typo, or no network): {e}") from e
     return list({info[4][0] for info in infos})
 
 
@@ -101,13 +114,40 @@ async def assert_safe_url(url: str) -> str:
 
     resolved = await p_resolve_host_async(host)
     if not resolved:
-        raise SSRFBlocked(f"No DNS records for {host}.")
+        raise DomainUnreachable(f"No DNS records for {host}.")
     for ip in resolved:
         if p_is_forbidden_ip(ip):
             raise SSRFBlocked(f"Host {host} resolves to forbidden IP {ip}.")
     return url
 
 
+async def p_read_capped(client: httpx.AsyncClient, request: httpx.Request,
+                        max_bytes: int) -> httpx.Response:
+    """Send `request` and buffer at most `max_bytes` of the body.
+
+    Returns a detached Response so callers keep the plain `.content` / `.text`
+    interface. Content-Encoding and Content-Length are dropped because
+    `aiter_bytes` already yields decoded bytes and the count may be short.
+    """
+    streamed = await client.send(request, stream=True)
+    chunks: list[bytes] = []
+    total = 0
+    try:
+        async for chunk in streamed.aiter_bytes():
+            chunks.append(chunk)
+            total += len(chunk)
+            if total >= max_bytes:
+                break
+    finally:
+        await streamed.aclose()
+    headers = httpx.Headers(
+        [(k, v) for k, v in streamed.headers.multi_items()
+         if k.lower() not in ("content-encoding", "content-length")]
+    )
+    return httpx.Response(status_code=streamed.status_code, headers=headers,
+                          content=b"".join(chunks)[:max_bytes], request=request)
+
+
 async def safe_fetch(
     url: str,
     *,
@@ -117,6 +157,7 @@ async def safe_fetch(
     max_redirects: int = 5,
     json_body: dict | None = None,
     data: dict | None = None,
+    max_bytes: int = MAX_FETCH_BYTES,
 ) -> httpx.Response:
     """Fetch with per-redirect SSRF re-validation.
 
@@ -126,15 +167,14 @@ async def safe_fetch(
     current_url = await assert_safe_url(url)
     async with httpx.AsyncClient(timeout=timeout, follow_redirects=False, headers=headers or {}) as client:
         for _ in range(max_redirects + 1):
+            req_kwargs: dict = {}
             if method.upper() == "POST":
-                req_kwargs = {}
                 if json_body is not None:
                     req_kwargs["json"] = json_body
                 if data is not None:
                     req_kwargs["data"] = data
-                resp = await client.post(current_url, **req_kwargs)
-            else:
-                resp = await client.get(current_url)
+            request = client.build_request(method.upper(), current_url, **req_kwargs)
+            resp = await p_read_capped(client, request, max_bytes)
             if not (300 <= resp.status_code < 400):
                 return resp
             location = resp.headers.get("location")

backend/apps/agents/tools/web.py

@@ -14,9 +14,10 @@ from backend.apps.agents.tools.search.search_ddg import (
     HTTP_TIMEOUT,
     USER_AGENT,
 )
-from backend.apps.agents.tools.fetch.page_text import PageText, body_to_text, html_to_text, looks_like_pdf
+from backend.apps.agents.tools.fetch.html_to_text import html_to_text
+from backend.apps.agents.tools.fetch.page_text import PageText, body_to_text, looks_like_pdf
 from backend.apps.agents.tools.search.search_ddg import search_ddg as run_ddg_search
-from backend.apps.agents.tools.ssrf_guard import SSRFBlocked, safe_fetch
+from backend.apps.agents.tools.ssrf_guard import DomainUnreachable, SSRFBlocked, safe_fetch
 
 P_MAX_OUTPUT_BYTES = 250 * 1024  # ~250 KB covers ~95% of articles/wikis/docs.
 
@@ -180,6 +181,8 @@ class WebFetchTool(BaseTool):
                 timeout=HTTP_TIMEOUT,
             )
             resp.raise_for_status()
+        except DomainUnreachable as exc:
+            return PageText(text=f"Could not reach {url}: {exc}", kind="error")
         except SSRFBlocked as exc:
             return PageText(text=f"Refused to fetch {url}: {exc}", kind="error")
         except httpx.HTTPStatusError as exc:

backend/tests/test_html_to_text.py

@@ -0,0 +1,102 @@
+"""The extraction ladder, pinned against the failures that motivated it.
+
+`favor_precision=True` disables trafilatura's own rescue, so on real pages
+(allrecipes, Spiegel) it returned NOTHING and we shipped the regex strip's nav
+soup instead of the article. Flipping the flag off is not the fix either: on
+The Verge the plain call collapses to 441 chars where precision finds 6,602.
+Hence a ladder that keeps whichever rung found the most text.
+"""
+
+import pytest
+
+import backend.apps.agents.tools.fetch.html_to_text as HT
+from backend.apps.agents.tools.fetch.html_to_text import (
+    MIN_EXTRACT_CHARS,
+    THIN_EXTRACT_CHARS,
+    html_to_text,
+)
+
+
+def p_stub(monkeypatch, plain: str, precise: str, baseline: str = "", html2txt: str = ""):
+    def p_extract(raw_html, *, favor_precision):
+        return precise if favor_precision else plain
+    monkeypatch.setattr(HT, "p_trafilatura_extract", p_extract)
+
+    def p_floor(raw_html, fn_name):
+        return baseline if fn_name == "baseline" else html2txt
+    monkeypatch.setattr(HT, "p_trafilatura_floor", p_floor)
+
+
+def test_full_plain_extraction_never_pays_for_a_second_pass(monkeypatch):
+    calls = []
+
+    def p_extract(raw_html, *, favor_precision):
+        calls.append(favor_precision)
+        return "x" * (THIN_EXTRACT_CHARS + 10)
+    monkeypatch.setattr(HT, "p_trafilatura_extract", p_extract)
+    out = html_to_text("<html></html>")
+    assert len(out) == THIN_EXTRACT_CHARS + 10
+    assert calls == [False], "a healthy extraction must not trigger the precision pass"
+
+
+def test_thin_plain_falls_to_precision(monkeypatch):
+    """The Verge case: default recall collapses, precision finds the article."""
+    p_stub(monkeypatch, plain="short", precise="y" * 6000)
+    assert len(html_to_text("<html></html>")) == 6000
+
+
+def test_precision_returning_nothing_falls_to_baseline(monkeypatch):
+    """The allrecipes case: precision extracted nothing at all."""
+    p_stub(monkeypatch, plain="", precise="", baseline="b" * 5000)
+    assert len(html_to_text("<html></html>")) == 5000
+
+
+def test_ladder_keeps_the_fullest_rung(monkeypatch):
+    p_stub(monkeypatch, plain="a" * 300, precise="b" * 200, baseline="c" * 100)
+    assert html_to_text("<html></html>") == "a" * 300
+
+
+def test_everything_empty_falls_back_to_regex_strip(monkeypatch):
+    p_stub(monkeypatch, plain="", precise="", baseline="", html2txt="")
+    out = html_to_text("<html><body><p>hello &amp; goodbye</p></body></html>")
+    assert "hello & goodbye" in out
+
+
+def test_html2txt_only_rescues_a_truly_empty_read(monkeypatch):
+    p_stub(monkeypatch, plain="", precise="", baseline="", html2txt="z" * 400)
+    assert len(html_to_text("<html></html>")) == 400
+    p_stub(monkeypatch, plain="q" * (MIN_EXTRACT_CHARS + 1), precise="", baseline="", html2txt="z" * 400)
+    assert html_to_text("<html></html>") == "q" * (MIN_EXTRACT_CHARS + 1)
+
+
+def test_extractor_exception_degrades_instead_of_raising(monkeypatch):
+    def p_boom(raw_html, *, favor_precision):
+        raise ValueError("lxml exploded")
+    monkeypatch.setattr(HT, "p_trafilatura_extract", p_boom)
+    with pytest.raises(ValueError):
+        p_boom("", favor_precision=False)
+    # Through the real wrapper the same failure is swallowed and the floor answers.
+    monkeypatch.undo()
+    out = html_to_text("<html><body><p>" + "words " * 100 + "</p></body></html>")
+    assert "words" in out
+
+
+@pytest.mark.parametrize("markup,needle", [
+    ("<html><body><article><p>" + "Real body text. " * 60 + "</p></article></body></html>", "Real body text"),
+    ("<html><body><main><p>" + "Docs paragraph. " * 60 + "</p></main></body></html>", "Docs paragraph"),
+])
+def test_real_trafilatura_extracts_article_bodies(markup, needle):
+    """No mocks: the shipped library on the shipped call must find an article body."""
+    assert needle in html_to_text(markup)
+
+
+def test_real_trafilatura_drops_nav_chrome():
+    markup = (
+        "<html><body><nav>Home About Careers Privacy Policy Sign in</nav>"
+        "<article><p>" + "The measured content sentence. " * 40 + "</p></article>"
+        "<footer>Copyright Terms of Use Cookie Settings</footer></body></html>"
+    )
+    out = html_to_text(markup)
+    assert "measured content sentence" in out
+    assert "Cookie Settings" not in out
+    assert "Careers" not in out

backend/tests/test_wayback_snapshot.py

@@ -0,0 +1,69 @@
+"""The archive's redirect interstitial is not a copy of the page.
+
+Measured on instagram.com/nasa: the Wayback tier answered 200 and we handed
+the model 225 characters reading "Got an HTTP 302 response at crawl time /
+Redirecting to .../accounts/login". It passed the substance floor because the
+interstitial has real words in it, so only the wording gives it away.
+"""
+
+import pytest
+
+import backend.apps.agents.tools.fetch.wayback as WB
+from backend.apps.agents.tools.browser_http import HttpReply
+from backend.apps.agents.tools.fetch.wayback import fetch_wayback, snapshot_date
+
+P_ARCHIVED_URL = "https://web.archive.org/web/20260711073650/https://example.com/story"
+
+
+def p_patch(monkeypatch, status: int, text: str, url: str = P_ARCHIVED_URL):
+    async def p_req(u, **kw):
+        return HttpReply(status=status, text=text, content=text.encode(),
+                         content_type="text/html", url=url)
+    monkeypatch.setattr(WB, "browser_request", p_req)
+
+
+P_INTERSTITIAL = (
+    "<html><body><p>Loading...</p><p>https://www.instagram.com/nasa/</p>"
+    "<p>07:36:50 July 11, 2026</p><p>Got an HTTP 302 response at crawl time</p>"
+    "<p>Redirecting to...</p><p>https://www.instagram.com/accounts/login/?next=%2Fnasa%2F</p>"
+    "<p>Wayback Machine has not archived that URL beyond the redirect target given here.</p>"
+    "</body></html>"
+)
+
+P_REAL_ARTICLE = (
+    "<html><body><article><p>" + "The archived article body says something real. " * 20
+    + "</p></article></body></html>"
+)
+
+
+@pytest.mark.asyncio
+async def test_redirect_interstitial_is_not_content(monkeypatch):
+    p_patch(monkeypatch, 200, P_INTERSTITIAL)
+    assert await fetch_wayback("https://www.instagram.com/nasa/") is None
+
+
+@pytest.mark.asyncio
+async def test_real_snapshot_still_returns_with_its_date(monkeypatch):
+    p_patch(monkeypatch, 200, P_REAL_ARTICLE)
+    out = await fetch_wayback("https://example.com/story")
+    assert out is not None
+    assert "archived article body" in out
+    assert "2026-07-11" in out
+
+
+@pytest.mark.asyncio
+async def test_stub_snapshot_below_the_floor_is_rejected(monkeypatch):
+    p_patch(monkeypatch, 200, "<html><body><p>Loading...</p></body></html>")
+    assert await fetch_wayback("https://example.com/story") is None
+
+
+@pytest.mark.asyncio
+async def test_offsite_redirect_is_refused(monkeypatch):
+    """We hand the archive a caller-supplied URL, so landing anywhere else means no answer."""
+    p_patch(monkeypatch, 200, P_REAL_ARTICLE, url="https://evil.example/whatever")
+    assert await fetch_wayback("https://example.com/story") is None
+
+
+def test_snapshot_date_parsing():
+    assert snapshot_date(P_ARCHIVED_URL) == "2026-07-11"
+    assert snapshot_date("https://web.archive.org/nope") is None

backend/tests/test_web_fetch_reachability.py

@@ -0,0 +1,98 @@
+"""A dead domain is an archive lookup, not a security refusal.
+
+Measured: fetching a domain with no DNS records returned HTTP 400 "Refused:
+DNS resolution failed", which reads like we blocked it AND short-circuited the
+cascade before the Wayback tier, which exists for exactly that case. These pin
+the split: unresolvable falls through, forbidden ranges still 400.
+"""
+
+import httpx
+import pytest
+
+import backend.apps.agents.tools.fetch.wayback as WB
+import backend.apps.agents.tools.ssrf_guard as SG
+from backend.apps.agents.tools.ssrf_guard import DomainUnreachable, SSRFBlocked, safe_fetch
+from backend.apps.agents.tools.web import WebFetchTool
+from backend.apps.web.web import FetchBody, fetch
+from backend.tests.web_cascade_fixtures import *  # noqa: F401,F403
+
+
+def p_unresolvable(monkeypatch):
+    async def p_dns_dead(url):
+        raise DomainUnreachable("nowhere.invalid could not be resolved (dead domain, typo, or no network)")
+    monkeypatch.setattr(SG, "assert_safe_url", p_dns_dead)
+
+
+def test_domain_unreachable_is_an_ssrf_blocked_subclass():
+    """Every existing `except SSRFBlocked` must keep failing closed on it."""
+    assert issubclass(DomainUnreachable, SSRFBlocked)
+
+
+@pytest.mark.asyncio
+async def test_dead_domain_reaches_the_archive(monkeypatch):
+    p_unresolvable(monkeypatch)
+
+    async def p_snapshot(url):
+        return "Archived copy of the dead site\n\n" + "real archived text. " * 40
+    monkeypatch.setattr(WB, "fetch_wayback", p_snapshot)
+
+    out = await fetch(FetchBody(url="https://nowhere.invalid/page"))
+    assert out["backend"] == "wayback"
+    assert "real archived text" in out["content"]
+
+
+@pytest.mark.asyncio
+async def test_dead_domain_without_a_snapshot_says_unreachable_not_refused(monkeypatch):
+    p_unresolvable(monkeypatch)
+    out = await fetch(FetchBody(url="https://nowhere.invalid/page"))
+    assert "Could not reach" in out["content"]
+    assert "Refused" not in out["content"]
+
+
+@pytest.mark.asyncio
+async def test_forbidden_range_still_gets_a_hard_refusal(monkeypatch):
+    from fastapi import HTTPException
+
+    async def p_blocked(url):
+        raise SSRFBlocked("URL host 169.254.169.254 is in a blocked range.")
+    monkeypatch.setattr(SG, "assert_safe_url", p_blocked)
+
+    with pytest.raises(HTTPException) as exc:
+        await fetch(FetchBody(url="http://169.254.169.254/latest/meta-data/"))
+    assert exc.value.status_code == 400
+    assert "Refused" in str(exc.value.detail)
+
+
+@pytest.mark.asyncio
+async def test_fetch_page_reports_unreachable_as_error_kind(monkeypatch):
+    p_unresolvable(monkeypatch)
+    page = await WebFetchTool.fetch_page("https://nowhere.invalid/page")
+    assert page.kind == "error"
+    assert "Could not reach" in page.text
+
+
+@pytest.mark.asyncio
+async def test_oversized_body_is_capped_not_buffered(monkeypatch):
+    """A link to a disk image must not pull gigabytes into RAM before we truncate to 250KB."""
+    served = {"bytes": 0}
+
+    class p_HugeStream(httpx.AsyncByteStream):
+        async def __aiter__(self):
+            for _ in range(500):
+                served["bytes"] += 100_000
+                yield b"x" * 100_000
+
+        async def aclose(self) -> None:
+            return None
+
+    async def p_send(self, request, **kw):
+        return httpx.Response(200, headers={"content-type": "text/plain"}, stream=p_HugeStream())
+    monkeypatch.setattr(httpx.AsyncClient, "send", p_send)
+
+    async def p_ok(url):
+        return url
+    monkeypatch.setattr(SG, "assert_safe_url", p_ok)
+
+    resp = await safe_fetch("https://example.com/huge.bin", max_bytes=250_000)
+    assert len(resp.content) == 250_000
+    assert served["bytes"] < 50_000_000, "the stream must stop early, not download the whole file"