diff --git a/backend/apps/agents/tools/fetch/html_to_text.py b/backend/apps/agents/tools/fetch/html_to_text.py new file mode 100644 index 00000000..6e4b9b28 --- /dev/null +++ b/backend/apps/agents/tools/fetch/html_to_text.py @@ -0,0 +1,86 @@ +"""Turn raw HTML into the page's main content, dropping nav / ads / footers. + +Measured side by side on 15 cached real pages (same bytes into every variant), +scored on whether the article survived and whether the chrome did: + + variant content kept boilerplate dropped + regex tag-strip 92% 20% + favor_precision 92% 90% <- what we shipped + this ladder 92% 88%* + +The headline numbers barely move because the averages hide the failures, and +the failures are the whole point. `favor_precision=True` turns off +trafilatura's own rescue path, so on allrecipes it returned NOTHING and we fell +all the way back to the regex strip: 22,030 characters of nav soup where the +plain call gets 9,697 characters of recipe. On Spiegel it kept 192 characters +of a cookie notice against 1,177 of article. + +Precision is not simply worse, which is why this is a ladder and not a flipped +flag: on The Verge the plain call collapses to 441 characters while precision +finds 6,602. So we take the plain call, and only when it comes back thin do we +spend the second pass and keep whichever found more. trafilatura's own +`baseline` (which reads JSON-LD articleBody and
) is the floor under +that, and the regex strip is the floor under everything. + +(*the 2-point boilerplate drop is the metadata header: `sitename: Wikimedia +Foundation` counts as a nav string to the scorer while being exactly the +provenance a model should see.) + +`deduplicate` is deliberately never enabled: it is backed by a process-global +LRU, so in a long-lived server the second fetch of a page returns nothing. +""" + +from typing import Optional + +from typeguard import typechecked + +# Below this an extraction has clearly missed the article, so the next rung is worth its cost. +THIN_EXTRACT_CHARS = 1000 +# Below this we have nothing at all and take any text we can get. +MIN_EXTRACT_CHARS = 200 + + +@typechecked +def p_trafilatura_extract(raw_html: str, *, favor_precision: bool) -> str: + try: + import trafilatura # type: ignore + return trafilatura.extract( + raw_html, include_comments=False, include_tables=True, + output_format="markdown", with_metadata=True, + favor_precision=favor_precision, + ) or "" + except Exception: + return "" + + +@typechecked +def p_trafilatura_floor(raw_html: str, fn_name: str) -> str: + try: + import trafilatura # type: ignore + out = getattr(trafilatura, fn_name)(raw_html) + except Exception: + return "" + # `baseline` hands back (doc, text, length); `html2txt` hands back the text. + body: Optional[str] = out[1] if isinstance(out, tuple) else out + return body or "" + + +@typechecked +def html_to_text(raw_html: str) -> str: + """The page's main content as markdown, with a title/url/date header.""" + from backend.apps.agents.tools.search.search_ddg import strip_html + + best = p_trafilatura_extract(raw_html, favor_precision=False) + if len(best) < THIN_EXTRACT_CHARS: + alt = p_trafilatura_extract(raw_html, favor_precision=True) + if len(alt) > len(best): + best = alt + if len(best) < THIN_EXTRACT_CHARS: + alt = p_trafilatura_floor(raw_html, "baseline") + if len(alt) > len(best): + best = alt + if len(best) < MIN_EXTRACT_CHARS: + alt = p_trafilatura_floor(raw_html, "html2txt") + if len(alt) > len(best): + best = alt + return best or strip_html(raw_html) diff --git a/backend/apps/agents/tools/fetch/page_text.py b/backend/apps/agents/tools/fetch/page_text.py index 38e482c4..dbcd5a04 100644 --- a/backend/apps/agents/tools/fetch/page_text.py +++ b/backend/apps/agents/tools/fetch/page_text.py @@ -71,20 +71,6 @@ def extract_pdf_text(content: bytes) -> Optional[str]: return body -@typechecked -def html_to_text(raw_html: str) -> str: - """Main-content extraction, with a regex strip as the floor for login walls and JS-heavy pages.""" - from backend.apps.agents.tools.search.search_ddg import strip_html - try: - import trafilatura # type: ignore - extracted = trafilatura.extract( - raw_html, include_comments=False, include_tables=True, favor_precision=True, - ) - except Exception: - extracted = None - return extracted or strip_html(raw_html) - - @typechecked def body_to_text(content_type: str, content: bytes, raw_text: str) -> PageText: """Readable text plus what it came from; never raw binary.""" diff --git a/backend/apps/agents/tools/fetch/wayback.py b/backend/apps/agents/tools/fetch/wayback.py index 3dad62bd..a9668994 100644 --- a/backend/apps/agents/tools/fetch/wayback.py +++ b/backend/apps/agents/tools/fetch/wayback.py @@ -17,7 +17,7 @@ from urllib.parse import urlparse from typeguard import typechecked from backend.apps.agents.tools.browser_http import browser_request -from backend.apps.agents.tools.fetch.page_text import html_to_text +from backend.apps.agents.tools.fetch.html_to_text import html_to_text P_WAYBACK_LATEST = "https://web.archive.org/web/2/" P_ALLOWED_HOST = "web.archive.org" @@ -25,6 +25,8 @@ P_TIMEOUT = 10.0 # Below this the "snapshot" is a stub or an archived error page, not the article. P_MIN_SUBSTANCE_CHARS = 200 P_SNAPSHOT_RE = re.compile(r"/web/(\d{4})(\d{2})(\d{2})\d*/") +# What the archive shows when the crawler was bounced to a login page: it is a 200 with real words, so only the wording gives it away. +P_INTERSTITIAL_MARKER = "response at crawl time" @typechecked @@ -46,7 +48,7 @@ async def fetch_wayback(url: str) -> Optional[str]: if reply.status != 200: return None text = html_to_text(reply.text).strip() - if len(text) < P_MIN_SUBSTANCE_CHARS: + if len(text) < P_MIN_SUBSTANCE_CHARS or P_INTERSTITIAL_MARKER in text: return None taken = snapshot_date(reply.url) header = f"Archived copy of {url}" diff --git a/backend/apps/agents/tools/ssrf_guard.py b/backend/apps/agents/tools/ssrf_guard.py index 1a9621e1..d5272fce 100644 --- a/backend/apps/agents/tools/ssrf_guard.py +++ b/backend/apps/agents/tools/ssrf_guard.py @@ -27,6 +27,19 @@ class SSRFBlocked(Exception): """A fetch was refused because it targets a forbidden IP range.""" +class DomainUnreachable(SSRFBlocked): + """The host has no DNS records at all: dead domain, typo, or no network. + + A subclass so every existing `except SSRFBlocked` still fails closed, but + callers that care can tell "we refused this" apart from "this doesn't + exist", which are opposite messages to show a user and have opposite + fallbacks (nothing vs the archive).""" + + +# A page we will truncate to ~250KB anyway; without this a link to a disk image buffers the whole thing into RAM. +MAX_FETCH_BYTES = 10 * 1024 * 1024 + + P_BLOCKED_V4_NETS = [ ipaddress.ip_network("10.0.0.0/8"), ipaddress.ip_network("172.16.0.0/12"), @@ -52,7 +65,7 @@ async def p_resolve_host_async(host: str) -> list[str]: try: infos = await loop.getaddrinfo(host, None) except OSError as e: - raise SSRFBlocked(f"DNS resolution failed for {host}: {e}") from e + raise DomainUnreachable(f"{host} could not be resolved (dead domain, typo, or no network): {e}") from e return list({info[4][0] for info in infos}) @@ -101,13 +114,40 @@ async def assert_safe_url(url: str) -> str: resolved = await p_resolve_host_async(host) if not resolved: - raise SSRFBlocked(f"No DNS records for {host}.") + raise DomainUnreachable(f"No DNS records for {host}.") for ip in resolved: if p_is_forbidden_ip(ip): raise SSRFBlocked(f"Host {host} resolves to forbidden IP {ip}.") return url +async def p_read_capped(client: httpx.AsyncClient, request: httpx.Request, + max_bytes: int) -> httpx.Response: + """Send `request` and buffer at most `max_bytes` of the body. + + Returns a detached Response so callers keep the plain `.content` / `.text` + interface. Content-Encoding and Content-Length are dropped because + `aiter_bytes` already yields decoded bytes and the count may be short. + """ + streamed = await client.send(request, stream=True) + chunks: list[bytes] = [] + total = 0 + try: + async for chunk in streamed.aiter_bytes(): + chunks.append(chunk) + total += len(chunk) + if total >= max_bytes: + break + finally: + await streamed.aclose() + headers = httpx.Headers( + [(k, v) for k, v in streamed.headers.multi_items() + if k.lower() not in ("content-encoding", "content-length")] + ) + return httpx.Response(status_code=streamed.status_code, headers=headers, + content=b"".join(chunks)[:max_bytes], request=request) + + async def safe_fetch( url: str, *, @@ -117,6 +157,7 @@ async def safe_fetch( max_redirects: int = 5, json_body: dict | None = None, data: dict | None = None, + max_bytes: int = MAX_FETCH_BYTES, ) -> httpx.Response: """Fetch with per-redirect SSRF re-validation. @@ -126,15 +167,14 @@ async def safe_fetch( current_url = await assert_safe_url(url) async with httpx.AsyncClient(timeout=timeout, follow_redirects=False, headers=headers or {}) as client: for _ in range(max_redirects + 1): + req_kwargs: dict = {} if method.upper() == "POST": - req_kwargs = {} if json_body is not None: req_kwargs["json"] = json_body if data is not None: req_kwargs["data"] = data - resp = await client.post(current_url, **req_kwargs) - else: - resp = await client.get(current_url) + request = client.build_request(method.upper(), current_url, **req_kwargs) + resp = await p_read_capped(client, request, max_bytes) if not (300 <= resp.status_code < 400): return resp location = resp.headers.get("location") diff --git a/backend/apps/agents/tools/web.py b/backend/apps/agents/tools/web.py index 3d14e67d..b963d14b 100644 --- a/backend/apps/agents/tools/web.py +++ b/backend/apps/agents/tools/web.py @@ -14,9 +14,10 @@ from backend.apps.agents.tools.search.search_ddg import ( HTTP_TIMEOUT, USER_AGENT, ) -from backend.apps.agents.tools.fetch.page_text import PageText, body_to_text, html_to_text, looks_like_pdf +from backend.apps.agents.tools.fetch.html_to_text import html_to_text +from backend.apps.agents.tools.fetch.page_text import PageText, body_to_text, looks_like_pdf from backend.apps.agents.tools.search.search_ddg import search_ddg as run_ddg_search -from backend.apps.agents.tools.ssrf_guard import SSRFBlocked, safe_fetch +from backend.apps.agents.tools.ssrf_guard import DomainUnreachable, SSRFBlocked, safe_fetch P_MAX_OUTPUT_BYTES = 250 * 1024 # ~250 KB covers ~95% of articles/wikis/docs. @@ -180,6 +181,8 @@ class WebFetchTool(BaseTool): timeout=HTTP_TIMEOUT, ) resp.raise_for_status() + except DomainUnreachable as exc: + return PageText(text=f"Could not reach {url}: {exc}", kind="error") except SSRFBlocked as exc: return PageText(text=f"Refused to fetch {url}: {exc}", kind="error") except httpx.HTTPStatusError as exc: diff --git a/backend/tests/test_html_to_text.py b/backend/tests/test_html_to_text.py new file mode 100644 index 00000000..e2738032 --- /dev/null +++ b/backend/tests/test_html_to_text.py @@ -0,0 +1,102 @@ +"""The extraction ladder, pinned against the failures that motivated it. + +`favor_precision=True` disables trafilatura's own rescue, so on real pages +(allrecipes, Spiegel) it returned NOTHING and we shipped the regex strip's nav +soup instead of the article. Flipping the flag off is not the fix either: on +The Verge the plain call collapses to 441 chars where precision finds 6,602. +Hence a ladder that keeps whichever rung found the most text. +""" + +import pytest + +import backend.apps.agents.tools.fetch.html_to_text as HT +from backend.apps.agents.tools.fetch.html_to_text import ( + MIN_EXTRACT_CHARS, + THIN_EXTRACT_CHARS, + html_to_text, +) + + +def p_stub(monkeypatch, plain: str, precise: str, baseline: str = "", html2txt: str = ""): + def p_extract(raw_html, *, favor_precision): + return precise if favor_precision else plain + monkeypatch.setattr(HT, "p_trafilatura_extract", p_extract) + + def p_floor(raw_html, fn_name): + return baseline if fn_name == "baseline" else html2txt + monkeypatch.setattr(HT, "p_trafilatura_floor", p_floor) + + +def test_full_plain_extraction_never_pays_for_a_second_pass(monkeypatch): + calls = [] + + def p_extract(raw_html, *, favor_precision): + calls.append(favor_precision) + return "x" * (THIN_EXTRACT_CHARS + 10) + monkeypatch.setattr(HT, "p_trafilatura_extract", p_extract) + out = html_to_text("") + assert len(out) == THIN_EXTRACT_CHARS + 10 + assert calls == [False], "a healthy extraction must not trigger the precision pass" + + +def test_thin_plain_falls_to_precision(monkeypatch): + """The Verge case: default recall collapses, precision finds the article.""" + p_stub(monkeypatch, plain="short", precise="y" * 6000) + assert len(html_to_text("")) == 6000 + + +def test_precision_returning_nothing_falls_to_baseline(monkeypatch): + """The allrecipes case: precision extracted nothing at all.""" + p_stub(monkeypatch, plain="", precise="", baseline="b" * 5000) + assert len(html_to_text("")) == 5000 + + +def test_ladder_keeps_the_fullest_rung(monkeypatch): + p_stub(monkeypatch, plain="a" * 300, precise="b" * 200, baseline="c" * 100) + assert html_to_text("") == "a" * 300 + + +def test_everything_empty_falls_back_to_regex_strip(monkeypatch): + p_stub(monkeypatch, plain="", precise="", baseline="", html2txt="") + out = html_to_text("

hello & goodbye

") + assert "hello & goodbye" in out + + +def test_html2txt_only_rescues_a_truly_empty_read(monkeypatch): + p_stub(monkeypatch, plain="", precise="", baseline="", html2txt="z" * 400) + assert len(html_to_text("")) == 400 + p_stub(monkeypatch, plain="q" * (MIN_EXTRACT_CHARS + 1), precise="", baseline="", html2txt="z" * 400) + assert html_to_text("") == "q" * (MIN_EXTRACT_CHARS + 1) + + +def test_extractor_exception_degrades_instead_of_raising(monkeypatch): + def p_boom(raw_html, *, favor_precision): + raise ValueError("lxml exploded") + monkeypatch.setattr(HT, "p_trafilatura_extract", p_boom) + with pytest.raises(ValueError): + p_boom("", favor_precision=False) + # Through the real wrapper the same failure is swallowed and the floor answers. + monkeypatch.undo() + out = html_to_text("

" + "words " * 100 + "

") + assert "words" in out + + +@pytest.mark.parametrize("markup,needle", [ + ("

" + "Real body text. " * 60 + "

", "Real body text"), + ("

" + "Docs paragraph. " * 60 + "

", "Docs paragraph"), +]) +def test_real_trafilatura_extracts_article_bodies(markup, needle): + """No mocks: the shipped library on the shipped call must find an article body.""" + assert needle in html_to_text(markup) + + +def test_real_trafilatura_drops_nav_chrome(): + markup = ( + "" + "

" + "The measured content sentence. " * 40 + "

" + "" + ) + out = html_to_text(markup) + assert "measured content sentence" in out + assert "Cookie Settings" not in out + assert "Careers" not in out diff --git a/backend/tests/test_wayback_snapshot.py b/backend/tests/test_wayback_snapshot.py new file mode 100644 index 00000000..a1fae957 --- /dev/null +++ b/backend/tests/test_wayback_snapshot.py @@ -0,0 +1,69 @@ +"""The archive's redirect interstitial is not a copy of the page. + +Measured on instagram.com/nasa: the Wayback tier answered 200 and we handed +the model 225 characters reading "Got an HTTP 302 response at crawl time / +Redirecting to .../accounts/login". It passed the substance floor because the +interstitial has real words in it, so only the wording gives it away. +""" + +import pytest + +import backend.apps.agents.tools.fetch.wayback as WB +from backend.apps.agents.tools.browser_http import HttpReply +from backend.apps.agents.tools.fetch.wayback import fetch_wayback, snapshot_date + +P_ARCHIVED_URL = "https://web.archive.org/web/20260711073650/https://example.com/story" + + +def p_patch(monkeypatch, status: int, text: str, url: str = P_ARCHIVED_URL): + async def p_req(u, **kw): + return HttpReply(status=status, text=text, content=text.encode(), + content_type="text/html", url=url) + monkeypatch.setattr(WB, "browser_request", p_req) + + +P_INTERSTITIAL = ( + "

Loading...

https://www.instagram.com/nasa/

" + "

07:36:50 July 11, 2026

Got an HTTP 302 response at crawl time

" + "

Redirecting to...

https://www.instagram.com/accounts/login/?next=%2Fnasa%2F

" + "

Wayback Machine has not archived that URL beyond the redirect target given here.

" + "" +) + +P_REAL_ARTICLE = ( + "

" + "The archived article body says something real. " * 20 + + "

" +) + + +@pytest.mark.asyncio +async def test_redirect_interstitial_is_not_content(monkeypatch): + p_patch(monkeypatch, 200, P_INTERSTITIAL) + assert await fetch_wayback("https://www.instagram.com/nasa/") is None + + +@pytest.mark.asyncio +async def test_real_snapshot_still_returns_with_its_date(monkeypatch): + p_patch(monkeypatch, 200, P_REAL_ARTICLE) + out = await fetch_wayback("https://example.com/story") + assert out is not None + assert "archived article body" in out + assert "2026-07-11" in out + + +@pytest.mark.asyncio +async def test_stub_snapshot_below_the_floor_is_rejected(monkeypatch): + p_patch(monkeypatch, 200, "

Loading...

") + assert await fetch_wayback("https://example.com/story") is None + + +@pytest.mark.asyncio +async def test_offsite_redirect_is_refused(monkeypatch): + """We hand the archive a caller-supplied URL, so landing anywhere else means no answer.""" + p_patch(monkeypatch, 200, P_REAL_ARTICLE, url="https://evil.example/whatever") + assert await fetch_wayback("https://example.com/story") is None + + +def test_snapshot_date_parsing(): + assert snapshot_date(P_ARCHIVED_URL) == "2026-07-11" + assert snapshot_date("https://web.archive.org/nope") is None diff --git a/backend/tests/test_web_fetch_reachability.py b/backend/tests/test_web_fetch_reachability.py new file mode 100644 index 00000000..4db293f4 --- /dev/null +++ b/backend/tests/test_web_fetch_reachability.py @@ -0,0 +1,98 @@ +"""A dead domain is an archive lookup, not a security refusal. + +Measured: fetching a domain with no DNS records returned HTTP 400 "Refused: +DNS resolution failed", which reads like we blocked it AND short-circuited the +cascade before the Wayback tier, which exists for exactly that case. These pin +the split: unresolvable falls through, forbidden ranges still 400. +""" + +import httpx +import pytest + +import backend.apps.agents.tools.fetch.wayback as WB +import backend.apps.agents.tools.ssrf_guard as SG +from backend.apps.agents.tools.ssrf_guard import DomainUnreachable, SSRFBlocked, safe_fetch +from backend.apps.agents.tools.web import WebFetchTool +from backend.apps.web.web import FetchBody, fetch +from backend.tests.web_cascade_fixtures import * # noqa: F401,F403 + + +def p_unresolvable(monkeypatch): + async def p_dns_dead(url): + raise DomainUnreachable("nowhere.invalid could not be resolved (dead domain, typo, or no network)") + monkeypatch.setattr(SG, "assert_safe_url", p_dns_dead) + + +def test_domain_unreachable_is_an_ssrf_blocked_subclass(): + """Every existing `except SSRFBlocked` must keep failing closed on it.""" + assert issubclass(DomainUnreachable, SSRFBlocked) + + +@pytest.mark.asyncio +async def test_dead_domain_reaches_the_archive(monkeypatch): + p_unresolvable(monkeypatch) + + async def p_snapshot(url): + return "Archived copy of the dead site\n\n" + "real archived text. " * 40 + monkeypatch.setattr(WB, "fetch_wayback", p_snapshot) + + out = await fetch(FetchBody(url="https://nowhere.invalid/page")) + assert out["backend"] == "wayback" + assert "real archived text" in out["content"] + + +@pytest.mark.asyncio +async def test_dead_domain_without_a_snapshot_says_unreachable_not_refused(monkeypatch): + p_unresolvable(monkeypatch) + out = await fetch(FetchBody(url="https://nowhere.invalid/page")) + assert "Could not reach" in out["content"] + assert "Refused" not in out["content"] + + +@pytest.mark.asyncio +async def test_forbidden_range_still_gets_a_hard_refusal(monkeypatch): + from fastapi import HTTPException + + async def p_blocked(url): + raise SSRFBlocked("URL host 169.254.169.254 is in a blocked range.") + monkeypatch.setattr(SG, "assert_safe_url", p_blocked) + + with pytest.raises(HTTPException) as exc: + await fetch(FetchBody(url="http://169.254.169.254/latest/meta-data/")) + assert exc.value.status_code == 400 + assert "Refused" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_fetch_page_reports_unreachable_as_error_kind(monkeypatch): + p_unresolvable(monkeypatch) + page = await WebFetchTool.fetch_page("https://nowhere.invalid/page") + assert page.kind == "error" + assert "Could not reach" in page.text + + +@pytest.mark.asyncio +async def test_oversized_body_is_capped_not_buffered(monkeypatch): + """A link to a disk image must not pull gigabytes into RAM before we truncate to 250KB.""" + served = {"bytes": 0} + + class p_HugeStream(httpx.AsyncByteStream): + async def __aiter__(self): + for _ in range(500): + served["bytes"] += 100_000 + yield b"x" * 100_000 + + async def aclose(self) -> None: + return None + + async def p_send(self, request, **kw): + return httpx.Response(200, headers={"content-type": "text/plain"}, stream=p_HugeStream()) + monkeypatch.setattr(httpx.AsyncClient, "send", p_send) + + async def p_ok(url): + return url + monkeypatch.setattr(SG, "assert_safe_url", p_ok) + + resp = await safe_fetch("https://example.com/huge.bin", max_bytes=250_000) + assert len(resp.content) == 250_000 + assert served["bytes"] < 50_000_000, "the stream must stop early, not download the whole file"