diff --git a/backend/apps/agents/tools/fetch/bot_wall.py b/backend/apps/agents/tools/fetch/bot_wall.py new file mode 100644 index 00000000..15d0ba7f --- /dev/null +++ b/backend/apps/agents/tools/fetch/bot_wall.py @@ -0,0 +1,50 @@ +"""Tell a bot wall apart from a page, so the cascade keeps going instead of +handing the model a challenge screen and calling it content. + +An HTTP error is easy: the fetch tier already falls through on any 4xx/5xx. The +gap was the wall that answers 200. Measured live, Reddit serves "Reddit - +Please wait for verification" with a 200, and Cloudflare's interstitial is a +normal 200 whose whole body is "Enable JavaScript and cookies to continue". +Those are exactly the pages our own offscreen Chromium CAN read, because it +runs a real browser on the user's own residential IP, so treating them as a +successful fetch spends the one tier that would have worked. + +Both conditions must hold: a challenge phrase AND a page too small to be an +article. A news story about Cloudflare outages contains the phrase and must not +be thrown away for it. +""" + +import re +from typing import Tuple + +from typeguard import typechecked + +# A wall is a stub page. Real articles that merely mention these run long. +MAX_WALL_CHARS = 2000 + +P_WALL_MARKERS: Tuple[str, ...] = ( + "just a moment...", + "enable javascript and cookies to continue", + "checking your browser before accessing", + "verifying you are human", + "please wait for verification", + "attention required! | cloudflare", + "you need to enable javascript to run this app", + "please enable js and disable any ad blocker", + "sorry, you have been blocked", + "why have i been blocked", + "performed triggered the security solution", + "confirm you are a human", + "press & hold", +) + +P_SPACE_RE = re.compile(r"\s+") + + +@typechecked +def looks_like_bot_wall(text: str) -> bool: + """True when this 200 is a challenge screen rather than the page.""" + if len(text) > MAX_WALL_CHARS: + return False + # Rendered walls arrive line-wrapped ("confirm you are\na human"), so match on flattened text. + return any(marker in P_SPACE_RE.sub(" ", text.lower()) for marker in P_WALL_MARKERS) diff --git a/backend/apps/web/web.py b/backend/apps/web/web.py index 5a5cf754..597b8a5f 100644 --- a/backend/apps/web/web.py +++ b/backend/apps/web/web.py @@ -263,13 +263,21 @@ async def fetch(body: FetchBody) -> Dict: body_text = page.text.split("\n\n", 1)[-1] if len(body_text.strip()) < 200: return None + # A challenge screen answers 200 and reads like prose, and it is exactly what our own Chromium on the user's own IP can get past, so never stop here for one. + from backend.apps.agents.tools.fetch.bot_wall import looks_like_bot_wall + if looks_like_bot_wall(body_text): + return None return {"url": body.url, "content": page.text, "backend": "local"} async def try_browser_fetch() -> Optional[Dict]: # Packaged-app tier: renders the page in a real offscreen Chromium and returns its visible text, so JS-only / SPA / soft-paywall pages that give httpx nothing actually resolve. Shares the user's browser cookies, so pages they're logged into fetch authed. + from backend.apps.agents.tools.fetch.bot_wall import looks_like_bot_wall res = await p_browser_bridge("browser_fetch", {"url": body.url}) if not res or not res.get("text"): return None + # Measured live: Cloudflare and PerimeterX beat even a real browser on some sites, and their challenge screen is not the page; the archive still has the real one. + if looks_like_bot_wall(res["text"]): + return None return {"url": body.url, "content": f"Contents of {body.url}:\n\n{res['text']}", "backend": "browser"} async def try_wayback() -> Optional[Dict]: diff --git a/backend/tests/test_bot_wall.py b/backend/tests/test_bot_wall.py new file mode 100644 index 00000000..71bd544d --- /dev/null +++ b/backend/tests/test_bot_wall.py @@ -0,0 +1,137 @@ +"""A 200 that is a bot wall must not end the fetch cascade. + +Any 4xx already falls through. The gap was the wall that answers 200 with +enough prose to clear the thin-page floor, which stopped the cascade one tier +before the offscreen Chromium that can actually read the page. +""" + +import pytest + +import backend.apps.web.web as W +from backend.apps.agents.tools.fetch.bot_wall import MAX_WALL_CHARS, looks_like_bot_wall +from backend.apps.agents.tools.fetch.page_text import PageText +from backend.apps.agents.tools.web import WebFetchTool +from backend.apps.web.web import FetchBody, fetch +from backend.tests.web_cascade_fixtures import * # noqa: F401,F403 +from backend.tests.web_cascade_fixtures import patch_browser_bridge + +P_CLOUDFLARE = ( + "Just a moment...\nwww.example.com needs to review the security of your connection " + "before proceeding.\nEnable JavaScript and cookies to continue\n" + "spacer text. " * 30 +) +P_REDDIT = "Reddit - Please wait for verification\n" + "checking. " * 40 + + +@pytest.mark.parametrize("text", [P_CLOUDFLARE, P_REDDIT]) +def test_challenge_screens_are_recognised(text): + assert looks_like_bot_wall(text) + + +def test_a_long_article_mentioning_the_phrase_is_not_a_wall(): + """A story about Cloudflare outages must not be thrown away for quoting the interstitial.""" + article = ("Cloudflare's interstitial, the one that says Just a moment... while it runs its " + "checks, was at the centre of today's outage. " + "Reporting continues. " * 200) + assert len(article) > MAX_WALL_CHARS + assert not looks_like_bot_wall(article) + + +def test_ordinary_page_text_is_not_a_wall(): + assert not looks_like_bot_wall("Coroutines and tasks. This section outlines asyncio APIs.") + + +def p_local_returns(monkeypatch, text: str, kind: str = "html"): + async def p_page(url, prompt=None): + return PageText(text=f"Contents of {url}:\n\n{text}", kind=kind) + monkeypatch.setattr(WebFetchTool, "fetch_page", staticmethod(p_page)) + + +@pytest.mark.asyncio +async def test_bot_wall_hands_the_page_to_the_browser_tier(monkeypatch): + p_local_returns(monkeypatch, P_CLOUDFLARE) + patch_browser_bridge(monkeypatch, {"text": "The real rendered article body."}) + out = await fetch(FetchBody(url="https://walled.example/article")) + assert out["backend"] == "browser" + assert "real rendered article body" in out["content"] + + +@pytest.mark.asyncio +async def test_a_real_page_never_reaches_the_browser_tier(monkeypatch): + """The browser tier costs a window and seconds; a page we already read must not pay for it.""" + p_local_returns(monkeypatch, "The genuine article body. " * 40) + calls = [] + + async def p_bridge(action, params): + calls.append(action) + return {"text": "should never be used"} + monkeypatch.setattr(W, "p_browser_bridge", p_bridge) + + out = await fetch(FetchBody(url="https://fine.example/article")) + assert out["backend"] == "local" + assert calls == [] + + +@pytest.mark.asyncio +async def test_browser_tier_is_tried_before_the_archive(monkeypatch): + """The live page in our own browser beats an archived copy of it.""" + import backend.apps.agents.tools.fetch.wayback as WB + p_local_returns(monkeypatch, P_REDDIT) + patch_browser_bridge(monkeypatch, {"text": "Live rendered thread."}) + + async def p_snapshot(url): + raise AssertionError("the archive must not be consulted while the browser tier can serve") + monkeypatch.setattr(WB, "fetch_wayback", p_snapshot) + + out = await fetch(FetchBody(url="https://www.reddit.com/r/programming/")) + assert out["backend"] == "browser" + + +@pytest.mark.asyncio +async def test_headless_backend_skips_the_browser_tier_without_erroring(monkeypatch): + """`bash backend/run.sh` has no Electron main bridge, so the tier must be a no-op, not a failure.""" + import backend.apps.agents.tools.fetch.wayback as WB + p_local_returns(monkeypatch, P_REDDIT) + patch_browser_bridge(monkeypatch, None) + + async def p_snapshot(url): + return "Archived copy\n\n" + "archived thread text. " * 30 + monkeypatch.setattr(WB, "fetch_wayback", p_snapshot) + + out = await fetch(FetchBody(url="https://www.reddit.com/r/programming/")) + assert out["backend"] == "wayback" + + +P_PERIMETERX = "Press & Hold to confirm you are\na human (and not a bot).\nReference ID eb3f614f" +P_CF_BLOCK = ("Sorry, you have been blocked\nYou are unable to access crunchbase.com\n" + "Why have I been blocked?\n" + "boilerplate. " * 20) + + +@pytest.mark.parametrize("text", [P_PERIMETERX, P_CF_BLOCK]) +def test_line_wrapped_challenges_are_recognised(text): + """A rendered wall arrives wrapped ('confirm you are\\na human'), so matching must flatten it.""" + assert looks_like_bot_wall(text) + + +@pytest.mark.asyncio +async def test_a_walled_browser_result_falls_through_to_the_archive(monkeypatch): + """Measured live: Cloudflare and PerimeterX beat even our real Chromium on some sites, and + Zillow's archived copy had 6,888 characters where the live challenge screen had 106.""" + import backend.apps.agents.tools.fetch.wayback as WB + p_local_returns(monkeypatch, "HTTP error 403 fetching it", kind="error") + patch_browser_bridge(monkeypatch, {"text": P_PERIMETERX}) + + async def p_snapshot(url): + return "Archived copy\n\n" + "real archived listings. " * 30 + monkeypatch.setattr(WB, "fetch_wayback", p_snapshot) + + out = await fetch(FetchBody(url="https://www.zillow.com/homes/for_sale/")) + assert out["backend"] == "wayback" + assert "real archived listings" in out["content"] + + +@pytest.mark.asyncio +async def test_a_genuine_browser_render_is_kept(monkeypatch): + p_local_returns(monkeypatch, "tiny", kind="html") + patch_browser_bridge(monkeypatch, {"text": "nasa\n104M followers\n" + "real profile content. " * 30}) + out = await fetch(FetchBody(url="https://www.instagram.com/nasa/")) + assert out["backend"] == "browser" + assert "real profile content" in out["content"]