From 37100e9b5c4c14a526aed08fde9576fad0116a17 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Thu, 30 Jul 2026 16:29:24 -0700 Subject: [PATCH] [eric] web: ask the archive for the raw snapshot so we stop returning its calendar toolbar as the article --- backend/apps/agents/tools/fetch/wayback.py | 13 ++++++++++--- backend/tests/test_wayback_snapshot.py | 19 +++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/backend/apps/agents/tools/fetch/wayback.py b/backend/apps/agents/tools/fetch/wayback.py index a9668994..4f843d4d 100644 --- a/backend/apps/agents/tools/fetch/wayback.py +++ b/backend/apps/agents/tools/fetch/wayback.py @@ -4,7 +4,14 @@ When a page is gone (404, domain dead, article pulled) or sits behind a wall that no client-side trick beats, the archive usually still has the REAL text, which beats a grounded model's summary of a page it also couldn't read. -Uses `web.archive.org/web/2/`, which redirects to the closest snapshot. +Uses `web.archive.org/web/2id_/`, which redirects to the closest snapshot +and serves the ORIGINAL bytes. Without the `id_` the archive injects its own +calendar toolbar into the page, and on a Reddit snapshot that toolbar was all +the text there was: we returned "Jun JUL Aug 30 2025 2026 2027 success fail +About this capture" as if it were the article, and it cleared the substance +floor because it is made of real words. `id_` also keeps the page's own URL in +the extracted metadata instead of stamping it `hostname: archive.org`. + The documented `archive.org/wayback/available` JSON API is NOT used: it is aggressively throttled and answered 429 on every probe from this machine, while the redirect path answered in 0.5-3s. @@ -19,12 +26,12 @@ from typeguard import typechecked from backend.apps.agents.tools.browser_http import browser_request from backend.apps.agents.tools.fetch.html_to_text import html_to_text -P_WAYBACK_LATEST = "https://web.archive.org/web/2/" +P_WAYBACK_LATEST = "https://web.archive.org/web/2id_/" P_ALLOWED_HOST = "web.archive.org" P_TIMEOUT = 10.0 # Below this the "snapshot" is a stub or an archived error page, not the article. P_MIN_SUBSTANCE_CHARS = 200 -P_SNAPSHOT_RE = re.compile(r"/web/(\d{4})(\d{2})(\d{2})\d*/") +P_SNAPSHOT_RE = re.compile(r"/web/(\d{4})(\d{2})(\d{2})\d*(?:id_)?/") # What the archive shows when the crawler was bounced to a login page: it is a 200 with real words, so only the wording gives it away. P_INTERSTITIAL_MARKER = "response at crawl time" diff --git a/backend/tests/test_wayback_snapshot.py b/backend/tests/test_wayback_snapshot.py index a1fae957..b7225a6c 100644 --- a/backend/tests/test_wayback_snapshot.py +++ b/backend/tests/test_wayback_snapshot.py @@ -67,3 +67,22 @@ async def test_offsite_redirect_is_refused(monkeypatch): def test_snapshot_date_parsing(): assert snapshot_date(P_ARCHIVED_URL) == "2026-07-11" assert snapshot_date("https://web.archive.org/nope") is None + + +def test_raw_snapshot_form_is_requested(monkeypatch): + """Without `id_` the archive injects its calendar toolbar, and on a Reddit snapshot + that toolbar WAS the whole extracted text.""" + from backend.apps.agents.tools.fetch.wayback import P_WAYBACK_LATEST + assert P_WAYBACK_LATEST.endswith("id_/") + + +def test_snapshot_date_parsing_survives_the_raw_form(): + assert snapshot_date("https://web.archive.org/web/20260727222838id_/https://x.example/") == "2026-07-27" + + +@pytest.mark.asyncio +async def test_archive_toolbar_text_alone_is_below_the_floor(monkeypatch): + toolbar = ("Jun JUL Aug 30 2025 2026 2027 success fail About this capture " + "COLLECTED BY Collection: Save Page Now TIMESTAMPS") + p_patch(monkeypatch, 200, toolbar) + assert await fetch_wayback("https://www.reddit.com/r/programming/") is None