WEBDIENSTE/openswarm
1
0
Fork 0
mirror of https://github.com/openswarm-ai/openswarm.git synced 2026-09-26 11:34:50 +02:00
Code Issues Packages Projects Releases Wiki Activity

[eric] web: ask the archive for the raw snapshot so we stop returning its calendar toolbar as the article

Browse Source
This commit is contained in:
ciregenz
2026-07-30 16:29:24 -07:00
parent a78d6276c0
commit 37100e9b5c
2 changed files with 29 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/apps/agents/tools/fetch/wayback.py
+10 -3
View File
@@ -4,7 +4,14 @@ When a page is gone (404, domain dead, article pulled) or sits behind a wall
that no client-side trick beats, the archive usually still has the REAL text,
which beats a grounded model's summary of a page it also couldn't read.
Uses `web.archive.org/web/2/<url>`, which redirects to the closest snapshot.
Uses `web.archive.org/web/2id_/<url>`, which redirects to the closest snapshot
and serves the ORIGINAL bytes. Without the `id_` the archive injects its own
calendar toolbar into the page, and on a Reddit snapshot that toolbar was all
the text there was: we returned "Jun JUL Aug 30 2025 2026 2027 success fail
About this capture" as if it were the article, and it cleared the substance
floor because it is made of real words. `id_` also keeps the page's own URL in
the extracted metadata instead of stamping it `hostname: archive.org`.
The documented `archive.org/wayback/available` JSON API is NOT used: it is
aggressively throttled and answered 429 on every probe from this machine, while
the redirect path answered in 0.5-3s.
@@ -19,12 +26,12 @@ from typeguard import typechecked
from backend.apps.agents.tools.browser_http import browser_request
from backend.apps.agents.tools.fetch.html_to_text import html_to_text
P_WAYBACK_LATEST = "https://web.archive.org/web/2/"
P_WAYBACK_LATEST = "https://web.archive.org/web/2id_/"
P_ALLOWED_HOST = "web.archive.org"
P_TIMEOUT = 10.0
# Below this the "snapshot" is a stub or an archived error page, not the article.
P_MIN_SUBSTANCE_CHARS = 200
P_SNAPSHOT_RE = re.compile(r"/web/(\d{4})(\d{2})(\d{2})\d*/")
P_SNAPSHOT_RE = re.compile(r"/web/(\d{4})(\d{2})(\d{2})\d*(?:id_)?/")
# What the archive shows when the crawler was bounced to a login page: it is a 200 with real words, so only the wording gives it away.
P_INTERSTITIAL_MARKER = "response at crawl time"
backend/tests/test_wayback_snapshot.py
+19
View File
@@ -67,3 +67,22 @@ async def test_offsite_redirect_is_refused(monkeypatch):
def test_snapshot_date_parsing():
assert snapshot_date(P_ARCHIVED_URL) == "2026-07-11"
assert snapshot_date("https://web.archive.org/nope") is None
def test_raw_snapshot_form_is_requested(monkeypatch):
"""Without `id_` the archive injects its calendar toolbar, and on a Reddit snapshot
that toolbar WAS the whole extracted text."""
from backend.apps.agents.tools.fetch.wayback import P_WAYBACK_LATEST
assert P_WAYBACK_LATEST.endswith("id_/")
def test_snapshot_date_parsing_survives_the_raw_form():
assert snapshot_date("https://web.archive.org/web/20260727222838id_/https://x.example/") == "2026-07-27"
@pytest.mark.asyncio
async def test_archive_toolbar_text_alone_is_below_the_floor(monkeypatch):
toolbar = ("<html><body>Jun JUL Aug 30 2025 2026 2027 success fail About this capture "
"COLLECTED BY Collection: Save Page Now TIMESTAMPS</body></html>")
p_patch(monkeypatch, 200, toolbar)
assert await fetch_wayback("https://www.reddit.com/r/programming/") is None