convert to flask and jinja from blade and php source & sync update bug and logic

2026-08-17 18:35:41 +02:00 · 2026-08-02 02:07:43 +07:00
parent f47507207b
commit e1b59d1b4f
11 changed files with 858 additions and 209 deletions
@@ -5,8 +5,16 @@
 1. Xquik API 
 2. Cookie (your account cookie session) 
 3. Wayback Machine (Cdx API)
+
+## Update Note
+
+1. Update infinity scroll and load new data 
+2. Update data corelation 
+3. Fix business logic flow 
 4. Monitoring (Soon)
 5. MCP (Soon)
+6. Add more parameter for enrichment 
+7. Add no rate limit (throttle)

 ## Setup

@@ -64,4 +72,8 @@ Xquik API DOC

 Offc doc: https://docs.xquik.com/api-reference/overview

-Soon i will check more detail about Twitter or X mechanism and business logic also endpoint API was listed in Mobile and Web 
+Soon i will check more detail about Twitter or X mechanism and business logic also endpoint API was listed in Mobile and Web 
+
+Wayback archive data source 
+
+The server connection to the Wayback Machine archive is often down, so try bumping the thread and don't set the throttle too high, and try checking the connection manually using curl.
@@ -3,6 +3,7 @@ import re
 import secrets
 import shutil
 import threading
+import time
 from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime
 import requests as _req
@@ -32,6 +33,36 @@ ACQUIRE_TIMEOUT         = 15  # seconds to wait before returning 429

 _sem = threading.Semaphore(MAX_CONCURRENT_REQUESTS)

+# ── Rate-limit protection for paginated load-more requests ─────────────────
+# Separate from the concurrency semaphore above, which limits how many
+# requests run *at once* — this limits how *often* the same external
+# account/service gets hit, regardless of which tab/tool triggered it. All
+# cookie-mode calls hit the same logged-in X account, so they share one
+# clock (a fresh search 2s after a scroll-load-more should still wait);
+# Wayback calls hit archive.org, unrelated to X's ban risk, so they get
+# their own independent clock.
+_THROTTLE_SECONDS = 5.0
+_throttle_lock     = threading.Lock()
+_last_call_at: dict = {"cookie": 0.0, "wayback": 0.0}
+
+_COOKIE_TOOLS = {
+    "tweet_search_extractor", "follower_explorer", "post_extractor",
+    "community_post_extractor", "tweet_replies_extractor",
+    "tweet_retweeters_extractor", "geo_post_extractor",
+}
+
+
+def _check_throttle(source: str):
+    """None if the call may proceed (and starts the next cooldown window);
+    otherwise the number of seconds still left to wait."""
+    now = time.monotonic()
+    with _throttle_lock:
+        elapsed = now - _last_call_at[source]
+        if elapsed < _THROTTLE_SECONDS:
+            return round(_THROTTLE_SECONDS - elapsed, 1)
+        _last_call_at[source] = now
+        return None
+
 # ── Cookie & session security ─────────────────────────────────────────────────

 _https = config.getboolean("server", "https", fallback=False)
@@ -202,11 +233,13 @@ def _filter_by_date(items: list, from_date: str, to_date: str) -> list:


 def _multi_source_search(query: str, count: int, from_date: str = "", to_date: str = "") -> list:
+    # Pagination isn't wired up for multi-source search yet (cookie + wayback
+    # only, per current scope) — grab just the items, discard the cursor.
    twitter_query = _apply_date_operators(query, from_date, to_date)
    jobs = {
-        "cookie":  lambda: cookie_tweet_search(twitter_query, count=count, config=config),
+        "cookie":  lambda: cookie_tweet_search(twitter_query, count=count, config=config)[0],
        "xquik":   lambda: XquikClient(config).tweet_search(twitter_query),
-        "wayback": lambda: wayback_search(query, count=count, from_date=from_date, to_date=to_date),
+        "wayback": lambda: wayback_search(query, count=count, from_date=from_date, to_date=to_date)[0],
    }
    with ThreadPoolExecutor(max_workers=len(jobs)) as pool:
        futures = {key: pool.submit(fn) for key, fn in jobs.items()}
@@ -254,6 +287,24 @@ def run_tool():
    tool_type = body.get("toolType")
    mode      = body.get("mode", "api")   # "api" | "cookie"
    count     = max(1, min(int(body.get("count", 20)), 200))
+    cursor    = body.get("cursor") or None   # opaque page token from a previous response's nextCursor
+
+    # Cookie/Wayback calls are throttled to one per 5s per source — checked
+    # up front, before taking a concurrency slot, so a request that's about
+    # to be rejected doesn't waste one.
+    throttle_source = None
+    if mode == "cookie" and tool_type in _COOKIE_TOOLS:
+        throttle_source = "cookie"
+    elif tool_type == "wayback_archive_search":
+        throttle_source = "wayback"
+    if throttle_source:
+        wait = _check_throttle(throttle_source)
+        if wait is not None:
+            return jsonify({
+                "ok": False,
+                "error": f"Please wait {wait}s before the next {throttle_source} request — this protects the account from rate limiting.",
+                "retryAfter": wait,
+            }), 429

    if not _sem.acquire(blocking=True, timeout=ACQUIRE_TIMEOUT):
        return jsonify({
@@ -261,18 +312,20 @@ def run_tool():
            "error": "Server is busy — max concurrent requests reached. Please try again shortly.",
        }), 429

+    next_cursor = None   # stays None for tools/modes that don't paginate
+
    try:
        if tool_type == "tweet_search_extractor":
            query = body.get("searchQuery", "")
            if mode == "cookie":
-                data = cookie_tweet_search(query, count=count, config=config)
+                data, next_cursor = cookie_tweet_search(query, count=count, config=config, cursor=cursor)
            else:
                data = XquikClient(config).tweet_search(query)

        elif tool_type == "follower_explorer":
            username = body.get("targetUsername", "")
            if mode == "cookie":
-                data = cookie_follower_explorer(username, count=count, config=config)
+                data, next_cursor = cookie_follower_explorer(username, count=count, config=config, cursor=cursor)
            else:
                data = XquikClient(config).follower_explorer(username)

@@ -286,14 +339,14 @@ def run_tool():
        elif tool_type == "community_post_extractor":
            community_id = body.get("targetCommunityId", "")
            if mode == "cookie":
-                data = cookie_community_post_extractor(community_id, count=count, config=config)
+                data, next_cursor = cookie_community_post_extractor(community_id, count=count, config=config, cursor=cursor)
            else:
                data = XquikClient(config).community_post_extractor(community_id)

        elif tool_type == "post_extractor":
            username = body.get("targetUsername", "")
            if mode == "cookie":
-                data = cookie_post_extractor(username, count=count, config=config)
+                data, next_cursor = cookie_post_extractor(username, count=count, config=config, cursor=cursor)
            else:
                data = XquikClient(config).post_extractor(username)

@@ -301,19 +354,19 @@ def run_tool():
            tweet_id = body.get("targetTweetId", "")
            if mode != "cookie":
                return jsonify({"ok": False, "error": "tweet_replies_extractor requires cookie mode"}), 400
-            data = cookie_tweet_replies(tweet_id, count=count, config=config)
+            data, next_cursor = cookie_tweet_replies(tweet_id, count=count, config=config, cursor=cursor)

        elif tool_type == "tweet_retweeters_extractor":
            tweet_id = body.get("targetTweetId", "")
            if mode != "cookie":
                return jsonify({"ok": False, "error": "tweet_retweeters_extractor requires cookie mode"}), 400
-            data = cookie_tweet_retweeters(tweet_id, count=count, config=config)
+            data, next_cursor = cookie_tweet_retweeters(tweet_id, count=count, config=config, cursor=cursor)

        elif tool_type == "geo_post_extractor":
            keyword = body.get("searchQuery", "")
            if mode != "cookie":
                return jsonify({"ok": False, "error": "geo_post_extractor requires cookie mode"}), 400
-            data = cookie_geo_search(keyword, count=count, config=config)
+            data, next_cursor = cookie_geo_search(keyword, count=count, config=config, cursor=cursor)

        elif tool_type == "wayback_archive_search":
            target    = body.get("searchQuery", "")
@@ -322,7 +375,7 @@ def run_tool():
            for label, val in (("waybackFrom", from_date), ("waybackTo", to_date)):
                if val and not _valid_date8(val):
                    return jsonify({"ok": False, "error": f"{label} must be an 8-digit date (YYYYMMDD)"}), 400
-            data = wayback_search(target, count=count, from_date=from_date, to_date=to_date)
+            data, next_cursor = wayback_search(target, count=count, from_date=from_date, to_date=to_date, cursor=cursor)

        elif tool_type == "multi_source_search":
            query     = body.get("searchQuery", "")
@@ -342,7 +395,7 @@ def run_tool():
        # in archive.py's _pick_fields).
        data = enrich_account_age(data)

-        return jsonify({"ok": True, "data": data})
+        return jsonify({"ok": True, "data": data, "nextCursor": next_cursor})

    except (XquikError, CookieClientError, WaybackError) as e:
        return jsonify({"ok": False, "error": str(e)}), 400
@@ -395,12 +448,19 @@ def archive_delete(archive_id):

@app.route("/api/archive", methods=["POST"])
 def archive_start():
-    body       = request.get_json(silent=True) or {}
-    tool_type  = body.get("toolType", "unknown")
-    data       = body.get("data")
-    query_info = body.get("queryInfo", {})
+    body        = request.get_json(silent=True) or {}
+    tool_type   = body.get("toolType", "unknown")
+    data        = body.get("data")
+    query_info  = body.get("queryInfo", {})
+    existing_id = body.get("archiveId")   # present -> checkpoint update, not a new archive
    if not data:
        return jsonify({"ok": False, "error": "No data provided"}), 400
+
+    if existing_id:
+        if not _archive.update(existing_id, tool_type, data, query_info):
+            return jsonify({"ok": False, "error": "Archive not found"}), 404
+        return jsonify({"ok": True, "archiveId": existing_id})
+
    archive_id = _archive.start(tool_type, data, query_info)
    return jsonify({"ok": True, "archiveId": archive_id})

@@ -64,7 +64,8 @@ def _pick_fields(item: dict) -> dict:
    keys = ["id", "text", "full_text", "article_text", "user", "user_id",
            "created_at", "reply_count", "retweet_count", "favorite_count",
            "view_count", "in_reply_to_tweet_id", "name", "screen_name",
-            "description", "followers_count", "following_count",
+            "verified", "is_blue_verified",
+            "description", "followers_count", "following_count", "tweet_count",
            "lat", "lon", "place",
            "retweeted_text", "retweeted_by_user", "retweeted_by_name",
            "retweeted_by_bio", "retweeted_at", "retweeted_tweet_id",
@@ -107,21 +108,39 @@ def _run(archive_id: str, tool_type: str, data, query_info: dict) -> None:
            fname = f"{item.get('id', 'unknown')}_{mtype}_{midx}.{ext}"
            dest  = media_dir / fname
            local_media.append(f"media/{fname}")
-            media_queue.append((url, dest))
+            # Skip re-downloading media that's already on disk — matters for
+            # checkpoint updates, which re-run this on a growing dataset that
+            # mostly overlaps with what was already archived.
+            if not dest.exists():
+                media_queue.append((url, dest))

        if local_media:
            record["archived_media"] = local_media
        enriched.append(record)

+    # Preserve the original archived_at across checkpoint updates (re-runs
+    # of this on an archive_id that already exists) rather than overwriting it.
+    meta_path     = archive_dir / "meta.json"
+    existing_meta = {}
+    if meta_path.exists():
+        try:
+            existing_meta = json.loads(meta_path.read_text())
+        except Exception:
+            existing_meta = {}
+
+    now_iso     = datetime.now().isoformat(timespec="seconds")
+    total_media = sum(len(r.get("archived_media", [])) for r in enriched if isinstance(r, dict))
+
    # Write metadata + results immediately (no waiting on media)
    meta = {
        "tool":        tool_type,
        "query":       query_info,
-        "archived_at": datetime.now().isoformat(timespec="seconds"),
+        "archived_at": existing_meta.get("archived_at", now_iso),
+        "updated_at":  now_iso,
        "total_items": len(items),
-        "media_count": len(media_queue),
+        "media_count": total_media,
    }
-    (archive_dir / "meta.json").write_text(
+    meta_path.write_text(
        json.dumps(meta, indent=2, ensure_ascii=False)
    )
    (archive_dir / "results.json").write_text(
@@ -160,6 +179,28 @@ def start(tool_type: str, data, query_info: dict) -> str:
    return archive_id


+def update(archive_id: str, tool_type: str, data, query_info: dict) -> bool:
+    """Re-run the archive pipeline against an EXISTING archive_id — the
+    "save checkpoint" flow, used once scroll/expand-load-more has fetched
+    more data than what was first archived. Overwrites results.json/meta.json
+    with the current (larger) dataset; media already on disk is skipped
+    rather than re-downloaded. Returns False if archive_id doesn't exist."""
+    archive_dir = ARCHIVE_ROOT / archive_id
+    if not archive_dir.exists():
+        return False
+
+    with _lock:
+        _registry[archive_id] = {"status": "saving", "progress": 0, "total": 0, "path": None}
+
+    t = threading.Thread(
+        target=_run,
+        args=(archive_id, tool_type, data, query_info),
+        daemon=True,
+    )
+    t.start()
+    return True
+
+
 def status(archive_id: str) -> dict | None:
    with _lock:
        entry = _registry.get(archive_id)
@@ -167,11 +208,16 @@ def status(archive_id: str) -> dict | None:


 def list_all() -> list[dict]:
-    """Read meta.json from every archive folder, newest first."""
+    """Read meta.json from every archive folder, newest first — sorted by the
+    archive's own archived_at/updated_at timestamp, not folder name. Folder
+    names are prefixed by tool_type (e.g. "graph_tweet_search_..." vs
+    "tweet_search_..."), so sorting by name doesn't actually sort
+    chronologically once more than one tool has been archived. A checkpoint
+    update bumps updated_at, so it resurfaces near the top too."""
    if not ARCHIVE_ROOT.exists():
        return []
    results = []
-    for d in sorted(ARCHIVE_ROOT.iterdir(), key=lambda p: p.name, reverse=True):
+    for d in ARCHIVE_ROOT.iterdir():
        meta_file = d / "meta.json"
        if not meta_file.exists():
            continue
@@ -181,4 +227,5 @@ def list_all() -> list[dict]:
            results.append(meta)
        except Exception:
            pass
+    results.sort(key=lambda m: m.get("updated_at") or m.get("archived_at") or "", reverse=True)
    return results
@@ -80,6 +80,11 @@ def _tweet_to_dict(t: object) -> dict:
        "user":           getattr(user_obj, "screen_name", None) if user_obj else None,
        "user_id":        _id_str(getattr(user_obj, "id", None)) if user_obj else None,
        "user_location":  getattr(user_obj, "location", None) if user_obj else None,
+        # Display name + verification badge — Twitter's own reply UI shows both
+        # next to the handle; twikit already exposes them on the tweet's user.
+        "name":                 getattr(user_obj, "name", None) if user_obj else None,
+        "verified":             getattr(user_obj, "verified", None) if user_obj else None,
+        "is_blue_verified":     getattr(user_obj, "is_blue_verified", None) if user_obj else None,
        "reply_count":          getattr(t, "reply_count", None),
        "retweet_count":        getattr(t, "retweet_count", None),
        "favorite_count":       getattr(t, "favorite_count", None),
@@ -117,24 +122,35 @@ async def _resolve_user(client, identifier: str):
    return await client.get_user_by_screen_name(clean)


-async def _tweet_search_async(query: str, auth_token: str, ct0: str, count: int) -> list:
+def _next_cursor(result) -> str | None:
+    """A zero-item page always means "exhausted," regardless of what cursor
+    value twikit hands back — avoids chasing a stale/looping cursor."""
+    if not len(result):
+        return None
+    return getattr(result, "next_cursor", None) or None
+
+
+async def _tweet_search_async(query: str, auth_token: str, ct0: str, count: int, cursor: str | None) -> tuple[list, str | None]:
    client  = await _make_client(auth_token, ct0)
-    results = await client.search_tweet(query, "Latest", count=count)
-    return [_tweet_to_dict(t) for t in results]
+    results = await client.search_tweet(query, "Latest", count=count, cursor=cursor)
+    return [_tweet_to_dict(t) for t in results], _next_cursor(results)


-async def _follower_explorer_async(username: str, auth_token: str, ct0: str, count: int) -> list:
+async def _follower_explorer_async(username: str, auth_token: str, ct0: str, count: int, cursor: str | None) -> tuple[list, str | None]:
    client    = await _make_client(auth_token, ct0)
    user      = await _resolve_user(client, username)
-    followers = await user.get_followers(count=count)
-    return [_user_to_dict(u) for u in followers]
+    # Bypass the User.get_followers() convenience wrapper — it doesn't accept
+    # a cursor at all, so it can't be resumed across requests.
+    followers = await client.get_user_followers(str(user.id), count=count, cursor=cursor)
+    return [_user_to_dict(u) for u in followers], _next_cursor(followers)


-async def _post_extractor_async(username: str, auth_token: str, ct0: str, count: int) -> list:
+async def _post_extractor_async(username: str, auth_token: str, ct0: str, count: int, cursor: str | None) -> tuple[list, str | None]:
    client = await _make_client(auth_token, ct0)
    user   = await _resolve_user(client, username)
-    tweets = await user.get_tweets("Tweets", count=count)
-    return [_tweet_to_dict(t) for t in tweets]
+    # Bypass User.get_tweets() for the same reason as followers above.
+    tweets = await client.get_user_tweets(str(user.id), "Tweets", count=count, cursor=cursor)
+    return [_tweet_to_dict(t) for t in tweets], _next_cursor(tweets)


 async def _article_extractor_async(tweet_id: str, auth_token: str, ct0: str) -> dict:
@@ -150,50 +166,52 @@ async def _article_extractor_async(tweet_id: str, auth_token: str, ct0: str) ->
    return result


-async def _community_posts_async(community_id: str, auth_token: str, ct0: str, count: int) -> list:
+async def _community_posts_async(community_id: str, auth_token: str, ct0: str, count: int, cursor: str | None) -> tuple[list, str | None]:
    client = await _make_client(auth_token, ct0)
-    posts  = await client.get_community_tweets(community_id, "Latest", count=count)
-    return [_tweet_to_dict(t) for t in posts]
+    posts  = await client.get_community_tweets(community_id, "Latest", count=count, cursor=cursor)
+    return [_tweet_to_dict(t) for t in posts], _next_cursor(posts)


-async def _tweet_replies_async(tweet_id: str, auth_token: str, ct0: str, count: int) -> list:
+async def _tweet_replies_async(tweet_id: str, auth_token: str, ct0: str, count: int, cursor: str | None) -> tuple[list, str | None]:
+    """Direct top-level replies to `tweet_id`, fetched via the same GraphQL
+    TweetDetail call that powers Twitter's own UI reply view — not a keyword
+    search. `search_tweet(f"conversation_id:...")` was tried first, but it
+    caps out around ~20 raw results with no way to page further regardless
+    of the requested count, and mixes in replies-to-replies from anywhere in
+    the thread (a 29-reply post returned 20 raw items with only 3 actually
+    replying to the target). TweetDetail already separates "direct replies to
+    this exact tweet" cleanly and supports proper cursor pagination.
+
+    `count` is advisory only here — `get_tweet_by_id` has no page-size knob,
+    X's TweetDetail backend decides how many replies come back per page. We
+    return the page whole rather than slicing to `count`: once the cursor is
+    handed back to the caller for real cross-request resumption, slicing
+    would permanently strand whatever got cut (the cursor already points
+    past those rows). Getting the rest is what the next paginated request
+    (scroll / expand-again) is for, not a bigger `count`.
+
+    Continuation pages need a different call than the first page: X's
+    TweetDetail response for a cursor-based request does NOT include the
+    root tweet's own entry (only reply entries + a trailing cursor), but
+    `get_tweet_by_id` unconditionally requires finding that entry — passing
+    it a cursor beyond page 1 raises `AttributeError: 'NoneType' object has
+    no attribute 'replies'` (confirmed empirically). `Client._get_more_replies`
+    is twikit's own handler for exactly this response shape — it's what
+    `Result.next()` calls internally — so we call it directly for page 2+.
+    It's a private method (fragile to twikit internals changing), but
+    there's no public equivalent for resuming pagination across a fresh
+    request/session rather than an in-memory `Result` object.
+    """
    client = await _make_client(auth_token, ct0)
-
-    # Walk up the in_reply_to chain to find the conversation root.
-    # tweet.in_reply_to → _legacy['in_reply_to_status_id_str'] (parent tweet ID string).
-    # There is no conversation_id attribute on twikit Tweet objects — the only way
-    # to reach the root is to follow the chain until in_reply_to is None.
-    conversation_id = tweet_id
-    current_id      = tweet_id
-
-    for _ in range(6):   # guard: max 6 hops up the thread
-        try:
-            node      = await client.get_tweet_by_id(current_id)
-            parent_id = getattr(node, "in_reply_to", None)
-            if not parent_id:
-                conversation_id = current_id   # reached root
-                break
-            current_id      = str(parent_id)
-            conversation_id = current_id
-        except Exception:
-            break
-
-    # Fetch all tweets in the conversation thread
-    results = await client.search_tweet(
-        f"conversation_id:{conversation_id}", "Latest", count=count
-    )
-
-    tweets = [_tweet_to_dict(t) for t in results]
-
-    # If the user clicked on a non-root reply, filter to that reply's direct children
-    if conversation_id != tweet_id:
-        direct = [d for d in tweets if d.get("in_reply_to_tweet_id") == tweet_id]
-        return direct if direct else tweets   # fallback: full thread
-
-    return tweets
+    if cursor:
+        replies = await client._get_more_replies(tweet_id, cursor)
+    else:
+        tweet   = await client.get_tweet_by_id(tweet_id)
+        replies = tweet.replies
+    return [_tweet_to_dict(t) for t in replies], _next_cursor(replies)


-async def _tweet_retweeters_async(tweet_id: str, auth_token: str, ct0: str, count: int) -> list:
+async def _tweet_retweeters_async(tweet_id: str, auth_token: str, ct0: str, count: int, cursor: str | None) -> tuple[list, str | None]:
    client = await _make_client(auth_token, ct0)

    # Fetch the original tweet once so every retweeter card shows what was retweeted
@@ -212,42 +230,45 @@ async def _tweet_retweeters_async(tweet_id: str, auth_token: str, ct0: str, coun
    except Exception:
        pass

-    retweeters = await client.get_retweeters(tweet_id, count=count)
+    retweeters = await client.get_retweeters(tweet_id, count=count, cursor=cursor)

    result = []
    for u in retweeters:
        # retweeted content first → shows prominently in the card
        d = {**rt_info, **_user_to_dict(u)}
        result.append(d)
-    return result
+    return result, _next_cursor(retweeters)


-async def _geo_search_async(keyword: str, auth_token: str, ct0: str, count: int) -> list:
+async def _geo_search_async(keyword: str, auth_token: str, ct0: str, count: int, cursor: str | None) -> tuple[list, str | None]:
    """Keyword search; user_location (profile location string) is included in every
    result so the frontend can geocode and plot it on a map."""
    client  = await _make_client(auth_token, ct0)
-    results = await client.search_tweet(keyword, "Latest", count=count)
-    return [_tweet_to_dict(t) for t in results]
+    results = await client.search_tweet(keyword, "Latest", count=count, cursor=cursor)
+    return [_tweet_to_dict(t) for t in results], _next_cursor(results)


 # ── Public sync wrappers ──────────────────────────────────────────────────────
+# Each pagination-capable wrapper returns (items, next_cursor). Pass the
+# previous response's next_cursor back in as `cursor` to fetch the next page;
+# `next_cursor` is None once there's nothing more to load.

-def cookie_tweet_search(query: str, count: int = 20, config: configparser.ConfigParser = None) -> list:
+def cookie_tweet_search(query: str, count: int = 20, config: configparser.ConfigParser = None, cursor: str | None = None) -> tuple[list, str | None]:
    cfg = config or load_config()
    auth, ct0 = _get_creds(cfg)
-    return asyncio.run(_tweet_search_async(query, auth, ct0, count))
+    return asyncio.run(_tweet_search_async(query, auth, ct0, count, cursor))


-def cookie_follower_explorer(username: str, count: int = 20, config: configparser.ConfigParser = None) -> list:
+def cookie_follower_explorer(username: str, count: int = 20, config: configparser.ConfigParser = None, cursor: str | None = None) -> tuple[list, str | None]:
    cfg = config or load_config()
    auth, ct0 = _get_creds(cfg)
-    return asyncio.run(_follower_explorer_async(username, auth, ct0, count))
+    return asyncio.run(_follower_explorer_async(username, auth, ct0, count, cursor))


-def cookie_post_extractor(username: str, count: int = 20, config: configparser.ConfigParser = None) -> list:
+def cookie_post_extractor(username: str, count: int = 20, config: configparser.ConfigParser = None, cursor: str | None = None) -> tuple[list, str | None]:
    cfg = config or load_config()
    auth, ct0 = _get_creds(cfg)
-    return asyncio.run(_post_extractor_async(username, auth, ct0, count))
+    return asyncio.run(_post_extractor_async(username, auth, ct0, count, cursor))


 def cookie_article_extractor(tweet_id: str, config: configparser.ConfigParser = None) -> dict:
@@ -257,31 +278,31 @@ def cookie_article_extractor(tweet_id: str, config: configparser.ConfigParser =


 def cookie_community_post_extractor(
-    community_id: str, count: int = 20, config: configparser.ConfigParser = None
-) -> list:
+    community_id: str, count: int = 20, config: configparser.ConfigParser = None, cursor: str | None = None
+) -> tuple[list, str | None]:
    cfg = config or load_config()
    auth, ct0 = _get_creds(cfg)
-    return asyncio.run(_community_posts_async(community_id, auth, ct0, count))
+    return asyncio.run(_community_posts_async(community_id, auth, ct0, count, cursor))


-def cookie_tweet_replies(tweet_id: str, count: int = 50, config: configparser.ConfigParser = None) -> list:
+def cookie_tweet_replies(tweet_id: str, count: int = 50, config: configparser.ConfigParser = None, cursor: str | None = None) -> tuple[list, str | None]:
    cfg = config or load_config()
    auth, ct0 = _get_creds(cfg)
-    return asyncio.run(_tweet_replies_async(tweet_id, auth, ct0, count))
+    return asyncio.run(_tweet_replies_async(tweet_id, auth, ct0, count, cursor))


-def cookie_tweet_retweeters(tweet_id: str, count: int = 50, config: configparser.ConfigParser = None) -> list:
+def cookie_tweet_retweeters(tweet_id: str, count: int = 50, config: configparser.ConfigParser = None, cursor: str | None = None) -> tuple[list, str | None]:
    cfg = config or load_config()
    auth, ct0 = _get_creds(cfg)
-    return asyncio.run(_tweet_retweeters_async(tweet_id, auth, ct0, count))
+    return asyncio.run(_tweet_retweeters_async(tweet_id, auth, ct0, count, cursor))


 def cookie_geo_search(
-    keyword: str, count: int = 20, config: configparser.ConfigParser = None,
-) -> list:
+    keyword: str, count: int = 20, config: configparser.ConfigParser = None, cursor: str | None = None,
+) -> tuple[list, str | None]:
    cfg = config or load_config()
    auth, ct0 = _get_creds(cfg)
-    return asyncio.run(_geo_search_async(keyword, auth, ct0, count))
+    return asyncio.run(_geo_search_async(keyword, auth, ct0, count, cursor))


 # Legacy alias — kept for any external scripts that import this name directly
@@ -316,15 +337,15 @@ if __name__ == "__main__":
    args = parser.parse_args()
    try:
        if args.cmd == "tweet_search":
-            out = cookie_tweet_search(args.query)
+            out, _ = cookie_tweet_search(args.query)
        elif args.cmd == "follower_explorer":
-            out = cookie_follower_explorer(args.username, count=args.count)
+            out, _ = cookie_follower_explorer(args.username, count=args.count)
        elif args.cmd == "post_extractor":
-            out = cookie_post_extractor(args.username, count=args.count)
+            out, _ = cookie_post_extractor(args.username, count=args.count)
        elif args.cmd == "article_extractor":
            out = cookie_article_extractor(args.tweet_id)
        elif args.cmd == "community_post_extractor":
-            out = cookie_community_post_extractor(args.community_id, count=args.count)
+            out, _ = cookie_community_post_extractor(args.community_id, count=args.count)
        print(json.dumps(out, indent=2, ensure_ascii=False))
    except CookieClientError as e:
        print(f"[ERROR] {e}")
@@ -0,0 +1,30 @@
+// ── Shared card-rendering rules — kept byte-identical across index.html and
+// archive.html by loading this one file, so field order / drill-down /
+// badges never drift between "live search" and "saved archive" views of the
+// same data. (Previously included via Jinja {% include %} directly inside a
+// <script> block — moved to a real .js file so editor/JS tooling can lint it
+// normally instead of flagging the Jinja syntax as invalid JavaScript.)
+const PRIORITY = [
+  'source', 'account_age_flag', 'account_age', 'account_created',
+  'user', 'screen_name', 'name', 'user_id', 'username',
+  'verified', 'is_blue_verified',
+  'text', 'full_text', 'article_text', 'post_title', 'post_text', 'content', 'title', 'description', 'bio',
+  'reply_count', 'retweet_count', 'favorite_count', 'view_count',
+  'followers_count', 'following_count', 'tweet_count',
+  'created_at', 'in_reply_to_tweet_id',
+  'retweeted_by_user', 'retweeted_by_name', 'retweeted_text', 'retweeted_by_bio', 'retweeted_at', 'retweeted_tweet_id',
+  'lat', 'lon', 'place', 'user_location',
+  'tweet_url', 'archive_url', 'preview_image',
+  'iso_date', 'original', 'statuscode', 'mimetype', 'length',
+];
+
+// Which fields link out to a fresh extraction for that tweet — same anchor
+// behavior whether you're looking at a live result or a saved archive.
+const DRILLABLE = {
+  reply_count:          'tweet_replies_extractor',
+  retweet_count:        'tweet_retweeters_extractor',
+  in_reply_to_tweet_id: 'tweet_replies_extractor',
+};
+
+const SOURCE_CLASS = { 'Twitter Cookie': 'src-cookie', 'Xquik API': 'src-xquik', 'Wayback Machine': 'src-wayback' };
+const AGE_LABELS   = { new: 'New account', recent: 'Recent account', established: 'Established account' };
@@ -20,24 +20,34 @@
 <div class="param-row"><code class="param-key">is_blue_verified</code> Paid X Premium checkmark</div>

 <div class="param-heading">Threading &amp; retweets</div>
-<div class="param-row"><code class="param-key">in_reply_to_tweet_id</code> Parent tweet this replies to</div>
-<div class="param-row"><code class="param-key">retweeted_*</code> Details of the original tweet being retweeted</div>
+<div class="param-row"><code class="param-key">in_reply_to_tweet_id</code> Parent tweet this replies to — also the drill-down anchor that fetches its replies</div>
+<div class="param-row"><code class="param-key">retweeted_text / retweeted_by_user / retweeted_by_name / retweeted_by_bio</code> Content and author of the original tweet being retweeted</div>
+<div class="param-row"><code class="param-key">retweeted_at / retweeted_tweet_id</code> When the original was posted, and its own ID</div>

 <div class="param-heading">Location</div>
-<div class="param-row"><code class="param-key">lat / lon / place / user_location</code> Geodata — geo search only</div>
+<div class="param-row"><code class="param-key">lat / lon / place</code> Coordinates plotted on the map view (Geo Post Search)</div>
+<div class="param-row"><code class="param-key">user_location</code> Free-text profile location string, geocoded client-side to produce lat/lon</div>

 <div class="param-heading">Account age (forensics)</div>
-<div class="param-row"><code class="param-key">account_created / account_age</code> Derived from the account's numeric ID, not the API — see badges above</div>
-<div class="param-row"><code class="param-key">account_age_flag</code> New / Recent / Established bucket</div>
-<div class="param-row"><code class="param-key">account_age_precision</code> exact / estimated / unknown — how much to trust the date</div>
+<div class="param-row"><code class="param-key">account_created / account_age</code> Derived from the account's numeric ID, not the API — see the Account age badges above</div>
+<div class="param-row"><code class="param-key">account_age_flag</code> New / Recent / Established bucket, drives the badge color</div>
+<div class="param-row"><code class="param-key">account_age_precision</code> exact (true Snowflake decode) / estimated (pre-Snowflake id, interpolated) / unknown</div>

 <div class="param-heading">Archive &amp; Wayback</div>
 <div class="param-row"><code class="param-key">archive_url</code> Link to the Wayback Machine snapshot</div>
 <div class="param-row"><code class="param-key">iso_date</code> When that snapshot was captured</div>
 <div class="param-row"><code class="param-key">original</code> The original URL that was archived</div>
-<div class="param-row"><code class="param-key">statuscode / mimetype / length</code> HTTP status / type / size of the snapshot</div>
-<div class="param-row"><code class="param-key">post_title / post_text / preview_image</code> Scraped from the archived page's meta tags</div>
-<div class="param-row"><code class="param-key">tweet_url</code> Direct link to the live tweet</div>
+<div class="param-row"><code class="param-key">statuscode / mimetype / length</code> HTTP status / content type / size of the snapshot</div>
+<div class="param-row"><code class="param-key">post_title / post_text / preview_image</code> Scraped from the archived page's own meta tags (og:/twitter: tags, or &lt;title&gt;/&lt;meta name="description"&gt; on older captures)</div>
+<div class="param-row"><code class="param-key">tweet_url</code> Direct link to the live tweet, built from user + id</div>

 <div class="param-heading">Source</div>
-<div class="param-row"><code class="param-key">source</code> Which of the 3 data sources this result came from (multi-source search only)</div>
+<div class="param-row"><code class="param-key">source</code> Which of the 3 data sources this result came from — Twitter Cookie / Xquik API / Wayback Machine (Multi-Source Search only)</div>
+
+<div class="param-heading">Graph node types (graph page only)</div>
+<div class="param-row"><code class="param-key">Search root</code> The diamond node — the query you ran</div>
+<div class="param-row"><code class="param-key">Tweet</code> A tweet/post returned by search or by expanding a node</div>
+<div class="param-row"><code class="param-key">Reply</code> A tweet fetched via Expand Replies on a tweet node</div>
+<div class="param-row"><code class="param-key">User / Retweeter</code> A person — from Follower Explorer, or via Expand Retweets on a tweet node</div>
+<div class="param-row"><code class="param-key">Wayback snapshot</code> An archived-page result, from Wayback Archive Search or the Wayback portion of Multi-Source Search</div>
+<div class="param-row"><code class="param-key">Viewed</code> Turns a node solid white once you've clicked it — a visual "already looked at this" marker, not part of the underlying data</div>
@@ -168,6 +168,19 @@
  }
  .sidebar-title { font-size: 11px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.06em; color: var(--muted); }
  .sidebar-count { font-size: 11px; color: var(--muted); background: var(--bg); padding: 2px 7px; border-radius: 10px; }
+  .sidebar-search {
+    width: 100%;
+    padding: 6px 10px;
+    background: var(--bg);
+    border: 1px solid var(--border);
+    color: var(--text);
+    font-family: var(--font);
+    font-size: 12px;
+    border-radius: 6px;
+    transition: border-color 0.15s;
+  }
+  .sidebar-search:focus { outline: none; border-color: var(--accent); }
+  .sidebar-search::placeholder { color: var(--muted); }
  .archive-list { overflow-y: auto; flex: 1; padding: 8px; }
  .archive-entry {
    padding: 10px 12px;
@@ -289,6 +302,16 @@
  .card-link { color: var(--accent); text-decoration: none; font-size: 12px; }
  .card-link:hover { text-decoration: underline; }

+  /* Drill-down links on reply / retweet counts — same as the live search page */
+  .drill-link {
+    color: var(--accent);
+    font-family: var(--font);
+    font-size: 13px;
+    text-decoration: underline;
+    text-underline-offset: 2px;
+  }
+  .drill-link:hover { color: #818cf8; }
+
  .source-badge {
    display: inline-block;
    font-size: 11px;
@@ -373,6 +396,7 @@
  }
  @keyframes spin { to { transform: rotate(360deg); } }
 </style>
+<script src="{{ url_for('static', filename='js/card_constants.js') }}" nonce="{{ g.csp_nonce }}"></script>
 </head>
 <body>

@@ -418,6 +442,9 @@
      <span class="sidebar-title">Archives</span>
      <span class="sidebar-count" id="archiveCount">—</span>
    </div>
+    <div style="padding:8px 12px 0">
+      <input type="text" class="sidebar-search" id="archiveSearch" placeholder="Search archives…">
+    </div>
    <div class="archive-list" id="archiveList">
      <div class="no-archives"><span class="spinner"></span> Loading…</div>
    </div>
@@ -471,9 +498,11 @@

 let allItems      = [];
 let activeId      = null;
+let allArchives   = [];   // full list from the server; archiveSearch filters this client-side

-const archiveList  = document.getElementById('archiveList');
-const archiveCount = document.getElementById('archiveCount');
+const archiveList   = document.getElementById('archiveList');
+const archiveCount  = document.getElementById('archiveCount');
+const archiveSearch = document.getElementById('archiveSearch');
 const viewerTop    = document.getElementById('viewerTop');
 const viewerTitle  = document.getElementById('viewerTitle');
 const viewerSub    = document.getElementById('viewerSub');
@@ -483,28 +512,52 @@ const deleteBtn    = document.getElementById('deleteBtn');
 const cardsBox     = document.getElementById('cardsBox');

 // ── Sidebar ───────────────────────────────────────────────────────────────────
+// The server already returns archives newest-first (by archived_at/updated_at,
+// not folder name — folder names are prefixed by tool_type so sorting by name
+// doesn't actually sort chronologically once more than one tool's been used).

 async function loadSidebar() {
  const res  = await fetch('/api/archive/list');
  const json = await res.json();

  if (!json.ok || !json.archives.length) {
+    allArchives = [];
    archiveList.innerHTML  = '<div class="no-archives">No archives yet.<br>Enable Auto Archive in the tool and run a search.</div>';
    archiveCount.textContent = '0';
    return;
  }

-  archiveCount.textContent = json.archives.length;
-  archiveList.innerHTML    = json.archives.map(a => entryHtml(a)).join('');
-
-  archiveList.querySelectorAll('.archive-entry').forEach(el => {
-    el.addEventListener('click', () => loadArchive(el.dataset.id));
-  });
+  allArchives = json.archives;
+  renderArchiveList(archiveSearch.value);

  // Auto-open the newest one
-  loadArchive(json.archives[0].id);
+  loadArchive(allArchives[0].id);
 }

+function archiveMatchText(a) {
+  return [a.tool || '', queryLabel(a.query || {}), a.id || ''].join(' ').toLowerCase();
+}
+
+function renderArchiveList(filterText = '') {
+  const q        = filterText.toLowerCase().trim();
+  const filtered = q ? allArchives.filter(a => archiveMatchText(a).includes(q)) : allArchives;
+
+  archiveCount.textContent = q ? `${filtered.length} / ${allArchives.length}` : allArchives.length;
+
+  if (!filtered.length) {
+    archiveList.innerHTML = '<div class="no-archives">No archives match your search.</div>';
+    return;
+  }
+
+  archiveList.innerHTML = filtered.map(a => entryHtml(a)).join('');
+  archiveList.querySelectorAll('.archive-entry').forEach(el => {
+    el.classList.toggle('active', el.dataset.id === activeId);
+    el.addEventListener('click', () => loadArchive(el.dataset.id));
+  });
+}
+
+archiveSearch.addEventListener('input', () => renderArchiveList(archiveSearch.value));
+
 function entryHtml(a) {
  const tool  = (a.tool || 'unknown').replace(/_/g, ' ');
  const date  = a.archived_at ? a.archived_at.replace('T', ' ') : '—';
@@ -572,10 +625,10 @@ function flatText(obj) {

 // ── Cards ─────────────────────────────────────────────────────────────────────

-const PRIORITY = ['source','account_age_flag','account_age','account_created','user','user_id','text','full_text','article_text','post_title','post_text','created_at','tweet_url','archive_url','preview_image','iso_date','original','statuscode'];
+// PRIORITY / DRILLABLE / SOURCE_CLASS / AGE_LABELS come from
+// static/js/card_constants.js, loaded above — shared with index.html.
 const SKIP     = ['archived_media'];
 const CLAMP    = new Set(['text','full_text','article_text','description','post_text']);
-const SOURCE_CLASS = { 'Twitter Cookie': 'src-cookie', 'Xquik API': 'src-xquik', 'Wayback Machine': 'src-wayback' };

 function renderCards(items, query) {
  const q        = query.toLowerCase().trim();
@@ -593,13 +646,24 @@ function renderCards(items, query) {
 function buildCard(item) {
  if (typeof item !== 'object' || !item) return '';

+  const tweetId = item.id ? String(item.id) : null;
  const entries = Object.entries(item);
-  const pri  = entries.filter(([k]) => PRIORITY.includes(k));
-  const rest = entries.filter(([k]) => !PRIORITY.includes(k) && !SKIP.includes(k));
+  // Same rule as the live search page: order follows PRIORITY's own sequence,
+  // and nothing gets capped off — a saved archive should show exactly the
+  // same fields, in the same order, as when it was first fetched.
+  const priKeys = new Set(entries.map(([k]) => k).filter(k => PRIORITY.includes(k)));
+  const pri     = PRIORITY.filter(k => priKeys.has(k)).map(k => [k, item[k]]);
+  const rest    = entries.filter(([k]) => !PRIORITY.includes(k) && !SKIP.includes(k));

-  const rows = [...pri, ...rest].slice(0, 14).map(([k, v]) => {
+  const rows = [...pri, ...rest].map(([k, v]) => {
    let display;
-    if (v === null || v === undefined) {
+    // Drillable check first — reply_count / retweet_count on an archived
+    // tweet still link out to a fresh live extraction, same as a live result.
+    if (DRILLABLE[k] && tweetId) {
+      const _p  = new URLSearchParams({ tool: DRILLABLE[k], id: tweetId, autorun: '1' });
+      const lbl = (v !== null && v !== undefined) ? esc(String(v)) : '—';
+      display = `<a class="drill-link" href="/?${_p}" target="_blank" rel="noopener noreferrer">${lbl} ↗</a>`;
+    } else if (v === null || v === undefined) {
      display = `<span style="color:var(--muted)">—</span>`;
    } else if (k === 'tweet_url' || k === 'archive_url' || k === 'preview_image') {
      display = `<a href="${esc(String(v))}" target="_blank" rel="noopener" class="card-link">${esc(String(v))}</a>`;
@@ -607,8 +671,7 @@ function buildCard(item) {
      const cls = SOURCE_CLASS[v] || '';
      display = `<span class="source-badge ${cls}">${esc(String(v))}</span>`;
    } else if (k === 'account_age_flag') {
-      const label = { new: 'New account', recent: 'Recent account', established: 'Established account' }[v] || v;
-      display = `<span class="age-badge age-${esc(String(v))}">${esc(label)}</span>`;
+      display = `<span class="age-badge age-${esc(String(v))}">${esc(AGE_LABELS[v] || v)}</span>`;
    } else if (typeof v === 'object') {
      const s = JSON.stringify(v);
      display = `<span style="color:var(--muted);font-size:11px">${esc(s.length > 100 ? s.slice(0,100)+'…' : s)}</span>`;
@@ -137,6 +137,29 @@
  .btn-danger:hover:not(:disabled)  { background: rgba(239,68,68,0.1); border-color: var(--danger); }
  .btn:disabled { opacity: 0.38; cursor: not-allowed; }

+  /* ── Archive progress bar — same look as the non-graph search page ── */
+  .archive-bar {
+    padding: 6px 16px;
+    background: var(--surface);
+    border-bottom: 1px solid var(--border);
+    font-size: 12px;
+    color: var(--muted);
+    flex-shrink: 0;
+  }
+  .archive-bar-fill-wrap {
+    height: 3px;
+    background: var(--border);
+    border-radius: 2px;
+    margin-top: 6px;
+    overflow: hidden;
+  }
+  .archive-bar-fill {
+    height: 100%;
+    background: var(--accent);
+    border-radius: 2px;
+    transition: width 0.3s;
+  }
+
  /* ── Main area ── */
  #main {
    flex: 1;
@@ -245,20 +268,14 @@
    border: 1px solid var(--border);
    display: block;
  }
-  .info-media a.video-link {
-    display: flex;
-    align-items: center;
-    justify-content: center;
-    width: 80px;
-    height: 60px;
-    background: var(--surface2);
+  .info-media video.info-video {
+    display: block;
+    width: 100%;
+    max-width: 260px;
    border-radius: 4px;
    border: 1px solid var(--border);
-    color: var(--muted);
-    font-size: 10px;
-    text-decoration: none;
+    background: #000;
  }
-  .info-media a.video-link:hover { border-color: var(--accent); color: var(--accent); }

  #infoActions {
    padding: 10px 12px;
@@ -479,6 +496,8 @@
  <button id="btnClear"      class="btn btn-danger">Clear</button>
 </div>

+<div id="archiveBar" class="archive-bar" style="display:none"></div>
+
 <div id="main">
  <div id="cy"></div>

@@ -793,8 +812,28 @@ async function apiFetch(body) {
    body: JSON.stringify(body),
  });
  var json = await res.json();
-  if (!json.ok) throw new Error(json.error || 'API error');
-  return json.data;
+  if (!json.ok) {
+    var err = new Error(json.error || 'API error');
+    if (res.status === 429 && json.retryAfter) err.retryAfter = json.retryAfter;
+    throw err;
+  }
+  return { items: json.data, nextCursor: json.nextCursor || null };
+}
+
+// ── Cookie/Wayback cooldown (mirrors the server's 5s-per-source throttle) ──
+var cooldownUntil = { cookie: 0, wayback: 0 };
+var cooldownTimer = null;
+
+function throttleSourceFor(tool, mode) {
+  if (tool === 'wayback_archive_search') return 'wayback';
+  var COOKIE_TOOLS = ['tweet_search_extractor', 'follower_explorer', 'post_extractor',
+    'community_post_extractor', 'tweet_replies_extractor', 'tweet_retweeters_extractor', 'geo_post_extractor'];
+  if (mode === 'cookie' && COOKIE_TOOLS.indexOf(tool) !== -1) return 'cookie';
+  return null;
+}
+
+function stampCooldown(source) {
+  if (source) cooldownUntil[source] = Date.now() + 5000;
 }

 // ── Label builder ─────────────────────────────────────────────────────────────
@@ -887,8 +926,13 @@ async function runSearch() {
  setStatus('Searching…', false, true);

  try {
-    var data  = await apiFetch(buildBody(tool, query, count, mode));
-    var items = Array.isArray(data) ? data : [data];
+    var result = await apiFetch(buildBody(tool, query, count, mode));
+    var items  = Array.isArray(result.items) ? result.items : [result.items];
+    stampCooldown(throttleSourceFor(tool, mode));
+    // Root-search pagination (continuing this same search past one page) is
+    // not wired up in the graph UI yet — only reply/retweet node-expand
+    // supports "load more" this round. result.nextCursor is intentionally
+    // unused here.

    // Central search node
    var searchId = 'search_' + Date.now();
@@ -907,34 +951,75 @@ async function runSearch() {
 }

 // ── Expand replies / retweets from selected node ──────────────────────────────
+// First click on a node = fresh fetch. If more is available, the button
+// relabels itself and a second click continues from the stored cursor —
+// same cursor/cooldown mechanism as the non-graph page's scroll load-more.
+var CURSOR_KEY    = { tweet_replies_extractor: 'repliesCursor',   tweet_retweeters_extractor: 'retweetersCursor' };
+var EXHAUSTED_KEY = { tweet_replies_extractor: 'repliesExhausted', tweet_retweeters_extractor: 'retweetersExhausted' };
+
 async function expandNode(expandTool) {
  if (!selectedNode) return;
  var data    = selectedNode.data();
  var tweetId = data.tweetId;
  if (!tweetId) { setStatus('Node has no tweet ID', true); return; }

+  var source = throttleSourceFor(expandTool, 'cookie');   // always 'cookie' — both expand tools are cookie-only
+  var remaining = cooldownUntil[source] - Date.now();
+  if (remaining > 0) {
+    armExpandCountdown(remaining);
+    return;
+  }
+
  var count    = Math.max(1, Math.min(200, parseInt(document.getElementById('countInput').value, 10) || 50));
  var parentId = selectedNode.id();
+  var cursor   = selectedNode.data(CURSOR_KEY[expandTool]) || null;

  document.getElementById('btnExpandReplies').disabled  = true;
  document.getElementById('btnExpandRetweets').disabled = true;
-  setStatus('Expanding…', false, true);
+  setStatus(cursor ? 'Loading more…' : 'Expanding…', false, true);

  try {
-    var fetched  = await apiFetch({ toolType: expandTool, mode: 'cookie', count: count, targetTweetId: tweetId });
-    var items    = Array.isArray(fetched) ? fetched : [fetched];
+    var body = { toolType: expandTool, mode: 'cookie', count: count, targetTweetId: tweetId };
+    if (cursor) body.cursor = cursor;
+    var result   = await apiFetch(body);
+    stampCooldown(source);
+    var items    = Array.isArray(result.items) ? result.items : [result.items];
    var nodeType = expandTool === 'tweet_replies_extractor' ? 'reply' : 'retweeter';

    var added = addNodes(items, nodeType, parentId);
    runLayout(false, false);
    updateNodeCount();
    setStatus('Expanded — ' + items.length + ' result(s) (' + added + ' new nodes)');
+
+    selectedNode.data(CURSOR_KEY[expandTool], result.nextCursor || null);
+    selectedNode.data(EXHAUSTED_KEY[expandTool], !result.nextCursor);
+    showPanel(selectedNode.data());   // refresh button labels/disabled state (e.g. "all loaded")
  } catch (e) {
-    setStatus('Error: ' + e.message, true);
-  } finally {
+    if (e.retryAfter) {
+      // Keep both buttons disabled for the cooldown window — armExpandCountdown
+      // re-enables them itself once it elapses, so don't touch them here.
+      armExpandCountdown(e.retryAfter * 1000);
+    } else {
+      setStatus('Error: ' + e.message, true);
+      document.getElementById('btnExpandReplies').disabled  = false;
+      document.getElementById('btnExpandRetweets').disabled = false;
+    }
+  }
+}
+
+function armExpandCountdown(msRemaining) {
+  var secs = Math.max(1, Math.ceil(msRemaining / 1000));
+  setStatus('Rate-limit cooldown — retry in ' + secs + 's…', false, true);
+  // Both Expand buttons share one underlying X-account clock, so both wait together.
+  document.getElementById('btnExpandReplies').disabled  = true;
+  document.getElementById('btnExpandRetweets').disabled = true;
+  if (cooldownTimer) clearTimeout(cooldownTimer);
+  cooldownTimer = setTimeout(function () {
+    cooldownTimer = null;
    document.getElementById('btnExpandReplies').disabled  = false;
    document.getElementById('btnExpandRetweets').disabled = false;
-  }
+    setStatus('Ready — click Expand again', false, false);
+  }, msRemaining);
 }

 // ── Info panel ────────────────────────────────────────────────────────────────
@@ -961,6 +1046,32 @@ var SKIP_KEYS = new Set(['id', 'media', 'card', 'user_id', 'retweeted_by_user_id

 var SOURCE_CLASS = { 'Twitter Cookie': 'src-cookie', 'Xquik API': 'src-xquik', 'Wayback Machine': 'src-wayback' };

+// ── Media helpers — same normalization as the non-graph search page ───────────
+function extractMedia(item) {
+  // Cookie mode: item.media = [{type, thumb, url}]
+  if (Array.isArray(item.media) && item.media.length) {
+    var first = item.media[0];
+    if (first && typeof first === 'object' && ('thumb' in first || 'url' in first)) {
+      return item.media;
+    }
+  }
+  // API mode: item.extended_entities.media[] or item.entities.media[]
+  var src = (item.extended_entities && item.extended_entities.media)
+    || (item.entities && item.entities.media);
+  if (!Array.isArray(src)) return [];
+  return src.map(function (m) {
+    var mtype = m.type || 'photo';
+    var thumb = m.media_url_https || m.media_url || '';
+    var url   = thumb;
+    if (mtype === 'video' || mtype === 'animated_gif') {
+      var variants = (m.video_info && m.video_info.variants) || [];
+      var mp4s = variants.filter(function (v) { return v.content_type === 'video/mp4'; });
+      if (mp4s.length) url = mp4s.reduce(function (b, v) { return (v.bitrate || 0) > (b.bitrate || 0) ? v : b; }).url;
+    }
+    return { type: mtype, thumb: thumb, url: url };
+  }).filter(function (m) { return m.thumb; });
+}
+
 function showPanel(data) {
  var raw  = data.raw || {};
  var type = data.type;
@@ -1000,23 +1111,24 @@ function showPanel(data) {
    return '<div class="info-row"><div class="info-key">' + label + '</div><div class="info-val">' + val + '</div></div>';
  }).join('');

-  // Media thumbnails
-  var media = Array.isArray(raw.media) ? raw.media : [];
+  // Media thumbnails — normalized cookie/API shape; videos play inline via
+  // the backend proxy instead of just linking out to the raw stream.
+  var media = extractMedia(raw);
  if (media.length) {
    var mediaParts = ['<div class="info-media">'];
    media.forEach(function (m) {
-      var thumb = m.thumb || '';
-      var url   = m.url || thumb;
-      var mtype = m.type || 'photo';
-      if (!thumb) return;
-      if (mtype === 'photo') {
+      if (!m.thumb) return;
+      if (m.type === 'video' || m.type === 'animated_gif') {
+        var proxied = '/api/video?url=' + encodeURIComponent(m.url);
+        var loop    = m.type === 'animated_gif' ? 'loop muted' : '';
        mediaParts.push(
-          '<a href="' + esc(url) + '" target="_blank" rel="noopener noreferrer">' +
-          '<img src="' + esc(thumb) + '" loading="lazy" alt="media"></a>'
+          '<video class="info-video" controls ' + loop + ' poster="' + esc(m.thumb) + '" preload="none">' +
+          '<source src="' + proxied + '" type="video/mp4"></video>'
        );
      } else {
        mediaParts.push(
-          '<a href="/api/video?url=' + encodeURIComponent(url) + '" target="_blank" rel="noopener noreferrer" class="video-link">▶ ' + esc(mtype) + '</a>'
+          '<a href="' + esc(m.url) + '" target="_blank" rel="noopener noreferrer">' +
+          '<img src="' + esc(m.thumb) + '" loading="lazy" alt="media"></a>'
        );
      }
    });
@@ -1032,8 +1144,21 @@ function showPanel(data) {
  var tweetId   = data.tweetId;
  var user      = raw.user || raw.screen_name || '';

-  document.getElementById('btnExpandReplies').style.display  = isTweet ? '' : 'none';
-  document.getElementById('btnExpandRetweets').style.display = isTweet ? '' : 'none';
+  var repliesBtn    = document.getElementById('btnExpandReplies');
+  var retweetersBtn = document.getElementById('btnExpandRetweets');
+  repliesBtn.style.display    = isTweet ? '' : 'none';
+  retweetersBtn.style.display = isTweet ? '' : 'none';
+  if (isTweet) {
+    // Label reflects whether this node's already been expanded and whether
+    // more is available — neither twikit nor the Wayback CDX API expose a
+    // total count, only presence/absence of a next page, so no item count.
+    repliesBtn.textContent = data.repliesExhausted ? '↩ Expand Replies (all loaded)'
+      : data.repliesCursor ? '↩ Expand Replies (more available)' : '↩ Expand Replies';
+    repliesBtn.disabled = !!data.repliesExhausted;
+    retweetersBtn.textContent = data.retweetersExhausted ? '↗ Expand Retweets (all loaded)'
+      : data.retweetersCursor ? '↗ Expand Retweets (more available)' : '↗ Expand Retweets';
+    retweetersBtn.disabled = !!data.retweetersExhausted;
+  }

  var openBtn = document.getElementById('btnOpenTweet');
  if (isTweet && tweetId && user) {
@@ -1058,30 +1183,69 @@ function hidePanel() {
 }

 // ── Archive All ───────────────────────────────────────────────────────────────
+// First click creates a new archive; once that succeeds, graphArchivedId is
+// set and subsequent clicks (e.g. after expanding more reply/retweet nodes)
+// UPDATE that same archive in place instead of creating a new one each time
+// — same checkpoint behavior as the non-graph search page.
+var graphArchivedId = null;
+
 async function archiveAll() {
  if (!allItems.length) { setStatus('Nothing to archive yet', true); return; }
  var query = document.getElementById('queryInput').value.trim();
  var tool  = document.getElementById('toolSelect').value;

-  setStatus('Archiving…', false, true);
+  var archiveBar = document.getElementById('archiveBar');
+  archiveBar.style.display = '';
+  var label = graphArchivedId ? 'Saving checkpoint…' : 'Archiving…';
+  archiveBar.innerHTML = '<span>' + label + '</span><div class="archive-bar-fill-wrap"><div class="archive-bar-fill" style="width:0%"></div></div>';
+
  try {
+    var body = {
+      toolType:  'graph_' + tool,
+      data:      allItems.map(function (d) { return d.item; }),
+      queryInfo: { query: query, source: 'graph', nodes: cy.nodes().length },
+    };
+    if (graphArchivedId) body.archiveId = graphArchivedId;
+
    var res = await fetch('/api/archive', {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify({
-        toolType:  'graph_' + tool,
-        data:      allItems.map(function (d) { return d.item; }),
-        queryInfo: { query: query, source: 'graph', nodes: cy.nodes().length },
-      }),
+      body: JSON.stringify(body),
    });
    var json = await res.json();
    if (!json.ok) throw new Error(json.error);
-    setStatus('Archived — ID: ' + json.archiveId);
+    graphArchivedId = json.archiveId;
+    document.getElementById('btnArchiveAll').textContent = 'Update Archive';
+    pollGraphArchive(json.archiveId);
  } catch (e) {
-    setStatus('Archive error: ' + e.message, true);
+    archiveBar.innerHTML = '<span style="color:var(--danger)">Archive error: ' + esc(e.message) + '</span>';
  }
 }

+function pollGraphArchive(archiveId) {
+  var archiveBar = document.getElementById('archiveBar');
+  var interval = setInterval(async function () {
+    var res  = await fetch('/api/archive/' + archiveId + '/status');
+    var json = await res.json();
+    if (!json.ok) { clearInterval(interval); return; }
+
+    var status = json.status, progress = json.progress, total = json.total;
+    var fill = archiveBar.querySelector('.archive-bar-fill');
+
+    if (status === 'saving') {
+      archiveBar.querySelector('span').textContent = 'Saving results…';
+    } else if (status === 'downloading') {
+      var pct = total > 0 ? Math.round((progress / total) * 100) : 0;
+      archiveBar.querySelector('span').textContent = 'Downloading media ' + progress + '/' + total;
+      if (fill) fill.style.width = pct + '%';
+    } else if (status === 'done') {
+      clearInterval(interval);
+      if (fill) fill.style.width = '100%';
+      archiveBar.querySelector('span').textContent = 'Archived ✓ (' + total + ' media files saved) — ID: ' + archiveId;
+    }
+  }, 1200);
+}
+
 // ── Dump JSON ─────────────────────────────────────────────────────────────────
 function dumpJSON() {
  if (!allItems.length) { setStatus('Nothing to export yet', true); return; }
@@ -1110,6 +1274,9 @@ function clearGraph() {
  allItems.length = 0;
  hidePanel();
  updateNodeCount();
+  graphArchivedId = null;
+  document.getElementById('btnArchiveAll').textContent = 'Archive All';
+  document.getElementById('archiveBar').style.display = 'none';
  setStatus('Graph cleared');
 }

@@ -447,6 +447,24 @@
    text-align: center;
  }

+  .load-more-status {
+    text-align: center;
+    font-size: 12px;
+    color: var(--muted);
+    padding: 14px 0 20px;
+    display: none;
+  }
+  .load-more-status.visible { display: block; }
+  .load-more-status .spin-dot {
+    display: inline-block;
+    width: 6px; height: 6px;
+    border-radius: 50%;
+    background: var(--accent);
+    margin-right: 6px;
+    animation: lm-pulse 1s ease-in-out infinite;
+  }
+  @keyframes lm-pulse { 0%, 100% { opacity: 0.3; } 50% { opacity: 1; } }
+
  /* Media (gambar / video) */
  .card-media {
    display: flex;
@@ -556,6 +574,7 @@
  .leaflet-control-attribution { background: rgba(26,29,39,0.8) !important; color: var(--muted) !important; font-size: 9px; }
 </style>
 <script src="https://unpkg.com/leaflet@1.9.4/dist/leaflet.js" crossorigin="anonymous"></script>
+<script src="{{ url_for('static', filename='js/card_constants.js') }}" nonce="{{ g.csp_nonce }}"></script>
 </head>
 <body>

@@ -627,6 +646,9 @@
      <span class="toggle-label">Auto Archive</span>
    </div>
    <div id="archiveBar" class="archive-bar" style="display:none"></div>
+    <button id="checkpointBtn" class="btn-download" style="display:none;width:100%;margin-top:8px">
+      Save Checkpoint... Saved latest data 
+    </button>

    <!-- Recent archives -->
    <div class="archives-section" id="archivesSection" style="display:none">
@@ -649,6 +671,8 @@
    <div id="resultBox">
      <div class="empty-state">Nothing found :(</div>
    </div>
+    <div id="loadMoreSentinel" style="height:1px"></div>
+    <div id="loadMoreStatus" class="load-more-status"></div>
    <div id="mapStatus" style="display:none;font-size:12px;color:var(--muted);padding:6px 2px"></div>
    <div id="mapContainer"></div>
  </div>
@@ -693,11 +717,37 @@ const toggleTrack    = document.getElementById('toggleTrack');
 const archiveBar     = document.getElementById('archiveBar');
 const archivesSection= document.getElementById('archivesSection');
 const archivesList   = document.getElementById('archivesList');
+const loadMoreSentinel = document.getElementById('loadMoreSentinel');
+const loadMoreStatus   = document.getElementById('loadMoreStatus');
+const checkpointBtn    = document.getElementById('checkpointBtn');

 let currentMode    = 'api';
 let currentData    = null;
 let autoArchive    = false;
 let currentPayload = null;   // last run payload (for archive queryInfo)
+let archivedId     = null;   // archive id for the current run, once auto-archived — lets later load-more pages update it in place instead of creating a new archive
+
+// ── Scroll-triggered load-more (Cookie mode + Wayback only) ────────────────
+// Tools/modes the backend actually paginates — everything else (xquik/API
+// mode, multi-source search) just gets a single page, same as before.
+const PAGINATED_TOOLS = new Set([
+  'tweet_search_extractor', 'follower_explorer', 'post_extractor',
+  'community_post_extractor', 'tweet_replies_extractor',
+  'tweet_retweeters_extractor', 'geo_post_extractor',
+  'wayback_archive_search',
+]);
+const THROTTLE_SECONDS = 5;
+
+let nextCursor    = null;
+let loadingMore    = false;
+let cooldownUntil  = { cookie: 0, wayback: 0 };   // Date.now()-based timestamps
+let cooldownTimer  = null;
+
+function throttleSourceFor(tool, mode) {
+  if (tool === 'wayback_archive_search') return 'wayback';
+  if (mode === 'cookie' && PAGINATED_TOOLS.has(tool)) return 'cookie';
+  return null;
+}

 const HINTS = {
  tweet_search_extractor:    'Search for tweets by keyword',
@@ -850,16 +900,10 @@ function flatText(obj) {
  return Object.values(obj).map(flatText).join(' ');
 }

-const PRIORITY  = ['source','account_age_flag','account_age','account_created','retweeted_by_user','retweeted_by_name','retweeted_text','retweeted_by_bio','name','username','screen_name','user','user_id','text','full_text','content','title','description','bio','article_text','post_title','post_text','created_at','in_reply_to_tweet_id','archive_url','preview_image','iso_date','original','statuscode'];
+// PRIORITY / DRILLABLE / SOURCE_CLASS / AGE_LABELS come from
+// static/js/card_constants.js, loaded above — shared with archive.html.
 const SKIP      = ['profile_image_url','profile_banner_url','entities','extended_entities','urls','media','indices'];
 const CLAMP     = new Set(['text','full_text','content','description','bio','article_text','retweeted_text','retweeted_by_bio','post_text']);
-const SOURCE_CLASS = { 'Twitter Cookie': 'src-cookie', 'Xquik API': 'src-xquik', 'Wayback Machine': 'src-wayback' };
-// Keys that link out to a drill-down tool (key → target toolType)
-const DRILLABLE = {
-  reply_count:          'tweet_replies_extractor',
-  retweet_count:        'tweet_retweeters_extractor',
-  in_reply_to_tweet_id: 'tweet_replies_extractor',   // follow thread upward
-};

 // ── Media helpers ─────────────────────────────────────────────────────────────

@@ -935,9 +979,17 @@ function buildCard(item) {
  }
  const tweetId = item.id ? String(item.id) : null;
  const entries = Object.entries(item);
-  const pri     = entries.filter(([k]) => PRIORITY.includes(k));
+  // Order follows PRIORITY's own sequence (not object insertion order), so
+  // field position is deterministic and consistent across every tool/record
+  // shape. No cap on row count — this is an OSINT/analysis tool, so more
+  // extracted metadata is the point, not something to hide past a limit
+  // (this is what silently dropped the reply_count/retweet_count drill-down
+  // anchors once enough other fields — account age, verification, etc. —
+  // were present on a record).
+  const priKeys = new Set(entries.map(([k]) => k).filter(k => PRIORITY.includes(k)));
+  const pri     = PRIORITY.filter(k => priKeys.has(k)).map(k => [k, item[k]]);
  const rest    = entries.filter(([k]) => !PRIORITY.includes(k) && !SKIP.includes(k));
-  const rows    = [...pri, ...rest].slice(0, 14).map(([k, v]) => {
+  const rows    = [...pri, ...rest].map(([k, v]) => {
    let display;
    // Drillable check runs FIRST — reply_count / retweet_count always get a link
    // when we have a tweet ID, even if the count is 0 or null (twikit often under-reports)
@@ -953,8 +1005,7 @@ function buildCard(item) {
      const cls = SOURCE_CLASS[v] || '';
      display = `<span class="source-badge ${cls}">${esc(String(v))}</span>`;
    } else if (k === 'account_age_flag' && v) {
-      const label = { new: 'New account', recent: 'Recent account', established: 'Established account' }[v] || v;
-      display = `<span class="age-badge age-${esc(String(v))}">${esc(label)}</span>`;
+      display = `<span class="age-badge age-${esc(String(v))}">${esc(AGE_LABELS[v] || v)}</span>`;
    } else if (v === null || v === undefined) {
      display = `<span style="color:var(--muted)">—</span>`;
    } else if (typeof v === 'object') {
@@ -971,6 +1022,129 @@ function buildCard(item) {
  return `<div class="card">${rows}${renderMedia(media)}</div>`;
 }

+// ── Load more (scroll-triggered pagination) ─────────────────────────────────
+// The sentinel lives as a SIBLING of #resultBox, not inside it — renderCards()
+// replaces #resultBox.innerHTML wholesale on every run and every keystroke in
+// the search box, which would destroy an in-box sentinel and its observer tie.
+
+function appendCards(newItems, query = '') {
+  const q = query.toLowerCase().trim();
+  const toShow = q ? newItems.filter(i => flatText(i).toLowerCase().includes(q)) : newItems;
+  const grid = resultBox.querySelector('.cards-grid');
+  if (grid && toShow.length) {
+    grid.insertAdjacentHTML('beforeend', toShow.map(buildCard).join(''));
+  }
+  const totalShown = resultBox.querySelectorAll('.card').length;
+  resultCount.textContent = totalShown + (q ? ' found' : ' results');
+}
+
+function resetPagination() {
+  nextCursor   = null;
+  loadingMore  = false;
+  archivedId   = null;
+  if (cooldownTimer) { clearTimeout(cooldownTimer); cooldownTimer = null; }
+  loadMoreStatus.classList.remove('visible');
+  loadMoreStatus.textContent = '';
+  checkpointBtn.style.display = 'none';
+}
+
+function buildQueryInfo() {
+  const queryInfo = { ...currentPayload };
+  delete queryInfo.toolType;
+  delete queryInfo.mode;
+  delete queryInfo.count;
+  return queryInfo;
+}
+
+function stampCooldown(source) {
+  if (!source) return;
+  cooldownUntil[source] = Date.now() + THROTTLE_SECONDS * 1000;
+}
+
+function maybeLoadMore() {
+  if (!nextCursor || loadingMore || !currentPayload) return;
+  const source = throttleSourceFor(currentPayload.toolType, currentPayload.mode);
+  if (!source) return;   // shouldn't happen if nextCursor is set, but be safe
+
+  const remaining = cooldownUntil[source] - Date.now();
+  if (remaining > 0) {
+    armCountdown(remaining);
+    return;
+  }
+  doLoadMore(source);
+}
+
+function armCountdown(msRemaining) {
+  if (cooldownTimer) return;   // already counting down
+  const secs = Math.max(1, Math.ceil(msRemaining / 1000));
+  loadMoreStatus.classList.add('visible');
+  loadMoreStatus.innerHTML = `<span class="spin-dot"></span>Loading more in ${secs}s…`;
+  cooldownTimer = setTimeout(() => {
+    cooldownTimer = null;
+    maybeLoadMore();   // re-check: fires the load once the cooldown has cleared
+  }, msRemaining);
+}
+
+async function doLoadMore(source) {
+  loadingMore = true;
+  loadMoreStatus.classList.add('visible');
+  loadMoreStatus.innerHTML = `<span class="spin-dot"></span>Loading more…`;
+
+  try {
+    const res  = await fetch('/api/run', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ ...currentPayload, cursor: nextCursor }),
+    });
+    const json = await res.json();
+    stampCooldown(source);
+
+    if (!json.ok) {
+      if (res.status === 429 && json.retryAfter) {
+        loadingMore = false;
+        armCountdown(json.retryAfter * 1000);
+        return;
+      }
+      loadMoreStatus.textContent = `Load more failed: ${json.error}`;
+      nextCursor = null;
+      return;
+    }
+
+    const newItems = Array.isArray(json.data) ? json.data : [json.data];
+    currentData = (Array.isArray(currentData) ? currentData : [currentData]).concat(newItems);
+    appendCards(newItems, searchInput.value);
+    nextCursor = json.nextCursor || null;
+    loadMoreStatus.classList.remove('visible');
+
+    // Auto-archive only covered whatever was loaded at the initial Run — new
+    // pages from scroll need their own save. Once we've reached the end
+    // (no more pages), save automatically; otherwise surface a manual
+    // checkpoint button so the user can save without waiting to hit the end.
+    if (autoArchive && archivedId) {
+      if (!nextCursor) {
+        checkpointBtn.style.display = 'none';
+        triggerArchive(currentData, currentPayload.toolType, buildQueryInfo(), archivedId);
+      } else {
+        checkpointBtn.style.display = '';
+      }
+    }
+  } catch (e) {
+    loadMoreStatus.textContent = `Load more failed: ${e}`;
+  } finally {
+    loadingMore = false;
+  }
+}
+
+// root: null (the browser viewport) — .output has `overflow: auto` but is
+// never actually height-constrained (body/.layout only set min-height), so
+// it never becomes a real scroll container; the page/viewport is what
+// actually scrolls. Pointing the observer at .output meant it could never
+// detect the sentinel entering view.
+const loadMoreObserver = new IntersectionObserver((entries) => {
+  if (entries.some(e => e.isIntersecting)) maybeLoadMore();
+}, { root: null, rootMargin: '200px' });
+loadMoreObserver.observe(loadMoreSentinel);
+
 // ── Run ───────────────────────────────────────────────────────────────────────

 runBtn.addEventListener('click', async () => {
@@ -998,6 +1172,7 @@ runBtn.addEventListener('click', async () => {
  }

  currentPayload = payload;
+  resetPagination();

  runBtn.disabled            = true;
  runBtn.textContent         = 'Running…';
@@ -1018,9 +1193,15 @@ runBtn.addEventListener('click', async () => {
    const json = await res.json();

    if (json.ok) {
+      // Keep the client's cooldown clock in sync even if the user never
+      // scrolls to trigger a load-more — a fast scroll right after Run
+      // should still wait out the same 5s window the server just started.
+      stampCooldown(throttleSourceFor(t, currentMode));
+
      statusBadge.textContent    = 'done';
      statusBadge.className      = 'badge ok';
      currentData                = json.data;
+      nextCursor                 = json.nextCursor || null;
      const isMany               = Array.isArray(json.data) && json.data.length > 1;
      downloadBtn.style.display  = '';

@@ -1041,11 +1222,7 @@ runBtn.addEventListener('click', async () => {
      renderCards(currentData);

      if (autoArchive) {
-        const queryInfo = { ...currentPayload };
-        delete queryInfo.toolType;
-        delete queryInfo.mode;
-        delete queryInfo.count;
-        triggerArchive(json.data, t, queryInfo);
+        triggerArchive(json.data, t, buildQueryInfo());
      } else {
        archiveBar.style.display = 'none';
      }
@@ -1072,21 +1249,32 @@ archiveToggle.addEventListener('click', () => {
  if (autoArchive) loadRecentArchives();
 });

-async function triggerArchive(data, toolType, queryInfo) {
+async function triggerArchive(data, toolType, queryInfo, existingId) {
  archiveBar.style.display = '';
-  archiveBar.innerHTML     = '<span>Archiving…</span><div class="archive-bar-fill-wrap"><div class="archive-bar-fill" style="width:0%"></div></div>';
+  const label = existingId ? 'Saving checkpoint…' : 'Archiving…';
+  archiveBar.innerHTML = `<span>${label}</span><div class="archive-bar-fill-wrap"><div class="archive-bar-fill" style="width:0%"></div></div>`;
+
+  const body = { data, toolType, queryInfo };
+  if (existingId) body.archiveId = existingId;

  const res  = await fetch('/api/archive', {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify({ data, toolType, queryInfo }),
+    body: JSON.stringify(body),
  });
  const json = await res.json();
  if (!json.ok) { archiveBar.innerHTML = `<span style="color:var(--danger)">Archive failed: ${esc(json.error)}</span>`; return; }

+  archivedId = json.archiveId;
  pollArchive(json.archiveId);
 }

+checkpointBtn.addEventListener('click', () => {
+  if (!archivedId || !currentPayload) return;
+  checkpointBtn.style.display = 'none';
+  triggerArchive(currentData, currentPayload.toolType, buildQueryInfo(), archivedId);
+});
+
 function pollArchive(archiveId) {
  const interval = setInterval(async () => {
    const res  = await fetch(`/api/archive/${archiveId}/status`);
@@ -1189,6 +1377,9 @@ function setView(v) {
  const mapStatus    = document.getElementById('mapStatus');
  const searchInput  = document.getElementById('searchInput');

+  // No "bottom of the list" concept on a Leaflet map — suspend load-more there.
+  loadMoreSentinel.style.display = isCards ? '' : 'none';
+
  if (isCards) {
    mapContainer.style.display = 'none';
    mapStatus.style.display    = 'none';
@@ -13,6 +13,7 @@ the live site is a JS shell.
 """

 import html
+import json
 import re
 from concurrent.futures import ThreadPoolExecutor

@@ -51,13 +52,14 @@ def _normalize_target(raw: str) -> str:


 def _fetch_cdx(url: str, limit: int, from_date: str = "", to_date: str = "",
-                match_type: str | None = None) -> list[dict]:
+                match_type: str | None = None, resume_key: str | None = None) -> tuple[list[dict], str | None]:
    params = {
-        "url":      url,
-        "output":   "json",
-        "fl":       "timestamp,original,statuscode,mimetype,length",
-        "collapse": "digest",
-        "limit":    str(limit),
+        "url":           url,
+        "output":        "json",
+        "fl":            "timestamp,original,statuscode,mimetype,length",
+        "collapse":      "digest",
+        "limit":         str(limit),
+        "showResumeKey": "true",
    }
    if match_type:
        params["matchType"] = match_type
@@ -65,11 +67,13 @@ def _fetch_cdx(url: str, limit: int, from_date: str = "", to_date: str = "",
        params["from"] = from_date
    if to_date:
        params["to"] = to_date
+    if resume_key:
+        params["resumeKey"] = resume_key

    try:
        r = requests.get(
            WAYBACK_CDX_URL, params=params, timeout=REQUEST_TIMEOUT,
-            headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.10240"},
+            headers={"User-Agent": "Mozilla/5.0"},
        )
        r.raise_for_status()
    except requests.RequestException as e:
@@ -78,12 +82,20 @@ def _fetch_cdx(url: str, limit: int, from_date: str = "", to_date: str = "",
    try:
        rows = r.json()
    except ValueError:
-        return []
+        return [], None
    if not rows or len(rows) < 2:
-        return []
+        return [], None
+
+    # With showResumeKey=true, a truncated page ends with an empty-array
+    # sentinel followed by a one-element array holding the opaque resume
+    # key: [header, row..., [], ["<key>"]]. A full/last page has neither.
+    next_resume = None
+    if len(rows) >= 2 and rows[-2] == [] and isinstance(rows[-1], list) and len(rows[-1]) == 1:
+        next_resume = rows[-1][0]
+        rows = rows[:-2]

    header, *data_rows = rows
-    return [dict(zip(header, row)) for row in data_rows]
+    return [dict(zip(header, row)) for row in data_rows], next_resume


 def _row_to_record(row: dict) -> dict:
@@ -187,38 +199,74 @@ def _enrich_records(records: list[dict]) -> None:

 # ── Public API ───────────────────────────────────────────────────────────────

-def wayback_search(raw_target: str, count: int = 50, from_date: str = "", to_date: str = "") -> list[dict]:
+def wayback_search(raw_target: str, count: int = 50, from_date: str = "", to_date: str = "",
+                    cursor: str | None = None) -> tuple[list[dict], str | None]:
+    """Returns (records, next_cursor). `cursor` is an opaque JSON string from
+    a previous call's next_cursor — pass it back to fetch the next page.
+    None once there's nothing more to load."""
    _validate_date("from_date", from_date)
    _validate_date("to_date", to_date)
    target       = _normalize_target(raw_target)
    is_permalink = "/status/" in target

+    try:
+        incoming_cursor = json.loads(cursor) if cursor else {}
+        if not isinstance(incoming_cursor, dict):
+            incoming_cursor = {}
+    except ValueError:
+        incoming_cursor = {}
+
    rows: list[dict] = []
+    outgoing_cursor: dict[str, str] = {}
+    multi_domain = False

    if is_permalink:
-        rows = _fetch_cdx(target, count, from_date, to_date)
+        page_rows, next_resume = _fetch_cdx(target, count, from_date, to_date,
+                                             resume_key=incoming_cursor.get("main"))
+        rows = page_rows
+        if next_resume:
+            outgoing_cursor["main"] = next_resume
    else:
        domain, _, path = target.partition("/")
-        candidates = [target]
+        candidates = [("x" if domain == "x.com" else "tw", target)]
        alt_domain = "twitter.com" if domain == "x.com" else ("x.com" if domain == "twitter.com" else None)
        if alt_domain:
-            candidates.append(f"{alt_domain}/{path}")
+            candidates.append(("tw" if alt_domain == "twitter.com" else "x", f"{alt_domain}/{path}"))
+
+        # Page 1 (no incoming cursor): query every candidate domain. Later
+        # pages: only re-query a domain that still had a resume key on the
+        # previous page — a domain missing from incoming_cursor already ran
+        # dry, so skip it rather than restarting it from scratch.
+        active = [(key, url) for key, url in candidates if not incoming_cursor or key in incoming_cursor]
+        multi_domain = len(active) > 1

        seen = set()
-        for cand in candidates:
-            for row in _fetch_cdx(cand, count, from_date, to_date, match_type="prefix"):
-                key = (row.get("timestamp"), row.get("original"))
-                if key in seen:
+        for key, cand in active:
+            page_rows, next_resume = _fetch_cdx(cand, count, from_date, to_date,
+                                                 match_type="prefix", resume_key=incoming_cursor.get(key))
+            for row in page_rows:
+                dedupe_key = (row.get("timestamp"), row.get("original"))
+                if dedupe_key in seen:
                    continue
-                seen.add(key)
+                seen.add(dedupe_key)
                rows.append(row)
+            if next_resume:
+                outgoing_cursor[key] = next_resume

    records = [_row_to_record(r) for r in rows]
    records.sort(key=lambda r: r["timestamp"], reverse=True)
-    records = records[:count]
+    # Slicing to `count` is only safe when exactly one source was queried
+    # this page — its own resume key already accounts for exactly its own
+    # raw fetch. Slicing a page that merged >1 domain would silently strand
+    # whatever got cut, since each domain's cursor has already moved past
+    # everything it returned this round.
+    if not multi_domain:
+        records = records[:count]

    _enrich_records(records)

    for r in records:
        r.pop("timestamp", None)
-    return records
+
+    next_cursor = json.dumps(outgoing_cursor) if outgoing_cursor else None
+    return records, next_cursor
@@ -11,7 +11,7 @@ class XquikError(Exception):

 def load_config(path: str = CONFIG_PATH) -> configparser.ConfigParser:
    if not os.path.exists(path):
-        raise XquikError(f"Config are not set: {path}")
+        raise XquikError(f"Config tidak ditemukan: {path}")
    cfg = configparser.ConfigParser()
    cfg.read(path)
    return cfg
@@ -26,7 +26,7 @@ class XquikClient:
        ).strip()

        if not self.api_key or self.api_key == "xq_YOUR_KEY":
-            raise XquikError("No API key provided. Please check config.ini")
+            raise XquikError("api_key belum diisi di config.ini [xquik]")

    def _post(self, payload: dict) -> dict:
        headers = {
@@ -36,7 +36,7 @@ class XquikClient:
        try:
            resp = requests.post(self.base_url, headers=headers, json=payload, timeout=30)
        except requests.RequestException as e:
-            raise XquikError(f"Request Failed: {e}") from e
+            raise XquikError(f"Request gagal: {e}") from e

        if resp.status_code >= 400:
            raise XquikError(f"xquik API error {resp.status_code}: {resp.text}")
@@ -44,7 +44,7 @@ class XquikClient:
        try:
            return resp.json()
        except ValueError as e:
-            raise XquikError(f"Response are not json: {resp.text[:300]}") from e
+            raise XquikError(f"Response bukan JSON valid: {resp.text[:300]}") from e

    def tweet_search(self, search_query: str) -> dict:
        return self._post({"toolType": "tweet_search_extractor", "searchQuery": search_query})
@@ -61,7 +61,7 @@ class XquikClient:
        )

    def post_extractor(self, target_username: str) -> dict:
-        """User timeline via Xquik"""
+        """User timeline lewat xquik (pakai kuota API)."""
        return self._post({"toolType": "post_extractor", "targetUsername": target_username})