convert to flask and jinja from blade and php source & sync update bug and logic

Script/SOCMINT-Twitter/templates/_field_glossary.html

@@ -20,24 +20,34 @@
 <div class="param-row"><code class="param-key">is_blue_verified</code> Paid X Premium checkmark</div>
 
 <div class="param-heading">Threading &amp; retweets</div>
-<div class="param-row"><code class="param-key">in_reply_to_tweet_id</code> Parent tweet this replies to</div>
-<div class="param-row"><code class="param-key">retweeted_*</code> Details of the original tweet being retweeted</div>
+<div class="param-row"><code class="param-key">in_reply_to_tweet_id</code> Parent tweet this replies to — also the drill-down anchor that fetches its replies</div>
+<div class="param-row"><code class="param-key">retweeted_text / retweeted_by_user / retweeted_by_name / retweeted_by_bio</code> Content and author of the original tweet being retweeted</div>
+<div class="param-row"><code class="param-key">retweeted_at / retweeted_tweet_id</code> When the original was posted, and its own ID</div>
 
 <div class="param-heading">Location</div>
-<div class="param-row"><code class="param-key">lat / lon / place / user_location</code> Geodata — geo search only</div>
+<div class="param-row"><code class="param-key">lat / lon / place</code> Coordinates plotted on the map view (Geo Post Search)</div>
+<div class="param-row"><code class="param-key">user_location</code> Free-text profile location string, geocoded client-side to produce lat/lon</div>
 
 <div class="param-heading">Account age (forensics)</div>
-<div class="param-row"><code class="param-key">account_created / account_age</code> Derived from the account's numeric ID, not the API — see badges above</div>
-<div class="param-row"><code class="param-key">account_age_flag</code> New / Recent / Established bucket</div>
-<div class="param-row"><code class="param-key">account_age_precision</code> exact / estimated / unknown — how much to trust the date</div>
+<div class="param-row"><code class="param-key">account_created / account_age</code> Derived from the account's numeric ID, not the API — see the Account age badges above</div>
+<div class="param-row"><code class="param-key">account_age_flag</code> New / Recent / Established bucket, drives the badge color</div>
+<div class="param-row"><code class="param-key">account_age_precision</code> exact (true Snowflake decode) / estimated (pre-Snowflake id, interpolated) / unknown</div>
 
 <div class="param-heading">Archive &amp; Wayback</div>
 <div class="param-row"><code class="param-key">archive_url</code> Link to the Wayback Machine snapshot</div>
 <div class="param-row"><code class="param-key">iso_date</code> When that snapshot was captured</div>
 <div class="param-row"><code class="param-key">original</code> The original URL that was archived</div>
-<div class="param-row"><code class="param-key">statuscode / mimetype / length</code> HTTP status / type / size of the snapshot</div>
-<div class="param-row"><code class="param-key">post_title / post_text / preview_image</code> Scraped from the archived page's meta tags</div>
-<div class="param-row"><code class="param-key">tweet_url</code> Direct link to the live tweet</div>
+<div class="param-row"><code class="param-key">statuscode / mimetype / length</code> HTTP status / content type / size of the snapshot</div>
+<div class="param-row"><code class="param-key">post_title / post_text / preview_image</code> Scraped from the archived page's own meta tags (og:/twitter: tags, or &lt;title&gt;/&lt;meta name="description"&gt; on older captures)</div>
+<div class="param-row"><code class="param-key">tweet_url</code> Direct link to the live tweet, built from user + id</div>
 
 <div class="param-heading">Source</div>
-<div class="param-row"><code class="param-key">source</code> Which of the 3 data sources this result came from (multi-source search only)</div>
+<div class="param-row"><code class="param-key">source</code> Which of the 3 data sources this result came from — Twitter Cookie / Xquik API / Wayback Machine (Multi-Source Search only)</div>
+
+<div class="param-heading">Graph node types (graph page only)</div>
+<div class="param-row"><code class="param-key">Search root</code> The diamond node — the query you ran</div>
+<div class="param-row"><code class="param-key">Tweet</code> A tweet/post returned by search or by expanding a node</div>
+<div class="param-row"><code class="param-key">Reply</code> A tweet fetched via Expand Replies on a tweet node</div>
+<div class="param-row"><code class="param-key">User / Retweeter</code> A person — from Follower Explorer, or via Expand Retweets on a tweet node</div>
+<div class="param-row"><code class="param-key">Wayback snapshot</code> An archived-page result, from Wayback Archive Search or the Wayback portion of Multi-Source Search</div>
+<div class="param-row"><code class="param-key">Viewed</code> Turns a node solid white once you've clicked it — a visual "already looked at this" marker, not part of the underlying data</div>

Script/SOCMINT-Twitter/templates/archive.html

@@ -168,6 +168,19 @@
   }
   .sidebar-title { font-size: 11px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.06em; color: var(--muted); }
   .sidebar-count { font-size: 11px; color: var(--muted); background: var(--bg); padding: 2px 7px; border-radius: 10px; }
+  .sidebar-search {
+    width: 100%;
+    padding: 6px 10px;
+    background: var(--bg);
+    border: 1px solid var(--border);
+    color: var(--text);
+    font-family: var(--font);
+    font-size: 12px;
+    border-radius: 6px;
+    transition: border-color 0.15s;
+  }
+  .sidebar-search:focus { outline: none; border-color: var(--accent); }
+  .sidebar-search::placeholder { color: var(--muted); }
   .archive-list { overflow-y: auto; flex: 1; padding: 8px; }
   .archive-entry {
     padding: 10px 12px;
@@ -289,6 +302,16 @@
   .card-link { color: var(--accent); text-decoration: none; font-size: 12px; }
   .card-link:hover { text-decoration: underline; }
 
+  /* Drill-down links on reply / retweet counts — same as the live search page */
+  .drill-link {
+    color: var(--accent);
+    font-family: var(--font);
+    font-size: 13px;
+    text-decoration: underline;
+    text-underline-offset: 2px;
+  }
+  .drill-link:hover { color: #818cf8; }
+
   .source-badge {
     display: inline-block;
     font-size: 11px;
@@ -373,6 +396,7 @@
   }
   @keyframes spin { to { transform: rotate(360deg); } }
 </style>
+<script src="{{ url_for('static', filename='js/card_constants.js') }}" nonce="{{ g.csp_nonce }}"></script>
 </head>
 <body>
 
@@ -418,6 +442,9 @@
       <span class="sidebar-title">Archives</span>
       <span class="sidebar-count" id="archiveCount">—</span>
     </div>
+    <div style="padding:8px 12px 0">
+      <input type="text" class="sidebar-search" id="archiveSearch" placeholder="Search archives…">
+    </div>
     <div class="archive-list" id="archiveList">
       <div class="no-archives"><span class="spinner"></span> Loading…</div>
     </div>
@@ -471,9 +498,11 @@
 
 let allItems      = [];
 let activeId      = null;
+let allArchives   = [];   // full list from the server; archiveSearch filters this client-side
 
-const archiveList  = document.getElementById('archiveList');
-const archiveCount = document.getElementById('archiveCount');
+const archiveList   = document.getElementById('archiveList');
+const archiveCount  = document.getElementById('archiveCount');
+const archiveSearch = document.getElementById('archiveSearch');
 const viewerTop    = document.getElementById('viewerTop');
 const viewerTitle  = document.getElementById('viewerTitle');
 const viewerSub    = document.getElementById('viewerSub');
@@ -483,28 +512,52 @@ const deleteBtn    = document.getElementById('deleteBtn');
 const cardsBox     = document.getElementById('cardsBox');
 
 // ── Sidebar ───────────────────────────────────────────────────────────────────
+// The server already returns archives newest-first (by archived_at/updated_at,
+// not folder name — folder names are prefixed by tool_type so sorting by name
+// doesn't actually sort chronologically once more than one tool's been used).
 
 async function loadSidebar() {
   const res  = await fetch('/api/archive/list');
   const json = await res.json();
 
   if (!json.ok || !json.archives.length) {
+    allArchives = [];
     archiveList.innerHTML  = '<div class="no-archives">No archives yet.<br>Enable Auto Archive in the tool and run a search.</div>';
     archiveCount.textContent = '0';
     return;
   }
 
-  archiveCount.textContent = json.archives.length;
-  archiveList.innerHTML    = json.archives.map(a => entryHtml(a)).join('');
-
-  archiveList.querySelectorAll('.archive-entry').forEach(el => {
-    el.addEventListener('click', () => loadArchive(el.dataset.id));
-  });
+  allArchives = json.archives;
+  renderArchiveList(archiveSearch.value);
 
   // Auto-open the newest one
-  loadArchive(json.archives[0].id);
+  loadArchive(allArchives[0].id);
 }
 
+function archiveMatchText(a) {
+  return [a.tool || '', queryLabel(a.query || {}), a.id || ''].join(' ').toLowerCase();
+}
+
+function renderArchiveList(filterText = '') {
+  const q        = filterText.toLowerCase().trim();
+  const filtered = q ? allArchives.filter(a => archiveMatchText(a).includes(q)) : allArchives;
+
+  archiveCount.textContent = q ? `${filtered.length} / ${allArchives.length}` : allArchives.length;
+
+  if (!filtered.length) {
+    archiveList.innerHTML = '<div class="no-archives">No archives match your search.</div>';
+    return;
+  }
+
+  archiveList.innerHTML = filtered.map(a => entryHtml(a)).join('');
+  archiveList.querySelectorAll('.archive-entry').forEach(el => {
+    el.classList.toggle('active', el.dataset.id === activeId);
+    el.addEventListener('click', () => loadArchive(el.dataset.id));
+  });
+}
+
+archiveSearch.addEventListener('input', () => renderArchiveList(archiveSearch.value));
+
 function entryHtml(a) {
   const tool  = (a.tool || 'unknown').replace(/_/g, ' ');
   const date  = a.archived_at ? a.archived_at.replace('T', ' ') : '—';
@@ -572,10 +625,10 @@ function flatText(obj) {
 
 // ── Cards ─────────────────────────────────────────────────────────────────────
 
-const PRIORITY = ['source','account_age_flag','account_age','account_created','user','user_id','text','full_text','article_text','post_title','post_text','created_at','tweet_url','archive_url','preview_image','iso_date','original','statuscode'];
+// PRIORITY / DRILLABLE / SOURCE_CLASS / AGE_LABELS come from
+// static/js/card_constants.js, loaded above — shared with index.html.
 const SKIP     = ['archived_media'];
 const CLAMP    = new Set(['text','full_text','article_text','description','post_text']);
-const SOURCE_CLASS = { 'Twitter Cookie': 'src-cookie', 'Xquik API': 'src-xquik', 'Wayback Machine': 'src-wayback' };
 
 function renderCards(items, query) {
   const q        = query.toLowerCase().trim();
@@ -593,13 +646,24 @@ function renderCards(items, query) {
 function buildCard(item) {
   if (typeof item !== 'object' || !item) return '';
 
+  const tweetId = item.id ? String(item.id) : null;
   const entries = Object.entries(item);
-  const pri  = entries.filter(([k]) => PRIORITY.includes(k));
-  const rest = entries.filter(([k]) => !PRIORITY.includes(k) && !SKIP.includes(k));
+  // Same rule as the live search page: order follows PRIORITY's own sequence,
+  // and nothing gets capped off — a saved archive should show exactly the
+  // same fields, in the same order, as when it was first fetched.
+  const priKeys = new Set(entries.map(([k]) => k).filter(k => PRIORITY.includes(k)));
+  const pri     = PRIORITY.filter(k => priKeys.has(k)).map(k => [k, item[k]]);
+  const rest    = entries.filter(([k]) => !PRIORITY.includes(k) && !SKIP.includes(k));
 
-  const rows = [...pri, ...rest].slice(0, 14).map(([k, v]) => {
+  const rows = [...pri, ...rest].map(([k, v]) => {
     let display;
-    if (v === null || v === undefined) {
+    // Drillable check first — reply_count / retweet_count on an archived
+    // tweet still link out to a fresh live extraction, same as a live result.
+    if (DRILLABLE[k] && tweetId) {
+      const _p  = new URLSearchParams({ tool: DRILLABLE[k], id: tweetId, autorun: '1' });
+      const lbl = (v !== null && v !== undefined) ? esc(String(v)) : '—';
+      display = `<a class="drill-link" href="/?${_p}" target="_blank" rel="noopener noreferrer">${lbl} ↗</a>`;
+    } else if (v === null || v === undefined) {
       display = `<span style="color:var(--muted)">—</span>`;
     } else if (k === 'tweet_url' || k === 'archive_url' || k === 'preview_image') {
       display = `<a href="${esc(String(v))}" target="_blank" rel="noopener" class="card-link">${esc(String(v))}</a>`;
@@ -607,8 +671,7 @@ function buildCard(item) {
       const cls = SOURCE_CLASS[v] || '';
       display = `<span class="source-badge ${cls}">${esc(String(v))}</span>`;
     } else if (k === 'account_age_flag') {
-      const label = { new: 'New account', recent: 'Recent account', established: 'Established account' }[v] || v;
-      display = `<span class="age-badge age-${esc(String(v))}">${esc(label)}</span>`;
+      display = `<span class="age-badge age-${esc(String(v))}">${esc(AGE_LABELS[v] || v)}</span>`;
     } else if (typeof v === 'object') {
       const s = JSON.stringify(v);
       display = `<span style="color:var(--muted);font-size:11px">${esc(s.length > 100 ? s.slice(0,100)+'…' : s)}</span>`;

Script/SOCMINT-Twitter/templates/graph.html

@@ -137,6 +137,29 @@
   .btn-danger:hover:not(:disabled)  { background: rgba(239,68,68,0.1); border-color: var(--danger); }
   .btn:disabled { opacity: 0.38; cursor: not-allowed; }
 
+  /* ── Archive progress bar — same look as the non-graph search page ── */
+  .archive-bar {
+    padding: 6px 16px;
+    background: var(--surface);
+    border-bottom: 1px solid var(--border);
+    font-size: 12px;
+    color: var(--muted);
+    flex-shrink: 0;
+  }
+  .archive-bar-fill-wrap {
+    height: 3px;
+    background: var(--border);
+    border-radius: 2px;
+    margin-top: 6px;
+    overflow: hidden;
+  }
+  .archive-bar-fill {
+    height: 100%;
+    background: var(--accent);
+    border-radius: 2px;
+    transition: width 0.3s;
+  }
+
   /* ── Main area ── */
   #main {
     flex: 1;
@@ -245,20 +268,14 @@
     border: 1px solid var(--border);
     display: block;
   }
-  .info-media a.video-link {
-    display: flex;
-    align-items: center;
-    justify-content: center;
-    width: 80px;
-    height: 60px;
-    background: var(--surface2);
+  .info-media video.info-video {
+    display: block;
+    width: 100%;
+    max-width: 260px;
     border-radius: 4px;
     border: 1px solid var(--border);
-    color: var(--muted);
-    font-size: 10px;
-    text-decoration: none;
+    background: #000;
   }
-  .info-media a.video-link:hover { border-color: var(--accent); color: var(--accent); }
 
   #infoActions {
     padding: 10px 12px;
@@ -479,6 +496,8 @@
   <button id="btnClear"      class="btn btn-danger">Clear</button>
 </div>
 
+<div id="archiveBar" class="archive-bar" style="display:none"></div>
+
 <div id="main">
   <div id="cy"></div>
 
@@ -793,8 +812,28 @@ async function apiFetch(body) {
     body: JSON.stringify(body),
   });
   var json = await res.json();
-  if (!json.ok) throw new Error(json.error || 'API error');
-  return json.data;
+  if (!json.ok) {
+    var err = new Error(json.error || 'API error');
+    if (res.status === 429 && json.retryAfter) err.retryAfter = json.retryAfter;
+    throw err;
+  }
+  return { items: json.data, nextCursor: json.nextCursor || null };
+}
+
+// ── Cookie/Wayback cooldown (mirrors the server's 5s-per-source throttle) ──
+var cooldownUntil = { cookie: 0, wayback: 0 };
+var cooldownTimer = null;
+
+function throttleSourceFor(tool, mode) {
+  if (tool === 'wayback_archive_search') return 'wayback';
+  var COOKIE_TOOLS = ['tweet_search_extractor', 'follower_explorer', 'post_extractor',
+    'community_post_extractor', 'tweet_replies_extractor', 'tweet_retweeters_extractor', 'geo_post_extractor'];
+  if (mode === 'cookie' && COOKIE_TOOLS.indexOf(tool) !== -1) return 'cookie';
+  return null;
+}
+
+function stampCooldown(source) {
+  if (source) cooldownUntil[source] = Date.now() + 5000;
 }
 
 // ── Label builder ─────────────────────────────────────────────────────────────
@@ -887,8 +926,13 @@ async function runSearch() {
   setStatus('Searching…', false, true);
 
   try {
-    var data  = await apiFetch(buildBody(tool, query, count, mode));
-    var items = Array.isArray(data) ? data : [data];
+    var result = await apiFetch(buildBody(tool, query, count, mode));
+    var items  = Array.isArray(result.items) ? result.items : [result.items];
+    stampCooldown(throttleSourceFor(tool, mode));
+    // Root-search pagination (continuing this same search past one page) is
+    // not wired up in the graph UI yet — only reply/retweet node-expand
+    // supports "load more" this round. result.nextCursor is intentionally
+    // unused here.
 
     // Central search node
     var searchId = 'search_' + Date.now();
@@ -907,34 +951,75 @@ async function runSearch() {
 }
 
 // ── Expand replies / retweets from selected node ──────────────────────────────
+// First click on a node = fresh fetch. If more is available, the button
+// relabels itself and a second click continues from the stored cursor —
+// same cursor/cooldown mechanism as the non-graph page's scroll load-more.
+var CURSOR_KEY    = { tweet_replies_extractor: 'repliesCursor',   tweet_retweeters_extractor: 'retweetersCursor' };
+var EXHAUSTED_KEY = { tweet_replies_extractor: 'repliesExhausted', tweet_retweeters_extractor: 'retweetersExhausted' };
+
 async function expandNode(expandTool) {
   if (!selectedNode) return;
   var data    = selectedNode.data();
   var tweetId = data.tweetId;
   if (!tweetId) { setStatus('Node has no tweet ID', true); return; }
 
+  var source = throttleSourceFor(expandTool, 'cookie');   // always 'cookie' — both expand tools are cookie-only
+  var remaining = cooldownUntil[source] - Date.now();
+  if (remaining > 0) {
+    armExpandCountdown(remaining);
+    return;
+  }
+
   var count    = Math.max(1, Math.min(200, parseInt(document.getElementById('countInput').value, 10) || 50));
   var parentId = selectedNode.id();
+  var cursor   = selectedNode.data(CURSOR_KEY[expandTool]) || null;
 
   document.getElementById('btnExpandReplies').disabled  = true;
   document.getElementById('btnExpandRetweets').disabled = true;
-  setStatus('Expanding…', false, true);
+  setStatus(cursor ? 'Loading more…' : 'Expanding…', false, true);
 
   try {
-    var fetched  = await apiFetch({ toolType: expandTool, mode: 'cookie', count: count, targetTweetId: tweetId });
-    var items    = Array.isArray(fetched) ? fetched : [fetched];
+    var body = { toolType: expandTool, mode: 'cookie', count: count, targetTweetId: tweetId };
+    if (cursor) body.cursor = cursor;
+    var result   = await apiFetch(body);
+    stampCooldown(source);
+    var items    = Array.isArray(result.items) ? result.items : [result.items];
     var nodeType = expandTool === 'tweet_replies_extractor' ? 'reply' : 'retweeter';
 
     var added = addNodes(items, nodeType, parentId);
     runLayout(false, false);
     updateNodeCount();
     setStatus('Expanded — ' + items.length + ' result(s) (' + added + ' new nodes)');
+
+    selectedNode.data(CURSOR_KEY[expandTool], result.nextCursor || null);
+    selectedNode.data(EXHAUSTED_KEY[expandTool], !result.nextCursor);
+    showPanel(selectedNode.data());   // refresh button labels/disabled state (e.g. "all loaded")
   } catch (e) {
-    setStatus('Error: ' + e.message, true);
-  } finally {
+    if (e.retryAfter) {
+      // Keep both buttons disabled for the cooldown window — armExpandCountdown
+      // re-enables them itself once it elapses, so don't touch them here.
+      armExpandCountdown(e.retryAfter * 1000);
+    } else {
+      setStatus('Error: ' + e.message, true);
+      document.getElementById('btnExpandReplies').disabled  = false;
+      document.getElementById('btnExpandRetweets').disabled = false;
+    }
+  }
+}
+
+function armExpandCountdown(msRemaining) {
+  var secs = Math.max(1, Math.ceil(msRemaining / 1000));
+  setStatus('Rate-limit cooldown — retry in ' + secs + 's…', false, true);
+  // Both Expand buttons share one underlying X-account clock, so both wait together.
+  document.getElementById('btnExpandReplies').disabled  = true;
+  document.getElementById('btnExpandRetweets').disabled = true;
+  if (cooldownTimer) clearTimeout(cooldownTimer);
+  cooldownTimer = setTimeout(function () {
+    cooldownTimer = null;
     document.getElementById('btnExpandReplies').disabled  = false;
     document.getElementById('btnExpandRetweets').disabled = false;
-  }
+    setStatus('Ready — click Expand again', false, false);
+  }, msRemaining);
 }
 
 // ── Info panel ────────────────────────────────────────────────────────────────
@@ -961,6 +1046,32 @@ var SKIP_KEYS = new Set(['id', 'media', 'card', 'user_id', 'retweeted_by_user_id
 
 var SOURCE_CLASS = { 'Twitter Cookie': 'src-cookie', 'Xquik API': 'src-xquik', 'Wayback Machine': 'src-wayback' };
 
+// ── Media helpers — same normalization as the non-graph search page ───────────
+function extractMedia(item) {
+  // Cookie mode: item.media = [{type, thumb, url}]
+  if (Array.isArray(item.media) && item.media.length) {
+    var first = item.media[0];
+    if (first && typeof first === 'object' && ('thumb' in first || 'url' in first)) {
+      return item.media;
+    }
+  }
+  // API mode: item.extended_entities.media[] or item.entities.media[]
+  var src = (item.extended_entities && item.extended_entities.media)
+    || (item.entities && item.entities.media);
+  if (!Array.isArray(src)) return [];
+  return src.map(function (m) {
+    var mtype = m.type || 'photo';
+    var thumb = m.media_url_https || m.media_url || '';
+    var url   = thumb;
+    if (mtype === 'video' || mtype === 'animated_gif') {
+      var variants = (m.video_info && m.video_info.variants) || [];
+      var mp4s = variants.filter(function (v) { return v.content_type === 'video/mp4'; });
+      if (mp4s.length) url = mp4s.reduce(function (b, v) { return (v.bitrate || 0) > (b.bitrate || 0) ? v : b; }).url;
+    }
+    return { type: mtype, thumb: thumb, url: url };
+  }).filter(function (m) { return m.thumb; });
+}
+
 function showPanel(data) {
   var raw  = data.raw || {};
   var type = data.type;
@@ -1000,23 +1111,24 @@ function showPanel(data) {
     return '<div class="info-row"><div class="info-key">' + label + '</div><div class="info-val">' + val + '</div></div>';
   }).join('');
 
-  // Media thumbnails
-  var media = Array.isArray(raw.media) ? raw.media : [];
+  // Media thumbnails — normalized cookie/API shape; videos play inline via
+  // the backend proxy instead of just linking out to the raw stream.
+  var media = extractMedia(raw);
   if (media.length) {
     var mediaParts = ['<div class="info-media">'];
     media.forEach(function (m) {
-      var thumb = m.thumb || '';
-      var url   = m.url || thumb;
-      var mtype = m.type || 'photo';
-      if (!thumb) return;
-      if (mtype === 'photo') {
+      if (!m.thumb) return;
+      if (m.type === 'video' || m.type === 'animated_gif') {
+        var proxied = '/api/video?url=' + encodeURIComponent(m.url);
+        var loop    = m.type === 'animated_gif' ? 'loop muted' : '';
         mediaParts.push(
-          '<a href="' + esc(url) + '" target="_blank" rel="noopener noreferrer">' +
-          '<img src="' + esc(thumb) + '" loading="lazy" alt="media"></a>'
+          '<video class="info-video" controls ' + loop + ' poster="' + esc(m.thumb) + '" preload="none">' +
+          '<source src="' + proxied + '" type="video/mp4"></video>'
         );
       } else {
         mediaParts.push(
-          '<a href="/api/video?url=' + encodeURIComponent(url) + '" target="_blank" rel="noopener noreferrer" class="video-link">▶ ' + esc(mtype) + '</a>'
+          '<a href="' + esc(m.url) + '" target="_blank" rel="noopener noreferrer">' +
+          '<img src="' + esc(m.thumb) + '" loading="lazy" alt="media"></a>'
         );
       }
     });
@@ -1032,8 +1144,21 @@ function showPanel(data) {
   var tweetId   = data.tweetId;
   var user      = raw.user || raw.screen_name || '';
 
-  document.getElementById('btnExpandReplies').style.display  = isTweet ? '' : 'none';
-  document.getElementById('btnExpandRetweets').style.display = isTweet ? '' : 'none';
+  var repliesBtn    = document.getElementById('btnExpandReplies');
+  var retweetersBtn = document.getElementById('btnExpandRetweets');
+  repliesBtn.style.display    = isTweet ? '' : 'none';
+  retweetersBtn.style.display = isTweet ? '' : 'none';
+  if (isTweet) {
+    // Label reflects whether this node's already been expanded and whether
+    // more is available — neither twikit nor the Wayback CDX API expose a
+    // total count, only presence/absence of a next page, so no item count.
+    repliesBtn.textContent = data.repliesExhausted ? '↩ Expand Replies (all loaded)'
+      : data.repliesCursor ? '↩ Expand Replies (more available)' : '↩ Expand Replies';
+    repliesBtn.disabled = !!data.repliesExhausted;
+    retweetersBtn.textContent = data.retweetersExhausted ? '↗ Expand Retweets (all loaded)'
+      : data.retweetersCursor ? '↗ Expand Retweets (more available)' : '↗ Expand Retweets';
+    retweetersBtn.disabled = !!data.retweetersExhausted;
+  }
 
   var openBtn = document.getElementById('btnOpenTweet');
   if (isTweet && tweetId && user) {
@@ -1058,30 +1183,69 @@ function hidePanel() {
 }
 
 // ── Archive All ───────────────────────────────────────────────────────────────
+// First click creates a new archive; once that succeeds, graphArchivedId is
+// set and subsequent clicks (e.g. after expanding more reply/retweet nodes)
+// UPDATE that same archive in place instead of creating a new one each time
+// — same checkpoint behavior as the non-graph search page.
+var graphArchivedId = null;
+
 async function archiveAll() {
   if (!allItems.length) { setStatus('Nothing to archive yet', true); return; }
   var query = document.getElementById('queryInput').value.trim();
   var tool  = document.getElementById('toolSelect').value;
 
-  setStatus('Archiving…', false, true);
+  var archiveBar = document.getElementById('archiveBar');
+  archiveBar.style.display = '';
+  var label = graphArchivedId ? 'Saving checkpoint…' : 'Archiving…';
+  archiveBar.innerHTML = '<span>' + label + '</span><div class="archive-bar-fill-wrap"><div class="archive-bar-fill" style="width:0%"></div></div>';
+
   try {
+    var body = {
+      toolType:  'graph_' + tool,
+      data:      allItems.map(function (d) { return d.item; }),
+      queryInfo: { query: query, source: 'graph', nodes: cy.nodes().length },
+    };
+    if (graphArchivedId) body.archiveId = graphArchivedId;
+
     var res = await fetch('/api/archive', {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify({
-        toolType:  'graph_' + tool,
-        data:      allItems.map(function (d) { return d.item; }),
-        queryInfo: { query: query, source: 'graph', nodes: cy.nodes().length },
-      }),
+      body: JSON.stringify(body),
     });
     var json = await res.json();
     if (!json.ok) throw new Error(json.error);
-    setStatus('Archived — ID: ' + json.archiveId);
+    graphArchivedId = json.archiveId;
+    document.getElementById('btnArchiveAll').textContent = 'Update Archive';
+    pollGraphArchive(json.archiveId);
   } catch (e) {
-    setStatus('Archive error: ' + e.message, true);
+    archiveBar.innerHTML = '<span style="color:var(--danger)">Archive error: ' + esc(e.message) + '</span>';
   }
 }
 
+function pollGraphArchive(archiveId) {
+  var archiveBar = document.getElementById('archiveBar');
+  var interval = setInterval(async function () {
+    var res  = await fetch('/api/archive/' + archiveId + '/status');
+    var json = await res.json();
+    if (!json.ok) { clearInterval(interval); return; }
+
+    var status = json.status, progress = json.progress, total = json.total;
+    var fill = archiveBar.querySelector('.archive-bar-fill');
+
+    if (status === 'saving') {
+      archiveBar.querySelector('span').textContent = 'Saving results…';
+    } else if (status === 'downloading') {
+      var pct = total > 0 ? Math.round((progress / total) * 100) : 0;
+      archiveBar.querySelector('span').textContent = 'Downloading media ' + progress + '/' + total;
+      if (fill) fill.style.width = pct + '%';
+    } else if (status === 'done') {
+      clearInterval(interval);
+      if (fill) fill.style.width = '100%';
+      archiveBar.querySelector('span').textContent = 'Archived ✓ (' + total + ' media files saved) — ID: ' + archiveId;
+    }
+  }, 1200);
+}
+
 // ── Dump JSON ─────────────────────────────────────────────────────────────────
 function dumpJSON() {
   if (!allItems.length) { setStatus('Nothing to export yet', true); return; }
@@ -1110,6 +1274,9 @@ function clearGraph() {
   allItems.length = 0;
   hidePanel();
   updateNodeCount();
+  graphArchivedId = null;
+  document.getElementById('btnArchiveAll').textContent = 'Archive All';
+  document.getElementById('archiveBar').style.display = 'none';
   setStatus('Graph cleared');
 }
 

Script/SOCMINT-Twitter/templates/index.html

@@ -447,6 +447,24 @@
     text-align: center;
   }
 
+  .load-more-status {
+    text-align: center;
+    font-size: 12px;
+    color: var(--muted);
+    padding: 14px 0 20px;
+    display: none;
+  }
+  .load-more-status.visible { display: block; }
+  .load-more-status .spin-dot {
+    display: inline-block;
+    width: 6px; height: 6px;
+    border-radius: 50%;
+    background: var(--accent);
+    margin-right: 6px;
+    animation: lm-pulse 1s ease-in-out infinite;
+  }
+  @keyframes lm-pulse { 0%, 100% { opacity: 0.3; } 50% { opacity: 1; } }
+
   /* Media (gambar / video) */
   .card-media {
     display: flex;
@@ -556,6 +574,7 @@
   .leaflet-control-attribution { background: rgba(26,29,39,0.8) !important; color: var(--muted) !important; font-size: 9px; }
 </style>
 <script src="https://unpkg.com/leaflet@1.9.4/dist/leaflet.js" crossorigin="anonymous"></script>
+<script src="{{ url_for('static', filename='js/card_constants.js') }}" nonce="{{ g.csp_nonce }}"></script>
 </head>
 <body>
 
@@ -627,6 +646,9 @@
       <span class="toggle-label">Auto Archive</span>
     </div>
     <div id="archiveBar" class="archive-bar" style="display:none"></div>
+    <button id="checkpointBtn" class="btn-download" style="display:none;width:100%;margin-top:8px">
+      Save Checkpoint... Saved latest data 
+    </button>
 
     <!-- Recent archives -->
     <div class="archives-section" id="archivesSection" style="display:none">
@@ -649,6 +671,8 @@
     <div id="resultBox">
       <div class="empty-state">Nothing found :(</div>
     </div>
+    <div id="loadMoreSentinel" style="height:1px"></div>
+    <div id="loadMoreStatus" class="load-more-status"></div>
     <div id="mapStatus" style="display:none;font-size:12px;color:var(--muted);padding:6px 2px"></div>
     <div id="mapContainer"></div>
   </div>
@@ -693,11 +717,37 @@ const toggleTrack    = document.getElementById('toggleTrack');
 const archiveBar     = document.getElementById('archiveBar');
 const archivesSection= document.getElementById('archivesSection');
 const archivesList   = document.getElementById('archivesList');
+const loadMoreSentinel = document.getElementById('loadMoreSentinel');
+const loadMoreStatus   = document.getElementById('loadMoreStatus');
+const checkpointBtn    = document.getElementById('checkpointBtn');
 
 let currentMode    = 'api';
 let currentData    = null;
 let autoArchive    = false;
 let currentPayload = null;   // last run payload (for archive queryInfo)
+let archivedId     = null;   // archive id for the current run, once auto-archived — lets later load-more pages update it in place instead of creating a new archive
+
+// ── Scroll-triggered load-more (Cookie mode + Wayback only) ────────────────
+// Tools/modes the backend actually paginates — everything else (xquik/API
+// mode, multi-source search) just gets a single page, same as before.
+const PAGINATED_TOOLS = new Set([
+  'tweet_search_extractor', 'follower_explorer', 'post_extractor',
+  'community_post_extractor', 'tweet_replies_extractor',
+  'tweet_retweeters_extractor', 'geo_post_extractor',
+  'wayback_archive_search',
+]);
+const THROTTLE_SECONDS = 5;
+
+let nextCursor    = null;
+let loadingMore    = false;
+let cooldownUntil  = { cookie: 0, wayback: 0 };   // Date.now()-based timestamps
+let cooldownTimer  = null;
+
+function throttleSourceFor(tool, mode) {
+  if (tool === 'wayback_archive_search') return 'wayback';
+  if (mode === 'cookie' && PAGINATED_TOOLS.has(tool)) return 'cookie';
+  return null;
+}
 
 const HINTS = {
   tweet_search_extractor:    'Search for tweets by keyword',
@@ -850,16 +900,10 @@ function flatText(obj) {
   return Object.values(obj).map(flatText).join(' ');
 }
 
-const PRIORITY  = ['source','account_age_flag','account_age','account_created','retweeted_by_user','retweeted_by_name','retweeted_text','retweeted_by_bio','name','username','screen_name','user','user_id','text','full_text','content','title','description','bio','article_text','post_title','post_text','created_at','in_reply_to_tweet_id','archive_url','preview_image','iso_date','original','statuscode'];
+// PRIORITY / DRILLABLE / SOURCE_CLASS / AGE_LABELS come from
+// static/js/card_constants.js, loaded above — shared with archive.html.
 const SKIP      = ['profile_image_url','profile_banner_url','entities','extended_entities','urls','media','indices'];
 const CLAMP     = new Set(['text','full_text','content','description','bio','article_text','retweeted_text','retweeted_by_bio','post_text']);
-const SOURCE_CLASS = { 'Twitter Cookie': 'src-cookie', 'Xquik API': 'src-xquik', 'Wayback Machine': 'src-wayback' };
-// Keys that link out to a drill-down tool (key → target toolType)
-const DRILLABLE = {
-  reply_count:          'tweet_replies_extractor',
-  retweet_count:        'tweet_retweeters_extractor',
-  in_reply_to_tweet_id: 'tweet_replies_extractor',   // follow thread upward
-};
 
 // ── Media helpers ─────────────────────────────────────────────────────────────
 
@@ -935,9 +979,17 @@ function buildCard(item) {
   }
   const tweetId = item.id ? String(item.id) : null;
   const entries = Object.entries(item);
-  const pri     = entries.filter(([k]) => PRIORITY.includes(k));
+  // Order follows PRIORITY's own sequence (not object insertion order), so
+  // field position is deterministic and consistent across every tool/record
+  // shape. No cap on row count — this is an OSINT/analysis tool, so more
+  // extracted metadata is the point, not something to hide past a limit
+  // (this is what silently dropped the reply_count/retweet_count drill-down
+  // anchors once enough other fields — account age, verification, etc. —
+  // were present on a record).
+  const priKeys = new Set(entries.map(([k]) => k).filter(k => PRIORITY.includes(k)));
+  const pri     = PRIORITY.filter(k => priKeys.has(k)).map(k => [k, item[k]]);
   const rest    = entries.filter(([k]) => !PRIORITY.includes(k) && !SKIP.includes(k));
-  const rows    = [...pri, ...rest].slice(0, 14).map(([k, v]) => {
+  const rows    = [...pri, ...rest].map(([k, v]) => {
     let display;
     // Drillable check runs FIRST — reply_count / retweet_count always get a link
     // when we have a tweet ID, even if the count is 0 or null (twikit often under-reports)
@@ -953,8 +1005,7 @@ function buildCard(item) {
       const cls = SOURCE_CLASS[v] || '';
       display = `<span class="source-badge ${cls}">${esc(String(v))}</span>`;
     } else if (k === 'account_age_flag' && v) {
-      const label = { new: 'New account', recent: 'Recent account', established: 'Established account' }[v] || v;
-      display = `<span class="age-badge age-${esc(String(v))}">${esc(label)}</span>`;
+      display = `<span class="age-badge age-${esc(String(v))}">${esc(AGE_LABELS[v] || v)}</span>`;
     } else if (v === null || v === undefined) {
       display = `<span style="color:var(--muted)">—</span>`;
     } else if (typeof v === 'object') {
@@ -971,6 +1022,129 @@ function buildCard(item) {
   return `<div class="card">${rows}${renderMedia(media)}</div>`;
 }
 
+// ── Load more (scroll-triggered pagination) ─────────────────────────────────
+// The sentinel lives as a SIBLING of #resultBox, not inside it — renderCards()
+// replaces #resultBox.innerHTML wholesale on every run and every keystroke in
+// the search box, which would destroy an in-box sentinel and its observer tie.
+
+function appendCards(newItems, query = '') {
+  const q = query.toLowerCase().trim();
+  const toShow = q ? newItems.filter(i => flatText(i).toLowerCase().includes(q)) : newItems;
+  const grid = resultBox.querySelector('.cards-grid');
+  if (grid && toShow.length) {
+    grid.insertAdjacentHTML('beforeend', toShow.map(buildCard).join(''));
+  }
+  const totalShown = resultBox.querySelectorAll('.card').length;
+  resultCount.textContent = totalShown + (q ? ' found' : ' results');
+}
+
+function resetPagination() {
+  nextCursor   = null;
+  loadingMore  = false;
+  archivedId   = null;
+  if (cooldownTimer) { clearTimeout(cooldownTimer); cooldownTimer = null; }
+  loadMoreStatus.classList.remove('visible');
+  loadMoreStatus.textContent = '';
+  checkpointBtn.style.display = 'none';
+}
+
+function buildQueryInfo() {
+  const queryInfo = { ...currentPayload };
+  delete queryInfo.toolType;
+  delete queryInfo.mode;
+  delete queryInfo.count;
+  return queryInfo;
+}
+
+function stampCooldown(source) {
+  if (!source) return;
+  cooldownUntil[source] = Date.now() + THROTTLE_SECONDS * 1000;
+}
+
+function maybeLoadMore() {
+  if (!nextCursor || loadingMore || !currentPayload) return;
+  const source = throttleSourceFor(currentPayload.toolType, currentPayload.mode);
+  if (!source) return;   // shouldn't happen if nextCursor is set, but be safe
+
+  const remaining = cooldownUntil[source] - Date.now();
+  if (remaining > 0) {
+    armCountdown(remaining);
+    return;
+  }
+  doLoadMore(source);
+}
+
+function armCountdown(msRemaining) {
+  if (cooldownTimer) return;   // already counting down
+  const secs = Math.max(1, Math.ceil(msRemaining / 1000));
+  loadMoreStatus.classList.add('visible');
+  loadMoreStatus.innerHTML = `<span class="spin-dot"></span>Loading more in ${secs}s…`;
+  cooldownTimer = setTimeout(() => {
+    cooldownTimer = null;
+    maybeLoadMore();   // re-check: fires the load once the cooldown has cleared
+  }, msRemaining);
+}
+
+async function doLoadMore(source) {
+  loadingMore = true;
+  loadMoreStatus.classList.add('visible');
+  loadMoreStatus.innerHTML = `<span class="spin-dot"></span>Loading more…`;
+
+  try {
+    const res  = await fetch('/api/run', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ ...currentPayload, cursor: nextCursor }),
+    });
+    const json = await res.json();
+    stampCooldown(source);
+
+    if (!json.ok) {
+      if (res.status === 429 && json.retryAfter) {
+        loadingMore = false;
+        armCountdown(json.retryAfter * 1000);
+        return;
+      }
+      loadMoreStatus.textContent = `Load more failed: ${json.error}`;
+      nextCursor = null;
+      return;
+    }
+
+    const newItems = Array.isArray(json.data) ? json.data : [json.data];
+    currentData = (Array.isArray(currentData) ? currentData : [currentData]).concat(newItems);
+    appendCards(newItems, searchInput.value);
+    nextCursor = json.nextCursor || null;
+    loadMoreStatus.classList.remove('visible');
+
+    // Auto-archive only covered whatever was loaded at the initial Run — new
+    // pages from scroll need their own save. Once we've reached the end
+    // (no more pages), save automatically; otherwise surface a manual
+    // checkpoint button so the user can save without waiting to hit the end.
+    if (autoArchive && archivedId) {
+      if (!nextCursor) {
+        checkpointBtn.style.display = 'none';
+        triggerArchive(currentData, currentPayload.toolType, buildQueryInfo(), archivedId);
+      } else {
+        checkpointBtn.style.display = '';
+      }
+    }
+  } catch (e) {
+    loadMoreStatus.textContent = `Load more failed: ${e}`;
+  } finally {
+    loadingMore = false;
+  }
+}
+
+// root: null (the browser viewport) — .output has `overflow: auto` but is
+// never actually height-constrained (body/.layout only set min-height), so
+// it never becomes a real scroll container; the page/viewport is what
+// actually scrolls. Pointing the observer at .output meant it could never
+// detect the sentinel entering view.
+const loadMoreObserver = new IntersectionObserver((entries) => {
+  if (entries.some(e => e.isIntersecting)) maybeLoadMore();
+}, { root: null, rootMargin: '200px' });
+loadMoreObserver.observe(loadMoreSentinel);
+
 // ── Run ───────────────────────────────────────────────────────────────────────
 
 runBtn.addEventListener('click', async () => {
@@ -998,6 +1172,7 @@ runBtn.addEventListener('click', async () => {
   }
 
   currentPayload = payload;
+  resetPagination();
 
   runBtn.disabled            = true;
   runBtn.textContent         = 'Running…';
@@ -1018,9 +1193,15 @@ runBtn.addEventListener('click', async () => {
     const json = await res.json();
 
     if (json.ok) {
+      // Keep the client's cooldown clock in sync even if the user never
+      // scrolls to trigger a load-more — a fast scroll right after Run
+      // should still wait out the same 5s window the server just started.
+      stampCooldown(throttleSourceFor(t, currentMode));
+
       statusBadge.textContent    = 'done';
       statusBadge.className      = 'badge ok';
       currentData                = json.data;
+      nextCursor                 = json.nextCursor || null;
       const isMany               = Array.isArray(json.data) && json.data.length > 1;
       downloadBtn.style.display  = '';
 
@@ -1041,11 +1222,7 @@ runBtn.addEventListener('click', async () => {
       renderCards(currentData);
 
       if (autoArchive) {
-        const queryInfo = { ...currentPayload };
-        delete queryInfo.toolType;
-        delete queryInfo.mode;
-        delete queryInfo.count;
-        triggerArchive(json.data, t, queryInfo);
+        triggerArchive(json.data, t, buildQueryInfo());
       } else {
         archiveBar.style.display = 'none';
       }
@@ -1072,21 +1249,32 @@ archiveToggle.addEventListener('click', () => {
   if (autoArchive) loadRecentArchives();
 });
 
-async function triggerArchive(data, toolType, queryInfo) {
+async function triggerArchive(data, toolType, queryInfo, existingId) {
   archiveBar.style.display = '';
-  archiveBar.innerHTML     = '<span>Archiving…</span><div class="archive-bar-fill-wrap"><div class="archive-bar-fill" style="width:0%"></div></div>';
+  const label = existingId ? 'Saving checkpoint…' : 'Archiving…';
+  archiveBar.innerHTML = `<span>${label}</span><div class="archive-bar-fill-wrap"><div class="archive-bar-fill" style="width:0%"></div></div>`;
+
+  const body = { data, toolType, queryInfo };
+  if (existingId) body.archiveId = existingId;
 
   const res  = await fetch('/api/archive', {
     method: 'POST',
     headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify({ data, toolType, queryInfo }),
+    body: JSON.stringify(body),
   });
   const json = await res.json();
   if (!json.ok) { archiveBar.innerHTML = `<span style="color:var(--danger)">Archive failed: ${esc(json.error)}</span>`; return; }
 
+  archivedId = json.archiveId;
   pollArchive(json.archiveId);
 }
 
+checkpointBtn.addEventListener('click', () => {
+  if (!archivedId || !currentPayload) return;
+  checkpointBtn.style.display = 'none';
+  triggerArchive(currentData, currentPayload.toolType, buildQueryInfo(), archivedId);
+});
+
 function pollArchive(archiveId) {
   const interval = setInterval(async () => {
     const res  = await fetch(`/api/archive/${archiveId}/status`);
@@ -1189,6 +1377,9 @@ function setView(v) {
   const mapStatus    = document.getElementById('mapStatus');
   const searchInput  = document.getElementById('searchInput');
 
+  // No "bottom of the list" concept on a Leaflet map — suspend load-more there.
+  loadMoreSentinel.style.display = isCards ? '' : 'none';
+
   if (isCards) {
     mapContainer.style.display = 'none';
     mapStatus.style.display    = 'none';