WEBDIENSTE/OSINT-Cheat-sheet
1
0
Fork 0
mirror of https://github.com/Jieyab89/OSINT-Cheat-sheet.git synced 2026-08-17 18:35:41 +02:00
Code Issues Packages Projects Releases Wiki Activity

enhancement new featrues and fix bug

Browse Source
This commit is contained in:
Jieyab89
2026-08-11 22:23:46 +07:00
parent 88905e8493
commit ee5658fc0a
7 changed files with 1101 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Script/SOCMINT-Twitter/Readme.md
+11
View File
@@ -21,6 +21,10 @@
15. Update view as graph in archive
16. Add date and timestamp pattern in sentiment analysis
17. Delete entry node in graph visualizer
18. Add new features save project (load case or load new case)
19. Checkpoint or save data
20. Auto resume archive and dump data with state (state data to checkpoint)
21. Add date or timestamp paramater for all search module
## Features
@@ -289,6 +293,13 @@ Dasboard Home
<img width="2423" height="1217" alt="Image" src="https://github.com/user-attachments/assets/6ba91a78-5845-4e17-87e2-8171da3cec19" />
Load Case / Project
<img width="2532" height="1217" alt="image" src="https://github.com/user-attachments/assets/433d6f8a-a10f-4add-afeb-8935fc643e67" />
Checkpoint or Save the Project
<img width="2557" height="1218" alt="image" src="https://github.com/user-attachments/assets/d2ef01d6-108c-4ba7-ac90-52bb1fd881ce" />
Archive
<img width="2511" height="1228" alt="Image" src="https://github.com/user-attachments/assets/9e4fab45-edda-45f5-a88c-963e6bfdaeaf" />
Script/SOCMINT-Twitter/app.py
+173 -15
View File
@@ -7,6 +7,7 @@ import threading
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
from pathlib import Path
import requests as _req
from flask import Flask, g, jsonify, render_template, request, Response, stream_with_context, send_from_directory
@@ -416,12 +417,22 @@ def run_tool():
source_errors = None # multi_source_search only — {source: error} for lanes that failed this page
try:
# Common date range — validated once; all tools except wayback (which uses its
# own waybackFrom/waybackTo keys) and article_extractor (single item, no date)
# can receive these to narrow results.
from_date = body.get("dateFrom", "").strip()
to_date = body.get("dateTo", "").strip()
for d_label, d_val in (("dateFrom", from_date), ("dateTo", to_date)):
if d_val and not _valid_date8(d_val):
return jsonify({"ok": False, "error": f"{d_label} must be YYYYMMDD"}), 400
if tool_type == "tweet_search_extractor":
query = body.get("searchQuery", "")
query = _apply_date_operators(body.get("searchQuery", ""), from_date, to_date)
if mode == "cookie":
data, next_cursor = cookie_tweet_search(query, count=count, config=config, cursor=cursor)
else:
data = XquikClient(config).tweet_search(query)
data = _filter_by_date(data, from_date, to_date)
elif tool_type == "follower_explorer":
username = body.get("targetUsername", "")
@@ -429,12 +440,14 @@ def run_tool():
data, next_cursor = cookie_follower_explorer(username, count=count, config=config, cursor=cursor)
else:
data = XquikClient(config).follower_explorer(username)
data = _filter_by_date(data, from_date, to_date)
elif tool_type == "following_explorer":
username = body.get("targetUsername", "")
if mode != "cookie":
return jsonify({"ok": False, "error": "following_explorer requires cookie mode"}), 400
data, next_cursor = cookie_following_explorer(username, count=count, config=config, cursor=cursor)
data = _filter_by_date(data, from_date, to_date)
elif tool_type == "article_extractor":
tweet_id = body.get("targetTweetId", "")
@@ -449,6 +462,7 @@ def run_tool():
data, next_cursor = cookie_community_post_extractor(community_id, count=count, config=config, cursor=cursor)
else:
data = XquikClient(config).community_post_extractor(community_id)
data = _filter_by_date(data, from_date, to_date)
elif tool_type == "post_extractor":
username = body.get("targetUsername", "")
@@ -456,41 +470,40 @@ def run_tool():
data, next_cursor = cookie_post_extractor(username, count=count, config=config, cursor=cursor)
else:
data = XquikClient(config).post_extractor(username)
data = _filter_by_date(data, from_date, to_date)
elif tool_type == "tweet_replies_extractor":
tweet_id = body.get("targetTweetId", "")
if mode != "cookie":
return jsonify({"ok": False, "error": "tweet_replies_extractor requires cookie mode"}), 400
data, next_cursor = cookie_tweet_replies(tweet_id, count=count, config=config, cursor=cursor)
data = _filter_by_date(data, from_date, to_date)
elif tool_type == "tweet_retweeters_extractor":
tweet_id = body.get("targetTweetId", "")
if mode != "cookie":
return jsonify({"ok": False, "error": "tweet_retweeters_extractor requires cookie mode"}), 400
data, next_cursor = cookie_tweet_retweeters(tweet_id, count=count, config=config, cursor=cursor)
data = _filter_by_date(data, from_date, to_date)
elif tool_type == "geo_post_extractor":
keyword = body.get("searchQuery", "")
keyword = _apply_date_operators(body.get("searchQuery", ""), from_date, to_date)
if mode != "cookie":
return jsonify({"ok": False, "error": "geo_post_extractor requires cookie mode"}), 400
data, next_cursor = cookie_geo_search(keyword, count=count, config=config, cursor=cursor)
data = _filter_by_date(data, from_date, to_date)
elif tool_type == "wayback_archive_search":
target = body.get("searchQuery", "")
from_date = body.get("waybackFrom", "")
to_date = body.get("waybackTo", "")
for label, val in (("waybackFrom", from_date), ("waybackTo", to_date)):
if val and not _valid_date8(val):
return jsonify({"ok": False, "error": f"{label} must be an 8-digit date (YYYYMMDD)"}), 400
data, next_cursor = wayback_search(target, count=count, from_date=from_date, to_date=to_date, cursor=cursor)
target = body.get("searchQuery", "")
wb_from = body.get("waybackFrom", "")
wb_to = body.get("waybackTo", "")
for d_label, d_val in (("waybackFrom", wb_from), ("waybackTo", wb_to)):
if d_val and not _valid_date8(d_val):
return jsonify({"ok": False, "error": f"{d_label} must be an 8-digit date (YYYYMMDD)"}), 400
data, next_cursor = wayback_search(target, count=count, from_date=wb_from, to_date=wb_to, cursor=cursor)
elif tool_type == "multi_source_search":
query = body.get("searchQuery", "")
from_date = body.get("dateFrom", "")
to_date = body.get("dateTo", "")
for label, val in (("dateFrom", from_date), ("dateTo", to_date)):
if val and not _valid_date8(val):
return jsonify({"ok": False, "error": f"{label} must be an 8-digit date (YYYYMMDD)"}), 400
query = body.get("searchQuery", "")
data, next_cursor, source_errors = _multi_source_search(query, count=count, from_date=from_date, to_date=to_date, cursor=cursor)
else:
@@ -608,6 +621,151 @@ def archive_list():
return jsonify({"ok": True, "archives": _archive.list_all()})
# ── Cases (save / resume investigation sessions) ──────────────────────────────
CASES_ROOT = Path(__file__).parent / "cases"
_CASE_ID_RE = re.compile(r"^case_\d{8}_\d{6}_[0-9a-f]{6}$")
def _valid_case_id(cid: str) -> bool:
return bool(_CASE_ID_RE.match(cid))
@app.route("/api/cases")
def cases_list():
CASES_ROOT.mkdir(exist_ok=True)
out = []
for d in CASES_ROOT.iterdir():
if not d.is_dir():
continue
mf = d / "meta.json"
if not mf.exists():
continue
try:
out.append(json.loads(mf.read_text()))
except Exception:
continue
out.sort(key=lambda c: c.get("updated_at", ""), reverse=True)
return jsonify({"ok": True, "cases": out})
@app.route("/api/cases", methods=["POST"])
def cases_create():
body = request.get_json(silent=True) or {}
name = str(body.get("name", "Unnamed Case")).strip()[:120]
page = str(body.get("page", "index"))[:16]
state = body.get("state")
if state is None:
return jsonify({"ok": False, "error": "No state provided"}), 400
now = datetime.now()
case_id = f"case_{now.strftime('%Y%m%d_%H%M%S')}_{secrets.token_hex(3)}"
case_dir = CASES_ROOT / case_id
case_dir.mkdir(parents=True, exist_ok=True)
tool = str(body.get("tool", ""))[:80].strip()
ts = now.strftime("%Y-%m-%d %H:%M:%S")
meta = {"id": case_id, "name": name, "page": page, "tool": tool, "created_at": ts, "updated_at": ts}
(case_dir / "meta.json").write_text(json.dumps(meta, indent=2))
(case_dir / "state.json").write_text(json.dumps(state))
return jsonify({"ok": True, "caseId": case_id, "meta": meta})
@app.route("/api/cases/<case_id>")
def cases_get(case_id):
if not _valid_case_id(case_id):
return jsonify({"ok": False, "error": "Invalid case ID"}), 400
mf = CASES_ROOT / case_id / "meta.json"
sf = CASES_ROOT / case_id / "state.json"
if not mf.exists():
return jsonify({"ok": False, "error": "Case not found"}), 404
try:
meta = json.loads(mf.read_text())
state = json.loads(sf.read_text())
except Exception as e:
return jsonify({"ok": False, "error": str(e)}), 500
return jsonify({"ok": True, "meta": meta, "state": state})
@app.route("/api/cases/<case_id>", methods=["PUT"])
def cases_update(case_id):
if not _valid_case_id(case_id):
return jsonify({"ok": False, "error": "Invalid case ID"}), 400
mf = CASES_ROOT / case_id / "meta.json"
if not mf.exists():
return jsonify({"ok": False, "error": "Case not found"}), 404
body = request.get_json(silent=True) or {}
try:
meta = json.loads(mf.read_text())
except Exception as e:
return jsonify({"ok": False, "error": str(e)}), 500
if "name" in body:
meta["name"] = str(body["name"]).strip()[:120]
if "tool" in body:
meta["tool"] = str(body["tool"]).strip()[:80]
meta["updated_at"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if "state" in body:
(CASES_ROOT / case_id / "state.json").write_text(json.dumps(body["state"]))
mf.write_text(json.dumps(meta, indent=2))
return jsonify({"ok": True, "meta": meta})
@app.route("/api/cases/<case_id>", methods=["DELETE"])
def cases_delete(case_id):
if not _valid_case_id(case_id):
return jsonify({"ok": False, "error": "Invalid case ID"}), 400
case_dir = CASES_ROOT / case_id
if not case_dir.exists():
return jsonify({"ok": False, "error": "Case not found"}), 404
shutil.rmtree(case_dir)
return jsonify({"ok": True})
@app.route("/api/cases/beacon", methods=["POST"])
def cases_beacon():
"""navigator.sendBeacon() target — called on beforeunload. Always 204; response is ignored."""
try:
body = request.get_json(silent=True) or {}
state = body.get("state")
if not state:
return "", 204
tool = str(body.get("tool", ""))[:80].strip()
case_id = str(body.get("caseId", "")).strip()
CASES_ROOT.mkdir(exist_ok=True)
if case_id and _valid_case_id(case_id):
mf = CASES_ROOT / case_id / "meta.json"
sf = CASES_ROOT / case_id / "state.json"
if mf.exists():
try:
meta = json.loads(mf.read_text())
except Exception:
meta = {}
if tool:
meta["tool"] = tool
meta["updated_at"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
mf.write_text(json.dumps(meta, indent=2))
sf.write_text(json.dumps(state))
else:
name = str(body.get("name", ""))[:120].strip() or "Auto-save"
page = str(body.get("page", "index"))[:16]
now = datetime.now()
cid = f"case_{now.strftime('%Y%m%d_%H%M%S')}_{secrets.token_hex(3)}"
cdir = CASES_ROOT / cid
cdir.mkdir(parents=True, exist_ok=True)
ts = now.strftime("%Y-%m-%d %H:%M:%S")
meta = {"id": cid, "name": name, "page": page, "tool": tool, "created_at": ts, "updated_at": ts}
(cdir / "meta.json").write_text(json.dumps(meta, indent=2))
(cdir / "state.json").write_text(json.dumps(state))
except Exception:
pass
return "", 204
@app.route("/cases")
def cases_page():
return render_template("cases.html")
# ── Analytics (sentiment / clustering) ──────────────────────────────────────
# Same background-thread + polling shape archive.py's downloads already use
# (start() kicks off a thread and returns immediately, status() reports
Script/SOCMINT-Twitter/templates/analytics.html
+25 -2
View File
@@ -256,6 +256,15 @@
background: var(--accent-bg); border: 1px solid var(--accent); border-radius: 20px;
padding: 3px 10px; cursor: pointer; margin-left: 8px;
}
.cal-date-stat {
margin-top: 8px;
padding: 7px 12px;
background: var(--bg);
border: 1px solid var(--border);
border-radius: 6px;
font-size: 12px;
color: var(--text);
}
/* ── Sentiment badge (shared by tiles/list) ── */
.sent-badge {
@@ -389,6 +398,7 @@
<a href="/archives">Archives</a>
<hr class="nav-divider">
<a href="/analytics" class="current">Analytics</a>
<a href="/cases">Cases</a>
</nav>
</header>
@@ -725,7 +735,7 @@ function renderDashboard() {
<div class="method-note">
${d.method === 'ml'
? 'Sentiment is scored by a multilingual model (XLM-RoBERTa, fine-tuned on tweets across 8 languages and reasonably capable well beyond those) — works on non-Indonesian text, not just the lexicon fallback below. Still a heuristic, not ground truth: short text, sarcasm and irony all still degrade accuracy. Click a tile below to see which items landed in it and the model\'s confidence per item.'
: 'Sentiment is scored against an Indonesian positive/negative word lexicon with negation handling (e.g. "tidak bagus" flips to con) — a transparent rule-based heuristic, Indonesian-only. Install torch+transformers (see requirements.txt) for multilingual ML-based scoring instead. Click a tile below to see exactly which items landed in it and which words drove each score.'}
: 'Sentiment is scored con an Indonesian positive/negative word lexicon with negation handling (e.g. "tidak bagus" flips to con) — a transparent rule-based heuristic, Indonesian-only. Install torch+transformers (see requirements.txt) for multilingual ML-based scoring instead. Click a tile below to see exactly which items landed in it and which words drove each score.'}
${d.total_items - d.total_scored > 0 ? `${d.total_items - d.total_scored} of ${d.total_items} record(s) had no text to score (bare follower/following/user entries) and are excluded below.` : ''}
</div>
@@ -1135,6 +1145,18 @@ function calendarHtml(daily) {
return `<div class="cal-week">${cells}</div>`;
}).join('');
const selDay = activeDateFilter ? days.get(activeDateFilter) : null;
const selStat = selDay
? (() => {
const d = selDay;
const parts = [];
if (d.pro > 0) parts.push(`${d.pro} positive`);
if (d.neutral > 0) parts.push(`${d.neutral} neutral`);
if (d.con > 0) parts.push(`${d.con} con`);
return `${d.total} item${d.total !== 1 ? 's' : ''} collected${parts.length ? ' — ' + parts.join(' / ') : ''}`;
})()
: null;
return `
<div class="cal-outer">
<div class="cal-scroll">
@@ -1150,8 +1172,9 @@ function calendarHtml(daily) {
<span class="cal-cell cal-has-data cal-level-4"></span>
More
${undated ? `· ${undated} item(s) with no usable date not shown` : ''}
${activeDateFilter ? `<span class="cal-filter-chip" id="calClearFilter">Filtered: ${esc(activeDateFilter)} ×</span>` : ''}
${activeDateFilter ? `<span class="cal-filter-chip" id="calClearFilter">${esc(activeDateFilter)} &times;</span>` : ''}
</div>
${selStat ? `<div class="cal-date-stat">${esc(activeDateFilter)}: ${esc(selStat)}</div>` : ''}
</div>`;
}
Script/SOCMINT-Twitter/templates/archive.html
+2
View File
@@ -578,6 +578,8 @@
<hr class="nav-divider">
<a href="/archives" class="current">Archives</a>
<a href="/analytics">Analytics</a>
<hr class="nav-divider">
<a href="/cases">Cases</a>
</nav>
</header>
Script/SOCMINT-Twitter/templates/cases.html
+260
View File
@@ -0,0 +1,260 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Jieyab89 SOCMINT X — Cases</title>
<style>
:root {
--bg: #0f1117;
--surface: #1a1d27;
--border: #2a2d3a;
--text: #e8eaf0;
--muted: #8890a4;
--accent: #5865f2;
--accent-bg: #1e2240;
--danger: #ef4444;
--font: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", sans-serif;
}
* { box-sizing: border-box; margin: 0; padding: 0; }
body { background: var(--bg); color: var(--text); font-family: var(--font); font-size: 14px; line-height: 1.6; min-height: 100vh; }
header {
padding: 14px 24px; border-bottom: 1px solid var(--border);
display: flex; align-items: center; gap: 10px;
background: var(--surface); position: sticky; top: 0; z-index: 100;
}
header h1 { font-size: 14px; font-weight: 600; letter-spacing: 0.02em; }
header .sep { color: var(--border); }
header .sub { color: var(--muted); font-size: 12px; }
.hamburger-btn {
margin-left: auto; background: none; border: 1px solid var(--border);
border-radius: 5px; color: var(--muted); cursor: pointer;
padding: 5px 9px; font-size: 16px; line-height: 1; transition: all 0.15s;
}
.hamburger-btn:hover { color: var(--text); border-color: var(--accent); }
.nav-menu {
position: absolute; top: 100%; right: 12px; margin-top: 6px;
background: var(--surface); border: 1px solid var(--border);
border-radius: 7px; padding: 5px; min-width: 175px;
box-shadow: 0 8px 28px rgba(0,0,0,0.5); z-index: 200;
display: flex; flex-direction: column; gap: 1px;
}
.nav-menu.hidden { display: none; }
.nav-menu a {
padding: 7px 12px; border-radius: 5px; color: var(--muted);
text-decoration: none; font-size: 13px; transition: all 0.12s; display: block;
}
.nav-menu a:hover { color: var(--text); background: var(--accent-bg); }
.nav-menu a.current { color: var(--accent); background: var(--accent-bg); }
.nav-divider { border: none; border-top: 1px solid var(--border); margin: 3px 0; }
.page { max-width: 900px; margin: 0 auto; padding: 28px 20px; }
.topbar { display: flex; align-items: center; gap: 12px; margin-bottom: 20px; flex-wrap: wrap; }
.topbar h2 { font-size: 15px; font-weight: 600; flex: 1; }
.search-wrap { position: relative; }
.search-input {
padding: 7px 12px; background: var(--surface); border: 1px solid var(--border);
border-radius: 6px; color: var(--text); font-family: var(--font);
font-size: 13px; width: 240px; transition: border-color 0.15s;
}
.search-input:focus { outline: none; border-color: var(--accent); }
.search-input::placeholder { color: var(--muted); }
.cases-table-wrap { overflow-x: auto; border-radius: 8px; border: 1px solid var(--border); }
table { width: 100%; border-collapse: collapse; }
thead th {
padding: 10px 14px; text-align: left; font-size: 11px; font-weight: 600;
text-transform: uppercase; letter-spacing: 0.04em; color: var(--muted);
border-bottom: 1px solid var(--border); background: var(--surface); white-space: nowrap;
}
thead th.th-right { text-align: right; }
tbody tr { border-bottom: 1px solid var(--border); transition: background 0.1s; }
tbody tr:last-child { border-bottom: none; }
tbody tr:hover { background: rgba(88,101,242,0.04); }
tbody td { padding: 10px 14px; vertical-align: middle; font-size: 13px; }
.td-name { max-width: 280px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; font-weight: 500; }
.td-meta { color: var(--muted); font-size: 12px; font-variant-numeric: tabular-nums; white-space: nowrap; }
.td-actions { display: flex; gap: 6px; justify-content: flex-end; }
.page-badge {
display: inline-block; font-size: 10px; font-weight: 600;
text-transform: uppercase; letter-spacing: 0.04em;
padding: 2px 7px; border-radius: 3px; white-space: nowrap;
}
.pg-index { background: var(--accent-bg); color: #818cf8; border: 1px solid #34348a; }
.pg-graph { background: #1e1535; color: #a78bfa; border: 1px solid #4c3a8f; }
.btn-load {
padding: 4px 12px; font-size: 12px; border-radius: 5px;
background: transparent; border: 1px solid var(--accent); color: var(--accent);
font-family: var(--font); cursor: pointer; transition: all 0.12s; white-space: nowrap;
}
.btn-load:hover { background: var(--accent-bg); }
.btn-del {
padding: 4px 12px; font-size: 12px; border-radius: 5px;
background: transparent; border: 1px solid var(--border); color: var(--muted);
font-family: var(--font); cursor: pointer; transition: all 0.12s; white-space: nowrap;
}
.btn-del:hover { border-color: var(--danger); color: var(--danger); }
.empty-row td { padding: 48px 14px; text-align: center; color: var(--muted); font-size: 13px; }
.status-bar { font-size: 12px; color: var(--muted); margin-top: 10px; }
</style>
</head>
<body>
<header>
<h1>Jieyab89 SOCMINT X</h1>
<span class="sep">|</span>
<span class="sub">Cases</span>
<button id="navToggle" class="hamburger-btn" aria-label="Navigation menu">&#9776;</button>
<nav id="navMenu" class="nav-menu hidden">
<a href="/">Home</a>
<a href="/graph">Graph</a>
<hr class="nav-divider">
<a href="/archives">Archives</a>
<a href="/analytics">Analytics</a>
<hr class="nav-divider">
<a href="/cases" class="current">Cases</a>
</nav>
</header>
<div class="page">
<div class="topbar">
<h2>Saved Cases</h2>
<div class="search-wrap">
<input id="searchInput" class="search-input" type="text" placeholder="Search cases…" autocomplete="off" spellcheck="false">
</div>
</div>
<div class="cases-table-wrap">
<table>
<thead>
<tr>
<th>Name</th>
<th>Type</th>
<th>Last Saved</th>
<th class="th-right">Actions</th>
</tr>
</thead>
<tbody id="casesTbody">
<tr class="empty-row"><td colspan="4">Loading…</td></tr>
</tbody>
</table>
</div>
<div id="statusBar" class="status-bar"></div>
</div>
<script nonce="{{ g.csp_nonce }}">
(function() {
var cases = [];
function esc(s) {
return String(s)
.replace(/&/g,'&amp;').replace(/</g,'&lt;')
.replace(/>/g,'&gt;').replace(/"/g,'&quot;');
}
function fmtDate(iso) {
if (!iso) return '—';
try {
var d = new Date(iso);
return d.toLocaleDateString() + ' ' + d.toLocaleTimeString([], {hour:'2-digit',minute:'2-digit'});
} catch(e) { return iso; }
}
function render(list) {
var tbody = document.getElementById('casesTbody');
if (!list.length) {
tbody.innerHTML = '<tr class="empty-row"><td colspan="4">No saved cases. Run a search on the Home or Graph page and click Checkpoint to save a case.</td></tr>';
return;
}
tbody.innerHTML = list.map(function(c) {
var typeLabel = c.tool || (c.page === 'graph' ? 'Graph' : 'Home');
var isGraph = c.page === 'graph';
var badge = '<span class="page-badge ' + (isGraph ? 'pg-graph' : 'pg-index') + '">' + esc(typeLabel) + '</span>';
var updated = c.updated_at || c.created_at || '';
return '<tr data-id="' + esc(c.id) + '" data-page="' + esc(c.page || 'index') + '" data-name="' + esc(c.name) + '">' +
'<td class="td-name" title="' + esc(c.name) + '">' + esc(c.name) + '</td>' +
'<td>' + badge + '</td>' +
'<td class="td-meta">' + esc(fmtDate(updated)) + '</td>' +
'<td><div class="td-actions">' +
'<button class="btn-load" data-action="load">Load</button>' +
'<button class="btn-del" data-action="del">Delete</button>' +
'</div></td>' +
'</tr>';
}).join('');
}
function applyFilter() {
var q = document.getElementById('searchInput').value.toLowerCase();
var filtered = q ? cases.filter(function(c) {
return c.name.toLowerCase().indexOf(q) !== -1 || (c.page || '').indexOf(q) !== -1;
}) : cases;
render(filtered);
document.getElementById('statusBar').textContent =
filtered.length + ' / ' + cases.length + ' case' + (cases.length !== 1 ? 's' : '');
}
async function loadCases() {
try {
var res = await fetch('/api/cases');
var json = await res.json();
cases = (json.ok && Array.isArray(json.cases)) ? json.cases : [];
applyFilter();
} catch(e) {
document.getElementById('casesTbody').innerHTML =
'<tr class="empty-row"><td colspan="4">Failed to load cases: ' + esc(e.message) + '</td></tr>';
}
}
document.getElementById('searchInput').addEventListener('input', applyFilter);
document.getElementById('casesTbody').addEventListener('click', async function(e) {
var btn = e.target.closest('[data-action]');
if (!btn) return;
var row = btn.closest('tr');
var caseId = row.dataset.id;
var page = row.dataset.page;
if (btn.dataset.action === 'load') {
if (page === 'graph') {
window.location.href = '/graph?caseId=' + encodeURIComponent(caseId);
} else {
window.location.href = '/?caseId=' + encodeURIComponent(caseId);
}
return;
}
if (btn.dataset.action === 'del') {
var name = row.dataset.name;
if (!confirm('Delete case "' + name + '"?')) return;
btn.textContent = '…';
btn.disabled = true;
try {
var res = await fetch('/api/cases/' + encodeURIComponent(caseId), { method: 'DELETE' });
var json = await res.json();
if (!json.ok) throw new Error(json.error || 'Delete failed');
cases = cases.filter(function(c) { return c.id !== caseId; });
applyFilter();
} catch(err) {
btn.textContent = 'Delete';
btn.disabled = false;
alert('Delete failed: ' + err.message);
}
}
});
var navToggle = document.getElementById('navToggle');
var navMenu = document.getElementById('navMenu');
navToggle.addEventListener('click', function(e) { e.stopPropagation(); navMenu.classList.toggle('hidden'); });
document.addEventListener('click', function() { navMenu.classList.add('hidden'); });
navMenu.addEventListener('click', function(e) { e.stopPropagation(); });
loadCases();
})();
</script>
</body>
</html>
Script/SOCMINT-Twitter/templates/graph.html
+287 -6
View File
@@ -89,7 +89,6 @@
.nav-menu a:hover { color: var(--text); background: var(--accent-bg); }
.nav-menu a.current { color: var(--accent); background: var(--accent-bg); }
.nav-divider { border: none; border-top: 1px solid var(--border); margin: 3px 0; }
/* ── Toolbar ── */
#toolbar {
display: flex;
@@ -559,6 +558,53 @@
z-index: 5;
}
#emptyHint.hidden { display: none; }
/* Active case indicator in toolbar */
.case-indicator {
display: none; align-items: center; gap: 6px;
padding: 3px 8px; border-radius: 5px;
background: var(--accent-bg); border: 1px solid var(--accent); font-size: 11px;
}
.case-indicator.visible { display: flex; }
.case-indicator-name {
max-width: 120px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
color: var(--accent); font-weight: 500; color: #fff;
}
.case-checkpoint-btn {
padding: 4px 10px; font-size: 12px; border-radius: 5px;
background: transparent; border: 1px solid var(--accent); color: var(--accent);
font-family: var(--font); cursor: pointer; transition: all 0.12s; flex-shrink: 0; color: #fff;
}
.case-checkpoint-btn:hover:not(:disabled) { background: var(--accent-bg); }
.case-checkpoint-btn:disabled { opacity: 0.5; cursor: wait; }
.case-save-bar {
background: var(--surface); border-bottom: 1px solid var(--border);
padding: 10px 16px; display: flex; flex-direction: column; gap: 8px;
}
.case-save-bar[hidden] { display: none; }
.case-save-label { font-size: 12px; color: var(--muted); font-weight: 500; }
.case-save-input {
width: 100%; padding: 7px 10px; box-sizing: border-box;
background: var(--bg); border: 1px solid var(--border); border-radius: 6px;
color: var(--text); font-family: var(--font); font-size: 13px;
transition: border-color 0.15s;
}
.case-save-input:focus { outline: none; border-color: var(--accent); }
.case-save-input::placeholder { color: var(--muted); }
.case-save-btns { display: flex; gap: 6px; }
.case-save-confirm {
padding: 6px 16px; font-size: 12px; border-radius: 5px; white-space: nowrap;
background: var(--accent); color: #fff; border: none;
font-family: var(--font); font-weight: 500; cursor: pointer; transition: opacity 0.15s;
}
.case-save-confirm:disabled { opacity: 0.45; cursor: wait; }
.case-save-confirm:not(:disabled):hover { opacity: 0.85; }
.case-save-cancel {
padding: 6px 10px; font-size: 12px; border-radius: 5px; white-space: nowrap;
background: transparent; color: var(--muted); border: 1px solid var(--border);
font-family: var(--font); cursor: pointer; transition: all 0.12s;
}
.case-save-cancel:hover { color: var(--text); border-color: var(--muted); }
</style>
</head>
<body>
@@ -574,6 +620,8 @@
<hr class="nav-divider">
<a href="/archives">Archives</a>
<a href="/analytics">Analytics</a>
<hr class="nav-divider">
<a href="/cases">Cases</a>
</nav>
</header>
@@ -602,6 +650,10 @@
<input id="countInput" type="number" value="20" min="1" max="2000" title="Result count">
<input id="graphFilterInput" type="text" placeholder="Filter graph…" spellcheck="false" autocomplete="off" style="display:none;width:160px" title="Show only nodes whose data matches this text">
<span id="nodeCount">0 nodes</span>
<span id="caseIndicator" class="case-indicator">
<span class="case-indicator-name" id="caseIndicatorName"></span>
</span>
<button id="caseCheckpointBtn" class="case-checkpoint-btn">Save Case</button>
<button id="btnSearch" class="btn btn-primary">Search</button>
<button id="btnArchiveAll" class="btn btn-ghost">Archive All</button>
<button id="btnDump" class="btn btn-ghost">Dump JSON</button>
@@ -609,6 +661,14 @@
</div>
<div id="archiveBar" class="archive-bar" style="display:none"></div>
<div id="caseSaveForm" class="case-save-bar" hidden>
<span class="case-save-label">Case name</span>
<input id="caseSaveName" class="case-save-input" type="text" placeholder="Case name…" maxlength="120" autocomplete="off" spellcheck="false">
<div class="case-save-btns">
<button id="caseSaveConfirm" class="case-save-confirm">Save</button>
<button id="caseSaveCancel" class="case-save-cancel">Cancel</button>
</div>
</div>
<div id="main">
<div id="cy"></div>
@@ -690,6 +750,7 @@
{% include "_field_glossary.html" %}
</div>
<script src="https://unpkg.com/cytoscape@3.28.1/dist/cytoscape.min.js" crossorigin="anonymous"></script>
<script nonce="{{ g.csp_nonce }}">
(function () {
@@ -998,6 +1059,18 @@ function init() {
var archiveIdParam = new URLSearchParams(location.search).get('archiveId');
if (archiveIdParam) enterArchiveReadOnlyMode(archiveIdParam);
var caseIdParam = new URLSearchParams(location.search).get('caseId');
if (caseIdParam && !archiveIdParam) {
history.replaceState(null, '', location.pathname);
fetch('/api/cases/' + encodeURIComponent(caseIdParam))
.then(function(r) { return r.json(); })
.then(function(json) {
if (!json.ok) throw new Error(json.error || 'Case not found');
cmGRestoreState(json.state, json.meta);
})
.catch(function(e) { setStatus('Error loading case: ' + e.message, true); });
}
}
// ── Archive read-only bootstrap ─────────────────────────────────────────────
@@ -1142,10 +1215,11 @@ var NO_AUTH_TOOLS = ['wayback_archive_search', 'multi_source_search'];
function updateModeVisibility() {
var tool = document.getElementById('toolSelect').value;
var show = NO_AUTH_TOOLS.indexOf(tool) !== -1;
document.getElementById('modeSelect').style.display = show ? 'none' : '';
document.getElementById('dateFromInput').style.display = show ? '' : 'none';
document.getElementById('dateToInput').style.display = show ? '' : 'none';
var noAuth = NO_AUTH_TOOLS.indexOf(tool) !== -1;
var noDate = (tool === 'article_extractor');
document.getElementById('modeSelect').style.display = noAuth ? 'none' : '';
document.getElementById('dateFromInput').style.display = noDate ? 'none' : '';
document.getElementById('dateToInput').style.display = noDate ? 'none' : '';
}
// ── Status bar ────────────────────────────────────────────────────────────────
@@ -1235,7 +1309,7 @@ function buildBody(tool, query, count, mode) {
body.targetCommunityId = query;
}
if (tool === 'wayback_archive_search' || tool === 'multi_source_search') {
if (tool !== 'article_extractor') {
var from = document.getElementById('dateFromInput').value.trim();
var to = document.getElementById('dateToInput').value.trim();
if (tool === 'wayback_archive_search') {
@@ -2250,6 +2324,7 @@ async function archiveAll() {
if (!json.ok) throw new Error(json.error);
graphArchivedId = json.archiveId;
document.getElementById('btnArchiveAll').textContent = 'Update Archive';
autoCheckpoint(); // persist graphArchivedId into the case so progress survives a page loss
pollGraphArchive(json.archiveId);
} catch (e) {
archiveBar.innerHTML = '<span style="color:var(--danger)">Archive error: ' + esc(e.message) + '</span>';
@@ -2300,6 +2375,212 @@ function dumpJSON() {
document.body.removeChild(a);
URL.revokeObjectURL(url);
setStatus('Exported ' + allItems.length + ' item(s)');
autoCheckpoint(); // checkpoint so archiveId is preserved if the page is lost after export
}
// ── Cases (save / resume investigation sessions) ─────────────────────────────
var currentCaseId = null;
var currentCaseName = '';
function updateCaseIndicator() {
var ind = document.getElementById('caseIndicator');
var name = document.getElementById('caseIndicatorName');
var btn = document.getElementById('caseCheckpointBtn');
if (currentCaseId) {
name.textContent = currentCaseName;
ind.classList.add('visible');
if (btn && btn.textContent !== 'Saving…' && btn.textContent !== 'Saved!' && btn.textContent !== 'Failed') btn.textContent = 'Checkpoint';
} else {
ind.classList.remove('visible');
if (btn && btn.textContent !== 'Saving…' && btn.textContent !== 'Saved!' && btn.textContent !== 'Failed') btn.textContent = 'Save Case';
}
}
function _graphToolName() {
var sel = document.getElementById('toolSelect');
return 'Graph · ' + sel.options[sel.selectedIndex].text;
}
document.getElementById('caseCheckpointBtn').addEventListener('click', async function() {
var toolName = _graphToolName();
if (currentCaseId) {
// update existing case directly
var state = cmGCaptureState();
if (!state) return;
this.disabled = true; this.textContent = 'Saving…';
try {
var res = await fetch('/api/cases/' + currentCaseId, {
method: 'PUT', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ state: state, tool: toolName }),
});
var json = await res.json();
if (!json.ok) throw new Error(json.error);
this.textContent = 'Saved!';
setTimeout(function() { var b = document.getElementById('caseCheckpointBtn'); if (b) b.textContent = 'Checkpoint'; }, 1800);
} catch (e) {
this.textContent = 'Failed';
setTimeout(function() { var b = document.getElementById('caseCheckpointBtn'); if (b) b.textContent = 'Checkpoint'; }, 1800);
} finally {
this.disabled = false;
}
return;
}
// no active case → toggle save form
var state = cmGCaptureState();
var form = document.getElementById('caseSaveForm');
if (!state) {
this.textContent = 'No data yet';
setTimeout(function() { var b = document.getElementById('caseCheckpointBtn'); if (b) b.textContent = 'Save Case'; }, 1500);
return;
}
if (form.hidden) {
var dateStr = new Date().toLocaleDateString('en-CA');
document.getElementById('caseSaveName').value = toolName + ' · ' + dateStr;
form.hidden = false;
requestAnimationFrame(function() {
var inp = document.getElementById('caseSaveName');
inp.focus(); inp.select();
});
} else {
form.hidden = true;
}
});
document.getElementById('caseSaveName').addEventListener('keydown', function(e) {
if (e.key === 'Enter') { e.preventDefault(); document.getElementById('caseSaveConfirm').click(); }
});
document.getElementById('caseSaveCancel').addEventListener('click', function() {
document.getElementById('caseSaveForm').hidden = true;
});
document.getElementById('caseSaveConfirm').addEventListener('click', async function() {
var nameEl = document.getElementById('caseSaveName');
var name = nameEl.value.trim() || 'Unnamed Case';
var state = cmGCaptureState();
if (!state) return;
var toolName = _graphToolName();
this.disabled = true; this.textContent = 'Saving…';
try {
var res = await fetch('/api/cases', {
method: 'POST', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ name: name, page: 'graph', tool: toolName, state: state }),
});
var json = await res.json();
if (!json.ok) throw new Error(json.error);
currentCaseId = json.caseId;
currentCaseName = json.meta.name;
updateCaseIndicator();
document.getElementById('caseSaveForm').hidden = true;
nameEl.value = '';
} catch (e) {
// re-enable for retry
} finally {
this.disabled = false; this.textContent = 'Save';
}
});
// ── Auto-checkpoint (triggered by Archive All and Dump JSON) ─────────────────
async function autoCheckpoint() {
var state = cmGCaptureState();
if (!state) return;
var toolName = _graphToolName();
var dateStr = new Date().toLocaleDateString('en-CA');
try {
if (currentCaseId) {
await fetch('/api/cases/' + currentCaseId, {
method: 'PUT', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ state: state, tool: toolName }),
});
} else {
var res = await fetch('/api/cases', {
method: 'POST', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ name: 'Auto-save · ' + toolName + ' ' + dateStr, page: 'graph', tool: toolName, state: state }),
});
var json = await res.json();
if (json.ok) {
currentCaseId = json.caseId;
currentCaseName = json.meta.name;
updateCaseIndicator();
}
}
} catch (e) { /* silent */ }
}
// ── Periodic auto-save (every 60 s) — guards against power loss / crash ───────
setInterval(function() { autoCheckpoint(); }, 60000);
// ── beforeunload beacon — fires on tab/browser close ─────────────────────────
window.addEventListener('beforeunload', function() {
var state = cmGCaptureState();
if (!state) return;
var toolName = _graphToolName();
var dateStr = new Date().toLocaleDateString('en-CA');
var payload = currentCaseId
? { caseId: currentCaseId, state: state, tool: toolName }
: { name: 'Auto-save · ' + toolName + ' ' + dateStr, page: 'graph', tool: toolName, state: state };
navigator.sendBeacon('/api/cases/beacon', new Blob([JSON.stringify(payload)], { type: 'application/json' }));
});
function cmGCaptureState() {
if (!cy || !cy.nodes().length) return null;
return {
elements: cy.elements().jsons(),
toolbar: {
tool: document.getElementById('toolSelect').value,
mode: document.getElementById('modeSelect').value,
query: document.getElementById('queryInput').value.trim(),
count: document.getElementById('countInput').value,
dateFrom: document.getElementById('dateFromInput').value.trim(),
dateTo: document.getElementById('dateToInput').value.trim(),
},
graphArchivedId: graphArchivedId || null,
};
}
function cmGRestoreState(state, meta) {
if (!state || !meta) return;
clearGraph();
if (state.toolbar) {
var t = state.toolbar;
if (t.tool) { document.getElementById('toolSelect').value = t.tool; updatePlaceholder(); updateModeVisibility(); }
if (t.mode) document.getElementById('modeSelect').value = t.mode;
if (t.query) document.getElementById('queryInput').value = t.query;
if (t.count) document.getElementById('countInput').value = t.count;
if (t.dateFrom) document.getElementById('dateFromInput').value = t.dateFrom;
if (t.dateTo) document.getElementById('dateToInput').value = t.dateTo;
}
if (Array.isArray(state.elements) && state.elements.length) {
try {
cy.add(state.elements);
} catch (e) {
setStatus('Error restoring graph: ' + e.message, true);
return;
}
// Rebuild allItems from node data (type + raw) without double-storing
cy.nodes().forEach(function(node) {
var raw = node.data('raw');
var type = node.data('type');
if (raw && type) allItems.push({ type: type, item: raw });
});
cy.layout({ name: 'preset' }).run();
cy.fit(undefined, 60);
updateNodeCount();
}
currentCaseId = meta.id;
currentCaseName = meta.name;
updateCaseIndicator();
if (state.graphArchivedId) {
graphArchivedId = state.graphArchivedId;
document.getElementById('btnArchiveAll').textContent = 'Update Archive';
}
setStatus('Case "' + esc(meta.name) + '" restored — ' + cy.nodes().length + ' node(s)');
}
// ── Clear ─────────────────────────────────────────────────────────────────────
Script/SOCMINT-Twitter/templates/index.html
+343 -15
View File
@@ -717,6 +717,53 @@
}
.leaflet-bar a:hover { background: var(--accent-bg) !important; color: #818cf8 !important; }
.leaflet-control-attribution { background: rgba(26,29,39,0.8) !important; color: var(--muted) !important; font-size: 9px; }
/* Active-case indicator (sidebar) */
.case-indicator {
margin-top: 7px; padding: 6px 10px;
background: var(--accent-bg); border: 1px solid var(--accent); border-radius: 6px;
font-size: 11px; color: var(--text); gap: 8px; align-items: center; display: none;
}
.case-indicator.visible { display: flex; }
.case-indicator-name {
flex: 1; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
color: var(--accent); font-weight: 500; color: #fff;
}
.case-checkpoint-btn {
width: 100%; padding: 8px; margin-top: 7px;
background: transparent; color: var(--accent);
border: 1px solid var(--accent); border-radius: 6px;
font-family: var(--font); font-size: 12px; font-weight: 500;
cursor: pointer; transition: all 0.15s; color: #fff;
}
.case-checkpoint-btn:hover:not(:disabled) { background: var(--accent-bg); }
.case-checkpoint-btn:disabled { opacity: 0.5; cursor: wait; }
.case-save-form {
margin-top: 7px; padding: 10px;
background: var(--accent-bg); border: 1px solid var(--accent); border-radius: 6px;
}
.case-save-form[hidden] { display: none; }
.case-save-input {
width: 100%; padding: 6px 9px; margin-bottom: 8px;
background: var(--bg); border: 1px solid var(--border); border-radius: 5px;
color: var(--text); font-family: var(--font); font-size: 12px;
}
.case-save-input:focus { outline: none; border-color: var(--accent); }
.case-save-input::placeholder { color: var(--muted); }
.case-save-btns { display: flex; gap: 6px; }
.case-save-confirm {
flex: 1; padding: 5px 0; font-size: 12px; border-radius: 5px;
background: var(--accent); color: #fff; border: none;
font-family: var(--font); font-weight: 500; cursor: pointer; transition: opacity 0.15s;
}
.case-save-confirm:disabled { opacity: 0.45; cursor: wait; }
.case-save-confirm:not(:disabled):hover { opacity: 0.85; }
.case-save-cancel {
padding: 5px 10px; font-size: 12px; border-radius: 5px;
background: transparent; color: var(--muted); border: 1px solid var(--border);
font-family: var(--font); cursor: pointer; transition: all 0.12s;
}
.case-save-cancel:hover { color: var(--text); border-color: var(--muted); }
</style>
<script src="https://unpkg.com/leaflet@1.9.4/dist/leaflet.js" crossorigin="anonymous"></script>
<script src="{{ url_for('static', filename='js/card_constants.js') }}" nonce="{{ g.csp_nonce }}"></script>
@@ -734,6 +781,8 @@
<hr class="nav-divider">
<a href="/archives">Archives</a>
<a href="/analytics">Analytics</a>
<hr class="nav-divider">
<a href="/cases">Cases</a>
</nav>
</header>
@@ -791,6 +840,17 @@
<div id="dynamicFields"></div>
<button class="btn-run" id="runBtn">Run</button>
<button id="caseCheckpointBtn" class="case-checkpoint-btn">Save Case</button>
<div id="caseSaveForm" class="case-save-form" hidden>
<input id="caseSaveName" class="case-save-input" type="text" placeholder="Case name…" maxlength="120" autocomplete="off" spellcheck="false">
<div class="case-save-btns">
<button id="caseSaveConfirm" class="case-save-confirm">Save</button>
<button id="caseSaveCancel" class="case-save-cancel">Cancel</button>
</div>
</div>
<div id="caseIndicator" class="case-indicator">
<span class="case-indicator-name" id="caseIndicatorName"></span>
</div>
<div id="archiveBar" class="archive-bar" style="display:none"></div>
@@ -823,6 +883,7 @@
</div>
</div>
<script nonce="{{ g.csp_nonce }}">
// Hamburger nav
(function() {
@@ -886,6 +947,29 @@ let currentPayload = null; // last run payload (for archive queryInfo)
// on every new Run, same as graph.html's clearGraph() resets its own.
let archivedId = null;
// ── Card render virtual scroll ───────────────────────────────────────────────
const CARD_BATCH = 15;
let _cardItems = [];
let _cardOffset = 0;
let _cardObserver = null;
function _flushCardObserver() {
if (_cardObserver) { _cardObserver.disconnect(); _cardObserver = null; }
}
function _loadCardBatch() {
if (_cardOffset >= _cardItems.length) {
_flushCardObserver();
const s = resultBox.querySelector('#_crs');
if (s) s.remove();
return;
}
const batch = _cardItems.slice(_cardOffset, _cardOffset + CARD_BATCH);
_cardOffset += batch.length;
const grid = resultBox.querySelector('.cards-grid');
if (grid) grid.insertAdjacentHTML('beforeend', batch.map(buildCard).join(''));
}
// ── Scroll-triggered load-more ──────────────────────────────────────────────
// Tools/modes the backend actually paginates — everything else (plain xquik/API
// mode calls, article_extractor) just gets a single page.
@@ -988,28 +1072,35 @@ function renderFields() {
}
// Tool-specific input fields
const DATE_FIELDS = () => field('dateFrom', 'From date (YYYYMMDD, optional)', '20200101') +
field('dateTo', 'To date (YYYYMMDD, optional)', '20231231');
if (t === 'tweet_search_extractor') {
html += field('searchQuery', 'Search query', 'bitcoin');
html += DATE_FIELDS();
} else if (t === 'follower_explorer' || t === 'following_explorer') {
html += field('targetUsername', 'Username or User ID', 'elonmusk or 44196397');
html += DATE_FIELDS();
} else if (t === 'article_extractor') {
html += field('targetTweetId', 'Tweet ID', '1234567890');
} else if (t === 'community_post_extractor') {
html += field('targetCommunityId', 'Community ID', '1234567890');
html += DATE_FIELDS();
} else if (t === 'post_extractor') {
html += field('targetUsername', 'Username or User ID', 'elonmusk or 44196397');
html += DATE_FIELDS();
} else if (t === 'tweet_replies_extractor' || t === 'tweet_retweeters_extractor') {
html += field('targetTweetId', 'Tweet ID', '1234567890');
html += DATE_FIELDS();
} else if (t === 'geo_post_extractor') {
html += field('searchQuery', 'Search query', 'kopi jakarta');
html += DATE_FIELDS();
} else if (t === 'wayback_archive_search') {
html += field('searchQuery', 'Username or tweet URL', 'elonmusk or https://x.com/user/status/123');
html += field('waybackFrom', 'From date (YYYYMMDD, optional)', '20200101');
html += field('waybackTo', 'To date (YYYYMMDD, optional)', '20231231');
} else if (t === 'multi_source_search') {
html += field('searchQuery', 'Keyword, mention, username or URL', 'elonmusk');
html += field('dateFrom', 'From date (YYYYMMDD, optional)', '20200101');
html += field('dateTo', 'To date (YYYYMMDD, optional)', '20231231');
html += DATE_FIELDS();
}
// Count — for tools that support it, visible in cookie mode (or for no-auth tools)
@@ -1062,12 +1153,21 @@ document.getElementById('viewToggle').addEventListener('click', e => {
// ── Search / filter results ───────────────────────────────────────────────────
searchInput.addEventListener('input', () => renderCards(currentData, searchInput.value));
let _searchDebounce = null;
searchInput.addEventListener('input', () => {
clearTimeout(_searchDebounce);
_searchDebounce = setTimeout(() => renderCards(currentData, searchInput.value), 180);
});
const _ftCache = new WeakMap();
function flatText(obj) {
if (obj === null || obj === undefined) return '';
if (typeof obj !== 'object') return String(obj);
return Object.values(obj).map(flatText).join(' ');
let c = _ftCache.get(obj);
if (c !== undefined) return c;
c = Object.values(obj).map(flatText).join(' ');
_ftCache.set(obj, c);
return c;
}
// PRIORITY / DRILLABLE / SOURCE_CLASS / AGE_LABELS come from
@@ -1126,21 +1226,33 @@ function renderMedia(mediaList) {
}
function renderCards(data, query = '') {
_flushCardObserver();
if (!data) {
resultBox.innerHTML = '<div class="empty-state">No results yet.</div>';
return;
}
const items = Array.isArray(data) ? data : [data];
const q = query.toLowerCase().trim();
const filtered = q ? items.filter(i => flatText(i).toLowerCase().includes(q)) : items;
const items = Array.isArray(data) ? data : [data];
const q = query.toLowerCase().trim();
_cardItems = q ? items.filter(i => flatText(i).toLowerCase().includes(q)) : items;
_cardOffset = 0;
resultCount.textContent = filtered.length + (q ? ' found' : ' results');
resultCount.textContent = _cardItems.length + (q ? ' found' : ' results');
if (!filtered.length) {
if (!_cardItems.length) {
resultBox.innerHTML = '<div class="empty-state">No matching results.</div>';
return;
}
resultBox.innerHTML = '<div class="cards-grid">' + filtered.map(buildCard).join('') + '</div>';
resultBox.innerHTML = '<div class="cards-grid"></div><div id="_crs" style="height:4px"></div>';
_loadCardBatch();
const sentinel = resultBox.querySelector('#_crs');
if (sentinel) {
_cardObserver = new IntersectionObserver(entries => {
if (entries[0].isIntersecting) _loadCardBatch();
}, { rootMargin: '300px' });
_cardObserver.observe(sentinel);
}
}
// Tapping "Reply" on X auto-prefixes the compose box with every account the
@@ -1244,9 +1356,9 @@ function buildCard(item, nested = false) {
}
// ── Load more (scroll-triggered pagination) ─────────────────────────────────
// The sentinel lives as a SIBLING of #resultBox, not inside it — renderCards()
// replaces #resultBox.innerHTML wholesale on every run and every keystroke in
// the search box, which would destroy an in-box sentinel and its observer tie.
// loadMoreSentinel lives as a SIBLING of #resultBox (API-pagination sentinel).
// The card-render sentinel #_crs lives inside #resultBox and is intentionally
// recreated on each renderCards() call after disconnecting the old observer.
function appendCards(newItems, query = '') {
const q = query.toLowerCase().trim();
@@ -1595,8 +1707,12 @@ runBtn.addEventListener('click', async () => {
}
if (t === 'multi_source_search') {
payload.searchQuery = val('searchQuery');
payload.dateFrom = val('dateFrom');
payload.dateTo = val('dateTo');
}
// Date range — sent for all tools that expose the date fields (not wayback which uses its own keys, not article_extractor)
if (t !== 'wayback_archive_search' && t !== 'article_extractor') {
const df = val('dateFrom'); const dt = val('dateTo');
if (df) payload.dateFrom = df;
if (dt) payload.dateTo = dt;
}
currentPayload = payload;
@@ -1703,6 +1819,7 @@ async function archiveResults() {
archivedId = json.archiveId;
archiveBtn.textContent = 'Update Archive';
autoCheckpoint(); // persist archiveId into the case so the user can resume if the page is lost
pollArchive(json.archiveId);
} catch (e) {
archiveBar.innerHTML = `<span style="color:var(--danger)">Archive failed: ${esc(e.message)}</span>`;
@@ -1761,6 +1878,7 @@ downloadBtn.addEventListener('click', () => {
a.download = filename;
a.click();
URL.revokeObjectURL(url);
autoCheckpoint(); // checkpoint so the archiveId is preserved if the page is lost after export
});
function val(id) { const el = document.getElementById(id); return el ? el.value.trim() : ''; }
@@ -1933,6 +2051,216 @@ async function updateMap() {
else leafletMap.setView([20, 10], 2);
}
// ── Cases (save / resume investigation sessions) ─────────────────────────────
var currentCaseId = null;
var currentCaseName = '';
function updateCaseIndicator() {
const ind = document.getElementById('caseIndicator');
const name = document.getElementById('caseIndicatorName');
const btn = document.getElementById('caseCheckpointBtn');
if (currentCaseId) {
name.textContent = currentCaseName;
ind.classList.add('visible');
if (btn && btn.textContent !== 'Saving…' && btn.textContent !== 'Saved!' && btn.textContent !== 'Failed') btn.textContent = 'Checkpoint';
} else {
ind.classList.remove('visible');
if (btn && btn.textContent !== 'Saving…' && btn.textContent !== 'Saved!' && btn.textContent !== 'Failed') btn.textContent = 'Save Case';
}
}
document.getElementById('caseCheckpointBtn').addEventListener('click', async function() {
if (currentCaseId) {
// update existing case directly — no form needed
const state = cmCaptureState();
if (!state) return;
const tool = toolType.options[toolType.selectedIndex].text;
this.disabled = true; this.textContent = 'Saving…';
try {
const res = await fetch('/api/cases/' + currentCaseId, {
method: 'PUT', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ state, tool }),
});
const json = await res.json();
if (!json.ok) throw new Error(json.error);
this.textContent = 'Saved!';
setTimeout(() => { const b = document.getElementById('caseCheckpointBtn'); if (b) b.textContent = 'Checkpoint'; }, 1800);
} catch {
this.textContent = 'Failed';
setTimeout(() => { const b = document.getElementById('caseCheckpointBtn'); if (b) b.textContent = 'Checkpoint'; }, 1800);
} finally {
this.disabled = false;
}
return;
}
// no active case → toggle save form
const state = cmCaptureState();
const form = document.getElementById('caseSaveForm');
if (!state) {
// flash the button briefly if nothing to save
this.textContent = 'No data yet';
setTimeout(() => { const b = document.getElementById('caseCheckpointBtn'); if (b) b.textContent = 'Save Case'; }, 1500);
return;
}
if (form.hidden) {
const dateStr = new Date().toLocaleDateString('en-CA');
document.getElementById('caseSaveName').value = toolType.options[toolType.selectedIndex].text + ' · ' + dateStr;
form.hidden = false;
const inp = document.getElementById('caseSaveName');
inp.focus(); inp.select();
} else {
form.hidden = true;
}
});
document.getElementById('caseSaveCancel').addEventListener('click', function() {
document.getElementById('caseSaveForm').hidden = true;
});
document.getElementById('caseSaveConfirm').addEventListener('click', async function() {
const nameEl = document.getElementById('caseSaveName');
const name = nameEl.value.trim() || 'Unnamed Case';
const state = cmCaptureState();
if (!state) return;
const tool = toolType.options[toolType.selectedIndex].text;
this.disabled = true; this.textContent = 'Saving…';
try {
const res = await fetch('/api/cases', {
method: 'POST', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ name, page: 'index', tool, state }),
});
const json = await res.json();
if (!json.ok) throw new Error(json.error);
currentCaseId = json.caseId;
currentCaseName = json.meta.name;
updateCaseIndicator();
document.getElementById('caseSaveForm').hidden = true;
nameEl.value = '';
} catch {
// re-enable so user can retry
} finally {
this.disabled = false; this.textContent = 'Save';
}
});
function cmCaptureState() {
if (!currentData && !currentPayload) return null;
return {
payload: currentPayload || null,
data: Array.isArray(currentData) ? currentData : (currentData ? [currentData] : []),
nextCursor: nextCursor || null,
archiveId: archivedId || null,
};
}
function cmRestoreState(state, meta) {
if (!state || !meta) return;
currentCaseId = meta.id;
currentCaseName = meta.name;
updateCaseIndicator();
resetPagination();
currentData = null;
currentPayload = null;
if (state.payload) {
const p = state.payload;
if (p.toolType) toolType.value = p.toolType;
if (p.mode) currentMode = p.mode;
renderFields();
const fieldMap = {
searchQuery: p.searchQuery, targetUsername: p.targetUsername,
targetTweetId: p.targetTweetId, targetCommunityId: p.targetCommunityId,
waybackFrom: p.waybackFrom, waybackTo: p.waybackTo,
dateFrom: p.dateFrom, dateTo: p.dateTo,
};
Object.entries(fieldMap).forEach(([id, v]) => {
const el = document.getElementById(id);
if (el && v) el.value = v;
});
const cnt = document.getElementById('countInput');
if (cnt && p.count) cnt.value = p.count;
currentPayload = p;
}
if (Array.isArray(state.data) && state.data.length) {
currentData = state.data;
nextCursor = state.nextCursor || null;
statusBadge.textContent = 'restored';
statusBadge.className = 'badge ok';
searchInput.style.display = state.data.length > 1 ? '' : 'none';
downloadBtn.style.display = '';
archiveBtn.style.display = '';
renderCards(currentData);
resultCount.textContent = state.data.length + ' results';
}
if (state.archiveId) {
archivedId = state.archiveId;
archiveBtn.textContent = 'Update Archive';
}
}
// ── Auto-checkpoint (triggered by Archive and JSON dump) ─────────────────────
// Silently saves to the active case if one is loaded, or creates a new
// auto-save case if not. Never blocks the calling flow — errors are swallowed.
async function autoCheckpoint() {
const state = cmCaptureState();
if (!state) return;
const tool = toolType.options[toolType.selectedIndex].text;
const dateStr = new Date().toLocaleDateString('en-CA');
try {
if (currentCaseId) {
await fetch('/api/cases/' + currentCaseId, {
method: 'PUT', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ state, tool }),
});
} else {
const res = await fetch('/api/cases', {
method: 'POST', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ name: 'Auto-save · ' + tool + ' ' + dateStr, page: 'index', tool, state }),
});
const json = await res.json();
if (json.ok) {
currentCaseId = json.caseId;
currentCaseName = json.meta.name;
updateCaseIndicator();
}
}
} catch { /* silent — autosave must never interrupt the user's flow */ }
}
// ── Periodic auto-save (every 60 s) — guards against power loss / crash ───────
setInterval(function() { autoCheckpoint(); }, 60000);
// ── beforeunload beacon — fires on tab/browser close ─────────────────────────
window.addEventListener('beforeunload', function() {
const state = cmCaptureState();
if (!state) return;
const tool = toolType.options[toolType.selectedIndex].text;
const dateStr = new Date().toLocaleDateString('en-CA');
const payload = currentCaseId
? { caseId: currentCaseId, state, tool }
: { name: 'Auto-save · ' + tool + ' ' + dateStr, page: 'index', tool, state };
navigator.sendBeacon('/api/cases/beacon', new Blob([JSON.stringify(payload)], { type: 'application/json' }));
});
// Load case from URL param (e.g. when graph page redirects here with ?caseId=)
(async function() {
const caseId = new URLSearchParams(location.search).get('caseId');
if (!caseId) return;
history.replaceState(null, '', location.pathname);
try {
const res = await fetch('/api/cases/' + encodeURIComponent(caseId));
const json = await res.json();
if (!json.ok) throw new Error(json.error);
cmRestoreState(json.state, json.meta);
} catch (e) {
statusBadge.textContent = 'error';
statusBadge.className = 'badge err';
resultBox.innerHTML = `<div class="card"><div class="card-row"><div class="card-key">case load error</div><div class="card-val" style="color:#f87171">${esc(e.message)}</div></div></div>`;
}
})();
// ── Auto-run from URL params (drill-down new-tab entry point) ─────────────────
(function () {