mirror of
https://github.com/Jieyab89/OSINT-Cheat-sheet.git
synced 2026-08-17 18:35:41 +02:00
edit script socmint twitter
This commit is contained in:
Jieyab89
@@ -22,6 +22,18 @@ Xquik Dashboard
|
||||
|
||||
<img width="2556" height="1193" alt="image" src="https://github.com/user-attachments/assets/51e9d0f3-d079-44ce-9841-378a3e1ad7e4" />
|
||||
|
||||
Jieyab SOCMINT Twitter Dashboard
|
||||
Dasboard Home
|
||||
|
||||
<img width="2556" height="1261" alt="image" src="https://github.com/user-attachments/assets/dcca42be-1257-4e15-945b-41cd3913c857" />
|
||||
<img width="2546" height="1087" alt="image" src="https://github.com/user-attachments/assets/404ad67d-6b80-4448-8962-604548a3c56c" />
|
||||
|
||||
Archive
|
||||
|
||||
<img width="2558" height="1103" alt="image" src="https://github.com/user-attachments/assets/72053c57-f61b-4e7e-846d-a5d024d37e04" />
|
||||
|
||||
Graph
|
||||
|
||||
<img width="2557" height="1095" alt="image" src="https://github.com/user-attachments/assets/8046ce75-ebec-411b-91e3-786fbd0d4fe4" />
|
||||
|
||||
Dir Output
|
||||
|
||||
<img width="2312" height="790" alt="image" src="https://github.com/user-attachments/assets/334e4ab5-7ebb-46fa-82f3-be40f53d5094" />
|
||||
@@ -1,6 +1,11 @@
|
||||
import json
|
||||
import secrets
|
||||
import shutil
|
||||
import threading
|
||||
from flask import Flask, jsonify, render_template, request
|
||||
import requests as _req
|
||||
from flask import Flask, g, jsonify, render_template, request, Response, stream_with_context, send_from_directory
|
||||
|
||||
import archive as _archive
|
||||
from xquik_client import XquikClient, XquikError, load_config
|
||||
from cookie_client import (
|
||||
cookie_tweet_search,
|
||||
@@ -8,6 +13,9 @@ from cookie_client import (
|
||||
cookie_post_extractor,
|
||||
cookie_article_extractor,
|
||||
cookie_community_post_extractor,
|
||||
cookie_tweet_replies,
|
||||
cookie_tweet_retweeters,
|
||||
cookie_geo_search,
|
||||
CookieClientError,
|
||||
)
|
||||
|
||||
@@ -19,12 +27,102 @@ ACQUIRE_TIMEOUT = 15 # seconds to wait before returning 429
|
||||
|
||||
_sem = threading.Semaphore(MAX_CONCURRENT_REQUESTS)
|
||||
|
||||
# ── Cookie & session security ─────────────────────────────────────────────────
|
||||
|
||||
_https = config.getboolean("server", "https", fallback=False)
|
||||
|
||||
app.config.update(
|
||||
SECRET_KEY = config.get("server", "secret_key", fallback=secrets.token_hex(32)),
|
||||
SESSION_COOKIE_HTTPONLY = True,
|
||||
SESSION_COOKIE_SAMESITE = "Strict",
|
||||
SESSION_COOKIE_SECURE = _https, # True only when TLS is terminated at Flask
|
||||
)
|
||||
|
||||
# ── Security headers ──────────────────────────────────────────────────────────
|
||||
|
||||
@app.before_request
|
||||
def _make_nonce():
|
||||
g.csp_nonce = secrets.token_urlsafe(16)
|
||||
|
||||
|
||||
@app.after_request
|
||||
def _set_security_headers(response):
|
||||
nonce = getattr(g, "csp_nonce", "")
|
||||
csp = (
|
||||
"default-src 'none'; "
|
||||
f"script-src 'nonce-{nonce}' https://unpkg.com; "
|
||||
"style-src 'unsafe-inline' https://unpkg.com; "
|
||||
"img-src 'self' data: blob: "
|
||||
"https://*.twimg.com "
|
||||
"https://*.tile.openstreetmap.org "
|
||||
"https://server.arcgisonline.com "
|
||||
"https://*.tile.opentopomap.org "
|
||||
"https://unpkg.com; "
|
||||
"media-src 'self'; "
|
||||
"connect-src 'self' https://nominatim.openstreetmap.org; "
|
||||
"font-src 'none'; "
|
||||
"frame-src 'none'; "
|
||||
"object-src 'none'; "
|
||||
"base-uri 'self'; "
|
||||
"form-action 'self';"
|
||||
)
|
||||
response.headers["Content-Security-Policy"] = csp
|
||||
response.headers["X-Content-Type-Options"] = "nosniff"
|
||||
response.headers["X-Frame-Options"] = "DENY"
|
||||
response.headers["Referrer-Policy"] = "no-referrer"
|
||||
response.headers["Permissions-Policy"] = "geolocation=(), camera=(), microphone=()"
|
||||
response.headers["X-XSS-Protection"] = "0" # disable legacy broken auditor
|
||||
|
||||
# Harden every Set-Cookie header regardless of where it originates
|
||||
raw_cookies = response.headers.getlist("Set-Cookie")
|
||||
if raw_cookies:
|
||||
response.headers.remove("Set-Cookie")
|
||||
for raw in raw_cookies:
|
||||
parts = [p.strip() for p in raw.split(";")]
|
||||
flags = {p.split("=")[0].strip().lower() for p in parts[1:]}
|
||||
if "httponly" not in flags:
|
||||
parts.append("HttpOnly")
|
||||
if "samesite" not in flags:
|
||||
parts.append("SameSite=Strict")
|
||||
if "secure" not in flags and _https:
|
||||
parts.append("Secure")
|
||||
response.headers.add("Set-Cookie", "; ".join(parts))
|
||||
|
||||
return response
|
||||
|
||||
|
||||
@app.route("/")
|
||||
def index():
|
||||
return render_template("index.html")
|
||||
|
||||
|
||||
# Whitelist: only proxy Twitter's video CDN to prevent SSRF
|
||||
_VIDEO_CDN = ("https://video.twimg.com/",)
|
||||
|
||||
@app.route("/api/video")
|
||||
def video_proxy():
|
||||
url = request.args.get("url", "").strip()
|
||||
if not any(url.startswith(prefix) for prefix in _VIDEO_CDN):
|
||||
return jsonify({"ok": False, "error": "URL not allowed"}), 403
|
||||
try:
|
||||
upstream = _req.get(
|
||||
url,
|
||||
stream=True,
|
||||
timeout=20,
|
||||
headers={"Referer": "https://x.com/", "User-Agent": "Mozilla/5.0"},
|
||||
)
|
||||
headers = {"Content-Type": upstream.headers.get("Content-Type", "video/mp4")}
|
||||
if "Content-Length" in upstream.headers:
|
||||
headers["Content-Length"] = upstream.headers["Content-Length"]
|
||||
return Response(
|
||||
stream_with_context(upstream.iter_content(chunk_size=32768)),
|
||||
status=upstream.status_code,
|
||||
headers=headers,
|
||||
)
|
||||
except _req.RequestException as e:
|
||||
return jsonify({"ok": False, "error": str(e)}), 502
|
||||
|
||||
|
||||
@app.route("/api/run", methods=["POST"])
|
||||
def run_tool():
|
||||
body = request.get_json(silent=True) or {}
|
||||
@@ -74,6 +172,24 @@ def run_tool():
|
||||
else:
|
||||
data = XquikClient(config).post_extractor(username)
|
||||
|
||||
elif tool_type == "tweet_replies_extractor":
|
||||
tweet_id = body.get("targetTweetId", "")
|
||||
if mode != "cookie":
|
||||
return jsonify({"ok": False, "error": "tweet_replies_extractor requires cookie mode"}), 400
|
||||
data = cookie_tweet_replies(tweet_id, count=count, config=config)
|
||||
|
||||
elif tool_type == "tweet_retweeters_extractor":
|
||||
tweet_id = body.get("targetTweetId", "")
|
||||
if mode != "cookie":
|
||||
return jsonify({"ok": False, "error": "tweet_retweeters_extractor requires cookie mode"}), 400
|
||||
data = cookie_tweet_retweeters(tweet_id, count=count, config=config)
|
||||
|
||||
elif tool_type == "geo_post_extractor":
|
||||
keyword = body.get("searchQuery", "")
|
||||
if mode != "cookie":
|
||||
return jsonify({"ok": False, "error": "geo_post_extractor requires cookie mode"}), 400
|
||||
data = cookie_geo_search(keyword, count=count, config=config)
|
||||
|
||||
else:
|
||||
return jsonify({"ok": False, "error": f"Unknown toolType: {tool_type}"}), 400
|
||||
|
||||
@@ -87,6 +203,72 @@ def run_tool():
|
||||
_sem.release()
|
||||
|
||||
|
||||
# ── Archive routes ────────────────────────────────────────────────────────────
|
||||
|
||||
@app.route("/graph")
|
||||
def graph_viewer():
|
||||
return render_template("graph.html")
|
||||
|
||||
|
||||
@app.route("/archives")
|
||||
def archive_viewer():
|
||||
return render_template("archive.html")
|
||||
|
||||
|
||||
@app.route("/api/archive/<archive_id>/results")
|
||||
def archive_results(archive_id):
|
||||
base = _archive.ARCHIVE_ROOT / archive_id
|
||||
results_file = base / "results.json"
|
||||
meta_file = base / "meta.json"
|
||||
if not results_file.exists():
|
||||
return jsonify({"ok": False, "error": "Archive not found"}), 404
|
||||
return jsonify({
|
||||
"ok": True,
|
||||
"results": json.loads(results_file.read_text()),
|
||||
"meta": json.loads(meta_file.read_text()) if meta_file.exists() else {},
|
||||
})
|
||||
|
||||
|
||||
@app.route("/api/archive/<archive_id>/media/<path:filename>")
|
||||
def archive_media(archive_id, filename):
|
||||
media_dir = _archive.ARCHIVE_ROOT / archive_id / "media"
|
||||
return send_from_directory(str(media_dir), filename)
|
||||
|
||||
|
||||
@app.route("/api/archive/<archive_id>", methods=["DELETE"])
|
||||
def archive_delete(archive_id):
|
||||
archive_dir = _archive.ARCHIVE_ROOT / archive_id
|
||||
if not archive_dir.exists():
|
||||
return jsonify({"ok": False, "error": "Archive not found"}), 404
|
||||
shutil.rmtree(archive_dir)
|
||||
return jsonify({"ok": True})
|
||||
|
||||
|
||||
@app.route("/api/archive", methods=["POST"])
|
||||
def archive_start():
|
||||
body = request.get_json(silent=True) or {}
|
||||
tool_type = body.get("toolType", "unknown")
|
||||
data = body.get("data")
|
||||
query_info = body.get("queryInfo", {})
|
||||
if not data:
|
||||
return jsonify({"ok": False, "error": "No data provided"}), 400
|
||||
archive_id = _archive.start(tool_type, data, query_info)
|
||||
return jsonify({"ok": True, "archiveId": archive_id})
|
||||
|
||||
|
||||
@app.route("/api/archive/<archive_id>/status")
|
||||
def archive_status(archive_id):
|
||||
s = _archive.status(archive_id)
|
||||
if s is None:
|
||||
return jsonify({"ok": False, "error": "Archive not found"}), 404
|
||||
return jsonify({"ok": True, **s})
|
||||
|
||||
|
||||
@app.route("/api/archive/list")
|
||||
def archive_list():
|
||||
return jsonify({"ok": True, "archives": _archive.list_all()})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
host = config.get("server", "host", fallback="127.0.0.1")
|
||||
port = config.getint("server", "port", fallback=5000)
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
ARCHIVE_ROOT = Path(__file__).parent / "archives"
|
||||
DELAY_MIN = 1.5 # seconds between media downloads (anti-ban)
|
||||
DELAY_MAX = 3.5
|
||||
REQUEST_TIMEOUT = 25
|
||||
|
||||
_registry: dict[str, dict] = {} # archive_id → status dict
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
# ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
def _make_id(tool_type: str) -> str:
|
||||
return f"{tool_type}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
|
||||
|
||||
|
||||
def _tweet_url(item: dict) -> str | None:
|
||||
tid = str(item.get("id", "")).strip()
|
||||
user = str(item.get("user", "")).strip()
|
||||
if tid and user:
|
||||
return f"https://x.com/{user}/status/{tid}"
|
||||
return None
|
||||
|
||||
|
||||
def _media_ext(url: str, mtype: str) -> str:
|
||||
if mtype in ("video", "animated_gif"):
|
||||
return "mp4"
|
||||
for ext in (".jpg", ".jpeg", ".png", ".webp", ".gif"):
|
||||
if ext in url.lower():
|
||||
return ext.lstrip(".")
|
||||
return "jpg"
|
||||
|
||||
|
||||
def _download_file(url: str, dest: Path) -> bool:
|
||||
"""Download a single file. Returns True on success."""
|
||||
try:
|
||||
r = requests.get(
|
||||
url,
|
||||
timeout=REQUEST_TIMEOUT,
|
||||
headers={"Referer": "https://x.com/", "User-Agent": "Mozilla/5.0"},
|
||||
stream=True,
|
||||
)
|
||||
r.raise_for_status()
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(dest, "wb") as f:
|
||||
for chunk in r.iter_content(chunk_size=65536):
|
||||
f.write(chunk)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _pick_fields(item: dict) -> dict:
|
||||
"""Keep only the fields we want to archive."""
|
||||
keys = ["id", "text", "full_text", "article_text", "user", "user_id",
|
||||
"created_at", "reply_count", "retweet_count", "favorite_count",
|
||||
"view_count", "in_reply_to_tweet_id", "name", "screen_name",
|
||||
"description", "followers_count", "following_count",
|
||||
"lat", "lon", "place",
|
||||
"retweeted_text", "retweeted_by_user", "retweeted_by_name",
|
||||
"retweeted_by_bio", "retweeted_at", "retweeted_tweet_id"]
|
||||
return {k: item[k] for k in keys if k in item and item[k] is not None}
|
||||
|
||||
|
||||
# ── Core archive runner (runs in background thread) ───────────────────────────
|
||||
|
||||
def _run(archive_id: str, tool_type: str, data, query_info: dict) -> None:
|
||||
archive_dir = ARCHIVE_ROOT / archive_id
|
||||
media_dir = archive_dir / "media"
|
||||
archive_dir.mkdir(parents=True, exist_ok=True)
|
||||
media_dir.mkdir(exist_ok=True)
|
||||
|
||||
items = data if isinstance(data, list) else [data]
|
||||
|
||||
# Build enriched records + collect media download queue
|
||||
enriched = []
|
||||
media_queue = [] # list of (url, dest_path)
|
||||
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
enriched.append(item)
|
||||
continue
|
||||
|
||||
record = _pick_fields(item)
|
||||
record["tweet_url"] = _tweet_url(item)
|
||||
local_media = []
|
||||
|
||||
for midx, m in enumerate(item.get("media", [])):
|
||||
url = m.get("url") or m.get("thumb", "")
|
||||
if not url:
|
||||
continue
|
||||
mtype = m.get("type", "photo")
|
||||
ext = _media_ext(url, mtype)
|
||||
fname = f"{item.get('id', 'unknown')}_{mtype}_{midx}.{ext}"
|
||||
dest = media_dir / fname
|
||||
local_media.append(f"media/{fname}")
|
||||
media_queue.append((url, dest))
|
||||
|
||||
if local_media:
|
||||
record["archived_media"] = local_media
|
||||
enriched.append(record)
|
||||
|
||||
# Write metadata + results immediately (no waiting on media)
|
||||
meta = {
|
||||
"tool": tool_type,
|
||||
"query": query_info,
|
||||
"archived_at": datetime.now().isoformat(timespec="seconds"),
|
||||
"total_items": len(items),
|
||||
"media_count": len(media_queue),
|
||||
}
|
||||
(archive_dir / "meta.json").write_text(
|
||||
json.dumps(meta, indent=2, ensure_ascii=False)
|
||||
)
|
||||
(archive_dir / "results.json").write_text(
|
||||
json.dumps(enriched, indent=2, ensure_ascii=False)
|
||||
)
|
||||
|
||||
with _lock:
|
||||
_registry[archive_id].update({"status": "downloading", "total": len(media_queue)})
|
||||
|
||||
# Download media one-by-one with anti-ban delays
|
||||
for i, (url, dest) in enumerate(media_queue):
|
||||
if i > 0:
|
||||
time.sleep(random.uniform(DELAY_MIN, DELAY_MAX))
|
||||
_download_file(url, dest)
|
||||
with _lock:
|
||||
_registry[archive_id]["progress"] = i + 1
|
||||
|
||||
with _lock:
|
||||
_registry[archive_id].update({"status": "done", "path": str(archive_dir)})
|
||||
|
||||
|
||||
# ── Public API ────────────────────────────────────────────────────────────────
|
||||
|
||||
def start(tool_type: str, data, query_info: dict) -> str:
|
||||
"""Kick off archiving in a background thread. Returns archive_id."""
|
||||
archive_id = _make_id(tool_type)
|
||||
with _lock:
|
||||
_registry[archive_id] = {"status": "saving", "progress": 0, "total": 0, "path": None}
|
||||
|
||||
t = threading.Thread(
|
||||
target=_run,
|
||||
args=(archive_id, tool_type, data, query_info),
|
||||
daemon=True,
|
||||
)
|
||||
t.start()
|
||||
return archive_id
|
||||
|
||||
|
||||
def status(archive_id: str) -> dict | None:
|
||||
with _lock:
|
||||
entry = _registry.get(archive_id)
|
||||
return dict(entry) if entry else None
|
||||
|
||||
|
||||
def list_all() -> list[dict]:
|
||||
"""Read meta.json from every archive folder, newest first."""
|
||||
if not ARCHIVE_ROOT.exists():
|
||||
return []
|
||||
results = []
|
||||
for d in sorted(ARCHIVE_ROOT.iterdir(), key=lambda p: p.name, reverse=True):
|
||||
meta_file = d / "meta.json"
|
||||
if not meta_file.exists():
|
||||
continue
|
||||
try:
|
||||
meta = json.loads(meta_file.read_text())
|
||||
meta["id"] = d.name
|
||||
results.append(meta)
|
||||
except Exception:
|
||||
pass
|
||||
return results
|
||||
@@ -1,10 +1,8 @@
|
||||
[xquik]
|
||||
|
||||
api_key = xxxxxxxxxxxxxxxxxxxxxxxxxxx
|
||||
base_url = https://xquik.com/api/v1/extractions
|
||||
|
||||
[twitter_cookies]
|
||||
|
||||
auth_token = xxxxxxxxxxxxxxxxxx
|
||||
ct0 = xxxxxxxxxxxx
|
||||
|
||||
|
||||
@@ -1,19 +1,3 @@
|
||||
"""
|
||||
cookie_client.py
|
||||
Cookie-based wrapper for all five xquik tools, hitting X.com directly via
|
||||
twikit/twifork — no xquik API quota consumed.
|
||||
|
||||
Install: pip install twifork
|
||||
(NOT the upstream `twikit` package — it is broken since X changed ondemand.s.js.
|
||||
twifork is a drop-in replacement; imports remain `from twikit import Client`.)
|
||||
|
||||
How to obtain cookies:
|
||||
1. Log in to x.com in your browser.
|
||||
2. Open DevTools > Application > Cookies > https://x.com
|
||||
3. Copy the values of `auth_token` and `ct0`.
|
||||
4. Paste them into config.ini under [twitter_cookies].
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import configparser
|
||||
import os
|
||||
@@ -82,16 +66,25 @@ def _extract_media(t: object) -> list:
|
||||
return result
|
||||
|
||||
|
||||
def _id_str(val) -> str | None:
|
||||
"""Return ID as string, or None. Prevents 64-bit integer precision loss in JSON/JS."""
|
||||
return str(val) if val is not None else None
|
||||
|
||||
|
||||
def _tweet_to_dict(t: object) -> dict:
|
||||
user_obj = getattr(t, "user", None)
|
||||
d = {
|
||||
"id": getattr(t, "id", None),
|
||||
"id": _id_str(getattr(t, "id", None)),
|
||||
"created_at": getattr(t, "created_at", None),
|
||||
"text": getattr(t, "text", None),
|
||||
"user": getattr(t.user, "screen_name", None) if getattr(t, "user", None) else None,
|
||||
"reply_count": getattr(t, "reply_count", None),
|
||||
"retweet_count": getattr(t, "retweet_count", None),
|
||||
"favorite_count": getattr(t, "favorite_count", None),
|
||||
"view_count": getattr(t, "view_count", None),
|
||||
"user": getattr(user_obj, "screen_name", None) if user_obj else None,
|
||||
"user_id": _id_str(getattr(user_obj, "id", None)) if user_obj else None,
|
||||
"user_location": getattr(user_obj, "location", None) if user_obj else None,
|
||||
"reply_count": getattr(t, "reply_count", None),
|
||||
"retweet_count": getattr(t, "retweet_count", None),
|
||||
"favorite_count": getattr(t, "favorite_count", None),
|
||||
"view_count": getattr(t, "view_count", None),
|
||||
"in_reply_to_tweet_id": getattr(t, "in_reply_to", None), # returns id_str directly
|
||||
}
|
||||
media = _extract_media(t)
|
||||
if media:
|
||||
@@ -101,7 +94,7 @@ def _tweet_to_dict(t: object) -> dict:
|
||||
|
||||
def _user_to_dict(u: object) -> dict:
|
||||
return {
|
||||
"id": getattr(u, "id", None),
|
||||
"id": _id_str(getattr(u, "id", None)),
|
||||
"name": getattr(u, "name", None),
|
||||
"screen_name": getattr(u, "screen_name", None),
|
||||
"description": getattr(u, "description", None),
|
||||
@@ -116,6 +109,14 @@ def _user_to_dict(u: object) -> dict:
|
||||
|
||||
# ── Async implementations ─────────────────────────────────────────────────────
|
||||
|
||||
async def _resolve_user(client, identifier: str):
|
||||
"""Accept either a screen_name or a numeric user ID string."""
|
||||
clean = identifier.lstrip("@").strip()
|
||||
if clean.isdigit():
|
||||
return await client.get_user_by_id(clean)
|
||||
return await client.get_user_by_screen_name(clean)
|
||||
|
||||
|
||||
async def _tweet_search_async(query: str, auth_token: str, ct0: str, count: int) -> list:
|
||||
client = await _make_client(auth_token, ct0)
|
||||
results = await client.search_tweet(query, "Latest", count=count)
|
||||
@@ -124,14 +125,14 @@ async def _tweet_search_async(query: str, auth_token: str, ct0: str, count: int)
|
||||
|
||||
async def _follower_explorer_async(username: str, auth_token: str, ct0: str, count: int) -> list:
|
||||
client = await _make_client(auth_token, ct0)
|
||||
user = await client.get_user_by_screen_name(username)
|
||||
user = await _resolve_user(client, username)
|
||||
followers = await user.get_followers(count=count)
|
||||
return [_user_to_dict(u) for u in followers]
|
||||
|
||||
|
||||
async def _post_extractor_async(username: str, auth_token: str, ct0: str, count: int) -> list:
|
||||
client = await _make_client(auth_token, ct0)
|
||||
user = await client.get_user_by_screen_name(username)
|
||||
user = await _resolve_user(client, username)
|
||||
tweets = await user.get_tweets("Tweets", count=count)
|
||||
return [_tweet_to_dict(t) for t in tweets]
|
||||
|
||||
@@ -155,6 +156,80 @@ async def _community_posts_async(community_id: str, auth_token: str, ct0: str, c
|
||||
return [_tweet_to_dict(t) for t in posts]
|
||||
|
||||
|
||||
async def _tweet_replies_async(tweet_id: str, auth_token: str, ct0: str, count: int) -> list:
|
||||
client = await _make_client(auth_token, ct0)
|
||||
|
||||
# Walk up the in_reply_to chain to find the conversation root.
|
||||
# tweet.in_reply_to → _legacy['in_reply_to_status_id_str'] (parent tweet ID string).
|
||||
# There is no conversation_id attribute on twikit Tweet objects — the only way
|
||||
# to reach the root is to follow the chain until in_reply_to is None.
|
||||
conversation_id = tweet_id
|
||||
current_id = tweet_id
|
||||
|
||||
for _ in range(6): # guard: max 6 hops up the thread
|
||||
try:
|
||||
node = await client.get_tweet_by_id(current_id)
|
||||
parent_id = getattr(node, "in_reply_to", None)
|
||||
if not parent_id:
|
||||
conversation_id = current_id # reached root
|
||||
break
|
||||
current_id = str(parent_id)
|
||||
conversation_id = current_id
|
||||
except Exception:
|
||||
break
|
||||
|
||||
# Fetch all tweets in the conversation thread
|
||||
results = await client.search_tweet(
|
||||
f"conversation_id:{conversation_id}", "Latest", count=count
|
||||
)
|
||||
|
||||
tweets = [_tweet_to_dict(t) for t in results]
|
||||
|
||||
# If the user clicked on a non-root reply, filter to that reply's direct children
|
||||
if conversation_id != tweet_id:
|
||||
direct = [d for d in tweets if d.get("in_reply_to_tweet_id") == tweet_id]
|
||||
return direct if direct else tweets # fallback: full thread
|
||||
|
||||
return tweets
|
||||
|
||||
|
||||
async def _tweet_retweeters_async(tweet_id: str, auth_token: str, ct0: str, count: int) -> list:
|
||||
client = await _make_client(auth_token, ct0)
|
||||
|
||||
# Fetch the original tweet once so every retweeter card shows what was retweeted
|
||||
rt_info: dict = {}
|
||||
try:
|
||||
orig = await client.get_tweet_by_id(tweet_id)
|
||||
orig_user = getattr(orig, "user", None)
|
||||
rt_info = {
|
||||
"retweeted_text": getattr(orig, "text", None),
|
||||
"retweeted_by_user": getattr(orig_user, "screen_name", None) if orig_user else None,
|
||||
"retweeted_by_name": getattr(orig_user, "name", None) if orig_user else None,
|
||||
"retweeted_by_bio": getattr(orig_user, "description", None) if orig_user else None,
|
||||
"retweeted_at": getattr(orig, "created_at", None),
|
||||
"retweeted_tweet_id": _id_str(getattr(orig, "id", None)),
|
||||
}
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
retweeters = await client.get_retweeters(tweet_id, count=count)
|
||||
|
||||
result = []
|
||||
for u in retweeters:
|
||||
# retweeted content first → shows prominently in the card
|
||||
d = {**rt_info, **_user_to_dict(u)}
|
||||
result.append(d)
|
||||
return result
|
||||
|
||||
|
||||
async def _geo_search_async(keyword: str, auth_token: str, ct0: str, count: int) -> list:
|
||||
"""Keyword search; user_location (profile location string) is included in every
|
||||
result so the frontend can geocode and plot it on a map."""
|
||||
client = await _make_client(auth_token, ct0)
|
||||
results = await client.search_tweet(keyword, "Latest", count=count)
|
||||
return [_tweet_to_dict(t) for t in results]
|
||||
|
||||
|
||||
# ── Public sync wrappers ──────────────────────────────────────────────────────
|
||||
|
||||
def cookie_tweet_search(query: str, count: int = 20, config: configparser.ConfigParser = None) -> list:
|
||||
@@ -189,6 +264,26 @@ def cookie_community_post_extractor(
|
||||
return asyncio.run(_community_posts_async(community_id, auth, ct0, count))
|
||||
|
||||
|
||||
def cookie_tweet_replies(tweet_id: str, count: int = 50, config: configparser.ConfigParser = None) -> list:
|
||||
cfg = config or load_config()
|
||||
auth, ct0 = _get_creds(cfg)
|
||||
return asyncio.run(_tweet_replies_async(tweet_id, auth, ct0, count))
|
||||
|
||||
|
||||
def cookie_tweet_retweeters(tweet_id: str, count: int = 50, config: configparser.ConfigParser = None) -> list:
|
||||
cfg = config or load_config()
|
||||
auth, ct0 = _get_creds(cfg)
|
||||
return asyncio.run(_tweet_retweeters_async(tweet_id, auth, ct0, count))
|
||||
|
||||
|
||||
def cookie_geo_search(
|
||||
keyword: str, count: int = 20, config: configparser.ConfigParser = None,
|
||||
) -> list:
|
||||
cfg = config or load_config()
|
||||
auth, ct0 = _get_creds(cfg)
|
||||
return asyncio.run(_geo_search_async(keyword, auth, ct0, count))
|
||||
|
||||
|
||||
# Legacy alias — kept for any external scripts that import this name directly
|
||||
fetch_user_timeline = cookie_post_extractor
|
||||
|
||||
|
||||
@@ -0,0 +1,550 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<!-- Link Bootstrap Icons CDN -->
|
||||
<link rel="stylesheet" href="https://jsdelivr.net">
|
||||
<title>Jieyab89 SOCMINT X — Archives</title>
|
||||
<style>
|
||||
:root {
|
||||
--bg: #0f1117;
|
||||
--surface: #1a1d27;
|
||||
--surface2: #21253a;
|
||||
--border: #2a2d3a;
|
||||
--text: #e8eaf0;
|
||||
--muted: #8890a4;
|
||||
--accent: #5865f2;
|
||||
--accent-bg: #1e2240;
|
||||
--success: #22c55e;
|
||||
--danger: #ef4444;
|
||||
--font: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||||
}
|
||||
* { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
body { background: var(--bg); color: var(--text); font-family: var(--font); font-size: 14px; line-height: 1.6; min-height: 100vh; }
|
||||
|
||||
/* ── Header ── */
|
||||
header {
|
||||
padding: 14px 24px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
background: var(--surface);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
position: relative;
|
||||
}
|
||||
header h1 { font-size: 14px; font-weight: 600; }
|
||||
header .sep { color: var(--border); }
|
||||
header .sub { color: var(--muted); font-size: 12px; flex: 1; }
|
||||
.hamburger-btn {
|
||||
margin-left: auto;
|
||||
background: none;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 5px;
|
||||
color: var(--muted);
|
||||
cursor: pointer;
|
||||
padding: 5px 9px;
|
||||
font-size: 16px;
|
||||
line-height: 1;
|
||||
transition: all 0.15s;
|
||||
}
|
||||
.hamburger-btn:hover { color: var(--text); border-color: var(--accent); }
|
||||
.nav-menu {
|
||||
position: absolute;
|
||||
top: 100%;
|
||||
right: 12px;
|
||||
margin-top: 6px;
|
||||
background: var(--surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 7px;
|
||||
padding: 5px;
|
||||
min-width: 175px;
|
||||
box-shadow: 0 8px 28px rgba(0,0,0,0.5);
|
||||
z-index: 200;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 1px;
|
||||
}
|
||||
.nav-menu.hidden { display: none; }
|
||||
.nav-menu a {
|
||||
padding: 7px 12px;
|
||||
border-radius: 5px;
|
||||
color: var(--muted);
|
||||
text-decoration: none;
|
||||
font-size: 13px;
|
||||
transition: all 0.12s;
|
||||
display: block;
|
||||
}
|
||||
.nav-menu a:hover { color: var(--text); background: var(--accent-bg); }
|
||||
.nav-menu a.current { color: var(--accent); background: var(--accent-bg); }
|
||||
.nav-divider { border: none; border-top: 1px solid var(--border); margin: 3px 0; }
|
||||
|
||||
/* ── Layout ── */
|
||||
.layout { display: grid; grid-template-columns: 270px 1fr; min-height: calc(100vh - 49px); }
|
||||
@media (max-width: 760px) { .layout { grid-template-columns: 1fr; } }
|
||||
|
||||
/* ── Sidebar ── */
|
||||
.sidebar {
|
||||
border-right: 1px solid var(--border);
|
||||
background: var(--surface);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
.sidebar-head {
|
||||
padding: 14px 16px 10px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
}
|
||||
.sidebar-title { font-size: 11px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.06em; color: var(--muted); }
|
||||
.sidebar-count { font-size: 11px; color: var(--muted); background: var(--bg); padding: 2px 7px; border-radius: 10px; }
|
||||
.archive-list { overflow-y: auto; flex: 1; padding: 8px; }
|
||||
.archive-entry {
|
||||
padding: 10px 12px;
|
||||
border-radius: 7px;
|
||||
border: 1px solid transparent;
|
||||
cursor: pointer;
|
||||
transition: all 0.12s;
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
.archive-entry:hover { background: var(--surface2); border-color: var(--border); }
|
||||
.archive-entry.active { background: var(--accent-bg); border-color: var(--accent); }
|
||||
.ae-tool {
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
color: var(--text);
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
.ae-tool .tool-pill {
|
||||
display: inline-block;
|
||||
font-size: 10px;
|
||||
padding: 1px 6px;
|
||||
border-radius: 3px;
|
||||
background: var(--accent-bg);
|
||||
color: #818cf8;
|
||||
margin-right: 4px;
|
||||
font-weight: 500;
|
||||
}
|
||||
.ae-meta { font-size: 11px; color: var(--muted); margin-top: 2px; }
|
||||
.ae-stats { font-size: 11px; color: var(--muted); display: flex; gap: 8px; margin-top: 3px; }
|
||||
.ae-stat { display: flex; align-items: center; gap: 3px; }
|
||||
.no-archives {
|
||||
padding: 40px 16px;
|
||||
text-align: center;
|
||||
color: var(--muted);
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
/* ── Viewer ── */
|
||||
.viewer { display: flex; flex-direction: column; overflow: hidden; }
|
||||
.viewer-top {
|
||||
padding: 14px 18px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
background: var(--surface);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
.viewer-meta { flex: 1; }
|
||||
.viewer-meta-title { font-size: 13px; font-weight: 600; }
|
||||
.viewer-meta-sub { font-size: 11px; color: var(--muted); margin-top: 1px; }
|
||||
.search-bar {
|
||||
padding: 6px 10px;
|
||||
background: var(--bg);
|
||||
border: 1px solid var(--border);
|
||||
color: var(--text);
|
||||
font-family: var(--font);
|
||||
font-size: 13px;
|
||||
border-radius: 6px;
|
||||
width: 220px;
|
||||
transition: border-color 0.15s;
|
||||
}
|
||||
.search-bar:focus { outline: none; border-color: var(--accent); }
|
||||
.search-bar::placeholder { color: var(--muted); }
|
||||
.result-count { font-size: 12px; color: var(--muted); white-space: nowrap; }
|
||||
.btn-delete {
|
||||
padding: 5px 12px;
|
||||
background: transparent;
|
||||
color: var(--muted);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 6px;
|
||||
font-family: var(--font);
|
||||
font-size: 12px;
|
||||
cursor: pointer;
|
||||
transition: all 0.15s;
|
||||
white-space: nowrap;
|
||||
}
|
||||
.btn-delete:hover { color: var(--danger); border-color: var(--danger); }
|
||||
|
||||
.cards-area { padding: 16px 18px; overflow-y: auto; flex: 1; }
|
||||
.cards-grid { display: grid; gap: 8px; }
|
||||
|
||||
/* ── Cards ── */
|
||||
.card {
|
||||
background: var(--surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 8px;
|
||||
padding: 12px 14px;
|
||||
transition: border-color 0.12s;
|
||||
}
|
||||
.card:hover { border-color: #3a3d50; }
|
||||
.card-row {
|
||||
display: flex;
|
||||
gap: 10px;
|
||||
padding: 4px 0;
|
||||
font-size: 13px;
|
||||
border-bottom: 1px solid rgba(42,45,58,0.6);
|
||||
}
|
||||
.card-row:last-child { border-bottom: none; padding-bottom: 0; }
|
||||
.card-row:first-child { padding-top: 0; }
|
||||
.card-key {
|
||||
color: var(--muted);
|
||||
min-width: 100px;
|
||||
flex-shrink: 0;
|
||||
font-size: 11px;
|
||||
padding-top: 2px;
|
||||
font-weight: 500;
|
||||
}
|
||||
.card-val { color: var(--text); word-break: break-word; flex: 1; }
|
||||
.card-val.clamp {
|
||||
overflow: hidden;
|
||||
display: -webkit-box;
|
||||
-webkit-line-clamp: 3;
|
||||
-webkit-box-orient: vertical;
|
||||
}
|
||||
.card-link { color: var(--accent); text-decoration: none; font-size: 12px; }
|
||||
.card-link:hover { text-decoration: underline; }
|
||||
|
||||
/* ── Media ── */
|
||||
.card-media {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 6px;
|
||||
margin-top: 10px;
|
||||
padding-top: 10px;
|
||||
border-top: 1px solid rgba(42,45,58,0.6);
|
||||
}
|
||||
.media-item { position: relative; display: inline-block; }
|
||||
.media-thumb {
|
||||
display: block;
|
||||
width: 130px;
|
||||
height: 86px;
|
||||
object-fit: cover;
|
||||
border-radius: 5px;
|
||||
border: 1px solid var(--border);
|
||||
cursor: pointer;
|
||||
transition: opacity 0.15s;
|
||||
}
|
||||
.media-thumb:hover { opacity: 0.82; }
|
||||
.media-video {
|
||||
display: block;
|
||||
width: 220px;
|
||||
height: 140px;
|
||||
border-radius: 5px;
|
||||
border: 1px solid var(--border);
|
||||
background: #000;
|
||||
}
|
||||
.media-badge {
|
||||
position: absolute;
|
||||
bottom: 5px; left: 5px;
|
||||
background: rgba(0,0,0,0.72);
|
||||
color: #fff;
|
||||
font-size: 10px;
|
||||
font-weight: 600;
|
||||
padding: 2px 6px;
|
||||
border-radius: 3px;
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
/* ── Empty / loading states ── */
|
||||
.empty-state {
|
||||
color: var(--muted);
|
||||
font-size: 13px;
|
||||
padding: 60px 20px;
|
||||
text-align: center;
|
||||
}
|
||||
.spinner {
|
||||
display: inline-block;
|
||||
width: 14px; height: 14px;
|
||||
border: 2px solid var(--border);
|
||||
border-top-color: var(--accent);
|
||||
border-radius: 50%;
|
||||
animation: spin 0.7s linear infinite;
|
||||
vertical-align: middle;
|
||||
margin-right: 6px;
|
||||
}
|
||||
@keyframes spin { to { transform: rotate(360deg); } }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<header>
|
||||
<h1>Jieyab89 SOCMINT X</h1>
|
||||
<span class="sep">|</span>
|
||||
<span class="sub">Saved Archives</span>
|
||||
<button id="navToggle" class="hamburger-btn" aria-label="Navigation menu">☰</button>
|
||||
<nav id="navMenu" class="nav-menu hidden">
|
||||
<a href="/">Home</a>
|
||||
<a href="/graph">Graph</a>
|
||||
<hr class="nav-divider">
|
||||
<a href="/archives" class="current">Archives</a>
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<div class="layout">
|
||||
|
||||
<!-- Sidebar: archive list -->
|
||||
<div class="sidebar">
|
||||
<div class="sidebar-head">
|
||||
<span class="sidebar-title">Archives</span>
|
||||
<span class="sidebar-count" id="archiveCount">—</span>
|
||||
</div>
|
||||
<div class="archive-list" id="archiveList">
|
||||
<div class="no-archives"><span class="spinner"></span> Loading…</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Main viewer -->
|
||||
<div class="viewer">
|
||||
<div class="viewer-top" id="viewerTop" style="display:none">
|
||||
<div class="viewer-meta">
|
||||
<div class="viewer-meta-title" id="viewerTitle">—</div>
|
||||
<div class="viewer-meta-sub" id="viewerSub">—</div>
|
||||
</div>
|
||||
<input type="text" class="search-bar" id="searchInput" placeholder="Search in archive…">
|
||||
<span class="result-count" id="resultCount"></span>
|
||||
<button class="btn-delete" id="deleteBtn">Delete</button>
|
||||
</div>
|
||||
|
||||
<div class="cards-area">
|
||||
<div id="cardsBox">
|
||||
<div class="empty-state">Select an archive from the sidebar to view its contents.</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<script nonce="{{ g.csp_nonce }}">
|
||||
// Hamburger nav
|
||||
(function() {
|
||||
var btn = document.getElementById('navToggle');
|
||||
var menu = document.getElementById('navMenu');
|
||||
btn.addEventListener('click', function(e) {
|
||||
e.stopPropagation();
|
||||
menu.classList.toggle('hidden');
|
||||
});
|
||||
document.addEventListener('click', function() { menu.classList.add('hidden'); });
|
||||
menu.addEventListener('click', function(e) { e.stopPropagation(); });
|
||||
}());
|
||||
|
||||
let allItems = [];
|
||||
let activeId = null;
|
||||
|
||||
const archiveList = document.getElementById('archiveList');
|
||||
const archiveCount = document.getElementById('archiveCount');
|
||||
const viewerTop = document.getElementById('viewerTop');
|
||||
const viewerTitle = document.getElementById('viewerTitle');
|
||||
const viewerSub = document.getElementById('viewerSub');
|
||||
const searchInput = document.getElementById('searchInput');
|
||||
const resultCount = document.getElementById('resultCount');
|
||||
const deleteBtn = document.getElementById('deleteBtn');
|
||||
const cardsBox = document.getElementById('cardsBox');
|
||||
|
||||
// ── Sidebar ───────────────────────────────────────────────────────────────────
|
||||
|
||||
async function loadSidebar() {
|
||||
const res = await fetch('/api/archive/list');
|
||||
const json = await res.json();
|
||||
|
||||
if (!json.ok || !json.archives.length) {
|
||||
archiveList.innerHTML = '<div class="no-archives">No archives yet.<br>Enable Auto Archive in the tool and run a search.</div>';
|
||||
archiveCount.textContent = '0';
|
||||
return;
|
||||
}
|
||||
|
||||
archiveCount.textContent = json.archives.length;
|
||||
archiveList.innerHTML = json.archives.map(a => entryHtml(a)).join('');
|
||||
|
||||
archiveList.querySelectorAll('.archive-entry').forEach(el => {
|
||||
el.addEventListener('click', () => loadArchive(el.dataset.id));
|
||||
});
|
||||
|
||||
// Auto-open the newest one
|
||||
loadArchive(json.archives[0].id);
|
||||
}
|
||||
|
||||
function entryHtml(a) {
|
||||
const tool = (a.tool || 'unknown').replace(/_/g, ' ');
|
||||
const date = a.archived_at ? a.archived_at.replace('T', ' ') : '—';
|
||||
const query = queryLabel(a.query || {});
|
||||
return `
|
||||
<div class="archive-entry" data-id="${esc(a.id)}">
|
||||
<div class="ae-tool">${esc(tool)}</div>
|
||||
${query ? `<div class="ae-meta">${esc(query)}</div>` : ''}
|
||||
<div class="ae-stats">
|
||||
<span class="ae-stat">· ${a.total_items || 0} items</span>
|
||||
<span class="ae-stat">· ${a.media_count || 0} media</span>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function queryLabel(q) {
|
||||
return q.searchQuery || q.targetUsername || q.targetTweetId || q.targetCommunityId || '';
|
||||
}
|
||||
|
||||
// ── Viewer ────────────────────────────────────────────────────────────────────
|
||||
|
||||
async function loadArchive(id) {
|
||||
activeId = id;
|
||||
|
||||
// Highlight sidebar entry
|
||||
archiveList.querySelectorAll('.archive-entry').forEach(el => {
|
||||
el.classList.toggle('active', el.dataset.id === id);
|
||||
});
|
||||
|
||||
cardsBox.innerHTML = '<div class="empty-state"><span class="spinner"></span> Loading…</div>';
|
||||
viewerTop.style.display = 'none';
|
||||
searchInput.value = '';
|
||||
|
||||
const res = await fetch(`/api/archive/${id}/results`);
|
||||
const json = await res.json();
|
||||
if (!json.ok) {
|
||||
cardsBox.innerHTML = `<div class="empty-state" style="color:var(--danger)">${esc(json.error)}</div>`;
|
||||
return;
|
||||
}
|
||||
|
||||
allItems = json.results || [];
|
||||
const meta = json.meta || {};
|
||||
|
||||
const tool = (meta.tool || 'unknown').replace(/_/g, ' ');
|
||||
const query = queryLabel(meta.query || {});
|
||||
viewerTitle.textContent = tool + (query ? ` — "${query}"` : '');
|
||||
viewerSub.textContent = [
|
||||
meta.archived_at ? meta.archived_at.replace('T', ' ') : '',
|
||||
`${allItems.length} items`,
|
||||
meta.media_count ? `${meta.media_count} media files` : '',
|
||||
].filter(Boolean).join(' · ');
|
||||
|
||||
viewerTop.style.display = '';
|
||||
renderCards(allItems, '');
|
||||
}
|
||||
|
||||
// ── Search ────────────────────────────────────────────────────────────────────
|
||||
|
||||
searchInput.addEventListener('input', () => renderCards(allItems, searchInput.value));
|
||||
|
||||
function flatText(obj) {
|
||||
if (!obj || typeof obj !== 'object') return String(obj ?? '');
|
||||
return Object.values(obj).map(flatText).join(' ');
|
||||
}
|
||||
|
||||
// ── Cards ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
const PRIORITY = ['user','user_id','text','full_text','article_text','created_at','tweet_url'];
|
||||
const SKIP = ['archived_media'];
|
||||
const CLAMP = new Set(['text','full_text','article_text','description']);
|
||||
|
||||
function renderCards(items, query) {
|
||||
const q = query.toLowerCase().trim();
|
||||
const filtered = q ? items.filter(i => flatText(i).toLowerCase().includes(q)) : items;
|
||||
|
||||
resultCount.textContent = filtered.length + (q ? ' found' : ' results');
|
||||
|
||||
if (!filtered.length) {
|
||||
cardsBox.innerHTML = '<div class="empty-state">No matching results.</div>';
|
||||
return;
|
||||
}
|
||||
cardsBox.innerHTML = '<div class="cards-grid">' + filtered.map(buildCard).join('') + '</div>';
|
||||
}
|
||||
|
||||
function buildCard(item) {
|
||||
if (typeof item !== 'object' || !item) return '';
|
||||
|
||||
const entries = Object.entries(item);
|
||||
const pri = entries.filter(([k]) => PRIORITY.includes(k));
|
||||
const rest = entries.filter(([k]) => !PRIORITY.includes(k) && !SKIP.includes(k));
|
||||
|
||||
const rows = [...pri, ...rest].slice(0, 14).map(([k, v]) => {
|
||||
let display;
|
||||
if (v === null || v === undefined) {
|
||||
display = `<span style="color:var(--muted)">—</span>`;
|
||||
} else if (k === 'tweet_url') {
|
||||
display = `<a href="${esc(String(v))}" target="_blank" rel="noopener" class="card-link">${esc(String(v))}</a>`;
|
||||
} else if (typeof v === 'object') {
|
||||
const s = JSON.stringify(v);
|
||||
display = `<span style="color:var(--muted);font-size:11px">${esc(s.length > 100 ? s.slice(0,100)+'…' : s)}</span>`;
|
||||
} else {
|
||||
const cls = CLAMP.has(k) ? ' clamp' : '';
|
||||
display = `<span class="${cls}">${esc(String(v))}</span>`;
|
||||
}
|
||||
return `<div class="card-row"><div class="card-key">${esc(k)}</div><div class="card-val">${display}</div></div>`;
|
||||
}).join('');
|
||||
|
||||
const media = buildMedia(item);
|
||||
return `<div class="card">${rows}${media}</div>`;
|
||||
}
|
||||
|
||||
function buildMedia(item) {
|
||||
const paths = item.archived_media;
|
||||
if (!Array.isArray(paths) || !paths.length) return '';
|
||||
|
||||
const archiveId = activeId;
|
||||
const html = paths.map(p => {
|
||||
// p = "media/filename.ext"
|
||||
const filename = p.replace(/^media\//, '');
|
||||
const src = `/api/archive/${encodeURIComponent(archiveId)}/media/${encodeURIComponent(filename)}`;
|
||||
const isVideo = filename.endsWith('.mp4');
|
||||
const isGif = filename.includes('animated_gif');
|
||||
|
||||
if (isVideo || isGif) {
|
||||
const loop = isGif ? 'loop muted' : '';
|
||||
const label = isGif ? '<span class="media-badge">GIF</span>' : '';
|
||||
return `<div class="media-item">
|
||||
<video class="media-video" controls ${loop} preload="none">
|
||||
<source src="${src}" type="video/mp4">
|
||||
</video>${label}
|
||||
</div>`;
|
||||
}
|
||||
return `<a href="${src}" target="_blank" rel="noopener" class="media-item">
|
||||
<img src="${src}" class="media-thumb" alt="media" loading="lazy">
|
||||
</a>`;
|
||||
}).join('');
|
||||
|
||||
return `<div class="card-media">${html}</div>`;
|
||||
}
|
||||
|
||||
// ── Delete ────────────────────────────────────────────────────────────────────
|
||||
|
||||
deleteBtn.addEventListener('click', async () => {
|
||||
if (!activeId) return;
|
||||
if (!confirm('Delete this archive? This will permanently remove all saved files and media.')) return;
|
||||
|
||||
const res = await fetch(`/api/archive/${activeId}`, { method: 'DELETE' });
|
||||
const json = await res.json();
|
||||
if (!json.ok) { alert('Delete failed: ' + json.error); return; }
|
||||
|
||||
activeId = null;
|
||||
allItems = [];
|
||||
cardsBox.innerHTML = '<div class="empty-state">Archive deleted. Select another from the sidebar.</div>';
|
||||
viewerTop.style.display = 'none';
|
||||
loadSidebar();
|
||||
});
|
||||
|
||||
// ── Utilities ─────────────────────────────────────────────────────────────────
|
||||
|
||||
function esc(s) {
|
||||
return String(s).replace(/[&<>"']/g, m => ({'&':'&','<':'<','>':'>','"':'"',"'":'''})[m]);
|
||||
}
|
||||
|
||||
// ── Boot ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
loadSidebar();
|
||||
</script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,947 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<link rel="stylesheet" href="https://jsdelivr.net">
|
||||
<title>Jieyab89 SOCMINT X — Graph</title>
|
||||
<style>
|
||||
:root {
|
||||
--bg: #0f1117;
|
||||
--surface: #1a1d27;
|
||||
--surface2: #21253a;
|
||||
--border: #2a2d3a;
|
||||
--text: #e8eaf0;
|
||||
--muted: #8890a4;
|
||||
--accent: #5865f2;
|
||||
--accent-bg: #1e2240;
|
||||
--success: #22c55e;
|
||||
--danger: #ef4444;
|
||||
--warn: #f59e0b;
|
||||
--purple: #8b5cf6;
|
||||
--purple-bg: #1e1535;
|
||||
--font: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", sans-serif;
|
||||
}
|
||||
* { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
html, body {
|
||||
height: 100%;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
font-family: var(--font);
|
||||
font-size: 14px;
|
||||
overflow: hidden;
|
||||
}
|
||||
body { display: flex; flex-direction: column; }
|
||||
|
||||
/* ── Header ── */
|
||||
header {
|
||||
padding: 10px 20px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
background: var(--surface);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
flex-shrink: 0;
|
||||
position: relative;
|
||||
}
|
||||
header h1 { font-size: 14px; font-weight: 600; }
|
||||
header .sep { color: var(--border); }
|
||||
header .sub { color: var(--muted); font-size: 12px; flex: 1; }
|
||||
.hamburger-btn {
|
||||
margin-left: auto;
|
||||
background: none;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 5px;
|
||||
color: var(--muted);
|
||||
cursor: pointer;
|
||||
padding: 5px 9px;
|
||||
font-size: 16px;
|
||||
line-height: 1;
|
||||
transition: all 0.15s;
|
||||
}
|
||||
.hamburger-btn:hover { color: var(--text); border-color: var(--accent); }
|
||||
.nav-menu {
|
||||
position: absolute;
|
||||
top: 100%;
|
||||
right: 12px;
|
||||
margin-top: 6px;
|
||||
background: var(--surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 7px;
|
||||
padding: 5px;
|
||||
min-width: 175px;
|
||||
box-shadow: 0 8px 28px rgba(0,0,0,0.5);
|
||||
z-index: 200;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 1px;
|
||||
}
|
||||
.nav-menu.hidden { display: none; }
|
||||
.nav-menu a {
|
||||
padding: 7px 12px;
|
||||
border-radius: 5px;
|
||||
color: var(--muted);
|
||||
text-decoration: none;
|
||||
font-size: 13px;
|
||||
transition: all 0.12s;
|
||||
display: block;
|
||||
}
|
||||
.nav-menu a:hover { color: var(--text); background: var(--accent-bg); }
|
||||
.nav-menu a.current { color: var(--accent); background: var(--accent-bg); }
|
||||
.nav-divider { border: none; border-top: 1px solid var(--border); margin: 3px 0; }
|
||||
|
||||
/* ── Toolbar ── */
|
||||
#toolbar {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 7px;
|
||||
padding: 7px 16px;
|
||||
background: var(--surface);
|
||||
border-bottom: 1px solid var(--border);
|
||||
flex-shrink: 0;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
#toolbar select, #toolbar input[type="text"], #toolbar input[type="number"] {
|
||||
background: var(--bg);
|
||||
border: 1px solid var(--border);
|
||||
color: var(--text);
|
||||
border-radius: 5px;
|
||||
padding: 5px 8px;
|
||||
font-size: 12px;
|
||||
font-family: var(--font);
|
||||
line-height: 1.4;
|
||||
}
|
||||
#toolbar select:focus, #toolbar input:focus { outline: none; border-color: var(--accent); }
|
||||
#queryInput { flex: 1; min-width: 200px; }
|
||||
#countInput { width: 68px; }
|
||||
|
||||
/* ── Buttons ── */
|
||||
.btn {
|
||||
padding: 5px 12px;
|
||||
border-radius: 5px;
|
||||
border: 1px solid var(--border);
|
||||
font-size: 12px;
|
||||
font-family: var(--font);
|
||||
cursor: pointer;
|
||||
transition: all 0.15s;
|
||||
white-space: nowrap;
|
||||
line-height: 1.4;
|
||||
}
|
||||
.btn-primary { background: var(--accent); color: #fff; border-color: var(--accent); }
|
||||
.btn-primary:hover:not(:disabled) { background: #4752c4; }
|
||||
.btn-ghost { background: transparent; color: var(--muted); }
|
||||
.btn-ghost:hover:not(:disabled) { color: var(--text); border-color: var(--accent); background: var(--accent-bg); }
|
||||
.btn-success { background: transparent; color: var(--success); border-color: #1a4d2e; }
|
||||
.btn-success:hover:not(:disabled) { background: rgba(34,197,94,0.1); border-color: var(--success); }
|
||||
.btn-danger { background: transparent; color: var(--danger); border-color: #4d1a1a; }
|
||||
.btn-danger:hover:not(:disabled) { background: rgba(239,68,68,0.1); border-color: var(--danger); }
|
||||
.btn:disabled { opacity: 0.38; cursor: not-allowed; }
|
||||
|
||||
/* ── Main area ── */
|
||||
#main {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
min-height: 0;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
/* ── Cytoscape canvas ── */
|
||||
#cy {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
background: var(--bg);
|
||||
}
|
||||
|
||||
/* ── Info panel ── */
|
||||
#infoPanel {
|
||||
width: 310px;
|
||||
border-left: 1px solid var(--border);
|
||||
background: var(--surface);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
flex-shrink: 0;
|
||||
overflow: hidden;
|
||||
transition: width 0.2s ease;
|
||||
}
|
||||
#infoPanel.hidden { width: 0; border-left: none; }
|
||||
|
||||
#infoHeader {
|
||||
padding: 9px 12px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
flex-shrink: 0;
|
||||
gap: 8px;
|
||||
}
|
||||
#infoTitle {
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
color: var(--accent);
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.05em;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
#btnClosePanel {
|
||||
background: none;
|
||||
border: none;
|
||||
color: var(--muted);
|
||||
cursor: pointer;
|
||||
font-size: 18px;
|
||||
line-height: 1;
|
||||
padding: 0 2px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
#btnClosePanel:hover { color: var(--text); }
|
||||
|
||||
#infoContent {
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
padding: 10px 12px;
|
||||
font-size: 12px;
|
||||
scrollbar-width: thin;
|
||||
scrollbar-color: var(--border) transparent;
|
||||
}
|
||||
.info-row {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
padding: 5px 0;
|
||||
border-bottom: 1px solid var(--border);
|
||||
align-items: flex-start;
|
||||
}
|
||||
.info-key {
|
||||
color: var(--muted);
|
||||
font-size: 10px;
|
||||
min-width: 95px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.04em;
|
||||
flex-shrink: 0;
|
||||
padding-top: 1px;
|
||||
line-height: 1.5;
|
||||
}
|
||||
.info-val {
|
||||
color: var(--text);
|
||||
flex: 1;
|
||||
word-break: break-word;
|
||||
line-height: 1.5;
|
||||
}
|
||||
.info-val a { color: var(--accent); text-decoration: none; }
|
||||
.info-val a:hover { text-decoration: underline; }
|
||||
.info-media {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 5px;
|
||||
margin-top: 8px;
|
||||
padding-top: 8px;
|
||||
border-top: 1px solid var(--border);
|
||||
}
|
||||
.info-media a img {
|
||||
width: 80px;
|
||||
height: 60px;
|
||||
object-fit: cover;
|
||||
border-radius: 4px;
|
||||
border: 1px solid var(--border);
|
||||
display: block;
|
||||
}
|
||||
.info-media a.video-link {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 80px;
|
||||
height: 60px;
|
||||
background: var(--surface2);
|
||||
border-radius: 4px;
|
||||
border: 1px solid var(--border);
|
||||
color: var(--muted);
|
||||
font-size: 10px;
|
||||
text-decoration: none;
|
||||
}
|
||||
.info-media a.video-link:hover { border-color: var(--accent); color: var(--accent); }
|
||||
|
||||
#infoActions {
|
||||
padding: 10px 12px;
|
||||
border-top: 1px solid var(--border);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 5px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
#infoActions .btn { width: 100%; text-align: center; justify-content: center; }
|
||||
#infoActions a.btn { text-decoration: none; display: block; }
|
||||
|
||||
/* ── Legend ── */
|
||||
#legend {
|
||||
position: absolute;
|
||||
bottom: 14px;
|
||||
left: 14px;
|
||||
background: rgba(26,29,39,0.93);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 7px;
|
||||
padding: 9px 13px;
|
||||
font-size: 11px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 5px;
|
||||
pointer-events: none;
|
||||
z-index: 10;
|
||||
}
|
||||
.leg-row { display: flex; align-items: center; gap: 8px; color: var(--muted); }
|
||||
.leg-icon {
|
||||
width: 14px;
|
||||
height: 14px;
|
||||
flex-shrink: 0;
|
||||
border: 2px solid;
|
||||
}
|
||||
.leg-icon.search { background: var(--accent); border-color: var(--accent); transform: rotate(45deg); border-radius: 1px; }
|
||||
.leg-icon.tweet { background: var(--accent-bg); border-color: var(--accent); border-radius: 2px; }
|
||||
.leg-icon.reply { background: rgba(34,197,94,0.12); border-color: var(--success); border-radius: 2px; }
|
||||
.leg-icon.user { background: var(--purple-bg); border-color: var(--purple); border-radius: 50%; }
|
||||
|
||||
/* ── Status bar ── */
|
||||
#statusBar {
|
||||
position: absolute;
|
||||
top: 10px;
|
||||
left: 50%;
|
||||
transform: translateX(-50%);
|
||||
background: rgba(26,29,39,0.94);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 5px;
|
||||
padding: 4px 16px;
|
||||
font-size: 12px;
|
||||
color: var(--muted);
|
||||
pointer-events: none;
|
||||
z-index: 20;
|
||||
transition: opacity 0.3s;
|
||||
max-width: 420px;
|
||||
text-align: center;
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
#statusBar.error { color: var(--danger); border-color: #4d1a1a; }
|
||||
#statusBar.hidden { opacity: 0; }
|
||||
|
||||
/* ── Node count badge ── */
|
||||
#nodeCount {
|
||||
font-size: 11px;
|
||||
color: var(--muted);
|
||||
padding: 3px 9px;
|
||||
background: var(--bg);
|
||||
border-radius: 10px;
|
||||
border: 1px solid var(--border);
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
/* ── Spinner ── */
|
||||
.spin {
|
||||
display: inline-block;
|
||||
animation: spin 1s linear infinite;
|
||||
}
|
||||
@keyframes spin { to { transform: rotate(360deg); } }
|
||||
|
||||
/* ── Empty state hint ── */
|
||||
#emptyHint {
|
||||
position: absolute;
|
||||
top: 50%;
|
||||
left: 50%;
|
||||
transform: translate(-50%, -50%);
|
||||
color: var(--border);
|
||||
font-size: 13px;
|
||||
pointer-events: none;
|
||||
text-align: center;
|
||||
line-height: 2;
|
||||
z-index: 5;
|
||||
}
|
||||
#emptyHint.hidden { display: none; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<header>
|
||||
<h1>Jieyab89 SOCMINT X</h1>
|
||||
<span class="sep">|</span>
|
||||
<span class="sub">Graph Visualization</span>
|
||||
<button id="navToggle" class="hamburger-btn" aria-label="Navigation menu">☰</button>
|
||||
<nav id="navMenu" class="nav-menu hidden">
|
||||
<a href="/">Home</a>
|
||||
<a href="/graph" class="current">Graph</a>
|
||||
<hr class="nav-divider">
|
||||
<a href="/archives">Archives</a>
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<div id="toolbar">
|
||||
<select id="toolSelect">
|
||||
<option value="tweet_search_extractor">Tweet Search</option>
|
||||
<option value="follower_explorer">Follower Explorer</option>
|
||||
<option value="post_extractor">Post Extractor</option>
|
||||
<option value="article_extractor">Article Extractor</option>
|
||||
<option value="community_post_extractor">Community Posts</option>
|
||||
<option value="tweet_replies_extractor">Tweet Replies [Cookie]</option>
|
||||
<option value="tweet_retweeters_extractor">Tweet Retweeters [Cookie]</option>
|
||||
<option value="geo_post_extractor">Geo Post Search [Cookie]</option>
|
||||
</select>
|
||||
<select id="modeSelect">
|
||||
<option value="cookie">Cookie</option>
|
||||
<option value="api">API</option>
|
||||
</select>
|
||||
<input id="queryInput" type="text" placeholder="Search query…" spellcheck="false" autocomplete="off">
|
||||
<input id="countInput" type="number" value="20" min="1" max="200" title="Result count">
|
||||
<span id="nodeCount">0 nodes</span>
|
||||
<button id="btnSearch" class="btn btn-primary">Search</button>
|
||||
<button id="btnArchiveAll" class="btn btn-ghost">Archive All</button>
|
||||
<button id="btnDump" class="btn btn-ghost">Dump JSON</button>
|
||||
<button id="btnClear" class="btn btn-danger">Clear</button>
|
||||
</div>
|
||||
|
||||
<div id="main">
|
||||
<div id="cy"></div>
|
||||
|
||||
<!-- Right info panel -->
|
||||
<div id="infoPanel" class="hidden">
|
||||
<div id="infoHeader">
|
||||
<span id="infoTitle">Node Details</span>
|
||||
<button id="btnClosePanel" title="Close">×</button>
|
||||
</div>
|
||||
<div id="infoContent"></div>
|
||||
<div id="infoActions">
|
||||
<button id="btnExpandReplies" class="btn btn-ghost" style="display:none">↩ Expand Replies</button>
|
||||
<button id="btnExpandRetweets" class="btn btn-ghost" style="display:none">↗ Expand Retweets</button>
|
||||
<a id="btnOpenTweet" class="btn btn-success" style="display:none"
|
||||
href="#" target="_blank" rel="noopener noreferrer">Open Tweet ↗</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Status bar (floats over cy) -->
|
||||
<div id="statusBar" class="hidden">Ready</div>
|
||||
|
||||
<!-- Empty state -->
|
||||
<div id="emptyHint">
|
||||
Use the toolbar to search<br>
|
||||
Click any node to inspect · Click tweet nodes to expand replies / retweets
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Legend (floats over cy, bottom-left) -->
|
||||
<div id="legend">
|
||||
<div class="leg-row"><div class="leg-icon search"></div> Search root</div>
|
||||
<div class="leg-row"><div class="leg-icon tweet"></div> Tweet</div>
|
||||
<div class="leg-row"><div class="leg-icon reply"></div> Reply</div>
|
||||
<div class="leg-row"><div class="leg-icon user"></div> User / Retweeter</div>
|
||||
</div>
|
||||
|
||||
<script src="https://unpkg.com/cytoscape@3.28.1/dist/cytoscape.min.js" crossorigin="anonymous"></script>
|
||||
<script nonce="{{ g.csp_nonce }}">
|
||||
(function () {
|
||||
'use strict';
|
||||
|
||||
// ── State ─────────────────────────────────────────────────────────────────────
|
||||
const allItems = []; // { type, item }
|
||||
let selectedNode = null;
|
||||
let cy;
|
||||
|
||||
// ── Cytoscape styles ──────────────────────────────────────────────────────────
|
||||
const CY_STYLE = [
|
||||
{ selector: 'node[type="search"]', style: {
|
||||
shape: 'diamond',
|
||||
'background-color': '#5865f2',
|
||||
label: 'data(label)',
|
||||
color: '#e8eaf0',
|
||||
'font-size': '11px',
|
||||
'text-valign': 'bottom',
|
||||
'text-halign': 'center',
|
||||
'text-margin-y': '9px',
|
||||
width: '46px',
|
||||
height: '46px',
|
||||
'font-family': '-apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif',
|
||||
}},
|
||||
{ selector: 'node[type="tweet"]', style: {
|
||||
shape: 'roundrectangle',
|
||||
'background-color': '#1e2240',
|
||||
'border-color': '#5865f2',
|
||||
'border-width': '2px',
|
||||
label: 'data(label)',
|
||||
color: '#e8eaf0',
|
||||
'font-size': '9px',
|
||||
'text-valign': 'center',
|
||||
'text-halign': 'center',
|
||||
'text-wrap': 'wrap',
|
||||
'text-max-width': '138px',
|
||||
width: '158px',
|
||||
height: '62px',
|
||||
'font-family': '-apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif',
|
||||
}},
|
||||
{ selector: 'node[type="reply"]', style: {
|
||||
shape: 'roundrectangle',
|
||||
'background-color': '#0d2018',
|
||||
'border-color': '#22c55e',
|
||||
'border-width': '2px',
|
||||
label: 'data(label)',
|
||||
color: '#e8eaf0',
|
||||
'font-size': '9px',
|
||||
'text-valign': 'center',
|
||||
'text-halign': 'center',
|
||||
'text-wrap': 'wrap',
|
||||
'text-max-width': '138px',
|
||||
width: '158px',
|
||||
height: '62px',
|
||||
'font-family': '-apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif',
|
||||
}},
|
||||
{ selector: 'node[type="retweeter"]', style: {
|
||||
shape: 'ellipse',
|
||||
'background-color': '#1e1535',
|
||||
'border-color': '#8b5cf6',
|
||||
'border-width': '2px',
|
||||
label: 'data(label)',
|
||||
color: '#e8eaf0',
|
||||
'font-size': '10px',
|
||||
'text-valign': 'bottom',
|
||||
'text-halign': 'center',
|
||||
'text-margin-y': '7px',
|
||||
width: '74px',
|
||||
height: '74px',
|
||||
'font-family': '-apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif',
|
||||
}},
|
||||
{ selector: 'node[type="user"]', style: {
|
||||
shape: 'ellipse',
|
||||
'background-color': '#1e1535',
|
||||
'border-color': '#8b5cf6',
|
||||
'border-width': '2px',
|
||||
label: 'data(label)',
|
||||
color: '#e8eaf0',
|
||||
'font-size': '10px',
|
||||
'text-valign': 'bottom',
|
||||
'text-halign': 'center',
|
||||
'text-margin-y': '7px',
|
||||
width: '74px',
|
||||
height: '74px',
|
||||
'font-family': '-apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif',
|
||||
}},
|
||||
{ selector: 'node:selected', style: {
|
||||
'border-color': '#f59e0b',
|
||||
'border-width': '3px',
|
||||
'background-color': '#2a2510',
|
||||
}},
|
||||
{ selector: 'edge', style: {
|
||||
'line-color': '#FFFFFF',
|
||||
'line-style': 'dashed',
|
||||
width: '1.5px',
|
||||
'target-arrow-color': '#FFFFFF',
|
||||
'target-arrow-shape': 'triangle',
|
||||
'curve-style': 'bezier',
|
||||
'arrow-scale': 1.0,
|
||||
}},
|
||||
];
|
||||
|
||||
// ── Init ──────────────────────────────────────────────────────────────────────
|
||||
function init() {
|
||||
cy = cytoscape({
|
||||
container: document.getElementById('cy'),
|
||||
style: CY_STYLE,
|
||||
layout: { name: 'preset' },
|
||||
minZoom: 0.08,
|
||||
maxZoom: 4,
|
||||
wheelSensitivity: 0.25,
|
||||
boxSelectionEnabled: false,
|
||||
});
|
||||
|
||||
cy.on('tap', 'node', function (evt) {
|
||||
selectedNode = evt.target;
|
||||
showPanel(selectedNode.data());
|
||||
});
|
||||
|
||||
cy.on('tap', function (evt) {
|
||||
if (evt.target === cy) hidePanel();
|
||||
});
|
||||
|
||||
// Hamburger nav
|
||||
var navBtn = document.getElementById('navToggle');
|
||||
var navMenu = document.getElementById('navMenu');
|
||||
navBtn.addEventListener('click', function(e) {
|
||||
e.stopPropagation();
|
||||
navMenu.classList.toggle('hidden');
|
||||
});
|
||||
document.addEventListener('click', function() { navMenu.classList.add('hidden'); });
|
||||
navMenu.addEventListener('click', function(e) { e.stopPropagation(); });
|
||||
|
||||
document.getElementById('btnSearch').addEventListener('click', runSearch);
|
||||
document.getElementById('queryInput').addEventListener('keydown', function (e) {
|
||||
if (e.key === 'Enter') runSearch();
|
||||
});
|
||||
document.getElementById('toolSelect').addEventListener('change', updatePlaceholder);
|
||||
document.getElementById('btnArchiveAll').addEventListener('click', archiveAll);
|
||||
document.getElementById('btnDump').addEventListener('click', dumpJSON);
|
||||
document.getElementById('btnClear').addEventListener('click', clearGraph);
|
||||
document.getElementById('btnClosePanel').addEventListener('click', hidePanel);
|
||||
document.getElementById('btnExpandReplies').addEventListener('click', function () {
|
||||
expandNode('tweet_replies_extractor');
|
||||
});
|
||||
document.getElementById('btnExpandRetweets').addEventListener('click', function () {
|
||||
expandNode('tweet_retweeters_extractor');
|
||||
});
|
||||
|
||||
updatePlaceholder();
|
||||
}
|
||||
|
||||
// ── Placeholder by tool type ──────────────────────────────────────────────────
|
||||
function updatePlaceholder() {
|
||||
var map = {
|
||||
tweet_search_extractor: 'Search query…',
|
||||
follower_explorer: 'Username or user ID…',
|
||||
post_extractor: 'Username or user ID…',
|
||||
article_extractor: 'Tweet ID…',
|
||||
community_post_extractor: 'Community ID…',
|
||||
tweet_replies_extractor: 'Tweet ID…',
|
||||
tweet_retweeters_extractor: 'Tweet ID…',
|
||||
geo_post_extractor: 'Keyword (geocodes profile location)…',
|
||||
};
|
||||
var tool = document.getElementById('toolSelect').value;
|
||||
document.getElementById('queryInput').placeholder = map[tool] || 'Query…';
|
||||
}
|
||||
|
||||
// ── Status bar ────────────────────────────────────────────────────────────────
|
||||
var _statusTimer = null;
|
||||
|
||||
function setStatus(msg, isError, persist) {
|
||||
var bar = document.getElementById('statusBar');
|
||||
bar.textContent = msg;
|
||||
bar.classList.toggle('error', !!isError);
|
||||
bar.classList.remove('hidden');
|
||||
if (_statusTimer) clearTimeout(_statusTimer);
|
||||
if (!persist) {
|
||||
_statusTimer = setTimeout(function () { bar.classList.add('hidden'); }, 4500);
|
||||
}
|
||||
}
|
||||
|
||||
function updateNodeCount() {
|
||||
var n = cy.nodes().length;
|
||||
document.getElementById('nodeCount').textContent = n + ' node' + (n !== 1 ? 's' : '');
|
||||
document.getElementById('emptyHint').classList.toggle('hidden', n > 0);
|
||||
}
|
||||
|
||||
// ── API helpers ───────────────────────────────────────────────────────────────
|
||||
function buildBody(tool, query, count, mode) {
|
||||
var body = { toolType: tool, mode: mode, count: count };
|
||||
if (tool === 'tweet_search_extractor' || tool === 'geo_post_extractor') {
|
||||
body.searchQuery = query;
|
||||
} else if (tool === 'follower_explorer' || tool === 'post_extractor') {
|
||||
body.targetUsername = query;
|
||||
} else if (
|
||||
tool === 'article_extractor' ||
|
||||
tool === 'tweet_replies_extractor' ||
|
||||
tool === 'tweet_retweeters_extractor'
|
||||
) {
|
||||
body.targetTweetId = query;
|
||||
} else if (tool === 'community_post_extractor') {
|
||||
body.targetCommunityId = query;
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
async function apiFetch(body) {
|
||||
var res = await fetch('/api/run', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
var json = await res.json();
|
||||
if (!json.ok) throw new Error(json.error || 'API error');
|
||||
return json.data;
|
||||
}
|
||||
|
||||
// ── Label builder ─────────────────────────────────────────────────────────────
|
||||
function makeLabel(item, type) {
|
||||
if (type === 'user' || type === 'retweeter') {
|
||||
var handle = item.screen_name || item.user || '';
|
||||
return handle ? '@' + handle : 'Unknown';
|
||||
}
|
||||
var text = (item.text || item.retweeted_text || '').replace(/\s+/g, ' ').trim();
|
||||
return text.length > 60 ? text.substring(0, 60) + '…' : (text || ('Tweet ' + (item.id || '?')));
|
||||
}
|
||||
|
||||
// ── Graph operations ──────────────────────────────────────────────────────────
|
||||
function addNodes(items, nodeType, parentId) {
|
||||
var added = 0;
|
||||
items.forEach(function (item, i) {
|
||||
var tweetId = String(item.id || '');
|
||||
var cyId = tweetId ? 'n_' + tweetId : 'n_' + Date.now() + '_' + i;
|
||||
|
||||
if (cy.getElementById(cyId).length) {
|
||||
if (parentId) addEdge(parentId, cyId);
|
||||
return;
|
||||
}
|
||||
|
||||
cy.add({
|
||||
data: {
|
||||
id: cyId,
|
||||
type: nodeType,
|
||||
label: makeLabel(item, nodeType),
|
||||
tweetId: tweetId,
|
||||
raw: item,
|
||||
}
|
||||
});
|
||||
|
||||
if (parentId) addEdge(parentId, cyId);
|
||||
allItems.push({ type: nodeType, item: item });
|
||||
added++;
|
||||
});
|
||||
return added;
|
||||
}
|
||||
|
||||
function addEdge(src, tgt) {
|
||||
var eId = 'e_' + src + '_' + tgt;
|
||||
if (!cy.getElementById(eId).length) {
|
||||
cy.add({ data: { id: eId, source: src, target: tgt } });
|
||||
}
|
||||
}
|
||||
|
||||
function runLayout(fit, randomize) {
|
||||
cy.layout({
|
||||
name: 'cose',
|
||||
animate: true,
|
||||
animationDuration: 450,
|
||||
randomize: !!randomize,
|
||||
fit: !!fit,
|
||||
nodeRepulsion: 10000,
|
||||
idealEdgeLength: 130,
|
||||
gravity: 0.35,
|
||||
numIter: 1000,
|
||||
padding: 40,
|
||||
}).run();
|
||||
}
|
||||
|
||||
// ── Search ────────────────────────────────────────────────────────────────────
|
||||
async function runSearch() {
|
||||
var tool = document.getElementById('toolSelect').value;
|
||||
var query = document.getElementById('queryInput').value.trim();
|
||||
var count = Math.max(1, Math.min(200, parseInt(document.getElementById('countInput').value, 10) || 20));
|
||||
var mode = document.getElementById('modeSelect').value;
|
||||
|
||||
if (!query) { setStatus('Enter a query first', true); return; }
|
||||
|
||||
document.getElementById('btnSearch').disabled = true;
|
||||
setStatus('Searching…', false, true);
|
||||
|
||||
try {
|
||||
var data = await apiFetch(buildBody(tool, query, count, mode));
|
||||
var items = Array.isArray(data) ? data : [data];
|
||||
|
||||
// Central search node
|
||||
var searchId = 'search_' + Date.now();
|
||||
var shortQ = query.length > 40 ? query.substring(0, 40) + '…' : query;
|
||||
cy.add({ data: { id: searchId, type: 'search', label: shortQ, tweetId: '', raw: { query: query } } });
|
||||
|
||||
// Node type by tool
|
||||
var nodeType = 'tweet';
|
||||
if (tool === 'follower_explorer') nodeType = 'user';
|
||||
else if (tool === 'tweet_retweeters_extractor') nodeType = 'retweeter';
|
||||
|
||||
var added = addNodes(items, nodeType, searchId);
|
||||
runLayout(true, true);
|
||||
updateNodeCount();
|
||||
setStatus('Done — ' + items.length + ' result(s) (' + added + ' new nodes)');
|
||||
} catch (e) {
|
||||
setStatus('Error: ' + e.message, true);
|
||||
} finally {
|
||||
document.getElementById('btnSearch').disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Expand replies / retweets from selected node ──────────────────────────────
|
||||
async function expandNode(expandTool) {
|
||||
if (!selectedNode) return;
|
||||
var data = selectedNode.data();
|
||||
var tweetId = data.tweetId;
|
||||
if (!tweetId) { setStatus('Node has no tweet ID', true); return; }
|
||||
|
||||
var count = Math.max(1, Math.min(200, parseInt(document.getElementById('countInput').value, 10) || 50));
|
||||
var parentId = selectedNode.id();
|
||||
|
||||
document.getElementById('btnExpandReplies').disabled = true;
|
||||
document.getElementById('btnExpandRetweets').disabled = true;
|
||||
setStatus('Expanding…', false, true);
|
||||
|
||||
try {
|
||||
var fetched = await apiFetch({ toolType: expandTool, mode: 'cookie', count: count, targetTweetId: tweetId });
|
||||
var items = Array.isArray(fetched) ? fetched : [fetched];
|
||||
var nodeType = expandTool === 'tweet_replies_extractor' ? 'reply' : 'retweeter';
|
||||
|
||||
var added = addNodes(items, nodeType, parentId);
|
||||
runLayout(false, false);
|
||||
updateNodeCount();
|
||||
setStatus('Expanded — ' + items.length + ' result(s) (' + added + ' new nodes)');
|
||||
} catch (e) {
|
||||
setStatus('Error: ' + e.message, true);
|
||||
} finally {
|
||||
document.getElementById('btnExpandReplies').disabled = false;
|
||||
document.getElementById('btnExpandRetweets').disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Info panel ────────────────────────────────────────────────────────────────
|
||||
function esc(s) {
|
||||
return String(s)
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"');
|
||||
}
|
||||
|
||||
var PRIORITY_KEYS = [
|
||||
'user', 'screen_name', 'name', 'text', 'created_at',
|
||||
'retweeted_by_user', 'retweeted_by_name', 'retweeted_text', 'retweeted_by_bio',
|
||||
'reply_count', 'retweet_count', 'favorite_count', 'view_count',
|
||||
'followers_count', 'following_count', 'tweet_count',
|
||||
'description', 'user_location', 'in_reply_to_tweet_id',
|
||||
'retweeted_tweet_id', 'retweeted_at', 'verified', 'is_blue_verified',
|
||||
];
|
||||
|
||||
var SKIP_KEYS = new Set(['id', 'media', 'card', 'user_id', 'retweeted_by_user_id']);
|
||||
|
||||
function showPanel(data) {
|
||||
var raw = data.raw || {};
|
||||
var type = data.type;
|
||||
|
||||
var titles = { search: 'Search Node', tweet: 'Tweet', reply: 'Reply', user: 'User', retweeter: 'Retweeter' };
|
||||
document.getElementById('infoTitle').textContent = titles[type] || 'Node';
|
||||
|
||||
// Build key list (priority first, then remaining)
|
||||
var allKeys = Object.keys(raw);
|
||||
var ordered = PRIORITY_KEYS.concat(allKeys.filter(function (k) { return PRIORITY_KEYS.indexOf(k) === -1; }));
|
||||
var keys = ordered.filter(function (k) {
|
||||
return !SKIP_KEYS.has(k) && raw[k] !== null && raw[k] !== undefined && raw[k] !== '';
|
||||
});
|
||||
|
||||
var rows = keys.map(function (k) {
|
||||
var v = raw[k];
|
||||
var val = esc(String(v));
|
||||
|
||||
// Format tweet ID fields as links
|
||||
if ((k === 'retweeted_tweet_id' || k === 'in_reply_to_tweet_id') && /^\d+$/.test(String(v))) {
|
||||
var handle = raw.retweeted_by_user || raw.user || '';
|
||||
if (handle) {
|
||||
val = '<a href="https://x.com/' + encodeURIComponent(handle) + '/status/' + encodeURIComponent(String(v)) +
|
||||
'" target="_blank" rel="noopener noreferrer">' + esc(String(v)) + ' ↗</a>';
|
||||
}
|
||||
}
|
||||
|
||||
var label = esc(k.replace(/_/g, ' '));
|
||||
return '<div class="info-row"><div class="info-key">' + label + '</div><div class="info-val">' + val + '</div></div>';
|
||||
}).join('');
|
||||
|
||||
// Media thumbnails
|
||||
var media = Array.isArray(raw.media) ? raw.media : [];
|
||||
if (media.length) {
|
||||
var mediaParts = ['<div class="info-media">'];
|
||||
media.forEach(function (m) {
|
||||
var thumb = m.thumb || '';
|
||||
var url = m.url || thumb;
|
||||
var mtype = m.type || 'photo';
|
||||
if (!thumb) return;
|
||||
if (mtype === 'photo') {
|
||||
mediaParts.push(
|
||||
'<a href="' + esc(url) + '" target="_blank" rel="noopener noreferrer">' +
|
||||
'<img src="' + esc(thumb) + '" loading="lazy" alt="media"></a>'
|
||||
);
|
||||
} else {
|
||||
mediaParts.push(
|
||||
'<a href="/api/video?url=' + encodeURIComponent(url) + '" target="_blank" rel="noopener noreferrer" class="video-link">▶ ' + esc(mtype) + '</a>'
|
||||
);
|
||||
}
|
||||
});
|
||||
mediaParts.push('</div>');
|
||||
rows += mediaParts.join('');
|
||||
}
|
||||
|
||||
document.getElementById('infoContent').innerHTML = rows;
|
||||
|
||||
// Action buttons
|
||||
var isTweet = type === 'tweet' || type === 'reply';
|
||||
var tweetId = data.tweetId;
|
||||
var user = raw.user || raw.screen_name || '';
|
||||
|
||||
document.getElementById('btnExpandReplies').style.display = isTweet ? '' : 'none';
|
||||
document.getElementById('btnExpandRetweets').style.display = isTweet ? '' : 'none';
|
||||
|
||||
var openBtn = document.getElementById('btnOpenTweet');
|
||||
if (isTweet && tweetId && user) {
|
||||
openBtn.href = 'https://x.com/' + encodeURIComponent(user) + '/status/' + encodeURIComponent(tweetId);
|
||||
openBtn.style.display = '';
|
||||
} else {
|
||||
openBtn.style.display = 'none';
|
||||
}
|
||||
|
||||
document.getElementById('infoPanel').classList.remove('hidden');
|
||||
}
|
||||
|
||||
function hidePanel() {
|
||||
document.getElementById('infoPanel').classList.add('hidden');
|
||||
selectedNode = null;
|
||||
cy.elements().unselect();
|
||||
}
|
||||
|
||||
// ── Archive All ───────────────────────────────────────────────────────────────
|
||||
async function archiveAll() {
|
||||
if (!allItems.length) { setStatus('Nothing to archive yet', true); return; }
|
||||
var query = document.getElementById('queryInput').value.trim();
|
||||
var tool = document.getElementById('toolSelect').value;
|
||||
|
||||
setStatus('Archiving…', false, true);
|
||||
try {
|
||||
var res = await fetch('/api/archive', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
toolType: 'graph_' + tool,
|
||||
data: allItems.map(function (d) { return d.item; }),
|
||||
queryInfo: { query: query, source: 'graph', nodes: cy.nodes().length },
|
||||
}),
|
||||
});
|
||||
var json = await res.json();
|
||||
if (!json.ok) throw new Error(json.error);
|
||||
setStatus('Archived — ID: ' + json.archiveId);
|
||||
} catch (e) {
|
||||
setStatus('Archive error: ' + e.message, true);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Dump JSON ─────────────────────────────────────────────────────────────────
|
||||
function dumpJSON() {
|
||||
if (!allItems.length) { setStatus('Nothing to export yet', true); return; }
|
||||
var payload = {
|
||||
exported_at: new Date().toISOString(),
|
||||
query: document.getElementById('queryInput').value.trim(),
|
||||
tool: document.getElementById('toolSelect').value,
|
||||
node_count: cy.nodes().length,
|
||||
items: allItems,
|
||||
};
|
||||
var blob = new Blob([JSON.stringify(payload, null, 2)], { type: 'application/json' });
|
||||
var url = URL.createObjectURL(blob);
|
||||
var a = document.createElement('a');
|
||||
a.href = url;
|
||||
a.download = 'graph_export_' + Date.now() + '.json';
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
document.body.removeChild(a);
|
||||
URL.revokeObjectURL(url);
|
||||
setStatus('Exported ' + allItems.length + ' item(s)');
|
||||
}
|
||||
|
||||
// ── Clear ─────────────────────────────────────────────────────────────────────
|
||||
function clearGraph() {
|
||||
cy.elements().remove();
|
||||
allItems.length = 0;
|
||||
hidePanel();
|
||||
updateNodeCount();
|
||||
setStatus('Graph cleared');
|
||||
}
|
||||
|
||||
// ── Boot ──────────────────────────────────────────────────────────────────────
|
||||
document.addEventListener('DOMContentLoaded', init);
|
||||
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -46,8 +46,6 @@ class XquikClient:
|
||||
except ValueError as e:
|
||||
raise XquikError(f"Response bukan JSON valid: {resp.text[:300]}") from e
|
||||
|
||||
# --- 5 tool type sesuai docs ---
|
||||
|
||||
def tweet_search(self, search_query: str) -> dict:
|
||||
return self._post({"toolType": "tweet_search_extractor", "searchQuery": search_query})
|
||||
|
||||
|
||||
@@ -53,6 +53,7 @@ def green(text):
|
||||
|
||||
# Wordlist (TLD + ccTLD + gTLD + vanity)
|
||||
COMMON_TLDS = [
|
||||
# Arr extentions tld domain lists
|
||||
|
||||
# All region goverment
|
||||
"gov","mil","go.id","mil.id","gov.au","gov.br",
|
||||
@@ -327,8 +328,7 @@ COMMON_TLDS = [
|
||||
"xn--t60b56a","xn--tckwe","xn--tiq49xqyj","xn--unup4y","xn--vermgensberater-ctb","xn--vermgensberatung-pwb","xn--vhquv","xn--vuq861b",
|
||||
"xn--w4r85el8fhu5dnra","xn--w4rs40l","xn--wgbh1c","xn--wgbl6a","xn--xhq521b","xn--xkc2al3hye2a","xn--xkc2dl3a5ee0h","xn--y9a3aq","xn--yfro4i67o",
|
||||
"xn--ygbi2ammx","xn--zfr164b","xxx","xyz","yachts","yahoo","yamaxun","yandex","ye","yodobashi","yoga","yokohama","you","youtube","yun","za",
|
||||
"zappos","zara","zero","zm","zone","zuerich","zw",
|
||||
|
||||
"zappos","zara","zero","zm","zone","zuerich","zw",
|
||||
]
|
||||
|
||||
def generate_brute_tlds(max_len=3, min_len=1):
|
||||
|
||||
Reference in New Issue
Block a user