mirror of
https://github.com/Jieyab89/OSINT-Cheat-sheet.git
synced 2026-08-17 18:35:41 +02:00
Edit and enhancement twitter scapper tool
This commit is contained in:
Jieyab89
@@ -9,3 +9,16 @@ Happy learning
|
||||
If any bug, missue and logic please let me know with mention the issue also the script was provided
|
||||
|
||||
Thanks
|
||||
|
||||
# Table Of Content
|
||||
|
||||
- [CNN-News-Scrapping-and-Archive](https://github.com/Jieyab89/OSINT-Cheat-sheet/tree/main/Script/CNN-News-Scrapping-and-Archive)
|
||||
- [Darkweb-Scrapping](https://github.com/Jieyab89/OSINT-Cheat-sheet/tree/main/Script/Darkweb-Scrapping)
|
||||
- [Dorking](https://github.com/Jieyab89/OSINT-Cheat-sheet/tree/main/Script/Dorking)
|
||||
- [Github-Search](https://github.com/Jieyab89/OSINT-Cheat-sheet/tree/main/Script/Github-Search)
|
||||
- [Linkedin-Script](https://github.com/Jieyab89/OSINT-Cheat-sheet/tree/main/Script/Linkedin-Script)
|
||||
- [OSM-QUERY-LIST](https://github.com/Jieyab89/OSINT-Cheat-sheet/tree/main/Script/OSM-QUERY-LIST)
|
||||
- [Red-Team](https://github.com/Jieyab89/OSINT-Cheat-sheet/tree/main/Script/Red-Team)
|
||||
- [SENTINEL-SAT-SCRIPT-LIST](https://github.com/Jieyab89/OSINT-Cheat-sheet/tree/main/Script/SENTINEL-SAT-SCRIPT-LIST)
|
||||
- [SOCMINT-Twitter](https://github.com/Jieyab89/OSINT-Cheat-sheet/tree/main/Script/SOCMINT-Twitter)
|
||||
- [Web-TLD-Enumerate-NS-Check](https://github.com/Jieyab89/OSINT-Cheat-sheet/tree/main/Script/Web-TLD-Enumerate-NS-Check)
|
||||
@@ -1,6 +1,41 @@
|
||||
# Jieyab ft Xquik
|
||||
|
||||
<img width="2553" height="1218" alt="Image" src="https://github.com/user-attachments/assets/eb01c1ca-c577-432d-a266-5533b5b71ce0" />
|
||||
<img width="2552" height="1237" alt="Image" src="https://github.com/user-attachments/assets/20465836-89f0-453b-a0d5-fa2700b8ace1" />
|
||||
|
||||
## Features
|
||||
|
||||
- **Multi-source search** — one query fans out to Cookie, Xquik API,
|
||||
Wayback Machine, and Google CSE in parallel, each result tagged with
|
||||
which source it came from.
|
||||
- **10 extraction tools** — tweet search, follower/following explorer, post
|
||||
(timeline) extractor, article extractor, community posts, tweet replies,
|
||||
tweet retweeters, geo-tagged search with a map view, Wayback archive
|
||||
search, and multi-source search.
|
||||
- **Infinite scroll + inline nested reply threads** — scroll to load more
|
||||
pages automatically; expand a reply's own replies in place, recursively,
|
||||
the way X's own UI threads a conversation.
|
||||
- **Relationship graph** (`/graph`) — a Cytoscape-based node graph. Select a
|
||||
node to expand its replies, retweeters, posts, followers, or following;
|
||||
pivot from any tweet straight to its author (no extra request — the data's
|
||||
already on the tweet); box-select multiple nodes and drag them together;
|
||||
edges are labeled by relationship (*replied by*, *followed by*, *authored
|
||||
by*, ...) and a distinct color flags where two different paths through the
|
||||
graph converge on the same account or tweet.
|
||||
- **Archive** (`/archives`) — one-click (or auto-) save of a search's full
|
||||
raw results, including downloaded media, browsable and re-searchable later
|
||||
independent of whether the source is still reachable.
|
||||
- **Analytics** (`/analytics`) — sentiment clustering (pro / neutral / con)
|
||||
over a saved archive, plus who posts most, what's driving the most
|
||||
engagement, and word frequency — see [Sentiment Analysis](#sentiment-analysis)
|
||||
below.
|
||||
- **Account-age forensics** — every result with a numeric X/Twitter ID gets
|
||||
its account creation date decoded straight from the ID's Snowflake bits
|
||||
(no extra API call), flagged New / Recent / Established.
|
||||
- **Security-conscious by default** — strict CSP with per-request nonces,
|
||||
hardened cookies, a whitelisted SSRF-safe video proxy, per-source request
|
||||
throttling to protect the logged-in account from rate limits, and
|
||||
URL-scheme validation everywhere a scraped link is rendered as an image or
|
||||
embedded in CSS.
|
||||
|
||||
# Sett up
|
||||
|
||||
@@ -31,9 +66,96 @@ Check the result in the table
|
||||
3. Wayback Machine (Cdx API)
|
||||
4. Goole CSE API (free quota 100 per day u can increase u limit with buy the service)
|
||||
|
||||
## Usage
|
||||
|
||||
### Search (`/`)
|
||||
|
||||
Pick a tool from the dropdown, fill in the field it asks for (query,
|
||||
username, tweet ID, ...), and hit **Run**. Cookie-only tools (marked
|
||||
`[Cookie]`) always use your logged-in session; everything else lets you
|
||||
toggle between **Cookie** (your session, no xquik quota used) and **xquik
|
||||
API** (uses your API key's quota) mode.
|
||||
|
||||
- Scroll down to auto-load more pages on any tool that supports pagination.
|
||||
- Any card whose post has replies of its own gets an **Expand N replies**
|
||||
button — click it to thread the conversation inline, as deep as it
|
||||
actually goes.
|
||||
- Toggle **Auto Archive** before running a search to save results as you go
|
||||
(including every scrolled-in page) — see [Archives](#archives-archives).
|
||||
- **Geo Post Extractor** switches the results view to a map, geocoding each
|
||||
author's profile location.
|
||||
- The `↓ JSON` button downloads exactly what's on screen as raw JSON.
|
||||
|
||||
### Graph (`/graph`)
|
||||
|
||||
Same 10 tools, rendered as a node graph instead of a card list.
|
||||
|
||||
- Click a node to inspect its full raw data in the side panel.
|
||||
- **Expand Replies / Retweets** (tweet or reply nodes), **Expand Posts /
|
||||
Followers / Following** (user or retweeter nodes), and **View Author
|
||||
Profile** (tweet/reply nodes — pulls the author out as their own node,
|
||||
for free) all attach new nodes with a labeled edge showing the
|
||||
relationship.
|
||||
- **Shift/Alt/Ctrl + drag** on empty canvas to box-select multiple nodes;
|
||||
drag any one of them to move the whole selection together. Plain drag
|
||||
still pans, scroll still zooms.
|
||||
- A **yellow edge** means two different paths through the graph converged
|
||||
on the same node — worth a second look.
|
||||
- **Archive All** / **Dump JSON** export everything currently on canvas.
|
||||
- Click the **?** button (bottom-left) for the full legend.
|
||||
|
||||
### Archives (`/archives`)
|
||||
|
||||
Browse everything saved from Search or Graph. Pick an archive from the
|
||||
sidebar to see its full raw results and any downloaded media. Archives
|
||||
persist independent of whether the original source is still reachable —
|
||||
useful for content that gets deleted or a session that expires.
|
||||
|
||||
### Analytics (`/analytics`)
|
||||
|
||||
Pick a saved archive to run sentiment clustering and the surrounding
|
||||
aggregates over it:
|
||||
|
||||
- **Sentiment clustering** — every item with text gets classified **pro** /
|
||||
**neutral** / **con**, shown as a diverging bar plus per-category tiles
|
||||
you can click to filter the item list below.
|
||||
- **Most active accounts** — who shows up most often in that archive.
|
||||
- **Most engagement** — which items drove the most reply+retweet+like
|
||||
activity.
|
||||
- **Word frequency** — a word cloud sized by how often each word appears.
|
||||
- **Item browser** — every scored item, filterable by sentiment and
|
||||
searchable by text/author, with the model's confidence (or, in lexicon
|
||||
fallback mode, the exact matched words) shown per item — a classification
|
||||
is always inspectable, never a black box.
|
||||
|
||||
## Sentiment Analysis
|
||||
|
||||
Two backends, tried in this order (see `sentiment.py`):
|
||||
|
||||
1. **ML (preferred)** — [`cardiffnlp/twitter-xlm-roberta-base-sentiment`](https://huggingface.co/cardiffnlp/twitter-xlm-roberta-base-sentiment),
|
||||
an XLM-RoBERTa model fine-tuned for tweet sentiment across 8 languages
|
||||
(Arabic, English, French, German, Hindi, Italian, Portuguese, Spanish).
|
||||
Its underlying pretraining covers roughly 100 languages, so it degrades
|
||||
gracefully rather than failing outright on a language outside that
|
||||
fine-tuned set — this is what makes the tool usable for a global
|
||||
audience, not just Indonesian speakers. Requires `torch` + `transformers`
|
||||
(see `requirements.txt`) and downloads ~1.1GB of model weights from
|
||||
Hugging Face the first time it runs.
|
||||
2. **Lexicon fallback** — a hand-built Indonesian positive/negative word
|
||||
list with negation handling (e.g. *"tidak bagus"* flips from positive to
|
||||
negative). Used automatically whenever `torch`/`transformers` aren't
|
||||
installed, so a lightweight install still has a working — if
|
||||
Indonesian-only — sentiment feature instead of a hard failure.
|
||||
|
||||
The `/api/analytics/<archive_id>` response always reports which backend
|
||||
(`"ml"` or `"lexicon"`) produced its results, and the Analytics page's
|
||||
banner reflects it. Neither backend is a ground-truth classifier — short
|
||||
text, sarcasm, and irony degrade accuracy either way. Treat results as a
|
||||
starting point for investigation, not a verdict.
|
||||
|
||||
## Update Note
|
||||
|
||||
1. Update infinity scroll and load new data for twitter reply and retweets
|
||||
1. Update infinity scroll and load new data for each search also in graph
|
||||
2. Update data corelation
|
||||
3. Fix business logic flow
|
||||
4. Monitoring (Soon)
|
||||
@@ -41,8 +163,12 @@ Check the result in the table
|
||||
6. Add more parameter for enrichment
|
||||
7. Add no rate limit (throttle)
|
||||
8. Add Google CSE data source
|
||||
9. Expand data user profile post, follower and following, reply post, retweet post in graph —
|
||||
10. Add sentiment analysis for clustering data, pro, neutral, con based on archive data and dump data
|
||||
11. Add more data source and other parameter (soon) still research
|
||||
12. Add more detail data source for the context
|
||||
|
||||
## Setup
|
||||
## Settup
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
@@ -50,6 +176,83 @@ pip install -r requirements.txt
|
||||
|
||||
Edit `config.ini.example` to config.ini
|
||||
|
||||
## Installing the ML sentiment model (optional, recommended)
|
||||
|
||||
The Analytics page's sentiment scoring works two ways — see
|
||||
[Sentiment Analysis](#sentiment-analysis) above. Skipping everything on this
|
||||
page is completely fine: the app still runs and Analytics still works, just
|
||||
using the Indonesian-only lexicon instead of the multilingual model. This
|
||||
section is only for turning on the better (multilingual) one. **You don't
|
||||
need to know Python or be a developer to follow this — it's copy/paste.**
|
||||
|
||||
**Why a "virtual environment" (venv)?** It's just a private, throwaway
|
||||
folder for this project's Python packages, kept separate from anything else
|
||||
Python-related already on your computer. The ML packages (`torch`,
|
||||
`transformers`) are large and can conflict with other unrelated tools if
|
||||
installed system-wide — a venv avoids that entirely, and if anything ever
|
||||
goes wrong, you just delete the `.venv` folder and start over, nothing else
|
||||
on your machine is touched.
|
||||
|
||||
**1. Open a terminal in this project's folder** (the same folder as
|
||||
`app.py`).
|
||||
|
||||
**2. Create the venv** (only needs to be done once):
|
||||
|
||||
```bash
|
||||
python3 -m venv .venv
|
||||
```
|
||||
|
||||
**3. Activate it** (needs to be done every time you open a new terminal to
|
||||
work on this project):
|
||||
|
||||
- macOS / Linux:
|
||||
```bash
|
||||
source .venv/bin/activate
|
||||
```
|
||||
- Windows (Command Prompt):
|
||||
```bat
|
||||
.venv\Scripts\activate.bat
|
||||
```
|
||||
- Windows (PowerShell):
|
||||
```powershell
|
||||
.venv\Scripts\Activate.ps1
|
||||
```
|
||||
|
||||
Your terminal prompt should now start with `(.venv)` — that means it's
|
||||
active and every `pip install` from here on stays inside this project's
|
||||
private folder.
|
||||
|
||||
**4. Install everything** (the base app + the ML packages), in this order:
|
||||
|
||||
```bash
|
||||
pip install --upgrade pip
|
||||
pip install flask requests twifork
|
||||
pip install --index-url https://download.pytorch.org/whl/cpu torch
|
||||
pip install transformers sentencepiece protobuf
|
||||
```
|
||||
|
||||
The third line is deliberately its own command — installing `torch` the
|
||||
plain way can pull down a multi-gigabyte GPU-enabled build depending on your
|
||||
system, when this app only ever needs the much smaller CPU version. Using
|
||||
that exact command is what keeps the download small.
|
||||
|
||||
**5. Run the app as usual** (make sure `.venv` is still active — you'll see
|
||||
`(.venv)` in your prompt):
|
||||
|
||||
```bash
|
||||
python app.py
|
||||
```
|
||||
|
||||
The **first time** you use the Analytics page after this, it will download
|
||||
the sentiment model itself (~1.1GB) from Hugging Face automatically — this
|
||||
needs an internet connection and can take a few minutes depending on your
|
||||
connection, but only happens once. After that, it's cached on your computer
|
||||
and loads instantly.
|
||||
|
||||
**Next time you come back to work on this project**, you only need step 3
|
||||
again (activate) before running the app — steps 1, 2, and 4 are one-time
|
||||
setup.
|
||||
|
||||
## Run Local Web Server
|
||||
|
||||
```bash
|
||||
@@ -66,33 +269,41 @@ Xquik Dashboard
|
||||
|
||||
Dasboard Home
|
||||
|
||||
<img width="2545" height="1085" alt="image" src="https://github.com/user-attachments/assets/232e02de-4587-4b15-be28-01e09f0e367f" />
|
||||
<img width="2423" height="1217" alt="Image" src="https://github.com/user-attachments/assets/6ba91a78-5845-4e17-87e2-8171da3cec19" />
|
||||
|
||||
Archive
|
||||
|
||||
<img width="2556" height="1108" alt="image" src="https://github.com/user-attachments/assets/a80b4211-d1a8-4a84-9f11-64df3d878372" />
|
||||
<img width="2511" height="1228" alt="Image" src="https://github.com/user-attachments/assets/9e4fab45-edda-45f5-a88c-963e6bfdaeaf" />
|
||||
|
||||
<img width="2551" height="1118" alt="image" src="https://github.com/user-attachments/assets/27c554c0-3c3e-4fae-818d-f9f5aa076fcb" />
|
||||
<img width="2540" height="1235" alt="Image" src="https://github.com/user-attachments/assets/a0c0c42a-d16d-41c9-aac2-c95740eb5dc6" />
|
||||
|
||||
Graph
|
||||
|
||||
<img width="2560" height="1098" alt="image" src="https://github.com/user-attachments/assets/0c5710e8-26eb-4c25-9bd0-4385053e2cb8" />
|
||||
<img width="2556" height="1215" alt="Image" src="https://github.com/user-attachments/assets/98253ea7-2db4-4425-bc43-21be28b99596" />
|
||||
|
||||
Vidio
|
||||
|
||||
<img width="2556" height="1243" alt="image" src="https://github.com/user-attachments/assets/c40bae79-7dd5-48fb-9642-17f27bf06d18" />
|
||||
<img width="2553" height="1300" alt="Image" src="https://github.com/user-attachments/assets/2bb138ac-1605-4c98-a38e-7e0716d9f4d1" />
|
||||
|
||||
Dir Output
|
||||
|
||||
<img width="2131" height="1065" alt="image" src="https://github.com/user-attachments/assets/22c8900f-1418-45aa-a773-f85ac0f3f8a3" />
|
||||
|
||||
Sentiment Analysis
|
||||
|
||||
<img width="2556" height="1243" alt="Image" src="https://github.com/user-attachments/assets/9138564b-9777-4697-a15d-0fda1898c250" />
|
||||
|
||||
<img width="2550" height="1231" alt="Image" src="https://github.com/user-attachments/assets/cddea86b-30ea-4f2d-ad14-c5e8751aa482" />
|
||||
|
||||
<img width="2553" height="1225" alt="Image" src="https://github.com/user-attachments/assets/0b10340a-1503-434a-9015-1719e169a664" />
|
||||
|
||||
# Help
|
||||
|
||||
About SnowflakeID -> Twitter userid : https://en.wikipedia.org/wiki/Snowflake_ID
|
||||
|
||||
About paramater was provided in data and dump with json file type
|
||||
|
||||
<img width="2555" height="1105" alt="image" src="https://github.com/user-attachments/assets/0646b1e8-c46e-49d5-a29f-f511cea9ee47" />
|
||||
<img width="2512" height="1230" alt="Image" src="https://github.com/user-attachments/assets/9d1a3b1e-03a4-45d0-9146-c171eaa6dbce" />
|
||||
|
||||
Xquik API DOC
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import requests as _req
|
||||
from flask import Flask, g, jsonify, render_template, request, Response, stream_with_context, send_from_directory
|
||||
|
||||
import archive as _archive
|
||||
import sentiment as _sentiment
|
||||
from xquik_client import XquikClient, XquikError, load_config
|
||||
from wayback_client import wayback_search, WaybackError
|
||||
from google_cse_client import google_cse_search, GoogleCSEError
|
||||
@@ -17,6 +18,7 @@ from id_forensics import enrich_account_age
|
||||
from cookie_client import (
|
||||
cookie_tweet_search,
|
||||
cookie_follower_explorer,
|
||||
cookie_following_explorer,
|
||||
cookie_post_extractor,
|
||||
cookie_article_extractor,
|
||||
cookie_community_post_extractor,
|
||||
@@ -31,6 +33,7 @@ config = load_config()
|
||||
|
||||
MAX_CONCURRENT_REQUESTS = 3 # parallel execution slots
|
||||
ACQUIRE_TIMEOUT = 15 # seconds to wait before returning 429
|
||||
MAX_COUNT = 2000 # upper bound on a single page's requested item count
|
||||
|
||||
_sem = threading.Semaphore(MAX_CONCURRENT_REQUESTS)
|
||||
|
||||
@@ -47,20 +50,32 @@ _throttle_lock = threading.Lock()
|
||||
_last_call_at: dict = {"cookie": 0.0, "wayback": 0.0}
|
||||
|
||||
_COOKIE_TOOLS = {
|
||||
"tweet_search_extractor", "follower_explorer", "post_extractor",
|
||||
"tweet_search_extractor", "follower_explorer", "following_explorer", "post_extractor",
|
||||
"community_post_extractor", "tweet_replies_extractor",
|
||||
"tweet_retweeters_extractor", "geo_post_extractor",
|
||||
}
|
||||
|
||||
|
||||
def _check_throttle(source: str):
|
||||
"""None if the call may proceed (and starts the next cooldown window);
|
||||
otherwise the number of seconds still left to wait."""
|
||||
def _check_throttle(sources):
|
||||
"""None if the call may proceed (and starts the next cooldown window for
|
||||
every source in `sources`); otherwise the number of seconds still left to
|
||||
wait. Checking is atomic across all requested sources — if any one of
|
||||
them is still cooling down, none of the clocks are touched, so a
|
||||
rejected multi-source call (e.g. multi_source_search, which hits both
|
||||
the cookie and wayback clocks) never partially starts a window for the
|
||||
sources that *did* have room."""
|
||||
if isinstance(sources, str):
|
||||
sources = (sources,)
|
||||
now = time.monotonic()
|
||||
with _throttle_lock:
|
||||
wait = 0.0
|
||||
for source in sources:
|
||||
elapsed = now - _last_call_at[source]
|
||||
if elapsed < _THROTTLE_SECONDS:
|
||||
return round(_THROTTLE_SECONDS - elapsed, 1)
|
||||
wait = max(wait, _THROTTLE_SECONDS - elapsed)
|
||||
if wait:
|
||||
return round(wait, 1)
|
||||
for source in sources:
|
||||
_last_call_at[source] = now
|
||||
return None
|
||||
|
||||
@@ -267,30 +282,61 @@ def _filter_by_date(items: list, from_date: str, to_date: str) -> list:
|
||||
return kept
|
||||
|
||||
|
||||
def _multi_source_search(query: str, count: int, from_date: str = "", to_date: str = "") -> list:
|
||||
# Pagination isn't wired up for multi-source search yet (cookie + wayback
|
||||
# only, per current scope) — grab just the items, discard the cursor.
|
||||
def _multi_source_search(
|
||||
query: str, count: int, from_date: str = "", to_date: str = "", cursor: str | None = None,
|
||||
) -> tuple[list, str | None]:
|
||||
"""Fans out across every source in parallel. cursor (if given) is an
|
||||
opaque JSON object of {source: source_cursor} built from a previous
|
||||
call's returned cursor — each key present in it is a source that still
|
||||
had more to give, so only those get re-queried. xquik has no pagination
|
||||
at all (no cursor concept), so it's only ever queried on the first page
|
||||
(cursor=None); every load-more page after that is cookie/wayback/cse only.
|
||||
A cursor value that doesn't parse as a JSON object is treated as "no
|
||||
cursor" (first page) rather than raising — same tolerant-of-garbage-input
|
||||
posture as the rest of this file's client-supplied-field handling."""
|
||||
twitter_query = _apply_date_operators(query, from_date, to_date)
|
||||
jobs = {
|
||||
"cookie": lambda: cookie_tweet_search(twitter_query, count=count, config=config)[0],
|
||||
"xquik": lambda: XquikClient(config).tweet_search(twitter_query),
|
||||
"wayback": lambda: wayback_search(query, count=count, from_date=from_date, to_date=to_date)[0],
|
||||
|
||||
try:
|
||||
incoming = json.loads(cursor) if cursor else {}
|
||||
if not isinstance(incoming, dict):
|
||||
incoming = {}
|
||||
except (TypeError, ValueError):
|
||||
incoming = {}
|
||||
first_page = not incoming
|
||||
|
||||
jobs = {}
|
||||
if first_page or "cookie" in incoming:
|
||||
c = incoming.get("cookie")
|
||||
jobs["cookie"] = lambda c=c: cookie_tweet_search(twitter_query, count=count, config=config, cursor=c)
|
||||
if first_page:
|
||||
jobs["xquik"] = lambda: (XquikClient(config).tweet_search(twitter_query), None)
|
||||
if first_page or "wayback" in incoming:
|
||||
c = incoming.get("wayback")
|
||||
jobs["wayback"] = lambda c=c: wayback_search(query, count=count, from_date=from_date, to_date=to_date, cursor=c)
|
||||
if first_page or "cse" in incoming:
|
||||
c = incoming.get("cse")
|
||||
# Google has no since:/until: query syntax like Twitter/Wayback do, so
|
||||
# this lane runs unbounded by date — _filter_by_date below keeps
|
||||
# results whose own timestamp it can't verify rather than dropping them.
|
||||
"cse": lambda: google_cse_search(query, count=count, config=config)[0],
|
||||
}
|
||||
jobs["cse"] = lambda c=c: google_cse_search(query, count=count, config=config, cursor=c)
|
||||
|
||||
results = []
|
||||
next_cursor_parts = {}
|
||||
with ThreadPoolExecutor(max_workers=len(jobs)) as pool:
|
||||
futures = {key: pool.submit(fn) for key, fn in jobs.items()}
|
||||
results = []
|
||||
for key in ("cookie", "xquik", "wayback", "cse"): # deterministic display order
|
||||
if key not in futures:
|
||||
continue
|
||||
try:
|
||||
data = futures[key].result()
|
||||
data, next_c = futures[key].result()
|
||||
except Exception:
|
||||
continue # a source failing (missing creds, network, ...) shouldn't sink the others
|
||||
results.extend(_tag_source(data, SOURCE_LABELS[key]))
|
||||
if next_c:
|
||||
next_cursor_parts[key] = next_c
|
||||
|
||||
return _filter_by_date(results, from_date, to_date)
|
||||
next_cursor = json.dumps(next_cursor_parts) if next_cursor_parts else None
|
||||
return _filter_by_date(results, from_date, to_date), next_cursor
|
||||
|
||||
|
||||
# Whitelist: only proxy Twitter's video CDN to prevent SSRF
|
||||
@@ -325,23 +371,27 @@ def run_tool():
|
||||
body = request.get_json(silent=True) or {}
|
||||
tool_type = body.get("toolType")
|
||||
mode = body.get("mode", "api") # "api" | "cookie"
|
||||
count = max(1, min(int(body.get("count", 20)), 200))
|
||||
count = max(1, min(int(body.get("count", 20)), MAX_COUNT))
|
||||
cursor = body.get("cursor") or None # opaque page token from a previous response's nextCursor
|
||||
|
||||
# Cookie/Wayback calls are throttled to one per 5s per source — checked
|
||||
# up front, before taking a concurrency slot, so a request that's about
|
||||
# to be rejected doesn't waste one.
|
||||
throttle_source = None
|
||||
# to be rejected doesn't waste one. multi_source_search fans out to both
|
||||
# cookie and wayback internally, so it's checked (and, once it proceeds,
|
||||
# starts the cooldown) against both clocks at once.
|
||||
throttle_sources = []
|
||||
if mode == "cookie" and tool_type in _COOKIE_TOOLS:
|
||||
throttle_source = "cookie"
|
||||
throttle_sources = ["cookie"]
|
||||
elif tool_type == "wayback_archive_search":
|
||||
throttle_source = "wayback"
|
||||
if throttle_source:
|
||||
wait = _check_throttle(throttle_source)
|
||||
throttle_sources = ["wayback"]
|
||||
elif tool_type == "multi_source_search":
|
||||
throttle_sources = ["cookie", "wayback"]
|
||||
if throttle_sources:
|
||||
wait = _check_throttle(throttle_sources)
|
||||
if wait is not None:
|
||||
return jsonify({
|
||||
"ok": False,
|
||||
"error": f"Please wait {wait}s before the next {throttle_source} request — this protects the account from rate limiting.",
|
||||
"error": f"Please wait {wait}s before the next {'/'.join(throttle_sources)} request — this protects the account from rate limiting.",
|
||||
"retryAfter": wait,
|
||||
}), 429
|
||||
|
||||
@@ -368,6 +418,12 @@ def run_tool():
|
||||
else:
|
||||
data = XquikClient(config).follower_explorer(username)
|
||||
|
||||
elif tool_type == "following_explorer":
|
||||
username = body.get("targetUsername", "")
|
||||
if mode != "cookie":
|
||||
return jsonify({"ok": False, "error": "following_explorer requires cookie mode"}), 400
|
||||
data, next_cursor = cookie_following_explorer(username, count=count, config=config, cursor=cursor)
|
||||
|
||||
elif tool_type == "article_extractor":
|
||||
tweet_id = body.get("targetTweetId", "")
|
||||
if mode == "cookie":
|
||||
@@ -423,7 +479,7 @@ def run_tool():
|
||||
for label, val in (("dateFrom", from_date), ("dateTo", to_date)):
|
||||
if val and not _valid_date8(val):
|
||||
return jsonify({"ok": False, "error": f"{label} must be an 8-digit date (YYYYMMDD)"}), 400
|
||||
data = _multi_source_search(query, count=count, from_date=from_date, to_date=to_date)
|
||||
data, next_cursor = _multi_source_search(query, count=count, from_date=from_date, to_date=to_date, cursor=cursor)
|
||||
|
||||
else:
|
||||
return jsonify({"ok": False, "error": f"Unknown toolType: {tool_type}"}), 400
|
||||
@@ -431,7 +487,7 @@ def run_tool():
|
||||
# Single choke point: every tool's output passes through here, so the
|
||||
# account-age label, fetch timestamp, and tweet_url all show up
|
||||
# everywhere downstream for free — cards, graph nodes, JSON dump, and
|
||||
# archives (once the fields are whitelisted in archive.py's _pick_fields).
|
||||
# archives (which now store the item's full raw shape verbatim).
|
||||
data = enrich_account_age(data)
|
||||
data = _stamp_fetched_at(data)
|
||||
data = _stamp_tweet_url(data)
|
||||
@@ -519,6 +575,22 @@ def archive_list():
|
||||
return jsonify({"ok": True, "archives": _archive.list_all()})
|
||||
|
||||
|
||||
# ── Analytics (sentiment / clustering) ──────────────────────────────────────
|
||||
|
||||
@app.route("/analytics")
|
||||
def analytics_viewer():
|
||||
return render_template("analytics.html")
|
||||
|
||||
|
||||
@app.route("/api/analytics/<archive_id>")
|
||||
def analytics_run(archive_id):
|
||||
results_file = _archive.ARCHIVE_ROOT / archive_id / "results.json"
|
||||
if not results_file.exists():
|
||||
return jsonify({"ok": False, "error": "Archive not found"}), 404
|
||||
items = json.loads(results_file.read_text())
|
||||
return jsonify({"ok": True, **_sentiment.analyze(items)})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
host = config.get("server", "host", fallback="127.0.0.1")
|
||||
port = config.getint("server", "port", fallback=5000)
|
||||
|
||||
@@ -102,24 +102,6 @@ def _download_file(url: str, dest: Path) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _pick_fields(item: dict) -> dict:
|
||||
"""Keep only the fields we want to archive."""
|
||||
keys = ["id", "text", "full_text", "article_text", "user", "user_id",
|
||||
"created_at", "reply_count", "retweet_count", "favorite_count",
|
||||
"view_count", "in_reply_to_tweet_id", "name", "screen_name",
|
||||
"verified", "is_blue_verified",
|
||||
"description", "followers_count", "following_count", "tweet_count",
|
||||
"lat", "lon", "place",
|
||||
"retweeted_text", "retweeted_by_user", "retweeted_by_name",
|
||||
"retweeted_by_bio", "retweeted_at", "retweeted_tweet_id",
|
||||
"iso_date", "original", "statuscode", "mimetype", "length",
|
||||
"archive_url", "post_title", "post_text", "preview_image",
|
||||
"result_url", "display_link", "fetched_at",
|
||||
"source", "account_created", "account_age", "account_age_flag",
|
||||
"account_age_precision"]
|
||||
return {k: item[k] for k in keys if k in item and item[k] is not None}
|
||||
|
||||
|
||||
# ── Core archive runner (runs in background thread) ───────────────────────────
|
||||
|
||||
def _run(archive_id: str, tool_type: str, data, query_info: dict) -> None:
|
||||
@@ -139,7 +121,10 @@ def _run(archive_id: str, tool_type: str, data, query_info: dict) -> None:
|
||||
enriched.append(item)
|
||||
continue
|
||||
|
||||
record = _pick_fields(item)
|
||||
# Archive the item's full raw shape (every field the source API gave
|
||||
# us), not a whitelisted subset — downstream sentiment analysis needs
|
||||
# the raw record, not just the fields the card UI happens to display.
|
||||
record = dict(item)
|
||||
record["tweet_url"] = item.get("tweet_url") or build_tweet_url(item)
|
||||
local_media = []
|
||||
|
||||
|
||||
@@ -71,12 +71,78 @@ def _id_str(val) -> str | None:
|
||||
return str(val) if val is not None else None
|
||||
|
||||
|
||||
def _full_text(t: object) -> str | None:
|
||||
"""twikit's `.text` is Twitter's own legacy `full_text` field — despite the
|
||||
name, X still truncates *that* mid-sentence into a t.co link for anything
|
||||
past the classic length limit (long-form "Note" tweets, e.g. Premium/Blue
|
||||
posts). twikit's `.full_text` PROPERTY (a different thing from the legacy
|
||||
field of the same name) checks the tweet's note_tweet payload first and
|
||||
returns the real complete text when one exists, falling back to `.text`
|
||||
itself otherwise — so it's always at least as complete, strictly more so
|
||||
for long tweets. Swallows a malformed note_tweet shape rather than letting
|
||||
one tweet's data take down the whole batch."""
|
||||
try:
|
||||
return getattr(t, "full_text", None)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _hashtags(t: object) -> list | None:
|
||||
"""Same note_tweet-aware source as _full_text — a long-form tweet's
|
||||
hashtags live in the note_tweet entity set, not the legacy entities twikit
|
||||
falls back to otherwise."""
|
||||
try:
|
||||
tags = getattr(t, "hashtags", None)
|
||||
return tags or None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _leading_reply_mentions(t: object) -> list | None:
|
||||
"""Tapping "Reply" on X auto-prefixes the compose box with every account
|
||||
the reply-chain already has tagged — not just the tweet being replied to
|
||||
— and that prefix is genuinely stored as the literal start of the
|
||||
reply's own full_text. X's own web/app UI never shows it inline though:
|
||||
it reads `display_text_range` (the slice of full_text actually meant to
|
||||
be *shown*) and renders anything before that start index as a separate
|
||||
"Replying to @x @y" line instead. Skipping this meant our raw `text`
|
||||
looked like the replier had typed those @mentions themselves — e.g. a
|
||||
reply that only ever says "Proyek kepentingan, bukan untuk rakyat..."
|
||||
displayed as if it opened with "@regar_op0sisi @prabowo ...", which is
|
||||
exactly what looked wrong compared to the tweet on x.com. This only
|
||||
covers the un-extended legacy text/entities — a long-form Note tweet's
|
||||
entity indices belong to its own separate note_tweet string, which this
|
||||
intentionally does not touch rather than risk slicing the wrong string.
|
||||
Reads twikit's private `_legacy`/`_note_tweet_results` (no public
|
||||
equivalent exists) — same trade-off already made for `_get_more_replies`
|
||||
elsewhere in this file. Returns None rather than raising on any
|
||||
unexpected shape, since this is purely a display aid, never the record
|
||||
of truth `text` already is."""
|
||||
try:
|
||||
if t._note_tweet_results:
|
||||
return None
|
||||
legacy = t._legacy
|
||||
start = (legacy.get("display_text_range") or [0])[0]
|
||||
if not start:
|
||||
return None
|
||||
names = []
|
||||
for m in (legacy.get("entities") or {}).get("user_mentions", []) or []:
|
||||
idx = m.get("indices") or [None, None]
|
||||
if idx[0] is not None and idx[1] is not None and idx[1] <= start:
|
||||
sn = m.get("screen_name")
|
||||
if sn:
|
||||
names.append(sn)
|
||||
return names or None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _tweet_to_dict(t: object) -> dict:
|
||||
user_obj = getattr(t, "user", None)
|
||||
d = {
|
||||
"id": _id_str(getattr(t, "id", None)),
|
||||
"created_at": getattr(t, "created_at", None),
|
||||
"text": getattr(t, "text", None),
|
||||
"text": _full_text(t) or getattr(t, "text", None),
|
||||
"user": getattr(user_obj, "screen_name", None) if user_obj else None,
|
||||
"user_id": _id_str(getattr(user_obj, "id", None)) if user_obj else None,
|
||||
"user_location": getattr(user_obj, "location", None) if user_obj else None,
|
||||
@@ -85,6 +151,11 @@ def _tweet_to_dict(t: object) -> dict:
|
||||
"name": getattr(user_obj, "name", None) if user_obj else None,
|
||||
"verified": getattr(user_obj, "verified", None) if user_obj else None,
|
||||
"is_blue_verified": getattr(user_obj, "is_blue_verified", None) if user_obj else None,
|
||||
# Author's avatar/cover/bio — twikit's embedded user object on a tweet
|
||||
# already carries these, no extra lookup needed.
|
||||
"user_avatar": getattr(user_obj, "profile_image_url", None) if user_obj else None,
|
||||
"user_banner": getattr(user_obj, "profile_banner_url", None) if user_obj else None,
|
||||
"user_bio": getattr(user_obj, "description", None) if user_obj else None,
|
||||
"reply_count": getattr(t, "reply_count", None),
|
||||
"retweet_count": getattr(t, "retweet_count", None),
|
||||
"favorite_count": getattr(t, "favorite_count", None),
|
||||
@@ -94,6 +165,12 @@ def _tweet_to_dict(t: object) -> dict:
|
||||
media = _extract_media(t)
|
||||
if media:
|
||||
d["media"] = media
|
||||
tags = _hashtags(t)
|
||||
if tags:
|
||||
d["hashtags"] = tags
|
||||
mentions = _leading_reply_mentions(t)
|
||||
if mentions:
|
||||
d["reply_to_mentions"] = mentions
|
||||
return d
|
||||
|
||||
|
||||
@@ -103,6 +180,8 @@ def _user_to_dict(u: object) -> dict:
|
||||
"name": getattr(u, "name", None),
|
||||
"screen_name": getattr(u, "screen_name", None),
|
||||
"description": getattr(u, "description", None),
|
||||
"avatar": getattr(u, "profile_image_url", None),
|
||||
"banner": getattr(u, "profile_banner_url", None),
|
||||
"followers_count": getattr(u, "followers_count", None),
|
||||
"following_count": getattr(u, "following_count", None),
|
||||
"tweet_count": getattr(u, "statuses_count", None),
|
||||
@@ -145,6 +224,16 @@ async def _follower_explorer_async(username: str, auth_token: str, ct0: str, cou
|
||||
return [_user_to_dict(u) for u in followers], _next_cursor(followers)
|
||||
|
||||
|
||||
async def _following_explorer_async(username: str, auth_token: str, ct0: str, count: int, cursor: str | None) -> tuple[list, str | None]:
|
||||
"""Who `username` follows — the other half of follower_explorer. twikit
|
||||
exposes this as Client.get_user_following, same shape/cursor contract as
|
||||
get_user_followers, so this mirrors _follower_explorer_async exactly."""
|
||||
client = await _make_client(auth_token, ct0)
|
||||
user = await _resolve_user(client, username)
|
||||
following = await client.get_user_following(str(user.id), count=count, cursor=cursor)
|
||||
return [_user_to_dict(u) for u in following], _next_cursor(following)
|
||||
|
||||
|
||||
async def _post_extractor_async(username: str, auth_token: str, ct0: str, count: int, cursor: str | None) -> tuple[list, str | None]:
|
||||
client = await _make_client(auth_token, ct0)
|
||||
user = await _resolve_user(client, username)
|
||||
@@ -265,6 +354,12 @@ def cookie_follower_explorer(username: str, count: int = 20, config: configparse
|
||||
return asyncio.run(_follower_explorer_async(username, auth, ct0, count, cursor))
|
||||
|
||||
|
||||
def cookie_following_explorer(username: str, count: int = 20, config: configparser.ConfigParser = None, cursor: str | None = None) -> tuple[list, str | None]:
|
||||
cfg = config or load_config()
|
||||
auth, ct0 = _get_creds(cfg)
|
||||
return asyncio.run(_following_explorer_async(username, auth, ct0, count, cursor))
|
||||
|
||||
|
||||
def cookie_post_extractor(username: str, count: int = 20, config: configparser.ConfigParser = None, cursor: str | None = None) -> tuple[list, str | None]:
|
||||
cfg = config or load_config()
|
||||
auth, ct0 = _get_creds(cfg)
|
||||
|
||||
@@ -46,6 +46,14 @@ _META_TAG_RE = re.compile(r"<meta\b[^>]*>", re.IGNORECASE)
|
||||
_ATTR_RE = re.compile(r'''([\w:-]+)\s*=\s*"([^"]*)"|([\w:-]+)\s*=\s*'([^']*)\'''')
|
||||
_TITLE_TAG_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)
|
||||
_TWEET_ID_RE = re.compile(r"/status/(\d+)")
|
||||
_X_HOST_RE = re.compile(r"^https?://(?:www\.)?(?:x|twitter)\.com(?:/|$)", re.IGNORECASE)
|
||||
_PROFILE_RE = re.compile(r"^https?://(?:www\.)?(?:x|twitter)\.com/([^/?#]+)/?(?:\?.*)?$", re.IGNORECASE)
|
||||
# Path segments that look like a profile URL shape but aren't a person/org
|
||||
# account — X's own site-nav pages live at this same depth.
|
||||
_NON_PROFILE_PATHS = {
|
||||
"home", "explore", "notifications", "messages", "i", "search", "settings",
|
||||
"compose", "login", "logout", "signup", "tos", "privacy", "about",
|
||||
}
|
||||
|
||||
|
||||
def _tweet_created_at(url: str) -> str | None:
|
||||
@@ -64,6 +72,26 @@ def _tweet_created_at(url: str) -> str | None:
|
||||
return dt.strftime("%a %b %d %H:%M:%S +0000 %Y")
|
||||
|
||||
|
||||
def _classify_url(url: str) -> str:
|
||||
"""A search result linking to x.com/someone with no other context reads
|
||||
as "a Twitter profile" whether it's actually a specific tweet, a bare
|
||||
profile page, or some other X page entirely — this is a real user report:
|
||||
a result was shown for a keyword match with no way to tell that it was
|
||||
(or wasn't) an actual tweet permalink. Purely a label derived from the
|
||||
URL's own shape; never touches the URL/title/snippet themselves.
|
||||
Returns 'tweet' | 'profile' | 'twitter_other' | 'other'."""
|
||||
if not url:
|
||||
return "other"
|
||||
if _TWEET_ID_RE.search(url):
|
||||
return "tweet"
|
||||
if not _X_HOST_RE.match(url):
|
||||
return "other"
|
||||
m = _PROFILE_RE.match(url)
|
||||
if m and m.group(1).lower() not in _NON_PROFILE_PATHS:
|
||||
return "profile"
|
||||
return "twitter_other"
|
||||
|
||||
|
||||
class GoogleCSEError(Exception):
|
||||
pass
|
||||
|
||||
@@ -95,13 +123,27 @@ def _row_to_record(item: dict) -> dict:
|
||||
record = {}
|
||||
title = item.get("title")
|
||||
if title:
|
||||
record["post_title"] = html.unescape(title).strip()
|
||||
# Kept as serp_title even after _enrich_records below potentially
|
||||
# overwrites post_title with a live re-fetch — X serves bots a
|
||||
# generic/gated page for most URLs, so a live fetch of an X link
|
||||
# often returns less specific content than Google's own SERP/cache
|
||||
# already had. Without this, that overwrite silently threw away the
|
||||
# more useful value with no way to get it back.
|
||||
record["serp_title"] = html.unescape(title).strip()
|
||||
record["post_title"] = record["serp_title"]
|
||||
snippet = item.get("snippet")
|
||||
if snippet:
|
||||
record["post_text"] = html.unescape(snippet).strip()
|
||||
# Google truncates this itself (ends in "…" mid-sentence) — that's
|
||||
# the SERP snippet as Google's own API hands it back, not something
|
||||
# this scraper cuts short. Named `description` (not `serp_snippet`)
|
||||
# to read clearly as "what this page is about" next to post_text
|
||||
# (the live-fetched og:description, which may or may not agree).
|
||||
record["description"] = html.unescape(snippet).strip()
|
||||
record["post_text"] = record["description"]
|
||||
link = item.get("link")
|
||||
if link and _SAFE_URL_RE.match(link):
|
||||
record["result_url"] = link
|
||||
record["content_type"] = _classify_url(link)
|
||||
created_at = _tweet_created_at(link)
|
||||
if created_at:
|
||||
record["created_at"] = created_at # when the post itself was actually made
|
||||
@@ -173,8 +215,13 @@ def _fetch_live_meta(url: str) -> dict:
|
||||
|
||||
|
||||
def _enrich_records(records: list[dict]) -> None:
|
||||
"""Mutates each record in place. Runs in parallel — one slow/dead site
|
||||
shouldn't hold up the rest of the result set."""
|
||||
"""Mutates each record in place — post_title/post_text become "best known
|
||||
value," preferring a fresh live fetch over Google's SERP snapshot when
|
||||
one succeeds. serp_title/description (set in _row_to_record, before this
|
||||
runs) are never touched here, so Google's original values always survive
|
||||
even when this overwrites post_title/post_text with something less
|
||||
useful (X routinely serves bots a generic/gated page). Runs in parallel —
|
||||
one slow/dead site shouldn't hold up the rest of the result set."""
|
||||
candidates = [r for r in records if r.get("result_url")]
|
||||
if not candidates:
|
||||
return
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
flask
|
||||
requests
|
||||
twifork
|
||||
torch
|
||||
transformers
|
||||
sentencepiece
|
||||
protobuf
|
||||
@@ -0,0 +1,368 @@
|
||||
"""Sentiment clustering (pro / neutral / con) for archived X/Twitter data,
|
||||
plus the supporting "who's loudest, what's trending, what words dominate"
|
||||
aggregates — the same shape of dashboard Drone Emprit-style tools give a
|
||||
dataset.
|
||||
|
||||
Two scoring backends, tried in this order:
|
||||
|
||||
1. ML (preferred) — cardiffnlp/twitter-xlm-roberta-base-sentiment, an
|
||||
XLM-RoBERTa model fine-tuned for tweet sentiment across 8 languages
|
||||
(ar/en/fr/de/hi/it/pt/es). Its base pretraining covers ~100 languages, so
|
||||
it degrades gracefully rather than failing outright on a language outside
|
||||
that fine-tuning set — this is what makes the tool usable for an
|
||||
open-source audience that isn't Indonesian-only. Needs torch +
|
||||
transformers (see requirements.txt) and ~1.1GB of model weights
|
||||
downloaded from Hugging Face on first use.
|
||||
2. Lexicon fallback — a hand-built Indonesian positive/negative word list
|
||||
with basic negation handling ("tidak bagus" flips "bagus" from positive
|
||||
to negative). Used automatically whenever torch/transformers aren't
|
||||
installed, so a lightweight install (just requirements.txt's base deps)
|
||||
still has a working — if Indonesian-only — sentiment feature rather than
|
||||
a hard failure. Every scored item exposes exactly which backend produced
|
||||
it (`method` on the analyze() payload), so a classification's provenance
|
||||
is never ambiguous, and the lexicon path additionally exposes the exact
|
||||
words that drove its score for full transparency.
|
||||
|
||||
Neither backend is a ground-truth classifier — short text, sarcasm, and
|
||||
irony all degrade accuracy regardless of approach. Treat results as a
|
||||
starting point for investigation, not a verdict.
|
||||
"""
|
||||
|
||||
import re
|
||||
import threading
|
||||
from collections import Counter
|
||||
|
||||
# Local fallback
|
||||
|
||||
# ── Sentiment lexicon ───────────────────────────────────────────────────────
|
||||
# Indonesian words skew political/social-discourse (matches the kind of
|
||||
# content this tool actually pulls — keyword searches on public affairs,
|
||||
# government programs, public figures) as well as general register.
|
||||
|
||||
# Arr data words
|
||||
# Need to feedback and research to sett the all parameter for each words
|
||||
# Need to help netizen Indo, with slang and words
|
||||
|
||||
POSITIVE_WORDS = {
|
||||
"bagus", "baik", "hebat", "keren", "mantap", "mantul", "top", "terbaik",
|
||||
"sukses", "berhasil", "tepat", "benar", "setuju", "dukung", "dukungan",
|
||||
"mendukung", "apresiasi", "mengapresiasi", "bangga", "membanggakan",
|
||||
"senang", "gembira", "puas", "memuaskan", "bersyukur", "syukur",
|
||||
"alhamdulillah", "semoga", "maju", "kemajuan", "bijak", "bijaksana",
|
||||
"cerdas", "pintar", "amanah", "jujur", "kejujuran", "adil", "keadilan",
|
||||
"peduli", "kepedulian", "bermanfaat", "manfaat", "membantu", "bantuan",
|
||||
"solusi", "prestasi", "berprestasi", "unggul", "keunggulan", "luar biasa",
|
||||
"joss", "jos", "gas terus", "lanjutkan", "sip", "oke", "aman", "tenang",
|
||||
"damai", "sejahtera", "kesejahteraan", "makmur", "kemakmuran", "indah",
|
||||
"cantik", "tampan", "ramah", "sopan", "santun", "hormat", "menghormati",
|
||||
"salut", "kagum", "mengagumkan", "respect", "terharu", "terinspirasi",
|
||||
"inspiratif", "positif", "optimis", "optimisme", "harapan", "berharap",
|
||||
"cinta", "mencintai", "sayang", "suka", "menyukai", "rindu", "kangen",
|
||||
"gemas", "lucu", "menghibur", "menyenangkan", "menginspirasi", "tegas",
|
||||
"berani", "keberanian", "kuat", "gigih", "semangat", "bersemangat",
|
||||
"sukacita", "kompeten", "profesional", "berkualitas", "kualitas",
|
||||
"elegan", "canggih", "inovatif", "inovasi", "kreatif", "efisien",
|
||||
"efektif", "transparan", "transparansi", "akuntabel", "akuntabilitas",
|
||||
"merakyat", "membela rakyat", "pro rakyat", "berpihak pada rakyat",
|
||||
"terpuji", "membanggakan", "gemilang", "cemerlang", "berkah", "istimewa",
|
||||
}
|
||||
|
||||
NEGATIVE_WORDS = {
|
||||
"buruk", "keburukan", "jelek", "gagal", "kegagalan", "bodoh", "tolol",
|
||||
"goblok", "bego", "dungu", "idiot", "korupsi", "korup", "koruptor",
|
||||
"bohong", "kebohongan", "hoax", "menipu", "penipu", "penipuan", "tipu",
|
||||
"curang", "kecurangan", "culas", "zalim", "menzalimi", "kejam",
|
||||
"kekejaman", "jahat", "kejahatan", "rusak", "merusak", "hancur",
|
||||
"menghancurkan", "kacau", "mengacaukan", "parah", "memalukan",
|
||||
"memuakkan", "bejat", "biadab", "tolak", "menolak", "penolakan",
|
||||
"kecewa", "mengecewakan", "kekecewaan", "marah", "kemarahan", "murka",
|
||||
"benci", "membenci", "kebencian", "muak", "jijik", "menjijikkan",
|
||||
"sampah", "anjing", "bangsat", "bajingan", "kampret", "sialan",
|
||||
"kacung", "boneka", "munafik", "kemunafikan", "pengkhianat",
|
||||
"mengkhianati", "khianat", "pengkhianatan", "penjajah", "menjajah",
|
||||
"licik", "licin", "serakah", "keserakahan", "tamak", "otoriter",
|
||||
"diktator", "fasis", "salah", "kesalahan", "blunder", "konyol",
|
||||
"ngawur", "ngaco", "absurd", "aneh", "ironis", "ironi", "tragis",
|
||||
"miris", "prihatin", "keprihatinan", "sedih", "menyedihkan", "susah",
|
||||
"kesusahan", "sulit", "kesulitan", "sengsara", "menderita",
|
||||
"penderitaan", "korban", "dizalimi", "ditindas", "menindas",
|
||||
"penindasan", "kriminal", "pelanggaran", "melanggar", "ilegal",
|
||||
"pungli", "sogok", "menyuap", "disuap", "suap", "nepotisme", "kkn",
|
||||
"provokasi", "provokator", "memprovokasi", "fitnah", "memfitnah",
|
||||
"ancaman", "mengancam", "teror", "intimidasi", "mengintimidasi",
|
||||
"brutal", "kekerasan", "sadis", "tragedi", "bencana", "krisis",
|
||||
"darurat", "gawat", "resah", "keresahan", "meresahkan", "cemas",
|
||||
"kecemasan", "khawatir", "kekhawatiran", "takut", "ketakutan", "geram",
|
||||
"kesal", "jengkel", "dongkol", "malu", "hina", "menghina",
|
||||
"penghinaan", "murahan", "norak", "kampungan", "terbelakang", "mundur",
|
||||
"kemunduran", "ambruk", "bangkrut", "kebangkrutan", "defisit",
|
||||
"terlilit", "terjerat", "terjebak", "cengeng", "lemah", "kelemahan",
|
||||
"pengecut", "penakut", "plin-plan", "labil", "egois", "keegoisan",
|
||||
"sombong", "kesombongan", "angkuh", "arogan", "sok", "songong",
|
||||
"kurang ajar", "tidak becus", "amburadul", "berantakan", "semrawut",
|
||||
"menyengsarakan", "represif", "represi", "diskriminasi",
|
||||
"mendiskriminasi", "rasis", "rasisme", "intoleran", "intoleransi",
|
||||
"penjilat", "gila", "kontol", "memek", "paok", "stress", "goblog",
|
||||
"kepala batu", "oon",
|
||||
}
|
||||
|
||||
# Flips the polarity of a sentiment word found within NEGATION_WINDOW tokens
|
||||
# after it ("tidak bagus" -> negative even though "bagus" alone is positive).
|
||||
# Indonesian often puts more distance between the negation and the word it
|
||||
# actually governs than English does — "ngga ngajarin cara ngmg yg sopan"
|
||||
# ("doesn't teach how to speak politely") is 5 tokens from negation to the
|
||||
# sentiment word it negates. A wider window catches more of those at the
|
||||
# cost of occasionally flipping something the negation wasn't really about;
|
||||
# lexicon scoring is a heuristic either way, this just picks which failure
|
||||
# mode to lean toward.
|
||||
NEGATION_WORDS = {"tidak", "tak", "bukan", "belum", "jangan", "nggak", "ga", "gak", "kagak", "ngga"}
|
||||
NEGATION_WINDOW = 5
|
||||
|
||||
# Boosts a sentiment word's weight rather than changing its polarity.
|
||||
INTENSIFIERS = {"sangat", "banget", "sekali", "sungguh", "amat", "terlalu", "sangatlah"}
|
||||
INTENSIFIER_MULTIPLIER = 1.5
|
||||
|
||||
STOPWORDS = {
|
||||
"yang", "dan", "di", "ke", "dari", "untuk", "dengan", "ini", "itu", "ya",
|
||||
"nya", "adalah", "akan", "saya", "kamu", "kita", "kami", "mereka", "dia",
|
||||
"juga", "saja", "sudah", "belum", "atau", "karena", "jika", "kalau",
|
||||
"agar", "supaya", "pada", "oleh", "dalam", "luar", "atas", "bawah",
|
||||
"antara", "seperti", "sebagai", "tentang", "bahwa", "namun", "tetapi",
|
||||
"tapi", "hingga", "sampai", "sejak", "setelah", "sebelum", "ketika",
|
||||
"saat", "ada", "punya", "milik", "lah", "kah", "pun", "deh", "dong",
|
||||
"sih", "kok", "loh", "nih", "gitu", "gini", "dsb", "dll", "dst", "yg",
|
||||
"utk", "dgn", "krn", "gak", "ga", "nggak", "tak", "tidak", "bukan",
|
||||
"jangan", "apa", "apakah", "siapa", "mengapa", "kenapa", "bagaimana",
|
||||
"dimana", "kapan", "para", "si", "sang", "an", "kan", "in", "the", "is",
|
||||
"are", "was", "were", "be", "been", "being", "to", "of", "for", "on",
|
||||
"with", "as", "by", "at", "an", "a", "rt",
|
||||
} | NEGATION_WORDS | INTENSIFIERS
|
||||
|
||||
_WORD_RE = re.compile(r"[a-zA-ZÀ-ÿ]+(?:-[a-zA-ZÀ-ÿ]+)?")
|
||||
_URL_RE = re.compile(r"https?://\S+")
|
||||
_MENTION_RE = re.compile(r"@\w+")
|
||||
|
||||
|
||||
def _tokenize(text: str) -> list[str]:
|
||||
"""Lowercased word tokens with URLs/@mentions stripped first (both would
|
||||
otherwise pollute the lexicon match and the word cloud with usernames/
|
||||
link fragments neither list has any business scoring)."""
|
||||
if not text:
|
||||
return []
|
||||
cleaned = _URL_RE.sub(" ", text)
|
||||
cleaned = _MENTION_RE.sub(" ", cleaned)
|
||||
return [w.lower() for w in _WORD_RE.findall(cleaned)]
|
||||
|
||||
|
||||
def score_text(text: str) -> dict:
|
||||
"""Returns {label, score, matches} for one piece of text. label is one of
|
||||
"pro" / "neutral" / "con". matches lists (word, polarity, weight) for
|
||||
every lexicon hit, so a classification can be inspected rather than
|
||||
trusted blindly — a defining trait of a rule-based classifier is that you
|
||||
CAN see exactly why it decided what it decided."""
|
||||
tokens = _tokenize(text)
|
||||
score = 0.0
|
||||
matches = []
|
||||
|
||||
for i, tok in enumerate(tokens):
|
||||
polarity = 1 if tok in POSITIVE_WORDS else -1 if tok in NEGATIVE_WORDS else 0
|
||||
if polarity == 0:
|
||||
continue
|
||||
|
||||
weight = 1.0
|
||||
# Negation: any negation word in the preceding window flips polarity.
|
||||
window_start = max(0, i - NEGATION_WINDOW)
|
||||
if any(t in NEGATION_WORDS for t in tokens[window_start:i]):
|
||||
polarity = -polarity
|
||||
# Intensifier: any intensifier immediately before boosts magnitude
|
||||
# (checked after negation so "tidak sangat bagus" still flips first).
|
||||
if i > 0 and tokens[i - 1] in INTENSIFIERS:
|
||||
weight = INTENSIFIER_MULTIPLIER
|
||||
|
||||
contribution = polarity * weight
|
||||
score += contribution
|
||||
matches.append({"word": tok, "polarity": "pro" if polarity > 0 else "con", "weight": weight})
|
||||
|
||||
label = "pro" if score > 0 else "con" if score < 0 else "neutral"
|
||||
return {"label": label, "score": round(score, 2), "matches": matches}
|
||||
|
||||
|
||||
def _item_text(item: dict) -> str:
|
||||
"""The text worth scoring/tokenizing for a given archived record —
|
||||
varies by which tool produced it (a tweet's own text vs. a Wayback/CSE
|
||||
page's scraped title+description vs. a bare user's bio)."""
|
||||
parts = [
|
||||
item.get("text"), item.get("full_text"), item.get("article_text"),
|
||||
item.get("post_title"), item.get("post_text"), item.get("description"),
|
||||
]
|
||||
return " ".join(p for p in parts if p)
|
||||
|
||||
|
||||
def _item_author(item: dict) -> str | None:
|
||||
return item.get("screen_name") or item.get("user") or item.get("name")
|
||||
|
||||
|
||||
def _item_engagement(item: dict) -> int:
|
||||
total = 0
|
||||
for k in ("reply_count", "retweet_count", "favorite_count"):
|
||||
v = item.get(k)
|
||||
if isinstance(v, (int, float)):
|
||||
total += v
|
||||
return total
|
||||
|
||||
|
||||
def word_frequencies(items: list[dict], top_n: int = 60) -> list[dict]:
|
||||
"""Word-cloud data: [{word, count}], most frequent first. Stopwords and
|
||||
single-character tokens are dropped; everything else counts regardless
|
||||
of whether it happened to be in the sentiment lexicon."""
|
||||
counts = Counter()
|
||||
for item in items:
|
||||
for tok in _tokenize(_item_text(item)):
|
||||
if len(tok) < 3 or tok in STOPWORDS:
|
||||
continue
|
||||
counts[tok] += 1
|
||||
return [{"word": w, "count": c} for w, c in counts.most_common(top_n)]
|
||||
|
||||
|
||||
def top_users(items: list[dict], top_n: int = 20) -> list[dict]:
|
||||
"""Who shows up most often across the archive — every record with an
|
||||
identifiable author counts once, regardless of whether it's a tweet, a
|
||||
reply, a retweeter entry, or a bare follower/following record. Carries
|
||||
along the most recently seen avatar/name for that handle so the
|
||||
dashboard can show a face, not just a bare count."""
|
||||
counts: Counter = Counter()
|
||||
display: dict[str, dict] = {}
|
||||
for item in items:
|
||||
handle = item.get("screen_name") or item.get("user")
|
||||
if not handle:
|
||||
continue
|
||||
counts[handle] += 1
|
||||
display[handle] = {
|
||||
"screen_name": handle,
|
||||
"name": item.get("name"),
|
||||
"avatar": item.get("avatar") or item.get("user_avatar"),
|
||||
"verified": item.get("verified"),
|
||||
"is_blue_verified": item.get("is_blue_verified"),
|
||||
}
|
||||
ranked = []
|
||||
for handle, count in counts.most_common(top_n):
|
||||
ranked.append({**display[handle], "count": count})
|
||||
return ranked
|
||||
|
||||
|
||||
def top_engagement(items: list[dict], top_n: int = 15) -> list[dict]:
|
||||
"""Which posts drove the most reply+retweet+favorite activity — "paling
|
||||
ramai" (busiest/most-discussed), not just most recent."""
|
||||
scored = [(_item_engagement(it), it) for it in items if _item_text(it)]
|
||||
scored.sort(key=lambda pair: pair[0], reverse=True)
|
||||
return [{"engagement": score, "item": it} for score, it in scored[:top_n] if score > 0]
|
||||
|
||||
|
||||
# ── ML backend (preferred) ───────────────────────────────────────────────────
|
||||
# Lazy-loaded: importing torch/transformers and loading ~1.1GB of weights is
|
||||
# slow, and both packages are optional (requirements.txt notes how to add
|
||||
# them) — doing this at module import time would slow down every single use
|
||||
# of this app, including ones that never touch analytics, and would hard-crash
|
||||
# an install that skipped the ML deps entirely instead of just falling back.
|
||||
_ML_MODEL_NAME = "cardiffnlp/twitter-xlm-roberta-base-sentiment"
|
||||
_ML_LABEL_MAP = {"positive": "pro", "negative": "con", "neutral": "neutral"}
|
||||
_ml_pipeline = None
|
||||
_ml_unavailable = False # sticky once loading fails — stop retrying every call
|
||||
_ml_load_lock = threading.Lock()
|
||||
|
||||
|
||||
def _get_ml_pipeline():
|
||||
global _ml_pipeline, _ml_unavailable
|
||||
if _ml_pipeline is not None or _ml_unavailable:
|
||||
return _ml_pipeline
|
||||
with _ml_load_lock:
|
||||
if _ml_pipeline is not None or _ml_unavailable: # re-check post-lock
|
||||
return _ml_pipeline
|
||||
try:
|
||||
from transformers import pipeline
|
||||
_ml_pipeline = pipeline("sentiment-analysis", model=_ML_MODEL_NAME)
|
||||
except Exception:
|
||||
# Missing torch/transformers, no internet for the first-time
|
||||
# model download, out of memory, ... any of these should fall
|
||||
# back to the lexicon scorer rather than take the whole
|
||||
# analytics endpoint down.
|
||||
_ml_unavailable = True
|
||||
_ml_pipeline = None
|
||||
return _ml_pipeline
|
||||
|
||||
|
||||
def _score_texts_ml(texts: list[str]) -> list[dict] | None:
|
||||
"""Batch-scores every text in one call (far faster on CPU than one
|
||||
pipeline call per item). Returns None if the model isn't available, so
|
||||
the caller falls back to the lexicon scorer instead. `score` is signed
|
||||
(positive for pro, negative for con, 0 for neutral) to match the
|
||||
lexicon backend's convention; `confidence` carries the model's own
|
||||
unsigned probability for the label it picked."""
|
||||
clf = _get_ml_pipeline()
|
||||
if clf is None:
|
||||
return None
|
||||
raw = clf(texts, truncation=True, batch_size=16)
|
||||
results = []
|
||||
for r in raw:
|
||||
label = _ML_LABEL_MAP.get(str(r.get("label", "")).lower(), "neutral")
|
||||
confidence = float(r.get("score", 0.0))
|
||||
signed = confidence if label == "pro" else -confidence if label == "con" else 0.0
|
||||
results.append({
|
||||
"label": label, "score": round(signed, 3),
|
||||
"confidence": round(confidence, 3), "matches": [],
|
||||
})
|
||||
return results
|
||||
|
||||
|
||||
def analyze(items: list[dict]) -> dict:
|
||||
"""Full analytics payload for one archive's worth of raw items."""
|
||||
if not isinstance(items, list):
|
||||
items = [items]
|
||||
# archive.py's own _run() passes non-dict entries through as-is rather
|
||||
# than dropping them (see its "if not isinstance(item, dict)" branch),
|
||||
# so a saved archive can legitimately contain a stray non-dict item —
|
||||
# every function below assumes dict.get(), so those get filtered here
|
||||
# once rather than each helper needing its own isinstance guard.
|
||||
items = [it for it in items if isinstance(it, dict)]
|
||||
|
||||
text_items, texts = [], []
|
||||
for item in items:
|
||||
text = _item_text(item)
|
||||
if text: # skip e.g. a bare follower/following record with no post text of its own
|
||||
text_items.append(item)
|
||||
texts.append(text)
|
||||
|
||||
ml_results = _score_texts_ml(texts) if texts else None
|
||||
method = "ml" if ml_results is not None else "lexicon"
|
||||
if ml_results is None:
|
||||
ml_results = [score_text(t) for t in texts]
|
||||
|
||||
sentiment_counts = {"pro": 0, "neutral": 0, "con": 0}
|
||||
scored_items = []
|
||||
for item, text, result in zip(text_items, texts, ml_results):
|
||||
sentiment_counts[result["label"]] += 1
|
||||
scored_items.append({
|
||||
"label": result["label"],
|
||||
"score": result["score"],
|
||||
"confidence": result.get("confidence"),
|
||||
"matches": result.get("matches", []),
|
||||
"author": _item_author(item),
|
||||
"text": text,
|
||||
"item": item,
|
||||
})
|
||||
|
||||
total = len(scored_items)
|
||||
return {
|
||||
"method": method,
|
||||
"total_items": len(items),
|
||||
"total_scored": total,
|
||||
"sentiment_counts": sentiment_counts,
|
||||
"sentiment_pct": {
|
||||
k: round(v / total * 100, 1) if total else 0.0
|
||||
for k, v in sentiment_counts.items()
|
||||
},
|
||||
"scored_items": scored_items,
|
||||
"word_freq": word_frequencies(items),
|
||||
"top_users": top_users(items),
|
||||
"top_engagement": top_engagement(items),
|
||||
}
|
||||
@@ -8,7 +8,7 @@
|
||||
// Arr params
|
||||
|
||||
const PRIORITY = [
|
||||
'source', 'account_age_flag', 'account_age', 'account_created',
|
||||
'source', 'content_type', 'account_age_flag', 'account_age', 'account_created',
|
||||
'user', 'screen_name', 'name', 'user_id', 'username',
|
||||
'verified', 'is_blue_verified',
|
||||
'text', 'full_text', 'article_text', 'post_title', 'post_text', 'content', 'title', 'description', 'bio',
|
||||
@@ -18,9 +18,23 @@ const PRIORITY = [
|
||||
'retweeted_by_user', 'retweeted_by_name', 'retweeted_text', 'retweeted_by_bio', 'retweeted_at', 'retweeted_tweet_id',
|
||||
'lat', 'lon', 'place', 'user_location',
|
||||
'tweet_url', 'archive_url', 'result_url', 'preview_image', 'display_link',
|
||||
'serp_title',
|
||||
'iso_date', 'original', 'statuscode', 'mimetype', 'length',
|
||||
];
|
||||
|
||||
// content_type: what kind of X/Twitter page a Google CSE / Wayback result
|
||||
// actually is — a search hit for a keyword could be a specific tweet, a
|
||||
// bare profile page, some other X page, or (Google CSE only) a site off X
|
||||
// entirely. Cookie/Xquik records don't carry this field at all (a tweet
|
||||
// search result there is unambiguously always a tweet), so it only ever
|
||||
// shows up for the two sources it's meant to disambiguate.
|
||||
const CONTENT_TYPE_LABELS = {
|
||||
tweet: 'Tweet',
|
||||
profile: 'Profile page',
|
||||
twitter_other: 'Other X/Twitter page',
|
||||
other: 'External page (non-X)',
|
||||
};
|
||||
|
||||
// Which fields link out to a fresh extraction for that tweet — same anchor
|
||||
// behavior whether you're looking at a live result or a saved archive.
|
||||
const DRILLABLE = {
|
||||
|
||||
@@ -4,10 +4,13 @@
|
||||
<div class="param-row"><code class="param-key">id / user_id</code> Twitter's internal numeric ID for the tweet / account</div>
|
||||
<div class="param-row"><code class="param-key">user / screen_name</code> The @handle</div>
|
||||
<div class="param-row"><code class="param-key">name</code> Display name</div>
|
||||
<div class="param-row"><code class="param-key">avatar / banner</code> Profile photo / cover photo — direct CDN links, cookie mode only. On a tweet/reply record these are the <em>author's</em> photos, named <code class="param-key">user_avatar</code>/<code class="param-key">user_banner</code> instead so they don't collide with the tweet's own fields</div>
|
||||
|
||||
<div class="param-heading">Content</div>
|
||||
<div class="param-row"><code class="param-key">text / full_text / article_text</code> The post's own text</div>
|
||||
<div class="param-row"><code class="param-key">description</code> Profile bio</div>
|
||||
<div class="param-row"><code class="param-key">text / full_text / article_text</code> The post's own complete text — cookie mode always resolves the real full text even for long-form "Note" tweets that X itself truncates in its legacy API field</div>
|
||||
<div class="param-row"><code class="param-key">reply_to_mentions</code> The @handles X auto-prepended to a reply's text when it was composed (everyone already tagged in that conversation) — x.com's own UI hides these from the visible tweet body and shows them as a separate "Replying to" line instead, which is exactly what every card/panel here does with this field too. The raw <code class="param-key">text</code> itself still contains them untouched</div>
|
||||
<div class="param-row"><code class="param-key">hashtags</code> Hashtags used in the post</div>
|
||||
<div class="param-row"><code class="param-key">description</code> Profile bio (a user/follower/following record), or Google's own SERP snippet for that result (a CSE record) — which one depends on whether the record is a person or a web result</div>
|
||||
<div class="param-row"><code class="param-key">created_at</code> When the <em>post itself</em> was actually made. Same field, every source: Cookie/Xquik get it straight from the API; Wayback and Google CSE decode it from the tweet id's Snowflake bits whenever the result links to a tweet permalink (absent otherwise — e.g. a profile page or a non-X result, where there's no post id to decode)</div>
|
||||
<div class="param-row"><code class="param-key">fetched_at</code> When <em>this tool</em> pulled the record — same field, every source (Cookie, Xquik, Wayback, Google CSE), so results are comparable side by side. Not to be confused with <code class="param-key">created_at</code> (the post's own creation time) or <code class="param-key">iso_date</code> (a Wayback snapshot's capture time)</div>
|
||||
|
||||
@@ -47,9 +50,11 @@
|
||||
<div class="param-row"><code class="param-key">statuscode / mimetype / length</code> HTTP status / content type / size of the snapshot</div>
|
||||
<div class="param-row"><code class="param-key">post_title / post_text / preview_image</code> Scraped from the archived page's own meta tags (og:/twitter: tags, or <title>/<meta name="description"> on older captures)</div>
|
||||
|
||||
<div class="param-heading">Google CSE</div>
|
||||
<div class="param-row"><code class="param-key">post_title</code> Result title from Google</div>
|
||||
<div class="param-row"><code class="param-key">post_text</code> The page's own og:/twitter:/meta description, fetched live from <code class="param-key">result_url</code> — not Google's own SERP snippet, which is usually clipped mid-sentence. Falls back to Google's snippet only if that live fetch fails</div>
|
||||
<div class="param-heading">Google CSE / Wayback</div>
|
||||
<div class="param-row"><code class="param-key">content_type</code> What kind of X/Twitter page a result actually is — <code class="param-key">tweet</code> (a specific permalink) / <code class="param-key">profile</code> (a bare profile page) / <code class="param-key">twitter_other</code> (some other X page) / <code class="param-key">other</code> (a non-X site, Google CSE only). Derived purely from the result URL's own shape — a keyword match isn't always a specific tweet, and this is how to tell without opening the link</div>
|
||||
<div class="param-row"><code class="param-key">serp_title</code> Google's own SERP result title — always kept as-is, even if <code class="param-key">post_title</code> below gets replaced by a fresher live fetch</div>
|
||||
<div class="param-row"><code class="param-key">description</code> Google's own SERP snippet for this result — Google truncates this itself (ends mid-sentence in "…"), that's not this tool cutting it short. Kept untouched alongside <code class="param-key">post_text</code> below since a live re-fetch of an X link often gets a less specific, bot-facing page instead</div>
|
||||
<div class="param-row"><code class="param-key">post_title / post_text</code> "Best known" title/description — starts as Google's own SERP values, replaced by a live fetch of the page's own og:/twitter:/meta tags when one succeeds. See <code class="param-key">serp_title</code>/<code class="param-key">description</code> above for what Google originally had, in case the live fetch made things worse rather than better</div>
|
||||
<div class="param-row"><code class="param-key">result_url</code> The live page Google indexed, shown as its full raw address</div>
|
||||
<div class="param-row"><code class="param-key">display_link</code> The result's domain, as shown in Google's own results</div>
|
||||
<div class="param-row"><code class="param-key">preview_image</code> Thumbnail — from Google's own indexed metadata, or the live page's og:/twitter:image when Google didn't have one</div>
|
||||
@@ -59,9 +64,9 @@
|
||||
|
||||
<div class="param-heading">Graph node types (graph page only)</div>
|
||||
<div class="param-row"><code class="param-key">Search root</code> The diamond node — the query you ran</div>
|
||||
<div class="param-row"><code class="param-key">Tweet</code> A tweet/post returned by search or by expanding a node</div>
|
||||
<div class="param-row"><code class="param-key">Tweet</code> A tweet/post returned by search, by Expand Posts on a user node, or by expanding another node</div>
|
||||
<div class="param-row"><code class="param-key">Reply</code> A tweet fetched via Expand Replies on a tweet node</div>
|
||||
<div class="param-row"><code class="param-key">User / Retweeter</code> A person — from Follower Explorer, or via Expand Retweets on a tweet node</div>
|
||||
<div class="param-row"><code class="param-key">User / Retweeter</code> A person — from Follower/Following Explorer, via Expand Retweets/Followers/Following on a node, or via View Author Profile on a tweet/reply (pulled from that tweet's already-fetched author info, no extra request). Once on canvas, select it and Expand Posts/Followers/Following works the same regardless of how it got there</div>
|
||||
<div class="param-row"><code class="param-key">Wayback snapshot</code> An archived-page result, from Wayback Archive Search or the Wayback portion of Multi-Source Search</div>
|
||||
<div class="param-row"><code class="param-key">Web result</code> A Google CSE result, from the Google CSE portion of Multi-Source Search</div>
|
||||
<div class="param-row"><code class="param-key">Viewed</code> Turns a node solid white once you've clicked it — a visual "already looked at this" marker, not part of the underlying data</div>
|
||||
|
||||
@@ -0,0 +1,816 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Jieyab89 SOCMINT X — Analytics</title>
|
||||
<style>
|
||||
:root {
|
||||
--bg: #0f1117;
|
||||
--surface: #1a1d27;
|
||||
--surface2: #21253a;
|
||||
--border: #2a2d3a;
|
||||
--text: #e8eaf0;
|
||||
--muted: #8890a4;
|
||||
--accent: #5865f2;
|
||||
--accent-bg: #1e2240;
|
||||
--success: #22c55e;
|
||||
--success-bg: #052011;
|
||||
--danger: #ef4444;
|
||||
--danger-bg: #1c0505;
|
||||
--font: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||||
}
|
||||
* { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
body { background: var(--bg); color: var(--text); font-family: var(--font); font-size: 14px; line-height: 1.6; min-height: 100vh; }
|
||||
|
||||
/* ── Header (same shell as the other 3 pages) ── */
|
||||
header {
|
||||
padding: 14px 24px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
background: var(--surface);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 100;
|
||||
}
|
||||
header h1 { font-size: 14px; font-weight: 600; }
|
||||
header .sep { color: var(--border); }
|
||||
header .sub { color: var(--muted); font-size: 12px; flex: 1; }
|
||||
.hamburger-btn {
|
||||
margin-left: auto;
|
||||
background: none;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 5px;
|
||||
color: var(--muted);
|
||||
cursor: pointer;
|
||||
padding: 5px 9px;
|
||||
font-size: 16px;
|
||||
line-height: 1;
|
||||
transition: all 0.15s;
|
||||
}
|
||||
.hamburger-btn:hover { color: var(--text); border-color: var(--accent); }
|
||||
.nav-menu {
|
||||
position: absolute;
|
||||
top: 100%;
|
||||
right: 12px;
|
||||
margin-top: 6px;
|
||||
background: var(--surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 7px;
|
||||
padding: 5px;
|
||||
min-width: 175px;
|
||||
box-shadow: 0 8px 28px rgba(0,0,0,0.5);
|
||||
z-index: 200;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 1px;
|
||||
}
|
||||
.nav-menu.hidden { display: none; }
|
||||
.nav-menu a {
|
||||
padding: 7px 12px;
|
||||
border-radius: 5px;
|
||||
color: var(--muted);
|
||||
text-decoration: none;
|
||||
font-size: 13px;
|
||||
transition: all 0.12s;
|
||||
display: block;
|
||||
}
|
||||
.nav-menu a:hover { color: var(--text); background: var(--accent-bg); }
|
||||
.nav-menu a.current { color: var(--accent); background: var(--accent-bg); }
|
||||
.nav-divider { border: none; border-top: 1px solid var(--border); margin: 3px 0; }
|
||||
|
||||
.back-to-top {
|
||||
position: fixed;
|
||||
bottom: 14px;
|
||||
right: 14px;
|
||||
width: 36px;
|
||||
height: 36px;
|
||||
border-radius: 50%;
|
||||
background: var(--surface);
|
||||
border: 1px solid var(--border);
|
||||
color: #FFFFFF;
|
||||
font-size: 17px;
|
||||
cursor: pointer;
|
||||
z-index: 251;
|
||||
box-shadow: 0 4px 14px rgba(0,0,0,0.4);
|
||||
transition: all 0.15s;
|
||||
display: none;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
.back-to-top:hover { border-color: var(--accent); }
|
||||
.back-to-top.visible { display: flex; }
|
||||
|
||||
/* ── Layout ── */
|
||||
.layout { display: grid; grid-template-columns: 270px 1fr; min-height: calc(100vh - 49px); }
|
||||
@media (max-width: 900px) { .layout { grid-template-columns: 1fr; } }
|
||||
|
||||
/* ── Sidebar (archive picker) ── */
|
||||
.sidebar { border-right: 1px solid var(--border); background: var(--surface); display: flex; flex-direction: column; }
|
||||
.sidebar-head { padding: 14px 16px 10px; border-bottom: 1px solid var(--border); display: flex; align-items: center; justify-content: space-between; }
|
||||
.sidebar-title { font-size: 11px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.06em; color: var(--muted); }
|
||||
.sidebar-count { font-size: 11px; color: var(--muted); background: var(--bg); padding: 2px 7px; border-radius: 10px; }
|
||||
.sidebar-search {
|
||||
width: calc(100% - 32px); margin: 10px 16px 6px; padding: 6px 10px;
|
||||
background: var(--bg); border: 1px solid var(--border); color: var(--text);
|
||||
font-family: var(--font); font-size: 12px; border-radius: 6px; transition: border-color 0.15s;
|
||||
}
|
||||
.sidebar-search:focus { outline: none; border-color: var(--accent); }
|
||||
.sidebar-search::placeholder { color: var(--muted); }
|
||||
.archive-list { overflow-y: auto; flex: 1; padding: 8px; }
|
||||
.archive-entry { padding: 10px 12px; border-radius: 7px; border: 1px solid transparent; cursor: pointer; transition: all 0.12s; margin-bottom: 4px; }
|
||||
.archive-entry:hover { background: var(--surface2); border-color: var(--border); }
|
||||
.archive-entry.active { background: var(--accent-bg); border-color: var(--accent); }
|
||||
.ae-tool { font-size: 12px; font-weight: 600; color: var(--text); white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
|
||||
.ae-meta { font-size: 11px; color: var(--muted); margin-top: 2px; }
|
||||
.ae-date { font-size: 10.5px; color: var(--muted); opacity: 0.75; margin-top: 2px; }
|
||||
.ae-stats { font-size: 11px; color: var(--muted); margin-top: 3px; }
|
||||
.no-archives { padding: 40px 16px; text-align: center; color: var(--muted); font-size: 13px; }
|
||||
|
||||
/* ── Main viewer ── */
|
||||
.viewer { overflow-y: auto; padding: 20px 22px 60px; }
|
||||
.empty-state { color: var(--muted); font-size: 13px; padding: 80px 20px; text-align: center; }
|
||||
.section { margin-bottom: 26px; }
|
||||
.section-title {
|
||||
font-size: 11px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.06em;
|
||||
color: var(--muted); margin-bottom: 10px; display: flex; align-items: center; gap: 8px;
|
||||
}
|
||||
.section-note { font-size: 11px; color: var(--muted); font-weight: 400; text-transform: none; letter-spacing: 0; }
|
||||
.btn-toolbar {
|
||||
margin-left: auto;
|
||||
padding: 5px 12px;
|
||||
background: transparent;
|
||||
color: var(--muted);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 6px;
|
||||
font-family: var(--font);
|
||||
font-size: 11px;
|
||||
font-weight: 500;
|
||||
text-transform: none;
|
||||
letter-spacing: 0;
|
||||
cursor: pointer;
|
||||
transition: all 0.15s;
|
||||
}
|
||||
.btn-toolbar:hover { color: var(--text); border-color: var(--accent); }
|
||||
|
||||
/* ── Sentiment breakdown ── */
|
||||
.sentiment-bar {
|
||||
display: flex; height: 34px; border-radius: 8px; overflow: hidden;
|
||||
border: 1px solid var(--border); margin-bottom: 12px;
|
||||
}
|
||||
.sentiment-seg { display: flex; align-items: center; justify-content: center; font-size: 12px; font-weight: 600; color: var(--bg); transition: filter 0.15s; }
|
||||
.sentiment-seg:hover { filter: brightness(1.12); }
|
||||
.sentiment-seg.seg-pro { background: var(--success); }
|
||||
.sentiment-seg.seg-neutral { background: var(--muted); }
|
||||
.sentiment-seg.seg-con { background: var(--danger); }
|
||||
.sentiment-seg.empty { color: transparent; }
|
||||
.sentiment-tiles { display: grid; grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); gap: 10px; }
|
||||
.stile {
|
||||
background: var(--surface); border: 1px solid var(--border); border-radius: 8px;
|
||||
padding: 12px 14px; cursor: pointer; transition: all 0.12s;
|
||||
}
|
||||
.stile:hover { border-color: #3a3d50; }
|
||||
.stile.active { border-color: var(--accent); background: var(--accent-bg); }
|
||||
.stile-label { font-size: 11px; font-weight: 600; display: flex; align-items: center; gap: 6px; }
|
||||
.stile-dot { width: 9px; height: 9px; border-radius: 50%; flex-shrink: 0; }
|
||||
.stile-dot.dot-pro { background: var(--success); }
|
||||
.stile-dot.dot-neutral { background: var(--muted); }
|
||||
.stile-dot.dot-con { background: var(--danger); }
|
||||
.stile-value { font-size: 22px; font-weight: 700; margin-top: 4px; }
|
||||
.stile-pct { font-size: 11px; color: var(--muted); margin-top: 1px; }
|
||||
|
||||
/* ── Cards grid (top users / top engagement) ── */
|
||||
.grid-2 { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; }
|
||||
@media (max-width: 900px) { .grid-2 { grid-template-columns: 1fr; } }
|
||||
|
||||
.user-list { background: var(--surface); border: 1px solid var(--border); border-radius: 8px; overflow: hidden; }
|
||||
.user-row { display: flex; align-items: center; gap: 10px; padding: 9px 12px; border-bottom: 1px solid rgba(42,45,58,0.6); }
|
||||
.user-row:last-child { border-bottom: none; }
|
||||
.user-rank { font-size: 11px; color: var(--muted); width: 18px; flex-shrink: 0; text-align: right; }
|
||||
.user-avatar {
|
||||
width: 30px; height: 30px; border-radius: 50%; object-fit: cover; flex-shrink: 0;
|
||||
border: 1px solid var(--border); background: var(--bg);
|
||||
}
|
||||
.user-avatar-fallback { display: flex; align-items: center; justify-content: center; font-size: 12px; font-weight: 700; color: var(--muted); }
|
||||
.user-identity { flex: 1; min-width: 0; }
|
||||
.user-name { font-size: 12.5px; font-weight: 600; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
|
||||
.user-handle { font-size: 11px; color: var(--muted); }
|
||||
.user-count { font-size: 12px; color: var(--accent); font-weight: 600; flex-shrink: 0; }
|
||||
|
||||
.engagement-list { background: var(--surface); border: 1px solid var(--border); border-radius: 8px; overflow: hidden; }
|
||||
.eng-row { padding: 10px 12px; border-bottom: 1px solid rgba(42,45,58,0.6); }
|
||||
.eng-row:last-child { border-bottom: none; }
|
||||
.eng-top { display: flex; align-items: center; gap: 8px; margin-bottom: 4px; }
|
||||
.eng-handle { font-size: 12px; font-weight: 600; color: var(--text); }
|
||||
.eng-score { font-size: 11px; color: var(--accent); font-weight: 600; margin-left: auto; white-space: nowrap; }
|
||||
.eng-text { font-size: 12px; color: var(--text); opacity: 0.9; overflow: hidden; display: -webkit-box; -webkit-line-clamp: 2; -webkit-box-orient: vertical; }
|
||||
|
||||
/* ── Word cloud ── */
|
||||
.wordcloud {
|
||||
background: var(--surface); border: 1px solid var(--border); border-radius: 8px;
|
||||
padding: 18px 20px; display: flex; flex-wrap: wrap; align-items: baseline; gap: 8px 14px;
|
||||
}
|
||||
.wc-word { color: var(--text); font-weight: 600; line-height: 1; white-space: nowrap; cursor: default; transition: color 0.12s; }
|
||||
.wc-word:hover { color: var(--accent); }
|
||||
|
||||
/* ── Sentiment badge (shared by tiles/list) ── */
|
||||
.sent-badge {
|
||||
display: inline-block; font-size: 10px; font-weight: 700; padding: 2px 8px;
|
||||
border-radius: 20px; border: 1px solid; text-transform: uppercase; letter-spacing: 0.03em; flex-shrink: 0;
|
||||
}
|
||||
.sent-badge.sent-pro { color: var(--success); border-color: #14532d; background: var(--success-bg); }
|
||||
.sent-badge.sent-neutral { color: var(--muted); border-color: var(--border); background: var(--bg); }
|
||||
.sent-badge.sent-con { color: var(--danger); border-color: #7f1d1d; background: var(--danger-bg); }
|
||||
|
||||
/* ── Item browser ── */
|
||||
.item-toolbar { display: flex; align-items: center; gap: 10px; margin-bottom: 10px; flex-wrap: wrap; }
|
||||
.item-search {
|
||||
padding: 6px 10px; background: var(--surface); border: 1px solid var(--border); color: var(--text);
|
||||
font-family: var(--font); font-size: 12px; border-radius: 6px; width: 220px;
|
||||
}
|
||||
.item-search:focus { outline: none; border-color: var(--accent); }
|
||||
.item-count { font-size: 11px; color: var(--muted); }
|
||||
.item-list { display: flex; flex-direction: column; gap: 8px; }
|
||||
.item-card { background: var(--surface); border: 1px solid var(--border); border-radius: 8px; padding: 12px 14px; }
|
||||
.item-card-top { display: flex; align-items: center; gap: 8px; margin-bottom: 6px; flex-wrap: wrap; }
|
||||
.item-author { font-size: 12px; font-weight: 600; color: var(--text); }
|
||||
.item-score { font-size: 11px; color: var(--muted); margin-left: auto; }
|
||||
.item-text { font-size: 13px; color: var(--text); word-break: break-word; white-space: pre-wrap; }
|
||||
.item-matches { margin-top: 8px; display: flex; flex-wrap: wrap; gap: 5px; }
|
||||
.match-chip {
|
||||
font-size: 10px; padding: 1px 7px; border-radius: 10px; border: 1px solid;
|
||||
}
|
||||
.match-chip.mc-pro { color: var(--success); border-color: #14532d; background: var(--success-bg); }
|
||||
.match-chip.mc-con { color: var(--danger); border-color: #7f1d1d; background: var(--danger-bg); }
|
||||
|
||||
.spinner {
|
||||
display: inline-block; width: 13px; height: 13px; border: 2px solid var(--border);
|
||||
border-top-color: var(--accent); border-radius: 50%; animation: spin 0.7s linear infinite; vertical-align: -2px;
|
||||
}
|
||||
@keyframes spin { to { transform: rotate(360deg); } }
|
||||
|
||||
.method-note {
|
||||
font-size: 11px; color: var(--muted); line-height: 1.6; background: var(--surface);
|
||||
border: 1px solid var(--border); border-radius: 8px; padding: 10px 14px; margin-bottom: 22px;
|
||||
}
|
||||
|
||||
/* ── Social network graph ── */
|
||||
.sna-legend {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 14px;
|
||||
font-size: 11px;
|
||||
color: var(--muted);
|
||||
margin-bottom: 10px;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
.sna-legend-item { display: flex; align-items: center; gap: 5px; }
|
||||
.sna-legend-dot { width: 9px; height: 9px; border-radius: 50%; flex-shrink: 0; }
|
||||
.sna-legend-dot.dot-pro { background: var(--success); }
|
||||
.sna-legend-dot.dot-neutral { background: var(--muted); }
|
||||
.sna-legend-dot.dot-con { background: var(--danger); }
|
||||
.sna-wrap {
|
||||
display: none;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 8px;
|
||||
overflow: hidden;
|
||||
background: var(--surface);
|
||||
}
|
||||
.sna-wrap.visible { display: block; }
|
||||
.sna-graph {
|
||||
width: 100%;
|
||||
height: 480px;
|
||||
background: var(--bg);
|
||||
}
|
||||
.sna-node-info {
|
||||
display: none;
|
||||
padding: 12px 16px;
|
||||
border-top: 1px solid var(--border);
|
||||
align-items: center;
|
||||
gap: 14px;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
.sna-node-info.visible { display: flex; }
|
||||
.sna-node-identity { min-width: 140px; }
|
||||
.sna-node-name { font-size: 13px; font-weight: 600; color: var(--text); }
|
||||
.sna-node-handle { font-size: 11px; color: var(--muted); }
|
||||
.sna-node-stats { display: flex; gap: 6px; flex-wrap: wrap; }
|
||||
.sna-node-total { font-size: 11px; color: var(--muted); margin-left: auto; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<header>
|
||||
<h1>Jieyab89 SOCMINT X</h1>
|
||||
<span class="sep">/</span>
|
||||
<span class="sub">Analytics — sentiment clustering, top actors, word frequency</span>
|
||||
<button class="hamburger-btn" id="navToggle" aria-label="Menu">☰</button>
|
||||
<nav id="navMenu" class="nav-menu hidden">
|
||||
<a href="/">Home</a>
|
||||
<a href="/graph">Graph</a>
|
||||
<a href="/archives">Archives</a>
|
||||
<hr class="nav-divider">
|
||||
<a href="/analytics" class="current">Analytics</a>
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<button id="backToTop" class="back-to-top" aria-label="Back to top" title="Back to top">↑</button>
|
||||
|
||||
<div class="layout">
|
||||
<aside class="sidebar">
|
||||
<div class="sidebar-head">
|
||||
<span class="sidebar-title">Archives</span>
|
||||
<span class="sidebar-count" id="archiveCount">0</span>
|
||||
</div>
|
||||
<input type="text" class="sidebar-search" id="archiveSearch" placeholder="Filter archives...">
|
||||
<div class="archive-list" id="archiveList">
|
||||
<div class="no-archives">Loading…</div>
|
||||
</div>
|
||||
</aside>
|
||||
|
||||
<main class="viewer" id="viewer">
|
||||
<div class="empty-state">Pick an archive on the left to run sentiment clustering, top-actor, and word-frequency analysis on its raw data.</div>
|
||||
</main>
|
||||
</div>
|
||||
|
||||
<script src="https://unpkg.com/cytoscape@3.28.1/dist/cytoscape.min.js" crossorigin="anonymous"></script>
|
||||
<script nonce="{{ g.csp_nonce }}">
|
||||
(function () {
|
||||
'use strict';
|
||||
|
||||
function esc(s) { return String(s).replace(/[&<>"']/g, m => ({'&':'&','<':'<','>':'>','"':'"',"'":'''})[m]); }
|
||||
|
||||
// Same allowlist index.html/graph.html apply to avatar/banner URLs — https
|
||||
// only, no quote/angle-bracket/whitespace/paren characters. Not strictly
|
||||
// required for a plain <img src> (no second CSS-parsing pass involved the
|
||||
// way a background-image url() has), but kept consistent with the other
|
||||
// two pages rather than being the one spot that skips it.
|
||||
function isSafeImageUrl(u) {
|
||||
return typeof u === 'string' && /^https:\/\/[^\s'"<>()]+$/.test(u);
|
||||
}
|
||||
|
||||
// Hamburger nav
|
||||
(function () {
|
||||
const btn = document.getElementById('navToggle');
|
||||
const menu = document.getElementById('navMenu');
|
||||
btn.addEventListener('click', e => { e.stopPropagation(); menu.classList.toggle('hidden'); });
|
||||
document.addEventListener('click', () => menu.classList.add('hidden'));
|
||||
menu.addEventListener('click', e => e.stopPropagation());
|
||||
})();
|
||||
|
||||
// Back to top — the page itself is what scrolls (the sidebar/viewer split
|
||||
// is a grid column layout, not a fixed-height scroll pane).
|
||||
(function () {
|
||||
const btn = document.getElementById('backToTop');
|
||||
window.addEventListener('scroll', () => {
|
||||
btn.classList.toggle('visible', window.scrollY > 400);
|
||||
});
|
||||
btn.addEventListener('click', () => {
|
||||
window.scrollTo({ top: 0, behavior: 'smooth' });
|
||||
});
|
||||
})();
|
||||
|
||||
const archiveList = document.getElementById('archiveList');
|
||||
const archiveCount = document.getElementById('archiveCount');
|
||||
const archiveSearch = document.getElementById('archiveSearch');
|
||||
const viewer = document.getElementById('viewer');
|
||||
|
||||
let allArchives = [];
|
||||
let activeId = null;
|
||||
let activeData = null; // last /api/analytics/<id> payload
|
||||
let activeFilter = null; // 'pro' | 'neutral' | 'con' | null (item browser filter)
|
||||
|
||||
// ── Sidebar ──────────────────────────────────────────────────────────────────
|
||||
|
||||
async function loadSidebar() {
|
||||
const res = await fetch('/api/archive/list');
|
||||
const json = await res.json();
|
||||
|
||||
if (!json.ok || !json.archives.length) {
|
||||
allArchives = [];
|
||||
archiveList.innerHTML = '<div class="no-archives">No archives yet.<br>Save one from the search page, graph, or geo search first.</div>';
|
||||
archiveCount.textContent = '0';
|
||||
return;
|
||||
}
|
||||
|
||||
allArchives = json.archives;
|
||||
renderArchiveList(archiveSearch.value);
|
||||
}
|
||||
|
||||
function queryLabel(q) {
|
||||
return (q && (q.searchQuery || q.targetUsername || q.targetTweetId || q.targetCommunityId || q.query)) || '';
|
||||
}
|
||||
|
||||
function archiveMatchText(a) {
|
||||
return [a.tool || '', queryLabel(a.query || {}), a.id || ''].join(' ').toLowerCase();
|
||||
}
|
||||
|
||||
function renderArchiveList(filterText = '') {
|
||||
const q = filterText.toLowerCase().trim();
|
||||
const filtered = q ? allArchives.filter(a => archiveMatchText(a).includes(q)) : allArchives;
|
||||
|
||||
archiveCount.textContent = q ? `${filtered.length} / ${allArchives.length}` : String(allArchives.length);
|
||||
|
||||
if (!filtered.length) {
|
||||
archiveList.innerHTML = '<div class="no-archives">No archives match your search.</div>';
|
||||
return;
|
||||
}
|
||||
|
||||
archiveList.innerHTML = filtered.map(entryHtml).join('');
|
||||
archiveList.querySelectorAll('.archive-entry').forEach(el => {
|
||||
el.classList.toggle('active', el.dataset.id === activeId);
|
||||
el.addEventListener('click', () => loadAnalytics(el.dataset.id));
|
||||
});
|
||||
}
|
||||
|
||||
archiveSearch.addEventListener('input', () => renderArchiveList(archiveSearch.value));
|
||||
|
||||
function entryHtml(a) {
|
||||
const tool = (a.tool || 'unknown').replace(/_/g, ' ');
|
||||
const date = a.archived_at ? a.archived_at.replace('T', ' ') : '—';
|
||||
const query = queryLabel(a.query || {});
|
||||
return `
|
||||
<div class="archive-entry" data-id="${esc(a.id)}">
|
||||
<div class="ae-tool">${esc(tool)}</div>
|
||||
<div class="ae-date">${esc(date)}</div>
|
||||
${query ? `<div class="ae-meta">${esc(query)}</div>` : ''}
|
||||
<div class="ae-stats">${a.total_items || 0} items</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Analysis ──────────────────────────────────────────────────────────────────
|
||||
|
||||
async function loadAnalytics(id) {
|
||||
activeId = id;
|
||||
activeFilter = null;
|
||||
archiveList.querySelectorAll('.archive-entry').forEach(el => el.classList.toggle('active', el.dataset.id === id));
|
||||
|
||||
viewer.innerHTML = '<div class="empty-state"><span class="spinner"></span> Analyzing…</div>';
|
||||
|
||||
try {
|
||||
const res = await fetch('/api/analytics/' + encodeURIComponent(id));
|
||||
const json = await res.json();
|
||||
if (!json.ok) { viewer.innerHTML = `<div class="empty-state">${esc(json.error || 'Analysis failed')}</div>`; return; }
|
||||
activeData = json;
|
||||
renderDashboard();
|
||||
} catch (e) {
|
||||
viewer.innerHTML = `<div class="empty-state">${esc(String(e))}</div>`;
|
||||
}
|
||||
}
|
||||
|
||||
const SENT_LABELS = { pro: 'Pro', neutral: 'Neutral', con: 'Con' };
|
||||
let snaCy = null; // lazily built the first time "View as Graph" is clicked
|
||||
|
||||
function renderDashboard() {
|
||||
// A previous archive's graph instance (if any) is about to lose its
|
||||
// container to the innerHTML rewrite below — destroy it cleanly first
|
||||
// rather than leaving a detached Cytoscape instance running.
|
||||
if (snaCy) { snaCy.destroy(); snaCy = null; }
|
||||
|
||||
const d = activeData;
|
||||
if (!d.total_scored) {
|
||||
viewer.innerHTML = '<div class="empty-state">This archive has no text content to analyze (e.g. a pure follower/following list, or Wayback/CSE-only records with nothing scraped).</div>';
|
||||
return;
|
||||
}
|
||||
|
||||
viewer.innerHTML = `
|
||||
<div class="method-note">
|
||||
${d.method === 'ml'
|
||||
? 'Sentiment is scored by a multilingual model (XLM-RoBERTa, fine-tuned on tweets across 8 languages and reasonably capable well beyond those) — works on non-Indonesian text, not just the lexicon fallback below. Still a heuristic, not ground truth: short text, sarcasm and irony all still degrade accuracy. Click a tile below to see which items landed in it and the model\'s confidence per item.'
|
||||
: 'Sentiment is scored against an Indonesian positive/negative word lexicon with negation handling (e.g. "tidak bagus" flips to con) — a transparent rule-based heuristic, Indonesian-only. Install torch+transformers (see requirements.txt) for multilingual ML-based scoring instead. Click a tile below to see exactly which items landed in it and which words drove each score.'}
|
||||
${d.total_items - d.total_scored > 0 ? `${d.total_items - d.total_scored} of ${d.total_items} record(s) had no text to score (bare follower/following/user entries) and are excluded below.` : ''}
|
||||
</div>
|
||||
|
||||
<div class="section">
|
||||
<div class="section-title">Sentiment clustering <span class="section-note">${d.total_scored} scored</span></div>
|
||||
${sentimentBarHtml(d)}
|
||||
<div class="sentiment-tiles">
|
||||
${sentimentTileHtml('pro', d)}
|
||||
${sentimentTileHtml('neutral', d)}
|
||||
${sentimentTileHtml('con', d)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="grid-2 section">
|
||||
<div>
|
||||
<div class="section-title">Most active accounts <span class="section-note">by items in this archive</span></div>
|
||||
${topUsersHtml(d.top_users)}
|
||||
</div>
|
||||
<div>
|
||||
<div class="section-title">Most engagement <span class="section-note">replies + retweets + likes</span></div>
|
||||
${topEngagementHtml(d.top_engagement)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="section">
|
||||
<div class="section-title">
|
||||
Social network graph <span class="section-note">accounts, sized by activity, colored by sentiment mix</span>
|
||||
<button type="button" id="btnSnaGraph" class="btn-toolbar">View as Graph</button>
|
||||
</div>
|
||||
<div class="sna-legend">
|
||||
<span class="sna-legend-item"><span class="sna-legend-dot dot-pro"></span>Pro</span>
|
||||
<span class="sna-legend-item"><span class="sna-legend-dot dot-neutral"></span>Neutral</span>
|
||||
<span class="sna-legend-item"><span class="sna-legend-dot dot-con"></span>Con</span>
|
||||
<span class="sna-legend-item">Node size = post volume · edges = reply relationships found within this archive · click a node for details</span>
|
||||
</div>
|
||||
<div class="sna-wrap" id="snaWrap">
|
||||
<div class="sna-graph" id="snaGraph"></div>
|
||||
<div class="sna-node-info" id="snaNodeInfo"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="section">
|
||||
<div class="section-title">Word frequency</div>
|
||||
${wordCloudHtml(d.word_freq)}
|
||||
</div>
|
||||
|
||||
<div class="section">
|
||||
<div class="item-toolbar">
|
||||
<div class="section-title" style="margin-bottom:0">Items <span class="item-count" id="itemCount"></span></div>
|
||||
<input type="text" class="item-search" id="itemSearch" placeholder="Search text or author...">
|
||||
</div>
|
||||
<div class="item-list" id="itemList"></div>
|
||||
</div>
|
||||
`;
|
||||
|
||||
document.querySelectorAll('.stile').forEach(el => {
|
||||
el.addEventListener('click', () => {
|
||||
const label = el.dataset.label;
|
||||
activeFilter = activeFilter === label ? null : label;
|
||||
document.querySelectorAll('.stile').forEach(t => t.classList.toggle('active', t.dataset.label === activeFilter));
|
||||
renderItemList();
|
||||
});
|
||||
});
|
||||
document.getElementById('itemSearch').addEventListener('input', renderItemList);
|
||||
renderItemList();
|
||||
|
||||
document.getElementById('btnSnaGraph').addEventListener('click', toggleSnaGraph);
|
||||
}
|
||||
|
||||
// ── Social network graph ─────────────────────────────────────────────────────
|
||||
// Accounts as nodes — each one's fill is a native Cytoscape pie split by its
|
||||
// own pro/neutral/con proportions (a Maltego-style at-a-glance indicator:
|
||||
// mostly-green vs mostly-red vs mixed is readable across the whole graph at
|
||||
// once, not just one account at a time), sized by how many items in this
|
||||
// archive that account produced. Edges are reply relationships derived
|
||||
// entirely from data already loaded in scored_items — no extra request:
|
||||
// when an item's in_reply_to_tweet_id matches another item's own id within
|
||||
// this same archive, that's a real reply link between the two authors.
|
||||
function toggleSnaGraph() {
|
||||
const wrap = document.getElementById('snaWrap');
|
||||
const btn = document.getElementById('btnSnaGraph');
|
||||
const showing = wrap.classList.toggle('visible');
|
||||
btn.textContent = showing ? 'Hide Graph' : 'View as Graph';
|
||||
if (showing && !snaCy) buildSnaGraph();
|
||||
}
|
||||
|
||||
function buildSnaGraph() {
|
||||
const d = activeData;
|
||||
const accounts = {}; // handle -> { pro, neutral, con, total, name, avatar }
|
||||
const idToAuthor = {}; // raw item id -> handle, for resolving reply edges
|
||||
|
||||
d.scored_items.forEach(it => {
|
||||
const raw = it.item || {};
|
||||
const handle = it.author || raw.screen_name || raw.user;
|
||||
if (!handle) return;
|
||||
if (!accounts[handle]) {
|
||||
accounts[handle] = {
|
||||
pro: 0, neutral: 0, con: 0, total: 0,
|
||||
name: raw.name || '', avatar: raw.avatar || raw.user_avatar || '',
|
||||
};
|
||||
}
|
||||
accounts[handle][it.label]++;
|
||||
accounts[handle].total++;
|
||||
if (raw.id != null) idToAuthor[String(raw.id)] = handle;
|
||||
});
|
||||
|
||||
const handles = Object.keys(accounts);
|
||||
const graphEl = document.getElementById('snaGraph');
|
||||
if (!handles.length) {
|
||||
graphEl.innerHTML = '<div class="empty-state" style="padding:24px">No identifiable accounts to graph.</div>';
|
||||
return;
|
||||
}
|
||||
|
||||
const seenEdge = new Set();
|
||||
const edges = [];
|
||||
d.scored_items.forEach(it => {
|
||||
const raw = it.item || {};
|
||||
const replyTo = raw.in_reply_to_tweet_id;
|
||||
const fromHandle = it.author || raw.screen_name || raw.user;
|
||||
if (!replyTo || !fromHandle) return;
|
||||
const toHandle = idToAuthor[String(replyTo)];
|
||||
if (!toHandle || toHandle === fromHandle) return;
|
||||
const key = fromHandle + '→' + toHandle;
|
||||
if (seenEdge.has(key)) return;
|
||||
seenEdge.add(key);
|
||||
edges.push({ data: { id: 'e_' + edges.length, source: fromHandle, target: toHandle, label: 'replied to' } });
|
||||
});
|
||||
|
||||
const counts = handles.map(h => accounts[h].total);
|
||||
const maxCount = Math.max(...counts);
|
||||
const minCount = Math.min(...counts);
|
||||
const sameSize = maxCount === minCount;
|
||||
|
||||
const nodes = handles.map(h => {
|
||||
const a = accounts[h];
|
||||
const pct = n => a.total ? Math.round((n / a.total) * 100) : 0;
|
||||
return {
|
||||
data: {
|
||||
id: h,
|
||||
label: '@' + h + ' (' + a.total + ')',
|
||||
name: a.name, avatar: a.avatar,
|
||||
pro: a.pro, neutral: a.neutral, con: a.con, total: a.total,
|
||||
proPct: pct(a.pro), neutralPct: pct(a.neutral), conPct: pct(a.con),
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
const sizeStyle = sameSize
|
||||
? { width: 42, height: 42 }
|
||||
: { width: `mapData(total, ${minCount}, ${maxCount}, 26, 74)`, height: `mapData(total, ${minCount}, ${maxCount}, 26, 74)` };
|
||||
|
||||
snaCy = cytoscape({
|
||||
container: graphEl,
|
||||
elements: { nodes, edges },
|
||||
minZoom: 0.15,
|
||||
maxZoom: 3,
|
||||
style: [
|
||||
{ selector: 'node', style: Object.assign({
|
||||
shape: 'ellipse',
|
||||
label: 'data(label)',
|
||||
color: '#e8eaf0',
|
||||
'font-size': '9px',
|
||||
'min-zoomed-font-size': 6,
|
||||
'text-valign': 'bottom',
|
||||
'text-margin-y': '6px',
|
||||
'text-wrap': 'ellipsis',
|
||||
'text-max-width': '90px',
|
||||
'border-width': 2,
|
||||
'border-color': '#2a2d3a',
|
||||
'pie-size': '100%',
|
||||
'pie-1-background-color': '#22c55e',
|
||||
'pie-1-background-size': 'data(proPct)',
|
||||
'pie-2-background-color': '#8890a4',
|
||||
'pie-2-background-size': 'data(neutralPct)',
|
||||
'pie-3-background-color': '#ef4444',
|
||||
'pie-3-background-size': 'data(conPct)',
|
||||
}, sizeStyle) },
|
||||
{ selector: 'edge', style: {
|
||||
'line-color': '#FFFFFF',
|
||||
'line-opacity': 0.45,
|
||||
width: 1,
|
||||
'target-arrow-color': '#FFFFFF',
|
||||
'target-arrow-shape': 'triangle',
|
||||
'arrow-scale': 0.8,
|
||||
'curve-style': 'bezier',
|
||||
label: 'data(label)',
|
||||
'font-size': '7px',
|
||||
color: '#8890a4',
|
||||
'text-background-color': '#0f1117',
|
||||
'text-background-opacity': 0.85,
|
||||
'text-background-padding': '2px',
|
||||
}},
|
||||
{ selector: 'node:selected', style: {
|
||||
'border-width': 3,
|
||||
'border-color': '#5865f2',
|
||||
}},
|
||||
],
|
||||
layout: {
|
||||
name: 'cose', animate: true, animationDuration: 450,
|
||||
nodeRepulsion: 9000, idealEdgeLength: 90, gravity: 0.4, numIter: 1000, fit: true, padding: 30,
|
||||
},
|
||||
});
|
||||
|
||||
snaCy.on('tap', 'node', evt => showSnaNodeInfo(evt.target.data()));
|
||||
}
|
||||
|
||||
function showSnaNodeInfo(nd) {
|
||||
const panel = document.getElementById('snaNodeInfo');
|
||||
panel.classList.add('visible');
|
||||
panel.innerHTML = `
|
||||
<div class="sna-node-identity">
|
||||
<div class="sna-node-name">${esc(nd.name || nd.id)}</div>
|
||||
<div class="sna-node-handle">@${esc(nd.id)}</div>
|
||||
</div>
|
||||
<div class="sna-node-stats">
|
||||
<span class="sent-badge sent-pro">Pro ${nd.pro}</span>
|
||||
<span class="sent-badge sent-neutral">Neutral ${nd.neutral}</span>
|
||||
<span class="sent-badge sent-con">Con ${nd.con}</span>
|
||||
</div>
|
||||
<div class="sna-node-total">${nd.total} item(s) total</div>
|
||||
`;
|
||||
}
|
||||
|
||||
function sentimentBarHtml(d) {
|
||||
const total = d.total_scored || 1;
|
||||
const segs = ['pro', 'neutral', 'con'].map(label => {
|
||||
const pct = d.sentiment_pct[label] || 0;
|
||||
if (!pct) return '';
|
||||
return `<div class="sentiment-seg seg-${label}" style="flex:${d.sentiment_counts[label]}" title="${SENT_LABELS[label]}: ${pct}%">${pct >= 8 ? pct + '%' : ''}</div>`;
|
||||
}).join('');
|
||||
return `<div class="sentiment-bar">${segs}</div>`;
|
||||
}
|
||||
|
||||
function sentimentTileHtml(label, d) {
|
||||
return `
|
||||
<div class="stile" data-label="${label}">
|
||||
<div class="stile-label"><span class="stile-dot dot-${label}"></span>${SENT_LABELS[label]}</div>
|
||||
<div class="stile-value">${d.sentiment_counts[label] || 0}</div>
|
||||
<div class="stile-pct">${d.sentiment_pct[label] || 0}% of scored items</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function topUsersHtml(users) {
|
||||
if (!users || !users.length) return '<div class="empty-state" style="padding:24px">No identifiable authors in this archive.</div>';
|
||||
return '<div class="user-list">' + users.map((u, i) => {
|
||||
const avatarUrl = isSafeImageUrl(u.avatar) ? u.avatar : '';
|
||||
const avatar = avatarUrl
|
||||
? `<img class="user-avatar" src="${esc(avatarUrl)}" alt="" loading="lazy" referrerpolicy="no-referrer">`
|
||||
: `<div class="user-avatar user-avatar-fallback">${esc(((u.name || u.screen_name || '?').charAt(0)).toUpperCase())}</div>`;
|
||||
return `
|
||||
<div class="user-row">
|
||||
<span class="user-rank">${i + 1}</span>
|
||||
${avatar}
|
||||
<div class="user-identity">
|
||||
<div class="user-name">${esc(u.name || u.screen_name)}</div>
|
||||
<div class="user-handle">@${esc(u.screen_name)}</div>
|
||||
</div>
|
||||
<span class="user-count">${u.count}×</span>
|
||||
</div>`;
|
||||
}).join('') + '</div>';
|
||||
}
|
||||
|
||||
function topEngagementHtml(items) {
|
||||
if (!items || !items.length) return '<div class="empty-state" style="padding:24px">No engagement data (reply/retweet/favorite counts) in this archive.</div>';
|
||||
return '<div class="engagement-list">' + items.map(e => {
|
||||
const it = e.item;
|
||||
const handle = it.screen_name || it.user || it.name || 'unknown';
|
||||
const text = (it.text || it.full_text || it.post_title || it.post_text || it.description || '').trim();
|
||||
return `
|
||||
<div class="eng-row">
|
||||
<div class="eng-top">
|
||||
<span class="eng-handle">@${esc(handle)}</span>
|
||||
<span class="eng-score">${e.engagement.toLocaleString()} interactions</span>
|
||||
</div>
|
||||
<div class="eng-text">${esc(text)}</div>
|
||||
</div>`;
|
||||
}).join('') + '</div>';
|
||||
}
|
||||
|
||||
function wordCloudHtml(words) {
|
||||
if (!words || !words.length) return '<div class="empty-state" style="padding:24px">Not enough text to build a word list.</div>';
|
||||
const max = Math.max(...words.map(w => w.count));
|
||||
const min = Math.min(...words.map(w => w.count));
|
||||
const span = Math.max(1, max - min);
|
||||
return '<div class="wordcloud">' + words.map(w => {
|
||||
// Font size scaled by frequency (13px..38px) — size is the encoding here,
|
||||
// not color, so this stays legible without inventing a new color ramp.
|
||||
const t = (w.count - min) / span;
|
||||
const size = Math.round(13 + t * 25);
|
||||
const opacity = (0.55 + t * 0.45).toFixed(2);
|
||||
return `<span class="wc-word" style="font-size:${size}px;opacity:${opacity}" title="${w.count}×">${esc(w.word)}</span>`;
|
||||
}).join('') + '</div>';
|
||||
}
|
||||
|
||||
// ── Item browser ─────────────────────────────────────────────────────────────
|
||||
|
||||
function renderItemList() {
|
||||
const d = activeData;
|
||||
const q = (document.getElementById('itemSearch').value || '').toLowerCase().trim();
|
||||
|
||||
let items = d.scored_items;
|
||||
if (activeFilter) items = items.filter(it => it.label === activeFilter);
|
||||
if (q) items = items.filter(it => (it.text || '').toLowerCase().includes(q) || (it.author || '').toLowerCase().includes(q));
|
||||
|
||||
document.getElementById('itemCount').textContent = `${items.length} / ${d.scored_items.length}`;
|
||||
|
||||
const list = document.getElementById('itemList');
|
||||
if (!items.length) { list.innerHTML = '<div class="empty-state" style="padding:24px">No items match.</div>'; return; }
|
||||
|
||||
list.innerHTML = items.map(it => {
|
||||
const matches = (it.matches || []).map(m =>
|
||||
`<span class="match-chip mc-${m.polarity}">${esc(m.word)}</span>`
|
||||
).join('');
|
||||
const scoreLabel = typeof it.confidence === 'number'
|
||||
? `${Math.round(it.confidence * 100)}% confidence`
|
||||
: `score ${it.score > 0 ? '+' : ''}${it.score}`;
|
||||
return `
|
||||
<div class="item-card">
|
||||
<div class="item-card-top">
|
||||
<span class="item-author">${it.author ? '@' + esc(it.author) : 'Unknown'}</span>
|
||||
<span class="sent-badge sent-${it.label}">${SENT_LABELS[it.label]}</span>
|
||||
<span class="item-score">${scoreLabel}</span>
|
||||
</div>
|
||||
<div class="item-text">${esc(it.text)}</div>
|
||||
${matches ? `<div class="item-matches">${matches}</div>` : ''}
|
||||
</div>`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
loadSidebar();
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -31,7 +31,9 @@
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
position: relative;
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 100;
|
||||
}
|
||||
header h1 { font-size: 14px; font-weight: 600; }
|
||||
header .sep { color: var(--border); }
|
||||
@@ -96,6 +98,27 @@
|
||||
z-index: 251;
|
||||
transition: all 0.15s;
|
||||
}
|
||||
.back-to-top {
|
||||
position: fixed;
|
||||
bottom: 14px;
|
||||
right: 14px;
|
||||
width: 36px;
|
||||
height: 36px;
|
||||
border-radius: 50%;
|
||||
background: var(--surface);
|
||||
border: 1px solid var(--border);
|
||||
color: #FFFFFF;
|
||||
font-size: 17px;
|
||||
cursor: pointer;
|
||||
z-index: 251;
|
||||
box-shadow: 0 4px 14px rgba(0,0,0,0.4);
|
||||
transition: all 0.15s;
|
||||
display: none;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
.back-to-top:hover { border-color: var(--accent); }
|
||||
.back-to-top.visible { display: flex; }
|
||||
.help-toggle:hover { color: var(--text); border-color: var(--accent); }
|
||||
.help-popup {
|
||||
position: fixed;
|
||||
@@ -211,6 +234,7 @@
|
||||
font-weight: 500;
|
||||
}
|
||||
.ae-meta { font-size: 11px; color: var(--muted); margin-top: 2px; }
|
||||
.ae-date { font-size: 10.5px; color: var(--muted); opacity: 0.75; margin-top: 2px; }
|
||||
.ae-stats { font-size: 11px; color: var(--muted); display: flex; gap: 8px; margin-top: 3px; }
|
||||
.ae-stat { display: flex; align-items: center; gap: 3px; }
|
||||
.no-archives {
|
||||
@@ -337,6 +361,23 @@
|
||||
.age-badge.age-recent { color: #f59e0b; border-color: #78350f; background: #1c0e02; }
|
||||
.age-badge.age-established { color: var(--success); border-color: #14532d; background: #052011; }
|
||||
|
||||
/* What kind of X/Twitter page a Google CSE / Wayback result actually is —
|
||||
a keyword match linking to x.com/someone reads as "just a profile" with
|
||||
no way to tell a tweet permalink apart from a bare profile page or an
|
||||
unrelated page otherwise. */
|
||||
.content-type-badge {
|
||||
display: inline-block;
|
||||
font-size: 11px;
|
||||
font-weight: 500;
|
||||
padding: 2px 9px;
|
||||
border-radius: 20px;
|
||||
border: 1px solid var(--border);
|
||||
}
|
||||
.content-type-badge.ct-tweet { color: var(--success); border-color: #14532d; background: #052011; }
|
||||
.content-type-badge.ct-profile { color: #a78bfa; border-color: #4c3a8f; background: #1e1535; }
|
||||
.content-type-badge.ct-twitter_other { color: #f59e0b; border-color: #78350f; background: #1c0e02; }
|
||||
.content-type-badge.ct-other { color: var(--muted); border-color: var(--border); background: var(--bg); }
|
||||
|
||||
/* ── Media ── */
|
||||
.card-media {
|
||||
display: flex;
|
||||
@@ -411,17 +452,24 @@
|
||||
<a href="/graph">Graph</a>
|
||||
<hr class="nav-divider">
|
||||
<a href="/archives" class="current">Archives</a>
|
||||
<a href="/analytics">Analytics</a>
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<!-- Legend / help popup (floats over the page, bottom-left) -->
|
||||
<button id="helpToggle" class="help-toggle" aria-label="Legend / help" title="Legend / help">?</button>
|
||||
<button id="backToTop" class="back-to-top" aria-label="Back to top" title="Back to top">↑</button>
|
||||
<div id="helpPopup" class="help-popup hidden">
|
||||
<h4>Source</h4>
|
||||
<div class="help-row"><span class="source-badge src-cookie">Twitter Cookie</span> Live via your cookie session</div>
|
||||
<div class="help-row"><span class="source-badge src-xquik">Xquik API</span> Live via the xquik API</div>
|
||||
<div class="help-row"><span class="source-badge src-wayback">Wayback Machine</span> Archived snapshot</div>
|
||||
<div class="help-row"><span class="source-badge src-cse">Google CSE</span> Live web result via Google Custom Search</div>
|
||||
<h4>Content type — Google CSE / Wayback only</h4>
|
||||
<div class="help-row"><span class="content-type-badge ct-tweet">Tweet</span> A specific tweet permalink</div>
|
||||
<div class="help-row"><span class="content-type-badge ct-profile">Profile page</span> A bare account profile, not a specific post</div>
|
||||
<div class="help-row"><span class="content-type-badge ct-twitter_other">Other X/Twitter page</span> Some other X page</div>
|
||||
<div class="help-row"><span class="content-type-badge ct-other">External page (non-X)</span> A site off X entirely (Google CSE only)</div>
|
||||
<h4>Account age</h4>
|
||||
<div class="help-row"><span class="age-badge age-new">New account</span> Created < 30 days ago</div>
|
||||
<div class="help-row"><span class="age-badge age-recent">Recent account</span> Created < 1 year ago</div>
|
||||
@@ -498,6 +546,18 @@
|
||||
popup.addEventListener('click', function(e) { e.stopPropagation(); });
|
||||
}());
|
||||
|
||||
// Back to top — the page itself is what scrolls (the sidebar/viewer split
|
||||
// is a grid column layout, not a fixed-height scroll pane).
|
||||
(function() {
|
||||
var btn = document.getElementById('backToTop');
|
||||
window.addEventListener('scroll', function() {
|
||||
btn.classList.toggle('visible', window.scrollY > 400);
|
||||
});
|
||||
btn.addEventListener('click', function() {
|
||||
window.scrollTo({ top: 0, behavior: 'smooth' });
|
||||
});
|
||||
}());
|
||||
|
||||
let allItems = [];
|
||||
let activeId = null;
|
||||
let allArchives = []; // full list from the server; archiveSearch filters this client-side
|
||||
@@ -567,6 +627,7 @@ function entryHtml(a) {
|
||||
return `
|
||||
<div class="archive-entry" data-id="${esc(a.id)}">
|
||||
<div class="ae-tool">${esc(tool)}</div>
|
||||
<div class="ae-date">${esc(date)}</div>
|
||||
${query ? `<div class="ae-meta">${esc(query)}</div>` : ''}
|
||||
<div class="ae-stats">
|
||||
<span class="ae-stat">· ${a.total_items || 0} items</span>
|
||||
@@ -674,6 +735,8 @@ function buildCard(item) {
|
||||
display = `<span class="source-badge ${cls}">${esc(String(v))}</span>`;
|
||||
} else if (k === 'account_age_flag') {
|
||||
display = `<span class="age-badge age-${esc(String(v))}">${esc(AGE_LABELS[v] || v)}</span>`;
|
||||
} else if (k === 'content_type') {
|
||||
display = `<span class="content-type-badge ct-${esc(String(v))}">${esc(CONTENT_TYPE_LABELS[v] || String(v))}</span>`;
|
||||
} else if (typeof v === 'object') {
|
||||
const s = JSON.stringify(v);
|
||||
display = `<span style="color:var(--muted);font-size:11px">${esc(s.length > 100 ? s.slice(0,100)+'…' : s)}</span>`;
|
||||
|
||||
@@ -227,6 +227,56 @@
|
||||
scrollbar-width: thin;
|
||||
scrollbar-color: var(--border) transparent;
|
||||
}
|
||||
.panel-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
padding-bottom: 12px;
|
||||
margin-bottom: 8px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
.panel-avatar {
|
||||
width: 44px;
|
||||
height: 44px;
|
||||
border-radius: 50%;
|
||||
object-fit: cover;
|
||||
border: 2px solid var(--border);
|
||||
flex-shrink: 0;
|
||||
background: var(--bg);
|
||||
}
|
||||
.panel-avatar-fallback {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-size: 16px;
|
||||
font-weight: 700;
|
||||
color: var(--muted);
|
||||
}
|
||||
.panel-identity { min-width: 0; }
|
||||
.panel-name {
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
color: var(--text);
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
.panel-handle { font-size: 11px; color: var(--muted); }
|
||||
/* Compact variant for tweet/reply nodes — the node's own text is a row
|
||||
below, so this stays a small identity line rather than a full profile
|
||||
block (that treatment is reserved for user/retweeter nodes). */
|
||||
.panel-byline { padding-bottom: 10px; }
|
||||
.panel-byline .panel-avatar { width: 28px; height: 28px; }
|
||||
.panel-byline .panel-identity { display: flex; align-items: baseline; gap: 6px; }
|
||||
.panel-bio {
|
||||
font-size: 11px;
|
||||
color: var(--text);
|
||||
opacity: 0.85;
|
||||
line-height: 1.5;
|
||||
padding-bottom: 10px;
|
||||
margin-bottom: 4px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
.info-row {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
@@ -252,6 +302,14 @@
|
||||
}
|
||||
.info-val a { color: var(--accent); text-decoration: none; }
|
||||
.info-val a:hover { text-decoration: underline; }
|
||||
/* "Replying to @x @y" — x.com trims these from the visible tweet body and
|
||||
shows them as their own line instead; matches that here. Display-only,
|
||||
the underlying text value (Dump JSON / Archive All) is untouched. */
|
||||
.reply-context {
|
||||
font-size: 10px;
|
||||
color: var(--muted);
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
.info-media {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
@@ -341,6 +399,8 @@
|
||||
.leg-icon.wayback { background: #2a1f08; border-color: var(--warn); border-radius: 2px; }
|
||||
.leg-icon.cse { background: #2a0d0d; border-color: #ef4444; border-radius: 2px; }
|
||||
.leg-icon.viewed { background: #ffffff; border-color: #c7c7c7; border-radius: 2px; }
|
||||
.leg-line { width: 20px; height: 0; flex-shrink: 0; border-top: 2px solid; }
|
||||
.leg-line-correlation { border-top-color: #facc15; }
|
||||
.leg-heading {
|
||||
font-size: 10px;
|
||||
text-transform: uppercase;
|
||||
@@ -397,6 +457,23 @@
|
||||
.age-badge.age-recent { color: var(--warn); border-color: #78350f; background: #1c0e02; }
|
||||
.age-badge.age-established { color: var(--success); border-color: #14532d; background: #052011; }
|
||||
|
||||
/* What kind of X/Twitter page a Google CSE / Wayback result actually is —
|
||||
a keyword match linking to x.com/someone reads as "just a profile" with
|
||||
no way to tell a tweet permalink apart from a bare profile page or an
|
||||
unrelated page otherwise. */
|
||||
.content-type-badge {
|
||||
display: inline-block;
|
||||
font-size: 11px;
|
||||
font-weight: 500;
|
||||
padding: 2px 9px;
|
||||
border-radius: 20px;
|
||||
border: 1px solid var(--border);
|
||||
}
|
||||
.content-type-badge.ct-tweet { color: var(--success); border-color: #14532d; background: #052011; }
|
||||
.content-type-badge.ct-profile { color: #a78bfa; border-color: #4c3a8f; background: var(--purple-bg); }
|
||||
.content-type-badge.ct-twitter_other { color: var(--warn); border-color: #78350f; background: #1c0e02; }
|
||||
.content-type-badge.ct-other { color: var(--muted); border-color: var(--border); background: var(--bg); }
|
||||
|
||||
/* ── Status bar ── */
|
||||
#statusBar {
|
||||
position: absolute;
|
||||
@@ -467,6 +544,7 @@
|
||||
<a href="/graph" class="current">Graph</a>
|
||||
<hr class="nav-divider">
|
||||
<a href="/archives">Archives</a>
|
||||
<a href="/analytics">Analytics</a>
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
@@ -474,6 +552,7 @@
|
||||
<select id="toolSelect">
|
||||
<option value="tweet_search_extractor">Tweet Search</option>
|
||||
<option value="follower_explorer">Follower Explorer</option>
|
||||
<option value="following_explorer">Following Explorer [Cookie]</option>
|
||||
<option value="post_extractor">Post Extractor</option>
|
||||
<option value="article_extractor">Article Extractor</option>
|
||||
<option value="community_post_extractor">Community Posts</option>
|
||||
@@ -490,7 +569,7 @@
|
||||
<input id="queryInput" type="text" placeholder="Search query…" spellcheck="false" autocomplete="off">
|
||||
<input id="dateFromInput" type="text" placeholder="From YYYYMMDD" title="Optional start date" maxlength="8" style="width:100px;display:none">
|
||||
<input id="dateToInput" type="text" placeholder="To YYYYMMDD" title="Optional end date" maxlength="8" style="width:100px;display:none">
|
||||
<input id="countInput" type="number" value="20" min="1" max="200" title="Result count">
|
||||
<input id="countInput" type="number" value="20" min="1" max="2000" title="Result count">
|
||||
<span id="nodeCount">0 nodes</span>
|
||||
<button id="btnSearch" class="btn btn-primary">Search</button>
|
||||
<button id="btnArchiveAll" class="btn btn-ghost">Archive All</button>
|
||||
@@ -513,6 +592,11 @@
|
||||
<div id="infoActions">
|
||||
<button id="btnExpandReplies" class="btn btn-ghost" style="display:none">↩ Expand Replies</button>
|
||||
<button id="btnExpandRetweets" class="btn btn-ghost" style="display:none">↗ Expand Retweets</button>
|
||||
<button id="btnViewAuthor" class="btn btn-ghost" style="display:none">View Author Profile</button>
|
||||
<button id="btnExpandPosts" class="btn btn-ghost" style="display:none">Expand Posts</button>
|
||||
<button id="btnExpandFollowers" class="btn btn-ghost" style="display:none">Expand Followers</button>
|
||||
<button id="btnExpandFollowing" class="btn btn-ghost" style="display:none">Expand Following</button>
|
||||
<button id="btnLoadMoreSearch" class="btn btn-ghost" style="display:none">⤓ Load More Results</button>
|
||||
<a id="btnOpenTweet" class="btn btn-success" style="display:none"
|
||||
href="#" target="_blank" rel="noopener noreferrer">Open Tweet ↗</a>
|
||||
</div>
|
||||
@@ -524,20 +608,31 @@
|
||||
<!-- Empty state -->
|
||||
<div id="emptyHint">
|
||||
Use the toolbar to search<br>
|
||||
Click any node to inspect · Click tweet nodes to expand replies / retweets
|
||||
Click any node to inspect · Click tweet nodes to expand replies / retweets<br>
|
||||
Shift/Alt/Ctrl+drag to box-select multiple nodes, then drag any one to move them together
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Legend / help popup (floats over cy, bottom-left) -->
|
||||
<button id="legendToggle" aria-label="Legend / help" title="Legend / help">?</button>
|
||||
<div id="legend" class="hidden">
|
||||
<div class="leg-heading">Controls</div>
|
||||
<div class="leg-note">Drag background to pan · Scroll to zoom · Click a node to inspect it<br>Shift/Alt/Ctrl+drag to box-select multiple nodes, then drag any one to move the whole selection together</div>
|
||||
|
||||
<div class="leg-row"><div class="leg-icon search"></div> Search root</div>
|
||||
<div class="leg-row"><div class="leg-icon tweet"></div> Tweet</div>
|
||||
<div class="leg-row"><div class="leg-icon reply"></div> Reply</div>
|
||||
<div class="leg-row"><div class="leg-icon user"></div> User / Retweeter</div>
|
||||
<div class="leg-row"><div class="leg-icon user"></div> User / Retweeter / Follower / Following</div>
|
||||
<div class="leg-row"><div class="leg-icon wayback"></div> Wayback snapshot</div>
|
||||
<div class="leg-row"><div class="leg-icon cse"></div> Google CSE web result</div>
|
||||
<div class="leg-row"><div class="leg-icon viewed"></div> Viewed (clicked)</div>
|
||||
<div class="leg-note">Double border = this tweet/reply has replies of its own — click it, then Expand Replies to go a level deeper. Works on reply nodes too, so a thread nests as deep as it actually goes.</div>
|
||||
<div class="leg-note">A user/retweeter node's photo is their real avatar when one was captured (cookie mode) — the plain purple circle is the fallback when it wasn't.</div>
|
||||
|
||||
<div class="leg-heading">Expanding a node further</div>
|
||||
<div class="leg-note">Tweet/Reply — Expand Replies, Expand Retweets, or View Author Profile (pulls the author out as its own node, free — no request, the data's already on the tweet).<br>User/Retweeter — Expand Posts, Expand Followers, or Expand Following. Works on a "View Author" node too, so a reply's author can be looked up in full: posts, followers, following.</div>
|
||||
<div class="leg-note">Edge labels show how two nodes relate once expanded: <em>replied by</em>, <em>retweeted by</em>, <em>posted</em>, <em>followed by</em>, <em>follows</em>, <em>authored by</em>. A plain unlabeled edge just means "matched the search."</div>
|
||||
<div class="leg-row"><div class="leg-line leg-line-correlation"></div> Correlation — this connects to a node already reached some other way (e.g. a reply's author who also turns up as a follower). Marks where two separate paths through the graph converge on the same account/tweet.</div>
|
||||
|
||||
<div class="leg-heading">Source (info panel)</div>
|
||||
<div class="leg-row"><span class="source-badge src-cookie">Cookie</span></div>
|
||||
@@ -545,6 +640,13 @@
|
||||
<div class="leg-row"><span class="source-badge src-wayback">Wayback Machine</span></div>
|
||||
<div class="leg-row"><span class="source-badge src-cse">Google CSE</span></div>
|
||||
|
||||
<div class="leg-heading">Content type — Google CSE / Wayback only (info panel)</div>
|
||||
<div class="leg-row"><span class="content-type-badge ct-tweet">Tweet</span></div>
|
||||
<div class="leg-row"><span class="content-type-badge ct-profile">Profile page</span></div>
|
||||
<div class="leg-row"><span class="content-type-badge ct-twitter_other">Other X/Twitter page</span></div>
|
||||
<div class="leg-row"><span class="content-type-badge ct-other">External page (non-X)</span></div>
|
||||
<div class="leg-note">What kind of page a keyword match actually landed on — a search hit isn't always a specific tweet.</div>
|
||||
|
||||
<div class="leg-heading">Account age (info panel)</div>
|
||||
<div class="leg-row"><span class="age-badge age-new">New</span> < 30 days</div>
|
||||
<div class="leg-row"><span class="age-badge age-recent">Recent</span> < 1 year</div>
|
||||
@@ -646,6 +748,33 @@ const CY_STYLE = [
|
||||
'background-color': '#ffffff',
|
||||
'border-color': '#c7c7c7',
|
||||
}},
|
||||
// Avatar texture — replaces the flat fill with the person's actual profile
|
||||
// photo when one was captured (cookie mode only; xquik/API-mode records
|
||||
// don't carry it). The colored ring from the selectors above stays as a
|
||||
// type indicator; only the fill becomes a photo. Declared after the
|
||||
// "viewed" block so the photo wins regardless of viewed state.
|
||||
{ selector: 'node[type="user"][?hasAvatar], node[type="retweeter"][?hasAvatar]', style: {
|
||||
'background-image': 'data(avatarUrl)',
|
||||
'background-fit': 'cover',
|
||||
'background-clip': 'node',
|
||||
}},
|
||||
// Nested-reply indicator — a slightly thicker, double-lined border marks a
|
||||
// tweet/reply node that has further replies of its own to expand into,
|
||||
// glanceable on the canvas without opening the info panel.
|
||||
{ selector: 'node[?hasReplies]', style: {
|
||||
'border-width': '3px',
|
||||
'border-style': 'double',
|
||||
}},
|
||||
// Multi-select highlight (Maltego-style box select) — wins over every
|
||||
// type's own border-color above since it's declared last in this array
|
||||
// (Cytoscape applies matching selectors in declaration order, last wins).
|
||||
{ selector: 'node:selected', style: {
|
||||
'border-width': '3px',
|
||||
'border-color': '#5865f2',
|
||||
'overlay-color': '#5865f2',
|
||||
'overlay-opacity': 0.22,
|
||||
'overlay-padding': '6px',
|
||||
}},
|
||||
// Softer, thinner and semi-transparent — a hub with many edges (a popular
|
||||
// tweet's replies, a big multi-source search) stays legible instead of
|
||||
// turning into a bright white tangle.
|
||||
@@ -658,6 +787,30 @@ const CY_STYLE = [
|
||||
'target-arrow-shape': 'triangle',
|
||||
'arrow-scale': 1.0,
|
||||
'curve-style': 'bezier',
|
||||
// Empty for a plain "matched the search" edge (addEdge's label
|
||||
// defaults to ''), so most edges in a fresh search still draw with no
|
||||
// visible text — this only shows up on the relationship-specific
|
||||
// edges node-expand actions create (replied by / retweeted by /
|
||||
// posted / followed by / follows / authored by).
|
||||
label: 'data(label)',
|
||||
'font-size': '8px',
|
||||
color: '#c7c7c7',
|
||||
'text-rotation': 'autorotate',
|
||||
'text-background-color': '#0f1117',
|
||||
'text-background-opacity': 0.85,
|
||||
'text-background-padding': '2px',
|
||||
}},
|
||||
// Correlation — this edge connects to a node that already existed via
|
||||
// some OTHER path (a different search, a different expand, ...) rather
|
||||
// than one just discovered. Declared after the base 'edge' selector so it
|
||||
// wins on line/arrow color; solid + thicker on top of the color change so
|
||||
// it's still readable once a dense graph desaturates everything else.
|
||||
{ selector: 'edge[?correlation]', style: {
|
||||
'line-color': '#facc15',
|
||||
'line-style': 'solid',
|
||||
width: '1.8px',
|
||||
'target-arrow-color': '#facc15',
|
||||
color: '#facc15',
|
||||
}},
|
||||
];
|
||||
|
||||
@@ -670,7 +823,14 @@ function init() {
|
||||
minZoom: 0.08,
|
||||
maxZoom: 4,
|
||||
wheelSensitivity: 0.25,
|
||||
boxSelectionEnabled: false,
|
||||
// Maltego-style multi-select: hold shift/alt/ctrl/cmd and drag on empty
|
||||
// canvas to rubber-band select several nodes at once, then drag any one
|
||||
// of them to move the whole selection together — this is native
|
||||
// Cytoscape drag behavior once elements are selected, no extra code
|
||||
// needed. Plain drag-on-background still pans (Cytoscape requires a
|
||||
// modifier key for box select whenever panning is also enabled, so the
|
||||
// two gestures never conflict).
|
||||
boxSelectionEnabled: true,
|
||||
// Keep panning/zooming smooth once a graph has hundreds of nodes/edges —
|
||||
// edges & a rasterized texture stand in for the full render mid-gesture.
|
||||
hideEdgesOnViewport: true,
|
||||
@@ -687,6 +847,18 @@ function init() {
|
||||
if (evt.target === cy) hidePanel();
|
||||
});
|
||||
|
||||
// Multi-select bookkeeping — the single-node info panel only makes sense
|
||||
// for exactly one selected node, so a box-select of several hides it
|
||||
// (without touching the selection itself, unlike hidePanel()) and the
|
||||
// node-count badge picks up the slack to show what's selected.
|
||||
cy.on('select unselect', 'node', function () {
|
||||
updateNodeCount();
|
||||
if (cy.nodes(':selected').length > 1) {
|
||||
document.getElementById('infoPanel').classList.add('hidden');
|
||||
selectedNode = null;
|
||||
}
|
||||
});
|
||||
|
||||
// Hamburger nav
|
||||
var navBtn = document.getElementById('navToggle');
|
||||
var navMenu = document.getElementById('navMenu');
|
||||
@@ -725,6 +897,17 @@ function init() {
|
||||
document.getElementById('btnExpandRetweets').addEventListener('click', function () {
|
||||
expandNode('tweet_retweeters_extractor');
|
||||
});
|
||||
document.getElementById('btnViewAuthor').addEventListener('click', viewAuthor);
|
||||
document.getElementById('btnExpandPosts').addEventListener('click', function () {
|
||||
expandNode('post_extractor');
|
||||
});
|
||||
document.getElementById('btnExpandFollowers').addEventListener('click', function () {
|
||||
expandNode('follower_explorer');
|
||||
});
|
||||
document.getElementById('btnExpandFollowing').addEventListener('click', function () {
|
||||
expandNode('following_explorer');
|
||||
});
|
||||
document.getElementById('btnLoadMoreSearch').addEventListener('click', expandSearch);
|
||||
|
||||
updatePlaceholder();
|
||||
updateModeVisibility();
|
||||
@@ -735,6 +918,7 @@ function updatePlaceholder() {
|
||||
var map = {
|
||||
tweet_search_extractor: 'Search query…',
|
||||
follower_explorer: 'Username or user ID…',
|
||||
following_explorer: 'Username or user ID…',
|
||||
post_extractor: 'Username or user ID…',
|
||||
article_extractor: 'Tweet ID…',
|
||||
community_post_extractor: 'Community ID…',
|
||||
@@ -777,7 +961,10 @@ function setStatus(msg, isError, persist) {
|
||||
|
||||
function updateNodeCount() {
|
||||
var n = cy.nodes().length;
|
||||
document.getElementById('nodeCount').textContent = n + ' node' + (n !== 1 ? 's' : '');
|
||||
var selN = cy.nodes(':selected').length;
|
||||
var label = n + ' node' + (n !== 1 ? 's' : '');
|
||||
if (selN > 1) label += ' · ' + selN + ' selected';
|
||||
document.getElementById('nodeCount').textContent = label;
|
||||
document.getElementById('emptyHint').classList.toggle('hidden', n > 0);
|
||||
}
|
||||
|
||||
@@ -787,7 +974,7 @@ function buildBody(tool, query, count, mode) {
|
||||
if (tool === 'tweet_search_extractor' || tool === 'geo_post_extractor' ||
|
||||
tool === 'wayback_archive_search' || tool === 'multi_source_search') {
|
||||
body.searchQuery = query;
|
||||
} else if (tool === 'follower_explorer' || tool === 'post_extractor') {
|
||||
} else if (tool === 'follower_explorer' || tool === 'following_explorer' || tool === 'post_extractor') {
|
||||
body.targetUsername = query;
|
||||
} else if (
|
||||
tool === 'article_extractor' ||
|
||||
@@ -832,19 +1019,50 @@ async function apiFetch(body) {
|
||||
var cooldownUntil = { cookie: 0, wayback: 0 };
|
||||
var cooldownTimer = null;
|
||||
|
||||
// Returns the list of rate-limit clocks a request against this tool/mode
|
||||
// touches. multi_source_search fans out to both cookie and wayback
|
||||
// internally, so continuing it has to wait out (and then restart) both
|
||||
// cooldowns at once, not just one.
|
||||
function throttleSourceFor(tool, mode) {
|
||||
if (tool === 'wayback_archive_search') return 'wayback';
|
||||
var COOKIE_TOOLS = ['tweet_search_extractor', 'follower_explorer', 'post_extractor',
|
||||
if (tool === 'wayback_archive_search') return ['wayback'];
|
||||
if (tool === 'multi_source_search') return ['cookie', 'wayback'];
|
||||
var COOKIE_TOOLS = ['tweet_search_extractor', 'follower_explorer', 'following_explorer', 'post_extractor',
|
||||
'community_post_extractor', 'tweet_replies_extractor', 'tweet_retweeters_extractor', 'geo_post_extractor'];
|
||||
if (mode === 'cookie' && COOKIE_TOOLS.indexOf(tool) !== -1) return 'cookie';
|
||||
return null;
|
||||
if (mode === 'cookie' && COOKIE_TOOLS.indexOf(tool) !== -1) return ['cookie'];
|
||||
return [];
|
||||
}
|
||||
|
||||
function stampCooldown(source) {
|
||||
if (source) cooldownUntil[source] = Date.now() + 5000;
|
||||
function stampCooldown(sources) {
|
||||
(sources || []).forEach(function (source) { cooldownUntil[source] = Date.now() + 5000; });
|
||||
}
|
||||
|
||||
// Tools whose root search (not just node-expand) supports cursor pagination
|
||||
// — everything except article_extractor, which only ever returns one item.
|
||||
var ROOT_PAGINATED_TOOLS = ['tweet_search_extractor', 'follower_explorer', 'following_explorer', 'post_extractor',
|
||||
'community_post_extractor', 'tweet_replies_extractor', 'tweet_retweeters_extractor',
|
||||
'geo_post_extractor', 'wayback_archive_search', 'multi_source_search'];
|
||||
|
||||
// ── Label builder ─────────────────────────────────────────────────────────────
|
||||
// Tapping "Reply" on X auto-prefixes the compose box with every account the
|
||||
// reply-chain already has tagged, and that prefix is genuinely part of the
|
||||
// reply's own raw text — but x.com's own UI never shows it inline, trimming
|
||||
// it into a separate "Replying to @x @y" line instead. `reply_to_mentions`
|
||||
// (from cookie_client.py's _leading_reply_mentions) lists exactly which
|
||||
// leading @mentions those are; this only touches how text RENDERS — the
|
||||
// node's own raw data (and what Dump JSON / Archive All send) is untouched.
|
||||
function stripLeadingMentions(text, mentions) {
|
||||
if (!text || !mentions || !mentions.length) return text;
|
||||
var rest = text;
|
||||
for (var i = 0; i < mentions.length; i++) {
|
||||
var re = new RegExp('^\\s*@' + String(mentions[i]).replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '\\b');
|
||||
var match = rest.match(re);
|
||||
if (!match) break;
|
||||
rest = rest.slice(match[0].length);
|
||||
}
|
||||
rest = rest.replace(/^\s+/, '');
|
||||
return rest || text;
|
||||
}
|
||||
|
||||
function makeLabel(item, type) {
|
||||
if (type === 'user' || type === 'retweeter') {
|
||||
var handle = item.screen_name || item.user || '';
|
||||
@@ -858,7 +1076,8 @@ function makeLabel(item, type) {
|
||||
var web = (item.post_title || item.post_text || item.display_link || 'Web result').replace(/\s+/g, ' ').trim();
|
||||
return web.length > 60 ? web.substring(0, 60) + '…' : web;
|
||||
}
|
||||
var text = (item.text || item.retweeted_text || '').replace(/\s+/g, ' ').trim();
|
||||
var rawText = item.text ? stripLeadingMentions(item.text, item.reply_to_mentions) : item.text;
|
||||
var text = (rawText || item.retweeted_text || '').replace(/\s+/g, ' ').trim();
|
||||
return text.length > 60 ? text.substring(0, 60) + '…' : (text || ('Tweet ' + (item.id || '?')));
|
||||
}
|
||||
|
||||
@@ -866,7 +1085,7 @@ function makeLabel(item, type) {
|
||||
// snapshots and Google CSE web results in one result set, so the type has to
|
||||
// be resolved per item rather than fixed for the whole tool/search.
|
||||
function resolveNodeType(tool, item) {
|
||||
if (tool === 'follower_explorer') return 'user';
|
||||
if (tool === 'follower_explorer' || tool === 'following_explorer') return 'user';
|
||||
if (tool === 'tweet_retweeters_extractor') return 'retweeter';
|
||||
if (tool === 'wayback_archive_search') return 'wayback';
|
||||
if (tool === 'multi_source_search') {
|
||||
@@ -877,8 +1096,21 @@ function resolveNodeType(tool, item) {
|
||||
return 'tweet';
|
||||
}
|
||||
|
||||
// Only a plain https URL is ever handed to Cytoscape's background-image —
|
||||
// it ends up as a CSS url() token internally, so the same reasoning as
|
||||
// index.html's isSafeImageUrl applies: reject anything but a clean https
|
||||
// URL rather than trying to escape one value for two contexts.
|
||||
function isSafeImageUrl(u) {
|
||||
return typeof u === 'string' && /^https:\/\/[^\s'"<>()]+$/.test(u);
|
||||
}
|
||||
|
||||
// ── Graph operations ──────────────────────────────────────────────────────────
|
||||
function addNodes(items, nodeTypeOrFn, parentId) {
|
||||
// edgeLabel is optional — omitted for a plain "this matched the search"
|
||||
// connection (search root → result, continued-search pagination); every
|
||||
// node-expand action (replies/retweets/posts/followers/following/author)
|
||||
// passes one, since with 6+ relationship types now a bare arrow stopped
|
||||
// being enough to tell what an edge actually means.
|
||||
function addNodes(items, nodeTypeOrFn, parentId, edgeLabel) {
|
||||
var added = 0;
|
||||
items.forEach(function (item, i) {
|
||||
var nodeType = typeof nodeTypeOrFn === 'function' ? nodeTypeOrFn(item) : nodeTypeOrFn;
|
||||
@@ -886,10 +1118,19 @@ function addNodes(items, nodeTypeOrFn, parentId) {
|
||||
var cyId = tweetId ? 'n_' + tweetId : 'n_' + Date.now() + '_' + i;
|
||||
|
||||
if (cy.getElementById(cyId).length) {
|
||||
if (parentId) addEdge(parentId, cyId);
|
||||
// The node this would-be edge points to already exists — reached via
|
||||
// some OTHER path already (a different search, a different expand, a
|
||||
// reply's author who also turned up as a follower, ...). That's a
|
||||
// genuine correlation between two otherwise-separate parts of the
|
||||
// graph, worth calling out on its own rather than blending in with
|
||||
// every other edge once the graph gets big.
|
||||
if (parentId) addEdge(parentId, cyId, edgeLabel, true);
|
||||
return;
|
||||
}
|
||||
|
||||
var avatarUrl = item.avatar || item.user_avatar || '';
|
||||
if (!isSafeImageUrl(avatarUrl)) avatarUrl = '';
|
||||
|
||||
cy.add({
|
||||
data: {
|
||||
id: cyId,
|
||||
@@ -897,20 +1138,28 @@ function addNodes(items, nodeTypeOrFn, parentId) {
|
||||
label: makeLabel(item, nodeType),
|
||||
tweetId: tweetId,
|
||||
raw: item,
|
||||
hasReplies: (nodeType === 'tweet' || nodeType === 'reply') && Number(item.reply_count) > 0,
|
||||
hasAvatar: (nodeType === 'user' || nodeType === 'retweeter') && !!avatarUrl,
|
||||
avatarUrl: avatarUrl,
|
||||
}
|
||||
});
|
||||
|
||||
if (parentId) addEdge(parentId, cyId);
|
||||
if (parentId) addEdge(parentId, cyId, edgeLabel, false);
|
||||
allItems.push({ type: nodeType, item: item });
|
||||
added++;
|
||||
});
|
||||
return added;
|
||||
}
|
||||
|
||||
function addEdge(src, tgt) {
|
||||
// A node pair can only ever have one edge between them (id is derived purely
|
||||
// from src+tgt) — if a second expand action connects the same two nodes with
|
||||
// a different label (e.g. someone who's both a follower and a retweeter),
|
||||
// the first label/correlation state that got there wins rather than being
|
||||
// silently replaced.
|
||||
function addEdge(src, tgt, label, isCorrelation) {
|
||||
var eId = 'e_' + src + '_' + tgt;
|
||||
if (!cy.getElementById(eId).length) {
|
||||
cy.add({ data: { id: eId, source: src, target: tgt } });
|
||||
cy.add({ data: { id: eId, source: src, target: tgt, label: label || '', correlation: !!isCorrelation } });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -933,7 +1182,7 @@ function runLayout(fit, randomize) {
|
||||
async function runSearch() {
|
||||
var tool = document.getElementById('toolSelect').value;
|
||||
var query = document.getElementById('queryInput').value.trim();
|
||||
var count = Math.max(1, Math.min(200, parseInt(document.getElementById('countInput').value, 10) || 20));
|
||||
var count = Math.max(1, Math.min(2000, parseInt(document.getElementById('countInput').value, 10) || 20));
|
||||
var mode = document.getElementById('modeSelect').value;
|
||||
|
||||
if (!query) { setStatus('Enter a query first', true); return; }
|
||||
@@ -945,15 +1194,20 @@ async function runSearch() {
|
||||
var result = await apiFetch(buildBody(tool, query, count, mode));
|
||||
var items = Array.isArray(result.items) ? result.items : [result.items];
|
||||
stampCooldown(throttleSourceFor(tool, mode));
|
||||
// Root-search pagination (continuing this same search past one page) is
|
||||
// not wired up in the graph UI yet — only reply/retweet node-expand
|
||||
// supports "load more" this round. result.nextCursor is intentionally
|
||||
// unused here.
|
||||
|
||||
// Central search node
|
||||
// Central search node — carries everything expandSearch() needs to fetch
|
||||
// the next page later (tool/query/mode/count fixed at the time of this
|
||||
// search, cursor updated after every page).
|
||||
var searchId = 'search_' + Date.now();
|
||||
var shortQ = query.length > 40 ? query.substring(0, 40) + '…' : query;
|
||||
cy.add({ data: { id: searchId, type: 'search', label: shortQ, tweetId: '', raw: { query: query } } });
|
||||
cy.add({
|
||||
data: {
|
||||
id: searchId, type: 'search', label: shortQ, tweetId: '', raw: { query: query },
|
||||
tool: tool, query: query, mode: mode, count: count,
|
||||
searchCursor: result.nextCursor || null,
|
||||
searchExhausted: !result.nextCursor,
|
||||
}
|
||||
});
|
||||
|
||||
var added = addNodes(items, function (item) { return resolveNodeType(tool, item); }, searchId);
|
||||
runLayout(true, true);
|
||||
@@ -966,75 +1220,194 @@ async function runSearch() {
|
||||
}
|
||||
}
|
||||
|
||||
// ── Expand replies / retweets from selected node ──────────────────────────────
|
||||
// ── Expand a node further (replies/retweets/posts/followers/following) ────────
|
||||
// First click on a node = fresh fetch. If more is available, the button
|
||||
// relabels itself and a second click continues from the stored cursor —
|
||||
// same cursor/cooldown mechanism as the non-graph page's scroll load-more.
|
||||
var CURSOR_KEY = { tweet_replies_extractor: 'repliesCursor', tweet_retweeters_extractor: 'retweetersCursor' };
|
||||
var EXHAUSTED_KEY = { tweet_replies_extractor: 'repliesExhausted', tweet_retweeters_extractor: 'retweetersExhausted' };
|
||||
// targetKind picks which id the API call needs: a tweet node sends its own
|
||||
// tweetId as targetTweetId; a user/retweeter node sends the account's own
|
||||
// id (preferred) or handle as targetUsername.
|
||||
var EXPAND_TOOLS = {
|
||||
tweet_replies_extractor: { targetKind: 'tweet', nodeType: 'reply', button: 'btnExpandReplies', cursorKey: 'repliesCursor', exhaustedKey: 'repliesExhausted', label: 'Replies', edgeLabel: 'replied by', icon: '↩ ', countField: 'reply_count' },
|
||||
tweet_retweeters_extractor: { targetKind: 'tweet', nodeType: 'retweeter', button: 'btnExpandRetweets', cursorKey: 'retweetersCursor', exhaustedKey: 'retweetersExhausted', label: 'Retweets', edgeLabel: 'retweeted by', icon: '↗ ', countField: 'retweet_count' },
|
||||
post_extractor: { targetKind: 'user', nodeType: 'tweet', button: 'btnExpandPosts', cursorKey: 'postsCursor', exhaustedKey: 'postsExhausted', label: 'Posts', edgeLabel: 'posted', icon: '', countField: 'tweet_count' },
|
||||
follower_explorer: { targetKind: 'user', nodeType: 'user', button: 'btnExpandFollowers', cursorKey: 'followersCursor', exhaustedKey: 'followersExhausted', label: 'Followers', edgeLabel: 'followed by', icon: '', countField: 'followers_count' },
|
||||
following_explorer: { targetKind: 'user', nodeType: 'user', button: 'btnExpandFollowing', cursorKey: 'followingCursor', exhaustedKey: 'followingExhausted', label: 'Following', edgeLabel: 'follows', icon: '', countField: 'following_count' },
|
||||
};
|
||||
|
||||
// Shared by every Expand button in the panel — label reflects whether this
|
||||
// node's already been expanded and whether more is available; before the
|
||||
// first click, falls back to the account/tweet's own self-reported count
|
||||
// (reply_count, followers_count, ...) so it's never a blind "Expand".
|
||||
function updateExpandButtonLabel(btn, data, cfg, raw) {
|
||||
if (data[cfg.exhaustedKey]) {
|
||||
btn.textContent = cfg.icon + 'Expand ' + cfg.label + ' (all loaded)';
|
||||
} else if (data[cfg.cursorKey]) {
|
||||
btn.textContent = cfg.icon + 'Expand ' + cfg.label + ' (more available)';
|
||||
} else {
|
||||
var n = Number(raw[cfg.countField]) || 0;
|
||||
btn.textContent = n > 0 ? cfg.icon + 'Expand ~' + n + ' ' + cfg.label : cfg.icon + 'Expand ' + cfg.label;
|
||||
}
|
||||
btn.disabled = !!data[cfg.exhaustedKey];
|
||||
}
|
||||
|
||||
function targetIdFor(cfg, data) {
|
||||
if (cfg.targetKind === 'tweet') return data.tweetId || null;
|
||||
var raw = data.raw || {};
|
||||
// user_id is unambiguous (survives a screen_name change); screen_name/user
|
||||
// are the fallback for any record shape that never captured a numeric id.
|
||||
return raw.user_id || raw.id || raw.screen_name || raw.user || null;
|
||||
}
|
||||
|
||||
async function expandNode(expandTool) {
|
||||
if (!selectedNode) return;
|
||||
var cfg = EXPAND_TOOLS[expandTool];
|
||||
var data = selectedNode.data();
|
||||
var tweetId = data.tweetId;
|
||||
if (!tweetId) { setStatus('Node has no tweet ID', true); return; }
|
||||
var targetId = targetIdFor(cfg, data);
|
||||
if (!targetId) { setStatus('Node has no ' + (cfg.targetKind === 'tweet' ? 'tweet ID' : 'user ID/handle'), true); return; }
|
||||
|
||||
var source = throttleSourceFor(expandTool, 'cookie'); // always 'cookie' — both expand tools are cookie-only
|
||||
var remaining = cooldownUntil[source] - Date.now();
|
||||
var sources = throttleSourceFor(expandTool, 'cookie'); // always 'cookie' — every expand tool here is cookie-only
|
||||
var btn = document.getElementById(cfg.button);
|
||||
var remaining = Math.max(0, ...sources.map(function (s) { return cooldownUntil[s] - Date.now(); }));
|
||||
if (remaining > 0) {
|
||||
armExpandCountdown(remaining);
|
||||
armCooldownButtons(remaining, [btn], 'Ready — click Expand ' + cfg.label + ' again');
|
||||
return;
|
||||
}
|
||||
|
||||
var count = Math.max(1, Math.min(200, parseInt(document.getElementById('countInput').value, 10) || 50));
|
||||
var count = Math.max(1, Math.min(2000, parseInt(document.getElementById('countInput').value, 10) || 50));
|
||||
var parentId = selectedNode.id();
|
||||
var cursor = selectedNode.data(CURSOR_KEY[expandTool]) || null;
|
||||
var cursor = selectedNode.data(cfg.cursorKey) || null;
|
||||
|
||||
document.getElementById('btnExpandReplies').disabled = true;
|
||||
document.getElementById('btnExpandRetweets').disabled = true;
|
||||
var body = { toolType: expandTool, mode: 'cookie', count: count };
|
||||
if (cfg.targetKind === 'tweet') body.targetTweetId = targetId;
|
||||
else body.targetUsername = targetId;
|
||||
if (cursor) body.cursor = cursor;
|
||||
|
||||
btn.disabled = true;
|
||||
setStatus(cursor ? 'Loading more…' : 'Expanding…', false, true);
|
||||
|
||||
try {
|
||||
var body = { toolType: expandTool, mode: 'cookie', count: count, targetTweetId: tweetId };
|
||||
if (cursor) body.cursor = cursor;
|
||||
var result = await apiFetch(body);
|
||||
stampCooldown(source);
|
||||
stampCooldown(sources);
|
||||
var items = Array.isArray(result.items) ? result.items : [result.items];
|
||||
var nodeType = expandTool === 'tweet_replies_extractor' ? 'reply' : 'retweeter';
|
||||
|
||||
var added = addNodes(items, nodeType, parentId);
|
||||
var added = addNodes(items, cfg.nodeType, parentId, cfg.edgeLabel);
|
||||
runLayout(false, false);
|
||||
updateNodeCount();
|
||||
setStatus('Expanded — ' + items.length + ' result(s) (' + added + ' new nodes)');
|
||||
|
||||
selectedNode.data(CURSOR_KEY[expandTool], result.nextCursor || null);
|
||||
selectedNode.data(EXHAUSTED_KEY[expandTool], !result.nextCursor);
|
||||
selectedNode.data(cfg.cursorKey, result.nextCursor || null);
|
||||
selectedNode.data(cfg.exhaustedKey, !result.nextCursor);
|
||||
showPanel(selectedNode.data()); // refresh button labels/disabled state (e.g. "all loaded")
|
||||
} catch (e) {
|
||||
if (e.retryAfter) {
|
||||
// Keep both buttons disabled for the cooldown window — armExpandCountdown
|
||||
// re-enables them itself once it elapses, so don't touch them here.
|
||||
armExpandCountdown(e.retryAfter * 1000);
|
||||
// Keep the button disabled for the cooldown window — armCooldownButtons
|
||||
// re-enables it itself once it elapses, so don't touch it here.
|
||||
armCooldownButtons(e.retryAfter * 1000, [btn], 'Ready — click Expand ' + cfg.label + ' again');
|
||||
} else {
|
||||
setStatus('Error: ' + e.message, true);
|
||||
document.getElementById('btnExpandReplies').disabled = false;
|
||||
document.getElementById('btnExpandRetweets').disabled = false;
|
||||
btn.disabled = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function armExpandCountdown(msRemaining) {
|
||||
// ── View a tweet/reply node's author as its own node ───────────────────────
|
||||
// A reply, retweet, or search-result tweet already carries its author's
|
||||
// avatar/bio/account-age info (cookie_client.py's _tweet_to_dict embeds it
|
||||
// on every tweet) — this never hits the network, it just re-shapes that
|
||||
// already-fetched data into a standalone user node with the same field
|
||||
// names cookie_client.py's _user_to_dict itself produces, so the node
|
||||
// renders identically to a real follower/following result. The synthesized
|
||||
// node's id is the author's own user_id, so an author who's already on
|
||||
// canvas some other way (e.g. also appears as a follower elsewhere) merges
|
||||
// into that same node instead of duplicating it — this is exactly what
|
||||
// makes a reply's author "look-up-able": select them afterward and Expand
|
||||
// Posts/Followers/Following works on them like any other user node.
|
||||
var AUTHOR_REMAP = { user_avatar: 'avatar', user_banner: 'banner', user_bio: 'description' };
|
||||
var AUTHOR_PASSTHROUGH = ['name', 'verified', 'is_blue_verified', 'user_location',
|
||||
'account_created', 'account_age', 'account_age_flag', 'account_age_precision',
|
||||
'source', 'fetched_at'];
|
||||
|
||||
function authorFromRaw(raw) {
|
||||
if (!raw || (!raw.user_id && !raw.user)) return null;
|
||||
var out = { id: raw.user_id || null, screen_name: raw.user || null };
|
||||
Object.keys(raw).forEach(function (k) {
|
||||
if (AUTHOR_REMAP[k] && raw[k] != null) out[AUTHOR_REMAP[k]] = raw[k];
|
||||
});
|
||||
AUTHOR_PASSTHROUGH.forEach(function (k) {
|
||||
if (raw[k] != null) out[k] = raw[k];
|
||||
});
|
||||
return out;
|
||||
}
|
||||
|
||||
function viewAuthor() {
|
||||
if (!selectedNode) return;
|
||||
var raw = selectedNode.data('raw') || {};
|
||||
var author = authorFromRaw(raw);
|
||||
if (!author) { setStatus('No author info on this node', true); return; }
|
||||
|
||||
var added = addNodes([author], 'user', selectedNode.id(), 'authored by');
|
||||
runLayout(false, false);
|
||||
updateNodeCount();
|
||||
setStatus(added ? 'Author profile added — select it to Expand Posts/Followers/Following'
|
||||
: 'Author already on canvas — linked');
|
||||
}
|
||||
|
||||
// ── Load more results for the root search node ─────────────────────────────
|
||||
// Mirrors expandNode() above — same click+cursor+cooldown mechanism, applied
|
||||
// to the search node itself instead of a tweet node, so any tool (not just
|
||||
// replies/retweets) can page past its first result set.
|
||||
async function expandSearch() {
|
||||
if (!selectedNode) return;
|
||||
var data = selectedNode.data();
|
||||
if (data.type !== 'search' || !data.tool) return;
|
||||
|
||||
var sources = throttleSourceFor(data.tool, data.mode);
|
||||
var btn = document.getElementById('btnLoadMoreSearch');
|
||||
var remaining = Math.max(0, ...sources.map(function (s) { return cooldownUntil[s] - Date.now(); }));
|
||||
if (remaining > 0) {
|
||||
armCooldownButtons(remaining, [btn], 'Ready — click Load More again');
|
||||
return;
|
||||
}
|
||||
|
||||
var cursor = selectedNode.data('searchCursor') || null;
|
||||
btn.disabled = true;
|
||||
setStatus('Loading more…', false, true);
|
||||
|
||||
try {
|
||||
var body = buildBody(data.tool, data.query, data.count, data.mode);
|
||||
if (cursor) body.cursor = cursor;
|
||||
var result = await apiFetch(body);
|
||||
stampCooldown(sources);
|
||||
var items = Array.isArray(result.items) ? result.items : [result.items];
|
||||
|
||||
var added = addNodes(items, function (item) { return resolveNodeType(data.tool, item); }, selectedNode.id());
|
||||
runLayout(false, false);
|
||||
updateNodeCount();
|
||||
setStatus('Loaded more — ' + items.length + ' result(s) (' + added + ' new nodes)');
|
||||
|
||||
selectedNode.data('searchCursor', result.nextCursor || null);
|
||||
selectedNode.data('searchExhausted', !result.nextCursor);
|
||||
showPanel(selectedNode.data());
|
||||
} catch (e) {
|
||||
if (e.retryAfter) {
|
||||
armCooldownButtons(e.retryAfter * 1000, [btn], 'Ready — click Load More again');
|
||||
} else {
|
||||
setStatus('Error: ' + e.message, true);
|
||||
btn.disabled = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function armCooldownButtons(msRemaining, buttons, readyMessage) {
|
||||
var secs = Math.max(1, Math.ceil(msRemaining / 1000));
|
||||
setStatus('Rate-limit cooldown — retry in ' + secs + 's…', false, true);
|
||||
// Both Expand buttons share one underlying X-account clock, so both wait together.
|
||||
document.getElementById('btnExpandReplies').disabled = true;
|
||||
document.getElementById('btnExpandRetweets').disabled = true;
|
||||
buttons.forEach(function (b) { b.disabled = true; });
|
||||
if (cooldownTimer) clearTimeout(cooldownTimer);
|
||||
cooldownTimer = setTimeout(function () {
|
||||
cooldownTimer = null;
|
||||
document.getElementById('btnExpandReplies').disabled = false;
|
||||
document.getElementById('btnExpandRetweets').disabled = false;
|
||||
setStatus('Ready — click Expand again', false, false);
|
||||
buttons.forEach(function (b) { b.disabled = false; });
|
||||
setStatus(readyMessage, false, false);
|
||||
}, msRemaining);
|
||||
}
|
||||
|
||||
@@ -1050,17 +1423,31 @@ function esc(s) {
|
||||
// LIST ARR PARAMS
|
||||
|
||||
var PRIORITY_KEYS = [
|
||||
'source', 'account_age_flag', 'account_age', 'account_created',
|
||||
'source', 'content_type', 'account_age_flag', 'account_age', 'account_created',
|
||||
'user', 'screen_name', 'name', 'text', 'post_title', 'post_text', 'created_at', 'fetched_at',
|
||||
'retweeted_by_user', 'retweeted_by_name', 'retweeted_text', 'retweeted_by_bio',
|
||||
'reply_count', 'retweet_count', 'favorite_count', 'view_count',
|
||||
'followers_count', 'following_count', 'tweet_count',
|
||||
'description', 'user_location', 'in_reply_to_tweet_id',
|
||||
'retweeted_tweet_id', 'retweeted_at', 'verified', 'is_blue_verified',
|
||||
'archive_url', 'result_url', 'preview_image', 'display_link', 'iso_date', 'original', 'statuscode',
|
||||
'archive_url', 'result_url', 'preview_image', 'display_link', 'serp_title',
|
||||
'iso_date', 'original', 'statuscode',
|
||||
];
|
||||
|
||||
var SKIP_KEYS = new Set(['id', 'media', 'card', 'user_id', 'retweeted_by_user_id']);
|
||||
// Same labels/reasoning as card_constants.js's CONTENT_TYPE_LABELS — kept as
|
||||
// a separate copy here since graph.html doesn't load that shared file (its
|
||||
// PRIORITY_KEYS/SKIP_KEYS are already their own independent copy, not the
|
||||
// index.html/archive.html shared ones).
|
||||
var CONTENT_TYPE_LABELS = {
|
||||
tweet: 'Tweet',
|
||||
profile: 'Profile page',
|
||||
twitter_other: 'Other X/Twitter page',
|
||||
other: 'External page (non-X)',
|
||||
};
|
||||
|
||||
// avatar/banner URLs are rendered as an actual image in the panel header
|
||||
// below instead of a raw-link row.
|
||||
var SKIP_KEYS = new Set(['id', 'media', 'card', 'user_id', 'retweeted_by_user_id', 'avatar', 'banner', 'user_avatar', 'user_banner']);
|
||||
|
||||
var SOURCE_CLASS = { 'Twitter Cookie': 'src-cookie', 'Xquik API': 'src-xquik', 'Wayback Machine': 'src-wayback', 'Google CSE': 'src-cse' };
|
||||
|
||||
@@ -1090,6 +1477,44 @@ function extractMedia(item) {
|
||||
}).filter(function (m) { return m.thumb; });
|
||||
}
|
||||
|
||||
// Same header treatment index.html's cards give a user — avatar, name,
|
||||
// handle, bio — but for the info panel, which every node type funnels
|
||||
// through (search root, tweet, reply, user, retweeter, ...).
|
||||
function buildPanelHeader(raw) {
|
||||
var avatarRaw = raw.avatar || raw.user_avatar || '';
|
||||
var avatar = isSafeImageUrl(avatarRaw) ? avatarRaw : '';
|
||||
var name = raw.name || '';
|
||||
var handle = raw.screen_name || raw.user || '';
|
||||
|
||||
if (!avatar && !name && !handle) return '';
|
||||
|
||||
var avatarHtml = avatar
|
||||
? '<img class="panel-avatar" src="' + esc(avatar) + '" alt="" loading="lazy" referrerpolicy="no-referrer">'
|
||||
: (name || handle)
|
||||
? '<div class="panel-avatar panel-avatar-fallback">' + esc((name || handle).charAt(0).toUpperCase()) + '</div>'
|
||||
: '';
|
||||
var identityHtml = (name || handle)
|
||||
? '<div class="panel-identity">' +
|
||||
(name ? '<div class="panel-name">' + esc(name) + '</div>' : '') +
|
||||
(handle ? '<div class="panel-handle">@' + esc(handle) + '</div>' : '') +
|
||||
'</div>'
|
||||
: '';
|
||||
|
||||
// A tweet/reply node's own text already shows a few rows down — leading
|
||||
// with the author's full bio here would bury it the same way it did on
|
||||
// index.html's cards. Bio only makes sense when the node itself IS a
|
||||
// user (follower/retweeter), not a tweet someone happened to write.
|
||||
var isTweetLike = raw.text !== undefined || raw.full_text !== undefined || raw.article_text !== undefined;
|
||||
if (isTweetLike) {
|
||||
return '<div class="panel-header panel-byline">' + avatarHtml + identityHtml + '</div>';
|
||||
}
|
||||
|
||||
var bio = raw.description || raw.user_bio || '';
|
||||
var bioHtml = bio ? '<div class="panel-bio">' + esc(bio) + '</div>' : '';
|
||||
|
||||
return '<div class="panel-header">' + avatarHtml + identityHtml + '</div>' + bioHtml;
|
||||
}
|
||||
|
||||
function showPanel(data) {
|
||||
var raw = data.raw || {};
|
||||
var type = data.type;
|
||||
@@ -1124,6 +1549,12 @@ function showPanel(data) {
|
||||
} else if (k === 'account_age_flag') {
|
||||
var ageLabels = { new: 'New account', recent: 'Recent account', established: 'Established account' };
|
||||
val = '<span class="age-badge age-' + esc(String(v)) + '">' + esc(ageLabels[v] || String(v)) + '</span>';
|
||||
} else if (k === 'content_type') {
|
||||
val = '<span class="content-type-badge ct-' + esc(String(v)) + '">' + esc(CONTENT_TYPE_LABELS[v] || String(v)) + '</span>';
|
||||
} else if (k === 'text' && raw.reply_to_mentions && raw.reply_to_mentions.length) {
|
||||
var mentionBadge = '<div class="reply-context">↩ Replying to ' +
|
||||
raw.reply_to_mentions.map(function (m) { return '@' + esc(m); }).join(', ') + '</div>';
|
||||
val = mentionBadge + esc(stripLeadingMentions(String(v), raw.reply_to_mentions));
|
||||
}
|
||||
|
||||
var label = esc(k.replace(/_/g, ' '));
|
||||
@@ -1155,28 +1586,51 @@ function showPanel(data) {
|
||||
rows += mediaParts.join('');
|
||||
}
|
||||
|
||||
document.getElementById('infoContent').innerHTML = rows;
|
||||
document.getElementById('infoContent').innerHTML = buildPanelHeader(raw) + rows;
|
||||
|
||||
// Action buttons
|
||||
var isTweet = type === 'tweet' || type === 'reply';
|
||||
var isUserNode = type === 'user' || type === 'retweeter';
|
||||
var isWayback = type === 'wayback';
|
||||
var tweetId = data.tweetId;
|
||||
var user = raw.user || raw.screen_name || '';
|
||||
|
||||
var repliesBtn = document.getElementById('btnExpandReplies');
|
||||
var retweetersBtn = document.getElementById('btnExpandRetweets');
|
||||
var authorBtn = document.getElementById('btnViewAuthor');
|
||||
var postsBtn = document.getElementById('btnExpandPosts');
|
||||
var followersBtn = document.getElementById('btnExpandFollowers');
|
||||
var followingBtn = document.getElementById('btnExpandFollowing');
|
||||
var loadMoreBtn = document.getElementById('btnLoadMoreSearch');
|
||||
|
||||
repliesBtn.style.display = isTweet ? '' : 'none';
|
||||
retweetersBtn.style.display = isTweet ? '' : 'none';
|
||||
// A synthesized "view author" pivot only makes sense when there's
|
||||
// somewhere to pivot TO — a tweet/reply always embeds its author's
|
||||
// user_id/screen_name (cookie_client.py's _tweet_to_dict), but a bare
|
||||
// xquik/API-mode record might not have captured either.
|
||||
authorBtn.style.display = (isTweet && (raw.user_id || raw.user)) ? '' : 'none';
|
||||
|
||||
postsBtn.style.display = isUserNode ? '' : 'none';
|
||||
followersBtn.style.display = isUserNode ? '' : 'none';
|
||||
followingBtn.style.display = isUserNode ? '' : 'none';
|
||||
|
||||
var isPaginableSearch = type === 'search' && ROOT_PAGINATED_TOOLS.indexOf(data.tool) !== -1;
|
||||
loadMoreBtn.style.display = isPaginableSearch ? '' : 'none';
|
||||
if (isPaginableSearch) {
|
||||
loadMoreBtn.textContent = data.searchExhausted ? '⤓ Load More Results (all loaded)'
|
||||
: data.searchCursor ? '⤓ Load More Results (more available)' : '⤓ Load More Results';
|
||||
loadMoreBtn.disabled = !!data.searchExhausted;
|
||||
}
|
||||
|
||||
if (isTweet) {
|
||||
// Label reflects whether this node's already been expanded and whether
|
||||
// more is available — neither twikit nor the Wayback CDX API expose a
|
||||
// total count, only presence/absence of a next page, so no item count.
|
||||
repliesBtn.textContent = data.repliesExhausted ? '↩ Expand Replies (all loaded)'
|
||||
: data.repliesCursor ? '↩ Expand Replies (more available)' : '↩ Expand Replies';
|
||||
repliesBtn.disabled = !!data.repliesExhausted;
|
||||
retweetersBtn.textContent = data.retweetersExhausted ? '↗ Expand Retweets (all loaded)'
|
||||
: data.retweetersCursor ? '↗ Expand Retweets (more available)' : '↗ Expand Retweets';
|
||||
retweetersBtn.disabled = !!data.retweetersExhausted;
|
||||
updateExpandButtonLabel(repliesBtn, data, EXPAND_TOOLS.tweet_replies_extractor, raw);
|
||||
updateExpandButtonLabel(retweetersBtn, data, EXPAND_TOOLS.tweet_retweeters_extractor, raw);
|
||||
}
|
||||
if (isUserNode) {
|
||||
updateExpandButtonLabel(postsBtn, data, EXPAND_TOOLS.post_extractor, raw);
|
||||
updateExpandButtonLabel(followersBtn, data, EXPAND_TOOLS.follower_explorer, raw);
|
||||
updateExpandButtonLabel(followingBtn, data, EXPAND_TOOLS.following_explorer, raw);
|
||||
}
|
||||
|
||||
var openBtn = document.getElementById('btnOpenTweet');
|
||||
|
||||
@@ -39,7 +39,9 @@
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
background: var(--surface);
|
||||
position: relative;
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 100;
|
||||
}
|
||||
header h1 { font-size: 14px; font-weight: 600; letter-spacing: 0.02em; }
|
||||
header .sep { color: var(--border); }
|
||||
@@ -104,6 +106,27 @@
|
||||
z-index: 251;
|
||||
transition: all 0.15s;
|
||||
}
|
||||
.back-to-top {
|
||||
position: fixed;
|
||||
bottom: 14px;
|
||||
right: 14px;
|
||||
width: 36px;
|
||||
height: 36px;
|
||||
border-radius: 50%;
|
||||
background: var(--surface);
|
||||
border: 1px solid var(--border);
|
||||
color: #FFFFFF;
|
||||
font-size: 17px;
|
||||
cursor: pointer;
|
||||
z-index: 251;
|
||||
box-shadow: 0 4px 14px rgba(0,0,0,0.4);
|
||||
transition: all 0.15s;
|
||||
display: none;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
.back-to-top:hover { border-color: var(--accent); }
|
||||
.back-to-top.visible { display: flex; }
|
||||
.help-toggle:hover { color: var(--text); border-color: var(--accent); }
|
||||
.help-popup {
|
||||
position: fixed;
|
||||
@@ -290,6 +313,23 @@
|
||||
.age-badge.age-recent { color: var(--warn); border-color: #78350f; background: #1c0e02; }
|
||||
.age-badge.age-established { color: var(--success); border-color: #14532d; background: #052011; }
|
||||
|
||||
/* What kind of X/Twitter page a Google CSE / Wayback result actually is —
|
||||
a keyword match linking to x.com/someone reads as "just a profile" with
|
||||
no way to tell a tweet permalink apart from a bare profile page or an
|
||||
unrelated page otherwise. */
|
||||
.content-type-badge {
|
||||
display: inline-block;
|
||||
font-size: 11px;
|
||||
font-weight: 500;
|
||||
padding: 2px 9px;
|
||||
border-radius: 20px;
|
||||
border: 1px solid var(--border);
|
||||
}
|
||||
.content-type-badge.ct-tweet { color: var(--success); border-color: #14532d; background: #052011; }
|
||||
.content-type-badge.ct-profile { color: #a78bfa; border-color: #4c3a8f; background: var(--cookie-bg); }
|
||||
.content-type-badge.ct-twitter_other { color: var(--warn); border-color: #78350f; background: #1c0e02; }
|
||||
.content-type-badge.ct-other { color: var(--muted); border-color: var(--border); background: var(--bg); }
|
||||
|
||||
.search-bar {
|
||||
flex: 1;
|
||||
min-width: 160px;
|
||||
@@ -322,38 +362,6 @@
|
||||
}
|
||||
.btn-download:hover { color: var(--text); border-color: var(--accent); }
|
||||
|
||||
/* Auto Archive toggle */
|
||||
.archive-toggle {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
margin-top: 10px;
|
||||
padding-top: 10px;
|
||||
border-top: 1px solid var(--border);
|
||||
cursor: pointer;
|
||||
user-select: none;
|
||||
}
|
||||
.toggle-track {
|
||||
width: 32px;
|
||||
height: 18px;
|
||||
border-radius: 9px;
|
||||
background: var(--border);
|
||||
position: relative;
|
||||
transition: background 0.2s;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
.toggle-track.on { background: var(--accent); }
|
||||
.toggle-knob {
|
||||
position: absolute;
|
||||
top: 2px; left: 2px;
|
||||
width: 14px; height: 14px;
|
||||
border-radius: 50%;
|
||||
background: #fff;
|
||||
transition: left 0.2s;
|
||||
}
|
||||
.toggle-track.on .toggle-knob { left: 16px; }
|
||||
.toggle-label { font-size: 12px; color: var(--muted); }
|
||||
|
||||
/* Archive progress bar */
|
||||
.archive-bar {
|
||||
margin-top: 8px;
|
||||
@@ -411,9 +419,91 @@
|
||||
border-radius: 8px;
|
||||
padding: 12px 14px;
|
||||
transition: border-color 0.15s;
|
||||
overflow: hidden;
|
||||
}
|
||||
.card:hover { border-color: #3a3d50; }
|
||||
|
||||
/* Profile header — cover photo, avatar, name/handle, bio. Only rendered
|
||||
when the item actually carries any of that (a tweet's embedded author,
|
||||
a bare user/follower/retweeter record); everything else keeps the plain
|
||||
row list it always had. */
|
||||
.card-header { margin-bottom: 10px; }
|
||||
.card-header.has-banner {
|
||||
margin: -12px -14px 10px;
|
||||
}
|
||||
.card-banner {
|
||||
height: 64px;
|
||||
background-size: cover;
|
||||
background-position: center;
|
||||
background-color: var(--bg);
|
||||
}
|
||||
.card-header-row {
|
||||
display: flex;
|
||||
align-items: flex-end;
|
||||
gap: 10px;
|
||||
}
|
||||
.card-header.has-banner .card-header-row {
|
||||
padding: 0 14px;
|
||||
margin-top: -26px;
|
||||
}
|
||||
.card-avatar {
|
||||
width: 48px;
|
||||
height: 48px;
|
||||
border-radius: 50%;
|
||||
object-fit: cover;
|
||||
border: 2px solid var(--surface);
|
||||
background: var(--bg);
|
||||
flex-shrink: 0;
|
||||
}
|
||||
.card-avatar-fallback {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-size: 17px;
|
||||
font-weight: 700;
|
||||
color: var(--muted);
|
||||
}
|
||||
.card-identity { min-width: 0; padding-bottom: 3px; }
|
||||
.card-name {
|
||||
font-size: 14px;
|
||||
font-weight: 600;
|
||||
color: var(--text);
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
.card-handle { font-size: 12px; color: var(--muted); }
|
||||
.card-bio {
|
||||
margin-top: 8px;
|
||||
font-size: 12px;
|
||||
color: var(--text);
|
||||
opacity: 0.85;
|
||||
line-height: 1.45;
|
||||
}
|
||||
.card-header.has-banner .card-bio { padding: 0 14px; }
|
||||
|
||||
/* Compact byline — tweets/replies (already lead with their own text a few
|
||||
rows down) get just this instead of the full banner+bio header above,
|
||||
which is reserved for records that ARE a user rather than a tweet
|
||||
someone wrote. */
|
||||
.card-byline {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
.card-byline .card-avatar { width: 26px; height: 26px; border-width: 1px; }
|
||||
.card-byline .card-avatar-fallback { font-size: 11px; }
|
||||
.card-byline .card-identity {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
gap: 6px;
|
||||
padding-bottom: 0;
|
||||
min-width: 0;
|
||||
}
|
||||
.card-byline .card-name { font-size: 13px; max-width: 55%; }
|
||||
.card-byline .card-handle { flex-shrink: 0; }
|
||||
|
||||
.card-row {
|
||||
display: flex;
|
||||
gap: 10px;
|
||||
@@ -514,6 +604,16 @@
|
||||
.card-row.rt-origin .card-key { color: var(--accent); }
|
||||
.card-row.rt-origin .card-val { color: #c7d2fe; }
|
||||
|
||||
/* "Replying to @x @y" — x.com trims these from the visible tweet body and
|
||||
shows them as their own line instead; matches that here so the reply
|
||||
text itself isn't misread as having been written with those mentions
|
||||
inline. Underlying text value is untouched, this is display-only. */
|
||||
.reply-context {
|
||||
font-size: 11px;
|
||||
color: var(--muted);
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
|
||||
/* Drill-down links on reply / retweet counts */
|
||||
.drill-link {
|
||||
color: var(--accent);
|
||||
@@ -524,6 +624,49 @@
|
||||
}
|
||||
.drill-link:hover { color: #818cf8; }
|
||||
|
||||
/* Inline nested-reply threads — expand a reply's own replies in place,
|
||||
recursively, instead of only linking out to a new tab. */
|
||||
.reply-expand-btn {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
margin-top: 10px;
|
||||
background: none;
|
||||
border: 1px solid var(--border);
|
||||
color: var(--accent);
|
||||
font-family: var(--font);
|
||||
font-size: 11px;
|
||||
font-weight: 500;
|
||||
padding: 4px 10px;
|
||||
border-radius: 5px;
|
||||
cursor: pointer;
|
||||
}
|
||||
.reply-expand-btn:hover:not(:disabled) { border-color: var(--accent); }
|
||||
.reply-expand-btn:disabled { opacity: 0.5; cursor: wait; }
|
||||
|
||||
.reply-thread {
|
||||
margin: 10px 0 2px 16px;
|
||||
padding-left: 14px;
|
||||
border-left: 2px solid var(--border);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
}
|
||||
.reply-thread.hidden { display: none; }
|
||||
|
||||
.reply-load-more-btn {
|
||||
align-self: flex-start;
|
||||
background: none;
|
||||
border: none;
|
||||
color: var(--accent);
|
||||
font-family: var(--font);
|
||||
font-size: 11px;
|
||||
cursor: pointer;
|
||||
text-decoration: underline;
|
||||
text-underline-offset: 2px;
|
||||
padding: 2px 0;
|
||||
}
|
||||
.reply-load-more-btn:disabled { opacity: 0.5; cursor: wait; text-decoration: none; }
|
||||
|
||||
/* View toggle (Cards / Map) */
|
||||
.view-toggle {
|
||||
display: flex;
|
||||
@@ -582,24 +725,31 @@
|
||||
<header>
|
||||
<h1>Jieyab89 SOCMINT X</h1>
|
||||
<span class="sep">|</span>
|
||||
<span class="sub">Retrieve data using Xquik API or Cookie</span>
|
||||
<span class="sub">Retrieve Twitter Data With Many Data Source and Sentiment Analysis</span>
|
||||
<button id="navToggle" class="hamburger-btn" aria-label="Navigation menu">☰</button>
|
||||
<nav id="navMenu" class="nav-menu hidden">
|
||||
<a href="/" class="current">Home</a>
|
||||
<a href="/graph">Graph</a>
|
||||
<hr class="nav-divider">
|
||||
<a href="/archives">Archives</a>
|
||||
<a href="/analytics">Analytics</a>
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<!-- Legend / help popup (floats over the page, bottom-left) -->
|
||||
<button id="helpToggle" class="help-toggle" aria-label="Legend / help" title="Legend / help">?</button>
|
||||
<button id="backToTop" class="back-to-top" aria-label="Back to top" title="Back to top">↑</button>
|
||||
<div id="helpPopup" class="help-popup hidden">
|
||||
<h4>Source</h4>
|
||||
<div class="help-row"><span class="source-badge src-cookie">Twitter Cookie</span> Live via your cookie session</div>
|
||||
<div class="help-row"><span class="source-badge src-xquik">Xquik API</span> Live via the xquik API</div>
|
||||
<div class="help-row"><span class="source-badge src-wayback">Wayback Machine</span> Archived snapshot</div>
|
||||
<div class="help-row"><span class="source-badge src-cse">Google CSE</span> Live web result via Google Custom Search</div>
|
||||
<h4>Content type — Google CSE / Wayback only</h4>
|
||||
<div class="help-row"><span class="content-type-badge ct-tweet">Tweet</span> A specific tweet permalink</div>
|
||||
<div class="help-row"><span class="content-type-badge ct-profile">Profile page</span> A bare account profile, not a specific post</div>
|
||||
<div class="help-row"><span class="content-type-badge ct-twitter_other">Other X/Twitter page</span> Some other X page</div>
|
||||
<div class="help-row"><span class="content-type-badge ct-other">External page (non-X)</span> A site off X entirely (Google CSE only)</div>
|
||||
<h4>Account age</h4>
|
||||
<div class="help-row"><span class="age-badge age-new">New account</span> Created < 30 days ago</div>
|
||||
<div class="help-row"><span class="age-badge age-recent">Recent account</span> Created < 1 year ago</div>
|
||||
@@ -622,6 +772,7 @@
|
||||
<select id="toolType">
|
||||
<option value="tweet_search_extractor">Tweet Search Extractor</option>
|
||||
<option value="follower_explorer">Follower Explorer</option>
|
||||
<option value="following_explorer">Following Explorer [Cookie]</option>
|
||||
<option value="post_extractor">Post Extractor</option>
|
||||
<option value="article_extractor">Article Extractor</option>
|
||||
<option value="community_post_extractor">community Post Extractor</option>
|
||||
@@ -640,17 +791,7 @@
|
||||
|
||||
<button class="btn-run" id="runBtn">Run</button>
|
||||
|
||||
<!-- Auto Archive toggle -->
|
||||
<div class="archive-toggle" id="archiveToggle">
|
||||
<div class="toggle-track" id="toggleTrack">
|
||||
<div class="toggle-knob"></div>
|
||||
</div>
|
||||
<span class="toggle-label">Auto Archive</span>
|
||||
</div>
|
||||
<div id="archiveBar" class="archive-bar" style="display:none"></div>
|
||||
<button id="checkpointBtn" class="btn-download" style="display:none;width:100%;margin-top:8px">
|
||||
Save Checkpoint... Saved latest data
|
||||
</button>
|
||||
|
||||
<!-- Recent archives -->
|
||||
<div class="archives-section" id="archivesSection" style="display:none">
|
||||
@@ -668,6 +809,7 @@
|
||||
<button class="view-btn active" data-view="cards">Cards</button>
|
||||
<button class="view-btn" data-view="map">Map</button>
|
||||
</div>
|
||||
<button class="btn-download" id="archiveBtn" style="display:none">Archive</button>
|
||||
<button class="btn-download" id="downloadBtn" style="display:none">↓ JSON</button>
|
||||
</div>
|
||||
<div id="resultBox">
|
||||
@@ -705,6 +847,18 @@
|
||||
popup.addEventListener('click', function(e) { e.stopPropagation(); });
|
||||
}());
|
||||
|
||||
// Back to top — the page (not an inner pane) is what actually scrolls here,
|
||||
// same as the load-more sentinel further down relies on.
|
||||
(function() {
|
||||
var btn = document.getElementById('backToTop');
|
||||
window.addEventListener('scroll', function() {
|
||||
btn.classList.toggle('visible', window.scrollY > 400);
|
||||
});
|
||||
btn.addEventListener('click', function() {
|
||||
window.scrollTo({ top: 0, behavior: 'smooth' });
|
||||
});
|
||||
}());
|
||||
|
||||
const toolType = document.getElementById('toolType');
|
||||
const dynFields = document.getElementById('dynamicFields');
|
||||
const runBtn = document.getElementById('runBtn');
|
||||
@@ -714,29 +868,31 @@ const searchInput = document.getElementById('searchInput');
|
||||
const resultCount = document.getElementById('resultCount');
|
||||
const toolHint = document.getElementById('toolHint');
|
||||
const downloadBtn = document.getElementById('downloadBtn');
|
||||
const archiveToggle = document.getElementById('archiveToggle');
|
||||
const toggleTrack = document.getElementById('toggleTrack');
|
||||
const archiveBtn = document.getElementById('archiveBtn');
|
||||
const archiveBar = document.getElementById('archiveBar');
|
||||
const archivesSection= document.getElementById('archivesSection');
|
||||
const archivesList = document.getElementById('archivesList');
|
||||
const loadMoreSentinel = document.getElementById('loadMoreSentinel');
|
||||
const loadMoreStatus = document.getElementById('loadMoreStatus');
|
||||
const checkpointBtn = document.getElementById('checkpointBtn');
|
||||
|
||||
let currentMode = 'api';
|
||||
let currentData = null;
|
||||
let autoArchive = false;
|
||||
let currentPayload = null; // last run payload (for archive queryInfo)
|
||||
let archivedId = null; // archive id for the current run, once auto-archived — lets later load-more pages update it in place instead of creating a new archive
|
||||
// Archive id for the CURRENT run's data — same model as graph.html's
|
||||
// graphArchivedId: null until the Archive button is clicked once (creates),
|
||||
// then every later click (after scrolling/expanding loads more) checkpoints
|
||||
// that same archive in place instead of creating a new one each time. Reset
|
||||
// on every new Run, same as graph.html's clearGraph() resets its own.
|
||||
let archivedId = null;
|
||||
|
||||
// ── Scroll-triggered load-more (Cookie mode + Wayback only) ────────────────
|
||||
// Tools/modes the backend actually paginates — everything else (xquik/API
|
||||
// mode, multi-source search) just gets a single page, same as before.
|
||||
// ── Scroll-triggered load-more ──────────────────────────────────────────────
|
||||
// Tools/modes the backend actually paginates — everything else (plain xquik/API
|
||||
// mode calls, article_extractor) just gets a single page.
|
||||
const PAGINATED_TOOLS = new Set([
|
||||
'tweet_search_extractor', 'follower_explorer', 'post_extractor',
|
||||
'tweet_search_extractor', 'follower_explorer', 'following_explorer', 'post_extractor',
|
||||
'community_post_extractor', 'tweet_replies_extractor',
|
||||
'tweet_retweeters_extractor', 'geo_post_extractor',
|
||||
'wayback_archive_search',
|
||||
'wayback_archive_search', 'multi_source_search',
|
||||
]);
|
||||
const THROTTLE_SECONDS = 5;
|
||||
|
||||
@@ -745,15 +901,21 @@ let loadingMore = false;
|
||||
let cooldownUntil = { cookie: 0, wayback: 0 }; // Date.now()-based timestamps
|
||||
let cooldownTimer = null;
|
||||
|
||||
// Returns the list of rate-limit clocks a request against this tool/mode
|
||||
// touches. multi_source_search fans out to both cookie and wayback
|
||||
// internally, so a load-more page for it has to wait out (and then restart)
|
||||
// both cooldowns at once, not just one.
|
||||
function throttleSourceFor(tool, mode) {
|
||||
if (tool === 'wayback_archive_search') return 'wayback';
|
||||
if (mode === 'cookie' && PAGINATED_TOOLS.has(tool)) return 'cookie';
|
||||
return null;
|
||||
if (tool === 'wayback_archive_search') return ['wayback'];
|
||||
if (tool === 'multi_source_search') return ['cookie', 'wayback'];
|
||||
if (mode === 'cookie' && PAGINATED_TOOLS.has(tool)) return ['cookie'];
|
||||
return [];
|
||||
}
|
||||
|
||||
const HINTS = {
|
||||
tweet_search_extractor: 'Search for tweets by keyword',
|
||||
follower_explorer: 'List followers of an account',
|
||||
following_explorer: 'List accounts a user follows — cookie only',
|
||||
post_extractor: 'Tweet timeline from an account',
|
||||
article_extractor: 'Content of an X article from its tweet URL',
|
||||
community_post_extractor: 'Posts from an X Community',
|
||||
@@ -769,6 +931,7 @@ const COOKIE_ONLY = new Set([
|
||||
'tweet_replies_extractor',
|
||||
'tweet_retweeters_extractor',
|
||||
'geo_post_extractor',
|
||||
'following_explorer',
|
||||
]);
|
||||
|
||||
// These tools don't use the single Mode toggle: wayback needs no auth at all,
|
||||
@@ -781,6 +944,7 @@ const NO_AUTH = new Set([
|
||||
const HAS_COUNT = new Set([
|
||||
'tweet_search_extractor',
|
||||
'follower_explorer',
|
||||
'following_explorer',
|
||||
'post_extractor',
|
||||
'community_post_extractor',
|
||||
'tweet_replies_extractor',
|
||||
@@ -823,7 +987,7 @@ function renderFields() {
|
||||
// Tool-specific input fields
|
||||
if (t === 'tweet_search_extractor') {
|
||||
html += field('searchQuery', 'Search query', 'bitcoin');
|
||||
} else if (t === 'follower_explorer') {
|
||||
} else if (t === 'follower_explorer' || t === 'following_explorer') {
|
||||
html += field('targetUsername', 'Username or User ID', 'elonmusk or 44196397');
|
||||
} else if (t === 'article_extractor') {
|
||||
html += field('targetTweetId', 'Tweet ID', '1234567890');
|
||||
@@ -850,12 +1014,12 @@ function renderFields() {
|
||||
const showCount = NO_AUTH.has(t) || currentMode === 'cookie';
|
||||
const countLabels = {
|
||||
wayback_archive_search: 'Max snapshots',
|
||||
multi_source_search: 'Result count per source (max 200)',
|
||||
multi_source_search: 'Result count per source (max 2000)',
|
||||
};
|
||||
html += `
|
||||
<div class="field" id="countWrap" style="${showCount ? '' : 'display:none'}">
|
||||
<label>${countLabels[t] || 'Result count (max 200)'}</label>
|
||||
<input id="countInput" type="number" value="${NO_AUTH.has(t) ? 50 : 20}" min="1" max="200">
|
||||
<label>${countLabels[t] || 'Result count (max 2000)'}</label>
|
||||
<input id="countInput" type="number" value="${NO_AUTH.has(t) ? 50 : 20}" min="1" max="2000">
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
@@ -886,6 +1050,7 @@ function attachModeListeners() {
|
||||
|
||||
toolType.addEventListener('change', renderFields);
|
||||
renderFields();
|
||||
loadRecentArchives();
|
||||
|
||||
document.getElementById('viewToggle').addEventListener('click', e => {
|
||||
const btn = e.target.closest('[data-view]');
|
||||
@@ -975,12 +1140,109 @@ function renderCards(data, query = '') {
|
||||
resultBox.innerHTML = '<div class="cards-grid">' + filtered.map(buildCard).join('') + '</div>';
|
||||
}
|
||||
|
||||
function buildCard(item) {
|
||||
// Only a plain https URL (no quotes/angle-brackets/whitespace/parens) is
|
||||
// ever interpolated into the CSS url('...') below — background-image goes
|
||||
// through a second parsing pass CSS-side, so HTML-attribute escaping alone
|
||||
// isn't sufficient there the way it is for a plain <img src>. Rejecting
|
||||
// anything but a clean https URL up front closes that off rather than
|
||||
// trying to escape a value for two contexts (HTML attribute + CSS token)
|
||||
// wedged into one string.
|
||||
function isSafeImageUrl(u) {
|
||||
return typeof u === 'string' && /^https:\/\/[^\s'"<>()]+$/.test(u);
|
||||
}
|
||||
|
||||
// Tapping "Reply" on X auto-prefixes the compose box with every account the
|
||||
// reply-chain already has tagged, and that prefix is genuinely part of the
|
||||
// reply's own raw text — but x.com's own UI never shows it inline, trimming
|
||||
// it into a separate "Replying to @x @y" line instead. `reply_to_mentions`
|
||||
// (from cookie_client.py's _leading_reply_mentions) lists exactly which
|
||||
// leading @mentions those are; this only touches how the text RENDERS here,
|
||||
// never item.text itself, so JSON download / archive still get the
|
||||
// complete raw string untouched. mentions.length is walked in order and
|
||||
// stops at the first one that doesn't match — a data mismatch just leaves
|
||||
// the rest of the prefix visible instead of risking mangled text.
|
||||
function stripLeadingMentions(text, mentions) {
|
||||
if (!text || !mentions || !mentions.length) return text;
|
||||
let rest = text;
|
||||
for (const m of mentions) {
|
||||
const re = new RegExp('^\\s*@' + String(m).replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '\\b');
|
||||
const match = rest.match(re);
|
||||
if (!match) break;
|
||||
rest = rest.slice(match[0].length);
|
||||
}
|
||||
rest = rest.replace(/^\s+/, '');
|
||||
return rest || text;
|
||||
}
|
||||
|
||||
// Returns { html, usedFields } instead of just a string — buildCard() below
|
||||
// needs to know exactly which raw keys actually ended up rendered in the
|
||||
// header so it can drop only THOSE from the generic row list. A static
|
||||
// "always hide these field names" list doesn't work here: CSE/Wayback
|
||||
// records also have a `description` field (Google's own snippet, renamed
|
||||
// from serp_snippet) that this header never touches (no avatar/name/handle
|
||||
// on those records, so it returns empty) — hiding it unconditionally would
|
||||
// have silently deleted the one thing the user asked to see more clearly.
|
||||
function buildCardHeader(item) {
|
||||
const avatarRaw = item.avatar || item.user_avatar || '';
|
||||
const avatar = isSafeImageUrl(avatarRaw) ? avatarRaw : '';
|
||||
const name = item.name || '';
|
||||
const handle = item.screen_name || item.user || item.username || '';
|
||||
|
||||
if (!avatar && !name && !handle) return { html: '', usedFields: [] };
|
||||
|
||||
const usedFields = [];
|
||||
if (avatar) usedFields.push(item.avatar ? 'avatar' : 'user_avatar');
|
||||
if (name) usedFields.push('name');
|
||||
if (handle) usedFields.push(item.screen_name ? 'screen_name' : item.user ? 'user' : 'username');
|
||||
|
||||
const avatarHtml = avatar
|
||||
? `<img class="card-avatar" src="${esc(avatar)}" alt="" loading="lazy" referrerpolicy="no-referrer">`
|
||||
: (name || handle)
|
||||
? `<div class="card-avatar card-avatar-fallback">${esc((name || handle).charAt(0).toUpperCase())}</div>`
|
||||
: '';
|
||||
const identityHtml = (name || handle)
|
||||
? `<div class="card-identity">
|
||||
${name ? `<div class="card-name">${esc(name)}</div>` : ''}
|
||||
${handle ? `<div class="card-handle">@${esc(handle)}</div>` : ''}
|
||||
</div>`
|
||||
: '';
|
||||
|
||||
// A tweet or reply already leads with its own text a few rows down — a
|
||||
// full cover-photo-plus-bio header buries that under the *author's*
|
||||
// profile instead of the actual reply content (this is what made a reply
|
||||
// thread look like it was rendering "just the user"). Those get a small
|
||||
// inline byline only; the full profile-card treatment (banner + bio) is
|
||||
// reserved for records that ARE a user — follower/retweeter results, not
|
||||
// tweets a user happened to write.
|
||||
const isTweetLike = item.text !== undefined || item.full_text !== undefined || item.article_text !== undefined;
|
||||
if (isTweetLike) {
|
||||
return { html: `<div class="card-byline">${avatarHtml}${identityHtml}</div>`, usedFields };
|
||||
}
|
||||
|
||||
const bannerRaw = item.banner || item.user_banner || '';
|
||||
const banner = isSafeImageUrl(bannerRaw) ? bannerRaw : '';
|
||||
const bio = item.description || item.user_bio || '';
|
||||
if (banner) usedFields.push(item.banner ? 'banner' : 'user_banner');
|
||||
if (bio) usedFields.push(item.description ? 'description' : 'user_bio');
|
||||
|
||||
const bioHtml = bio ? `<div class="card-bio">${esc(bio)}</div>` : '';
|
||||
const bannerHtml = banner
|
||||
? `<div class="card-banner" style="background-image:url('${esc(banner)}')"></div>`
|
||||
: '';
|
||||
|
||||
return {
|
||||
html: `<div class="card-header${banner ? ' has-banner' : ''}">${bannerHtml}<div class="card-header-row">${avatarHtml}${identityHtml}</div>${bioHtml}</div>`,
|
||||
usedFields,
|
||||
};
|
||||
}
|
||||
|
||||
function buildCard(item, nested = false) {
|
||||
if (typeof item !== 'object' || item === null) {
|
||||
return `<div class="card"><div class="card-row"><div class="card-val">${esc(String(item))}</div></div></div>`;
|
||||
}
|
||||
const tweetId = item.id ? String(item.id) : null;
|
||||
const entries = Object.entries(item);
|
||||
const { html: header, usedFields: headerFields } = buildCardHeader(item);
|
||||
// Order follows PRIORITY's own sequence (not object insertion order), so
|
||||
// field position is deterministic and consistent across every tool/record
|
||||
// shape. No cap on row count — this is an OSINT/analysis tool, so more
|
||||
@@ -989,8 +1251,8 @@ function buildCard(item) {
|
||||
// anchors once enough other fields — account age, verification, etc. —
|
||||
// were present on a record).
|
||||
const priKeys = new Set(entries.map(([k]) => k).filter(k => PRIORITY.includes(k)));
|
||||
const pri = PRIORITY.filter(k => priKeys.has(k)).map(k => [k, item[k]]);
|
||||
const rest = entries.filter(([k]) => !PRIORITY.includes(k) && !SKIP.includes(k));
|
||||
const pri = PRIORITY.filter(k => priKeys.has(k) && !headerFields.includes(k)).map(k => [k, item[k]]);
|
||||
const rest = entries.filter(([k]) => !PRIORITY.includes(k) && !SKIP.includes(k) && !headerFields.includes(k));
|
||||
const rows = [...pri, ...rest].map(([k, v]) => {
|
||||
let display;
|
||||
// Drillable check runs FIRST — reply_count / retweet_count always get a link
|
||||
@@ -1009,6 +1271,12 @@ function buildCard(item) {
|
||||
display = `<span class="source-badge ${cls}">${esc(String(v))}</span>`;
|
||||
} else if (k === 'account_age_flag' && v) {
|
||||
display = `<span class="age-badge age-${esc(String(v))}">${esc(AGE_LABELS[v] || v)}</span>`;
|
||||
} else if (k === 'content_type' && v) {
|
||||
display = `<span class="content-type-badge ct-${esc(String(v))}">${esc(CONTENT_TYPE_LABELS[v] || String(v))}</span>`;
|
||||
} else if (k === 'text' && item.reply_to_mentions && item.reply_to_mentions.length) {
|
||||
const badge = `<div class="reply-context">↩ Replying to ${item.reply_to_mentions.map(m => '@' + esc(m)).join(', ')}</div>`;
|
||||
const clean = stripLeadingMentions(String(v), item.reply_to_mentions);
|
||||
display = `${badge}<span class="clamp">${esc(clean)}</span>`;
|
||||
} else if (v === null || v === undefined) {
|
||||
display = `<span style="color:var(--muted)">—</span>`;
|
||||
} else if (typeof v === 'object') {
|
||||
@@ -1022,7 +1290,24 @@ function buildCard(item) {
|
||||
return `<div class="card-row${rtCls}"><div class="card-key">${esc(k)}</div><div class="card-val">${display}</div></div>`;
|
||||
}).join('');
|
||||
const media = extractMedia(item);
|
||||
return `<div class="card">${rows}${renderMedia(media)}</div>`;
|
||||
|
||||
// Any tweet-shaped item with replies of its own can be expanded in place —
|
||||
// same tweet_replies_extractor call graph.html's node-expand uses, applies
|
||||
// uniformly across every tool's cards (search results, timeline, replies
|
||||
// themselves, ...) since it only keys off id + reply_count, not the tool
|
||||
// that produced the card. A reply rendered here recursively gets its own
|
||||
// Expand button too, since it's built by this same function.
|
||||
const replyCount = Number(item.reply_count);
|
||||
const canExpandReplies = tweetId && Number.isFinite(replyCount) && replyCount > 0;
|
||||
const expandBtn = canExpandReplies
|
||||
? `<button type="button" class="reply-expand-btn" data-tweet-id="${esc(tweetId)}" data-count="${replyCount}">↩ Expand ${replyCount} repl${replyCount === 1 ? 'y' : 'ies'}</button>`
|
||||
: '';
|
||||
|
||||
const cardCls = nested ? 'card reply-card' : 'card';
|
||||
const card = `<div class="${cardCls}">${header}${rows}${renderMedia(media)}${expandBtn}</div>`;
|
||||
if (!canExpandReplies) return card;
|
||||
|
||||
return `<div class="card-thread-wrap">${card}<div class="reply-thread hidden" data-tweet-id="${esc(tweetId)}"></div></div>`;
|
||||
}
|
||||
|
||||
// ── Load more (scroll-triggered pagination) ─────────────────────────────────
|
||||
@@ -1037,18 +1322,167 @@ function appendCards(newItems, query = '') {
|
||||
if (grid && toShow.length) {
|
||||
grid.insertAdjacentHTML('beforeend', toShow.map(buildCard).join(''));
|
||||
}
|
||||
const totalShown = resultBox.querySelectorAll('.card').length;
|
||||
// Excludes .reply-card — nested replies rendered inline via expand threads
|
||||
// aren't part of "the results," same reasoning renderCards' own count
|
||||
// (items.length, never a DOM query) already never counted them either.
|
||||
const totalShown = resultBox.querySelectorAll('.card:not(.reply-card)').length;
|
||||
resultCount.textContent = totalShown + (q ? ' found' : ' results');
|
||||
}
|
||||
|
||||
// ── Inline nested-reply threads ─────────────────────────────────────────────
|
||||
// Delegated once at the container level — buildCard() renders replies
|
||||
// recursively via itself, so an expand button can appear at any nesting
|
||||
// depth; delegation catches all of them without per-card listener wiring.
|
||||
resultBox.addEventListener('click', (e) => {
|
||||
const expandBtn = e.target.closest('.reply-expand-btn');
|
||||
if (expandBtn) { toggleReplyThread(expandBtn); return; }
|
||||
const moreBtn = e.target.closest('.reply-load-more-btn');
|
||||
if (moreBtn) { loadMoreReplies(moreBtn); }
|
||||
});
|
||||
|
||||
function replyBtnLabel(count) {
|
||||
return `↩ Expand ${count} repl${count === 1 ? 'y' : 'ies'}`;
|
||||
}
|
||||
|
||||
async function toggleReplyThread(btn) {
|
||||
const wrap = btn.closest('.card-thread-wrap');
|
||||
const thread = wrap.querySelector(':scope > .reply-thread');
|
||||
|
||||
// Already fetched once — just show/hide, no new request.
|
||||
if (thread.dataset.loaded === '1') {
|
||||
const nowHidden = thread.classList.toggle('hidden');
|
||||
btn.textContent = nowHidden ? replyBtnLabel(Number(btn.dataset.count)) : '▲ Hide replies';
|
||||
return;
|
||||
}
|
||||
|
||||
const tweetId = btn.dataset.tweetId;
|
||||
const remaining = cooldownUntil.cookie - Date.now();
|
||||
if (remaining > 0) {
|
||||
armReplyCountdown(btn, remaining, () => replyBtnLabel(Number(btn.dataset.count)));
|
||||
return;
|
||||
}
|
||||
|
||||
btn.disabled = true;
|
||||
btn.textContent = 'Loading…';
|
||||
|
||||
try {
|
||||
const res = await fetch('/api/run', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ toolType: 'tweet_replies_extractor', mode: 'cookie', count: 20, targetTweetId: tweetId }),
|
||||
});
|
||||
const json = await res.json();
|
||||
stampCooldown(['cookie']);
|
||||
|
||||
if (!json.ok) {
|
||||
if (res.status === 429 && json.retryAfter) {
|
||||
armReplyCountdown(btn, json.retryAfter * 1000, () => replyBtnLabel(Number(btn.dataset.count)));
|
||||
return;
|
||||
}
|
||||
btn.disabled = false;
|
||||
btn.textContent = 'Failed — retry';
|
||||
return;
|
||||
}
|
||||
|
||||
const items = Array.isArray(json.data) ? json.data : [json.data];
|
||||
thread.innerHTML = items.map(it => buildCard(it, true)).join('');
|
||||
thread.dataset.loaded = '1';
|
||||
if (json.nextCursor) {
|
||||
thread.dataset.cursor = json.nextCursor;
|
||||
thread.insertAdjacentHTML('beforeend',
|
||||
`<button type="button" class="reply-load-more-btn" data-tweet-id="${esc(tweetId)}">Load more replies…</button>`);
|
||||
}
|
||||
thread.classList.remove('hidden');
|
||||
btn.disabled = !items.length;
|
||||
btn.textContent = items.length ? '▲ Hide replies' : 'No replies found';
|
||||
|
||||
// Fold newly-discovered replies into the same dataset the JSON download
|
||||
// and Archive button both read from — otherwise data pulled via inline
|
||||
// expand would silently be excluded from both.
|
||||
if (items.length) {
|
||||
currentData = (Array.isArray(currentData) ? currentData : [currentData]).concat(items);
|
||||
}
|
||||
} catch (err) {
|
||||
btn.disabled = false;
|
||||
btn.textContent = 'Failed — retry';
|
||||
}
|
||||
}
|
||||
|
||||
async function loadMoreReplies(moreBtn) {
|
||||
const thread = moreBtn.closest('.reply-thread');
|
||||
const tweetId = moreBtn.dataset.tweetId;
|
||||
const cursor = thread.dataset.cursor;
|
||||
if (!cursor) { moreBtn.remove(); return; }
|
||||
|
||||
const remaining = cooldownUntil.cookie - Date.now();
|
||||
if (remaining > 0) {
|
||||
armReplyCountdown(moreBtn, remaining, () => 'Load more replies…');
|
||||
return;
|
||||
}
|
||||
|
||||
moreBtn.disabled = true;
|
||||
moreBtn.textContent = 'Loading…';
|
||||
|
||||
try {
|
||||
const res = await fetch('/api/run', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ toolType: 'tweet_replies_extractor', mode: 'cookie', count: 20, targetTweetId: tweetId, cursor }),
|
||||
});
|
||||
const json = await res.json();
|
||||
stampCooldown(['cookie']);
|
||||
|
||||
if (!json.ok) {
|
||||
if (res.status === 429 && json.retryAfter) {
|
||||
armReplyCountdown(moreBtn, json.retryAfter * 1000, () => 'Load more replies…');
|
||||
return;
|
||||
}
|
||||
moreBtn.disabled = false;
|
||||
moreBtn.textContent = 'Failed — retry';
|
||||
return;
|
||||
}
|
||||
|
||||
const items = Array.isArray(json.data) ? json.data : [json.data];
|
||||
moreBtn.insertAdjacentHTML('beforebegin', items.map(it => buildCard(it, true)).join(''));
|
||||
if (json.nextCursor) {
|
||||
thread.dataset.cursor = json.nextCursor;
|
||||
moreBtn.disabled = false;
|
||||
moreBtn.textContent = 'Load more replies…';
|
||||
} else {
|
||||
moreBtn.remove();
|
||||
}
|
||||
|
||||
if (items.length) {
|
||||
currentData = (Array.isArray(currentData) ? currentData : [currentData]).concat(items);
|
||||
}
|
||||
} catch (err) {
|
||||
moreBtn.disabled = false;
|
||||
moreBtn.textContent = 'Failed — retry';
|
||||
}
|
||||
}
|
||||
|
||||
// Shared by both reply-expand entry points above — mirrors armCountdown()
|
||||
// but scoped to a single button instead of the page-wide loadMoreStatus bar,
|
||||
// since several threads can be mid-cooldown independently of each other.
|
||||
function armReplyCountdown(btn, msRemaining, resetLabel) {
|
||||
const secs = Math.max(1, Math.ceil(msRemaining / 1000));
|
||||
btn.disabled = true;
|
||||
btn.textContent = `Wait ${secs}s…`;
|
||||
setTimeout(() => {
|
||||
btn.disabled = false;
|
||||
btn.textContent = resetLabel();
|
||||
}, msRemaining);
|
||||
}
|
||||
|
||||
function resetPagination() {
|
||||
nextCursor = null;
|
||||
loadingMore = false;
|
||||
archivedId = null;
|
||||
archiveBtn.textContent = 'Archive';
|
||||
archiveBar.style.display = 'none';
|
||||
if (cooldownTimer) { clearTimeout(cooldownTimer); cooldownTimer = null; }
|
||||
loadMoreStatus.classList.remove('visible');
|
||||
loadMoreStatus.textContent = '';
|
||||
checkpointBtn.style.display = 'none';
|
||||
}
|
||||
|
||||
function buildQueryInfo() {
|
||||
@@ -1059,39 +1493,63 @@ function buildQueryInfo() {
|
||||
return queryInfo;
|
||||
}
|
||||
|
||||
function stampCooldown(source) {
|
||||
if (!source) return;
|
||||
function stampCooldown(sources) {
|
||||
(sources || []).forEach(source => {
|
||||
cooldownUntil[source] = Date.now() + THROTTLE_SECONDS * 1000;
|
||||
});
|
||||
}
|
||||
|
||||
function currentItemCount() {
|
||||
return Array.isArray(currentData) ? currentData.length : (currentData ? 1 : 0);
|
||||
}
|
||||
|
||||
// count is "how many items I want," not just a per-page size hint — X's own
|
||||
// API caps how many it actually returns per page regardless of what's
|
||||
// requested (count=200 still only returns ~20 on the first page), so
|
||||
// reaching the requested count used to mean manually scrolling the sentinel
|
||||
// into view over and over. This chains load-more calls automatically (still
|
||||
// gated by the exact same per-source throttle everything else respects)
|
||||
// until either the requested count is reached or the source runs dry —
|
||||
// scrolling further past that point still works via the IntersectionObserver
|
||||
// below, for open-ended browsing beyond the original ask.
|
||||
function maybeAutoContinue() {
|
||||
if (!currentPayload || !currentPayload.count) return;
|
||||
if (currentItemCount() >= currentPayload.count) return;
|
||||
maybeLoadMore();
|
||||
}
|
||||
|
||||
function progressSuffix() {
|
||||
return (currentPayload && currentPayload.count) ? ` (${currentItemCount()}/${currentPayload.count})` : '';
|
||||
}
|
||||
|
||||
function maybeLoadMore() {
|
||||
if (!nextCursor || loadingMore || !currentPayload) return;
|
||||
const source = throttleSourceFor(currentPayload.toolType, currentPayload.mode);
|
||||
if (!source) return; // shouldn't happen if nextCursor is set, but be safe
|
||||
const sources = throttleSourceFor(currentPayload.toolType, currentPayload.mode);
|
||||
if (!sources.length) return; // shouldn't happen if nextCursor is set, but be safe
|
||||
|
||||
const remaining = cooldownUntil[source] - Date.now();
|
||||
const remaining = Math.max(0, ...sources.map(s => cooldownUntil[s] - Date.now()));
|
||||
if (remaining > 0) {
|
||||
armCountdown(remaining);
|
||||
return;
|
||||
}
|
||||
doLoadMore(source);
|
||||
doLoadMore(sources);
|
||||
}
|
||||
|
||||
function armCountdown(msRemaining) {
|
||||
if (cooldownTimer) return; // already counting down
|
||||
const secs = Math.max(1, Math.ceil(msRemaining / 1000));
|
||||
loadMoreStatus.classList.add('visible');
|
||||
loadMoreStatus.innerHTML = `<span class="spin-dot"></span>Loading more in ${secs}s…`;
|
||||
loadMoreStatus.innerHTML = `<span class="spin-dot"></span>Loading more in ${secs}s…${progressSuffix()}`;
|
||||
cooldownTimer = setTimeout(() => {
|
||||
cooldownTimer = null;
|
||||
maybeLoadMore(); // re-check: fires the load once the cooldown has cleared
|
||||
}, msRemaining);
|
||||
}
|
||||
|
||||
async function doLoadMore(source) {
|
||||
async function doLoadMore(sources) {
|
||||
loadingMore = true;
|
||||
loadMoreStatus.classList.add('visible');
|
||||
loadMoreStatus.innerHTML = `<span class="spin-dot"></span>Loading more…`;
|
||||
loadMoreStatus.innerHTML = `<span class="spin-dot"></span>Loading more…${progressSuffix()}`;
|
||||
|
||||
try {
|
||||
const res = await fetch('/api/run', {
|
||||
@@ -1100,7 +1558,7 @@ async function doLoadMore(source) {
|
||||
body: JSON.stringify({ ...currentPayload, cursor: nextCursor }),
|
||||
});
|
||||
const json = await res.json();
|
||||
stampCooldown(source);
|
||||
stampCooldown(sources);
|
||||
|
||||
if (!json.ok) {
|
||||
if (res.status === 429 && json.retryAfter) {
|
||||
@@ -1118,23 +1576,11 @@ async function doLoadMore(source) {
|
||||
appendCards(newItems, searchInput.value);
|
||||
nextCursor = json.nextCursor || null;
|
||||
loadMoreStatus.classList.remove('visible');
|
||||
|
||||
// Auto-archive only covered whatever was loaded at the initial Run — new
|
||||
// pages from scroll need their own save. Once we've reached the end
|
||||
// (no more pages), save automatically; otherwise surface a manual
|
||||
// checkpoint button so the user can save without waiting to hit the end.
|
||||
if (autoArchive && archivedId) {
|
||||
if (!nextCursor) {
|
||||
checkpointBtn.style.display = 'none';
|
||||
triggerArchive(currentData, currentPayload.toolType, buildQueryInfo(), archivedId);
|
||||
} else {
|
||||
checkpointBtn.style.display = '';
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
loadMoreStatus.textContent = `Load more failed: ${e}`;
|
||||
} finally {
|
||||
loadingMore = false;
|
||||
maybeAutoContinue();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1157,7 +1603,7 @@ runBtn.addEventListener('click', async () => {
|
||||
const payload = { toolType: t, mode: currentMode, count };
|
||||
|
||||
if (t === 'tweet_search_extractor') payload.searchQuery = val('searchQuery');
|
||||
if (t === 'follower_explorer' || t === 'post_extractor') payload.targetUsername = val('targetUsername');
|
||||
if (t === 'follower_explorer' || t === 'following_explorer' || t === 'post_extractor') payload.targetUsername = val('targetUsername');
|
||||
if (t === 'community_post_extractor') payload.targetCommunityId = val('targetCommunityId');
|
||||
if (t === 'article_extractor' ||
|
||||
t === 'tweet_replies_extractor' ||
|
||||
@@ -1183,6 +1629,7 @@ runBtn.addEventListener('click', async () => {
|
||||
statusBadge.className = 'badge running';
|
||||
searchInput.style.display = 'none';
|
||||
downloadBtn.style.display = 'none';
|
||||
archiveBtn.style.display = 'none';
|
||||
resultCount.textContent = '';
|
||||
currentData = null;
|
||||
resultBox.innerHTML = '<div class="empty-state">Fetching data…</div>';
|
||||
@@ -1207,6 +1654,7 @@ runBtn.addEventListener('click', async () => {
|
||||
nextCursor = json.nextCursor || null;
|
||||
const isMany = Array.isArray(json.data) && json.data.length > 1;
|
||||
downloadBtn.style.display = '';
|
||||
archiveBtn.style.display = '';
|
||||
|
||||
// Show map toggle only for geo results; reset to card view on each run
|
||||
const isGeo = (t === 'geo_post_extractor');
|
||||
@@ -1223,12 +1671,7 @@ runBtn.addEventListener('click', async () => {
|
||||
|
||||
searchInput.style.display = (isMany && !isGeo) ? '' : 'none';
|
||||
renderCards(currentData);
|
||||
|
||||
if (autoArchive) {
|
||||
triggerArchive(json.data, t, buildQueryInfo());
|
||||
} else {
|
||||
archiveBar.style.display = 'none';
|
||||
}
|
||||
maybeAutoContinue();
|
||||
} else {
|
||||
statusBadge.textContent = 'error';
|
||||
statusBadge.className = 'badge err';
|
||||
@@ -1244,21 +1687,32 @@ runBtn.addEventListener('click', async () => {
|
||||
}
|
||||
});
|
||||
|
||||
// ── Auto Archive ─────────────────────────────────────────────────────────────
|
||||
// ── Archive ───────────────────────────────────────────────────────────────────
|
||||
// Manual, on demand — same model as graph.html's "Archive All" button rather
|
||||
// than the old auto-checkpoint-on-every-scroll system (which had a real race:
|
||||
// a fast scroll landing between the archive-creation call setting its
|
||||
// in-flight flag and that call's fetch actually resolving could silently
|
||||
// drop that checkpoint for good). Click Archive whenever you want to save
|
||||
// exactly what's currently loaded; click it again after scrolling/expanding
|
||||
// loads more and it checkpoints the SAME archive in place instead of creating a
|
||||
// new one each time — identical behavior to graph.html's graphArchivedId.
|
||||
|
||||
archiveToggle.addEventListener('click', () => {
|
||||
autoArchive = !autoArchive;
|
||||
toggleTrack.classList.toggle('on', autoArchive);
|
||||
if (autoArchive) loadRecentArchives();
|
||||
});
|
||||
archiveBtn.addEventListener('click', archiveResults);
|
||||
|
||||
async function archiveResults() {
|
||||
if (!currentData || !currentPayload) return;
|
||||
|
||||
async function triggerArchive(data, toolType, queryInfo, existingId) {
|
||||
archiveBar.style.display = '';
|
||||
const label = existingId ? 'Saving checkpoint…' : 'Archiving…';
|
||||
const label = archivedId ? 'Saving checkpoint…' : 'Archiving…';
|
||||
archiveBar.innerHTML = `<span>${label}</span><div class="archive-bar-fill-wrap"><div class="archive-bar-fill" style="width:0%"></div></div>`;
|
||||
|
||||
const body = { data, toolType, queryInfo };
|
||||
if (existingId) body.archiveId = existingId;
|
||||
try {
|
||||
const body = {
|
||||
toolType: currentPayload.toolType,
|
||||
data: Array.isArray(currentData) ? currentData : [currentData],
|
||||
queryInfo: buildQueryInfo(),
|
||||
};
|
||||
if (archivedId) body.archiveId = archivedId;
|
||||
|
||||
const res = await fetch('/api/archive', {
|
||||
method: 'POST',
|
||||
@@ -1266,17 +1720,15 @@ async function triggerArchive(data, toolType, queryInfo, existingId) {
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
const json = await res.json();
|
||||
if (!json.ok) { archiveBar.innerHTML = `<span style="color:var(--danger)">Archive failed: ${esc(json.error)}</span>`; return; }
|
||||
if (!json.ok) throw new Error(json.error);
|
||||
|
||||
archivedId = json.archiveId;
|
||||
archiveBtn.textContent = 'Update Archive';
|
||||
pollArchive(json.archiveId);
|
||||
} catch (e) {
|
||||
archiveBar.innerHTML = `<span style="color:var(--danger)">Archive failed: ${esc(e.message)}</span>`;
|
||||
}
|
||||
}
|
||||
|
||||
checkpointBtn.addEventListener('click', () => {
|
||||
if (!archivedId || !currentPayload) return;
|
||||
checkpointBtn.style.display = 'none';
|
||||
triggerArchive(currentData, currentPayload.toolType, buildQueryInfo(), archivedId);
|
||||
});
|
||||
|
||||
function pollArchive(archiveId) {
|
||||
const interval = setInterval(async () => {
|
||||
@@ -1519,13 +1971,6 @@ async function updateMap() {
|
||||
const idInput = document.getElementById('targetTweetId');
|
||||
if (idInput) idInput.value = id;
|
||||
|
||||
// Restore auto-archive state if the parent tab had it on
|
||||
if (p.get('archive') === '1') {
|
||||
autoArchive = true;
|
||||
toggleTrack.classList.add('on');
|
||||
loadRecentArchives();
|
||||
}
|
||||
|
||||
// Kick off the run automatically
|
||||
runBtn.click();
|
||||
|
||||
|
||||
@@ -35,6 +35,34 @@ class WaybackError(Exception):
|
||||
_DATE8_RE = re.compile(r"^\d{8}$")
|
||||
_SAFE_URL_RE = re.compile(r"^https?://", re.IGNORECASE)
|
||||
_TWEET_ID_RE = re.compile(r"/status/(\d+)")
|
||||
# Wayback crawled every historical subdomain variant (mobile./m./www./bare),
|
||||
# unlike a live search result which is always the canonical domain — hence
|
||||
# the looser (?:[\w-]+\.)? host prefix here vs google_cse_client's (?:www\.)?.
|
||||
_X_HOST_RE = re.compile(r"^https?://(?:[\w-]+\.)?(?:x|twitter)\.com(?:/|$)", re.IGNORECASE)
|
||||
_PROFILE_RE = re.compile(r"^https?://(?:[\w-]+\.)?(?:x|twitter)\.com/([^/?#]+)/?(?:\?.*)?$", re.IGNORECASE)
|
||||
_NON_PROFILE_PATHS = {
|
||||
"home", "explore", "notifications", "messages", "i", "search", "settings",
|
||||
"compose", "login", "logout", "signup", "tos", "privacy", "about", "hashtag",
|
||||
}
|
||||
|
||||
|
||||
def _classify_url(url: str) -> str:
|
||||
"""Same reasoning as google_cse_client's copy of this — an archived
|
||||
snapshot of x.com/someone reads as "a Twitter profile" whether it's
|
||||
actually a tweet permalink, a bare profile page, or some other X page
|
||||
entirely. Purely a label derived from the URL's own shape; never touches
|
||||
original/archive_url themselves. Returns
|
||||
'tweet' | 'profile' | 'twitter_other' | 'other'."""
|
||||
if not url:
|
||||
return "other"
|
||||
if _TWEET_ID_RE.search(url):
|
||||
return "tweet"
|
||||
if not _X_HOST_RE.match(url):
|
||||
return "other"
|
||||
m = _PROFILE_RE.match(url)
|
||||
if m and m.group(1).lower() not in _NON_PROFILE_PATHS:
|
||||
return "profile"
|
||||
return "twitter_other"
|
||||
|
||||
|
||||
def _validate_date(label: str, value: str) -> None:
|
||||
@@ -127,6 +155,7 @@ def _row_to_record(row: dict) -> dict:
|
||||
"timestamp": ts, # internal only — stripped before returning to caller
|
||||
"iso_date": iso_date, # when Wayback captured this snapshot
|
||||
"original": original,
|
||||
"content_type": _classify_url(original),
|
||||
"statuscode": row.get("statuscode"),
|
||||
"mimetype": row.get("mimetype"),
|
||||
"length": row.get("length"),
|
||||
|
||||
@@ -9,6 +9,7 @@ urls = {
|
||||
"readme": "https://raw.githubusercontent.com/Jieyab89/OSINT-Cheat-sheet/refs/heads/main/README.md",
|
||||
"wiki": "https://github.com/Jieyab89/OSINT-Cheat-sheet/wiki",
|
||||
"articles": "https://raw.githubusercontent.com/Jieyab89/OSINT-Cheat-sheet/main/awesome-article.md"
|
||||
#"scripts": "https://github.com/Jieyab89/OSINT-Cheat-sheet/blob/main/Script/README.md" soon
|
||||
}
|
||||
|
||||
headers = {
|
||||
|
||||
Reference in New Issue
Block a user