mirror of
https://github.com/Jieyab89/OSINT-Cheat-sheet.git
synced 2026-08-17 18:35:41 +02:00
fix bug and bottle neck and enhancement
This commit is contained in:
Jieyab89
@@ -12,10 +12,12 @@
|
||||
6. Add more parameter for enrichment
|
||||
7. Add no rate limit (throttle)
|
||||
8. Add Google CSE data source
|
||||
9. Expand data user profile post, follower and following, reply post, retweet post in graph —
|
||||
10. Add sentiment analysis for clustering data, pro, neutral, con based on archive data and dump data
|
||||
9. Expand data user profile post, follower and following, reply post, retweet post in graph
|
||||
10. Add sentiment analysis for clustering data, pro, neutral, con. Based on archive data and dump data
|
||||
11. Add more data source and other parameter (soon) still research
|
||||
12. Add more detail data source for the context
|
||||
13. Auto repair broken archive and sentiment analysis data dump
|
||||
14. Update rendering data in sentiment analysis
|
||||
|
||||
## Features
|
||||
|
||||
@@ -233,6 +235,8 @@ starting point for investigation, not a verdict.
|
||||
|
||||
# Results
|
||||
|
||||
[](https://vimeo.com/1216677900)
|
||||
|
||||
Xquik Dashboard
|
||||
|
||||
<img width="2556" height="1193" alt="image" src="https://github.com/user-attachments/assets/51e9d0f3-d079-44ce-9841-378a3e1ad7e4" />
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import shutil
|
||||
@@ -535,11 +536,29 @@ def archive_results(archive_id):
|
||||
meta_file = base / "meta.json"
|
||||
if not results_file.exists():
|
||||
return jsonify({"ok": False, "error": "Archive not found"}), 404
|
||||
return jsonify({
|
||||
"ok": True,
|
||||
"results": json.loads(results_file.read_text()),
|
||||
"meta": json.loads(meta_file.read_text()) if meta_file.exists() else {},
|
||||
})
|
||||
|
||||
# json.loads() raising here used to fall through to Flask's default
|
||||
# error handler, which returns an HTML error page — the browser's
|
||||
# res.json() then fails with an opaque "SyntaxError: JSON.parse:
|
||||
# unexpected character..." instead of the actual problem. archive.py now
|
||||
# writes both files atomically (see _atomic_write_json), so a reader
|
||||
# should never see a torn file mid-checkpoint-update; this is the
|
||||
# backstop for any other cause (disk fault, a pre-existing archive
|
||||
# written before that fix, manual editing) — always answer with clean
|
||||
# JSON either way.
|
||||
try:
|
||||
results = json.loads(results_file.read_text())
|
||||
except (json.JSONDecodeError, OSError) as e:
|
||||
return jsonify({"ok": False, "error": f"Archive data is corrupted or unreadable ({e}). Try re-running the search and archiving again."}), 500
|
||||
|
||||
meta = {}
|
||||
if meta_file.exists():
|
||||
try:
|
||||
meta = json.loads(meta_file.read_text())
|
||||
except (json.JSONDecodeError, OSError):
|
||||
pass # meta is supplementary — a corrupt/missing meta shouldn't block viewing the results that DID load fine
|
||||
|
||||
return jsonify({"ok": True, "results": results, "meta": meta})
|
||||
|
||||
|
||||
@app.route("/api/archive/<archive_id>/media/<path:filename>")
|
||||
@@ -590,23 +609,95 @@ def archive_list():
|
||||
|
||||
|
||||
# ── Analytics (sentiment / clustering) ──────────────────────────────────────
|
||||
# Same background-thread + polling shape archive.py's downloads already use
|
||||
# (start() kicks off a thread and returns immediately, status() reports
|
||||
# progress) — ML sentiment scoring runs locally on CPU at ~11-12ms/item
|
||||
# (measured), so a large archive (thousands of items) can take a minute-plus.
|
||||
# A blocking request for that long leaves the browser with nothing to show
|
||||
# but a static spinner and no way to tell "still working" from "stuck."
|
||||
|
||||
_analytics_registry: dict[str, dict] = {} # archive_id -> job status dict
|
||||
_analytics_lock = threading.Lock()
|
||||
|
||||
|
||||
def _run_analytics(archive_id: str, items: list) -> None:
|
||||
def on_progress(done, total):
|
||||
with _analytics_lock:
|
||||
entry = _analytics_registry.get(archive_id)
|
||||
if entry is not None: # could've been cleared/overwritten by a re-run
|
||||
entry.update({"progress": done, "total": total})
|
||||
|
||||
try:
|
||||
result = _sentiment.analyze(items, on_progress=on_progress)
|
||||
with _analytics_lock:
|
||||
_analytics_registry[archive_id] = {
|
||||
"status": "done", "progress": result.get("total_scored", 0),
|
||||
"total": result.get("total_scored", 0), "result": result, "error": None,
|
||||
}
|
||||
except Exception as e: # noqa: BLE001
|
||||
with _analytics_lock:
|
||||
_analytics_registry[archive_id] = {
|
||||
"status": "error", "progress": 0, "total": 0, "result": None, "error": str(e),
|
||||
}
|
||||
|
||||
|
||||
@app.route("/analytics")
|
||||
def analytics_viewer():
|
||||
return render_template("analytics.html")
|
||||
|
||||
|
||||
@app.route("/api/analytics/<archive_id>")
|
||||
def analytics_run(archive_id):
|
||||
@app.route("/api/analytics/<archive_id>/start", methods=["POST"])
|
||||
def analytics_start(archive_id):
|
||||
results_file = _archive.ARCHIVE_ROOT / archive_id / "results.json"
|
||||
if not results_file.exists():
|
||||
return jsonify({"ok": False, "error": "Archive not found"}), 404
|
||||
items = json.loads(results_file.read_text())
|
||||
return jsonify({"ok": True, **_sentiment.analyze(items)})
|
||||
# Same reasoning as archive_results() above — never let a bad file turn
|
||||
# into an HTML error page here either, or the browser's res.json() call
|
||||
# fails with an opaque parse error instead of a readable message.
|
||||
try:
|
||||
items = json.loads(results_file.read_text())
|
||||
except (json.JSONDecodeError, OSError) as e:
|
||||
return jsonify({"ok": False, "error": f"Archive data is corrupted or unreadable ({e}). Try re-running the search and archiving again."}), 500
|
||||
|
||||
with _analytics_lock:
|
||||
_analytics_registry[archive_id] = {
|
||||
"status": "scoring", "progress": 0, "total": 0, "result": None, "error": None,
|
||||
}
|
||||
t = threading.Thread(target=_run_analytics, args=(archive_id, items), daemon=True)
|
||||
t.start()
|
||||
return jsonify({"ok": True})
|
||||
|
||||
|
||||
@app.route("/api/analytics/<archive_id>/status")
|
||||
def analytics_status(archive_id):
|
||||
with _analytics_lock:
|
||||
entry = _analytics_registry.get(archive_id)
|
||||
entry = dict(entry) if entry else None
|
||||
if entry is None:
|
||||
return jsonify({"ok": False, "error": "No analysis running for this archive — call start first"}), 404
|
||||
if entry["status"] == "error":
|
||||
return jsonify({"ok": False, "error": entry["error"]}), 500
|
||||
|
||||
resp = {"ok": True, "status": entry["status"], "progress": entry["progress"], "total": entry["total"]}
|
||||
if entry["status"] == "done":
|
||||
resp.update(entry["result"])
|
||||
return jsonify(resp)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
host = config.get("server", "host", fallback="127.0.0.1")
|
||||
port = config.getint("server", "port", fallback=5000)
|
||||
debug = config.getboolean("server", "debug", fallback=True)
|
||||
|
||||
# Warm up the ML sentiment model in the background so the FIRST
|
||||
# analytics request doesn't pay its ~15-20s one-time load cost live —
|
||||
# see sentiment.warm_up_ml()'s docstring. Skipped in the reloader's
|
||||
# outer "monitor" process (debug mode re-execs a child process to
|
||||
# actually serve requests, setting WERKZEUG_RUN_MAIN in that child
|
||||
# only) — otherwise a process that never serves a single request would
|
||||
# load ~1GB of model weights for nothing, repeating on every autoreload
|
||||
# during dev.
|
||||
if not debug or os.environ.get("WERKZEUG_RUN_MAIN") == "true":
|
||||
threading.Thread(target=_sentiment.warm_up_ml, daemon=True).start()
|
||||
|
||||
app.run(host=host, port=port, debug=debug, threaded=True)
|
||||
|
||||
@@ -83,6 +83,30 @@ def _media_ext(url: str, mtype: str) -> str:
|
||||
return "jpg"
|
||||
|
||||
|
||||
def _atomic_write_json(path: Path, data) -> None:
|
||||
"""Write JSON to `path` without ever leaving a reader able to observe a
|
||||
half-written file. Path.write_text() opens, writes, and closes in place —
|
||||
a GET landing on archive_results() mid-write (most likely during a
|
||||
checkpoint update() re-run, which can take a while on a large dataset)
|
||||
could read a truncated/malformed file and hand the browser invalid JSON,
|
||||
which is exactly what surfaces client-side as a raw
|
||||
"SyntaxError: JSON.parse: unexpected character..." instead of a clean
|
||||
error. Writing to a sibling temp file first and os.replace()-ing it into
|
||||
place is atomic on both POSIX and Windows: a concurrent reader always
|
||||
sees either the complete old file or the complete new one, never
|
||||
something in between."""
|
||||
# pid + thread id: os.getpid() alone collides if two checkpoint updates
|
||||
# for the same archive_id race inside this one (threaded=True) process —
|
||||
# e.g. a fast double-click on "Update Archive," or the browser retrying a
|
||||
# save right as the first one is still writing. Each writer then gets
|
||||
# its own temp file, so the two writes can't corrupt each other; the
|
||||
# last os.replace() to run simply wins, same as a normal last-write-wins
|
||||
# race would, but never with a torn/partial file in between.
|
||||
tmp = path.with_suffix(path.suffix + f".tmp{os.getpid()}_{threading.get_ident()}")
|
||||
tmp.write_text(json.dumps(data, indent=2, ensure_ascii=False))
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def _download_file(url: str, dest: Path) -> bool:
|
||||
"""Download a single file. Returns True on success."""
|
||||
try:
|
||||
@@ -169,12 +193,8 @@ def _run(archive_id: str, tool_type: str, data, query_info: dict) -> None:
|
||||
"total_items": len(items),
|
||||
"media_count": total_media,
|
||||
}
|
||||
meta_path.write_text(
|
||||
json.dumps(meta, indent=2, ensure_ascii=False)
|
||||
)
|
||||
(archive_dir / "results.json").write_text(
|
||||
json.dumps(enriched, indent=2, ensure_ascii=False)
|
||||
)
|
||||
_atomic_write_json(meta_path, meta)
|
||||
_atomic_write_json(archive_dir / "results.json", enriched)
|
||||
|
||||
with _lock:
|
||||
_registry[archive_id].update({"status": "downloading", "total": len(media_queue)})
|
||||
|
||||
@@ -38,6 +38,21 @@ from collections import Counter
|
||||
# Indonesian words skew political/social-discourse (matches the kind of
|
||||
# content this tool actually pulls — keyword searches on public affairs,
|
||||
# government programs, public figures) as well as general register.
|
||||
#
|
||||
# Lexicon-parity note: an earlier version of this list ran ~130 positive vs.
|
||||
# ~220 negative entries. That gap isn't neutral — with score_text() summing
|
||||
# one point per matched word, a lexicon with substantially more negative
|
||||
# coverage (more synonyms per concept: bohong/kebohongan/hoax/menipu/penipu/
|
||||
# penipuan/tipu for one idea, "lying," vs. jujur/kejujuran for its opposite)
|
||||
# structurally nudges mixed/ambiguous text toward "con" independent of the
|
||||
# text's actual sentiment, simply because there's more negative surface area
|
||||
# to match against. The additions below restore rough parity for the same
|
||||
# governance/social-discourse register the negative list already covers
|
||||
# (accountability, honesty, inclusion, rule of law) rather than padding with
|
||||
# unrelated filler — pair each new word against the negative concept it
|
||||
# offsets in a review. This is still a heuristic, not a bias-free scorer:
|
||||
# see analyze()'s docstring and the module docstring above for the standing
|
||||
# caveat that neither backend is ground truth.
|
||||
|
||||
# Arr data words
|
||||
# Need to feedback and research to sett the all parameter for each words
|
||||
@@ -65,6 +80,18 @@ POSITIVE_WORDS = {
|
||||
"efektif", "transparan", "transparansi", "akuntabel", "akuntabilitas",
|
||||
"merakyat", "membela rakyat", "pro rakyat", "berpihak pada rakyat",
|
||||
"terpuji", "membanggakan", "gemilang", "cemerlang", "berkah", "istimewa",
|
||||
# Governance/social-discourse counterparts added for lexicon parity
|
||||
# (offsetting korupsi/nepotisme/kkn, otoriter/diktator/fasis/represif,
|
||||
# rasis/intoleran, bohong/hoax/menipu, and pelanggaran/ilegal below).
|
||||
"bersih", "antikorupsi", "berintegritas", "integritas", "kredibel",
|
||||
"kredibilitas", "terpercaya", "dapat dipercaya", "netral", "imparsial",
|
||||
"objektif", "demokratis", "reformasi", "reformis", "inklusif",
|
||||
"inklusi", "toleran", "toleransi", "egaliter", "partisipatif",
|
||||
"aspiratif", "taat hukum", "patuh hukum", "sesuai aturan", "legal",
|
||||
"sah", "melindungi", "perlindungan", "membangun", "pembangunan",
|
||||
"sinergi", "berkolaborasi", "kolaboratif", "harmonis", "kondusif",
|
||||
"stabil", "stabilitas", "humanis", "empati", "berempati", "rendah hati",
|
||||
"dermawan",
|
||||
}
|
||||
|
||||
NEGATIVE_WORDS = {
|
||||
@@ -102,10 +129,10 @@ NEGATIVE_WORDS = {
|
||||
"kurang ajar", "tidak becus", "amburadul", "berantakan", "semrawut",
|
||||
"menyengsarakan", "represif", "represi", "diskriminasi",
|
||||
"mendiskriminasi", "rasis", "rasisme", "intoleran", "intoleransi",
|
||||
"penjilat", "gila", "kontol", "memek", "paok", "stress", "goblog",
|
||||
"penjilat", "kontol", "memek", "paok", "stress", "goblog", "kontlo",
|
||||
"kepala batu", "oon", "bacot", "asu", "gijil", "jembut", "kanjut",
|
||||
"ngentot", "puki", "meki", "jembot", "pukimak", "kimak", "tembelek",
|
||||
"tai",
|
||||
"tai", "bacod", "telaso",
|
||||
}
|
||||
|
||||
# Flips the polarity of a sentiment word found within NEGATION_WINDOW tokens
|
||||
@@ -192,10 +219,24 @@ def score_text(text: str) -> dict:
|
||||
def _item_text(item: dict) -> str:
|
||||
"""The text worth scoring/tokenizing for a given archived record —
|
||||
varies by which tool produced it (a tweet's own text vs. a Wayback/CSE
|
||||
page's scraped title+description vs. a bare user's bio)."""
|
||||
page's scraped title+description).
|
||||
|
||||
`description` is deliberately NOT pulled from a bare user record (a
|
||||
follower/following/retweeter entry — cookie_client.py's _user_to_dict
|
||||
always sets `followers_count`, even to None, which no tweet/CSE/Wayback
|
||||
record ever carries, so that key's mere presence identifies the shape
|
||||
reliably). For a CSE result, `description` is Google's own snippet of
|
||||
the matched page — genuinely relevant text. For a user record it's the
|
||||
account's own bio, which says nothing about the search topic; scoring
|
||||
"suka kucing dan kopi ☕" as pro/con toward whatever was searched would
|
||||
just be noise. Those accounts are still kept for clustering/leaderboard
|
||||
purposes (top_users() below runs over every item regardless of text) —
|
||||
they're just excluded from sentiment/word-cloud scoring specifically."""
|
||||
is_bare_user_record = "followers_count" in item
|
||||
parts = [
|
||||
item.get("text"), item.get("full_text"), item.get("article_text"),
|
||||
item.get("post_title"), item.get("post_text"), item.get("description"),
|
||||
item.get("post_title"), item.get("post_text"),
|
||||
None if is_bare_user_record else item.get("description"),
|
||||
]
|
||||
return " ".join(p for p in parts if p)
|
||||
|
||||
@@ -293,31 +334,81 @@ def _get_ml_pipeline():
|
||||
return _ml_pipeline
|
||||
|
||||
|
||||
def _score_texts_ml(texts: list[str]) -> list[dict] | None:
|
||||
"""Batch-scores every text in one call (far faster on CPU than one
|
||||
pipeline call per item). Returns None if the model isn't available, so
|
||||
the caller falls back to the lexicon scorer instead. `score` is signed
|
||||
(positive for pro, negative for con, 0 for neutral) to match the
|
||||
lexicon backend's convention; `confidence` carries the model's own
|
||||
unsigned probability for the label it picked."""
|
||||
def warm_up_ml() -> None:
|
||||
"""Loads the ML pipeline right now instead of waiting for the first real
|
||||
analytics request to trigger it lazily. Measured at ~15-20s the first
|
||||
time any process calls _get_ml_pipeline() (importing transformers,
|
||||
constructing the pipeline, reading the cached weights off disk) — vs.
|
||||
~11-12ms/item for actual scoring once loaded. Without this, that whole
|
||||
one-time cost lands inside the FIRST user's analytics job, during which
|
||||
the progress bar has nothing to report yet (on_progress only fires once
|
||||
scoring itself starts) and just sits at 0/0 looking stuck. Meant to be
|
||||
called from a background thread at server startup (see app.py) — still
|
||||
completely safe to skip calling this at all, or to have the first real
|
||||
request race it, since _get_ml_pipeline() is lock-protected and
|
||||
idempotent either way; this is purely a warm-up, not a dependency."""
|
||||
_get_ml_pipeline()
|
||||
|
||||
|
||||
_ML_PROGRESS_CHUNK = 64 # texts per pipeline call — see _score_texts_ml docstring
|
||||
|
||||
|
||||
def _score_texts_ml(texts: list[str], on_progress=None) -> list[dict] | None:
|
||||
"""Scores every text, chunked (rather than one giant pipeline call), so
|
||||
a caller running this in a background thread can report real progress —
|
||||
on a CPU this measures ~11-12ms/item (~1000 items ≈ 12s, ~5000 ≈ ~1min),
|
||||
linear with volume, so a large archive genuinely takes a while and a
|
||||
caller polling for status needs something better to show than a blind
|
||||
spinner. _ML_PROGRESS_CHUNK=64 batches (each itself pipelined
|
||||
batch_size=16 internally by HF) keeps ticks frequent enough to feel
|
||||
live (~0.7-0.8s apart) without paying per-call overhead for every
|
||||
single item. Returns None if the model isn't available, so the caller
|
||||
falls back to the lexicon scorer instead. `score` is signed (positive
|
||||
for pro, negative for con, 0 for neutral) to match the lexicon
|
||||
backend's convention; `confidence` carries the model's own unsigned
|
||||
probability for the label it picked."""
|
||||
clf = _get_ml_pipeline()
|
||||
if clf is None:
|
||||
return None
|
||||
raw = clf(texts, truncation=True, batch_size=16)
|
||||
|
||||
results = []
|
||||
for r in raw:
|
||||
label = _ML_LABEL_MAP.get(str(r.get("label", "")).lower(), "neutral")
|
||||
confidence = float(r.get("score", 0.0))
|
||||
signed = confidence if label == "pro" else -confidence if label == "con" else 0.0
|
||||
results.append({
|
||||
"label": label, "score": round(signed, 3),
|
||||
"confidence": round(confidence, 3), "matches": [],
|
||||
})
|
||||
for start in range(0, len(texts), _ML_PROGRESS_CHUNK):
|
||||
chunk = texts[start:start + _ML_PROGRESS_CHUNK]
|
||||
# truncation=True alone is NOT enough here: it truncates to the
|
||||
# tokenizer's own model_max_length, which for this tokenizer's
|
||||
# shipped config is left at HF's "unset" sentinel (~1e30, i.e.
|
||||
# effectively no limit) rather than the model's real 512-token
|
||||
# capacity. A single long post (a fact-check thread, an
|
||||
# article-length tweet — anything past ~512 tokens once
|
||||
# subword-tokenized) then sails through "truncation" untruncated,
|
||||
# overflows the model's position-embedding table, and crashes the
|
||||
# whole batch with a raw RuntimeError ("index 514 is out of bounds
|
||||
# for dimension 1 with size 514" — 514 = 512 + the 2-position
|
||||
# offset RoBERTa-style embeddings use). max_length=512 forces the
|
||||
# real limit regardless of what the tokenizer config claims.
|
||||
raw = clf(chunk, truncation=True, max_length=512, batch_size=16)
|
||||
for r in raw:
|
||||
label = _ML_LABEL_MAP.get(str(r.get("label", "")).lower(), "neutral")
|
||||
confidence = float(r.get("score", 0.0))
|
||||
signed = confidence if label == "pro" else -confidence if label == "con" else 0.0
|
||||
results.append({
|
||||
"label": label, "score": round(signed, 3),
|
||||
"confidence": round(confidence, 3), "matches": [],
|
||||
})
|
||||
if on_progress:
|
||||
on_progress(len(results), len(texts))
|
||||
return results
|
||||
|
||||
|
||||
def analyze(items: list[dict]) -> dict:
|
||||
"""Full analytics payload for one archive's worth of raw items."""
|
||||
def analyze(items: list[dict], on_progress=None) -> dict:
|
||||
"""Full analytics payload for one archive's worth of raw items.
|
||||
on_progress, if given, is called as on_progress(scored_count,
|
||||
total_to_score) — zero or more times during ML scoring (chunked, see
|
||||
_score_texts_ml), and always at least once at the very end regardless
|
||||
of which backend actually ran, so a caller polling for status always
|
||||
sees a final 100%-done tick even on the lexicon path (fast enough that
|
||||
per-chunk progress wouldn't mean anything, but a job registry watching
|
||||
for "did this reach total" still needs that terminal call)."""
|
||||
if not isinstance(items, list):
|
||||
items = [items]
|
||||
# archive.py's own _run() passes non-dict entries through as-is rather
|
||||
@@ -334,10 +425,12 @@ def analyze(items: list[dict]) -> dict:
|
||||
text_items.append(item)
|
||||
texts.append(text)
|
||||
|
||||
ml_results = _score_texts_ml(texts) if texts else None
|
||||
ml_results = _score_texts_ml(texts, on_progress=on_progress) if texts else None
|
||||
method = "ml" if ml_results is not None else "lexicon"
|
||||
if ml_results is None:
|
||||
ml_results = [score_text(t) for t in texts]
|
||||
if on_progress:
|
||||
on_progress(len(texts), len(texts))
|
||||
|
||||
sentiment_counts = {"pro": 0, "neutral": 0, "con": 0}
|
||||
scored_items = []
|
||||
|
||||
@@ -45,3 +45,85 @@ const DRILLABLE = {
|
||||
|
||||
const SOURCE_CLASS = { 'Twitter Cookie': 'src-cookie', 'Xquik API': 'src-xquik', 'Wayback Machine': 'src-wayback', 'Google CSE': 'src-cse' };
|
||||
const AGE_LABELS = { new: 'New account', recent: 'Recent account', established: 'Established account' };
|
||||
|
||||
// ── Shared profile header (avatar/name/handle/banner/bio) ──────────────────
|
||||
// Used by buildCard() in both index.html (live results) and archive.html
|
||||
// (saved archives) so a record's author identity renders byte-identically
|
||||
// whether you're looking at it live or after it's been archived — previously
|
||||
// archive.html had no equivalent at all, so a saved card silently dropped
|
||||
// the avatar/name/handle/bio/banner that the live card showed for the exact
|
||||
// same record. `esc()` is expected to already be defined as a global by the
|
||||
// time this actually runs (each page defines its own, loaded in a later
|
||||
// <script> block — this file only *calls* esc() inside functions, it never
|
||||
// runs at parse time, so load order is fine).
|
||||
|
||||
// Only a plain https URL (no quotes/angle-brackets/whitespace/parens) is
|
||||
// ever interpolated into the CSS url('...') background-image — that goes
|
||||
// through a second parsing pass CSS-side, so HTML-attribute escaping alone
|
||||
// isn't sufficient there the way it is for a plain <img src>. Rejecting
|
||||
// anything but a clean https URL up front closes that off rather than
|
||||
// trying to escape a value for two contexts (HTML attribute + CSS token)
|
||||
// wedged into one string.
|
||||
function isSafeImageUrl(u) {
|
||||
return typeof u === 'string' && /^https:\/\/[^\s'"<>()]+$/.test(u);
|
||||
}
|
||||
|
||||
// Returns { html, usedFields } instead of just a string — buildCard() needs
|
||||
// to know exactly which raw keys actually ended up rendered in the header
|
||||
// so it can drop only THOSE from the generic row list. A static "always hide
|
||||
// these field names" list doesn't work here: CSE/Wayback records also have a
|
||||
// `description` field (Google's own snippet) that this header never touches
|
||||
// (no avatar/name/handle on those records, so it returns empty) — hiding it
|
||||
// unconditionally would silently delete the one thing the user asked to see.
|
||||
function buildCardHeader(item) {
|
||||
const avatarRaw = item.avatar || item.user_avatar || '';
|
||||
const avatar = isSafeImageUrl(avatarRaw) ? avatarRaw : '';
|
||||
const name = item.name || '';
|
||||
const handle = item.screen_name || item.user || item.username || '';
|
||||
|
||||
if (!avatar && !name && !handle) return { html: '', usedFields: [] };
|
||||
|
||||
const usedFields = [];
|
||||
if (avatar) usedFields.push(item.avatar ? 'avatar' : 'user_avatar');
|
||||
if (name) usedFields.push('name');
|
||||
if (handle) usedFields.push(item.screen_name ? 'screen_name' : item.user ? 'user' : 'username');
|
||||
|
||||
const avatarHtml = avatar
|
||||
? `<img class="card-avatar" src="${esc(avatar)}" alt="" loading="lazy" referrerpolicy="no-referrer">`
|
||||
: (name || handle)
|
||||
? `<div class="card-avatar card-avatar-fallback">${esc((name || handle).charAt(0).toUpperCase())}</div>`
|
||||
: '';
|
||||
const identityHtml = (name || handle)
|
||||
? `<div class="card-identity">
|
||||
${name ? `<div class="card-name">${esc(name)}</div>` : ''}
|
||||
${handle ? `<div class="card-handle">@${esc(handle)}</div>` : ''}
|
||||
</div>`
|
||||
: '';
|
||||
|
||||
// A tweet or reply already leads with its own text a few rows down — a
|
||||
// full cover-photo-plus-bio header buries that under the *author's*
|
||||
// profile instead of the actual reply content. Those get a small inline
|
||||
// byline only; the full profile-card treatment (banner + bio) is reserved
|
||||
// for records that ARE a user — follower/retweeter results, not tweets a
|
||||
// user happened to write.
|
||||
const isTweetLike = item.text !== undefined || item.full_text !== undefined || item.article_text !== undefined;
|
||||
if (isTweetLike) {
|
||||
return { html: `<div class="card-byline">${avatarHtml}${identityHtml}</div>`, usedFields };
|
||||
}
|
||||
|
||||
const bannerRaw = item.banner || item.user_banner || '';
|
||||
const banner = isSafeImageUrl(bannerRaw) ? bannerRaw : '';
|
||||
const bio = item.description || item.user_bio || '';
|
||||
if (banner) usedFields.push(item.banner ? 'banner' : 'user_banner');
|
||||
if (bio) usedFields.push(item.description ? 'description' : 'user_bio');
|
||||
|
||||
const bioHtml = bio ? `<div class="card-bio">${esc(bio)}</div>` : '';
|
||||
const bannerHtml = banner
|
||||
? `<div class="card-banner" style="background-image:url('${esc(banner)}')"></div>`
|
||||
: '';
|
||||
|
||||
return {
|
||||
html: `<div class="card-header${banner ? ' has-banner' : ''}">${bannerHtml}<div class="card-header-row">${avatarHtml}${identityHtml}</div>${bioHtml}</div>`,
|
||||
usedFields,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -223,6 +223,33 @@
|
||||
.sent-badge.sent-pro { color: var(--success); border-color: #14532d; background: var(--success-bg); }
|
||||
.sent-badge.sent-neutral { color: var(--muted); border-color: var(--border); background: var(--bg); }
|
||||
.sent-badge.sent-con { color: var(--danger); border-color: #7f1d1d; background: var(--danger-bg); }
|
||||
.sent-badge.sent-mixed { color: var(--accent); border-color: #34348a; background: var(--accent-bg); }
|
||||
|
||||
/* ── Account sentiment breakdown — who's pro/con/neutral, as accounts, not
|
||||
just individual posts ── */
|
||||
.account-sent-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); gap: 16px; }
|
||||
.asg-head { display: flex; align-items: center; gap: 8px; margin-bottom: 8px; }
|
||||
.asg-count { font-size: 11px; color: var(--muted); }
|
||||
.user-row.clickable { cursor: pointer; }
|
||||
.user-row.clickable:hover { background: var(--surface2); }
|
||||
.user-breakdown { font-size: 10px; color: var(--muted); flex-shrink: 0; white-space: nowrap; }
|
||||
|
||||
/* Auto-scroll — same idea as index.html's scroll-triggered load-more, just
|
||||
scoped to each small box instead of the whole page: a fixed-height
|
||||
panel that scrolls internally, revealing more rows as you scroll near
|
||||
its bottom instead of dumping everything (or a hard cutoff) at once. */
|
||||
.user-list.scrollable {
|
||||
max-height: 320px;
|
||||
overflow-y: auto;
|
||||
scrollbar-width: thin;
|
||||
scrollbar-color: var(--border) transparent;
|
||||
}
|
||||
.load-more-note {
|
||||
font-size: 11px;
|
||||
color: var(--muted);
|
||||
text-align: center;
|
||||
padding: 10px 0 2px;
|
||||
}
|
||||
|
||||
/* ── Item browser ── */
|
||||
.item-toolbar { display: flex; align-items: center; gap: 10px; margin-bottom: 10px; flex-wrap: wrap; }
|
||||
@@ -251,6 +278,12 @@
|
||||
}
|
||||
@keyframes spin { to { transform: rotate(360deg); } }
|
||||
|
||||
/* Sentiment-analysis progress bar — same visual language as the archive
|
||||
media-download bar on index.html/graph.html/archive.html, so a
|
||||
long-running background job looks the same wherever one shows up. */
|
||||
.archive-bar-fill-wrap { height: 3px; background: var(--border); border-radius: 2px; margin-top: 8px; overflow: hidden; }
|
||||
.archive-bar-fill { height: 100%; background: var(--accent); border-radius: 2px; transition: width 0.3s; }
|
||||
|
||||
.method-note {
|
||||
font-size: 11px; color: var(--muted); line-height: 1.6; background: var(--surface);
|
||||
border: 1px solid var(--border); border-radius: 8px; padding: 10px 14px; margin-bottom: 22px;
|
||||
@@ -372,6 +405,24 @@ function isSafeImageUrl(u) {
|
||||
});
|
||||
})();
|
||||
|
||||
// Click an account row (top-actors list or the pro/con/neutral/mixed
|
||||
// breakdown) to filter the Items browser down to just that account — bound
|
||||
// once here on the persistent #viewer container (delegated), rather than
|
||||
// re-bound inside renderDashboard() every time an archive is opened, which
|
||||
// would stack a duplicate listener on every switch.
|
||||
document.getElementById('viewer').addEventListener('click', (e) => {
|
||||
const row = e.target.closest('.user-row.clickable');
|
||||
if (!row || !activeData) return;
|
||||
const handle = row.dataset.handle;
|
||||
const search = document.getElementById('itemSearch');
|
||||
if (!search) return;
|
||||
search.value = handle;
|
||||
activeFilter = null;
|
||||
document.querySelectorAll('.stile').forEach(t => t.classList.remove('active'));
|
||||
renderItemList();
|
||||
document.getElementById('itemList').scrollIntoView({ behavior: 'smooth', block: 'start' });
|
||||
});
|
||||
|
||||
const archiveList = document.getElementById('archiveList');
|
||||
const archiveCount = document.getElementById('archiveCount');
|
||||
const archiveSearch = document.getElementById('archiveSearch');
|
||||
@@ -382,15 +433,45 @@ let activeId = null;
|
||||
let activeData = null; // last /api/analytics/<id> payload
|
||||
let activeFilter = null; // 'pro' | 'neutral' | 'con' | null (item browser filter)
|
||||
|
||||
// ── Resilient fetch — auto-retry transient failures ─────────────────────────
|
||||
// res.json() throws a raw SyntaxError ("unexpected character at line 1
|
||||
// column 1...") whenever a response body isn't valid JSON — a dropped/
|
||||
// truncated connection can do this even when the server itself is fine.
|
||||
// Retries a few times with a short backoff (self-heals a one-off network
|
||||
// blip automatically) before surfacing a real, readable error. A
|
||||
// well-formed {ok:false,...} response is NOT retried — that's the server
|
||||
// correctly answering "not found" / "corrupted," retrying wouldn't change it.
|
||||
async function fetchJsonRetry(url, opts, attempts = 3) {
|
||||
let lastErr;
|
||||
for (let i = 0; i < attempts; i++) {
|
||||
try {
|
||||
const res = await fetch(url, opts);
|
||||
return await res.json();
|
||||
} catch (e) {
|
||||
lastErr = e;
|
||||
if (i < attempts - 1) await new Promise(r => setTimeout(r, 500 * (i + 1)));
|
||||
}
|
||||
}
|
||||
throw lastErr;
|
||||
}
|
||||
|
||||
// ── Sidebar ──────────────────────────────────────────────────────────────────
|
||||
|
||||
async function loadSidebar() {
|
||||
const res = await fetch('/api/archive/list');
|
||||
const json = await res.json();
|
||||
let json;
|
||||
try {
|
||||
json = await fetchJsonRetry('/api/archive/list');
|
||||
} catch (e) {
|
||||
archiveList.innerHTML = `<div class="no-archives" style="color:var(--danger)">Couldn't load archives — ${esc(e.message || String(e))}<br><button type="button" class="btn-toolbar" id="retrySidebarBtn" style="margin-top:8px">Retry</button></div>`;
|
||||
document.getElementById('retrySidebarBtn').addEventListener('click', loadSidebar);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!json.ok || !json.archives.length) {
|
||||
allArchives = [];
|
||||
archiveList.innerHTML = '<div class="no-archives">No archives yet.<br>Save one from the search page, graph, or geo search first.</div>';
|
||||
archiveList.innerHTML = json.ok
|
||||
? '<div class="no-archives">No archives yet.<br>Save one from the search page, graph, or geo search first.</div>'
|
||||
: `<div class="no-archives" style="color:var(--danger)">${esc(json.error)}</div>`;
|
||||
archiveCount.textContent = '0';
|
||||
return;
|
||||
}
|
||||
@@ -441,26 +522,80 @@ function entryHtml(a) {
|
||||
}
|
||||
|
||||
// ── Analysis ──────────────────────────────────────────────────────────────────
|
||||
// ML sentiment scoring runs locally on CPU at ~11-12ms/item (measured) — a
|
||||
// large archive (thousands of items) can genuinely take a minute-plus, so
|
||||
// this starts the job on the server (background thread, see app.py's
|
||||
// analytics_start/_run_analytics) and polls for progress instead of
|
||||
// blocking on one long request with nothing to show but a static spinner.
|
||||
|
||||
async function loadAnalytics(id) {
|
||||
activeId = id;
|
||||
activeFilter = null;
|
||||
archiveList.querySelectorAll('.archive-entry').forEach(el => el.classList.toggle('active', el.dataset.id === id));
|
||||
|
||||
viewer.innerHTML = '<div class="empty-state"><span class="spinner"></span> Analyzing…</div>';
|
||||
viewer.innerHTML = `
|
||||
<div class="empty-state">
|
||||
<div style="max-width:340px;margin:0 auto;text-align:left">
|
||||
<div><span class="spinner"></span> <span id="analyzeStatusText">Starting analysis…</span></div>
|
||||
<div class="archive-bar-fill-wrap"><div class="archive-bar-fill" id="analyzeBarFill" style="width:0%"></div></div>
|
||||
</div>
|
||||
</div>`;
|
||||
|
||||
try {
|
||||
const res = await fetch('/api/analytics/' + encodeURIComponent(id));
|
||||
const json = await res.json();
|
||||
if (!json.ok) { viewer.innerHTML = `<div class="empty-state">${esc(json.error || 'Analysis failed')}</div>`; return; }
|
||||
activeData = json;
|
||||
renderDashboard();
|
||||
const startJson = await fetchJsonRetry('/api/analytics/' + encodeURIComponent(id) + '/start', { method: 'POST' });
|
||||
if (activeId !== id) return; // user already clicked a different archive
|
||||
if (!startJson.ok) { showAnalyticsError(id, startJson.error || 'Analysis failed to start'); return; }
|
||||
pollAnalytics(id);
|
||||
} catch (e) {
|
||||
viewer.innerHTML = `<div class="empty-state">${esc(String(e))}</div>`;
|
||||
showAnalyticsError(id, e.message || String(e));
|
||||
}
|
||||
}
|
||||
|
||||
const SENT_LABELS = { pro: 'Pro', neutral: 'Neutral', con: 'Con' };
|
||||
function showAnalyticsError(id, message) {
|
||||
if (activeId !== id) return; // a later archive click already replaced this view
|
||||
viewer.innerHTML = `<div class="empty-state">Couldn't analyze this archive — ${esc(message)}<br><button type="button" class="btn-toolbar" id="retryAnalyticsBtn" style="margin-top:8px">Retry</button></div>`;
|
||||
document.getElementById('retryAnalyticsBtn').addEventListener('click', () => loadAnalytics(id));
|
||||
}
|
||||
|
||||
function pollAnalytics(id) {
|
||||
const interval = setInterval(async () => {
|
||||
// Stop polling for an archive the user has since clicked away from —
|
||||
// otherwise a slow poll response landing late could clobber whatever
|
||||
// (possibly already-finished) view is on screen now.
|
||||
if (activeId !== id) { clearInterval(interval); return; }
|
||||
|
||||
let json;
|
||||
try {
|
||||
json = await fetchJsonRetry('/api/analytics/' + encodeURIComponent(id) + '/status');
|
||||
} catch (e) {
|
||||
clearInterval(interval);
|
||||
showAnalyticsError(id, e.message || String(e));
|
||||
return;
|
||||
}
|
||||
if (activeId !== id) { clearInterval(interval); return; }
|
||||
|
||||
if (!json.ok) {
|
||||
clearInterval(interval);
|
||||
showAnalyticsError(id, json.error || 'Analysis failed');
|
||||
return;
|
||||
}
|
||||
if (json.status === 'done') {
|
||||
clearInterval(interval);
|
||||
activeData = json;
|
||||
renderDashboard();
|
||||
return;
|
||||
}
|
||||
// status === 'scoring' — update the bar and keep polling
|
||||
const total = json.total || 0;
|
||||
const pct = total > 0 ? Math.round((json.progress / total) * 100) : 0;
|
||||
const statusText = document.getElementById('analyzeStatusText');
|
||||
const fill = document.getElementById('analyzeBarFill');
|
||||
if (statusText) statusText.textContent = total > 0 ? `Scoring ${json.progress} / ${total} items…` : 'Preparing…';
|
||||
if (fill) fill.style.width = pct + '%';
|
||||
}, 700);
|
||||
}
|
||||
|
||||
const SENT_LABELS = { pro: 'Pro', neutral: 'Neutral', con: 'Con', mixed: 'Mixed' };
|
||||
let snaCy = null; // lazily built the first time "View as Graph" is clicked
|
||||
|
||||
function renderDashboard() {
|
||||
@@ -493,10 +628,15 @@ function renderDashboard() {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="section">
|
||||
<div class="section-title">Account sentiment breakdown <span class="section-note">accounts grouped by their own pro/con/neutral lean, not just individual posts — click an account to filter Items below</span></div>
|
||||
${accountSentimentHtml(d)}
|
||||
</div>
|
||||
|
||||
<div class="grid-2 section">
|
||||
<div>
|
||||
<div class="section-title">Most active accounts <span class="section-note">by items in this archive</span></div>
|
||||
${topUsersHtml(d.top_users)}
|
||||
${topUsersHtml(d.top_users, accountSentiment(d))}
|
||||
</div>
|
||||
<div>
|
||||
<div class="section-title">Most engagement <span class="section-note">replies + retweets + likes</span></div>
|
||||
@@ -532,6 +672,8 @@ function renderDashboard() {
|
||||
<input type="text" class="item-search" id="itemSearch" placeholder="Search text or author...">
|
||||
</div>
|
||||
<div class="item-list" id="itemList"></div>
|
||||
<div id="itemListSentinel" style="height:1px"></div>
|
||||
<div class="load-more-note" id="itemListStatus" style="display:none"></div>
|
||||
</div>
|
||||
`;
|
||||
|
||||
@@ -543,9 +685,26 @@ function renderDashboard() {
|
||||
renderItemList();
|
||||
});
|
||||
});
|
||||
document.getElementById('itemSearch').addEventListener('input', renderItemList);
|
||||
// Debounced — on a large archive (thousands of scored items) re-filtering
|
||||
// and re-rendering the whole list on every single keystroke makes typing
|
||||
// itself feel laggy; 180ms is short enough to still feel instant once
|
||||
// typing pauses, long enough to collapse a fast typist's keystrokes into
|
||||
// one render instead of one per character.
|
||||
let itemSearchDebounce = null;
|
||||
document.getElementById('itemSearch').addEventListener('input', () => {
|
||||
clearTimeout(itemSearchDebounce);
|
||||
itemSearchDebounce = setTimeout(renderItemList, 180);
|
||||
});
|
||||
renderItemList();
|
||||
|
||||
// Re-bind both scroll-observers to this render's freshly built sentinel
|
||||
// nodes (see the comments above itemListObserver / rebindAccountScrollObservers
|
||||
// for why re-binding is needed on every archive switch).
|
||||
itemListObserver.disconnect();
|
||||
const itemSentinel = document.getElementById('itemListSentinel');
|
||||
if (itemSentinel) itemListObserver.observe(itemSentinel);
|
||||
rebindAccountScrollObservers();
|
||||
|
||||
document.getElementById('btnSnaGraph').addEventListener('click', toggleSnaGraph);
|
||||
}
|
||||
|
||||
@@ -568,22 +727,13 @@ function toggleSnaGraph() {
|
||||
|
||||
function buildSnaGraph() {
|
||||
const d = activeData;
|
||||
const accounts = {}; // handle -> { pro, neutral, con, total, name, avatar }
|
||||
const accounts = accountSentiment(d); // handle -> { pro, neutral, con, total, name, avatar }
|
||||
const idToAuthor = {}; // raw item id -> handle, for resolving reply edges
|
||||
|
||||
d.scored_items.forEach(it => {
|
||||
const raw = it.item || {};
|
||||
const handle = it.author || raw.screen_name || raw.user;
|
||||
if (!handle) return;
|
||||
if (!accounts[handle]) {
|
||||
accounts[handle] = {
|
||||
pro: 0, neutral: 0, con: 0, total: 0,
|
||||
name: raw.name || '', avatar: raw.avatar || raw.user_avatar || '',
|
||||
};
|
||||
}
|
||||
accounts[handle][it.label]++;
|
||||
accounts[handle].total++;
|
||||
if (raw.id != null) idToAuthor[String(raw.id)] = handle;
|
||||
if (handle && raw.id != null) idToAuthor[String(raw.id)] = handle;
|
||||
});
|
||||
|
||||
const handles = Object.keys(accounts);
|
||||
@@ -722,26 +872,165 @@ function sentimentTileHtml(label, d) {
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function topUsersHtml(users) {
|
||||
// ── Account-level sentiment (not just per-post) ─────────────────────────────
|
||||
// Groups scored_items by author so "who's pro/con/neutral" can be answered
|
||||
// about ACCOUNTS, the same way top_users answers "who's most active" —
|
||||
// sentiment.py's own scoring stays per-item (a person can write both a pro
|
||||
// and a con post), this just aggregates what's already in the payload.
|
||||
// Memoized per activeData object since buildSnaGraph(), topUsersHtml() and
|
||||
// accountSentimentHtml() all need the identical aggregation for the same
|
||||
// archive and it's wasteful (and a drift risk) to recompute it three ways.
|
||||
let _acctSentCache = null;
|
||||
let _acctSentCacheFor = null;
|
||||
function accountSentiment(d) {
|
||||
if (_acctSentCacheFor === d) return _acctSentCache;
|
||||
const accounts = {}; // handle -> { pro, neutral, con, total, name, avatar }
|
||||
d.scored_items.forEach(it => {
|
||||
const raw = it.item || {};
|
||||
const handle = it.author || raw.screen_name || raw.user;
|
||||
if (!handle) return;
|
||||
if (!accounts[handle]) {
|
||||
accounts[handle] = {
|
||||
pro: 0, neutral: 0, con: 0, total: 0,
|
||||
name: raw.name || '', avatar: raw.avatar || raw.user_avatar || '',
|
||||
};
|
||||
}
|
||||
accounts[handle][it.label]++;
|
||||
accounts[handle].total++;
|
||||
});
|
||||
_acctSentCache = accounts;
|
||||
_acctSentCacheFor = d;
|
||||
return accounts;
|
||||
}
|
||||
|
||||
// An account's overall lean — majority label among ITS OWN scored items. A
|
||||
// tie between pro and con (the case that matters most: an account posting
|
||||
// equally strongly on both sides of a hot-button topic) is deliberately NOT
|
||||
// broken toward either side — arbitrarily crowning one would bake a
|
||||
// directional bias into exactly the accounts where the evidence is split
|
||||
// down the middle. Those land in "mixed", kept distinct from "neutral"
|
||||
// (which means mostly neutral-toned posts, i.e. never really took a side —
|
||||
// a different thing than "took both sides equally").
|
||||
function dominantLabel(a) {
|
||||
const max = Math.max(a.pro, a.neutral, a.con);
|
||||
if (max === 0) return null;
|
||||
const winners = ['pro', 'neutral', 'con'].filter(k => a[k] === max);
|
||||
return winners.length === 1 ? winners[0] : 'mixed';
|
||||
}
|
||||
|
||||
function userRowHtml(handle, u, opts = {}) {
|
||||
const avatarUrl = isSafeImageUrl(u.avatar) ? u.avatar : '';
|
||||
const avatar = avatarUrl
|
||||
? `<img class="user-avatar" src="${esc(avatarUrl)}" alt="" loading="lazy" referrerpolicy="no-referrer">`
|
||||
: `<div class="user-avatar user-avatar-fallback">${esc(((u.name || handle || '?').charAt(0)).toUpperCase())}</div>`;
|
||||
const rank = opts.rank != null ? `<span class="user-rank">${opts.rank}</span>` : '';
|
||||
const right = opts.right || '';
|
||||
return `
|
||||
<div class="user-row clickable" data-handle="${esc(handle)}">
|
||||
${rank}
|
||||
${avatar}
|
||||
<div class="user-identity">
|
||||
<div class="user-name">${esc(u.name || handle)}</div>
|
||||
<div class="user-handle">@${esc(handle)}</div>
|
||||
</div>
|
||||
${right}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function topUsersHtml(users, sentByHandle) {
|
||||
if (!users || !users.length) return '<div class="empty-state" style="padding:24px">No identifiable authors in this archive.</div>';
|
||||
return '<div class="user-list">' + users.map((u, i) => {
|
||||
const avatarUrl = isSafeImageUrl(u.avatar) ? u.avatar : '';
|
||||
const avatar = avatarUrl
|
||||
? `<img class="user-avatar" src="${esc(avatarUrl)}" alt="" loading="lazy" referrerpolicy="no-referrer">`
|
||||
: `<div class="user-avatar user-avatar-fallback">${esc(((u.name || u.screen_name || '?').charAt(0)).toUpperCase())}</div>`;
|
||||
return `
|
||||
<div class="user-row">
|
||||
<span class="user-rank">${i + 1}</span>
|
||||
${avatar}
|
||||
<div class="user-identity">
|
||||
<div class="user-name">${esc(u.name || u.screen_name)}</div>
|
||||
<div class="user-handle">@${esc(u.screen_name)}</div>
|
||||
</div>
|
||||
<span class="user-count">${u.count}×</span>
|
||||
</div>`;
|
||||
const s = sentByHandle && sentByHandle[u.screen_name];
|
||||
const label = s ? dominantLabel(s) : null;
|
||||
const badge = label ? `<span class="sent-badge sent-${label}" style="margin-right:6px">${SENT_LABELS[label]}</span>` : '';
|
||||
return userRowHtml(u.screen_name, u, { rank: i + 1, right: `${badge}<span class="user-count">${u.count}×</span>` });
|
||||
}).join('') + '</div>';
|
||||
}
|
||||
|
||||
// Explicit "which accounts are pro / con / neutral / mixed" listing — the
|
||||
// SNA graph already encodes this visually (pie-per-node), but that's a
|
||||
// click-to-open, hover-to-read view; this is the plain-text equivalent
|
||||
// requested directly: a scannable list per bucket, sorted by how many
|
||||
// scored items back that lean up (not just alphabetical).
|
||||
//
|
||||
// Auto-scroll rather than a hard cutoff: an archive with thousands of items
|
||||
// can have thousands of unique accounts per bucket, so all four lists start
|
||||
// at ACCOUNT_BATCH rows and grow by ACCOUNT_BATCH more each time you scroll
|
||||
// near the bottom of that bucket's own little scroll box (same idea as
|
||||
// index.html's page-level scroll-to-load-more, just scoped per box since
|
||||
// four buckets share one screen). The full sorted arrays stay in
|
||||
// `acctBuckets` in memory — nothing here ever throws data away, it just
|
||||
// paces how much gets built into HTML at once.
|
||||
const ACCOUNT_BATCH = 20;
|
||||
let acctBuckets = { pro: [], con: [], neutral: [], mixed: [] }; // full sorted [handle, data] arrays
|
||||
let acctShown = { pro: 0, con: 0, neutral: 0, mixed: 0 };
|
||||
const acctObservers = {};
|
||||
|
||||
function acctRowsHtml(key, from, to) {
|
||||
const slice = acctBuckets[key].slice(from, to);
|
||||
if (!slice.length) return from === 0 ? '<div class="empty-state" style="padding:16px;font-size:12px">None</div>' : '';
|
||||
return slice.map(([handle, a]) => userRowHtml(handle, a, {
|
||||
right: `<span class="user-breakdown">${a.pro}p · ${a.neutral}n · ${a.con}c</span>`,
|
||||
})).join('');
|
||||
}
|
||||
|
||||
function loadMoreAccounts(key) {
|
||||
const list = acctBuckets[key];
|
||||
if (acctShown[key] >= list.length) return;
|
||||
const prev = acctShown[key];
|
||||
acctShown[key] = Math.min(acctShown[key] + ACCOUNT_BATCH, list.length);
|
||||
const sentinel = document.getElementById('asgSentinel-' + key);
|
||||
if (sentinel) sentinel.insertAdjacentHTML('beforebegin', acctRowsHtml(key, prev, acctShown[key]));
|
||||
}
|
||||
|
||||
function accountSentimentHtml(d) {
|
||||
const accounts = accountSentiment(d);
|
||||
const buckets = { pro: [], neutral: [], con: [], mixed: [] };
|
||||
Object.keys(accounts).forEach(handle => {
|
||||
const a = accounts[handle];
|
||||
const label = dominantLabel(a);
|
||||
if (label) buckets[label].push([handle, a]);
|
||||
});
|
||||
Object.values(buckets).forEach(list => list.sort((x, y) => y[1].total - x[1].total));
|
||||
acctBuckets = buckets;
|
||||
acctShown = { pro: 0, con: 0, neutral: 0, mixed: 0 };
|
||||
|
||||
const sections = [
|
||||
['pro', 'Pro'], ['con', 'Con'], ['neutral', 'Neutral'], ['mixed', 'Mixed'],
|
||||
].map(([key, title]) => {
|
||||
const list = buckets[key];
|
||||
acctShown[key] = Math.min(ACCOUNT_BATCH, list.length);
|
||||
return `
|
||||
<div>
|
||||
<div class="asg-head"><span class="sent-badge sent-${key}">${title}</span><span class="asg-count">${list.length} account${list.length === 1 ? '' : 's'}</span></div>
|
||||
<div class="user-list scrollable" id="asgList-${key}">
|
||||
${acctRowsHtml(key, 0, acctShown[key])}
|
||||
<div class="asg-sentinel" id="asgSentinel-${key}" style="height:1px"></div>
|
||||
</div>
|
||||
</div>`;
|
||||
}).join('');
|
||||
|
||||
return `<div class="account-sent-grid">${sections}</div>`;
|
||||
}
|
||||
|
||||
// Re-bound after every render — the scroll box + sentinel for each bucket
|
||||
// are rebuilt fresh whenever an archive is opened (they're part of the
|
||||
// dashboard's innerHTML), so the previous archive's observer would be
|
||||
// watching a detached node. `root` is the scroll box itself (not the page),
|
||||
// since these are small internally-scrolling panels, not page-level scroll.
|
||||
function rebindAccountScrollObservers() {
|
||||
['pro', 'con', 'neutral', 'mixed'].forEach(key => {
|
||||
if (acctObservers[key]) acctObservers[key].disconnect();
|
||||
const root = document.getElementById('asgList-' + key);
|
||||
const sentinel = document.getElementById('asgSentinel-' + key);
|
||||
if (!root || !sentinel) return;
|
||||
acctObservers[key] = new IntersectionObserver((entries) => {
|
||||
if (entries.some(e => e.isIntersecting)) loadMoreAccounts(key);
|
||||
}, { root, rootMargin: '80px' });
|
||||
acctObservers[key].observe(sentinel);
|
||||
});
|
||||
}
|
||||
|
||||
function topEngagementHtml(items) {
|
||||
if (!items || !items.length) return '<div class="empty-state" style="padding:24px">No engagement data (reply/retweet/favorite counts) in this archive.</div>';
|
||||
return '<div class="engagement-list">' + items.map(e => {
|
||||
@@ -775,6 +1064,58 @@ function wordCloudHtml(words) {
|
||||
}
|
||||
|
||||
// ── Item browser ─────────────────────────────────────────────────────────────
|
||||
// Auto-scroll, same mechanism index.html uses for live search results
|
||||
// (IntersectionObserver watching a sentinel below the list) — but here
|
||||
// there's no server round-trip to paginate: every scored item is already in
|
||||
// memory, so "loading more" just means building more HTML for data that's
|
||||
// already there. ITEM_BATCH still matters even so — building a few thousand
|
||||
// .item-card divs in one go is real DOM-write cost, so this paces it out in
|
||||
// chunks as you scroll instead of paying for all of it up front.
|
||||
const ITEM_BATCH = 150;
|
||||
let filteredItems = [];
|
||||
let itemRenderCount = 0;
|
||||
|
||||
function itemCardHtml(it) {
|
||||
const matches = (it.matches || []).map(m =>
|
||||
`<span class="match-chip mc-${m.polarity}">${esc(m.word)}</span>`
|
||||
).join('');
|
||||
const scoreLabel = typeof it.confidence === 'number'
|
||||
? `${Math.round(it.confidence * 100)}% confidence`
|
||||
: `score ${it.score > 0 ? '+' : ''}${it.score}`;
|
||||
return `
|
||||
<div class="item-card">
|
||||
<div class="item-card-top">
|
||||
<span class="item-author">${it.author ? '@' + esc(it.author) : 'Unknown'}</span>
|
||||
<span class="sent-badge sent-${it.label}">${SENT_LABELS[it.label]}</span>
|
||||
<span class="item-score">${scoreLabel}</span>
|
||||
</div>
|
||||
<div class="item-text">${esc(it.text)}</div>
|
||||
${matches ? `<div class="item-matches">${matches}</div>` : ''}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function updateItemListStatus() {
|
||||
const status = document.getElementById('itemListStatus');
|
||||
if (!status) return;
|
||||
const remaining = filteredItems.length - itemRenderCount;
|
||||
if (remaining > 0) {
|
||||
status.style.display = '';
|
||||
status.textContent = `Showing ${itemRenderCount} of ${filteredItems.length} — scroll for more…`;
|
||||
} else {
|
||||
status.style.display = 'none';
|
||||
}
|
||||
}
|
||||
|
||||
function loadMoreItems() {
|
||||
if (itemRenderCount >= filteredItems.length) return;
|
||||
const prev = itemRenderCount;
|
||||
itemRenderCount = Math.min(itemRenderCount + ITEM_BATCH, filteredItems.length);
|
||||
document.getElementById('itemList').insertAdjacentHTML(
|
||||
'beforeend',
|
||||
filteredItems.slice(prev, itemRenderCount).map(itemCardHtml).join(''),
|
||||
);
|
||||
updateItemListStatus();
|
||||
}
|
||||
|
||||
function renderItemList() {
|
||||
const d = activeData;
|
||||
@@ -786,29 +1127,29 @@ function renderItemList() {
|
||||
|
||||
document.getElementById('itemCount').textContent = `${items.length} / ${d.scored_items.length}`;
|
||||
|
||||
const list = document.getElementById('itemList');
|
||||
if (!items.length) { list.innerHTML = '<div class="empty-state" style="padding:24px">No items match.</div>'; return; }
|
||||
filteredItems = items;
|
||||
itemRenderCount = Math.min(ITEM_BATCH, items.length);
|
||||
|
||||
list.innerHTML = items.map(it => {
|
||||
const matches = (it.matches || []).map(m =>
|
||||
`<span class="match-chip mc-${m.polarity}">${esc(m.word)}</span>`
|
||||
).join('');
|
||||
const scoreLabel = typeof it.confidence === 'number'
|
||||
? `${Math.round(it.confidence * 100)}% confidence`
|
||||
: `score ${it.score > 0 ? '+' : ''}${it.score}`;
|
||||
return `
|
||||
<div class="item-card">
|
||||
<div class="item-card-top">
|
||||
<span class="item-author">${it.author ? '@' + esc(it.author) : 'Unknown'}</span>
|
||||
<span class="sent-badge sent-${it.label}">${SENT_LABELS[it.label]}</span>
|
||||
<span class="item-score">${scoreLabel}</span>
|
||||
</div>
|
||||
<div class="item-text">${esc(it.text)}</div>
|
||||
${matches ? `<div class="item-matches">${matches}</div>` : ''}
|
||||
</div>`;
|
||||
}).join('');
|
||||
const list = document.getElementById('itemList');
|
||||
if (!items.length) {
|
||||
list.innerHTML = '<div class="empty-state" style="padding:24px">No items match.</div>';
|
||||
updateItemListStatus();
|
||||
return;
|
||||
}
|
||||
|
||||
list.innerHTML = items.slice(0, itemRenderCount).map(itemCardHtml).join('');
|
||||
updateItemListStatus();
|
||||
}
|
||||
|
||||
// root: null (page viewport) — same reasoning as index.html's own load-more
|
||||
// observer: the item list is never height-constrained itself, the page is
|
||||
// what actually scrolls. Re-observed (not just created once) after every
|
||||
// render since #itemListSentinel is rebuilt along with the rest of the
|
||||
// dashboard each time a different archive is opened.
|
||||
const itemListObserver = new IntersectionObserver((entries) => {
|
||||
if (entries.some(e => e.isIntersecting)) loadMoreItems();
|
||||
}, { root: null, rootMargin: '300px' });
|
||||
|
||||
loadSidebar();
|
||||
})();
|
||||
</script>
|
||||
|
||||
@@ -288,6 +288,12 @@
|
||||
|
||||
.cards-area { padding: 16px 18px; overflow-y: auto; flex: 1; }
|
||||
.cards-grid { display: grid; gap: 8px; }
|
||||
.load-more-note {
|
||||
font-size: 11px;
|
||||
color: var(--muted);
|
||||
text-align: center;
|
||||
padding: 14px 0 2px;
|
||||
}
|
||||
|
||||
/* ── Cards ── */
|
||||
.card {
|
||||
@@ -298,6 +304,124 @@
|
||||
transition: border-color 0.12s;
|
||||
}
|
||||
.card:hover { border-color: #3a3d50; }
|
||||
|
||||
/* Profile header — cover photo, avatar, name/handle, bio. Same rules as
|
||||
the live search page: only rendered when the item actually carries any
|
||||
of that (a tweet's embedded author, a bare user/follower/retweeter
|
||||
record); everything else keeps the plain row list it always had. */
|
||||
.card-header { margin-bottom: 10px; }
|
||||
.card-header.has-banner { margin: -12px -14px 10px; }
|
||||
.card-banner {
|
||||
height: 64px;
|
||||
background-size: cover;
|
||||
background-position: center;
|
||||
background-color: var(--bg);
|
||||
}
|
||||
.card-header-row { display: flex; align-items: flex-end; gap: 10px; }
|
||||
.card-header.has-banner .card-header-row { padding: 0 14px; margin-top: -26px; }
|
||||
.card-avatar {
|
||||
width: 48px;
|
||||
height: 48px;
|
||||
border-radius: 50%;
|
||||
object-fit: cover;
|
||||
border: 2px solid var(--surface);
|
||||
background: var(--bg);
|
||||
flex-shrink: 0;
|
||||
}
|
||||
.card-avatar-fallback {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-size: 17px;
|
||||
font-weight: 700;
|
||||
color: var(--muted);
|
||||
}
|
||||
.card-identity { min-width: 0; padding-bottom: 3px; }
|
||||
.card-name {
|
||||
font-size: 14px;
|
||||
font-weight: 600;
|
||||
color: var(--text);
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
.card-handle { font-size: 12px; color: var(--muted); }
|
||||
.card-bio { margin-top: 8px; font-size: 12px; color: var(--text); opacity: 0.85; line-height: 1.45; }
|
||||
.card-header.has-banner .card-bio { padding: 0 14px; }
|
||||
|
||||
/* Compact byline — tweets/replies get just this instead of the full
|
||||
banner+bio header above, which is reserved for records that ARE a user
|
||||
rather than a tweet someone wrote. */
|
||||
.card-byline { display: flex; align-items: center; gap: 8px; margin-bottom: 8px; }
|
||||
.card-byline .card-avatar { width: 26px; height: 26px; border-width: 1px; }
|
||||
.card-byline .card-avatar-fallback { font-size: 11px; }
|
||||
.card-byline .card-identity { display: flex; align-items: baseline; gap: 6px; padding-bottom: 0; min-width: 0; }
|
||||
.card-byline .card-name { font-size: 13px; max-width: 55%; }
|
||||
.card-byline .card-handle { flex-shrink: 0; }
|
||||
|
||||
/* Retweeted-content rows — subtle highlight, same as the live search page */
|
||||
.card-row.rt-origin .card-key { color: var(--accent); }
|
||||
.card-row.rt-origin .card-val { color: #c7d2fe; }
|
||||
|
||||
.reply-context { font-size: 11px; color: var(--muted); margin-bottom: 4px; }
|
||||
|
||||
/* Inline expand-and-save — replies/retweets fetched here get folded into
|
||||
this archive's own item list and checkpointed back to disk immediately,
|
||||
same as the live pages but persisted right away since there's no
|
||||
separate "Archive" button on this read view. */
|
||||
.reply-expand-btn, .retweet-expand-btn {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 5px;
|
||||
margin-top: 10px;
|
||||
margin-right: 6px;
|
||||
background: none;
|
||||
border: 1px solid var(--border);
|
||||
color: var(--accent);
|
||||
font-family: var(--font);
|
||||
font-size: 11px;
|
||||
font-weight: 500;
|
||||
padding: 4px 10px;
|
||||
border-radius: 5px;
|
||||
cursor: pointer;
|
||||
}
|
||||
.retweet-expand-btn { color: #f59e0b; }
|
||||
.reply-expand-btn:hover:not(:disabled), .retweet-expand-btn:hover:not(:disabled) { border-color: currentColor; }
|
||||
.reply-expand-btn:disabled, .retweet-expand-btn:disabled { opacity: 0.5; cursor: wait; }
|
||||
|
||||
.reply-thread, .retweet-thread {
|
||||
margin: 10px 0 2px 16px;
|
||||
padding-left: 14px;
|
||||
border-left: 2px solid var(--border);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
}
|
||||
.reply-thread.hidden, .retweet-thread.hidden { display: none; }
|
||||
|
||||
.reply-load-more-btn {
|
||||
align-self: flex-start;
|
||||
background: none;
|
||||
border: none;
|
||||
color: var(--accent);
|
||||
font-family: var(--font);
|
||||
font-size: 11px;
|
||||
cursor: pointer;
|
||||
text-decoration: underline;
|
||||
text-underline-offset: 2px;
|
||||
padding: 2px 0;
|
||||
}
|
||||
.reply-load-more-btn:disabled { opacity: 0.5; cursor: wait; text-decoration: none; }
|
||||
|
||||
.save-indicator {
|
||||
display: inline-block;
|
||||
margin-top: 10px;
|
||||
font-size: 11px;
|
||||
color: var(--muted);
|
||||
}
|
||||
.save-indicator.ok { color: var(--success); }
|
||||
.save-indicator.err { color: var(--danger); }
|
||||
|
||||
.card-row {
|
||||
display: flex;
|
||||
gap: 10px;
|
||||
@@ -516,6 +640,8 @@
|
||||
<div id="cardsBox">
|
||||
<div class="empty-state">Select an archive from the sidebar to view its contents.</div>
|
||||
</div>
|
||||
<div id="cardsSentinel" style="height:1px"></div>
|
||||
<div class="load-more-note" id="cardsLoadStatus" style="display:none"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -560,8 +686,20 @@
|
||||
|
||||
let allItems = [];
|
||||
let activeId = null;
|
||||
let activeMeta = {}; // meta.json for the open archive — tool/query, needed to checkpoint-save expansions back to it
|
||||
let allArchives = []; // full list from the server; archiveSearch filters this client-side
|
||||
|
||||
// ── Reply/retweet expand-and-save cooldown ──────────────────────────────────
|
||||
// Mirrors index.html's throttle handling — every expand here is a cookie-mode
|
||||
// call, sharing the backend's single 5s-per-source cookie clock with every
|
||||
// other tool, so a fast double-click here still respects it.
|
||||
const THROTTLE_SECONDS = 5;
|
||||
let cooldownUntil = { cookie: 0 };
|
||||
|
||||
function stampCooldown(sources) {
|
||||
(sources || []).forEach(source => { cooldownUntil[source] = Date.now() + THROTTLE_SECONDS * 1000; });
|
||||
}
|
||||
|
||||
const archiveList = document.getElementById('archiveList');
|
||||
const archiveCount = document.getElementById('archiveCount');
|
||||
const archiveSearch = document.getElementById('archiveSearch');
|
||||
@@ -573,18 +711,53 @@ const resultCount = document.getElementById('resultCount');
|
||||
const deleteBtn = document.getElementById('deleteBtn');
|
||||
const cardsBox = document.getElementById('cardsBox');
|
||||
|
||||
// ── Resilient fetch — auto-retry transient failures ─────────────────────────
|
||||
// `res.json()` throws a raw SyntaxError ("unexpected character at line 1
|
||||
// column 1...") whenever the response body isn't valid JSON — a dropped/
|
||||
// truncated connection (flaky wifi, the device sleeping mid-request) or a
|
||||
// server-side hiccup can both do this, and previously nothing here caught
|
||||
// it: the promise just rejected, the "Loading…" spinner never went away,
|
||||
// and the failure showed up nowhere the user could see WHY. This retries a
|
||||
// few times with a short backoff (self-heals a one-off network blip
|
||||
// automatically — no user action needed) before surfacing a real,
|
||||
// human-readable error. A well-formed {ok:false,...} response is NOT
|
||||
// retried here — that's the server correctly telling you something (e.g.
|
||||
// "Archive not found"), retrying wouldn't change the answer; only fetch/
|
||||
// parse failures (the connection or the body itself was the problem) do.
|
||||
async function fetchJsonRetry(url, opts, attempts = 3) {
|
||||
let lastErr;
|
||||
for (let i = 0; i < attempts; i++) {
|
||||
try {
|
||||
const res = await fetch(url, opts);
|
||||
return await res.json();
|
||||
} catch (e) {
|
||||
lastErr = e;
|
||||
if (i < attempts - 1) await new Promise(r => setTimeout(r, 500 * (i + 1)));
|
||||
}
|
||||
}
|
||||
throw lastErr;
|
||||
}
|
||||
|
||||
// ── Sidebar ───────────────────────────────────────────────────────────────────
|
||||
// The server already returns archives newest-first (by archived_at/updated_at,
|
||||
// not folder name — folder names are prefixed by tool_type so sorting by name
|
||||
// doesn't actually sort chronologically once more than one tool's been used).
|
||||
|
||||
async function loadSidebar() {
|
||||
const res = await fetch('/api/archive/list');
|
||||
const json = await res.json();
|
||||
let json;
|
||||
try {
|
||||
json = await fetchJsonRetry('/api/archive/list');
|
||||
} catch (e) {
|
||||
archiveList.innerHTML = `<div class="no-archives" style="color:var(--danger)">Couldn't load archives — ${esc(e.message || String(e))}<br><button type="button" class="reply-expand-btn" id="retrySidebarBtn" style="margin-top:8px">Retry</button></div>`;
|
||||
document.getElementById('retrySidebarBtn').addEventListener('click', loadSidebar);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!json.ok || !json.archives.length) {
|
||||
allArchives = [];
|
||||
archiveList.innerHTML = '<div class="no-archives">No archives yet.<br>Enable Auto Archive in the tool and run a search.</div>';
|
||||
archiveList.innerHTML = json.ok
|
||||
? '<div class="no-archives">No archives yet.<br>Enable Auto Archive in the tool and run a search.</div>'
|
||||
: `<div class="no-archives" style="color:var(--danger)">${esc(json.error)}</div>`;
|
||||
archiveCount.textContent = '0';
|
||||
return;
|
||||
}
|
||||
@@ -654,23 +827,29 @@ async function loadArchive(id) {
|
||||
viewerTop.style.display = 'none';
|
||||
searchInput.value = '';
|
||||
|
||||
const res = await fetch(`/api/archive/${id}/results`);
|
||||
const json = await res.json();
|
||||
let json;
|
||||
try {
|
||||
json = await fetchJsonRetry(`/api/archive/${id}/results`);
|
||||
} catch (e) {
|
||||
cardsBox.innerHTML = `<div class="empty-state" style="color:var(--danger)">Couldn't load this archive — ${esc(e.message || String(e))}<br><button type="button" class="reply-expand-btn" id="retryArchiveBtn" style="margin-top:8px">Retry</button></div>`;
|
||||
document.getElementById('retryArchiveBtn').addEventListener('click', () => loadArchive(id));
|
||||
return;
|
||||
}
|
||||
if (!json.ok) {
|
||||
cardsBox.innerHTML = `<div class="empty-state" style="color:var(--danger)">${esc(json.error)}</div>`;
|
||||
return;
|
||||
}
|
||||
|
||||
allItems = json.results || [];
|
||||
const meta = json.meta || {};
|
||||
allItems = json.results || [];
|
||||
activeMeta = json.meta || {};
|
||||
|
||||
const tool = (meta.tool || 'unknown').replace(/_/g, ' ');
|
||||
const query = queryLabel(meta.query || {});
|
||||
const tool = (activeMeta.tool || 'unknown').replace(/_/g, ' ');
|
||||
const query = queryLabel(activeMeta.query || {});
|
||||
viewerTitle.textContent = tool + (query ? ` — "${query}"` : '');
|
||||
viewerSub.textContent = [
|
||||
meta.archived_at ? meta.archived_at.replace('T', ' ') : '',
|
||||
activeMeta.archived_at ? activeMeta.archived_at.replace('T', ' ') : '',
|
||||
`${allItems.length} items`,
|
||||
meta.media_count ? `${meta.media_count} media files` : '',
|
||||
activeMeta.media_count ? `${activeMeta.media_count} media files` : '',
|
||||
].filter(Boolean).join(' · ');
|
||||
|
||||
viewerTop.style.display = '';
|
||||
@@ -678,8 +857,14 @@ async function loadArchive(id) {
|
||||
}
|
||||
|
||||
// ── Search ────────────────────────────────────────────────────────────────────
|
||||
|
||||
searchInput.addEventListener('input', () => renderCards(allItems, searchInput.value));
|
||||
// Debounced — same reasoning as analytics.html's item search: re-filtering
|
||||
// and re-rendering the full card list (avatars, media, everything) on every
|
||||
// single keystroke gets janky once an archive has hundreds/thousands of items.
|
||||
let cardSearchDebounce = null;
|
||||
searchInput.addEventListener('input', () => {
|
||||
clearTimeout(cardSearchDebounce);
|
||||
cardSearchDebounce = setTimeout(() => renderCards(allItems, searchInput.value), 180);
|
||||
});
|
||||
|
||||
function flatText(obj) {
|
||||
if (!obj || typeof obj !== 'object') return String(obj ?? '');
|
||||
@@ -688,10 +873,49 @@ function flatText(obj) {
|
||||
|
||||
// ── Cards ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
// PRIORITY / DRILLABLE / SOURCE_CLASS / AGE_LABELS come from
|
||||
// static/js/card_constants.js, loaded above — shared with index.html.
|
||||
const SKIP = ['archived_media'];
|
||||
const CLAMP = new Set(['text','full_text','article_text','description','post_text']);
|
||||
// PRIORITY / DRILLABLE / SOURCE_CLASS / AGE_LABELS / buildCardHeader come
|
||||
// from static/js/card_constants.js, loaded above — shared with index.html,
|
||||
// so a record's avatar/name/handle/banner/bio render identically whether
|
||||
// you're looking at a live result or a saved archive.
|
||||
const SKIP = ['archived_media', 'profile_image_url', 'profile_banner_url', 'entities', 'extended_entities', 'urls', 'media', 'indices'];
|
||||
const CLAMP = new Set(['text','full_text','content','description','bio','article_text','retweeted_text','retweeted_by_bio','post_text']);
|
||||
|
||||
// ── Auto-scroll — same mechanism index.html's live search results use
|
||||
// (IntersectionObserver watching a sentinel, root: null since — per this
|
||||
// page's own back-to-top comment above — the PAGE is what actually scrolls
|
||||
// here, .cards-area's overflow:auto never gets a bounded height to actually
|
||||
// clip against). Building the card HTML itself is cheap (measured well
|
||||
// under 20ms for 200+ cards) — what actually made a big first batch feel
|
||||
// slow is every card's <img> avatar firing its own request to Twitter's CDN
|
||||
// at once; 50 of those landing simultaneously is what dragged out the
|
||||
// initial paint, not the rendering logic. 10 keeps that burst small; the
|
||||
// tighter rootMargin below (vs. what analytics.html's plain-text item list
|
||||
// uses) matters just as much — a bigger margin pre-triggers the NEXT
|
||||
// image-heavy batch before you've actually scrolled toward it.
|
||||
const CARD_BATCH = 10;
|
||||
let filteredCardItems = [];
|
||||
let cardRenderCount = 0;
|
||||
|
||||
function updateCardsLoadStatus() {
|
||||
const status = document.getElementById('cardsLoadStatus');
|
||||
const remaining = filteredCardItems.length - cardRenderCount;
|
||||
if (remaining > 0) {
|
||||
status.style.display = '';
|
||||
status.textContent = `Showing ${cardRenderCount} of ${filteredCardItems.length} — scroll for more…`;
|
||||
} else {
|
||||
status.style.display = 'none';
|
||||
}
|
||||
}
|
||||
|
||||
function loadMoreCards() {
|
||||
if (cardRenderCount >= filteredCardItems.length) return;
|
||||
const prev = cardRenderCount;
|
||||
cardRenderCount = Math.min(cardRenderCount + CARD_BATCH, filteredCardItems.length);
|
||||
const grid = cardsBox.querySelector('.cards-grid');
|
||||
if (!grid) return;
|
||||
grid.insertAdjacentHTML('beforeend', filteredCardItems.slice(prev, cardRenderCount).map(it => buildCard(it)).join(''));
|
||||
updateCardsLoadStatus();
|
||||
}
|
||||
|
||||
function renderCards(items, query) {
|
||||
const q = query.toLowerCase().trim();
|
||||
@@ -699,24 +923,60 @@ function renderCards(items, query) {
|
||||
|
||||
resultCount.textContent = filtered.length + (q ? ' found' : ' results');
|
||||
|
||||
filteredCardItems = filtered;
|
||||
cardRenderCount = Math.min(CARD_BATCH, filtered.length);
|
||||
|
||||
if (!filtered.length) {
|
||||
cardsBox.innerHTML = '<div class="empty-state">No matching results.</div>';
|
||||
updateCardsLoadStatus();
|
||||
return;
|
||||
}
|
||||
cardsBox.innerHTML = '<div class="cards-grid">' + filtered.map(buildCard).join('') + '</div>';
|
||||
cardsBox.innerHTML = '<div class="cards-grid">' + filtered.slice(0, cardRenderCount).map(it => buildCard(it)).join('') + '</div>';
|
||||
updateCardsLoadStatus();
|
||||
}
|
||||
|
||||
function buildCard(item) {
|
||||
// #cardsSentinel and .cards-area are static page elements (only #cardsBox's
|
||||
// innerHTML gets replaced on every render), so — unlike analytics.html's
|
||||
// dashboard, which gets rebuilt from scratch per archive — this observer is
|
||||
// bound exactly once, right here, and just keeps working across archive
|
||||
// switches and searches.
|
||||
const cardsObserver = new IntersectionObserver((entries) => {
|
||||
if (entries.some(e => e.isIntersecting)) loadMoreCards();
|
||||
}, { root: null, rootMargin: '100px' });
|
||||
cardsObserver.observe(document.getElementById('cardsSentinel'));
|
||||
|
||||
// Same "Replying to @x @y" trim index.html's cards apply — reply_to_mentions
|
||||
// (from cookie_client.py) lists exactly which leading @mentions x.com itself
|
||||
// hides from the rendered tweet body. Display-only: item.text in the
|
||||
// archived JSON is untouched.
|
||||
function stripLeadingMentions(text, mentions) {
|
||||
if (!text || !mentions || !mentions.length) return text;
|
||||
let rest = text;
|
||||
for (const m of mentions) {
|
||||
const re = new RegExp('^\\s*@' + String(m).replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '\\b');
|
||||
const match = rest.match(re);
|
||||
if (!match) break;
|
||||
rest = rest.slice(match[0].length);
|
||||
}
|
||||
rest = rest.replace(/^\s+/, '');
|
||||
return rest || text;
|
||||
}
|
||||
|
||||
// nested=true for a reply/retweet rendered inside another card's expanded
|
||||
// thread — same recursive shape index.html's buildCard() uses, so a reply
|
||||
// that itself has replies still gets its own Expand button.
|
||||
function buildCard(item, nested = false) {
|
||||
if (typeof item !== 'object' || !item) return '';
|
||||
|
||||
const tweetId = item.id ? String(item.id) : null;
|
||||
const entries = Object.entries(item);
|
||||
const { html: header, usedFields: headerFields } = buildCardHeader(item);
|
||||
// Same rule as the live search page: order follows PRIORITY's own sequence,
|
||||
// and nothing gets capped off — a saved archive should show exactly the
|
||||
// same fields, in the same order, as when it was first fetched.
|
||||
const priKeys = new Set(entries.map(([k]) => k).filter(k => PRIORITY.includes(k)));
|
||||
const pri = PRIORITY.filter(k => priKeys.has(k)).map(k => [k, item[k]]);
|
||||
const rest = entries.filter(([k]) => !PRIORITY.includes(k) && !SKIP.includes(k));
|
||||
const pri = PRIORITY.filter(k => priKeys.has(k) && !headerFields.includes(k)).map(k => [k, item[k]]);
|
||||
const rest = entries.filter(([k]) => !PRIORITY.includes(k) && !SKIP.includes(k) && !headerFields.includes(k));
|
||||
|
||||
const rows = [...pri, ...rest].map(([k, v]) => {
|
||||
let display;
|
||||
@@ -737,6 +997,10 @@ function buildCard(item) {
|
||||
display = `<span class="age-badge age-${esc(String(v))}">${esc(AGE_LABELS[v] || v)}</span>`;
|
||||
} else if (k === 'content_type') {
|
||||
display = `<span class="content-type-badge ct-${esc(String(v))}">${esc(CONTENT_TYPE_LABELS[v] || String(v))}</span>`;
|
||||
} else if (k === 'text' && item.reply_to_mentions && item.reply_to_mentions.length) {
|
||||
const badge = `<div class="reply-context">↩ Replying to ${item.reply_to_mentions.map(m => '@' + esc(m)).join(', ')}</div>`;
|
||||
const clean = stripLeadingMentions(String(v), item.reply_to_mentions);
|
||||
display = `${badge}<span class="clamp">${esc(clean)}</span>`;
|
||||
} else if (typeof v === 'object') {
|
||||
const s = JSON.stringify(v);
|
||||
display = `<span style="color:var(--muted);font-size:11px">${esc(s.length > 100 ? s.slice(0,100)+'…' : s)}</span>`;
|
||||
@@ -744,42 +1008,321 @@ function buildCard(item) {
|
||||
const cls = CLAMP.has(k) ? ' clamp' : '';
|
||||
display = `<span class="${cls}">${esc(String(v))}</span>`;
|
||||
}
|
||||
return `<div class="card-row"><div class="card-key">${esc(k)}</div><div class="card-val">${display}</div></div>`;
|
||||
const rtCls = k.startsWith('retweeted_') ? ' rt-origin' : '';
|
||||
return `<div class="card-row${rtCls}"><div class="card-key">${esc(k)}</div><div class="card-val">${display}</div></div>`;
|
||||
}).join('');
|
||||
|
||||
const media = buildMedia(item);
|
||||
return `<div class="card">${rows}${media}</div>`;
|
||||
|
||||
// Expand-in-place — same trigger rule as index.html (id + a positive
|
||||
// count), but the fetched result here also gets folded into this
|
||||
// archive's saved results.json as soon as it lands (see saveExpansion()),
|
||||
// since this view has no separate "Archive" button to click afterward.
|
||||
const replyCount = Number(item.reply_count);
|
||||
const canReplies = tweetId && Number.isFinite(replyCount) && replyCount > 0;
|
||||
const rtCount = Number(item.retweet_count);
|
||||
const canRetweets = tweetId && Number.isFinite(rtCount) && rtCount > 0;
|
||||
|
||||
const replyBtn = canReplies
|
||||
? `<button type="button" class="reply-expand-btn" data-tweet-id="${esc(tweetId)}" data-count="${replyCount}">↩ Expand ${replyCount} repl${replyCount === 1 ? 'y' : 'ies'}</button>`
|
||||
: '';
|
||||
const retweetBtn = canRetweets
|
||||
? `<button type="button" class="retweet-expand-btn" data-tweet-id="${esc(tweetId)}" data-count="${rtCount}">↗ Expand ${rtCount} retweet${rtCount === 1 ? '' : 's'}</button>`
|
||||
: '';
|
||||
|
||||
const cardCls = nested ? 'card reply-card' : 'card';
|
||||
const card = `<div class="${cardCls}">${header}${rows}${media}${replyBtn}${retweetBtn}<span class="save-indicator" data-role="save-indicator"></span></div>`;
|
||||
if (!canReplies && !canRetweets) return card;
|
||||
|
||||
const threads = [
|
||||
canReplies ? `<div class="reply-thread hidden" data-kind="reply" data-tweet-id="${esc(tweetId)}"></div>` : '',
|
||||
canRetweets ? `<div class="retweet-thread hidden" data-kind="retweet" data-tweet-id="${esc(tweetId)}"></div>` : '',
|
||||
].join('');
|
||||
return `<div class="card-thread-wrap">${card}${threads}</div>`;
|
||||
}
|
||||
|
||||
// Cookie mode: item.media = [{type, thumb, url}]. API mode: raw Twitter
|
||||
// shape under extended_entities.media / entities.media. Same normalization
|
||||
// index.html's extractMedia() applies — needed here for items fetched
|
||||
// through an inline expand, which haven't been through archive.py's
|
||||
// download pipeline yet (no archived_media of their own until the
|
||||
// checkpoint save that follows lands).
|
||||
function extractMedia(item) {
|
||||
if (Array.isArray(item.media) && item.media.length) {
|
||||
const first = item.media[0];
|
||||
if (first && typeof first === 'object' && ('thumb' in first || 'url' in first)) {
|
||||
return item.media;
|
||||
}
|
||||
}
|
||||
const src = (item.extended_entities && item.extended_entities.media)
|
||||
|| (item.entities && item.entities.media);
|
||||
if (!Array.isArray(src)) return [];
|
||||
return src.map(m => {
|
||||
const mtype = m.type || 'photo';
|
||||
const thumb = m.media_url_https || m.media_url || '';
|
||||
let url = thumb;
|
||||
if (mtype === 'video' || mtype === 'animated_gif') {
|
||||
const variants = (m.video_info && m.video_info.variants) || [];
|
||||
const mp4s = variants.filter(v => v.content_type === 'video/mp4');
|
||||
if (mp4s.length) url = mp4s.reduce((b, v) => (v.bitrate||0) > (b.bitrate||0) ? v : b).url;
|
||||
}
|
||||
return { type: mtype, thumb, url };
|
||||
}).filter(m => m.thumb);
|
||||
}
|
||||
|
||||
function buildMedia(item) {
|
||||
// Already-archived item: serve from local disk via the media route.
|
||||
const paths = item.archived_media;
|
||||
if (!Array.isArray(paths) || !paths.length) return '';
|
||||
if (Array.isArray(paths) && paths.length) {
|
||||
const archiveId = activeId;
|
||||
const html = paths.map(p => {
|
||||
// p = "media/filename.ext"
|
||||
const filename = p.replace(/^media\//, '');
|
||||
const src = `/api/archive/${encodeURIComponent(archiveId)}/media/${encodeURIComponent(filename)}`;
|
||||
const isVideo = filename.endsWith('.mp4');
|
||||
const isGif = filename.includes('animated_gif');
|
||||
|
||||
const archiveId = activeId;
|
||||
const html = paths.map(p => {
|
||||
// p = "media/filename.ext"
|
||||
const filename = p.replace(/^media\//, '');
|
||||
const src = `/api/archive/${encodeURIComponent(archiveId)}/media/${encodeURIComponent(filename)}`;
|
||||
const isVideo = filename.endsWith('.mp4');
|
||||
const isGif = filename.includes('animated_gif');
|
||||
if (isVideo || isGif) {
|
||||
const loop = isGif ? 'loop muted' : '';
|
||||
const label = isGif ? '<span class="media-badge">GIF</span>' : '';
|
||||
return `<div class="media-item">
|
||||
<video class="media-video" controls ${loop} preload="none">
|
||||
<source src="${src}" type="video/mp4">
|
||||
</video>${label}
|
||||
</div>`;
|
||||
}
|
||||
return `<a href="${src}" target="_blank" rel="noopener" class="media-item">
|
||||
<img src="${src}" class="media-thumb" alt="media" loading="lazy">
|
||||
</a>`;
|
||||
}).join('');
|
||||
return `<div class="card-media">${html}</div>`;
|
||||
}
|
||||
|
||||
if (isVideo || isGif) {
|
||||
const loop = isGif ? 'loop muted' : '';
|
||||
const label = isGif ? '<span class="media-badge">GIF</span>' : '';
|
||||
// Freshly expanded item, not yet checkpointed to disk — render straight
|
||||
// from the source (video routed through the same CDN proxy the live
|
||||
// search page uses), same as index.html's renderMedia().
|
||||
const mediaList = extractMedia(item);
|
||||
if (!mediaList.length) return '';
|
||||
const html = mediaList.map(m => {
|
||||
if (m.type === 'video' || m.type === 'animated_gif') {
|
||||
const proxied = `/api/video?url=${encodeURIComponent(m.url)}`;
|
||||
const loop = m.type === 'animated_gif' ? 'loop muted' : '';
|
||||
const poster = m.thumb ? `poster="${esc(m.thumb)}"` : '';
|
||||
const label = m.type === 'animated_gif' ? '<span class="media-badge">GIF</span>' : '';
|
||||
return `<div class="media-item">
|
||||
<video class="media-video" controls ${loop} preload="none">
|
||||
<source src="${src}" type="video/mp4">
|
||||
<video class="media-video" controls ${loop} ${poster} preload="none">
|
||||
<source src="${proxied}" type="video/mp4">
|
||||
</video>${label}
|
||||
</div>`;
|
||||
}
|
||||
return `<a href="${src}" target="_blank" rel="noopener" class="media-item">
|
||||
<img src="${src}" class="media-thumb" alt="media" loading="lazy">
|
||||
return `<a href="${esc(m.url)}" target="_blank" rel="noopener" class="media-item">
|
||||
<img src="${esc(m.thumb)}" class="media-thumb" alt="media" loading="lazy">
|
||||
</a>`;
|
||||
}).join('');
|
||||
|
||||
return `<div class="card-media">${html}</div>`;
|
||||
}
|
||||
|
||||
// ── Inline expand-and-save (replies / retweets) ─────────────────────────────
|
||||
// Same interaction as index.html's reply threads and graph.html's node
|
||||
// expand, adapted to this read-only view: there's no separate "Archive"
|
||||
// button here, so a successful expand immediately checkpoints the fetched
|
||||
// items back into THIS SAME archive's results.json via archive.py's
|
||||
// update() (which skips re-downloading any media already on disk) — that's
|
||||
// what makes an expanded comment/reply/retweet count as saved digital
|
||||
// evidence rather than something that only ever existed in the browser tab.
|
||||
|
||||
const EXPAND_KIND = {
|
||||
reply: {
|
||||
toolType: 'tweet_replies_extractor', threadSel: '.reply-thread', btnSel: '.reply-expand-btn',
|
||||
label: n => `↩ Expand ${n} repl${n === 1 ? 'y' : 'ies'}`, hideLabel: '▲ Hide replies',
|
||||
loadMoreLabel: 'Load more replies…', noneLabel: 'No replies found',
|
||||
},
|
||||
retweet: {
|
||||
toolType: 'tweet_retweeters_extractor', threadSel: '.retweet-thread', btnSel: '.retweet-expand-btn',
|
||||
label: n => `↗ Expand ${n} retweet${n === 1 ? '' : 's'}`, hideLabel: '▲ Hide retweets',
|
||||
loadMoreLabel: 'Load more retweets…', noneLabel: 'No retweeters found',
|
||||
},
|
||||
};
|
||||
|
||||
cardsBox.addEventListener('click', (e) => {
|
||||
const replyBtn = e.target.closest('.reply-expand-btn');
|
||||
if (replyBtn) { toggleThread(replyBtn, 'reply'); return; }
|
||||
const rtBtn = e.target.closest('.retweet-expand-btn');
|
||||
if (rtBtn) { toggleThread(rtBtn, 'retweet'); return; }
|
||||
const moreBtn = e.target.closest('.reply-load-more-btn');
|
||||
if (moreBtn) { loadMoreThread(moreBtn); }
|
||||
});
|
||||
|
||||
function armCountdown(btn, msRemaining, resetLabel) {
|
||||
const secs = Math.max(1, Math.ceil(msRemaining / 1000));
|
||||
btn.disabled = true;
|
||||
btn.textContent = `Wait ${secs}s…`;
|
||||
setTimeout(() => { btn.disabled = false; btn.textContent = resetLabel(); }, msRemaining);
|
||||
}
|
||||
|
||||
async function toggleThread(btn, kind) {
|
||||
const cfg = EXPAND_KIND[kind];
|
||||
const wrap = btn.closest('.card-thread-wrap');
|
||||
const thread = wrap.querySelector(`:scope > ${cfg.threadSel}`);
|
||||
|
||||
// Already fetched once — just show/hide, no new request (mirrors
|
||||
// index.html's toggleReplyThread()).
|
||||
if (thread.dataset.loaded === '1') {
|
||||
const nowHidden = thread.classList.toggle('hidden');
|
||||
btn.textContent = nowHidden ? cfg.label(Number(btn.dataset.count)) : cfg.hideLabel;
|
||||
return;
|
||||
}
|
||||
|
||||
const tweetId = btn.dataset.tweetId;
|
||||
const remaining = cooldownUntil.cookie - Date.now();
|
||||
if (remaining > 0) {
|
||||
armCountdown(btn, remaining, () => cfg.label(Number(btn.dataset.count)));
|
||||
return;
|
||||
}
|
||||
|
||||
btn.disabled = true;
|
||||
btn.textContent = 'Loading…';
|
||||
|
||||
try {
|
||||
const res = await fetch('/api/run', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ toolType: cfg.toolType, mode: 'cookie', count: 20, targetTweetId: tweetId }),
|
||||
});
|
||||
const json = await res.json();
|
||||
stampCooldown(['cookie']);
|
||||
|
||||
if (!json.ok) {
|
||||
if (res.status === 429 && json.retryAfter) {
|
||||
armCountdown(btn, json.retryAfter * 1000, () => cfg.label(Number(btn.dataset.count)));
|
||||
return;
|
||||
}
|
||||
btn.disabled = false;
|
||||
btn.textContent = 'Failed — retry';
|
||||
return;
|
||||
}
|
||||
|
||||
const items = Array.isArray(json.data) ? json.data : [json.data];
|
||||
thread.innerHTML = items.map(it => buildCard(it, true)).join('');
|
||||
thread.dataset.loaded = '1';
|
||||
if (json.nextCursor) {
|
||||
thread.dataset.cursor = json.nextCursor;
|
||||
thread.insertAdjacentHTML('beforeend',
|
||||
`<button type="button" class="reply-load-more-btn" data-kind="${kind}" data-tweet-id="${esc(tweetId)}">${cfg.loadMoreLabel}</button>`);
|
||||
}
|
||||
thread.classList.remove('hidden');
|
||||
btn.disabled = !items.length;
|
||||
btn.textContent = items.length ? cfg.hideLabel : cfg.noneLabel;
|
||||
|
||||
// Fold the newly-fetched items into this archive's own dataset and
|
||||
// checkpoint-save immediately — this is the "counts as evidence" step.
|
||||
if (items.length) {
|
||||
allItems = allItems.concat(items);
|
||||
saveExpansion(wrap.querySelector('[data-role="save-indicator"]'));
|
||||
}
|
||||
} catch (err) {
|
||||
btn.disabled = false;
|
||||
btn.textContent = 'Failed — retry';
|
||||
}
|
||||
}
|
||||
|
||||
async function loadMoreThread(moreBtn) {
|
||||
const kind = moreBtn.dataset.kind;
|
||||
const cfg = EXPAND_KIND[kind];
|
||||
const thread = moreBtn.closest(cfg.threadSel);
|
||||
const tweetId = moreBtn.dataset.tweetId;
|
||||
const cursor = thread.dataset.cursor;
|
||||
if (!cursor) { moreBtn.remove(); return; }
|
||||
|
||||
const remaining = cooldownUntil.cookie - Date.now();
|
||||
if (remaining > 0) {
|
||||
armCountdown(moreBtn, remaining, () => cfg.loadMoreLabel);
|
||||
return;
|
||||
}
|
||||
|
||||
moreBtn.disabled = true;
|
||||
moreBtn.textContent = 'Loading…';
|
||||
|
||||
try {
|
||||
const res = await fetch('/api/run', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ toolType: cfg.toolType, mode: 'cookie', count: 20, targetTweetId: tweetId, cursor }),
|
||||
});
|
||||
const json = await res.json();
|
||||
stampCooldown(['cookie']);
|
||||
|
||||
if (!json.ok) {
|
||||
if (res.status === 429 && json.retryAfter) {
|
||||
armCountdown(moreBtn, json.retryAfter * 1000, () => cfg.loadMoreLabel);
|
||||
return;
|
||||
}
|
||||
moreBtn.disabled = false;
|
||||
moreBtn.textContent = 'Failed — retry';
|
||||
return;
|
||||
}
|
||||
|
||||
const items = Array.isArray(json.data) ? json.data : [json.data];
|
||||
moreBtn.insertAdjacentHTML('beforebegin', items.map(it => buildCard(it, true)).join(''));
|
||||
if (json.nextCursor) {
|
||||
thread.dataset.cursor = json.nextCursor;
|
||||
moreBtn.disabled = false;
|
||||
moreBtn.textContent = cfg.loadMoreLabel;
|
||||
} else {
|
||||
moreBtn.remove();
|
||||
}
|
||||
|
||||
if (items.length) {
|
||||
allItems = allItems.concat(items);
|
||||
const wrap = moreBtn.closest('.card-thread-wrap');
|
||||
saveExpansion(wrap.querySelector('[data-role="save-indicator"]'));
|
||||
}
|
||||
} catch (err) {
|
||||
moreBtn.disabled = false;
|
||||
moreBtn.textContent = 'Failed — retry';
|
||||
}
|
||||
}
|
||||
|
||||
// Checkpoints allItems (the archive's original contents plus everything
|
||||
// expanded so far) back into the SAME archive_id — archive.py's update()
|
||||
// overwrites results.json/meta.json in place and skips re-downloading any
|
||||
// media already on disk, so this is cheap enough to call after every expand.
|
||||
let savingCheckpoint = false;
|
||||
async function saveExpansion(indicatorEl) {
|
||||
if (!activeId || savingCheckpoint) return;
|
||||
savingCheckpoint = true;
|
||||
if (indicatorEl) { indicatorEl.textContent = 'Saving to archive…'; indicatorEl.className = 'save-indicator'; }
|
||||
|
||||
try {
|
||||
const res = await fetch('/api/archive', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
toolType: activeMeta.tool || 'unknown',
|
||||
data: allItems,
|
||||
queryInfo: activeMeta.query || {},
|
||||
archiveId: activeId,
|
||||
}),
|
||||
});
|
||||
const json = await res.json();
|
||||
if (!json.ok) throw new Error(json.error);
|
||||
if (indicatorEl) { indicatorEl.textContent = '✓ Saved to archive'; indicatorEl.className = 'save-indicator ok'; }
|
||||
|
||||
// Keep the sidebar's item count in sync without a full re-fetch.
|
||||
const entry = allArchives.find(a => a.id === activeId);
|
||||
if (entry) { entry.total_items = allItems.length; renderArchiveList(archiveSearch.value); }
|
||||
viewerSub.textContent = [
|
||||
activeMeta.archived_at ? activeMeta.archived_at.replace('T', ' ') : '',
|
||||
`${allItems.length} items`,
|
||||
activeMeta.media_count ? `${activeMeta.media_count} media files` : '',
|
||||
].filter(Boolean).join(' · ');
|
||||
} catch (e) {
|
||||
if (indicatorEl) { indicatorEl.textContent = 'Save failed: ' + e.message; indicatorEl.className = 'save-indicator err'; }
|
||||
} finally {
|
||||
savingCheckpoint = false;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Delete ────────────────────────────────────────────────────────────────────
|
||||
|
||||
deleteBtn.addEventListener('click', async () => {
|
||||
@@ -790,8 +1333,9 @@ deleteBtn.addEventListener('click', async () => {
|
||||
const json = await res.json();
|
||||
if (!json.ok) { alert('Delete failed: ' + json.error); return; }
|
||||
|
||||
activeId = null;
|
||||
allItems = [];
|
||||
activeId = null;
|
||||
allItems = [];
|
||||
activeMeta = {};
|
||||
cardsBox.innerHTML = '<div class="empty-state">Archive deleted. Select another from the sidebar.</div>';
|
||||
viewerTop.style.display = 'none';
|
||||
loadSidebar();
|
||||
|
||||
@@ -1141,17 +1141,6 @@ function renderCards(data, query = '') {
|
||||
resultBox.innerHTML = '<div class="cards-grid">' + filtered.map(buildCard).join('') + '</div>';
|
||||
}
|
||||
|
||||
// Only a plain https URL (no quotes/angle-brackets/whitespace/parens) is
|
||||
// ever interpolated into the CSS url('...') below — background-image goes
|
||||
// through a second parsing pass CSS-side, so HTML-attribute escaping alone
|
||||
// isn't sufficient there the way it is for a plain <img src>. Rejecting
|
||||
// anything but a clean https URL up front closes that off rather than
|
||||
// trying to escape a value for two contexts (HTML attribute + CSS token)
|
||||
// wedged into one string.
|
||||
function isSafeImageUrl(u) {
|
||||
return typeof u === 'string' && /^https:\/\/[^\s'"<>()]+$/.test(u);
|
||||
}
|
||||
|
||||
// Tapping "Reply" on X auto-prefixes the compose box with every account the
|
||||
// reply-chain already has tagged, and that prefix is genuinely part of the
|
||||
// reply's own raw text — but x.com's own UI never shows it inline, trimming
|
||||
@@ -1175,68 +1164,9 @@ function stripLeadingMentions(text, mentions) {
|
||||
return rest || text;
|
||||
}
|
||||
|
||||
// Returns { html, usedFields } instead of just a string — buildCard() below
|
||||
// needs to know exactly which raw keys actually ended up rendered in the
|
||||
// header so it can drop only THOSE from the generic row list. A static
|
||||
// "always hide these field names" list doesn't work here: CSE/Wayback
|
||||
// records also have a `description` field (Google's own snippet, renamed
|
||||
// from serp_snippet) that this header never touches (no avatar/name/handle
|
||||
// on those records, so it returns empty) — hiding it unconditionally would
|
||||
// have silently deleted the one thing the user asked to see more clearly.
|
||||
function buildCardHeader(item) {
|
||||
const avatarRaw = item.avatar || item.user_avatar || '';
|
||||
const avatar = isSafeImageUrl(avatarRaw) ? avatarRaw : '';
|
||||
const name = item.name || '';
|
||||
const handle = item.screen_name || item.user || item.username || '';
|
||||
|
||||
if (!avatar && !name && !handle) return { html: '', usedFields: [] };
|
||||
|
||||
const usedFields = [];
|
||||
if (avatar) usedFields.push(item.avatar ? 'avatar' : 'user_avatar');
|
||||
if (name) usedFields.push('name');
|
||||
if (handle) usedFields.push(item.screen_name ? 'screen_name' : item.user ? 'user' : 'username');
|
||||
|
||||
const avatarHtml = avatar
|
||||
? `<img class="card-avatar" src="${esc(avatar)}" alt="" loading="lazy" referrerpolicy="no-referrer">`
|
||||
: (name || handle)
|
||||
? `<div class="card-avatar card-avatar-fallback">${esc((name || handle).charAt(0).toUpperCase())}</div>`
|
||||
: '';
|
||||
const identityHtml = (name || handle)
|
||||
? `<div class="card-identity">
|
||||
${name ? `<div class="card-name">${esc(name)}</div>` : ''}
|
||||
${handle ? `<div class="card-handle">@${esc(handle)}</div>` : ''}
|
||||
</div>`
|
||||
: '';
|
||||
|
||||
// A tweet or reply already leads with its own text a few rows down — a
|
||||
// full cover-photo-plus-bio header buries that under the *author's*
|
||||
// profile instead of the actual reply content (this is what made a reply
|
||||
// thread look like it was rendering "just the user"). Those get a small
|
||||
// inline byline only; the full profile-card treatment (banner + bio) is
|
||||
// reserved for records that ARE a user — follower/retweeter results, not
|
||||
// tweets a user happened to write.
|
||||
const isTweetLike = item.text !== undefined || item.full_text !== undefined || item.article_text !== undefined;
|
||||
if (isTweetLike) {
|
||||
return { html: `<div class="card-byline">${avatarHtml}${identityHtml}</div>`, usedFields };
|
||||
}
|
||||
|
||||
const bannerRaw = item.banner || item.user_banner || '';
|
||||
const banner = isSafeImageUrl(bannerRaw) ? bannerRaw : '';
|
||||
const bio = item.description || item.user_bio || '';
|
||||
if (banner) usedFields.push(item.banner ? 'banner' : 'user_banner');
|
||||
if (bio) usedFields.push(item.description ? 'description' : 'user_bio');
|
||||
|
||||
const bioHtml = bio ? `<div class="card-bio">${esc(bio)}</div>` : '';
|
||||
const bannerHtml = banner
|
||||
? `<div class="card-banner" style="background-image:url('${esc(banner)}')"></div>`
|
||||
: '';
|
||||
|
||||
return {
|
||||
html: `<div class="card-header${banner ? ' has-banner' : ''}">${bannerHtml}<div class="card-header-row">${avatarHtml}${identityHtml}</div>${bioHtml}</div>`,
|
||||
usedFields,
|
||||
};
|
||||
}
|
||||
|
||||
// buildCardHeader() / isSafeImageUrl() come from static/js/card_constants.js
|
||||
// — shared with archive.html so avatar/name/handle/banner/bio render
|
||||
// identically whether you're looking at a live result or a saved archive.
|
||||
function buildCard(item, nested = false) {
|
||||
if (typeof item !== 'object' || item === null) {
|
||||
return `<div class="card"><div class="card-row"><div class="card-val">${esc(String(item))}</div></div></div>`;
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Test Render Graph</title>
|
||||
</head>
|
||||
<body>
|
||||
<p>just test</p>
|
||||
</body>
|
||||
</html>
|
||||
Reference in New Issue
Block a user