diff --git a/Script/SOCMINT-Twitter/Readme.md b/Script/SOCMINT-Twitter/Readme.md index dc7b134..0a148ba 100644 --- a/Script/SOCMINT-Twitter/Readme.md +++ b/Script/SOCMINT-Twitter/Readme.md @@ -12,10 +12,12 @@ 6. Add more parameter for enrichment 7. Add no rate limit (throttle) 8. Add Google CSE data source -9. Expand data user profile post, follower and following, reply post, retweet post in graph — -10. Add sentiment analysis for clustering data, pro, neutral, con based on archive data and dump data +9. Expand data user profile post, follower and following, reply post, retweet post in graph +10. Add sentiment analysis for clustering data, pro, neutral, con. Based on archive data and dump data 11. Add more data source and other parameter (soon) still research 12. Add more detail data source for the context +13. Auto repair broken archive and sentiment analysis data dump +14. Update rendering data in sentiment analysis ## Features @@ -233,6 +235,8 @@ starting point for investigation, not a verdict. # Results +[![Watch the video](https://vumbnail.com/1216677900.jpg)](https://vimeo.com/1216677900) + Xquik Dashboard image diff --git a/Script/SOCMINT-Twitter/app.py b/Script/SOCMINT-Twitter/app.py index bf35481..422c166 100644 --- a/Script/SOCMINT-Twitter/app.py +++ b/Script/SOCMINT-Twitter/app.py @@ -1,4 +1,5 @@ import json +import os import re import secrets import shutil @@ -535,11 +536,29 @@ def archive_results(archive_id): meta_file = base / "meta.json" if not results_file.exists(): return jsonify({"ok": False, "error": "Archive not found"}), 404 - return jsonify({ - "ok": True, - "results": json.loads(results_file.read_text()), - "meta": json.loads(meta_file.read_text()) if meta_file.exists() else {}, - }) + + # json.loads() raising here used to fall through to Flask's default + # error handler, which returns an HTML error page — the browser's + # res.json() then fails with an opaque "SyntaxError: JSON.parse: + # unexpected character..." instead of the actual problem. archive.py now + # writes both files atomically (see _atomic_write_json), so a reader + # should never see a torn file mid-checkpoint-update; this is the + # backstop for any other cause (disk fault, a pre-existing archive + # written before that fix, manual editing) — always answer with clean + # JSON either way. + try: + results = json.loads(results_file.read_text()) + except (json.JSONDecodeError, OSError) as e: + return jsonify({"ok": False, "error": f"Archive data is corrupted or unreadable ({e}). Try re-running the search and archiving again."}), 500 + + meta = {} + if meta_file.exists(): + try: + meta = json.loads(meta_file.read_text()) + except (json.JSONDecodeError, OSError): + pass # meta is supplementary — a corrupt/missing meta shouldn't block viewing the results that DID load fine + + return jsonify({"ok": True, "results": results, "meta": meta}) @app.route("/api/archive//media/") @@ -590,23 +609,95 @@ def archive_list(): # ── Analytics (sentiment / clustering) ────────────────────────────────────── +# Same background-thread + polling shape archive.py's downloads already use +# (start() kicks off a thread and returns immediately, status() reports +# progress) — ML sentiment scoring runs locally on CPU at ~11-12ms/item +# (measured), so a large archive (thousands of items) can take a minute-plus. +# A blocking request for that long leaves the browser with nothing to show +# but a static spinner and no way to tell "still working" from "stuck." + +_analytics_registry: dict[str, dict] = {} # archive_id -> job status dict +_analytics_lock = threading.Lock() + + +def _run_analytics(archive_id: str, items: list) -> None: + def on_progress(done, total): + with _analytics_lock: + entry = _analytics_registry.get(archive_id) + if entry is not None: # could've been cleared/overwritten by a re-run + entry.update({"progress": done, "total": total}) + + try: + result = _sentiment.analyze(items, on_progress=on_progress) + with _analytics_lock: + _analytics_registry[archive_id] = { + "status": "done", "progress": result.get("total_scored", 0), + "total": result.get("total_scored", 0), "result": result, "error": None, + } + except Exception as e: # noqa: BLE001 + with _analytics_lock: + _analytics_registry[archive_id] = { + "status": "error", "progress": 0, "total": 0, "result": None, "error": str(e), + } + @app.route("/analytics") def analytics_viewer(): return render_template("analytics.html") -@app.route("/api/analytics/") -def analytics_run(archive_id): +@app.route("/api/analytics//start", methods=["POST"]) +def analytics_start(archive_id): results_file = _archive.ARCHIVE_ROOT / archive_id / "results.json" if not results_file.exists(): return jsonify({"ok": False, "error": "Archive not found"}), 404 - items = json.loads(results_file.read_text()) - return jsonify({"ok": True, **_sentiment.analyze(items)}) + # Same reasoning as archive_results() above — never let a bad file turn + # into an HTML error page here either, or the browser's res.json() call + # fails with an opaque parse error instead of a readable message. + try: + items = json.loads(results_file.read_text()) + except (json.JSONDecodeError, OSError) as e: + return jsonify({"ok": False, "error": f"Archive data is corrupted or unreadable ({e}). Try re-running the search and archiving again."}), 500 + + with _analytics_lock: + _analytics_registry[archive_id] = { + "status": "scoring", "progress": 0, "total": 0, "result": None, "error": None, + } + t = threading.Thread(target=_run_analytics, args=(archive_id, items), daemon=True) + t.start() + return jsonify({"ok": True}) + + +@app.route("/api/analytics//status") +def analytics_status(archive_id): + with _analytics_lock: + entry = _analytics_registry.get(archive_id) + entry = dict(entry) if entry else None + if entry is None: + return jsonify({"ok": False, "error": "No analysis running for this archive — call start first"}), 404 + if entry["status"] == "error": + return jsonify({"ok": False, "error": entry["error"]}), 500 + + resp = {"ok": True, "status": entry["status"], "progress": entry["progress"], "total": entry["total"]} + if entry["status"] == "done": + resp.update(entry["result"]) + return jsonify(resp) if __name__ == "__main__": host = config.get("server", "host", fallback="127.0.0.1") port = config.getint("server", "port", fallback=5000) debug = config.getboolean("server", "debug", fallback=True) + + # Warm up the ML sentiment model in the background so the FIRST + # analytics request doesn't pay its ~15-20s one-time load cost live — + # see sentiment.warm_up_ml()'s docstring. Skipped in the reloader's + # outer "monitor" process (debug mode re-execs a child process to + # actually serve requests, setting WERKZEUG_RUN_MAIN in that child + # only) — otherwise a process that never serves a single request would + # load ~1GB of model weights for nothing, repeating on every autoreload + # during dev. + if not debug or os.environ.get("WERKZEUG_RUN_MAIN") == "true": + threading.Thread(target=_sentiment.warm_up_ml, daemon=True).start() + app.run(host=host, port=port, debug=debug, threaded=True) diff --git a/Script/SOCMINT-Twitter/archive.py b/Script/SOCMINT-Twitter/archive.py index 0a0178f..d324a0b 100644 --- a/Script/SOCMINT-Twitter/archive.py +++ b/Script/SOCMINT-Twitter/archive.py @@ -83,6 +83,30 @@ def _media_ext(url: str, mtype: str) -> str: return "jpg" +def _atomic_write_json(path: Path, data) -> None: + """Write JSON to `path` without ever leaving a reader able to observe a + half-written file. Path.write_text() opens, writes, and closes in place — + a GET landing on archive_results() mid-write (most likely during a + checkpoint update() re-run, which can take a while on a large dataset) + could read a truncated/malformed file and hand the browser invalid JSON, + which is exactly what surfaces client-side as a raw + "SyntaxError: JSON.parse: unexpected character..." instead of a clean + error. Writing to a sibling temp file first and os.replace()-ing it into + place is atomic on both POSIX and Windows: a concurrent reader always + sees either the complete old file or the complete new one, never + something in between.""" + # pid + thread id: os.getpid() alone collides if two checkpoint updates + # for the same archive_id race inside this one (threaded=True) process — + # e.g. a fast double-click on "Update Archive," or the browser retrying a + # save right as the first one is still writing. Each writer then gets + # its own temp file, so the two writes can't corrupt each other; the + # last os.replace() to run simply wins, same as a normal last-write-wins + # race would, but never with a torn/partial file in between. + tmp = path.with_suffix(path.suffix + f".tmp{os.getpid()}_{threading.get_ident()}") + tmp.write_text(json.dumps(data, indent=2, ensure_ascii=False)) + os.replace(tmp, path) + + def _download_file(url: str, dest: Path) -> bool: """Download a single file. Returns True on success.""" try: @@ -169,12 +193,8 @@ def _run(archive_id: str, tool_type: str, data, query_info: dict) -> None: "total_items": len(items), "media_count": total_media, } - meta_path.write_text( - json.dumps(meta, indent=2, ensure_ascii=False) - ) - (archive_dir / "results.json").write_text( - json.dumps(enriched, indent=2, ensure_ascii=False) - ) + _atomic_write_json(meta_path, meta) + _atomic_write_json(archive_dir / "results.json", enriched) with _lock: _registry[archive_id].update({"status": "downloading", "total": len(media_queue)}) diff --git a/Script/SOCMINT-Twitter/sentiment.py b/Script/SOCMINT-Twitter/sentiment.py index 51d6c41..0bec3c8 100644 --- a/Script/SOCMINT-Twitter/sentiment.py +++ b/Script/SOCMINT-Twitter/sentiment.py @@ -38,6 +38,21 @@ from collections import Counter # Indonesian words skew political/social-discourse (matches the kind of # content this tool actually pulls — keyword searches on public affairs, # government programs, public figures) as well as general register. +# +# Lexicon-parity note: an earlier version of this list ran ~130 positive vs. +# ~220 negative entries. That gap isn't neutral — with score_text() summing +# one point per matched word, a lexicon with substantially more negative +# coverage (more synonyms per concept: bohong/kebohongan/hoax/menipu/penipu/ +# penipuan/tipu for one idea, "lying," vs. jujur/kejujuran for its opposite) +# structurally nudges mixed/ambiguous text toward "con" independent of the +# text's actual sentiment, simply because there's more negative surface area +# to match against. The additions below restore rough parity for the same +# governance/social-discourse register the negative list already covers +# (accountability, honesty, inclusion, rule of law) rather than padding with +# unrelated filler — pair each new word against the negative concept it +# offsets in a review. This is still a heuristic, not a bias-free scorer: +# see analyze()'s docstring and the module docstring above for the standing +# caveat that neither backend is ground truth. # Arr data words # Need to feedback and research to sett the all parameter for each words @@ -65,6 +80,18 @@ POSITIVE_WORDS = { "efektif", "transparan", "transparansi", "akuntabel", "akuntabilitas", "merakyat", "membela rakyat", "pro rakyat", "berpihak pada rakyat", "terpuji", "membanggakan", "gemilang", "cemerlang", "berkah", "istimewa", + # Governance/social-discourse counterparts added for lexicon parity + # (offsetting korupsi/nepotisme/kkn, otoriter/diktator/fasis/represif, + # rasis/intoleran, bohong/hoax/menipu, and pelanggaran/ilegal below). + "bersih", "antikorupsi", "berintegritas", "integritas", "kredibel", + "kredibilitas", "terpercaya", "dapat dipercaya", "netral", "imparsial", + "objektif", "demokratis", "reformasi", "reformis", "inklusif", + "inklusi", "toleran", "toleransi", "egaliter", "partisipatif", + "aspiratif", "taat hukum", "patuh hukum", "sesuai aturan", "legal", + "sah", "melindungi", "perlindungan", "membangun", "pembangunan", + "sinergi", "berkolaborasi", "kolaboratif", "harmonis", "kondusif", + "stabil", "stabilitas", "humanis", "empati", "berempati", "rendah hati", + "dermawan", } NEGATIVE_WORDS = { @@ -102,10 +129,10 @@ NEGATIVE_WORDS = { "kurang ajar", "tidak becus", "amburadul", "berantakan", "semrawut", "menyengsarakan", "represif", "represi", "diskriminasi", "mendiskriminasi", "rasis", "rasisme", "intoleran", "intoleransi", - "penjilat", "gila", "kontol", "memek", "paok", "stress", "goblog", + "penjilat", "kontol", "memek", "paok", "stress", "goblog", "kontlo", "kepala batu", "oon", "bacot", "asu", "gijil", "jembut", "kanjut", "ngentot", "puki", "meki", "jembot", "pukimak", "kimak", "tembelek", - "tai", + "tai", "bacod", "telaso", } # Flips the polarity of a sentiment word found within NEGATION_WINDOW tokens @@ -192,10 +219,24 @@ def score_text(text: str) -> dict: def _item_text(item: dict) -> str: """The text worth scoring/tokenizing for a given archived record — varies by which tool produced it (a tweet's own text vs. a Wayback/CSE - page's scraped title+description vs. a bare user's bio).""" + page's scraped title+description). + + `description` is deliberately NOT pulled from a bare user record (a + follower/following/retweeter entry — cookie_client.py's _user_to_dict + always sets `followers_count`, even to None, which no tweet/CSE/Wayback + record ever carries, so that key's mere presence identifies the shape + reliably). For a CSE result, `description` is Google's own snippet of + the matched page — genuinely relevant text. For a user record it's the + account's own bio, which says nothing about the search topic; scoring + "suka kucing dan kopi ☕" as pro/con toward whatever was searched would + just be noise. Those accounts are still kept for clustering/leaderboard + purposes (top_users() below runs over every item regardless of text) — + they're just excluded from sentiment/word-cloud scoring specifically.""" + is_bare_user_record = "followers_count" in item parts = [ item.get("text"), item.get("full_text"), item.get("article_text"), - item.get("post_title"), item.get("post_text"), item.get("description"), + item.get("post_title"), item.get("post_text"), + None if is_bare_user_record else item.get("description"), ] return " ".join(p for p in parts if p) @@ -293,31 +334,81 @@ def _get_ml_pipeline(): return _ml_pipeline -def _score_texts_ml(texts: list[str]) -> list[dict] | None: - """Batch-scores every text in one call (far faster on CPU than one - pipeline call per item). Returns None if the model isn't available, so - the caller falls back to the lexicon scorer instead. `score` is signed - (positive for pro, negative for con, 0 for neutral) to match the - lexicon backend's convention; `confidence` carries the model's own - unsigned probability for the label it picked.""" +def warm_up_ml() -> None: + """Loads the ML pipeline right now instead of waiting for the first real + analytics request to trigger it lazily. Measured at ~15-20s the first + time any process calls _get_ml_pipeline() (importing transformers, + constructing the pipeline, reading the cached weights off disk) — vs. + ~11-12ms/item for actual scoring once loaded. Without this, that whole + one-time cost lands inside the FIRST user's analytics job, during which + the progress bar has nothing to report yet (on_progress only fires once + scoring itself starts) and just sits at 0/0 looking stuck. Meant to be + called from a background thread at server startup (see app.py) — still + completely safe to skip calling this at all, or to have the first real + request race it, since _get_ml_pipeline() is lock-protected and + idempotent either way; this is purely a warm-up, not a dependency.""" + _get_ml_pipeline() + + +_ML_PROGRESS_CHUNK = 64 # texts per pipeline call — see _score_texts_ml docstring + + +def _score_texts_ml(texts: list[str], on_progress=None) -> list[dict] | None: + """Scores every text, chunked (rather than one giant pipeline call), so + a caller running this in a background thread can report real progress — + on a CPU this measures ~11-12ms/item (~1000 items ≈ 12s, ~5000 ≈ ~1min), + linear with volume, so a large archive genuinely takes a while and a + caller polling for status needs something better to show than a blind + spinner. _ML_PROGRESS_CHUNK=64 batches (each itself pipelined + batch_size=16 internally by HF) keeps ticks frequent enough to feel + live (~0.7-0.8s apart) without paying per-call overhead for every + single item. Returns None if the model isn't available, so the caller + falls back to the lexicon scorer instead. `score` is signed (positive + for pro, negative for con, 0 for neutral) to match the lexicon + backend's convention; `confidence` carries the model's own unsigned + probability for the label it picked.""" clf = _get_ml_pipeline() if clf is None: return None - raw = clf(texts, truncation=True, batch_size=16) + results = [] - for r in raw: - label = _ML_LABEL_MAP.get(str(r.get("label", "")).lower(), "neutral") - confidence = float(r.get("score", 0.0)) - signed = confidence if label == "pro" else -confidence if label == "con" else 0.0 - results.append({ - "label": label, "score": round(signed, 3), - "confidence": round(confidence, 3), "matches": [], - }) + for start in range(0, len(texts), _ML_PROGRESS_CHUNK): + chunk = texts[start:start + _ML_PROGRESS_CHUNK] + # truncation=True alone is NOT enough here: it truncates to the + # tokenizer's own model_max_length, which for this tokenizer's + # shipped config is left at HF's "unset" sentinel (~1e30, i.e. + # effectively no limit) rather than the model's real 512-token + # capacity. A single long post (a fact-check thread, an + # article-length tweet — anything past ~512 tokens once + # subword-tokenized) then sails through "truncation" untruncated, + # overflows the model's position-embedding table, and crashes the + # whole batch with a raw RuntimeError ("index 514 is out of bounds + # for dimension 1 with size 514" — 514 = 512 + the 2-position + # offset RoBERTa-style embeddings use). max_length=512 forces the + # real limit regardless of what the tokenizer config claims. + raw = clf(chunk, truncation=True, max_length=512, batch_size=16) + for r in raw: + label = _ML_LABEL_MAP.get(str(r.get("label", "")).lower(), "neutral") + confidence = float(r.get("score", 0.0)) + signed = confidence if label == "pro" else -confidence if label == "con" else 0.0 + results.append({ + "label": label, "score": round(signed, 3), + "confidence": round(confidence, 3), "matches": [], + }) + if on_progress: + on_progress(len(results), len(texts)) return results -def analyze(items: list[dict]) -> dict: - """Full analytics payload for one archive's worth of raw items.""" +def analyze(items: list[dict], on_progress=None) -> dict: + """Full analytics payload for one archive's worth of raw items. + on_progress, if given, is called as on_progress(scored_count, + total_to_score) — zero or more times during ML scoring (chunked, see + _score_texts_ml), and always at least once at the very end regardless + of which backend actually ran, so a caller polling for status always + sees a final 100%-done tick even on the lexicon path (fast enough that + per-chunk progress wouldn't mean anything, but a job registry watching + for "did this reach total" still needs that terminal call).""" if not isinstance(items, list): items = [items] # archive.py's own _run() passes non-dict entries through as-is rather @@ -334,10 +425,12 @@ def analyze(items: list[dict]) -> dict: text_items.append(item) texts.append(text) - ml_results = _score_texts_ml(texts) if texts else None + ml_results = _score_texts_ml(texts, on_progress=on_progress) if texts else None method = "ml" if ml_results is not None else "lexicon" if ml_results is None: ml_results = [score_text(t) for t in texts] + if on_progress: + on_progress(len(texts), len(texts)) sentiment_counts = {"pro": 0, "neutral": 0, "con": 0} scored_items = [] diff --git a/Script/SOCMINT-Twitter/static/js/card_constants.js b/Script/SOCMINT-Twitter/static/js/card_constants.js index e464744..2ed925a 100644 --- a/Script/SOCMINT-Twitter/static/js/card_constants.js +++ b/Script/SOCMINT-Twitter/static/js/card_constants.js @@ -45,3 +45,85 @@ const DRILLABLE = { const SOURCE_CLASS = { 'Twitter Cookie': 'src-cookie', 'Xquik API': 'src-xquik', 'Wayback Machine': 'src-wayback', 'Google CSE': 'src-cse' }; const AGE_LABELS = { new: 'New account', recent: 'Recent account', established: 'Established account' }; + +// ── Shared profile header (avatar/name/handle/banner/bio) ────────────────── +// Used by buildCard() in both index.html (live results) and archive.html +// (saved archives) so a record's author identity renders byte-identically +// whether you're looking at it live or after it's been archived — previously +// archive.html had no equivalent at all, so a saved card silently dropped +// the avatar/name/handle/bio/banner that the live card showed for the exact +// same record. `esc()` is expected to already be defined as a global by the +// time this actually runs (each page defines its own, loaded in a later +// diff --git a/Script/SOCMINT-Twitter/templates/archive.html b/Script/SOCMINT-Twitter/templates/archive.html index 9e8a459..61ec287 100644 --- a/Script/SOCMINT-Twitter/templates/archive.html +++ b/Script/SOCMINT-Twitter/templates/archive.html @@ -288,6 +288,12 @@ .cards-area { padding: 16px 18px; overflow-y: auto; flex: 1; } .cards-grid { display: grid; gap: 8px; } + .load-more-note { + font-size: 11px; + color: var(--muted); + text-align: center; + padding: 14px 0 2px; + } /* ── Cards ── */ .card { @@ -298,6 +304,124 @@ transition: border-color 0.12s; } .card:hover { border-color: #3a3d50; } + + /* Profile header — cover photo, avatar, name/handle, bio. Same rules as + the live search page: only rendered when the item actually carries any + of that (a tweet's embedded author, a bare user/follower/retweeter + record); everything else keeps the plain row list it always had. */ + .card-header { margin-bottom: 10px; } + .card-header.has-banner { margin: -12px -14px 10px; } + .card-banner { + height: 64px; + background-size: cover; + background-position: center; + background-color: var(--bg); + } + .card-header-row { display: flex; align-items: flex-end; gap: 10px; } + .card-header.has-banner .card-header-row { padding: 0 14px; margin-top: -26px; } + .card-avatar { + width: 48px; + height: 48px; + border-radius: 50%; + object-fit: cover; + border: 2px solid var(--surface); + background: var(--bg); + flex-shrink: 0; + } + .card-avatar-fallback { + display: flex; + align-items: center; + justify-content: center; + font-size: 17px; + font-weight: 700; + color: var(--muted); + } + .card-identity { min-width: 0; padding-bottom: 3px; } + .card-name { + font-size: 14px; + font-weight: 600; + color: var(--text); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + .card-handle { font-size: 12px; color: var(--muted); } + .card-bio { margin-top: 8px; font-size: 12px; color: var(--text); opacity: 0.85; line-height: 1.45; } + .card-header.has-banner .card-bio { padding: 0 14px; } + + /* Compact byline — tweets/replies get just this instead of the full + banner+bio header above, which is reserved for records that ARE a user + rather than a tweet someone wrote. */ + .card-byline { display: flex; align-items: center; gap: 8px; margin-bottom: 8px; } + .card-byline .card-avatar { width: 26px; height: 26px; border-width: 1px; } + .card-byline .card-avatar-fallback { font-size: 11px; } + .card-byline .card-identity { display: flex; align-items: baseline; gap: 6px; padding-bottom: 0; min-width: 0; } + .card-byline .card-name { font-size: 13px; max-width: 55%; } + .card-byline .card-handle { flex-shrink: 0; } + + /* Retweeted-content rows — subtle highlight, same as the live search page */ + .card-row.rt-origin .card-key { color: var(--accent); } + .card-row.rt-origin .card-val { color: #c7d2fe; } + + .reply-context { font-size: 11px; color: var(--muted); margin-bottom: 4px; } + + /* Inline expand-and-save — replies/retweets fetched here get folded into + this archive's own item list and checkpointed back to disk immediately, + same as the live pages but persisted right away since there's no + separate "Archive" button on this read view. */ + .reply-expand-btn, .retweet-expand-btn { + display: inline-flex; + align-items: center; + gap: 5px; + margin-top: 10px; + margin-right: 6px; + background: none; + border: 1px solid var(--border); + color: var(--accent); + font-family: var(--font); + font-size: 11px; + font-weight: 500; + padding: 4px 10px; + border-radius: 5px; + cursor: pointer; + } + .retweet-expand-btn { color: #f59e0b; } + .reply-expand-btn:hover:not(:disabled), .retweet-expand-btn:hover:not(:disabled) { border-color: currentColor; } + .reply-expand-btn:disabled, .retweet-expand-btn:disabled { opacity: 0.5; cursor: wait; } + + .reply-thread, .retweet-thread { + margin: 10px 0 2px 16px; + padding-left: 14px; + border-left: 2px solid var(--border); + display: flex; + flex-direction: column; + gap: 8px; + } + .reply-thread.hidden, .retweet-thread.hidden { display: none; } + + .reply-load-more-btn { + align-self: flex-start; + background: none; + border: none; + color: var(--accent); + font-family: var(--font); + font-size: 11px; + cursor: pointer; + text-decoration: underline; + text-underline-offset: 2px; + padding: 2px 0; + } + .reply-load-more-btn:disabled { opacity: 0.5; cursor: wait; text-decoration: none; } + + .save-indicator { + display: inline-block; + margin-top: 10px; + font-size: 11px; + color: var(--muted); + } + .save-indicator.ok { color: var(--success); } + .save-indicator.err { color: var(--danger); } + .card-row { display: flex; gap: 10px; @@ -516,6 +640,8 @@
Select an archive from the sidebar to view its contents.
+
+ @@ -560,8 +686,20 @@ let allItems = []; let activeId = null; +let activeMeta = {}; // meta.json for the open archive — tool/query, needed to checkpoint-save expansions back to it let allArchives = []; // full list from the server; archiveSearch filters this client-side +// ── Reply/retweet expand-and-save cooldown ────────────────────────────────── +// Mirrors index.html's throttle handling — every expand here is a cookie-mode +// call, sharing the backend's single 5s-per-source cookie clock with every +// other tool, so a fast double-click here still respects it. +const THROTTLE_SECONDS = 5; +let cooldownUntil = { cookie: 0 }; + +function stampCooldown(sources) { + (sources || []).forEach(source => { cooldownUntil[source] = Date.now() + THROTTLE_SECONDS * 1000; }); +} + const archiveList = document.getElementById('archiveList'); const archiveCount = document.getElementById('archiveCount'); const archiveSearch = document.getElementById('archiveSearch'); @@ -573,18 +711,53 @@ const resultCount = document.getElementById('resultCount'); const deleteBtn = document.getElementById('deleteBtn'); const cardsBox = document.getElementById('cardsBox'); +// ── Resilient fetch — auto-retry transient failures ───────────────────────── +// `res.json()` throws a raw SyntaxError ("unexpected character at line 1 +// column 1...") whenever the response body isn't valid JSON — a dropped/ +// truncated connection (flaky wifi, the device sleeping mid-request) or a +// server-side hiccup can both do this, and previously nothing here caught +// it: the promise just rejected, the "Loading…" spinner never went away, +// and the failure showed up nowhere the user could see WHY. This retries a +// few times with a short backoff (self-heals a one-off network blip +// automatically — no user action needed) before surfacing a real, +// human-readable error. A well-formed {ok:false,...} response is NOT +// retried here — that's the server correctly telling you something (e.g. +// "Archive not found"), retrying wouldn't change the answer; only fetch/ +// parse failures (the connection or the body itself was the problem) do. +async function fetchJsonRetry(url, opts, attempts = 3) { + let lastErr; + for (let i = 0; i < attempts; i++) { + try { + const res = await fetch(url, opts); + return await res.json(); + } catch (e) { + lastErr = e; + if (i < attempts - 1) await new Promise(r => setTimeout(r, 500 * (i + 1))); + } + } + throw lastErr; +} + // ── Sidebar ─────────────────────────────────────────────────────────────────── // The server already returns archives newest-first (by archived_at/updated_at, // not folder name — folder names are prefixed by tool_type so sorting by name // doesn't actually sort chronologically once more than one tool's been used). async function loadSidebar() { - const res = await fetch('/api/archive/list'); - const json = await res.json(); + let json; + try { + json = await fetchJsonRetry('/api/archive/list'); + } catch (e) { + archiveList.innerHTML = `
Couldn't load archives — ${esc(e.message || String(e))}
`; + document.getElementById('retrySidebarBtn').addEventListener('click', loadSidebar); + return; + } if (!json.ok || !json.archives.length) { allArchives = []; - archiveList.innerHTML = '
No archives yet.
Enable Auto Archive in the tool and run a search.
'; + archiveList.innerHTML = json.ok + ? '
No archives yet.
Enable Auto Archive in the tool and run a search.
' + : `
${esc(json.error)}
`; archiveCount.textContent = '0'; return; } @@ -654,23 +827,29 @@ async function loadArchive(id) { viewerTop.style.display = 'none'; searchInput.value = ''; - const res = await fetch(`/api/archive/${id}/results`); - const json = await res.json(); + let json; + try { + json = await fetchJsonRetry(`/api/archive/${id}/results`); + } catch (e) { + cardsBox.innerHTML = `
Couldn't load this archive — ${esc(e.message || String(e))}
`; + document.getElementById('retryArchiveBtn').addEventListener('click', () => loadArchive(id)); + return; + } if (!json.ok) { cardsBox.innerHTML = `
${esc(json.error)}
`; return; } - allItems = json.results || []; - const meta = json.meta || {}; + allItems = json.results || []; + activeMeta = json.meta || {}; - const tool = (meta.tool || 'unknown').replace(/_/g, ' '); - const query = queryLabel(meta.query || {}); + const tool = (activeMeta.tool || 'unknown').replace(/_/g, ' '); + const query = queryLabel(activeMeta.query || {}); viewerTitle.textContent = tool + (query ? ` — "${query}"` : ''); viewerSub.textContent = [ - meta.archived_at ? meta.archived_at.replace('T', ' ') : '', + activeMeta.archived_at ? activeMeta.archived_at.replace('T', ' ') : '', `${allItems.length} items`, - meta.media_count ? `${meta.media_count} media files` : '', + activeMeta.media_count ? `${activeMeta.media_count} media files` : '', ].filter(Boolean).join(' · '); viewerTop.style.display = ''; @@ -678,8 +857,14 @@ async function loadArchive(id) { } // ── Search ──────────────────────────────────────────────────────────────────── - -searchInput.addEventListener('input', () => renderCards(allItems, searchInput.value)); +// Debounced — same reasoning as analytics.html's item search: re-filtering +// and re-rendering the full card list (avatars, media, everything) on every +// single keystroke gets janky once an archive has hundreds/thousands of items. +let cardSearchDebounce = null; +searchInput.addEventListener('input', () => { + clearTimeout(cardSearchDebounce); + cardSearchDebounce = setTimeout(() => renderCards(allItems, searchInput.value), 180); +}); function flatText(obj) { if (!obj || typeof obj !== 'object') return String(obj ?? ''); @@ -688,10 +873,49 @@ function flatText(obj) { // ── Cards ───────────────────────────────────────────────────────────────────── -// PRIORITY / DRILLABLE / SOURCE_CLASS / AGE_LABELS come from -// static/js/card_constants.js, loaded above — shared with index.html. -const SKIP = ['archived_media']; -const CLAMP = new Set(['text','full_text','article_text','description','post_text']); +// PRIORITY / DRILLABLE / SOURCE_CLASS / AGE_LABELS / buildCardHeader come +// from static/js/card_constants.js, loaded above — shared with index.html, +// so a record's avatar/name/handle/banner/bio render identically whether +// you're looking at a live result or a saved archive. +const SKIP = ['archived_media', 'profile_image_url', 'profile_banner_url', 'entities', 'extended_entities', 'urls', 'media', 'indices']; +const CLAMP = new Set(['text','full_text','content','description','bio','article_text','retweeted_text','retweeted_by_bio','post_text']); + +// ── Auto-scroll — same mechanism index.html's live search results use +// (IntersectionObserver watching a sentinel, root: null since — per this +// page's own back-to-top comment above — the PAGE is what actually scrolls +// here, .cards-area's overflow:auto never gets a bounded height to actually +// clip against). Building the card HTML itself is cheap (measured well +// under 20ms for 200+ cards) — what actually made a big first batch feel +// slow is every card's avatar firing its own request to Twitter's CDN +// at once; 50 of those landing simultaneously is what dragged out the +// initial paint, not the rendering logic. 10 keeps that burst small; the +// tighter rootMargin below (vs. what analytics.html's plain-text item list +// uses) matters just as much — a bigger margin pre-triggers the NEXT +// image-heavy batch before you've actually scrolled toward it. +const CARD_BATCH = 10; +let filteredCardItems = []; +let cardRenderCount = 0; + +function updateCardsLoadStatus() { + const status = document.getElementById('cardsLoadStatus'); + const remaining = filteredCardItems.length - cardRenderCount; + if (remaining > 0) { + status.style.display = ''; + status.textContent = `Showing ${cardRenderCount} of ${filteredCardItems.length} — scroll for more…`; + } else { + status.style.display = 'none'; + } +} + +function loadMoreCards() { + if (cardRenderCount >= filteredCardItems.length) return; + const prev = cardRenderCount; + cardRenderCount = Math.min(cardRenderCount + CARD_BATCH, filteredCardItems.length); + const grid = cardsBox.querySelector('.cards-grid'); + if (!grid) return; + grid.insertAdjacentHTML('beforeend', filteredCardItems.slice(prev, cardRenderCount).map(it => buildCard(it)).join('')); + updateCardsLoadStatus(); +} function renderCards(items, query) { const q = query.toLowerCase().trim(); @@ -699,24 +923,60 @@ function renderCards(items, query) { resultCount.textContent = filtered.length + (q ? ' found' : ' results'); + filteredCardItems = filtered; + cardRenderCount = Math.min(CARD_BATCH, filtered.length); + if (!filtered.length) { cardsBox.innerHTML = '
No matching results.
'; + updateCardsLoadStatus(); return; } - cardsBox.innerHTML = '
' + filtered.map(buildCard).join('') + '
'; + cardsBox.innerHTML = '
' + filtered.slice(0, cardRenderCount).map(it => buildCard(it)).join('') + '
'; + updateCardsLoadStatus(); } -function buildCard(item) { +// #cardsSentinel and .cards-area are static page elements (only #cardsBox's +// innerHTML gets replaced on every render), so — unlike analytics.html's +// dashboard, which gets rebuilt from scratch per archive — this observer is +// bound exactly once, right here, and just keeps working across archive +// switches and searches. +const cardsObserver = new IntersectionObserver((entries) => { + if (entries.some(e => e.isIntersecting)) loadMoreCards(); +}, { root: null, rootMargin: '100px' }); +cardsObserver.observe(document.getElementById('cardsSentinel')); + +// Same "Replying to @x @y" trim index.html's cards apply — reply_to_mentions +// (from cookie_client.py) lists exactly which leading @mentions x.com itself +// hides from the rendered tweet body. Display-only: item.text in the +// archived JSON is untouched. +function stripLeadingMentions(text, mentions) { + if (!text || !mentions || !mentions.length) return text; + let rest = text; + for (const m of mentions) { + const re = new RegExp('^\\s*@' + String(m).replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '\\b'); + const match = rest.match(re); + if (!match) break; + rest = rest.slice(match[0].length); + } + rest = rest.replace(/^\s+/, ''); + return rest || text; +} + +// nested=true for a reply/retweet rendered inside another card's expanded +// thread — same recursive shape index.html's buildCard() uses, so a reply +// that itself has replies still gets its own Expand button. +function buildCard(item, nested = false) { if (typeof item !== 'object' || !item) return ''; const tweetId = item.id ? String(item.id) : null; const entries = Object.entries(item); + const { html: header, usedFields: headerFields } = buildCardHeader(item); // Same rule as the live search page: order follows PRIORITY's own sequence, // and nothing gets capped off — a saved archive should show exactly the // same fields, in the same order, as when it was first fetched. const priKeys = new Set(entries.map(([k]) => k).filter(k => PRIORITY.includes(k))); - const pri = PRIORITY.filter(k => priKeys.has(k)).map(k => [k, item[k]]); - const rest = entries.filter(([k]) => !PRIORITY.includes(k) && !SKIP.includes(k)); + const pri = PRIORITY.filter(k => priKeys.has(k) && !headerFields.includes(k)).map(k => [k, item[k]]); + const rest = entries.filter(([k]) => !PRIORITY.includes(k) && !SKIP.includes(k) && !headerFields.includes(k)); const rows = [...pri, ...rest].map(([k, v]) => { let display; @@ -737,6 +997,10 @@ function buildCard(item) { display = `${esc(AGE_LABELS[v] || v)}`; } else if (k === 'content_type') { display = `${esc(CONTENT_TYPE_LABELS[v] || String(v))}`; + } else if (k === 'text' && item.reply_to_mentions && item.reply_to_mentions.length) { + const badge = `
↩ Replying to ${item.reply_to_mentions.map(m => '@' + esc(m)).join(', ')}
`; + const clean = stripLeadingMentions(String(v), item.reply_to_mentions); + display = `${badge}${esc(clean)}`; } else if (typeof v === 'object') { const s = JSON.stringify(v); display = `${esc(s.length > 100 ? s.slice(0,100)+'…' : s)}`; @@ -744,42 +1008,321 @@ function buildCard(item) { const cls = CLAMP.has(k) ? ' clamp' : ''; display = `${esc(String(v))}`; } - return `
${esc(k)}
${display}
`; + const rtCls = k.startsWith('retweeted_') ? ' rt-origin' : ''; + return `
${esc(k)}
${display}
`; }).join(''); const media = buildMedia(item); - return `
${rows}${media}
`; + + // Expand-in-place — same trigger rule as index.html (id + a positive + // count), but the fetched result here also gets folded into this + // archive's saved results.json as soon as it lands (see saveExpansion()), + // since this view has no separate "Archive" button to click afterward. + const replyCount = Number(item.reply_count); + const canReplies = tweetId && Number.isFinite(replyCount) && replyCount > 0; + const rtCount = Number(item.retweet_count); + const canRetweets = tweetId && Number.isFinite(rtCount) && rtCount > 0; + + const replyBtn = canReplies + ? `` + : ''; + const retweetBtn = canRetweets + ? `` + : ''; + + const cardCls = nested ? 'card reply-card' : 'card'; + const card = `
${header}${rows}${media}${replyBtn}${retweetBtn}
`; + if (!canReplies && !canRetweets) return card; + + const threads = [ + canReplies ? `` : '', + canRetweets ? `` : '', + ].join(''); + return `
${card}${threads}
`; +} + +// Cookie mode: item.media = [{type, thumb, url}]. API mode: raw Twitter +// shape under extended_entities.media / entities.media. Same normalization +// index.html's extractMedia() applies — needed here for items fetched +// through an inline expand, which haven't been through archive.py's +// download pipeline yet (no archived_media of their own until the +// checkpoint save that follows lands). +function extractMedia(item) { + if (Array.isArray(item.media) && item.media.length) { + const first = item.media[0]; + if (first && typeof first === 'object' && ('thumb' in first || 'url' in first)) { + return item.media; + } + } + const src = (item.extended_entities && item.extended_entities.media) + || (item.entities && item.entities.media); + if (!Array.isArray(src)) return []; + return src.map(m => { + const mtype = m.type || 'photo'; + const thumb = m.media_url_https || m.media_url || ''; + let url = thumb; + if (mtype === 'video' || mtype === 'animated_gif') { + const variants = (m.video_info && m.video_info.variants) || []; + const mp4s = variants.filter(v => v.content_type === 'video/mp4'); + if (mp4s.length) url = mp4s.reduce((b, v) => (v.bitrate||0) > (b.bitrate||0) ? v : b).url; + } + return { type: mtype, thumb, url }; + }).filter(m => m.thumb); } function buildMedia(item) { + // Already-archived item: serve from local disk via the media route. const paths = item.archived_media; - if (!Array.isArray(paths) || !paths.length) return ''; + if (Array.isArray(paths) && paths.length) { + const archiveId = activeId; + const html = paths.map(p => { + // p = "media/filename.ext" + const filename = p.replace(/^media\//, ''); + const src = `/api/archive/${encodeURIComponent(archiveId)}/media/${encodeURIComponent(filename)}`; + const isVideo = filename.endsWith('.mp4'); + const isGif = filename.includes('animated_gif'); - const archiveId = activeId; - const html = paths.map(p => { - // p = "media/filename.ext" - const filename = p.replace(/^media\//, ''); - const src = `/api/archive/${encodeURIComponent(archiveId)}/media/${encodeURIComponent(filename)}`; - const isVideo = filename.endsWith('.mp4'); - const isGif = filename.includes('animated_gif'); + if (isVideo || isGif) { + const loop = isGif ? 'loop muted' : ''; + const label = isGif ? 'GIF' : ''; + return `
+ ${label} +
`; + } + return ` + media + `; + }).join(''); + return `
${html}
`; + } - if (isVideo || isGif) { - const loop = isGif ? 'loop muted' : ''; - const label = isGif ? 'GIF' : ''; + // Freshly expanded item, not yet checkpointed to disk — render straight + // from the source (video routed through the same CDN proxy the live + // search page uses), same as index.html's renderMedia(). + const mediaList = extractMedia(item); + if (!mediaList.length) return ''; + const html = mediaList.map(m => { + if (m.type === 'video' || m.type === 'animated_gif') { + const proxied = `/api/video?url=${encodeURIComponent(m.url)}`; + const loop = m.type === 'animated_gif' ? 'loop muted' : ''; + const poster = m.thumb ? `poster="${esc(m.thumb)}"` : ''; + const label = m.type === 'animated_gif' ? 'GIF' : ''; return `
-
`; } - return ` - media + return ` + media `; }).join(''); - return `
${html}
`; } +// ── Inline expand-and-save (replies / retweets) ───────────────────────────── +// Same interaction as index.html's reply threads and graph.html's node +// expand, adapted to this read-only view: there's no separate "Archive" +// button here, so a successful expand immediately checkpoints the fetched +// items back into THIS SAME archive's results.json via archive.py's +// update() (which skips re-downloading any media already on disk) — that's +// what makes an expanded comment/reply/retweet count as saved digital +// evidence rather than something that only ever existed in the browser tab. + +const EXPAND_KIND = { + reply: { + toolType: 'tweet_replies_extractor', threadSel: '.reply-thread', btnSel: '.reply-expand-btn', + label: n => `↩ Expand ${n} repl${n === 1 ? 'y' : 'ies'}`, hideLabel: '▲ Hide replies', + loadMoreLabel: 'Load more replies…', noneLabel: 'No replies found', + }, + retweet: { + toolType: 'tweet_retweeters_extractor', threadSel: '.retweet-thread', btnSel: '.retweet-expand-btn', + label: n => `↗ Expand ${n} retweet${n === 1 ? '' : 's'}`, hideLabel: '▲ Hide retweets', + loadMoreLabel: 'Load more retweets…', noneLabel: 'No retweeters found', + }, +}; + +cardsBox.addEventListener('click', (e) => { + const replyBtn = e.target.closest('.reply-expand-btn'); + if (replyBtn) { toggleThread(replyBtn, 'reply'); return; } + const rtBtn = e.target.closest('.retweet-expand-btn'); + if (rtBtn) { toggleThread(rtBtn, 'retweet'); return; } + const moreBtn = e.target.closest('.reply-load-more-btn'); + if (moreBtn) { loadMoreThread(moreBtn); } +}); + +function armCountdown(btn, msRemaining, resetLabel) { + const secs = Math.max(1, Math.ceil(msRemaining / 1000)); + btn.disabled = true; + btn.textContent = `Wait ${secs}s…`; + setTimeout(() => { btn.disabled = false; btn.textContent = resetLabel(); }, msRemaining); +} + +async function toggleThread(btn, kind) { + const cfg = EXPAND_KIND[kind]; + const wrap = btn.closest('.card-thread-wrap'); + const thread = wrap.querySelector(`:scope > ${cfg.threadSel}`); + + // Already fetched once — just show/hide, no new request (mirrors + // index.html's toggleReplyThread()). + if (thread.dataset.loaded === '1') { + const nowHidden = thread.classList.toggle('hidden'); + btn.textContent = nowHidden ? cfg.label(Number(btn.dataset.count)) : cfg.hideLabel; + return; + } + + const tweetId = btn.dataset.tweetId; + const remaining = cooldownUntil.cookie - Date.now(); + if (remaining > 0) { + armCountdown(btn, remaining, () => cfg.label(Number(btn.dataset.count))); + return; + } + + btn.disabled = true; + btn.textContent = 'Loading…'; + + try { + const res = await fetch('/api/run', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ toolType: cfg.toolType, mode: 'cookie', count: 20, targetTweetId: tweetId }), + }); + const json = await res.json(); + stampCooldown(['cookie']); + + if (!json.ok) { + if (res.status === 429 && json.retryAfter) { + armCountdown(btn, json.retryAfter * 1000, () => cfg.label(Number(btn.dataset.count))); + return; + } + btn.disabled = false; + btn.textContent = 'Failed — retry'; + return; + } + + const items = Array.isArray(json.data) ? json.data : [json.data]; + thread.innerHTML = items.map(it => buildCard(it, true)).join(''); + thread.dataset.loaded = '1'; + if (json.nextCursor) { + thread.dataset.cursor = json.nextCursor; + thread.insertAdjacentHTML('beforeend', + ``); + } + thread.classList.remove('hidden'); + btn.disabled = !items.length; + btn.textContent = items.length ? cfg.hideLabel : cfg.noneLabel; + + // Fold the newly-fetched items into this archive's own dataset and + // checkpoint-save immediately — this is the "counts as evidence" step. + if (items.length) { + allItems = allItems.concat(items); + saveExpansion(wrap.querySelector('[data-role="save-indicator"]')); + } + } catch (err) { + btn.disabled = false; + btn.textContent = 'Failed — retry'; + } +} + +async function loadMoreThread(moreBtn) { + const kind = moreBtn.dataset.kind; + const cfg = EXPAND_KIND[kind]; + const thread = moreBtn.closest(cfg.threadSel); + const tweetId = moreBtn.dataset.tweetId; + const cursor = thread.dataset.cursor; + if (!cursor) { moreBtn.remove(); return; } + + const remaining = cooldownUntil.cookie - Date.now(); + if (remaining > 0) { + armCountdown(moreBtn, remaining, () => cfg.loadMoreLabel); + return; + } + + moreBtn.disabled = true; + moreBtn.textContent = 'Loading…'; + + try { + const res = await fetch('/api/run', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ toolType: cfg.toolType, mode: 'cookie', count: 20, targetTweetId: tweetId, cursor }), + }); + const json = await res.json(); + stampCooldown(['cookie']); + + if (!json.ok) { + if (res.status === 429 && json.retryAfter) { + armCountdown(moreBtn, json.retryAfter * 1000, () => cfg.loadMoreLabel); + return; + } + moreBtn.disabled = false; + moreBtn.textContent = 'Failed — retry'; + return; + } + + const items = Array.isArray(json.data) ? json.data : [json.data]; + moreBtn.insertAdjacentHTML('beforebegin', items.map(it => buildCard(it, true)).join('')); + if (json.nextCursor) { + thread.dataset.cursor = json.nextCursor; + moreBtn.disabled = false; + moreBtn.textContent = cfg.loadMoreLabel; + } else { + moreBtn.remove(); + } + + if (items.length) { + allItems = allItems.concat(items); + const wrap = moreBtn.closest('.card-thread-wrap'); + saveExpansion(wrap.querySelector('[data-role="save-indicator"]')); + } + } catch (err) { + moreBtn.disabled = false; + moreBtn.textContent = 'Failed — retry'; + } +} + +// Checkpoints allItems (the archive's original contents plus everything +// expanded so far) back into the SAME archive_id — archive.py's update() +// overwrites results.json/meta.json in place and skips re-downloading any +// media already on disk, so this is cheap enough to call after every expand. +let savingCheckpoint = false; +async function saveExpansion(indicatorEl) { + if (!activeId || savingCheckpoint) return; + savingCheckpoint = true; + if (indicatorEl) { indicatorEl.textContent = 'Saving to archive…'; indicatorEl.className = 'save-indicator'; } + + try { + const res = await fetch('/api/archive', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + toolType: activeMeta.tool || 'unknown', + data: allItems, + queryInfo: activeMeta.query || {}, + archiveId: activeId, + }), + }); + const json = await res.json(); + if (!json.ok) throw new Error(json.error); + if (indicatorEl) { indicatorEl.textContent = '✓ Saved to archive'; indicatorEl.className = 'save-indicator ok'; } + + // Keep the sidebar's item count in sync without a full re-fetch. + const entry = allArchives.find(a => a.id === activeId); + if (entry) { entry.total_items = allItems.length; renderArchiveList(archiveSearch.value); } + viewerSub.textContent = [ + activeMeta.archived_at ? activeMeta.archived_at.replace('T', ' ') : '', + `${allItems.length} items`, + activeMeta.media_count ? `${activeMeta.media_count} media files` : '', + ].filter(Boolean).join(' · '); + } catch (e) { + if (indicatorEl) { indicatorEl.textContent = 'Save failed: ' + e.message; indicatorEl.className = 'save-indicator err'; } + } finally { + savingCheckpoint = false; + } +} + // ── Delete ──────────────────────────────────────────────────────────────────── deleteBtn.addEventListener('click', async () => { @@ -790,8 +1333,9 @@ deleteBtn.addEventListener('click', async () => { const json = await res.json(); if (!json.ok) { alert('Delete failed: ' + json.error); return; } - activeId = null; - allItems = []; + activeId = null; + allItems = []; + activeMeta = {}; cardsBox.innerHTML = '
Archive deleted. Select another from the sidebar.
'; viewerTop.style.display = 'none'; loadSidebar(); diff --git a/Script/SOCMINT-Twitter/templates/index.html b/Script/SOCMINT-Twitter/templates/index.html index cf4f09e..25d18b8 100644 --- a/Script/SOCMINT-Twitter/templates/index.html +++ b/Script/SOCMINT-Twitter/templates/index.html @@ -1141,17 +1141,6 @@ function renderCards(data, query = '') { resultBox.innerHTML = '
' + filtered.map(buildCard).join('') + '
'; } -// Only a plain https URL (no quotes/angle-brackets/whitespace/parens) is -// ever interpolated into the CSS url('...') below — background-image goes -// through a second parsing pass CSS-side, so HTML-attribute escaping alone -// isn't sufficient there the way it is for a plain . Rejecting -// anything but a clean https URL up front closes that off rather than -// trying to escape a value for two contexts (HTML attribute + CSS token) -// wedged into one string. -function isSafeImageUrl(u) { - return typeof u === 'string' && /^https:\/\/[^\s'"<>()]+$/.test(u); -} - // Tapping "Reply" on X auto-prefixes the compose box with every account the // reply-chain already has tagged, and that prefix is genuinely part of the // reply's own raw text — but x.com's own UI never shows it inline, trimming @@ -1175,68 +1164,9 @@ function stripLeadingMentions(text, mentions) { return rest || text; } -// Returns { html, usedFields } instead of just a string — buildCard() below -// needs to know exactly which raw keys actually ended up rendered in the -// header so it can drop only THOSE from the generic row list. A static -// "always hide these field names" list doesn't work here: CSE/Wayback -// records also have a `description` field (Google's own snippet, renamed -// from serp_snippet) that this header never touches (no avatar/name/handle -// on those records, so it returns empty) — hiding it unconditionally would -// have silently deleted the one thing the user asked to see more clearly. -function buildCardHeader(item) { - const avatarRaw = item.avatar || item.user_avatar || ''; - const avatar = isSafeImageUrl(avatarRaw) ? avatarRaw : ''; - const name = item.name || ''; - const handle = item.screen_name || item.user || item.username || ''; - - if (!avatar && !name && !handle) return { html: '', usedFields: [] }; - - const usedFields = []; - if (avatar) usedFields.push(item.avatar ? 'avatar' : 'user_avatar'); - if (name) usedFields.push('name'); - if (handle) usedFields.push(item.screen_name ? 'screen_name' : item.user ? 'user' : 'username'); - - const avatarHtml = avatar - ? `` - : (name || handle) - ? `
${esc((name || handle).charAt(0).toUpperCase())}
` - : ''; - const identityHtml = (name || handle) - ? `
- ${name ? `
${esc(name)}
` : ''} - ${handle ? `
@${esc(handle)}
` : ''} -
` - : ''; - - // A tweet or reply already leads with its own text a few rows down — a - // full cover-photo-plus-bio header buries that under the *author's* - // profile instead of the actual reply content (this is what made a reply - // thread look like it was rendering "just the user"). Those get a small - // inline byline only; the full profile-card treatment (banner + bio) is - // reserved for records that ARE a user — follower/retweeter results, not - // tweets a user happened to write. - const isTweetLike = item.text !== undefined || item.full_text !== undefined || item.article_text !== undefined; - if (isTweetLike) { - return { html: `
${avatarHtml}${identityHtml}
`, usedFields }; - } - - const bannerRaw = item.banner || item.user_banner || ''; - const banner = isSafeImageUrl(bannerRaw) ? bannerRaw : ''; - const bio = item.description || item.user_bio || ''; - if (banner) usedFields.push(item.banner ? 'banner' : 'user_banner'); - if (bio) usedFields.push(item.description ? 'description' : 'user_bio'); - - const bioHtml = bio ? `
${esc(bio)}
` : ''; - const bannerHtml = banner - ? `
` - : ''; - - return { - html: `
${bannerHtml}
${avatarHtml}${identityHtml}
${bioHtml}
`, - usedFields, - }; -} - +// buildCardHeader() / isSafeImageUrl() come from static/js/card_constants.js +// — shared with archive.html so avatar/name/handle/banner/bio render +// identically whether you're looking at a live result or a saved archive. function buildCard(item, nested = false) { if (typeof item !== 'object' || item === null) { return `
${esc(String(item))}
`; diff --git a/Script/SOCMINT-Twitter/templates/test.html b/Script/SOCMINT-Twitter/templates/test.html deleted file mode 100644 index 2058def..0000000 --- a/Script/SOCMINT-Twitter/templates/test.html +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - Test Render Graph - - -

just test

- - \ No newline at end of file