From ee5658fc0acb0c6c4770a78702f622edf7be5d05 Mon Sep 17 00:00:00 2001 From: Jieyab89 Date: Tue, 11 Aug 2026 22:23:46 +0700 Subject: [PATCH] enhancement new featrues and fix bug --- Script/SOCMINT-Twitter/Readme.md | 11 + Script/SOCMINT-Twitter/app.py | 188 ++++++++- .../SOCMINT-Twitter/templates/analytics.html | 27 +- Script/SOCMINT-Twitter/templates/archive.html | 2 + Script/SOCMINT-Twitter/templates/cases.html | 260 +++++++++++++ Script/SOCMINT-Twitter/templates/graph.html | 293 +++++++++++++- Script/SOCMINT-Twitter/templates/index.html | 358 +++++++++++++++++- 7 files changed, 1101 insertions(+), 38 deletions(-) create mode 100644 Script/SOCMINT-Twitter/templates/cases.html diff --git a/Script/SOCMINT-Twitter/Readme.md b/Script/SOCMINT-Twitter/Readme.md index 4993407..79209c8 100644 --- a/Script/SOCMINT-Twitter/Readme.md +++ b/Script/SOCMINT-Twitter/Readme.md @@ -21,6 +21,10 @@ 15. Update view as graph in archive 16. Add date and timestamp pattern in sentiment analysis 17. Delete entry node in graph visualizer +18. Add new features save project (load case or load new case) +19. Checkpoint or save data +20. Auto resume archive and dump data with state (state data to checkpoint) +21. Add date or timestamp paramater for all search module ## Features @@ -289,6 +293,13 @@ Dasboard Home Image +Load Case / Project +image + +Checkpoint or Save the Project + +image + Archive Image diff --git a/Script/SOCMINT-Twitter/app.py b/Script/SOCMINT-Twitter/app.py index 422c166..07de6a4 100644 --- a/Script/SOCMINT-Twitter/app.py +++ b/Script/SOCMINT-Twitter/app.py @@ -7,6 +7,7 @@ import threading import time from concurrent.futures import ThreadPoolExecutor from datetime import datetime +from pathlib import Path import requests as _req from flask import Flask, g, jsonify, render_template, request, Response, stream_with_context, send_from_directory @@ -416,12 +417,22 @@ def run_tool(): source_errors = None # multi_source_search only — {source: error} for lanes that failed this page try: + # Common date range — validated once; all tools except wayback (which uses its + # own waybackFrom/waybackTo keys) and article_extractor (single item, no date) + # can receive these to narrow results. + from_date = body.get("dateFrom", "").strip() + to_date = body.get("dateTo", "").strip() + for d_label, d_val in (("dateFrom", from_date), ("dateTo", to_date)): + if d_val and not _valid_date8(d_val): + return jsonify({"ok": False, "error": f"{d_label} must be YYYYMMDD"}), 400 + if tool_type == "tweet_search_extractor": - query = body.get("searchQuery", "") + query = _apply_date_operators(body.get("searchQuery", ""), from_date, to_date) if mode == "cookie": data, next_cursor = cookie_tweet_search(query, count=count, config=config, cursor=cursor) else: data = XquikClient(config).tweet_search(query) + data = _filter_by_date(data, from_date, to_date) elif tool_type == "follower_explorer": username = body.get("targetUsername", "") @@ -429,12 +440,14 @@ def run_tool(): data, next_cursor = cookie_follower_explorer(username, count=count, config=config, cursor=cursor) else: data = XquikClient(config).follower_explorer(username) + data = _filter_by_date(data, from_date, to_date) elif tool_type == "following_explorer": username = body.get("targetUsername", "") if mode != "cookie": return jsonify({"ok": False, "error": "following_explorer requires cookie mode"}), 400 data, next_cursor = cookie_following_explorer(username, count=count, config=config, cursor=cursor) + data = _filter_by_date(data, from_date, to_date) elif tool_type == "article_extractor": tweet_id = body.get("targetTweetId", "") @@ -449,6 +462,7 @@ def run_tool(): data, next_cursor = cookie_community_post_extractor(community_id, count=count, config=config, cursor=cursor) else: data = XquikClient(config).community_post_extractor(community_id) + data = _filter_by_date(data, from_date, to_date) elif tool_type == "post_extractor": username = body.get("targetUsername", "") @@ -456,41 +470,40 @@ def run_tool(): data, next_cursor = cookie_post_extractor(username, count=count, config=config, cursor=cursor) else: data = XquikClient(config).post_extractor(username) + data = _filter_by_date(data, from_date, to_date) elif tool_type == "tweet_replies_extractor": tweet_id = body.get("targetTweetId", "") if mode != "cookie": return jsonify({"ok": False, "error": "tweet_replies_extractor requires cookie mode"}), 400 data, next_cursor = cookie_tweet_replies(tweet_id, count=count, config=config, cursor=cursor) + data = _filter_by_date(data, from_date, to_date) elif tool_type == "tweet_retweeters_extractor": tweet_id = body.get("targetTweetId", "") if mode != "cookie": return jsonify({"ok": False, "error": "tweet_retweeters_extractor requires cookie mode"}), 400 data, next_cursor = cookie_tweet_retweeters(tweet_id, count=count, config=config, cursor=cursor) + data = _filter_by_date(data, from_date, to_date) elif tool_type == "geo_post_extractor": - keyword = body.get("searchQuery", "") + keyword = _apply_date_operators(body.get("searchQuery", ""), from_date, to_date) if mode != "cookie": return jsonify({"ok": False, "error": "geo_post_extractor requires cookie mode"}), 400 data, next_cursor = cookie_geo_search(keyword, count=count, config=config, cursor=cursor) + data = _filter_by_date(data, from_date, to_date) elif tool_type == "wayback_archive_search": - target = body.get("searchQuery", "") - from_date = body.get("waybackFrom", "") - to_date = body.get("waybackTo", "") - for label, val in (("waybackFrom", from_date), ("waybackTo", to_date)): - if val and not _valid_date8(val): - return jsonify({"ok": False, "error": f"{label} must be an 8-digit date (YYYYMMDD)"}), 400 - data, next_cursor = wayback_search(target, count=count, from_date=from_date, to_date=to_date, cursor=cursor) + target = body.get("searchQuery", "") + wb_from = body.get("waybackFrom", "") + wb_to = body.get("waybackTo", "") + for d_label, d_val in (("waybackFrom", wb_from), ("waybackTo", wb_to)): + if d_val and not _valid_date8(d_val): + return jsonify({"ok": False, "error": f"{d_label} must be an 8-digit date (YYYYMMDD)"}), 400 + data, next_cursor = wayback_search(target, count=count, from_date=wb_from, to_date=wb_to, cursor=cursor) elif tool_type == "multi_source_search": - query = body.get("searchQuery", "") - from_date = body.get("dateFrom", "") - to_date = body.get("dateTo", "") - for label, val in (("dateFrom", from_date), ("dateTo", to_date)): - if val and not _valid_date8(val): - return jsonify({"ok": False, "error": f"{label} must be an 8-digit date (YYYYMMDD)"}), 400 + query = body.get("searchQuery", "") data, next_cursor, source_errors = _multi_source_search(query, count=count, from_date=from_date, to_date=to_date, cursor=cursor) else: @@ -608,6 +621,151 @@ def archive_list(): return jsonify({"ok": True, "archives": _archive.list_all()}) +# ── Cases (save / resume investigation sessions) ────────────────────────────── + +CASES_ROOT = Path(__file__).parent / "cases" +_CASE_ID_RE = re.compile(r"^case_\d{8}_\d{6}_[0-9a-f]{6}$") + + +def _valid_case_id(cid: str) -> bool: + return bool(_CASE_ID_RE.match(cid)) + + +@app.route("/api/cases") +def cases_list(): + CASES_ROOT.mkdir(exist_ok=True) + out = [] + for d in CASES_ROOT.iterdir(): + if not d.is_dir(): + continue + mf = d / "meta.json" + if not mf.exists(): + continue + try: + out.append(json.loads(mf.read_text())) + except Exception: + continue + out.sort(key=lambda c: c.get("updated_at", ""), reverse=True) + return jsonify({"ok": True, "cases": out}) + + +@app.route("/api/cases", methods=["POST"]) +def cases_create(): + body = request.get_json(silent=True) or {} + name = str(body.get("name", "Unnamed Case")).strip()[:120] + page = str(body.get("page", "index"))[:16] + state = body.get("state") + if state is None: + return jsonify({"ok": False, "error": "No state provided"}), 400 + + now = datetime.now() + case_id = f"case_{now.strftime('%Y%m%d_%H%M%S')}_{secrets.token_hex(3)}" + case_dir = CASES_ROOT / case_id + case_dir.mkdir(parents=True, exist_ok=True) + + tool = str(body.get("tool", ""))[:80].strip() + ts = now.strftime("%Y-%m-%d %H:%M:%S") + meta = {"id": case_id, "name": name, "page": page, "tool": tool, "created_at": ts, "updated_at": ts} + (case_dir / "meta.json").write_text(json.dumps(meta, indent=2)) + (case_dir / "state.json").write_text(json.dumps(state)) + return jsonify({"ok": True, "caseId": case_id, "meta": meta}) + + +@app.route("/api/cases/") +def cases_get(case_id): + if not _valid_case_id(case_id): + return jsonify({"ok": False, "error": "Invalid case ID"}), 400 + mf = CASES_ROOT / case_id / "meta.json" + sf = CASES_ROOT / case_id / "state.json" + if not mf.exists(): + return jsonify({"ok": False, "error": "Case not found"}), 404 + try: + meta = json.loads(mf.read_text()) + state = json.loads(sf.read_text()) + except Exception as e: + return jsonify({"ok": False, "error": str(e)}), 500 + return jsonify({"ok": True, "meta": meta, "state": state}) + + +@app.route("/api/cases/", methods=["PUT"]) +def cases_update(case_id): + if not _valid_case_id(case_id): + return jsonify({"ok": False, "error": "Invalid case ID"}), 400 + mf = CASES_ROOT / case_id / "meta.json" + if not mf.exists(): + return jsonify({"ok": False, "error": "Case not found"}), 404 + body = request.get_json(silent=True) or {} + try: + meta = json.loads(mf.read_text()) + except Exception as e: + return jsonify({"ok": False, "error": str(e)}), 500 + if "name" in body: + meta["name"] = str(body["name"]).strip()[:120] + if "tool" in body: + meta["tool"] = str(body["tool"]).strip()[:80] + meta["updated_at"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + if "state" in body: + (CASES_ROOT / case_id / "state.json").write_text(json.dumps(body["state"])) + mf.write_text(json.dumps(meta, indent=2)) + return jsonify({"ok": True, "meta": meta}) + + +@app.route("/api/cases/", methods=["DELETE"]) +def cases_delete(case_id): + if not _valid_case_id(case_id): + return jsonify({"ok": False, "error": "Invalid case ID"}), 400 + case_dir = CASES_ROOT / case_id + if not case_dir.exists(): + return jsonify({"ok": False, "error": "Case not found"}), 404 + shutil.rmtree(case_dir) + return jsonify({"ok": True}) + + +@app.route("/api/cases/beacon", methods=["POST"]) +def cases_beacon(): + """navigator.sendBeacon() target — called on beforeunload. Always 204; response is ignored.""" + try: + body = request.get_json(silent=True) or {} + state = body.get("state") + if not state: + return "", 204 + tool = str(body.get("tool", ""))[:80].strip() + case_id = str(body.get("caseId", "")).strip() + CASES_ROOT.mkdir(exist_ok=True) + if case_id and _valid_case_id(case_id): + mf = CASES_ROOT / case_id / "meta.json" + sf = CASES_ROOT / case_id / "state.json" + if mf.exists(): + try: + meta = json.loads(mf.read_text()) + except Exception: + meta = {} + if tool: + meta["tool"] = tool + meta["updated_at"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + mf.write_text(json.dumps(meta, indent=2)) + sf.write_text(json.dumps(state)) + else: + name = str(body.get("name", ""))[:120].strip() or "Auto-save" + page = str(body.get("page", "index"))[:16] + now = datetime.now() + cid = f"case_{now.strftime('%Y%m%d_%H%M%S')}_{secrets.token_hex(3)}" + cdir = CASES_ROOT / cid + cdir.mkdir(parents=True, exist_ok=True) + ts = now.strftime("%Y-%m-%d %H:%M:%S") + meta = {"id": cid, "name": name, "page": page, "tool": tool, "created_at": ts, "updated_at": ts} + (cdir / "meta.json").write_text(json.dumps(meta, indent=2)) + (cdir / "state.json").write_text(json.dumps(state)) + except Exception: + pass + return "", 204 + + +@app.route("/cases") +def cases_page(): + return render_template("cases.html") + + # ── Analytics (sentiment / clustering) ────────────────────────────────────── # Same background-thread + polling shape archive.py's downloads already use # (start() kicks off a thread and returns immediately, status() reports diff --git a/Script/SOCMINT-Twitter/templates/analytics.html b/Script/SOCMINT-Twitter/templates/analytics.html index 6db7803..27eea89 100644 --- a/Script/SOCMINT-Twitter/templates/analytics.html +++ b/Script/SOCMINT-Twitter/templates/analytics.html @@ -256,6 +256,15 @@ background: var(--accent-bg); border: 1px solid var(--accent); border-radius: 20px; padding: 3px 10px; cursor: pointer; margin-left: 8px; } + .cal-date-stat { + margin-top: 8px; + padding: 7px 12px; + background: var(--bg); + border: 1px solid var(--border); + border-radius: 6px; + font-size: 12px; + color: var(--text); + } /* ── Sentiment badge (shared by tiles/list) ── */ .sent-badge { @@ -389,6 +398,7 @@ Archives Analytics + Cases @@ -725,7 +735,7 @@ function renderDashboard() {
${d.method === 'ml' ? 'Sentiment is scored by a multilingual model (XLM-RoBERTa, fine-tuned on tweets across 8 languages and reasonably capable well beyond those) — works on non-Indonesian text, not just the lexicon fallback below. Still a heuristic, not ground truth: short text, sarcasm and irony all still degrade accuracy. Click a tile below to see which items landed in it and the model\'s confidence per item.' - : 'Sentiment is scored against an Indonesian positive/negative word lexicon with negation handling (e.g. "tidak bagus" flips to con) — a transparent rule-based heuristic, Indonesian-only. Install torch+transformers (see requirements.txt) for multilingual ML-based scoring instead. Click a tile below to see exactly which items landed in it and which words drove each score.'} + : 'Sentiment is scored con an Indonesian positive/negative word lexicon with negation handling (e.g. "tidak bagus" flips to con) — a transparent rule-based heuristic, Indonesian-only. Install torch+transformers (see requirements.txt) for multilingual ML-based scoring instead. Click a tile below to see exactly which items landed in it and which words drove each score.'} ${d.total_items - d.total_scored > 0 ? `${d.total_items - d.total_scored} of ${d.total_items} record(s) had no text to score (bare follower/following/user entries) and are excluded below.` : ''}
@@ -1135,6 +1145,18 @@ function calendarHtml(daily) { return `
${cells}
`; }).join(''); + const selDay = activeDateFilter ? days.get(activeDateFilter) : null; + const selStat = selDay + ? (() => { + const d = selDay; + const parts = []; + if (d.pro > 0) parts.push(`${d.pro} positive`); + if (d.neutral > 0) parts.push(`${d.neutral} neutral`); + if (d.con > 0) parts.push(`${d.con} con`); + return `${d.total} item${d.total !== 1 ? 's' : ''} collected${parts.length ? ' — ' + parts.join(' / ') : ''}`; + })() + : null; + return `
@@ -1150,8 +1172,9 @@ function calendarHtml(daily) { More ${undated ? `· ${undated} item(s) with no usable date not shown` : ''} - ${activeDateFilter ? `Filtered: ${esc(activeDateFilter)} ×` : ''} + ${activeDateFilter ? `${esc(activeDateFilter)} ×` : ''}
+ ${selStat ? `
${esc(activeDateFilter)}: ${esc(selStat)}
` : ''}
`; } diff --git a/Script/SOCMINT-Twitter/templates/archive.html b/Script/SOCMINT-Twitter/templates/archive.html index 990f9e3..6ebb9b7 100644 --- a/Script/SOCMINT-Twitter/templates/archive.html +++ b/Script/SOCMINT-Twitter/templates/archive.html @@ -578,6 +578,8 @@ Archives Analytics + + Cases diff --git a/Script/SOCMINT-Twitter/templates/cases.html b/Script/SOCMINT-Twitter/templates/cases.html new file mode 100644 index 0000000..348e792 --- /dev/null +++ b/Script/SOCMINT-Twitter/templates/cases.html @@ -0,0 +1,260 @@ + + + + + +Jieyab89 SOCMINT X — Cases + + + + +
+

Jieyab89 SOCMINT X

+ | + Cases + + +
+ +
+
+

Saved Cases

+
+ +
+
+ +
+ + + + + + + + + + + + +
NameTypeLast SavedActions
Loading…
+
+
+
+ + + + diff --git a/Script/SOCMINT-Twitter/templates/graph.html b/Script/SOCMINT-Twitter/templates/graph.html index 036c7e3..2907a6e 100644 --- a/Script/SOCMINT-Twitter/templates/graph.html +++ b/Script/SOCMINT-Twitter/templates/graph.html @@ -89,7 +89,6 @@ .nav-menu a:hover { color: var(--text); background: var(--accent-bg); } .nav-menu a.current { color: var(--accent); background: var(--accent-bg); } .nav-divider { border: none; border-top: 1px solid var(--border); margin: 3px 0; } - /* ── Toolbar ── */ #toolbar { display: flex; @@ -559,6 +558,53 @@ z-index: 5; } #emptyHint.hidden { display: none; } + + /* Active case indicator in toolbar */ + .case-indicator { + display: none; align-items: center; gap: 6px; + padding: 3px 8px; border-radius: 5px; + background: var(--accent-bg); border: 1px solid var(--accent); font-size: 11px; + } + .case-indicator.visible { display: flex; } + .case-indicator-name { + max-width: 120px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; + color: var(--accent); font-weight: 500; color: #fff; + } + .case-checkpoint-btn { + padding: 4px 10px; font-size: 12px; border-radius: 5px; + background: transparent; border: 1px solid var(--accent); color: var(--accent); + font-family: var(--font); cursor: pointer; transition: all 0.12s; flex-shrink: 0; color: #fff; + } + .case-checkpoint-btn:hover:not(:disabled) { background: var(--accent-bg); } + .case-checkpoint-btn:disabled { opacity: 0.5; cursor: wait; } + .case-save-bar { + background: var(--surface); border-bottom: 1px solid var(--border); + padding: 10px 16px; display: flex; flex-direction: column; gap: 8px; + } + .case-save-bar[hidden] { display: none; } + .case-save-label { font-size: 12px; color: var(--muted); font-weight: 500; } + .case-save-input { + width: 100%; padding: 7px 10px; box-sizing: border-box; + background: var(--bg); border: 1px solid var(--border); border-radius: 6px; + color: var(--text); font-family: var(--font); font-size: 13px; + transition: border-color 0.15s; + } + .case-save-input:focus { outline: none; border-color: var(--accent); } + .case-save-input::placeholder { color: var(--muted); } + .case-save-btns { display: flex; gap: 6px; } + .case-save-confirm { + padding: 6px 16px; font-size: 12px; border-radius: 5px; white-space: nowrap; + background: var(--accent); color: #fff; border: none; + font-family: var(--font); font-weight: 500; cursor: pointer; transition: opacity 0.15s; + } + .case-save-confirm:disabled { opacity: 0.45; cursor: wait; } + .case-save-confirm:not(:disabled):hover { opacity: 0.85; } + .case-save-cancel { + padding: 6px 10px; font-size: 12px; border-radius: 5px; white-space: nowrap; + background: transparent; color: var(--muted); border: 1px solid var(--border); + font-family: var(--font); cursor: pointer; transition: all 0.12s; + } + .case-save-cancel:hover { color: var(--text); border-color: var(--muted); } @@ -574,6 +620,8 @@ Archives Analytics + + Cases @@ -602,6 +650,10 @@ 0 nodes + + + + @@ -609,6 +661,14 @@ +
@@ -690,6 +750,7 @@ {% include "_field_glossary.html" %}
+ @@ -734,6 +781,8 @@ Archives Analytics + + Cases @@ -791,6 +840,17 @@
+ + +
+ +
@@ -823,6 +883,7 @@ +