mirror of
https://github.com/Jieyab89/OSINT-Cheat-sheet.git
synced 2026-08-17 18:35:41 +02:00
convert to python flask and jinja from blade and php also fix json dump and text intel
This commit is contained in:
Jieyab89
@@ -2,21 +2,45 @@
|
||||
|
||||
<img width="2553" height="1218" alt="Image" src="https://github.com/user-attachments/assets/eb01c1ca-c577-432d-a266-5533b5b71ce0" />
|
||||
|
||||
# Sett up
|
||||
|
||||
1. Config your Google api console here, enable and manage API Custom Search by Google
|
||||
|
||||
<img width="2536" height="1210" alt="enable" src="https://github.com/user-attachments/assets/17aca5db-9869-40f0-8a9b-58eae51dce6c" />
|
||||
|
||||
2. Sett the api key in web console Google
|
||||
|
||||
<img width="891" height="1354" alt="g-api" src="https://github.com/user-attachments/assets/1df5201e-9d8d-4450-9c46-cc83cb35eaba" />
|
||||
|
||||
Check the result in the table
|
||||
|
||||
<img width="2121" height="1218" alt="g - api result" src="https://github.com/user-attachments/assets/1763dc2f-2382-4c94-8973-90f98678d477" />
|
||||
|
||||
3. Settings CSE Google to put the cx key
|
||||
|
||||
<img width="2533" height="1254" alt="cx key" src="https://github.com/user-attachments/assets/b8d04387-2ac9-4302-8b04-2ea1873e610a" />
|
||||
|
||||
4. Add site want to crawll e.g twitter.com and x.com
|
||||
|
||||
<img width="868" height="886" alt="add host and domain twitter" src="https://github.com/user-attachments/assets/caecb9d5-85c7-4fdb-8e38-e4acfc55630e" />
|
||||
|
||||
## Data Source
|
||||
|
||||
1. Xquik API
|
||||
2. Cookie (your account cookie session)
|
||||
1. Xquik API (subs there is a price)
|
||||
2. Cookie (your account cookie session)
|
||||
3. Wayback Machine (Cdx API)
|
||||
4. Goole CSE API (free quota 100 per day u can increase u limit with buy the service)
|
||||
|
||||
## Update Note
|
||||
|
||||
1. Update infinity scroll and load new data
|
||||
1. Update infinity scroll and load new data for twitter reply and retweets
|
||||
2. Update data corelation
|
||||
3. Fix business logic flow
|
||||
4. Monitoring (Soon)
|
||||
5. MCP (Soon)
|
||||
6. Add more parameter for enrichment
|
||||
7. Add no rate limit (throttle)
|
||||
8. Add Google CSE data source
|
||||
|
||||
## Setup
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ from flask import Flask, g, jsonify, render_template, request, Response, stream_
|
||||
import archive as _archive
|
||||
from xquik_client import XquikClient, XquikError, load_config
|
||||
from wayback_client import wayback_search, WaybackError
|
||||
from google_cse_client import google_cse_search, GoogleCSEError
|
||||
from id_forensics import enrich_account_age
|
||||
from cookie_client import (
|
||||
cookie_tweet_search,
|
||||
@@ -142,6 +143,7 @@ SOURCE_LABELS = {
|
||||
"cookie": "Twitter Cookie",
|
||||
"xquik": "Xquik API",
|
||||
"wayback": "Wayback Machine",
|
||||
"cse": "Google CSE",
|
||||
}
|
||||
|
||||
|
||||
@@ -151,6 +153,39 @@ def _tag_source(items, label):
|
||||
return [{**it, "source": label} if isinstance(it, dict) else it for it in items]
|
||||
|
||||
|
||||
def _stamp_fetched_at(data):
|
||||
"""Mutates every dict in `data` (list or single dict) in place, adding
|
||||
fetched_at — when *this app* pulled the record, as opposed to created_at
|
||||
(a tweet's own post time) or iso_date (a Wayback snapshot's capture time).
|
||||
Applied uniformly across every source so results are comparable no
|
||||
matter which tool/source produced them."""
|
||||
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
items = data if isinstance(data, list) else [data]
|
||||
for item in items:
|
||||
if isinstance(item, dict):
|
||||
item["fetched_at"] = now
|
||||
return data
|
||||
|
||||
|
||||
def _stamp_tweet_url(data):
|
||||
"""Mutates every dict in `data` (list or single dict) in place, adding
|
||||
tweet_url wherever id+user identify an actual tweet — the citable-link
|
||||
field Wayback (archive_url/original) and Google CSE (result_url) already
|
||||
carry directly in their own data. Cookie/xquik never set this on the raw
|
||||
tweet dict themselves — previously it only got built at archive time, so a
|
||||
live card, a JSON dump, and an archive of the same search could each show
|
||||
a different answer for "what's the URL." Building it once here means all
|
||||
three read the identical value. archive.py's build_tweet_url() reuses this
|
||||
same logic rather than recomputing it separately."""
|
||||
items = data if isinstance(data, list) else [data]
|
||||
for item in items:
|
||||
if isinstance(item, dict) and not item.get("tweet_url"):
|
||||
url = _archive.build_tweet_url(item)
|
||||
if url:
|
||||
item["tweet_url"] = url
|
||||
return data
|
||||
|
||||
|
||||
# ── Optional date-range narrowing ───────────────────────────────────────────
|
||||
# Dates come in from the client as free-text fields, so every value is run
|
||||
# through this strict YYYYMMDD check before it touches a search query string
|
||||
@@ -240,11 +275,15 @@ def _multi_source_search(query: str, count: int, from_date: str = "", to_date: s
|
||||
"cookie": lambda: cookie_tweet_search(twitter_query, count=count, config=config)[0],
|
||||
"xquik": lambda: XquikClient(config).tweet_search(twitter_query),
|
||||
"wayback": lambda: wayback_search(query, count=count, from_date=from_date, to_date=to_date)[0],
|
||||
# Google has no since:/until: query syntax like Twitter/Wayback do, so
|
||||
# this lane runs unbounded by date — _filter_by_date below keeps
|
||||
# results whose own timestamp it can't verify rather than dropping them.
|
||||
"cse": lambda: google_cse_search(query, count=count, config=config)[0],
|
||||
}
|
||||
with ThreadPoolExecutor(max_workers=len(jobs)) as pool:
|
||||
futures = {key: pool.submit(fn) for key, fn in jobs.items()}
|
||||
results = []
|
||||
for key in ("cookie", "xquik", "wayback"): # deterministic display order
|
||||
for key in ("cookie", "xquik", "wayback", "cse"): # deterministic display order
|
||||
try:
|
||||
data = futures[key].result()
|
||||
except Exception:
|
||||
@@ -267,7 +306,7 @@ def video_proxy():
|
||||
url,
|
||||
stream=True,
|
||||
timeout=20,
|
||||
headers={"Referer": "https://x.com/", "User-Agent": "Mozilla/5.0"},
|
||||
headers={"Referer": "https://x.com/", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.10240"},
|
||||
)
|
||||
headers = {"Content-Type": upstream.headers.get("Content-Type", "video/mp4")}
|
||||
if "Content-Length" in upstream.headers:
|
||||
@@ -390,14 +429,16 @@ def run_tool():
|
||||
return jsonify({"ok": False, "error": f"Unknown toolType: {tool_type}"}), 400
|
||||
|
||||
# Single choke point: every tool's output passes through here, so the
|
||||
# account-age label shows up everywhere downstream for free — cards,
|
||||
# graph nodes, JSON dump, and archives (once the fields are whitelisted
|
||||
# in archive.py's _pick_fields).
|
||||
# account-age label, fetch timestamp, and tweet_url all show up
|
||||
# everywhere downstream for free — cards, graph nodes, JSON dump, and
|
||||
# archives (once the fields are whitelisted in archive.py's _pick_fields).
|
||||
data = enrich_account_age(data)
|
||||
data = _stamp_fetched_at(data)
|
||||
data = _stamp_tweet_url(data)
|
||||
|
||||
return jsonify({"ok": True, "data": data, "nextCursor": next_cursor})
|
||||
|
||||
except (XquikError, CookieClientError, WaybackError) as e:
|
||||
except (XquikError, CookieClientError, WaybackError, GoogleCSEError) as e:
|
||||
return jsonify({"ok": False, "error": str(e)}), 400
|
||||
except Exception as e: # noqa: BLE001
|
||||
return jsonify({"ok": False, "error": f"Unexpected error: {e}"}), 500
|
||||
|
||||
@@ -23,7 +23,10 @@ def _make_id(tool_type: str) -> str:
|
||||
return f"{tool_type}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
|
||||
|
||||
|
||||
def _tweet_url(item: dict) -> str | None:
|
||||
def build_tweet_url(item: dict) -> str | None:
|
||||
"""Not prefixed private — app.py reuses this so a live result and its
|
||||
later archive always compute the identical tweet_url, instead of two
|
||||
separate copies of the same logic drifting apart."""
|
||||
tid = str(item.get("id", "")).strip()
|
||||
user = str(item.get("user", "")).strip()
|
||||
if tid and user:
|
||||
@@ -31,6 +34,46 @@ def _tweet_url(item: dict) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _extract_media(item: dict) -> list[dict]:
|
||||
"""Same normalization the frontend's extractMedia() applies for card
|
||||
display. Cookie mode already returns [{type, thumb, url}] under `media`.
|
||||
xquik/API mode instead carries raw Twitter API shape under
|
||||
entities.media / extended_entities.media, which needs unpacking first —
|
||||
otherwise xquik-sourced photos/videos never enter the download queue below."""
|
||||
media = item.get("media")
|
||||
if isinstance(media, list) and media and isinstance(media[0], dict) \
|
||||
and ("thumb" in media[0] or "url" in media[0]):
|
||||
return media
|
||||
|
||||
src = None
|
||||
ext_ent = item.get("extended_entities")
|
||||
if isinstance(ext_ent, dict):
|
||||
src = ext_ent.get("media")
|
||||
if not isinstance(src, list):
|
||||
ent = item.get("entities")
|
||||
if isinstance(ent, dict):
|
||||
src = ent.get("media")
|
||||
if not isinstance(src, list):
|
||||
return []
|
||||
|
||||
result = []
|
||||
for m in src:
|
||||
if not isinstance(m, dict):
|
||||
continue
|
||||
mtype = m.get("type", "photo")
|
||||
thumb = m.get("media_url_https") or m.get("media_url") or ""
|
||||
if not thumb:
|
||||
continue
|
||||
url = thumb
|
||||
if mtype in ("video", "animated_gif"):
|
||||
variants = ((m.get("video_info") or {}).get("variants")) or []
|
||||
mp4s = [v for v in variants if isinstance(v, dict) and v.get("content_type") == "video/mp4"]
|
||||
if mp4s:
|
||||
url = max(mp4s, key=lambda v: v.get("bitrate", 0) or 0).get("url", thumb)
|
||||
result.append({"type": mtype, "thumb": thumb, "url": url})
|
||||
return result
|
||||
|
||||
|
||||
def _media_ext(url: str, mtype: str) -> str:
|
||||
if mtype in ("video", "animated_gif"):
|
||||
return "mp4"
|
||||
@@ -46,7 +89,7 @@ def _download_file(url: str, dest: Path) -> bool:
|
||||
r = requests.get(
|
||||
url,
|
||||
timeout=REQUEST_TIMEOUT,
|
||||
headers={"Referer": "https://x.com/", "User-Agent": "Mozilla/5.0"},
|
||||
headers={"Referer": "https://x.com/", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.10240"},
|
||||
stream=True,
|
||||
)
|
||||
r.raise_for_status()
|
||||
@@ -71,6 +114,7 @@ def _pick_fields(item: dict) -> dict:
|
||||
"retweeted_by_bio", "retweeted_at", "retweeted_tweet_id",
|
||||
"iso_date", "original", "statuscode", "mimetype", "length",
|
||||
"archive_url", "post_title", "post_text", "preview_image",
|
||||
"result_url", "display_link", "fetched_at",
|
||||
"source", "account_created", "account_age", "account_age_flag",
|
||||
"account_age_precision"]
|
||||
return {k: item[k] for k in keys if k in item and item[k] is not None}
|
||||
@@ -96,10 +140,10 @@ def _run(archive_id: str, tool_type: str, data, query_info: dict) -> None:
|
||||
continue
|
||||
|
||||
record = _pick_fields(item)
|
||||
record["tweet_url"] = _tweet_url(item)
|
||||
record["tweet_url"] = item.get("tweet_url") or build_tweet_url(item)
|
||||
local_media = []
|
||||
|
||||
for midx, m in enumerate(item.get("media", [])):
|
||||
for midx, m in enumerate(_extract_media(item)):
|
||||
url = m.get("url") or m.get("thumb", "")
|
||||
if not url:
|
||||
continue
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
[xquik]
|
||||
; check dasboard Xquik to put the api key
|
||||
|
||||
api_key = xxxxxxxxxxxxxxxxxxxxxxxxxxx
|
||||
|
||||
; Soon there is a array for base_url and endpoint are listed in offc docs
|
||||
@@ -9,11 +11,25 @@ base_url = https://xquik.com/api/v1/extractions
|
||||
|
||||
;end arr endpoint
|
||||
|
||||
[google_cse]
|
||||
; put the api key in google console api https://console.cloud.google.com/ also dont forget check the security config
|
||||
|
||||
api_key = AIzaSxxxxxxxxx
|
||||
|
||||
; put the cx key in https://programmablesearchengine.google.com/ google cse and sett the site want to crawll check the readme.md
|
||||
|
||||
cx = cxxxxxxxxxx
|
||||
|
||||
[twitter_cookies]
|
||||
; Inspect element then check the tab application then click the cookie in storage or u can use burpsuite
|
||||
|
||||
auth_token = xxxxxxxxxxxxxxxxxx
|
||||
ct0 = xxxxxxxxxxxx
|
||||
|
||||
[server]
|
||||
|
||||
; config flask
|
||||
|
||||
host = 127.0.0.1
|
||||
port = 5000
|
||||
debug = true
|
||||
@@ -0,0 +1,266 @@
|
||||
"""Look up general web results via the Google Custom Search JSON API
|
||||
(https://developers.google.com/custom-search/v1/overview) — a 4th data
|
||||
source alongside Cookie/Xquik (live X data) and Wayback (archived X pages).
|
||||
|
||||
Unlike the other three, this source isn't X-specific: it searches whatever
|
||||
scope the Custom Search Engine (cx) itself is configured for on Google's
|
||||
side, so it's the one lane that can surface a username/keyword showing up
|
||||
*off* X entirely — news mentions, forum posts, cached pages, other social
|
||||
platforms — which is what makes it worth adding to Multi Source Search.
|
||||
|
||||
Requires two values in config.ini [google_cse]:
|
||||
api_key — issued via Google Cloud/API Console (enable "Custom Search API")
|
||||
cx — the Search Engine ID from https://programmablesearchengine.google.com/
|
||||
|
||||
Free tier: 100 queries/day. Each page here costs exactly one query
|
||||
(Google caps `num` at 10 results/request), so `count` is served in
|
||||
10-result pages up to a hard ceiling well under the daily quota.
|
||||
"""
|
||||
|
||||
import configparser
|
||||
import html
|
||||
import re
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import requests
|
||||
|
||||
from id_forensics import decode_snowflake
|
||||
|
||||
CSE_URL = "https://www.googleapis.com/customsearch/v1"
|
||||
REQUEST_TIMEOUT = 20
|
||||
PAGE_SIZE = 10 # Google's hard max for `num`
|
||||
MAX_RESULTS = 50 # ceiling on results served per call, regardless of `count`
|
||||
|
||||
# Google truncates BOTH the title and the snippet it hands back in the SERP
|
||||
# JSON — title gets clipped just like the snippet does (see the module intro
|
||||
# reasoning). For text-intel use we want the page's own full title and
|
||||
# description instead, so every result with a link gets a best-effort live
|
||||
# fetch to pull its real og:/twitter: tags — same technique wayback_client.py
|
||||
# uses on archived snapshots, just against the live page instead.
|
||||
ENRICH_TIMEOUT = 10
|
||||
ENRICH_WORKERS = 8
|
||||
META_PARSE_CAP = 300_000 # bytes of HTML scanned for meta tags
|
||||
|
||||
_SAFE_URL_RE = re.compile(r"^https?://", re.IGNORECASE)
|
||||
_META_TAG_RE = re.compile(r"<meta\b[^>]*>", re.IGNORECASE)
|
||||
_ATTR_RE = re.compile(r'''([\w:-]+)\s*=\s*"([^"]*)"|([\w:-]+)\s*=\s*'([^']*)\'''')
|
||||
_TITLE_TAG_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)
|
||||
_TWEET_ID_RE = re.compile(r"/status/(\d+)")
|
||||
|
||||
|
||||
def _tweet_created_at(url: str) -> str | None:
|
||||
"""When `url` is a tweet permalink (…/status/<id>), decode the actual
|
||||
post-creation time straight out of the id's Snowflake bits. Same field
|
||||
name/format cookie and xquik already populate (`created_at`, Twitter's
|
||||
own classic timestamp string), so every source is consistent — and
|
||||
absent entirely for non-X results, same as it's absent for anything
|
||||
without a usable id."""
|
||||
m = _TWEET_ID_RE.search(url or "")
|
||||
if not m:
|
||||
return None
|
||||
dt = decode_snowflake(m.group(1))
|
||||
if not dt:
|
||||
return None
|
||||
return dt.strftime("%a %b %d %H:%M:%S +0000 %Y")
|
||||
|
||||
|
||||
class GoogleCSEError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def _get_client_config(config: configparser.ConfigParser) -> tuple[str, str]:
|
||||
api_key = config.get("google_cse", "api_key", fallback="").strip()
|
||||
cx = config.get("google_cse", "cx", fallback="").strip()
|
||||
if not api_key or api_key == "YOUR_GOOGLE_API_KEY":
|
||||
raise GoogleCSEError("api_key belum diisi di config.ini [google_cse]")
|
||||
if not cx or cx == "YOUR_SEARCH_ENGINE_ID":
|
||||
raise GoogleCSEError("cx (Search Engine ID) belum diisi di config.ini [google_cse]")
|
||||
return api_key, cx
|
||||
|
||||
|
||||
def _extract_thumbnail(item: dict) -> str | None:
|
||||
pagemap = item.get("pagemap") or {}
|
||||
for key in ("cse_image", "cse_thumbnail"):
|
||||
candidates = pagemap.get(key) or []
|
||||
if candidates and isinstance(candidates, list):
|
||||
src = (candidates[0] or {}).get("src", "").strip()
|
||||
# Third-party page metadata — only trust it if it's a plain http(s)
|
||||
# link, since the frontend renders this straight into an <a href>.
|
||||
if src and _SAFE_URL_RE.match(src):
|
||||
return src
|
||||
return None
|
||||
|
||||
|
||||
def _row_to_record(item: dict) -> dict:
|
||||
record = {}
|
||||
title = item.get("title")
|
||||
if title:
|
||||
record["post_title"] = html.unescape(title).strip()
|
||||
snippet = item.get("snippet")
|
||||
if snippet:
|
||||
record["post_text"] = html.unescape(snippet).strip()
|
||||
link = item.get("link")
|
||||
if link and _SAFE_URL_RE.match(link):
|
||||
record["result_url"] = link
|
||||
created_at = _tweet_created_at(link)
|
||||
if created_at:
|
||||
record["created_at"] = created_at # when the post itself was actually made
|
||||
display_link = item.get("displayLink")
|
||||
if display_link:
|
||||
record["display_link"] = display_link
|
||||
thumb = _extract_thumbnail(item)
|
||||
if thumb:
|
||||
record["preview_image"] = thumb
|
||||
return record
|
||||
|
||||
|
||||
def _parse_meta_tags(text: str) -> dict:
|
||||
tags = {}
|
||||
for tag in _META_TAG_RE.findall(text):
|
||||
attrs = {}
|
||||
for m in _ATTR_RE.finditer(tag):
|
||||
if m.group(1):
|
||||
attrs[m.group(1).lower()] = m.group(2)
|
||||
else:
|
||||
attrs[m.group(3).lower()] = m.group(4)
|
||||
key = attrs.get("property") or attrs.get("name")
|
||||
val = attrs.get("content")
|
||||
if key and val is not None:
|
||||
tags[key.lower()] = val
|
||||
return tags
|
||||
|
||||
|
||||
def _extract_title_tag(text: str) -> str | None:
|
||||
m = _TITLE_TAG_RE.search(text)
|
||||
if not m:
|
||||
return None
|
||||
title = re.sub(r"\s+", " ", m.group(1)).strip()
|
||||
return title or None
|
||||
|
||||
|
||||
def _fetch_live_meta(url: str) -> dict:
|
||||
"""Best-effort live fetch of the result's own page — pulls its real
|
||||
title and og:/twitter:/meta description to replace Google's clipped SERP
|
||||
title+snippet. Any failure (timeout, non-200, no usable tags) is
|
||||
swallowed — the caller just keeps Google's own (possibly truncated)
|
||||
values as a fallback."""
|
||||
try:
|
||||
r = requests.get(url, timeout=ENRICH_TIMEOUT, headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.10240"})
|
||||
if r.status_code != 200 or not r.text:
|
||||
return {}
|
||||
except requests.RequestException:
|
||||
return {}
|
||||
|
||||
body = r.text[:META_PARSE_CAP]
|
||||
tags = _parse_meta_tags(body)
|
||||
out = {}
|
||||
|
||||
title = tags.get("og:title") or tags.get("twitter:title") or _extract_title_tag(body)
|
||||
if title:
|
||||
out["post_title"] = html.unescape(title).strip()
|
||||
|
||||
desc = tags.get("og:description") or tags.get("twitter:description") or tags.get("description")
|
||||
if desc:
|
||||
out["post_text"] = html.unescape(desc).strip()
|
||||
|
||||
image = tags.get("og:image") or tags.get("twitter:image")
|
||||
if image:
|
||||
image = html.unescape(image).strip()
|
||||
if _SAFE_URL_RE.match(image):
|
||||
out["preview_image"] = image
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def _enrich_records(records: list[dict]) -> None:
|
||||
"""Mutates each record in place. Runs in parallel — one slow/dead site
|
||||
shouldn't hold up the rest of the result set."""
|
||||
candidates = [r for r in records if r.get("result_url")]
|
||||
if not candidates:
|
||||
return
|
||||
|
||||
def _job(rec):
|
||||
extra = _fetch_live_meta(rec["result_url"])
|
||||
if extra.get("post_title"):
|
||||
rec["post_title"] = extra["post_title"]
|
||||
if extra.get("post_text"):
|
||||
rec["post_text"] = extra["post_text"]
|
||||
if extra.get("preview_image") and not rec.get("preview_image"):
|
||||
rec["preview_image"] = extra["preview_image"]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=ENRICH_WORKERS) as pool:
|
||||
list(pool.map(_job, candidates))
|
||||
|
||||
|
||||
def _fetch_page(query: str, api_key: str, cx: str, start: int) -> list[dict]:
|
||||
params = {
|
||||
"key": api_key,
|
||||
"cx": cx,
|
||||
"q": query,
|
||||
"num": PAGE_SIZE,
|
||||
"start": start,
|
||||
}
|
||||
try:
|
||||
r = requests.get(CSE_URL, params=params, timeout=REQUEST_TIMEOUT)
|
||||
except requests.RequestException as e:
|
||||
raise GoogleCSEError(f"Google CSE request failed: {e}") from e
|
||||
|
||||
if r.status_code == 429:
|
||||
raise GoogleCSEError("Google CSE daily quota exceeded (100 free queries/day)")
|
||||
if r.status_code == 403:
|
||||
raise GoogleCSEError("Google CSE request forbidden — check api_key/cx and that the "
|
||||
"Custom Search API is enabled for that key's project")
|
||||
if r.status_code >= 400:
|
||||
raise GoogleCSEError(f"Google CSE API error {r.status_code}: {r.text[:300]}")
|
||||
|
||||
try:
|
||||
body = r.json()
|
||||
except ValueError as e:
|
||||
raise GoogleCSEError(f"Response bukan JSON valid: {r.text[:300]}") from e
|
||||
|
||||
return body.get("items") or []
|
||||
|
||||
|
||||
# ── Public API ───────────────────────────────────────────────────────────────
|
||||
|
||||
def google_cse_search(raw_query: str, count: int = 20, config: configparser.ConfigParser = None,
|
||||
cursor: str | None = None) -> tuple[list[dict], str | None]:
|
||||
"""Returns (records, next_cursor). `cursor` is the opaque 1-based `start`
|
||||
index from a previous call's next_cursor — pass it back to fetch the next
|
||||
page. None once there's nothing more to load (or the MAX_RESULTS ceiling
|
||||
is hit, to keep one Multi Source Search from burning the whole daily quota)."""
|
||||
query = (raw_query or "").strip()
|
||||
if not query:
|
||||
raise GoogleCSEError("Search query is required")
|
||||
|
||||
api_key, cx = _get_client_config(config)
|
||||
|
||||
try:
|
||||
start = int(cursor) if cursor else 1
|
||||
except ValueError:
|
||||
start = 1
|
||||
|
||||
target = min(max(1, count), MAX_RESULTS)
|
||||
records: list[dict] = []
|
||||
next_cursor = None
|
||||
|
||||
while len(records) < target:
|
||||
items = _fetch_page(query, api_key, cx, start)
|
||||
if not items:
|
||||
break
|
||||
for item in items:
|
||||
records.append(_row_to_record(item))
|
||||
if len(records) >= target:
|
||||
break
|
||||
start += PAGE_SIZE
|
||||
if len(items) < PAGE_SIZE:
|
||||
break # Google itself signaled this was the last page
|
||||
if start > MAX_RESULTS:
|
||||
break
|
||||
|
||||
_enrich_records(records)
|
||||
|
||||
if records and start <= MAX_RESULTS:
|
||||
next_cursor = str(start)
|
||||
|
||||
return records, next_cursor
|
||||
@@ -4,6 +4,9 @@
|
||||
// same data. (Previously included via Jinja {% include %} directly inside a
|
||||
// <script> block — moved to a real .js file so editor/JS tooling can lint it
|
||||
// normally instead of flagging the Jinja syntax as invalid JavaScript.)
|
||||
|
||||
// Arr params
|
||||
|
||||
const PRIORITY = [
|
||||
'source', 'account_age_flag', 'account_age', 'account_created',
|
||||
'user', 'screen_name', 'name', 'user_id', 'username',
|
||||
@@ -11,10 +14,10 @@ const PRIORITY = [
|
||||
'text', 'full_text', 'article_text', 'post_title', 'post_text', 'content', 'title', 'description', 'bio',
|
||||
'reply_count', 'retweet_count', 'favorite_count', 'view_count',
|
||||
'followers_count', 'following_count', 'tweet_count',
|
||||
'created_at', 'in_reply_to_tweet_id',
|
||||
'created_at', 'fetched_at', 'in_reply_to_tweet_id',
|
||||
'retweeted_by_user', 'retweeted_by_name', 'retweeted_text', 'retweeted_by_bio', 'retweeted_at', 'retweeted_tweet_id',
|
||||
'lat', 'lon', 'place', 'user_location',
|
||||
'tweet_url', 'archive_url', 'preview_image',
|
||||
'tweet_url', 'archive_url', 'result_url', 'preview_image', 'display_link',
|
||||
'iso_date', 'original', 'statuscode', 'mimetype', 'length',
|
||||
];
|
||||
|
||||
@@ -26,5 +29,5 @@ const DRILLABLE = {
|
||||
in_reply_to_tweet_id: 'tweet_replies_extractor',
|
||||
};
|
||||
|
||||
const SOURCE_CLASS = { 'Twitter Cookie': 'src-cookie', 'Xquik API': 'src-xquik', 'Wayback Machine': 'src-wayback' };
|
||||
const SOURCE_CLASS = { 'Twitter Cookie': 'src-cookie', 'Xquik API': 'src-xquik', 'Wayback Machine': 'src-wayback', 'Google CSE': 'src-cse' };
|
||||
const AGE_LABELS = { new: 'New account', recent: 'Recent account', established: 'Established account' };
|
||||
|
||||
@@ -8,6 +8,8 @@
|
||||
<div class="param-heading">Content</div>
|
||||
<div class="param-row"><code class="param-key">text / full_text / article_text</code> The post's own text</div>
|
||||
<div class="param-row"><code class="param-key">description</code> Profile bio</div>
|
||||
<div class="param-row"><code class="param-key">created_at</code> When the <em>post itself</em> was actually made. Same field, every source: Cookie/Xquik get it straight from the API; Wayback and Google CSE decode it from the tweet id's Snowflake bits whenever the result links to a tweet permalink (absent otherwise — e.g. a profile page or a non-X result, where there's no post id to decode)</div>
|
||||
<div class="param-row"><code class="param-key">fetched_at</code> When <em>this tool</em> pulled the record — same field, every source (Cookie, Xquik, Wayback, Google CSE), so results are comparable side by side. Not to be confused with <code class="param-key">created_at</code> (the post's own creation time) or <code class="param-key">iso_date</code> (a Wayback snapshot's capture time)</div>
|
||||
|
||||
<div class="param-heading">Engagement</div>
|
||||
<div class="param-row"><code class="param-key">reply_count / retweet_count / favorite_count / view_count</code> Counts reported by Twitter (can under-report)</div>
|
||||
@@ -24,6 +26,11 @@
|
||||
<div class="param-row"><code class="param-key">retweeted_text / retweeted_by_user / retweeted_by_name / retweeted_by_bio</code> Content and author of the original tweet being retweeted</div>
|
||||
<div class="param-row"><code class="param-key">retweeted_at / retweeted_tweet_id</code> When the original was posted, and its own ID</div>
|
||||
|
||||
<div class="param-heading">Media & links (every source)</div>
|
||||
<div class="param-row"><code class="param-key">media</code> Photo/video attachments — normalized to <code class="param-key">[{type, thumb, url}]</code> regardless of whether the source was Cookie or Xquik/API (the two use different raw shapes internally, unified before display/archive)</div>
|
||||
<div class="param-row"><code class="param-key">archived_media</code> Local file paths once a result's media has actually been downloaded into an archive — archive view only</div>
|
||||
<div class="param-row"><code class="param-key">tweet_url</code> Cookie/Xquik: direct link to the tweet, built from user + id. Populated the moment a result is fetched — the same value on the live card, in a JSON dump, and in an archive of it, not built separately each time. Wayback and Google CSE carry their own equivalent instead — see <code class="param-key">archive_url</code>/<code class="param-key">original</code> and <code class="param-key">result_url</code> below</div>
|
||||
|
||||
<div class="param-heading">Location</div>
|
||||
<div class="param-row"><code class="param-key">lat / lon / place</code> Coordinates plotted on the map view (Geo Post Search)</div>
|
||||
<div class="param-row"><code class="param-key">user_location</code> Free-text profile location string, geocoded client-side to produce lat/lon</div>
|
||||
@@ -39,10 +46,16 @@
|
||||
<div class="param-row"><code class="param-key">original</code> The original URL that was archived</div>
|
||||
<div class="param-row"><code class="param-key">statuscode / mimetype / length</code> HTTP status / content type / size of the snapshot</div>
|
||||
<div class="param-row"><code class="param-key">post_title / post_text / preview_image</code> Scraped from the archived page's own meta tags (og:/twitter: tags, or <title>/<meta name="description"> on older captures)</div>
|
||||
<div class="param-row"><code class="param-key">tweet_url</code> Direct link to the live tweet, built from user + id</div>
|
||||
|
||||
<div class="param-heading">Google CSE</div>
|
||||
<div class="param-row"><code class="param-key">post_title</code> Result title from Google</div>
|
||||
<div class="param-row"><code class="param-key">post_text</code> The page's own og:/twitter:/meta description, fetched live from <code class="param-key">result_url</code> — not Google's own SERP snippet, which is usually clipped mid-sentence. Falls back to Google's snippet only if that live fetch fails</div>
|
||||
<div class="param-row"><code class="param-key">result_url</code> The live page Google indexed, shown as its full raw address</div>
|
||||
<div class="param-row"><code class="param-key">display_link</code> The result's domain, as shown in Google's own results</div>
|
||||
<div class="param-row"><code class="param-key">preview_image</code> Thumbnail — from Google's own indexed metadata, or the live page's og:/twitter:image when Google didn't have one</div>
|
||||
|
||||
<div class="param-heading">Source</div>
|
||||
<div class="param-row"><code class="param-key">source</code> Which of the 3 data sources this result came from — Twitter Cookie / Xquik API / Wayback Machine (Multi-Source Search only)</div>
|
||||
<div class="param-row"><code class="param-key">source</code> Which of the 4 data sources this result came from — Twitter Cookie / Xquik API / Wayback Machine / Google CSE (Multi-Source Search only)</div>
|
||||
|
||||
<div class="param-heading">Graph node types (graph page only)</div>
|
||||
<div class="param-row"><code class="param-key">Search root</code> The diamond node — the query you ran</div>
|
||||
@@ -50,4 +63,5 @@
|
||||
<div class="param-row"><code class="param-key">Reply</code> A tweet fetched via Expand Replies on a tweet node</div>
|
||||
<div class="param-row"><code class="param-key">User / Retweeter</code> A person — from Follower Explorer, or via Expand Retweets on a tweet node</div>
|
||||
<div class="param-row"><code class="param-key">Wayback snapshot</code> An archived-page result, from Wayback Archive Search or the Wayback portion of Multi-Source Search</div>
|
||||
<div class="param-row"><code class="param-key">Web result</code> A Google CSE result, from the Google CSE portion of Multi-Source Search</div>
|
||||
<div class="param-row"><code class="param-key">Viewed</code> Turns a node solid white once you've clicked it — a visual "already looked at this" marker, not part of the underlying data</div>
|
||||
|
||||
@@ -323,6 +323,7 @@
|
||||
.source-badge.src-cookie { color: #a78bfa; border-color: #4c3a8f; background: #1e1535; }
|
||||
.source-badge.src-xquik { color: #8891f7; border-color: #34348a; background: var(--accent-bg); }
|
||||
.source-badge.src-wayback { color: #f59e0b; border-color: #78350f; background: #1c0e02; }
|
||||
.source-badge.src-cse { color: #f87171; border-color: #7f1d1d; background: #2a0d0d; }
|
||||
|
||||
.age-badge {
|
||||
display: inline-block;
|
||||
@@ -420,6 +421,7 @@
|
||||
<div class="help-row"><span class="source-badge src-cookie">Twitter Cookie</span> Live via your cookie session</div>
|
||||
<div class="help-row"><span class="source-badge src-xquik">Xquik API</span> Live via the xquik API</div>
|
||||
<div class="help-row"><span class="source-badge src-wayback">Wayback Machine</span> Archived snapshot</div>
|
||||
<div class="help-row"><span class="source-badge src-cse">Google CSE</span> Live web result via Google Custom Search</div>
|
||||
<h4>Account age</h4>
|
||||
<div class="help-row"><span class="age-badge age-new">New account</span> Created < 30 days ago</div>
|
||||
<div class="help-row"><span class="age-badge age-recent">Recent account</span> Created < 1 year ago</div>
|
||||
@@ -665,7 +667,7 @@ function buildCard(item) {
|
||||
display = `<a class="drill-link" href="/?${_p}" target="_blank" rel="noopener noreferrer">${lbl} ↗</a>`;
|
||||
} else if (v === null || v === undefined) {
|
||||
display = `<span style="color:var(--muted)">—</span>`;
|
||||
} else if (k === 'tweet_url' || k === 'archive_url' || k === 'preview_image') {
|
||||
} else if (k === 'tweet_url' || k === 'archive_url' || k === 'result_url' || k === 'preview_image') {
|
||||
display = `<a href="${esc(String(v))}" target="_blank" rel="noopener" class="card-link">${esc(String(v))}</a>`;
|
||||
} else if (k === 'source') {
|
||||
const cls = SOURCE_CLASS[v] || '';
|
||||
|
||||
@@ -339,6 +339,7 @@
|
||||
.leg-icon.reply { background: rgba(34,197,94,0.12); border-color: var(--success); border-radius: 2px; }
|
||||
.leg-icon.user { background: var(--purple-bg); border-color: var(--purple); border-radius: 50%; }
|
||||
.leg-icon.wayback { background: #2a1f08; border-color: var(--warn); border-radius: 2px; }
|
||||
.leg-icon.cse { background: #2a0d0d; border-color: #ef4444; border-radius: 2px; }
|
||||
.leg-icon.viewed { background: #ffffff; border-color: #c7c7c7; border-radius: 2px; }
|
||||
.leg-heading {
|
||||
font-size: 10px;
|
||||
@@ -382,6 +383,7 @@
|
||||
.source-badge.src-cookie { color: #a78bfa; border-color: #4c3a8f; background: var(--purple-bg); }
|
||||
.source-badge.src-xquik { color: #8891f7; border-color: #34348a; background: var(--accent-bg); }
|
||||
.source-badge.src-wayback { color: var(--warn); border-color: #78350f; background: #1c0e02; }
|
||||
.source-badge.src-cse { color: #f87171; border-color: #7f1d1d; background: #2a0d0d; }
|
||||
|
||||
.age-badge {
|
||||
display: inline-block;
|
||||
@@ -534,12 +536,14 @@
|
||||
<div class="leg-row"><div class="leg-icon reply"></div> Reply</div>
|
||||
<div class="leg-row"><div class="leg-icon user"></div> User / Retweeter</div>
|
||||
<div class="leg-row"><div class="leg-icon wayback"></div> Wayback snapshot</div>
|
||||
<div class="leg-row"><div class="leg-icon cse"></div> Google CSE web result</div>
|
||||
<div class="leg-row"><div class="leg-icon viewed"></div> Viewed (clicked)</div>
|
||||
|
||||
<div class="leg-heading">Source (info panel)</div>
|
||||
<div class="leg-row"><span class="source-badge src-cookie">Cookie</span></div>
|
||||
<div class="leg-row"><span class="source-badge src-xquik">Xquik API</span></div>
|
||||
<div class="leg-row"><span class="source-badge src-wayback">Wayback Machine</span></div>
|
||||
<div class="leg-row"><span class="source-badge src-cse">Google CSE</span></div>
|
||||
|
||||
<div class="leg-heading">Account age (info panel)</div>
|
||||
<div class="leg-row"><span class="age-badge age-new">New</span> < 30 days</div>
|
||||
@@ -581,7 +585,7 @@ const CY_STYLE = [
|
||||
// text-wrap 'ellipsis' truncates long post text to fit instead of growing the
|
||||
// box or overflowing it, which keeps dense graphs (100s of nodes) readable.
|
||||
// Full untruncated text is always available in the side panel on click.
|
||||
{ selector: 'node[type="tweet"], node[type="reply"], node[type="wayback"]', style: {
|
||||
{ selector: 'node[type="tweet"], node[type="reply"], node[type="wayback"], node[type="cse"]', style: {
|
||||
shape: 'roundrectangle',
|
||||
label: 'data(label)',
|
||||
color: '#e8eaf0',
|
||||
@@ -608,6 +612,10 @@ const CY_STYLE = [
|
||||
'background-color': '#2a1f08',
|
||||
'border-color': '#f59e0b',
|
||||
}},
|
||||
{ selector: 'node[type="cse"]', style: {
|
||||
'background-color': '#2a0d0d',
|
||||
'border-color': '#ef4444',
|
||||
}},
|
||||
{ selector: 'node[type="retweeter"], node[type="user"]', style: {
|
||||
shape: 'ellipse',
|
||||
'background-color': '#1e1535',
|
||||
@@ -629,7 +637,7 @@ const CY_STYLE = [
|
||||
// Viewed marker — set once a node has been clicked/inspected. Deliberately
|
||||
// plain white with no other color coding so it reads as one thing: "seen".
|
||||
// Placed after the type selectors so it always wins on background/border.
|
||||
{ selector: 'node[?viewed][type="tweet"], node[?viewed][type="reply"], node[?viewed][type="wayback"]', style: {
|
||||
{ selector: 'node[?viewed][type="tweet"], node[?viewed][type="reply"], node[?viewed][type="wayback"], node[?viewed][type="cse"]', style: {
|
||||
'background-color': '#ffffff',
|
||||
'border-color': '#c7c7c7',
|
||||
color: '#0f1117',
|
||||
@@ -846,18 +854,26 @@ function makeLabel(item, type) {
|
||||
var snap = (item.post_title || item.post_text || item.original || 'Snapshot').replace(/\s+/g, ' ').trim();
|
||||
return snap.length > 60 ? snap.substring(0, 60) + '…' : snap;
|
||||
}
|
||||
if (type === 'cse') {
|
||||
var web = (item.post_title || item.post_text || item.display_link || 'Web result').replace(/\s+/g, ' ').trim();
|
||||
return web.length > 60 ? web.substring(0, 60) + '…' : web;
|
||||
}
|
||||
var text = (item.text || item.retweeted_text || '').replace(/\s+/g, ' ').trim();
|
||||
return text.length > 60 ? text.substring(0, 60) + '…' : (text || ('Tweet ' + (item.id || '?')));
|
||||
}
|
||||
|
||||
// Per-item node type — multi_source_search mixes live tweets with Wayback
|
||||
// snapshots in one result set, so the type has to be resolved per item rather
|
||||
// than fixed for the whole tool/search.
|
||||
// snapshots and Google CSE web results in one result set, so the type has to
|
||||
// be resolved per item rather than fixed for the whole tool/search.
|
||||
function resolveNodeType(tool, item) {
|
||||
if (tool === 'follower_explorer') return 'user';
|
||||
if (tool === 'tweet_retweeters_extractor') return 'retweeter';
|
||||
if (tool === 'wayback_archive_search') return 'wayback';
|
||||
if (tool === 'multi_source_search') return item.source === 'Wayback Machine' ? 'wayback' : 'tweet';
|
||||
if (tool === 'multi_source_search') {
|
||||
if (item.source === 'Wayback Machine') return 'wayback';
|
||||
if (item.source === 'Google CSE') return 'cse';
|
||||
return 'tweet';
|
||||
}
|
||||
return 'tweet';
|
||||
}
|
||||
|
||||
@@ -1031,20 +1047,22 @@ function esc(s) {
|
||||
.replace(/"/g, '"');
|
||||
}
|
||||
|
||||
// LIST ARR PARAMS
|
||||
|
||||
var PRIORITY_KEYS = [
|
||||
'source', 'account_age_flag', 'account_age', 'account_created',
|
||||
'user', 'screen_name', 'name', 'text', 'post_title', 'post_text', 'created_at',
|
||||
'user', 'screen_name', 'name', 'text', 'post_title', 'post_text', 'created_at', 'fetched_at',
|
||||
'retweeted_by_user', 'retweeted_by_name', 'retweeted_text', 'retweeted_by_bio',
|
||||
'reply_count', 'retweet_count', 'favorite_count', 'view_count',
|
||||
'followers_count', 'following_count', 'tweet_count',
|
||||
'description', 'user_location', 'in_reply_to_tweet_id',
|
||||
'retweeted_tweet_id', 'retweeted_at', 'verified', 'is_blue_verified',
|
||||
'archive_url', 'preview_image', 'iso_date', 'original', 'statuscode',
|
||||
'archive_url', 'result_url', 'preview_image', 'display_link', 'iso_date', 'original', 'statuscode',
|
||||
];
|
||||
|
||||
var SKIP_KEYS = new Set(['id', 'media', 'card', 'user_id', 'retweeted_by_user_id']);
|
||||
|
||||
var SOURCE_CLASS = { 'Twitter Cookie': 'src-cookie', 'Xquik API': 'src-xquik', 'Wayback Machine': 'src-wayback' };
|
||||
var SOURCE_CLASS = { 'Twitter Cookie': 'src-cookie', 'Xquik API': 'src-xquik', 'Wayback Machine': 'src-wayback', 'Google CSE': 'src-cse' };
|
||||
|
||||
// ── Media helpers — same normalization as the non-graph search page ───────────
|
||||
function extractMedia(item) {
|
||||
@@ -1076,7 +1094,7 @@ function showPanel(data) {
|
||||
var raw = data.raw || {};
|
||||
var type = data.type;
|
||||
|
||||
var titles = { search: 'Search Node', tweet: 'Tweet', reply: 'Reply', user: 'User', retweeter: 'Retweeter' };
|
||||
var titles = { search: 'Search Node', tweet: 'Tweet', reply: 'Reply', user: 'User', retweeter: 'Retweeter', wayback: 'Wayback Snapshot', cse: 'Web Result' };
|
||||
document.getElementById('infoTitle').textContent = titles[type] || 'Node';
|
||||
|
||||
// Build key list (priority first, then remaining)
|
||||
@@ -1097,9 +1115,10 @@ function showPanel(data) {
|
||||
val = '<a href="https://x.com/' + encodeURIComponent(handle) + '/status/' + encodeURIComponent(String(v)) +
|
||||
'" target="_blank" rel="noopener noreferrer">' + esc(String(v)) + ' ↗</a>';
|
||||
}
|
||||
} else if (k === 'archive_url' || k === 'preview_image') {
|
||||
val = '<a href="' + esc(String(v)) + '" target="_blank" rel="noopener noreferrer">' +
|
||||
(k === 'archive_url' ? 'Open snapshot ↗' : 'Preview image ↗') + '</a>';
|
||||
} else if (k === 'tweet_url' || k === 'archive_url' || k === 'result_url' || k === 'preview_image') {
|
||||
// Full raw URL as the link text itself, not hidden behind a generic
|
||||
// label — same treatment as the live search page and archive.html.
|
||||
val = '<a href="' + esc(String(v)) + '" target="_blank" rel="noopener noreferrer">' + esc(String(v)) + '</a>';
|
||||
} else if (k === 'source') {
|
||||
val = '<span class="source-badge ' + (SOURCE_CLASS[v] || '') + '">' + esc(String(v)) + '</span>';
|
||||
} else if (k === 'account_age_flag') {
|
||||
@@ -1169,6 +1188,10 @@ function showPanel(data) {
|
||||
openBtn.href = raw.archive_url;
|
||||
openBtn.textContent = 'Open Archived Snapshot ↗';
|
||||
openBtn.style.display = '';
|
||||
} else if (type === 'cse' && raw.result_url) {
|
||||
openBtn.href = raw.result_url;
|
||||
openBtn.textContent = 'Open Page ↗';
|
||||
openBtn.style.display = '';
|
||||
} else {
|
||||
openBtn.style.display = 'none';
|
||||
}
|
||||
|
||||
@@ -276,6 +276,7 @@
|
||||
.source-badge.src-cookie { color: #a78bfa; border-color: #4c3a8f; background: var(--cookie-bg); }
|
||||
.source-badge.src-xquik { color: #8891f7; border-color: #34348a; background: var(--accent-bg); }
|
||||
.source-badge.src-wayback { color: var(--warn); border-color: #78350f; background: #1c0e02; }
|
||||
.source-badge.src-cse { color: #f87171; border-color: #7f1d1d; background: #2a0d0d; }
|
||||
|
||||
.age-badge {
|
||||
display: inline-block;
|
||||
@@ -598,6 +599,7 @@
|
||||
<div class="help-row"><span class="source-badge src-cookie">Twitter Cookie</span> Live via your cookie session</div>
|
||||
<div class="help-row"><span class="source-badge src-xquik">Xquik API</span> Live via the xquik API</div>
|
||||
<div class="help-row"><span class="source-badge src-wayback">Wayback Machine</span> Archived snapshot</div>
|
||||
<div class="help-row"><span class="source-badge src-cse">Google CSE</span> Live web result via Google Custom Search</div>
|
||||
<h4>Account age</h4>
|
||||
<div class="help-row"><span class="age-badge age-new">New account</span> Created < 30 days ago</div>
|
||||
<div class="help-row"><span class="age-badge age-recent">Recent account</span> Created < 1 year ago</div>
|
||||
@@ -759,7 +761,7 @@ const HINTS = {
|
||||
tweet_retweeters_extractor:'Users who retweeted a tweet — cookie only',
|
||||
geo_post_extractor: 'Search tweets by keyword, then plot each author\'s profile location on a map — cookie only',
|
||||
wayback_archive_search: 'Find archived snapshots of an X/Twitter profile or tweet via the Wayback Machine — great for recovering deleted content',
|
||||
multi_source_search: 'Searches Cookie, Xquik API and the Wayback Machine at once and merges the results — each card is labeled with its source. Accepts a keyword, mention, username, or a full URL. Wayback matches a handle or URL directly; Cookie/Xquik treat it as search text (e.g. paste a link to find who\'s tweeted it). Date range below is optional.',
|
||||
multi_source_search: 'Searches Cookie, Xquik API, the Wayback Machine and Google CSE at once and merges the results — each card is labeled with its source. Accepts a keyword, mention, username, or a full URL. Wayback matches a handle or URL directly; Cookie/Xquik treat it as search text; Google CSE runs it as a plain web search (results depend on how the Custom Search Engine itself is scoped). Date range below is optional and only narrows Cookie/Xquik/Wayback — Google CSE has no date-range query syntax.',
|
||||
};
|
||||
|
||||
// These tools only work via cookie; API mode is disabled for them
|
||||
@@ -997,10 +999,11 @@ function buildCard(item) {
|
||||
const _p = new URLSearchParams({ tool: DRILLABLE[k], id: tweetId, autorun: '1' });
|
||||
const lbl = (v !== null && v !== undefined) ? esc(String(v)) : '—';
|
||||
display = `<a class="drill-link" href="/?${_p}" target="_blank" rel="noopener noreferrer">${lbl} ↗</a>`;
|
||||
} else if (k === 'archive_url' && v) {
|
||||
display = `<a class="drill-link" href="${esc(String(v))}" target="_blank" rel="noopener noreferrer">Open snapshot ↗</a>`;
|
||||
} else if (k === 'preview_image' && v) {
|
||||
display = `<a class="drill-link" href="${esc(String(v))}" target="_blank" rel="noopener noreferrer">Preview image ↗</a>`;
|
||||
} else if ((k === 'tweet_url' || k === 'archive_url' || k === 'result_url' || k === 'preview_image') && v) {
|
||||
// Full raw URL shown as the link text itself (not hidden behind a generic
|
||||
// label) — text-intel use needs the actual address visible/copyable, same
|
||||
// treatment archive.html already gives every link field, kept consistent here.
|
||||
display = `<a class="drill-link" href="${esc(String(v))}" target="_blank" rel="noopener noreferrer">${esc(String(v))}</a>`;
|
||||
} else if (k === 'source' && v) {
|
||||
const cls = SOURCE_CLASS[v] || '';
|
||||
display = `<span class="source-badge ${cls}">${esc(String(v))}</span>`;
|
||||
|
||||
@@ -19,6 +19,8 @@ from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import requests
|
||||
|
||||
from id_forensics import decode_snowflake
|
||||
|
||||
WAYBACK_CDX_URL = "https://web.archive.org/cdx/search/cdx"
|
||||
REQUEST_TIMEOUT = 30
|
||||
SNAPSHOT_TIMEOUT = 10 # per-snapshot content fetch, run in parallel
|
||||
@@ -30,8 +32,9 @@ class WaybackError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
_DATE8_RE = re.compile(r"^\d{8}$")
|
||||
_DATE8_RE = re.compile(r"^\d{8}$")
|
||||
_SAFE_URL_RE = re.compile(r"^https?://", re.IGNORECASE)
|
||||
_TWEET_ID_RE = re.compile(r"/status/(\d+)")
|
||||
|
||||
|
||||
def _validate_date(label: str, value: str) -> None:
|
||||
@@ -73,7 +76,7 @@ def _fetch_cdx(url: str, limit: int, from_date: str = "", to_date: str = "",
|
||||
try:
|
||||
r = requests.get(
|
||||
WAYBACK_CDX_URL, params=params, timeout=REQUEST_TIMEOUT,
|
||||
headers={"User-Agent": "Mozilla/5.0"},
|
||||
headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.10240"},
|
||||
)
|
||||
r.raise_for_status()
|
||||
except requests.RequestException as e:
|
||||
@@ -98,21 +101,41 @@ def _fetch_cdx(url: str, limit: int, from_date: str = "", to_date: str = "",
|
||||
return [dict(zip(header, row)) for row in data_rows], next_resume
|
||||
|
||||
|
||||
def _tweet_created_at(url: str) -> str | None:
|
||||
"""When `url` is a tweet permalink (…/status/<id>), decode the actual
|
||||
post-creation time straight out of the id's Snowflake bits — independent
|
||||
of when Wayback happened to crawl it. Same field name/format cookie and
|
||||
xquik already populate (`created_at`, Twitter's own classic timestamp
|
||||
string), so date-range filtering and card rendering treat every source
|
||||
the same way. None for non-tweet URLs or ids too old to be Snowflake."""
|
||||
m = _TWEET_ID_RE.search(url or "")
|
||||
if not m:
|
||||
return None
|
||||
dt = decode_snowflake(m.group(1))
|
||||
if not dt:
|
||||
return None
|
||||
return dt.strftime("%a %b %d %H:%M:%S +0000 %Y")
|
||||
|
||||
|
||||
def _row_to_record(row: dict) -> dict:
|
||||
ts = row.get("timestamp", "") or ""
|
||||
original = row.get("original", "") or ""
|
||||
iso_date = None
|
||||
if len(ts) >= 14:
|
||||
iso_date = f"{ts[0:4]}-{ts[4:6]}-{ts[6:8]} {ts[8:10]}:{ts[10:12]}:{ts[12:14]}"
|
||||
return {
|
||||
record = {
|
||||
"timestamp": ts, # internal only — stripped before returning to caller
|
||||
"iso_date": iso_date,
|
||||
"iso_date": iso_date, # when Wayback captured this snapshot
|
||||
"original": original,
|
||||
"statuscode": row.get("statuscode"),
|
||||
"mimetype": row.get("mimetype"),
|
||||
"length": row.get("length"),
|
||||
"archive_url": f"https://web.archive.org/web/{ts}/{original}" if ts and original else None,
|
||||
}
|
||||
created_at = _tweet_created_at(original)
|
||||
if created_at:
|
||||
record["created_at"] = created_at # when the post itself was actually made
|
||||
return record
|
||||
|
||||
|
||||
# ── Content enrichment ──────────────────────────────────────────────────────
|
||||
@@ -153,7 +176,7 @@ def _fetch_snapshot_meta(timestamp: str, original: str) -> dict:
|
||||
try:
|
||||
r = requests.get(
|
||||
snap_url, timeout=SNAPSHOT_TIMEOUT,
|
||||
headers={"User-Agent": "Mozilla/5.0"},
|
||||
headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.10240"},
|
||||
)
|
||||
if r.status_code != 200 or not r.text:
|
||||
return {}
|
||||
|
||||
@@ -22,11 +22,12 @@ class XquikClient:
|
||||
self.config = config or load_config()
|
||||
self.api_key = self.config.get("xquik", "api_key", fallback="").strip()
|
||||
self.base_url = self.config.get(
|
||||
# Endpoint list arr soon
|
||||
"xquik", "base_url", fallback="https://xquik.com/api/v1/extractions"
|
||||
).strip()
|
||||
|
||||
if not self.api_key or self.api_key == "xq_YOUR_KEY":
|
||||
raise XquikError("api_key belum diisi di config.ini [xquik]")
|
||||
raise XquikError("api_key Xquik are not set")
|
||||
|
||||
def _post(self, payload: dict) -> dict:
|
||||
headers = {
|
||||
@@ -36,7 +37,7 @@ class XquikClient:
|
||||
try:
|
||||
resp = requests.post(self.base_url, headers=headers, json=payload, timeout=30)
|
||||
except requests.RequestException as e:
|
||||
raise XquikError(f"Request gagal: {e}") from e
|
||||
raise XquikError(f"Request failed: {e}") from e
|
||||
|
||||
if resp.status_code >= 400:
|
||||
raise XquikError(f"xquik API error {resp.status_code}: {resp.text}")
|
||||
@@ -44,7 +45,10 @@ class XquikClient:
|
||||
try:
|
||||
return resp.json()
|
||||
except ValueError as e:
|
||||
raise XquikError(f"Response bukan JSON valid: {resp.text[:300]}") from e
|
||||
raise XquikError(f"Response are not json: {resp.text[:300]}") from e
|
||||
|
||||
# PARAMS IN Xquik docs
|
||||
# https://docs.xquik.com/
|
||||
|
||||
def tweet_search(self, search_query: str) -> dict:
|
||||
return self._post({"toolType": "tweet_search_extractor", "searchQuery": search_query})
|
||||
@@ -61,7 +65,7 @@ class XquikClient:
|
||||
)
|
||||
|
||||
def post_extractor(self, target_username: str) -> dict:
|
||||
"""User timeline lewat xquik (pakai kuota API)."""
|
||||
"""User timeline via Xquik"""
|
||||
return self._post({"toolType": "post_extractor", "targetUsername": target_username})
|
||||
|
||||
|
||||
@@ -69,6 +73,9 @@ if __name__ == "__main__":
|
||||
import argparse
|
||||
import json
|
||||
|
||||
# PARAMS IN Xquik docs
|
||||
# https://docs.xquik.com/
|
||||
|
||||
parser = argparse.ArgumentParser(description="xquik.com extraction CLI")
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
@@ -90,6 +97,9 @@ if __name__ == "__main__":
|
||||
args = parser.parse_args()
|
||||
client = XquikClient()
|
||||
|
||||
# PARAMS IN Xquik docs
|
||||
# https://docs.xquik.com/
|
||||
|
||||
try:
|
||||
if args.command == "tweet_search":
|
||||
out = client.tweet_search(args.query)
|
||||
|
||||
Reference in New Issue
Block a user