OSINT-Cheat-sheet/Script/SOCMINT-Twitter/sentiment.py

"""Sentiment clustering (pro / neutral / con) for archived X/Twitter data,
plus the supporting "who's loudest, what's trending, what words dominate"
aggregates — the same shape of dashboard Drone Emprit-style tools give a
dataset.

Two scoring backends, tried in this order:

1. ML (preferred) — cardiffnlp/twitter-xlm-roberta-base-sentiment, an
   XLM-RoBERTa model fine-tuned for tweet sentiment across 8 languages
   (ar/en/fr/de/hi/it/pt/es). Its base pretraining covers ~100 languages, so
   it degrades gracefully rather than failing outright on a language outside
   that fine-tuning set — this is what makes the tool usable for an
   open-source audience that isn't Indonesian-only. Needs torch +
   transformers (see requirements.txt) and ~1.1GB of model weights
   downloaded from Hugging Face on first use.
2. Lexicon fallback — a hand-built Indonesian positive/negative word list
   with basic negation handling ("tidak bagus" flips "bagus" from positive
   to negative). Used automatically whenever torch/transformers aren't
   installed, so a lightweight install (just requirements.txt's base deps)
   still has a working — if Indonesian-only — sentiment feature rather than
   a hard failure. Every scored item exposes exactly which backend produced
   it (`method` on the analyze() payload), so a classification's provenance
   is never ambiguous, and the lexicon path additionally exposes the exact
   words that drove its score for full transparency.

Neither backend is a ground-truth classifier — short text, sarcasm, and
irony all degrade accuracy regardless of approach. Treat results as a
starting point for investigation, not a verdict.
"""

import re
import threading
from collections import Counter

# Local fallback 

# ── Sentiment lexicon ───────────────────────────────────────────────────────
# Indonesian words skew political/social-discourse (matches the kind of
# content this tool actually pulls — keyword searches on public affairs,
# government programs, public figures) as well as general register.

# Arr data words 
# Need to feedback and research to sett the all parameter for each words 
# Need to help netizen Indo, with slang and words 

POSITIVE_WORDS = {
    "bagus", "baik", "hebat", "keren", "mantap", "mantul", "top", "terbaik",
    "sukses", "berhasil", "tepat", "benar", "setuju", "dukung", "dukungan",
    "mendukung", "apresiasi", "mengapresiasi", "bangga", "membanggakan",
    "senang", "gembira", "puas", "memuaskan", "bersyukur", "syukur",
    "alhamdulillah", "semoga", "maju", "kemajuan", "bijak", "bijaksana",
    "cerdas", "pintar", "amanah", "jujur", "kejujuran", "adil", "keadilan",
    "peduli", "kepedulian", "bermanfaat", "manfaat", "membantu", "bantuan",
    "solusi", "prestasi", "berprestasi", "unggul", "keunggulan", "luar biasa",
    "joss", "jos", "gas terus", "lanjutkan", "sip", "oke", "aman", "tenang",
    "damai", "sejahtera", "kesejahteraan", "makmur", "kemakmuran", "indah",
    "cantik", "tampan", "ramah", "sopan", "santun", "hormat", "menghormati",
    "salut", "kagum", "mengagumkan", "respect", "terharu", "terinspirasi",
    "inspiratif", "positif", "optimis", "optimisme", "harapan", "berharap",
    "cinta", "mencintai", "sayang", "suka", "menyukai", "rindu", "kangen",
    "gemas", "lucu", "menghibur", "menyenangkan", "menginspirasi", "tegas",
    "berani", "keberanian", "kuat", "gigih", "semangat", "bersemangat",
    "sukacita", "kompeten", "profesional", "berkualitas", "kualitas",
    "elegan", "canggih", "inovatif", "inovasi", "kreatif", "efisien",
    "efektif", "transparan", "transparansi", "akuntabel", "akuntabilitas",
    "merakyat", "membela rakyat", "pro rakyat", "berpihak pada rakyat",
    "terpuji", "membanggakan", "gemilang", "cemerlang", "berkah", "istimewa",
}

NEGATIVE_WORDS = {
    "buruk", "keburukan", "jelek", "gagal", "kegagalan", "bodoh", "tolol",
    "goblok", "bego", "dungu", "idiot", "korupsi", "korup", "koruptor",
    "bohong", "kebohongan", "hoax", "menipu", "penipu", "penipuan", "tipu",
    "curang", "kecurangan", "culas", "zalim", "menzalimi", "kejam",
    "kekejaman", "jahat", "kejahatan", "rusak", "merusak", "hancur",
    "menghancurkan", "kacau", "mengacaukan", "parah", "memalukan",
    "memuakkan", "bejat", "biadab", "tolak", "menolak", "penolakan",
    "kecewa", "mengecewakan", "kekecewaan", "marah", "kemarahan", "murka",
    "benci", "membenci", "kebencian", "muak", "jijik", "menjijikkan",
    "sampah", "anjing", "bangsat", "bajingan", "kampret", "sialan",
    "kacung", "boneka", "munafik", "kemunafikan", "pengkhianat",
    "mengkhianati", "khianat", "pengkhianatan", "penjajah", "menjajah",
    "licik", "licin", "serakah", "keserakahan", "tamak", "otoriter",
    "diktator", "fasis", "salah", "kesalahan", "blunder", "konyol",
    "ngawur", "ngaco", "absurd", "aneh", "ironis", "ironi", "tragis",
    "miris", "prihatin", "keprihatinan", "sedih", "menyedihkan", "susah",
    "kesusahan", "sulit", "kesulitan", "sengsara", "menderita",
    "penderitaan", "korban", "dizalimi", "ditindas", "menindas",
    "penindasan", "kriminal", "pelanggaran", "melanggar", "ilegal",
    "pungli", "sogok", "menyuap", "disuap", "suap", "nepotisme", "kkn",
    "provokasi", "provokator", "memprovokasi", "fitnah", "memfitnah",
    "ancaman", "mengancam", "teror", "intimidasi", "mengintimidasi",
    "brutal", "kekerasan", "sadis", "tragedi", "bencana", "krisis",
    "darurat", "gawat", "resah", "keresahan", "meresahkan", "cemas",
    "kecemasan", "khawatir", "kekhawatiran", "takut", "ketakutan", "geram",
    "kesal", "jengkel", "dongkol", "malu", "hina", "menghina",
    "penghinaan", "murahan", "norak", "kampungan", "terbelakang", "mundur",
    "kemunduran", "ambruk", "bangkrut", "kebangkrutan", "defisit",
    "terlilit", "terjerat", "terjebak", "cengeng", "lemah", "kelemahan",
    "pengecut", "penakut", "plin-plan", "labil", "egois", "keegoisan",
    "sombong", "kesombongan", "angkuh", "arogan", "sok", "songong",
    "kurang ajar", "tidak becus", "amburadul", "berantakan", "semrawut",
    "menyengsarakan", "represif", "represi", "diskriminasi",
    "mendiskriminasi", "rasis", "rasisme", "intoleran", "intoleransi",
    "penjilat", "gila", "kontol", "memek", "paok", "stress", "goblog",
    "kepala batu", "oon",
}

# Flips the polarity of a sentiment word found within NEGATION_WINDOW tokens
# after it ("tidak bagus" -> negative even though "bagus" alone is positive).
# Indonesian often puts more distance between the negation and the word it
# actually governs than English does — "ngga ngajarin cara ngmg yg sopan"
# ("doesn't teach how to speak politely") is 5 tokens from negation to the
# sentiment word it negates. A wider window catches more of those at the
# cost of occasionally flipping something the negation wasn't really about;
# lexicon scoring is a heuristic either way, this just picks which failure
# mode to lean toward.
NEGATION_WORDS = {"tidak", "tak", "bukan", "belum", "jangan", "nggak", "ga", "gak", "kagak", "ngga"}
NEGATION_WINDOW = 5

# Boosts a sentiment word's weight rather than changing its polarity.
INTENSIFIERS = {"sangat", "banget", "sekali", "sungguh", "amat", "terlalu", "sangatlah"}
INTENSIFIER_MULTIPLIER = 1.5

STOPWORDS = {
    "yang", "dan", "di", "ke", "dari", "untuk", "dengan", "ini", "itu", "ya",
    "nya", "adalah", "akan", "saya", "kamu", "kita", "kami", "mereka", "dia",
    "juga", "saja", "sudah", "belum", "atau", "karena", "jika", "kalau",
    "agar", "supaya", "pada", "oleh", "dalam", "luar", "atas", "bawah",
    "antara", "seperti", "sebagai", "tentang", "bahwa", "namun", "tetapi",
    "tapi", "hingga", "sampai", "sejak", "setelah", "sebelum", "ketika",
    "saat", "ada", "punya", "milik", "lah", "kah", "pun", "deh", "dong",
    "sih", "kok", "loh", "nih", "gitu", "gini", "dsb", "dll", "dst", "yg",
    "utk", "dgn", "krn", "gak", "ga", "nggak", "tak", "tidak", "bukan",
    "jangan", "apa", "apakah", "siapa", "mengapa", "kenapa", "bagaimana",
    "dimana", "kapan", "para", "si", "sang", "an", "kan", "in", "the", "is",
    "are", "was", "were", "be", "been", "being", "to", "of", "for", "on",
    "with", "as", "by", "at", "an", "a", "rt",
} | NEGATION_WORDS | INTENSIFIERS

_WORD_RE  = re.compile(r"[a-zA-ZÀ-ÿ]+(?:-[a-zA-ZÀ-ÿ]+)?")
_URL_RE   = re.compile(r"https?://\S+")
_MENTION_RE = re.compile(r"@\w+")


def _tokenize(text: str) -> list[str]:
    """Lowercased word tokens with URLs/@mentions stripped first (both would
    otherwise pollute the lexicon match and the word cloud with usernames/
    link fragments neither list has any business scoring)."""
    if not text:
        return []
    cleaned = _URL_RE.sub(" ", text)
    cleaned = _MENTION_RE.sub(" ", cleaned)
    return [w.lower() for w in _WORD_RE.findall(cleaned)]


def score_text(text: str) -> dict:
    """Returns {label, score, matches} for one piece of text. label is one of
    "pro" / "neutral" / "con". matches lists (word, polarity, weight) for
    every lexicon hit, so a classification can be inspected rather than
    trusted blindly — a defining trait of a rule-based classifier is that you
    CAN see exactly why it decided what it decided."""
    tokens  = _tokenize(text)
    score   = 0.0
    matches = []

    for i, tok in enumerate(tokens):
        polarity = 1 if tok in POSITIVE_WORDS else -1 if tok in NEGATIVE_WORDS else 0
        if polarity == 0:
            continue

        weight = 1.0
        # Negation: any negation word in the preceding window flips polarity.
        window_start = max(0, i - NEGATION_WINDOW)
        if any(t in NEGATION_WORDS for t in tokens[window_start:i]):
            polarity = -polarity
        # Intensifier: any intensifier immediately before boosts magnitude
        # (checked after negation so "tidak sangat bagus" still flips first).
        if i > 0 and tokens[i - 1] in INTENSIFIERS:
            weight = INTENSIFIER_MULTIPLIER

        contribution = polarity * weight
        score += contribution
        matches.append({"word": tok, "polarity": "pro" if polarity > 0 else "con", "weight": weight})

    label = "pro" if score > 0 else "con" if score < 0 else "neutral"
    return {"label": label, "score": round(score, 2), "matches": matches}


def _item_text(item: dict) -> str:
    """The text worth scoring/tokenizing for a given archived record —
    varies by which tool produced it (a tweet's own text vs. a Wayback/CSE
    page's scraped title+description vs. a bare user's bio)."""
    parts = [
        item.get("text"), item.get("full_text"), item.get("article_text"),
        item.get("post_title"), item.get("post_text"), item.get("description"),
    ]
    return " ".join(p for p in parts if p)


def _item_author(item: dict) -> str | None:
    return item.get("screen_name") or item.get("user") or item.get("name")


def _item_engagement(item: dict) -> int:
    total = 0
    for k in ("reply_count", "retweet_count", "favorite_count"):
        v = item.get(k)
        if isinstance(v, (int, float)):
            total += v
    return total


def word_frequencies(items: list[dict], top_n: int = 60) -> list[dict]:
    """Word-cloud data: [{word, count}], most frequent first. Stopwords and
    single-character tokens are dropped; everything else counts regardless
    of whether it happened to be in the sentiment lexicon."""
    counts = Counter()
    for item in items:
        for tok in _tokenize(_item_text(item)):
            if len(tok) < 3 or tok in STOPWORDS:
                continue
            counts[tok] += 1
    return [{"word": w, "count": c} for w, c in counts.most_common(top_n)]


def top_users(items: list[dict], top_n: int = 20) -> list[dict]:
    """Who shows up most often across the archive — every record with an
    identifiable author counts once, regardless of whether it's a tweet, a
    reply, a retweeter entry, or a bare follower/following record. Carries
    along the most recently seen avatar/name for that handle so the
    dashboard can show a face, not just a bare count."""
    counts: Counter = Counter()
    display: dict[str, dict] = {}
    for item in items:
        handle = item.get("screen_name") or item.get("user")
        if not handle:
            continue
        counts[handle] += 1
        display[handle] = {
            "screen_name": handle,
            "name":        item.get("name"),
            "avatar":      item.get("avatar") or item.get("user_avatar"),
            "verified":    item.get("verified"),
            "is_blue_verified": item.get("is_blue_verified"),
        }
    ranked = []
    for handle, count in counts.most_common(top_n):
        ranked.append({**display[handle], "count": count})
    return ranked


def top_engagement(items: list[dict], top_n: int = 15) -> list[dict]:
    """Which posts drove the most reply+retweet+favorite activity — "paling
    ramai" (busiest/most-discussed), not just most recent."""
    scored = [(_item_engagement(it), it) for it in items if _item_text(it)]
    scored.sort(key=lambda pair: pair[0], reverse=True)
    return [{"engagement": score, "item": it} for score, it in scored[:top_n] if score > 0]


# ── ML backend (preferred) ───────────────────────────────────────────────────
# Lazy-loaded: importing torch/transformers and loading ~1.1GB of weights is
# slow, and both packages are optional (requirements.txt notes how to add
# them) — doing this at module import time would slow down every single use
# of this app, including ones that never touch analytics, and would hard-crash
# an install that skipped the ML deps entirely instead of just falling back.
_ML_MODEL_NAME    = "cardiffnlp/twitter-xlm-roberta-base-sentiment"
_ML_LABEL_MAP     = {"positive": "pro", "negative": "con", "neutral": "neutral"}
_ml_pipeline      = None
_ml_unavailable   = False   # sticky once loading fails — stop retrying every call
_ml_load_lock     = threading.Lock()


def _get_ml_pipeline():
    global _ml_pipeline, _ml_unavailable
    if _ml_pipeline is not None or _ml_unavailable:
        return _ml_pipeline
    with _ml_load_lock:
        if _ml_pipeline is not None or _ml_unavailable:   # re-check post-lock
            return _ml_pipeline
        try:
            from transformers import pipeline
            _ml_pipeline = pipeline("sentiment-analysis", model=_ML_MODEL_NAME)
        except Exception:
            # Missing torch/transformers, no internet for the first-time
            # model download, out of memory, ... any of these should fall
            # back to the lexicon scorer rather than take the whole
            # analytics endpoint down.
            _ml_unavailable = True
            _ml_pipeline = None
    return _ml_pipeline


def _score_texts_ml(texts: list[str]) -> list[dict] | None:
    """Batch-scores every text in one call (far faster on CPU than one
    pipeline call per item). Returns None if the model isn't available, so
    the caller falls back to the lexicon scorer instead. `score` is signed
    (positive for pro, negative for con, 0 for neutral) to match the
    lexicon backend's convention; `confidence` carries the model's own
    unsigned probability for the label it picked."""
    clf = _get_ml_pipeline()
    if clf is None:
        return None
    raw = clf(texts, truncation=True, batch_size=16)
    results = []
    for r in raw:
        label      = _ML_LABEL_MAP.get(str(r.get("label", "")).lower(), "neutral")
        confidence = float(r.get("score", 0.0))
        signed     = confidence if label == "pro" else -confidence if label == "con" else 0.0
        results.append({
            "label": label, "score": round(signed, 3),
            "confidence": round(confidence, 3), "matches": [],
        })
    return results


def analyze(items: list[dict]) -> dict:
    """Full analytics payload for one archive's worth of raw items."""
    if not isinstance(items, list):
        items = [items]
    # archive.py's own _run() passes non-dict entries through as-is rather
    # than dropping them (see its "if not isinstance(item, dict)" branch),
    # so a saved archive can legitimately contain a stray non-dict item —
    # every function below assumes dict.get(), so those get filtered here
    # once rather than each helper needing its own isinstance guard.
    items = [it for it in items if isinstance(it, dict)]

    text_items, texts = [], []
    for item in items:
        text = _item_text(item)
        if text:   # skip e.g. a bare follower/following record with no post text of its own
            text_items.append(item)
            texts.append(text)

    ml_results = _score_texts_ml(texts) if texts else None
    method     = "ml" if ml_results is not None else "lexicon"
    if ml_results is None:
        ml_results = [score_text(t) for t in texts]

    sentiment_counts = {"pro": 0, "neutral": 0, "con": 0}
    scored_items = []
    for item, text, result in zip(text_items, texts, ml_results):
        sentiment_counts[result["label"]] += 1
        scored_items.append({
            "label":      result["label"],
            "score":      result["score"],
            "confidence": result.get("confidence"),
            "matches":    result.get("matches", []),
            "author":     _item_author(item),
            "text":       text,
            "item":       item,
        })

    total = len(scored_items)
    return {
        "method":            method,
        "total_items":       len(items),
        "total_scored":      total,
        "sentiment_counts":  sentiment_counts,
        "sentiment_pct": {
            k: round(v / total * 100, 1) if total else 0.0
            for k, v in sentiment_counts.items()
        },
        "scored_items":  scored_items,
        "word_freq":     word_frequencies(items),
        "top_users":     top_users(items),
        "top_engagement": top_engagement(items),
    }