"""Sentiment clustering (pro / neutral / con) for archived X/Twitter data, plus the supporting "who's loudest, what's trending, what words dominate" aggregates — the same shape of dashboard Drone Emprit-style tools give a dataset. Two scoring backends, tried in this order: 1. ML (preferred) — cardiffnlp/twitter-xlm-roberta-base-sentiment, an XLM-RoBERTa model fine-tuned for tweet sentiment across 8 languages (ar/en/fr/de/hi/it/pt/es). Its base pretraining covers ~100 languages, so it degrades gracefully rather than failing outright on a language outside that fine-tuning set — this is what makes the tool usable for an open-source audience that isn't Indonesian-only. Needs torch + transformers (see requirements.txt) and ~1.1GB of model weights downloaded from Hugging Face on first use. 2. Lexicon fallback — a hand-built Indonesian positive/negative word list with basic negation handling ("tidak bagus" flips "bagus" from positive to negative). Used automatically whenever torch/transformers aren't installed, so a lightweight install (just requirements.txt's base deps) still has a working — if Indonesian-only — sentiment feature rather than a hard failure. Every scored item exposes exactly which backend produced it (`method` on the analyze() payload), so a classification's provenance is never ambiguous, and the lexicon path additionally exposes the exact words that drove its score for full transparency. Neither backend is a ground-truth classifier — short text, sarcasm, and irony all degrade accuracy regardless of approach. Treat results as a starting point for investigation, not a verdict. """ import re import threading from collections import Counter # Local fallback # ── Sentiment lexicon ─────────────────────────────────────────────────────── # Indonesian words skew political/social-discourse (matches the kind of # content this tool actually pulls — keyword searches on public affairs, # government programs, public figures) as well as general register. # Arr data words # Need to feedback and research to sett the all parameter for each words # Need to help netizen Indo, with slang and words POSITIVE_WORDS = { "bagus", "baik", "hebat", "keren", "mantap", "mantul", "top", "terbaik", "sukses", "berhasil", "tepat", "benar", "setuju", "dukung", "dukungan", "mendukung", "apresiasi", "mengapresiasi", "bangga", "membanggakan", "senang", "gembira", "puas", "memuaskan", "bersyukur", "syukur", "alhamdulillah", "semoga", "maju", "kemajuan", "bijak", "bijaksana", "cerdas", "pintar", "amanah", "jujur", "kejujuran", "adil", "keadilan", "peduli", "kepedulian", "bermanfaat", "manfaat", "membantu", "bantuan", "solusi", "prestasi", "berprestasi", "unggul", "keunggulan", "luar biasa", "joss", "jos", "gas terus", "lanjutkan", "sip", "oke", "aman", "tenang", "damai", "sejahtera", "kesejahteraan", "makmur", "kemakmuran", "indah", "cantik", "tampan", "ramah", "sopan", "santun", "hormat", "menghormati", "salut", "kagum", "mengagumkan", "respect", "terharu", "terinspirasi", "inspiratif", "positif", "optimis", "optimisme", "harapan", "berharap", "cinta", "mencintai", "sayang", "suka", "menyukai", "rindu", "kangen", "gemas", "lucu", "menghibur", "menyenangkan", "menginspirasi", "tegas", "berani", "keberanian", "kuat", "gigih", "semangat", "bersemangat", "sukacita", "kompeten", "profesional", "berkualitas", "kualitas", "elegan", "canggih", "inovatif", "inovasi", "kreatif", "efisien", "efektif", "transparan", "transparansi", "akuntabel", "akuntabilitas", "merakyat", "membela rakyat", "pro rakyat", "berpihak pada rakyat", "terpuji", "membanggakan", "gemilang", "cemerlang", "berkah", "istimewa", } NEGATIVE_WORDS = { "buruk", "keburukan", "jelek", "gagal", "kegagalan", "bodoh", "tolol", "goblok", "bego", "dungu", "idiot", "korupsi", "korup", "koruptor", "bohong", "kebohongan", "hoax", "menipu", "penipu", "penipuan", "tipu", "curang", "kecurangan", "culas", "zalim", "menzalimi", "kejam", "kekejaman", "jahat", "kejahatan", "rusak", "merusak", "hancur", "menghancurkan", "kacau", "mengacaukan", "parah", "memalukan", "memuakkan", "bejat", "biadab", "tolak", "menolak", "penolakan", "kecewa", "mengecewakan", "kekecewaan", "marah", "kemarahan", "murka", "benci", "membenci", "kebencian", "muak", "jijik", "menjijikkan", "sampah", "anjing", "bangsat", "bajingan", "kampret", "sialan", "kacung", "boneka", "munafik", "kemunafikan", "pengkhianat", "mengkhianati", "khianat", "pengkhianatan", "penjajah", "menjajah", "licik", "licin", "serakah", "keserakahan", "tamak", "otoriter", "diktator", "fasis", "salah", "kesalahan", "blunder", "konyol", "ngawur", "ngaco", "absurd", "aneh", "ironis", "ironi", "tragis", "miris", "prihatin", "keprihatinan", "sedih", "menyedihkan", "susah", "kesusahan", "sulit", "kesulitan", "sengsara", "menderita", "penderitaan", "korban", "dizalimi", "ditindas", "menindas", "penindasan", "kriminal", "pelanggaran", "melanggar", "ilegal", "pungli", "sogok", "menyuap", "disuap", "suap", "nepotisme", "kkn", "provokasi", "provokator", "memprovokasi", "fitnah", "memfitnah", "ancaman", "mengancam", "teror", "intimidasi", "mengintimidasi", "brutal", "kekerasan", "sadis", "tragedi", "bencana", "krisis", "darurat", "gawat", "resah", "keresahan", "meresahkan", "cemas", "kecemasan", "khawatir", "kekhawatiran", "takut", "ketakutan", "geram", "kesal", "jengkel", "dongkol", "malu", "hina", "menghina", "penghinaan", "murahan", "norak", "kampungan", "terbelakang", "mundur", "kemunduran", "ambruk", "bangkrut", "kebangkrutan", "defisit", "terlilit", "terjerat", "terjebak", "cengeng", "lemah", "kelemahan", "pengecut", "penakut", "plin-plan", "labil", "egois", "keegoisan", "sombong", "kesombongan", "angkuh", "arogan", "sok", "songong", "kurang ajar", "tidak becus", "amburadul", "berantakan", "semrawut", "menyengsarakan", "represif", "represi", "diskriminasi", "mendiskriminasi", "rasis", "rasisme", "intoleran", "intoleransi", "penjilat", "gila", "kontol", "memek", "paok", "stress", "goblog", "kepala batu", "oon", } # Flips the polarity of a sentiment word found within NEGATION_WINDOW tokens # after it ("tidak bagus" -> negative even though "bagus" alone is positive). # Indonesian often puts more distance between the negation and the word it # actually governs than English does — "ngga ngajarin cara ngmg yg sopan" # ("doesn't teach how to speak politely") is 5 tokens from negation to the # sentiment word it negates. A wider window catches more of those at the # cost of occasionally flipping something the negation wasn't really about; # lexicon scoring is a heuristic either way, this just picks which failure # mode to lean toward. NEGATION_WORDS = {"tidak", "tak", "bukan", "belum", "jangan", "nggak", "ga", "gak", "kagak", "ngga"} NEGATION_WINDOW = 5 # Boosts a sentiment word's weight rather than changing its polarity. INTENSIFIERS = {"sangat", "banget", "sekali", "sungguh", "amat", "terlalu", "sangatlah"} INTENSIFIER_MULTIPLIER = 1.5 STOPWORDS = { "yang", "dan", "di", "ke", "dari", "untuk", "dengan", "ini", "itu", "ya", "nya", "adalah", "akan", "saya", "kamu", "kita", "kami", "mereka", "dia", "juga", "saja", "sudah", "belum", "atau", "karena", "jika", "kalau", "agar", "supaya", "pada", "oleh", "dalam", "luar", "atas", "bawah", "antara", "seperti", "sebagai", "tentang", "bahwa", "namun", "tetapi", "tapi", "hingga", "sampai", "sejak", "setelah", "sebelum", "ketika", "saat", "ada", "punya", "milik", "lah", "kah", "pun", "deh", "dong", "sih", "kok", "loh", "nih", "gitu", "gini", "dsb", "dll", "dst", "yg", "utk", "dgn", "krn", "gak", "ga", "nggak", "tak", "tidak", "bukan", "jangan", "apa", "apakah", "siapa", "mengapa", "kenapa", "bagaimana", "dimana", "kapan", "para", "si", "sang", "an", "kan", "in", "the", "is", "are", "was", "were", "be", "been", "being", "to", "of", "for", "on", "with", "as", "by", "at", "an", "a", "rt", } | NEGATION_WORDS | INTENSIFIERS _WORD_RE = re.compile(r"[a-zA-ZÀ-ÿ]+(?:-[a-zA-ZÀ-ÿ]+)?") _URL_RE = re.compile(r"https?://\S+") _MENTION_RE = re.compile(r"@\w+") def _tokenize(text: str) -> list[str]: """Lowercased word tokens with URLs/@mentions stripped first (both would otherwise pollute the lexicon match and the word cloud with usernames/ link fragments neither list has any business scoring).""" if not text: return [] cleaned = _URL_RE.sub(" ", text) cleaned = _MENTION_RE.sub(" ", cleaned) return [w.lower() for w in _WORD_RE.findall(cleaned)] def score_text(text: str) -> dict: """Returns {label, score, matches} for one piece of text. label is one of "pro" / "neutral" / "con". matches lists (word, polarity, weight) for every lexicon hit, so a classification can be inspected rather than trusted blindly — a defining trait of a rule-based classifier is that you CAN see exactly why it decided what it decided.""" tokens = _tokenize(text) score = 0.0 matches = [] for i, tok in enumerate(tokens): polarity = 1 if tok in POSITIVE_WORDS else -1 if tok in NEGATIVE_WORDS else 0 if polarity == 0: continue weight = 1.0 # Negation: any negation word in the preceding window flips polarity. window_start = max(0, i - NEGATION_WINDOW) if any(t in NEGATION_WORDS for t in tokens[window_start:i]): polarity = -polarity # Intensifier: any intensifier immediately before boosts magnitude # (checked after negation so "tidak sangat bagus" still flips first). if i > 0 and tokens[i - 1] in INTENSIFIERS: weight = INTENSIFIER_MULTIPLIER contribution = polarity * weight score += contribution matches.append({"word": tok, "polarity": "pro" if polarity > 0 else "con", "weight": weight}) label = "pro" if score > 0 else "con" if score < 0 else "neutral" return {"label": label, "score": round(score, 2), "matches": matches} def _item_text(item: dict) -> str: """The text worth scoring/tokenizing for a given archived record — varies by which tool produced it (a tweet's own text vs. a Wayback/CSE page's scraped title+description vs. a bare user's bio).""" parts = [ item.get("text"), item.get("full_text"), item.get("article_text"), item.get("post_title"), item.get("post_text"), item.get("description"), ] return " ".join(p for p in parts if p) def _item_author(item: dict) -> str | None: return item.get("screen_name") or item.get("user") or item.get("name") def _item_engagement(item: dict) -> int: total = 0 for k in ("reply_count", "retweet_count", "favorite_count"): v = item.get(k) if isinstance(v, (int, float)): total += v return total def word_frequencies(items: list[dict], top_n: int = 60) -> list[dict]: """Word-cloud data: [{word, count}], most frequent first. Stopwords and single-character tokens are dropped; everything else counts regardless of whether it happened to be in the sentiment lexicon.""" counts = Counter() for item in items: for tok in _tokenize(_item_text(item)): if len(tok) < 3 or tok in STOPWORDS: continue counts[tok] += 1 return [{"word": w, "count": c} for w, c in counts.most_common(top_n)] def top_users(items: list[dict], top_n: int = 20) -> list[dict]: """Who shows up most often across the archive — every record with an identifiable author counts once, regardless of whether it's a tweet, a reply, a retweeter entry, or a bare follower/following record. Carries along the most recently seen avatar/name for that handle so the dashboard can show a face, not just a bare count.""" counts: Counter = Counter() display: dict[str, dict] = {} for item in items: handle = item.get("screen_name") or item.get("user") if not handle: continue counts[handle] += 1 display[handle] = { "screen_name": handle, "name": item.get("name"), "avatar": item.get("avatar") or item.get("user_avatar"), "verified": item.get("verified"), "is_blue_verified": item.get("is_blue_verified"), } ranked = [] for handle, count in counts.most_common(top_n): ranked.append({**display[handle], "count": count}) return ranked def top_engagement(items: list[dict], top_n: int = 15) -> list[dict]: """Which posts drove the most reply+retweet+favorite activity — "paling ramai" (busiest/most-discussed), not just most recent.""" scored = [(_item_engagement(it), it) for it in items if _item_text(it)] scored.sort(key=lambda pair: pair[0], reverse=True) return [{"engagement": score, "item": it} for score, it in scored[:top_n] if score > 0] # ── ML backend (preferred) ─────────────────────────────────────────────────── # Lazy-loaded: importing torch/transformers and loading ~1.1GB of weights is # slow, and both packages are optional (requirements.txt notes how to add # them) — doing this at module import time would slow down every single use # of this app, including ones that never touch analytics, and would hard-crash # an install that skipped the ML deps entirely instead of just falling back. _ML_MODEL_NAME = "cardiffnlp/twitter-xlm-roberta-base-sentiment" _ML_LABEL_MAP = {"positive": "pro", "negative": "con", "neutral": "neutral"} _ml_pipeline = None _ml_unavailable = False # sticky once loading fails — stop retrying every call _ml_load_lock = threading.Lock() def _get_ml_pipeline(): global _ml_pipeline, _ml_unavailable if _ml_pipeline is not None or _ml_unavailable: return _ml_pipeline with _ml_load_lock: if _ml_pipeline is not None or _ml_unavailable: # re-check post-lock return _ml_pipeline try: from transformers import pipeline _ml_pipeline = pipeline("sentiment-analysis", model=_ML_MODEL_NAME) except Exception: # Missing torch/transformers, no internet for the first-time # model download, out of memory, ... any of these should fall # back to the lexicon scorer rather than take the whole # analytics endpoint down. _ml_unavailable = True _ml_pipeline = None return _ml_pipeline def _score_texts_ml(texts: list[str]) -> list[dict] | None: """Batch-scores every text in one call (far faster on CPU than one pipeline call per item). Returns None if the model isn't available, so the caller falls back to the lexicon scorer instead. `score` is signed (positive for pro, negative for con, 0 for neutral) to match the lexicon backend's convention; `confidence` carries the model's own unsigned probability for the label it picked.""" clf = _get_ml_pipeline() if clf is None: return None raw = clf(texts, truncation=True, batch_size=16) results = [] for r in raw: label = _ML_LABEL_MAP.get(str(r.get("label", "")).lower(), "neutral") confidence = float(r.get("score", 0.0)) signed = confidence if label == "pro" else -confidence if label == "con" else 0.0 results.append({ "label": label, "score": round(signed, 3), "confidence": round(confidence, 3), "matches": [], }) return results def analyze(items: list[dict]) -> dict: """Full analytics payload for one archive's worth of raw items.""" if not isinstance(items, list): items = [items] # archive.py's own _run() passes non-dict entries through as-is rather # than dropping them (see its "if not isinstance(item, dict)" branch), # so a saved archive can legitimately contain a stray non-dict item — # every function below assumes dict.get(), so those get filtered here # once rather than each helper needing its own isinstance guard. items = [it for it in items if isinstance(it, dict)] text_items, texts = [], [] for item in items: text = _item_text(item) if text: # skip e.g. a bare follower/following record with no post text of its own text_items.append(item) texts.append(text) ml_results = _score_texts_ml(texts) if texts else None method = "ml" if ml_results is not None else "lexicon" if ml_results is None: ml_results = [score_text(t) for t in texts] sentiment_counts = {"pro": 0, "neutral": 0, "con": 0} scored_items = [] for item, text, result in zip(text_items, texts, ml_results): sentiment_counts[result["label"]] += 1 scored_items.append({ "label": result["label"], "score": result["score"], "confidence": result.get("confidence"), "matches": result.get("matches", []), "author": _item_author(item), "text": text, "item": item, }) total = len(scored_items) return { "method": method, "total_items": len(items), "total_scored": total, "sentiment_counts": sentiment_counts, "sentiment_pct": { k: round(v / total * 100, 1) if total else 0.0 for k, v in sentiment_counts.items() }, "scored_items": scored_items, "word_freq": word_frequencies(items), "top_users": top_users(items), "top_engagement": top_engagement(items), }