OSINT-Cheat-sheet/Script/SOCMINT-Twitter/sentiment.py

"""Sentiment clustering (pro / neutral / con) for archived X/Twitter data,
plus the supporting "who's loudest, what's trending, what words dominate"
aggregates — the same shape of dashboard Drone Emprit-style tools give a
dataset.

Two scoring backends, tried in this order:

1. ML (preferred) — cardiffnlp/twitter-xlm-roberta-base-sentiment, an
   XLM-RoBERTa model fine-tuned for tweet sentiment across 8 languages
   (ar/en/fr/de/hi/it/pt/es). Its base pretraining covers ~100 languages, so
   it degrades gracefully rather than failing outright on a language outside
   that fine-tuning set — this is what makes the tool usable for an
   open-source audience that isn't Indonesian-only. Needs torch +
   transformers (see requirements.txt) and ~1.1GB of model weights
   downloaded from Hugging Face on first use.
2. Lexicon fallback — a hand-built Indonesian positive/negative word list
   with basic negation handling ("tidak bagus" flips "bagus" from positive
   to negative). Used automatically whenever torch/transformers aren't
   installed, so a lightweight install (just requirements.txt's base deps)
   still has a working — if Indonesian-only — sentiment feature rather than
   a hard failure. Every scored item exposes exactly which backend produced
   it (`method` on the analyze() payload), so a classification's provenance
   is never ambiguous, and the lexicon path additionally exposes the exact
   words that drove its score for full transparency.

Neither backend is a ground-truth classifier — short text, sarcasm, and
irony all degrade accuracy regardless of approach. Treat results as a
starting point for investigation, not a verdict.
"""

import re
import threading
from collections import Counter

# Local fallback 

# ── Sentiment lexicon ───────────────────────────────────────────────────────
# Indonesian words skew political/social-discourse (matches the kind of
# content this tool actually pulls — keyword searches on public affairs,
# government programs, public figures) as well as general register.
#
# Lexicon-parity note: an earlier version of this list ran ~130 positive vs.
# ~220 negative entries. That gap isn't neutral — with score_text() summing
# one point per matched word, a lexicon with substantially more negative
# coverage (more synonyms per concept: bohong/kebohongan/hoax/menipu/penipu/
# penipuan/tipu for one idea, "lying," vs. jujur/kejujuran for its opposite)
# structurally nudges mixed/ambiguous text toward "con" independent of the
# text's actual sentiment, simply because there's more negative surface area
# to match against. The additions below restore rough parity for the same
# governance/social-discourse register the negative list already covers
# (accountability, honesty, inclusion, rule of law) rather than padding with
# unrelated filler — pair each new word against the negative concept it
# offsets in a review. This is still a heuristic, not a bias-free scorer:
# see analyze()'s docstring and the module docstring above for the standing
# caveat that neither backend is ground truth.

# Arr data words 
# Need to feedback and research to sett the all parameter for each words 
# Need to help netizen Indo, with slang and words 

POSITIVE_WORDS = {
    "bagus", "baik", "hebat", "keren", "mantap", "mantul", "top", "terbaik",
    "sukses", "berhasil", "tepat", "benar", "setuju", "dukung", "dukungan",
    "mendukung", "apresiasi", "mengapresiasi", "bangga", "membanggakan",
    "senang", "gembira", "puas", "memuaskan", "bersyukur", "syukur",
    "alhamdulillah", "semoga", "maju", "kemajuan", "bijak", "bijaksana",
    "cerdas", "pintar", "amanah", "jujur", "kejujuran", "adil", "keadilan",
    "peduli", "kepedulian", "bermanfaat", "manfaat", "membantu", "bantuan",
    "solusi", "prestasi", "berprestasi", "unggul", "keunggulan", "luar biasa",
    "joss", "jos", "gas terus", "lanjutkan", "sip", "oke", "aman", "tenang",
    "damai", "sejahtera", "kesejahteraan", "makmur", "kemakmuran", "indah",
    "cantik", "tampan", "ramah", "sopan", "santun", "hormat", "menghormati",
    "salut", "kagum", "mengagumkan", "respect", "terharu", "terinspirasi",
    "inspiratif", "positif", "optimis", "optimisme", "harapan", "berharap",
    "cinta", "mencintai", "sayang", "suka", "menyukai", "rindu", "kangen",
    "gemas", "lucu", "menghibur", "menyenangkan", "menginspirasi", "tegas",
    "berani", "keberanian", "kuat", "gigih", "semangat", "bersemangat",
    "sukacita", "kompeten", "profesional", "berkualitas", "kualitas",
    "elegan", "canggih", "inovatif", "inovasi", "kreatif", "efisien",
    "efektif", "transparan", "transparansi", "akuntabel", "akuntabilitas",
    "merakyat", "membela rakyat", "pro rakyat", "berpihak pada rakyat",
    "terpuji", "membanggakan", "gemilang", "cemerlang", "berkah", "istimewa",
    # Governance/social-discourse counterparts added for lexicon parity
    # (offsetting korupsi/nepotisme/kkn, otoriter/diktator/fasis/represif,
    # rasis/intoleran, bohong/hoax/menipu, and pelanggaran/ilegal below).
    "bersih", "antikorupsi", "berintegritas", "integritas", "kredibel",
    "kredibilitas", "terpercaya", "dapat dipercaya", "netral", "imparsial",
    "objektif", "demokratis", "reformasi", "reformis", "inklusif",
    "inklusi", "toleran", "toleransi", "egaliter", "partisipatif",
    "aspiratif", "taat hukum", "patuh hukum", "sesuai aturan", "legal",
    "sah", "melindungi", "perlindungan", "membangun", "pembangunan",
    "sinergi", "berkolaborasi", "kolaboratif", "harmonis", "kondusif",
    "stabil", "stabilitas", "humanis", "empati", "berempati", "rendah hati",
    "dermawan",
}

NEGATIVE_WORDS = {
    "buruk", "keburukan", "jelek", "gagal", "kegagalan", "bodoh", "tolol",
    "goblok", "bego", "dungu", "idiot", "korupsi", "korup", "koruptor",
    "bohong", "kebohongan", "hoax", "menipu", "penipu", "penipuan", "tipu",
    "curang", "kecurangan", "culas", "zalim", "menzalimi", "kejam",
    "kekejaman", "jahat", "kejahatan", "rusak", "merusak", "hancur",
    "menghancurkan", "kacau", "mengacaukan", "parah", "memalukan",
    "memuakkan", "bejat", "biadab", "tolak", "menolak", "penolakan",
    "kecewa", "mengecewakan", "kekecewaan", "marah", "kemarahan", "murka",
    "benci", "membenci", "kebencian", "muak", "jijik", "menjijikkan",
    "sampah", "anjing", "bangsat", "bajingan", "kampret", "sialan",
    "kacung", "boneka", "munafik", "kemunafikan", "pengkhianat",
    "mengkhianati", "khianat", "pengkhianatan", "penjajah", "menjajah",
    "licik", "licin", "serakah", "keserakahan", "tamak", "otoriter",
    "diktator", "fasis", "salah", "kesalahan", "blunder", "konyol",
    "ngawur", "ngaco", "absurd", "aneh", "ironis", "ironi", "tragis",
    "miris", "prihatin", "keprihatinan", "sedih", "menyedihkan", "susah",
    "kesusahan", "sulit", "kesulitan", "sengsara", "menderita",
    "penderitaan", "korban", "dizalimi", "ditindas", "menindas",
    "penindasan", "kriminal", "pelanggaran", "melanggar", "ilegal",
    "pungli", "sogok", "menyuap", "disuap", "suap", "nepotisme", "kkn",
    "provokasi", "provokator", "memprovokasi", "fitnah", "memfitnah",
    "ancaman", "mengancam", "teror", "intimidasi", "mengintimidasi",
    "brutal", "kekerasan", "sadis", "tragedi", "bencana", "krisis",
    "darurat", "gawat", "resah", "keresahan", "meresahkan", "cemas",
    "kecemasan", "khawatir", "kekhawatiran", "takut", "ketakutan", "geram",
    "kesal", "jengkel", "dongkol", "malu", "hina", "menghina",
    "penghinaan", "murahan", "norak", "kampungan", "terbelakang", "mundur",
    "kemunduran", "ambruk", "bangkrut", "kebangkrutan", "defisit",
    "terlilit", "terjerat", "terjebak", "cengeng", "lemah", "kelemahan",
    "pengecut", "penakut", "plin-plan", "labil", "egois", "keegoisan",
    "sombong", "kesombongan", "angkuh", "arogan", "sok", "songong",
    "kurang ajar", "tidak becus", "amburadul", "berantakan", "semrawut",
    "menyengsarakan", "represif", "represi", "diskriminasi", "ruwet",
    "mendiskriminasi", "rasis", "rasisme", "intoleran", "intoleransi",
    "penjilat", "kontol", "memek", "paok", "stress", "goblog", "kontlo",
    "kepala batu", "oon", "bacot", "asu", "gijil", "jembut", "kanjut",
    "ngentot", "puki", "meki", "jembot", "pukimak", "kimak", "tembelek", 
    "tai", "bacod", "telaso", "dongo", "pauk", "jemboot", "Komdongo", 
    "komintod", "komintol", "cabul", "pencabulan", "omdo", "kunyuk",
    "munyuk", "monyet", "nyet", "jawir", "j4wir", "j4w1r", "kentu",
    "kenthu", "komdonggo", "komdungu", "kolot", "wowok", "cok",  
    "cokil", "jancok", "jiancuk", "jancook", "ancok", "ancook", 
    "coli", "nyoli", "oten", "0ten", "ten oten", "kadrun", "drun",
    "k4drun", "bgst", "bangset", "jingan", "bgsd", "antek2", "antek",
    "carut marut", "parcok", "parjo", "BUZZER", "buzzer", "buzer", 
    "kimbek", "bodat", "lapet", "muncung kau", "pukimai", "babi",
    "setan", "khuontol", "khontol", "ngacau", "gemblung", "ancur",
    "anjj", "anj", "wowi", "owi", "owok", "wok", "suram", "ecek2",
    "ecek ecek", "anak abah", "najis", "bunted", "modar", "modyar",
    "lud4h", "gblk", "zionis", "laknat", "provokasi", "provokator",
    "zionist", "bodo", "ludahi", "ludahin", "penjajah", "pajet", 
    "pajeet", "cemoohan", "cemooh", "rusuh", "barbar", "paj3t", 
    "cuih", "ngibul", "boong", "pekok", "pea", "pantek", "pantat",
    "Anak haram", "asbun", "omon-omon", "omon2", "ndasmu", "Endasmu",
    "Nyenyenye",
}

# Flips the polarity of a sentiment word found within NEGATION_WINDOW tokens
# after it ("tidak bagus" -> negative even though "bagus" alone is positive).
# Indonesian often puts more distance between the negation and the word it
# actually governs than English does — "ngga ngajarin cara ngmg yg sopan"
# ("doesn't teach how to speak politely") is 5 tokens from negation to the
# sentiment word it negates. A wider window catches more of those at the
# cost of occasionally flipping something the negation wasn't really about;
# lexicon scoring is a heuristic either way, this just picks which failure
# mode to lean toward.
NEGATION_WORDS = {"tidak", "tak", "bukan", "belum", "jangan", "nggak", "ga", "gak", "kagak", "ngga"}
NEGATION_WINDOW = 5

# Boosts a sentiment word's weight rather than changing its polarity.
INTENSIFIERS = {"sangat", "banget", "sekali", "sungguh", "amat", "terlalu", "sangatlah"}
INTENSIFIER_MULTIPLIER = 1.5

STOPWORDS = {
    "yang", "dan", "di", "ke", "dari", "untuk", "dengan", "ini", "itu", "ya",
    "nya", "adalah", "akan", "saya", "kamu", "kita", "kami", "mereka", "dia",
    "juga", "saja", "sudah", "belum", "atau", "karena", "jika", "kalau",
    "agar", "supaya", "pada", "oleh", "dalam", "luar", "atas", "bawah",
    "antara", "seperti", "sebagai", "tentang", "bahwa", "namun", "tetapi",
    "tapi", "hingga", "sampai", "sejak", "setelah", "sebelum", "ketika",
    "saat", "ada", "punya", "milik", "lah", "kah", "pun", "deh", "dong",
    "sih", "kok", "loh", "nih", "gitu", "gini", "dsb", "dll", "dst", "yg",
    "utk", "dgn", "krn", "gak", "ga", "nggak", "tak", "tidak", "bukan",
    "jangan", "apa", "apakah", "siapa", "mengapa", "kenapa", "bagaimana",
    "dimana", "kapan", "para", "si", "sang", "an", "kan", "in", "the", "is",
    "are", "was", "were", "be", "been", "being", "to", "of", "for", "on",
    "with", "as", "by", "at", "an", "a", "rt",
} | NEGATION_WORDS | INTENSIFIERS

_WORD_RE  = re.compile(r"[a-zA-ZÀ-ÿ]+(?:-[a-zA-ZÀ-ÿ]+)?")
_URL_RE   = re.compile(r"https?://\S+")
_MENTION_RE = re.compile(r"@\w+")


def _tokenize(text: str) -> list[str]:
    """Lowercased word tokens with URLs/@mentions stripped first (both would
    otherwise pollute the lexicon match and the word cloud with usernames/
    link fragments neither list has any business scoring)."""
    if not text:
        return []
    cleaned = _URL_RE.sub(" ", text)
    cleaned = _MENTION_RE.sub(" ", cleaned)
    return [w.lower() for w in _WORD_RE.findall(cleaned)]


def score_text(text: str) -> dict:
    """Returns {label, score, matches} for one piece of text. label is one of
    "pro" / "neutral" / "con". matches lists (word, polarity, weight) for
    every lexicon hit, so a classification can be inspected rather than
    trusted blindly — a defining trait of a rule-based classifier is that you
    CAN see exactly why it decided what it decided."""
    tokens  = _tokenize(text)
    score   = 0.0
    matches = []

    for i, tok in enumerate(tokens):
        polarity = 1 if tok in POSITIVE_WORDS else -1 if tok in NEGATIVE_WORDS else 0
        if polarity == 0:
            continue

        weight = 1.0
        # Negation: any negation word in the preceding window flips polarity.
        window_start = max(0, i - NEGATION_WINDOW)
        if any(t in NEGATION_WORDS for t in tokens[window_start:i]):
            polarity = -polarity
        # Intensifier: any intensifier immediately before boosts magnitude
        # (checked after negation so "tidak sangat bagus" still flips first).
        if i > 0 and tokens[i - 1] in INTENSIFIERS:
            weight = INTENSIFIER_MULTIPLIER

        contribution = polarity * weight
        score += contribution
        matches.append({"word": tok, "polarity": "pro" if polarity > 0 else "con", "weight": weight})

    label = "pro" if score > 0 else "con" if score < 0 else "neutral"
    return {"label": label, "score": round(score, 2), "matches": matches}


def _item_text(item: dict) -> str:
    """The text worth scoring/tokenizing for a given archived record —
    varies by which tool produced it (a tweet's own text vs. a Wayback/CSE
    page's scraped title+description).

    `description` is deliberately NOT pulled from a bare user record (a
    follower/following/retweeter entry — cookie_client.py's _user_to_dict
    always sets `followers_count`, even to None, which no tweet/CSE/Wayback
    record ever carries, so that key's mere presence identifies the shape
    reliably). For a CSE result, `description` is Google's own snippet of
    the matched page — genuinely relevant text. For a user record it's the
    account's own bio, which says nothing about the search topic; scoring
    "suka kucing dan kopi ☕" as pro/con toward whatever was searched would
    just be noise. Those accounts are still kept for clustering/leaderboard
    purposes (top_users() below runs over every item regardless of text) —
    they're just excluded from sentiment/word-cloud scoring specifically."""
    is_bare_user_record = "followers_count" in item
    parts = [
        item.get("text"), item.get("full_text"), item.get("article_text"),
        item.get("post_title"), item.get("post_text"),
        None if is_bare_user_record else item.get("description"),
    ]
    return " ".join(p for p in parts if p)


def _item_author(item: dict) -> str | None:
    return item.get("screen_name") or item.get("user") or item.get("name")


def _item_engagement(item: dict) -> int:
    total = 0
    for k in ("reply_count", "retweet_count", "favorite_count"):
        v = item.get(k)
        if isinstance(v, (int, float)):
            total += v
    return total


def word_frequencies(items: list[dict], top_n: int = 60) -> list[dict]:
    """Word-cloud data: [{word, count}], most frequent first. Stopwords and
    single-character tokens are dropped; everything else counts regardless
    of whether it happened to be in the sentiment lexicon."""
    counts = Counter()
    for item in items:
        for tok in _tokenize(_item_text(item)):
            if len(tok) < 3 or tok in STOPWORDS:
                continue
            counts[tok] += 1
    return [{"word": w, "count": c} for w, c in counts.most_common(top_n)]


def top_users(items: list[dict], top_n: int | None = None) -> list[dict]:
    """Who shows up most often across the archive — every record with an
    identifiable author counts once, regardless of whether it's a tweet, a
    reply, a retweeter entry, or a bare follower/following record. Carries
    along the most recently seen avatar/name for that handle so the
    dashboard can show a face, not just a bare count.

    top_n=None (the default) returns EVERY account, not just the busiest N
    — this backs a digital-evidence archive, and silently dropping which
    accounts even show up here isn't something an OSINT tool gets to do.
    Counter.most_common(None) already returns everything sorted, so this
    costs nothing when unset; the dashboard paces rendering via scroll
    instead (see analytics.html's TOP_LIST_BATCH), not by the backend ever
    truncating the data. Pass an explicit top_n only if some future caller
    genuinely wants a fixed-size top list instead."""
    counts: Counter = Counter()
    display: dict[str, dict] = {}
    for item in items:
        handle = item.get("screen_name") or item.get("user")
        if not handle:
            continue
        counts[handle] += 1
        display[handle] = {
            "screen_name": handle,
            "name":        item.get("name"),
            "avatar":      item.get("avatar") or item.get("user_avatar"),
            "verified":    item.get("verified"),
            "is_blue_verified": item.get("is_blue_verified"),
            # Same account-age fields id_forensics.py's enrich_account_age()
            # already stamps onto every raw item at the /api/run choke point
            # (app.py) — index.html/archive.html/graph.html already surface
            # these as a badge; the analytics dashboard just wasn't pulling
            # them through to its own account list yet.
            "account_created":       item.get("account_created"),
            "account_age":           item.get("account_age"),
            "account_age_flag":      item.get("account_age_flag"),
            # account_age_flag is None whenever id_forensics.py's
            # enrich_account_age() lands in its own "unknown" bucket (an id
            # just outside the calibrated range — see that module's
            # docstring) — account_age_precision is what tells the frontend
            # "this account really is unresolvable," as opposed to just
            # never having had an account id to enrich in the first place.
            "account_age_precision": item.get("account_age_precision"),
        }
    ranked = []
    for handle, count in counts.most_common(top_n):
        ranked.append({**display[handle], "count": count})
    return ranked


def top_engagement(items: list[dict], top_n: int | None = None) -> list[dict]:
    """Which posts drove the most reply+retweet+favorite activity — "paling
    ramai" (busiest/most-discussed), not just most recent.

    top_n=None (the default) returns every scored item with positive
    engagement, not just the busiest N — same reasoning as top_users()
    above: this is digital evidence, the backend doesn't get to decide
    which posts are worth showing. `scored[:None]` is the full list."""
    scored = [(_item_engagement(it), it) for it in items if _item_text(it)]
    scored.sort(key=lambda pair: pair[0], reverse=True)
    return [{"engagement": score, "item": it} for score, it in scored[:top_n] if score > 0]


# ── ML backend (preferred) ───────────────────────────────────────────────────
# Lazy-loaded: importing torch/transformers and loading ~1.1GB of weights is
# slow, and both packages are optional (requirements.txt notes how to add
# them) — doing this at module import time would slow down every single use
# of this app, including ones that never touch analytics, and would hard-crash
# an install that skipped the ML deps entirely instead of just falling back.
_ML_MODEL_NAME    = "cardiffnlp/twitter-xlm-roberta-base-sentiment"
_ML_LABEL_MAP     = {"positive": "pro", "negative": "con", "neutral": "neutral"}
_ml_pipeline      = None
_ml_unavailable   = False   # sticky once loading fails — stop retrying every call
_ml_load_lock     = threading.Lock()


def _get_ml_pipeline():
    global _ml_pipeline, _ml_unavailable
    if _ml_pipeline is not None or _ml_unavailable:
        return _ml_pipeline
    with _ml_load_lock:
        if _ml_pipeline is not None or _ml_unavailable:   # re-check post-lock
            return _ml_pipeline
        try:
            from transformers import pipeline
            _ml_pipeline = pipeline("sentiment-analysis", model=_ML_MODEL_NAME)
        except Exception:
            # Missing torch/transformers, no internet for the first-time
            # model download, out of memory, ... any of these should fall
            # back to the lexicon scorer rather than take the whole
            # analytics endpoint down.
            _ml_unavailable = True
            _ml_pipeline = None
    return _ml_pipeline


def warm_up_ml() -> None:
    """Loads the ML pipeline right now instead of waiting for the first real
    analytics request to trigger it lazily. Measured at ~15-20s the first
    time any process calls _get_ml_pipeline() (importing transformers,
    constructing the pipeline, reading the cached weights off disk) — vs.
    ~11-12ms/item for actual scoring once loaded. Without this, that whole
    one-time cost lands inside the FIRST user's analytics job, during which
    the progress bar has nothing to report yet (on_progress only fires once
    scoring itself starts) and just sits at 0/0 looking stuck. Meant to be
    called from a background thread at server startup (see app.py) — still
    completely safe to skip calling this at all, or to have the first real
    request race it, since _get_ml_pipeline() is lock-protected and
    idempotent either way; this is purely a warm-up, not a dependency."""
    _get_ml_pipeline()


_ML_PROGRESS_CHUNK = 64   # texts per pipeline call — see _score_texts_ml docstring


def _score_texts_ml(texts: list[str], on_progress=None) -> list[dict] | None:
    """Scores every text, chunked (rather than one giant pipeline call), so
    a caller running this in a background thread can report real progress —
    on a CPU this measures ~11-12ms/item (~1000 items ≈ 12s, ~5000 ≈ ~1min),
    linear with volume, so a large archive genuinely takes a while and a
    caller polling for status needs something better to show than a blind
    spinner. _ML_PROGRESS_CHUNK=64 batches (each itself pipelined
    batch_size=16 internally by HF) keeps ticks frequent enough to feel
    live (~0.7-0.8s apart) without paying per-call overhead for every
    single item. Returns None if the model isn't available, so the caller
    falls back to the lexicon scorer instead. `score` is signed (positive
    for pro, negative for con, 0 for neutral) to match the lexicon
    backend's convention; `confidence` carries the model's own unsigned
    probability for the label it picked."""
    clf = _get_ml_pipeline()
    if clf is None:
        return None

    results = []
    for start in range(0, len(texts), _ML_PROGRESS_CHUNK):
        chunk = texts[start:start + _ML_PROGRESS_CHUNK]
        # truncation=True alone is NOT enough here: it truncates to the
        # tokenizer's own model_max_length, which for this tokenizer's
        # shipped config is left at HF's "unset" sentinel (~1e30, i.e.
        # effectively no limit) rather than the model's real 512-token
        # capacity. A single long post (a fact-check thread, an
        # article-length tweet — anything past ~512 tokens once
        # subword-tokenized) then sails through "truncation" untruncated,
        # overflows the model's position-embedding table, and crashes the
        # whole batch with a raw RuntimeError ("index 514 is out of bounds
        # for dimension 1 with size 514" — 514 = 512 + the 2-position
        # offset RoBERTa-style embeddings use). max_length=512 forces the
        # real limit regardless of what the tokenizer config claims.
        raw = clf(chunk, truncation=True, max_length=512, batch_size=16)
        for r in raw:
            label      = _ML_LABEL_MAP.get(str(r.get("label", "")).lower(), "neutral")
            confidence = float(r.get("score", 0.0))
            signed     = confidence if label == "pro" else -confidence if label == "con" else 0.0
            results.append({
                "label": label, "score": round(signed, 3),
                "confidence": round(confidence, 3), "matches": [],
            })
        if on_progress:
            on_progress(len(results), len(texts))
    return results


def analyze(items: list[dict], on_progress=None) -> dict:
    """Full analytics payload for one archive's worth of raw items.
    on_progress, if given, is called as on_progress(scored_count,
    total_to_score) — zero or more times during ML scoring (chunked, see
    _score_texts_ml), and always at least once at the very end regardless
    of which backend actually ran, so a caller polling for status always
    sees a final 100%-done tick even on the lexicon path (fast enough that
    per-chunk progress wouldn't mean anything, but a job registry watching
    for "did this reach total" still needs that terminal call)."""
    if not isinstance(items, list):
        items = [items]
    # archive.py's own _run() passes non-dict entries through as-is rather
    # than dropping them (see its "if not isinstance(item, dict)" branch),
    # so a saved archive can legitimately contain a stray non-dict item —
    # every function below assumes dict.get(), so those get filtered here
    # once rather than each helper needing its own isinstance guard.
    items = [it for it in items if isinstance(it, dict)]

    text_items, texts = [], []
    for item in items:
        text = _item_text(item)
        if text:   # skip e.g. a bare follower/following record with no post text of its own
            text_items.append(item)
            texts.append(text)

    ml_results = _score_texts_ml(texts, on_progress=on_progress) if texts else None
    method     = "ml" if ml_results is not None else "lexicon"
    if ml_results is None:
        ml_results = [score_text(t) for t in texts]
        if on_progress:
            on_progress(len(texts), len(texts))

    sentiment_counts = {"pro": 0, "neutral": 0, "con": 0}
    scored_items = []
    for item, text, result in zip(text_items, texts, ml_results):
        sentiment_counts[result["label"]] += 1
        scored_items.append({
            "label":      result["label"],
            "score":      result["score"],
            "confidence": result.get("confidence"),
            "matches":    result.get("matches", []),
            "author":     _item_author(item),
            "text":       text,
            "item":       item,
        })

    total = len(scored_items)
    return {
        "method":            method,
        "total_items":       len(items),
        "total_scored":      total,
        "sentiment_counts":  sentiment_counts,
        "sentiment_pct": {
            k: round(v / total * 100, 1) if total else 0.0
            for k, v in sentiment_counts.items()
        },
        "scored_items":  scored_items,
        "word_freq":     word_frequencies(items),
        "top_users":     top_users(items),
        "top_engagement": top_engagement(items),
    }