update twitter scrapper get detail about account and fix rate limiting

Script/SOCMINT-Twitter/Readme.md

@@ -25,6 +25,8 @@
 19. Checkpoint or save data 
 20. Auto resume archive and dump data with state (state data to checkpoint)
 21. Add date or timestamp paramater for all search module 
+22. Update logic rate limit for X
+23. Update add more paramater about account e.g account base location, total change username and etc 
 
 ## Features
 

Script/SOCMINT-Twitter/app.py

@@ -27,6 +27,7 @@ from cookie_client import (
     cookie_tweet_replies,
     cookie_tweet_retweeters,
     cookie_geo_search,
+    cookie_about_account,
     CookieClientError,
 )
 
@@ -516,6 +517,7 @@ def run_tool():
         data = enrich_account_age(data)
         data = _stamp_fetched_at(data)
         data = _stamp_tweet_url(data)
+        data = _enrich_about_profile(data)
 
         resp = {"ok": True, "data": data, "nextCursor": next_cursor}
         if source_errors:
@@ -774,6 +776,98 @@ def cases_page():
 # A blocking request for that long leaves the browser with nothing to show
 # but a static spinner and no way to tell "still working" from "stuck."
 
+_about_cache: dict[str, dict | None] = {}   # screen_name -> about_profile or None (never rate-limited entries)
+_about_lock  = threading.Lock()
+_ABOUT_CACHE_PATH = Path(__file__).parent / "about_cache.json"
+
+_MAX_ABOUT_ENRICH   = 20    # max unique users enriched per request
+_ABOUT_FETCH_DELAY  = 3.0   # seconds between sequential AboutAccountQuery calls / chnge this for evade rate limit X
+
+
+def _load_about_cache() -> None:
+    try:
+        if _ABOUT_CACHE_PATH.exists():
+            with open(_ABOUT_CACHE_PATH, "r", encoding="utf-8") as f:
+                data = json.load(f)
+            with _about_lock:
+                _about_cache.update(data)
+    except Exception:
+        pass
+
+
+def _save_about_cache() -> None:
+    try:
+        with _about_lock:
+            snapshot = dict(_about_cache)
+        with open(_ABOUT_CACHE_PATH, "w", encoding="utf-8") as f:
+            json.dump(snapshot, f, ensure_ascii=False, separators=(",", ":"))
+    except Exception:
+        pass
+
+
+_load_about_cache()
+
+
+def _enrich_about_profile(items: list) -> list:
+    """Add account_based_in, connected_via, username_changes to each item via AboutAccountQuery.
+
+    Fetches sequentially (0.8s delay) to stay under rate limits. Results are cached
+    to about_cache.json across restarts. Rate-limited accounts are NOT cached and
+    will be retried on the next search."""
+    if not isinstance(items, list):
+        return items
+    try:
+        auth, ct0 = config.get("twitter_cookies", "auth_token", fallback="").strip(), \
+                    config.get("twitter_cookies", "ct0", fallback="").strip()
+        if not auth or not ct0:
+            return items
+    except Exception:
+        return items
+
+    needed: list[str] = []
+    with _about_lock:
+        for item in items:
+            if not isinstance(item, dict):
+                continue
+            sn = item.get("user") or item.get("screen_name")
+            if sn and sn not in _about_cache and sn not in needed:
+                needed.append(sn)
+                if len(needed) >= _MAX_ABOUT_ENRICH:
+                    break
+
+    for i, sn in enumerate(needed):
+        if i > 0:
+            time.sleep(_ABOUT_FETCH_DELAY)
+        try:
+            result = cookie_about_account(sn, config=config)
+            with _about_lock:
+                _about_cache[sn] = result
+        except Exception as e:
+            if "TooManyRequests" in type(e).__name__ or "429" in str(e):
+                pass  # rate limited — skip cache so it's retried next time
+            else:
+                with _about_lock:
+                    _about_cache[sn] = None
+
+    if needed:
+        _save_about_cache()
+
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        sn = item.get("user") or item.get("screen_name")
+        if not sn:
+            continue
+        with _about_lock:
+            about = _about_cache.get(sn)
+        if about:
+            item["account_based_in"]  = about.get("account_based_in")
+            item["connected_via"]     = about.get("source")
+            item["username_changes"]  = (about.get("username_changes") or {}).get("count")
+
+    return items
+
+
 _analytics_registry: dict[str, dict] = {}   # archive_id -> job status dict
 _analytics_lock = threading.Lock()
 
@@ -799,6 +893,26 @@ def _run_analytics(archive_id: str, items: list) -> None:
             }
 
 
+@app.route("/api/about_account")
+def about_account():
+    screen_name = request.args.get("screen_name", "").strip()
+    if not screen_name:
+        return jsonify({"ok": False, "error": "screen_name required"}), 400
+    with _about_lock:
+        cached = _about_cache.get(screen_name, "MISS")
+    if cached != "MISS":
+        return jsonify({"ok": True, "about_profile": cached})
+    try:
+        result = cookie_about_account(screen_name, config=config)
+    except CookieClientError as e:
+        return jsonify({"ok": False, "error": str(e)}), 400
+    except Exception as e:
+        return jsonify({"ok": False, "error": str(e)}), 500
+    with _about_lock:
+        _about_cache[screen_name] = result
+    return jsonify({"ok": True, "about_profile": result})
+
+
 @app.route("/analytics")
 def analytics_viewer():
     return render_template("analytics.html")

Script/SOCMINT-Twitter/config.ini.example

@@ -27,7 +27,6 @@ auth_token = xxxxxxxxxxxxxxxxxx
 ct0 = xxxxxxxxxxxx
 
 [server]
-
 ; config flask 
 
 host = 127.0.0.1

Script/SOCMINT-Twitter/cookie_client.py

@@ -176,7 +176,6 @@ def _tweet_to_dict(t: object) -> dict:
         "text":           _full_text(t) or getattr(t, "text", None),
         "user":           getattr(user_obj, "screen_name", None) if user_obj else None,
         "user_id":        _id_str(getattr(user_obj, "id", None)) if user_obj else None,
-        "user_location":  getattr(user_obj, "location", None) if user_obj else None,
         # Display name + verification badge — Twitter's own reply UI shows both
         # next to the handle; twikit already exposes them on the tweet's user.
         "name":                 getattr(user_obj, "name", None) if user_obj else None,
@@ -371,8 +370,7 @@ async def _tweet_retweeters_async(tweet_id: str, auth_token: str, ct0: str, coun
 
 
 async def _geo_search_async(keyword: str, auth_token: str, ct0: str, count: int, cursor: str | None) -> tuple[list, str | None]:
-    """Keyword search; user_location (profile location string) is included in every
-    result so the frontend can geocode and plot it on a map."""
+    """Keyword search results; account_based_in is added later by _enrich_about_profile for map geocoding."""
     client  = await _make_client(auth_token, ct0)
     results = await client.search_tweet(keyword, "Latest", count=count, cursor=cursor)
     return [_tweet_to_dict(t) for t in results], _next_cursor(results)
@@ -441,6 +439,33 @@ def cookie_geo_search(
     return asyncio.run(_geo_search_async(keyword, auth, ct0, count, cursor))
 
 
+_ABOUT_ACCOUNT_URL = "https://x.com/i/api/graphql/TzOG2twZEfhr9KmClvVVqA/AboutAccountQuery"
+
+
+async def _about_account_async(screen_name: str, auth_token: str, ct0: str) -> dict | None:
+    from twikit.errors import TooManyRequests
+    client = await _make_client(auth_token, ct0)
+    try:
+        resp, _ = await client.gql.gql_get(_ABOUT_ACCOUNT_URL, {"screenName": screen_name}, {})
+        return resp["data"]["user_result_by_screen_name"]["result"].get("about_profile")
+    except TooManyRequests:
+        raise  # let caller decide — do not cache as permanent None
+    except Exception:
+        return None
+
+
+def cookie_about_account(screen_name: str, config: configparser.ConfigParser = None) -> dict | None:
+    """Fetch X's 'Account based in' and 'Connected via' data for a user.
+
+    Returns a dict with keys: account_based_in, source, location_accurate,
+    created_country_accurate, username_changes (count), learn_more_url.
+    Returns None if the account has no about_profile or on error.
+    """
+    cfg = config or load_config()
+    auth, ct0 = _get_creds(cfg)
+    return asyncio.run(_about_account_async(screen_name, auth, ct0))
+
+
 # Legacy alias — kept for any external scripts that import this name directly
 fetch_user_timeline = cookie_post_extractor
 

Script/SOCMINT-Twitter/static/js/card_constants.js

@@ -17,7 +17,8 @@ const PRIORITY = [
   'created_at', 'fetched_at', 'in_reply_to_tweet_id',
   'retweeted_by_user', 'retweeted_by_name', 'retweeted_text', 'retweeted_by_bio', 'retweeted_at', 'retweeted_tweet_id',
   'quoted_user', 'quoted_name', 'quoted_text', 'quoted_at', 'quoted_tweet_id',
-  'lat', 'lon', 'place', 'user_location',
+  'account_based_in', 'connected_via', 'username_changes',
+  'lat', 'lon', 'place',
   'tweet_url', 'archive_url', 'result_url', 'preview_image', 'display_link',
   'serp_title',
   'iso_date', 'original', 'statuscode', 'mimetype', 'length',

Script/SOCMINT-Twitter/templates/_field_glossary.html

@@ -38,7 +38,9 @@
 
 <div class="param-heading">Location</div>
 <div class="param-row"><code class="param-key">lat / lon / place</code> Coordinates plotted on the map view (Geo Post Search)</div>
-<div class="param-row"><code class="param-key">user_location</code> Free-text profile location string, geocoded client-side to produce lat/lon</div>
+<div class="param-row"><code class="param-key">account_based_in</code> X-inferred country/region where the account is operated from — determined by X from IP, phone number, and payment signals at registration and ongoing login. Can still be wrong if the user consistently uses a VPN or registered with a foreign SIM. Used as the signal for map geocoding</div>
+<div class="param-row"><code class="param-key">connected_via</code> Platform/store X detected as the account's primary connection source — e.g. "Indonesia App Store", "Southeast Asia Android App", "Web App". Sourced from X's <code class="param-key">AboutAccountQuery</code> endpoint, same data shown on x.com/username/about</div>
+<div class="param-row"><code class="param-key">username_changes</code> Number of times the account has changed its @handle — from X's <code class="param-key">AboutAccountQuery</code> endpoint. A high count can indicate identity-shifting behavior; 0 means the handle has never changed since account creation</div>
 
 <div class="param-heading">Account age (forensics)</div>
 <div class="param-row"><code class="param-key">account_created / account_age</code> Derived from the account's numeric ID, not the API — see the Account age badges above</div>

Script/SOCMINT-Twitter/templates/graph.html

@@ -1844,7 +1844,8 @@ async function expandNode(expandTool) {
 // makes a reply's author "look-up-able": select them afterward and Expand
 // Posts/Followers/Following works on them like any other user node.
 var AUTHOR_REMAP = { user_avatar: 'avatar', user_banner: 'banner', user_bio: 'description' };
-var AUTHOR_PASSTHROUGH = ['name', 'verified', 'is_blue_verified', 'user_location',
+var AUTHOR_PASSTHROUGH = ['name', 'verified', 'is_blue_verified',
+  'account_based_in', 'connected_via', 'username_changes',
   'account_created', 'account_age', 'account_age_flag', 'account_age_precision',
   'source', 'fetched_at'];
 
@@ -1949,7 +1950,7 @@ var PRIORITY_KEYS = [
   'quoted_user', 'quoted_name', 'quoted_text', 'quoted_at', 'quoted_tweet_id',
   'reply_count', 'retweet_count', 'favorite_count', 'view_count',
   'followers_count', 'following_count', 'tweet_count',
-  'description', 'user_location', 'in_reply_to_tweet_id',
+  'description', 'account_based_in', 'connected_via', 'username_changes', 'in_reply_to_tweet_id',
   'retweeted_tweet_id', 'retweeted_at', 'verified', 'is_blue_verified',
   'archive_url', 'result_url', 'preview_image', 'display_link', 'serp_title',
   'iso_date', 'original', 'statuscode',

Script/SOCMINT-Twitter/templates/index.html

@@ -1973,16 +1973,16 @@ async function updateMap() {
   const mapStatus = document.getElementById('mapStatus');
   const items     = Array.isArray(currentData) ? currentData : (currentData ? [currentData] : []);
 
-  // Collect tweets that have a non-empty profile location
-  const withLoc = items.filter(i => i.user_location && i.user_location.trim());
+  const locOf  = i => (i.account_based_in || '').trim();
+  const withLoc = items.filter(i => locOf(i));
   if (!withLoc.length) {
-    mapStatus.textContent = 'No profile locations found in results — users may not have set a location.';
+    mapStatus.textContent = 'No location data found in results.';
     leafletMap.setView([20, 10], 2);
     return;
   }
 
   // Deduplicate locations to minimise Nominatim calls
-  const uniqueLocs = [...new Set(withLoc.map(i => i.user_location.trim()))];
+  const uniqueLocs = [...new Set(withLoc.map(i => locOf(i)))];
   mapStatus.textContent = `Geocoding 0 / ${uniqueLocs.length} locations…`;
 
   let done = 0;
@@ -1995,7 +1995,7 @@ async function updateMap() {
   // Place a marker for every tweet whose location resolved
   let placed = 0;
   for (const item of withLoc) {
-    const loc    = item.user_location.trim();
+    const loc    = locOf(item);
     const coords = geoCache.get(loc.toLowerCase().trim());
     if (!coords) continue;