diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/settings.po b/docs/source/locale/zh_CN/LC_MESSAGES/settings.po index 7c95b9e..f23f873 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/settings.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/settings.po @@ -440,3 +440,60 @@ msgstr "" #~ "``settings.json`` 中的 ``cloudflare_bypass`` 配置块用于配置" #~ " :ref:`cloudflare-bypass` 所述的可选绕过机制。默认值如下:" + +#: ../../source/settings.rst:107 +msgid "Activation token cache" +msgstr "激活令牌缓存" + +#: ../../source/settings.rst:110 +msgid "" +"A handful of sites reject anonymous requests until a short-lived token is " +"obtained: Twitter needs a guest token, Vimeo a JWT, Weibo and Wikimapia a " +"session cookie, OnlyFans a signed header pair, ProtonMail a bearer token. " +"Maigret mints these during a run, the first time such a site answers with a " +"challenge instead of a profile page." +msgstr "" +"少数站点会拒绝匿名请求,必须先获取一个短期令牌:Twitter 需要 guest token,Vimeo 需要 " +"JWT,Weibo 和 Wikimapia 需要会话 cookie,OnlyFans 需要一对签名请求头,ProtonMail " +"需要 bearer token。当这类站点返回的是验证页面而非个人主页时,Maigret 会在本次运行中即时获取这些令牌。" + +#: ../../source/settings.rst:112 +msgid "Minted values are cached in:" +msgstr "获取到的值会缓存在:" + +#: ../../source/settings.rst:118 +msgid "" +"so the next run starts with a warm token and skips the extra round trip. The" +" file is created with ``0600`` permissions because these tokens are session " +"credentials — treat it as one, and do not attach it to a bug report." +msgstr "" +"这样下次运行即可直接使用仍然有效的令牌,省去一次额外的往返请求。该文件以 ``0600`` " +"权限创建,因为这些令牌属于会话凭据——请按凭据对待,不要将其附在问题报告中。" + +#: ../../source/settings.rst:120 +msgid "" +"**What is stored:** only the header values that differ from what the site " +"database shipped. Storing the whole header set would pin the shipped values," +" so a later database update could never change them again." +msgstr "" +"**缓存的内容:** 仅保存与站点数据库自带值不同的请求头。若把整组请求头都存下来,就会把自带值固定住," +"以后的数据库更新将再也无法修改它们。" + +#: ../../source/settings.rst:122 +msgid "" +"**Safe to delete:** yes. The next run mints whatever it needs again; the " +"only cost is one extra request per affected site." +msgstr "" +"**可以安全删除:** 可以。下次运行会重新获取所需的令牌,代价仅仅是每个相关站点多一次请求。" + +#: ../../source/settings.rst:124 +msgid "" +"**Why it is not kept in the site database:** the database is package data. " +"It is versioned, checksum-verified and replaced wholesale by :ref:`database-" +"auto-update`, and on a system-wide install (distribution package, snap, " +"``/nix/store``) its directory is not writable at all. Tokens are per-user " +"state with a different lifetime, so they live in a separate file." +msgstr "" +"**为什么不存放在站点数据库中:** 数据库属于软件包自带数据,它有版本、会做校验和验证,并且会被 " +":ref:`database-auto-update` 整体替换;而在系统级安装(发行版软件包、snap、``/nix/store``)下," +"它所在的目录根本不可写。令牌是每个用户各自的状态,生命周期也不同,因此单独存放。" diff --git a/docs/source/settings.rst b/docs/source/settings.rst index d5a4871..08ba0dc 100644 --- a/docs/source/settings.rst +++ b/docs/source/settings.rst @@ -102,6 +102,27 @@ This is recommended for **Docker containers**, **CI pipelines**, and **air-gappe **Using a custom database** with ``--db`` always skips auto-update — you are explicitly choosing your data source. +.. _activation-token-cache: + +Activation token cache +---------------------- + +A handful of sites reject anonymous requests until a short-lived token is obtained: Twitter needs a guest token, Vimeo a JWT, Weibo and Wikimapia a session cookie, OnlyFans a signed header pair, ProtonMail a bearer token. Maigret mints these during a run, the first time such a site answers with a challenge instead of a profile page. + +Minted values are cached in: + +.. code-block:: console + + ~/.maigret/activation.json + +so the next run starts with a warm token and skips the extra round trip. The file is created with ``0600`` permissions because these tokens are session credentials — treat it as one, and do not attach it to a bug report. + +**What is stored:** only the header values that differ from what the site database shipped. Storing the whole header set would pin the shipped values, so a later database update could never change them again. + +**Safe to delete:** yes. The next run mints whatever it needs again; the only cost is one extra request per affected site. + +**Why it is not kept in the site database:** the database is package data. It is versioned, checksum-verified and replaced wholesale by :ref:`database-auto-update`, and on a system-wide install (distribution package, snap, ``/nix/store``) its directory is not writable at all. Tokens are per-user state with a different lifetime, so they live in a separate file. + Cloudflare webgate ------------------ diff --git a/maigret/activation.py b/maigret/activation.py index 12285dd..e949993 100644 --- a/maigret/activation.py +++ b/maigret/activation.py @@ -1,10 +1,15 @@ import json +import os +import os.path as path import re from http.cookiejar import MozillaCookieJar from http.cookies import Morsel +from typing import Dict from aiohttp import ClientSession, CookieJar +from .db_updater import MAIGRET_HOME + class ParsingActivator: @staticmethod @@ -197,3 +202,73 @@ def import_aiohttp_cookies(cookiestxt_filename): cookies.update_cookies(cookies_list) return cookies + + +ACTIVATION_CACHE_PATH = path.join(MAIGRET_HOME, "activation.json") + + +def _activation_sites(db): + return [site for site in db.sites if site.activation] + + +def load_activation_cache(db, logger) -> Dict[str, Dict[str, str]]: + """Apply per-user cached activation headers to the sites database. + + Tokens minted at runtime (guest tokens, JWTs, session cookies) are user + state, not package data, so they are kept in the user's home instead of + being written back into the shipped database — which is read-only on any + system-wide install. + + Returns the pre-overlay headers, needed by :func:`save_activation_cache` + to tell a minted value from one that simply came with the database. + """ + baseline = {site.name: dict(site.headers) for site in _activation_sites(db)} + + try: + with open(ACTIVATION_CACHE_PATH, encoding="utf-8") as f: + cache = json.load(f) + except FileNotFoundError: + return baseline + except (OSError, ValueError) as e: + logger.debug(f"Ignoring unreadable activation cache: {e}") + return baseline + + for site in _activation_sites(db): + cached = cache.get(site.name) + if isinstance(cached, dict): + # Assign rather than update: MaigretSite.headers defaults to a + # mutable class attribute shared by every site that doesn't + # declare its own, and updating it in place would leak these + # tokens onto all of them. + site.headers = {**site.headers, **cached} + + return baseline + + +def save_activation_cache(db, baseline: Dict[str, Dict[str, str]], logger) -> None: + """Persist only the header values that this run actually minted. + + Storing the full header set would freeze whatever the database shipped at + the time, so a later database update could no longer change those headers. + """ + cache = {} + for site in _activation_sites(db): + was = baseline.get(site.name, {}) + minted = {k: v for k, v in site.headers.items() if was.get(k) != v} + if minted: + cache[site.name] = minted + + if not cache: + return + + try: + os.makedirs(MAIGRET_HOME, exist_ok=True) + # These are session credentials (guest tokens, JWTs, cookies), so the + # file is created 0600 rather than inheriting the umask. + fd = os.open( + ACTIVATION_CACHE_PATH, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600 + ) + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(cache, f, indent=2, ensure_ascii=False) + except OSError as e: + logger.debug(f"Could not write the activation cache: {e}") diff --git a/maigret/maigret.py b/maigret/maigret.py index 6672651..fe894b6 100755 --- a/maigret/maigret.py +++ b/maigret/maigret.py @@ -54,6 +54,7 @@ from .report import ( save_markdown_report, ) from .result import SiteResult +from .activation import load_activation_cache, save_activation_cache from .sites import MaigretDatabase from .submit import Submitter from .utils import get_dict_ascii_tree, is_plausible_username @@ -569,6 +570,22 @@ def setup_arguments_parser(settings: Settings): return parser +def save_db_safely(db: MaigretDatabase, db_file: str, logger) -> bool: + """Persist the sites database, tolerating a read-only installation. + + The bundled database lives inside the package, so on a system-wide + install (distro package, snap, /nix/store) its directory belongs to + root or is mounted read-only. Returns False instead of raising, so a + run that already produced its report doesn't die on a cache write. + """ + try: + db.save_to_file(db_file) + return True + except OSError as e: + logger.debug(f'Could not write the database to {db_file}: {e}') + return False + + async def main(): # Logging log_level = logging.ERROR @@ -704,6 +721,9 @@ async def main(): ) else: raise + + activation_baseline = load_activation_cache(db, logger) + get_top_sites_for_id = lambda x: db.ranked_sites_dict( top=args.top_sites, tags=args.tags, @@ -718,8 +738,10 @@ async def main(): if args.new_site_to_submit: submitter = Submitter(db=db, logger=logger, settings=settings, args=args) is_submitted = await submitter.dialog(args.new_site_to_submit, args.cookie_file) - if is_submitted: - db.save_to_file(db_file) + if is_submitted and not save_db_safely(db, db_file, logger): + query_notify.warning( + f'The new site was not saved: {db_file} is not writable' + ) await submitter.close() # Database self-checking @@ -754,8 +776,10 @@ async def main(): 'y', '', ): - db.save_to_file(db_file) - print('Database was successfully updated.') + if save_db_safely(db, db_file, logger): + print('Database was successfully updated.') + else: + print(f'Database was not updated: {db_file} is not writable.') else: print('Updates will be applied only for current search session.') @@ -1112,8 +1136,7 @@ async def main(): except Exception as e: query_notify.warning(f'AI analysis failed: {e}') - # update database - db.save_to_file(db_file) + save_activation_cache(db, activation_baseline, logger) def run(): diff --git a/tests/test_activation.py b/tests/test_activation.py index 0483edb..7051fa2 100644 --- a/tests/test_activation.py +++ b/tests/test_activation.py @@ -1,6 +1,9 @@ """Maigret activation test functions""" import inspect +import json +import os + import yarl import aiohttp @@ -8,7 +11,14 @@ import pytest from unittest.mock import Mock from tests.conftest import LOCAL_SERVER_PORT -from maigret.activation import ParsingActivator, import_aiohttp_cookies +from maigret import activation as activation_cache +from maigret.activation import ( + ParsingActivator, + import_aiohttp_cookies, + load_activation_cache, + save_activation_cache, +) +from maigret.sites import MaigretDatabase, MaigretSite COOKIES_TXT = """# HTTP Cookie File downloaded with cookies.txt by Genuinous @genuinous # This file can be used by wget, curl, aria2c and other standard compliant tools. @@ -409,3 +419,139 @@ async def test_wikimapia_activation_no_token_leaves_cookie_untouched(): await ParsingActivator.wikimapia(site, Mock(), html="no challenge here") assert site.headers["Cookie"] == "verified=1" + + +@pytest.fixture +def activation_db(monkeypatch, tmp_path): + """Two sites, one that mints tokens at runtime and one that doesn't.""" + db = MaigretDatabase() + db.update_site( + MaigretSite( + 'Twitter', + { + 'url': 'https://twitter.com/{username}', + 'urlMain': 'https://twitter.com/', + 'activation': {'method': 'twitter', 'marks': ['nope']}, + 'headers': {'x-guest-token': 'from-db', 'accept': 'text/html'}, + }, + ) + ) + db.update_site( + MaigretSite( + 'Plain', + { + 'url': 'https://example.com/{username}', + 'urlMain': 'https://example.com/', + 'headers': {'accept': 'text/html'}, + }, + ) + ) + monkeypatch.setattr(activation_cache, 'MAIGRET_HOME', str(tmp_path)) + monkeypatch.setattr( + activation_cache, 'ACTIVATION_CACHE_PATH', str(tmp_path / 'activation.json') + ) + return db + + +def test_activation_cache_persists_only_minted_headers(activation_db): + """The database ships baseline headers; only what the run minted is cached. + + Storing the full header set would pin whatever the database shipped, so a + later database update could never change those headers again. + """ + logger = Mock() + baseline = load_activation_cache(activation_db, logger) + + # simulate what ParsingActivator does mid-run + activation_db.sites_dict['Twitter'].headers['x-guest-token'] = 'minted' + + save_activation_cache(activation_db, baseline, logger) + + written = json.loads(open(activation_cache.ACTIVATION_CACHE_PATH).read()) + assert written == {'Twitter': {'x-guest-token': 'minted'}} + # the unchanged baseline header is not copied into the cache + assert 'accept' not in written['Twitter'] + + +def test_activation_cache_round_trip_applies_token(activation_db): + logger = Mock() + baseline = load_activation_cache(activation_db, logger) + activation_db.sites_dict['Twitter'].headers['x-guest-token'] = 'minted' + save_activation_cache(activation_db, baseline, logger) + + # a fresh run loads the database again, with the shipped value + activation_db.sites_dict['Twitter'].headers['x-guest-token'] = 'from-db' + load_activation_cache(activation_db, logger) + + assert activation_db.sites_dict['Twitter'].headers['x-guest-token'] == 'minted' + + +def test_activation_cache_skips_sites_without_activation(activation_db): + logger = Mock() + baseline = load_activation_cache(activation_db, logger) + activation_db.sites_dict['Plain'].headers['accept'] = 'application/json' + + save_activation_cache(activation_db, baseline, logger) + + assert not os.path.exists(activation_cache.ACTIVATION_CACHE_PATH) + + +def test_activation_cache_ignores_corrupt_file(activation_db): + logger = Mock() + with open(activation_cache.ACTIVATION_CACHE_PATH, 'w') as f: + f.write('{ this is not json') + + baseline = load_activation_cache(activation_db, logger) + + assert baseline['Twitter']['x-guest-token'] == 'from-db' + assert activation_db.sites_dict['Twitter'].headers['x-guest-token'] == 'from-db' + assert logger.debug.called + + +def test_activation_cache_is_not_world_readable(activation_db): + """The cache holds session credentials, so the umask must not decide.""" + logger = Mock() + baseline = load_activation_cache(activation_db, logger) + activation_db.sites_dict['Twitter'].headers['x-guest-token'] = 'minted' + + save_activation_cache(activation_db, baseline, logger) + + mode = os.stat(activation_cache.ACTIVATION_CACHE_PATH).st_mode & 0o777 + assert mode == 0o600, f"expected 0600, got {mode:o}" + + +def test_activation_cache_does_not_leak_into_shared_headers(monkeypatch, tmp_path): + """MaigretSite.headers defaults to a mutable class attribute. + + A site with an activation block but no headers of its own shares that + object with ~3000 other sites, so an in-place update would attach the + cached token to every one of them. + """ + logger = Mock() + db = MaigretDatabase() + db.update_site( + MaigretSite( + 'NoHeaders', + { + 'url': 'https://a.example/{username}', + 'urlMain': 'https://a.example/', + 'activation': {'method': 'twitter', 'marks': ['nope']}, + }, + ) + ) + db.update_site( + MaigretSite( + 'Bystander', + {'url': 'https://b.example/{username}', 'urlMain': 'https://b.example/'}, + ) + ) + cache_file = tmp_path / 'activation.json' + cache_file.write_text(json.dumps({'NoHeaders': {'x-guest-token': 'secret'}})) + monkeypatch.setattr(activation_cache, 'MAIGRET_HOME', str(tmp_path)) + monkeypatch.setattr(activation_cache, 'ACTIVATION_CACHE_PATH', str(cache_file)) + + load_activation_cache(db, logger) + + assert db.sites_dict['NoHeaders'].headers['x-guest-token'] == 'secret' + assert 'x-guest-token' not in db.sites_dict['Bystander'].headers + assert 'x-guest-token' not in MaigretSite.headers diff --git a/tests/test_maigret.py b/tests/test_maigret.py index d77f11c..22e6073 100644 --- a/tests/test_maigret.py +++ b/tests/test_maigret.py @@ -1,12 +1,14 @@ """Maigret main module test functions""" import asyncio +import json +import os import copy from unittest.mock import Mock, patch import pytest -from maigret.maigret import self_check, maigret +from maigret.maigret import self_check, maigret, save_db_safely from maigret.maigret import ( extract_ids_from_page, extract_ids_from_results, @@ -293,3 +295,37 @@ def test_main_entrypoint_handles_top_level_keyboard_interrupt_cleanly(): # No Python traceback may leak through assert "Traceback" not in result.stderr assert "KeyboardInterrupt" not in result.stderr + + +@pytest.mark.skipif(os.geteuid() == 0, reason="root ignores file permissions") +def test_save_db_safely_returns_false_on_readonly_target(default_db, tmp_path): + """A read-only install must not take down a run that already finished. + + Distro packages, snaps and /nix/store put the bundled database in a + directory the user cannot write, and it is written unconditionally at + the end of every run to cache activation tokens. + """ + logger = Mock() + readonly_dir = tmp_path / "site-packages" + readonly_dir.mkdir() + target = readonly_dir / "data.json" + target.write_text("{}") + readonly_dir.chmod(0o555) + target.chmod(0o444) + + try: + assert save_db_safely(default_db, str(target), logger) is False + assert logger.debug.called + # the untouched file is still the one that was there before + assert target.read_text() == "{}" + finally: + readonly_dir.chmod(0o755) + target.chmod(0o644) + + +def test_save_db_safely_writes_when_target_is_writable(default_db, tmp_path): + logger = Mock() + target = tmp_path / "data.json" + + assert save_db_safely(default_db, str(target), logger) is True + assert "sites" in json.loads(target.read_text())