diff --git a/maigret/activation.py b/maigret/activation.py index 1adb52f2..2fb3e232 100644 --- a/maigret/activation.py +++ b/maigret/activation.py @@ -2,36 +2,44 @@ import json from http.cookiejar import MozillaCookieJar from http.cookies import Morsel -from aiohttp import CookieJar +from aiohttp import ClientSession, CookieJar class ParsingActivator: @staticmethod - def twitter(site, logger, cookies={}, **kwargs): + async def twitter(site, logger, cookies={}, **kwargs): headers = dict(site.headers) - del headers["x-guest-token"] - import requests + headers.pop("x-guest-token", None) - r = requests.post(site.activation["url"], headers=headers) - logger.info(r) - j = r.json() + async with ClientSession(trust_env=True) as session: + async with session.post( + site.activation["url"], + headers=headers, + timeout=kwargs.get("timeout"), + ) as response: + logger.info(response) + j = await response.json(content_type=None) guest_token = j[site.activation["src"]] - site.headers["x-guest-token"] = guest_token + site.headers[site.activation.get("dst", "x-guest-token")] = guest_token @staticmethod - def vimeo(site, logger, cookies={}, **kwargs): + async def vimeo(site, logger, cookies={}, **kwargs): headers = dict(site.headers) - if "Authorization" in headers: - del headers["Authorization"] - import requests + headers.pop("Authorization", None) - r = requests.get(site.activation["url"], headers=headers) - logger.debug(f"Vimeo viewer activation: {json.dumps(r.json(), indent=4)}") - jwt_token = r.json()["jwt"] + async with ClientSession(trust_env=True) as session: + async with session.get( + site.activation["url"], + headers=headers, + timeout=kwargs.get("timeout"), + ) as response: + payload = await response.json(content_type=None) + logger.debug(f"Vimeo viewer activation: {json.dumps(payload, indent=4)}") + jwt_token = payload["jwt"] site.headers["Authorization"] = "jwt " + jwt_token @staticmethod - def onlyfans(site, logger, url=None, **kwargs): + async def onlyfans(site, logger, url=None, **kwargs): # Signing rules (static_param / checksum_indexes / checksum_constant / format / app_token) # live in data.json under OnlyFans.activation and rotate upstream every ~1–3 weeks. # If "Please refresh the page" keeps firing after activation, refresh them from: @@ -41,8 +49,6 @@ class ParsingActivator: import time as _time from urllib.parse import urlparse - import requests - act = site.activation static_param = act["static_param"] indexes = act["checksum_indexes"] @@ -69,11 +75,21 @@ class ParsingActivator: hdrs["time"] = t hdrs["sign"] = sg hdrs.pop("cookie", None) - r = requests.get(init_url, headers=hdrs, timeout=15) - jar = "; ".join(f"{k}={v}" for k, v in r.cookies.items()) + async with ClientSession(trust_env=True) as session: + async with session.get( + init_url, + headers=hdrs, + timeout=kwargs.get("timeout", 15), + ) as response: + jar = "; ".join( + f"{k}={getattr(v, 'value', v)}" + for k, v in response.cookies.items() + ) if jar: site.headers["cookie"] = jar - logger.debug(f"OnlyFans init: got cookies {list(r.cookies.keys())}") + logger.debug( + f"OnlyFans init: got cookies {list(response.cookies.keys())}" + ) target_path = urlparse(url).path if url else urlparse(init_url).path t, sg = _sign(target_path) @@ -82,38 +98,46 @@ class ParsingActivator: logger.debug(f"OnlyFans signed {target_path} time={t}") @staticmethod - def weibo(site, logger, **kwargs): + async def weibo(site, logger, **kwargs): headers = dict(site.headers) - import requests + timeout = kwargs.get("timeout") - session = requests.Session() - # 1 stage: get the redirect URL - r = session.get( - "https://weibo.com/clairekuo", headers=headers, allow_redirects=False - ) - logger.debug( - f"1 stage: {'success' if r.status_code == 302 else 'no 302 redirect, fail!'}" - ) - location = r.headers.get("Location", "") + async with ClientSession(trust_env=True) as session: + # 1 stage: get the redirect URL + async with session.get( + "https://weibo.com/clairekuo", + headers=headers, + allow_redirects=False, + timeout=timeout, + ) as response: + logger.debug( + f"1 stage: {'success' if response.status == 302 else 'no 302 redirect, fail!'}" + ) + location = response.headers.get("Location", "") - # 2 stage: go to passport visitor page - headers["Referer"] = location - r = session.get(location, headers=headers) - logger.debug( - f"2 stage: {'success' if r.status_code == 200 else 'no 200 response, fail!'}" - ) + # 2 stage: go to passport visitor page + headers["Referer"] = location + async with session.get( + location, + headers=headers, + timeout=timeout, + ) as response: + logger.debug( + f"2 stage: {'success' if response.status == 200 else 'no 200 response, fail!'}" + ) - # 3 stage: gen visitor token - headers["Referer"] = location - r = session.post( - "https://passport.weibo.com/visitor/genvisitor2", - headers=headers, - data={'cb': 'visitor_gray_callback', 'tid': '', 'from': 'weibo'}, - ) - cookies = r.headers.get('set-cookie') - logger.debug( - f"3 stage: {'success' if r.status_code == 200 and cookies else 'no 200 response and cookies, fail!'}" - ) + # 3 stage: gen visitor token + headers["Referer"] = location + async with session.post( + "https://passport.weibo.com/visitor/genvisitor2", + headers=headers, + data={'cb': 'visitor_gray_callback', 'tid': '', 'from': 'weibo'}, + timeout=timeout, + ) as response: + cookies = response.headers.get('set-cookie') + logger.debug( + f"3 stage: {'success' if response.status == 200 and cookies else 'no 200 response and cookies, fail!'}" + ) site.headers["Cookie"] = cookies diff --git a/maigret/checking.py b/maigret/checking.py index b47267de..51b914c1 100644 --- a/maigret/checking.py +++ b/maigret/checking.py @@ -628,6 +628,13 @@ class CheckerMock: return +def make_protocol_checker(options: QueryOptions, protocol: str): + checker_factory = options["checkers"][protocol] + if callable(checker_factory): + return checker_factory() + return checker_factory + + def debug_response_logging(url, html_text, status_code, check_error): with open("debug.log", "a") as f: status = status_code or "No response" @@ -681,29 +688,6 @@ def process_site_result( status_code, ) - # parsing activation - is_need_activation = any( - [s for s in site.activation.get("marks", []) if s in html_text] - ) - - if site.activation and html_text and is_need_activation: - logger.debug(f"Activation for {site.name}") - method = site.activation["method"] - try: - activate_fun = getattr(ParsingActivator(), method) - # TODO: async call - activate_fun(site, logger) - except AttributeError as e: - logger.warning( - f"Activation method {method} for site {site.name} not found!", - exc_info=True, - ) - except Exception as e: - logger.warning( - f"Failed activation {method} for site {site.name}: {str(e)}", - exc_info=True, - ) - site_name = site.pretty_name # presense flags # True by default @@ -889,9 +873,9 @@ def make_site_result( f"Site {site.name} requires TLS impersonation (curl_cffi) but it's not installed. " "Install with: pip install curl_cffi" ) - checker = options["checkers"][site.protocol] + checker = make_protocol_checker(options, site.protocol) else: - checker = options["checkers"][site.protocol] + checker = make_protocol_checker(options, site.protocol) # site check is disabled if site.disabled and not options['forced']: @@ -1024,8 +1008,13 @@ async def check_site_for_username( method = act["method"] try: activate_fun = getattr(ParsingActivator(), method) - activate_fun(site, logger, url=checker.url) - except AttributeError as e: + await activate_fun( + site, + logger, + url=checker.url, + timeout=options['timeout'], + ) + except AttributeError: logger.warning( f"Activation method {method} for site {site.name} not found!", exc_info=True, @@ -1152,31 +1141,32 @@ async def maigret( logger.debug(f"Using cookies jar file {cookies}") cookie_jar = import_aiohttp_cookies(cookies) - clearweb_checker = SimpleAiohttpChecker( - proxy=proxy, cookie_jar=cookie_jar, logger=logger, dns_resolver=dns_resolver - ) + def clearweb_checker(): + return SimpleAiohttpChecker( + proxy=proxy, cookie_jar=cookie_jar, logger=logger, dns_resolver=dns_resolver + ) - # TODO - tor_checker = CheckerMock() - if tor_proxy: - tor_checker = ProxiedAiohttpChecker( # type: ignore + def tor_checker(): + if not tor_proxy: + return CheckerMock() + return ProxiedAiohttpChecker( # type: ignore proxy=tor_proxy, cookie_jar=cookie_jar, logger=logger, dns_resolver=dns_resolver ) - # TODO - i2p_checker = CheckerMock() - if i2p_proxy: - i2p_checker = ProxiedAiohttpChecker( # type: ignore + def i2p_checker(): + if not i2p_proxy: + return CheckerMock() + return ProxiedAiohttpChecker( # type: ignore proxy=i2p_proxy, cookie_jar=cookie_jar, logger=logger, dns_resolver=dns_resolver ) - # TODO - dns_checker = CheckerMock() - if check_domains: - dns_checker = AiodnsDomainResolver(logger=logger) # type: ignore + def dns_checker(): + if not check_domains: + return CheckerMock() + return AiodnsDomainResolver(logger=logger) # type: ignore if logger.level == logging.DEBUG: - await debug_ip_request(clearweb_checker, logger) + await debug_ip_request(clearweb_checker(), logger) # setup parallel executor executor = AsyncioQueueGeneratorExecutor( @@ -1265,12 +1255,9 @@ async def maigret( all_results.update([result]) progress() except asyncio.CancelledError: - # Tear down HTTP sessions and re-raise so the caller's - # `except CancelledError` runs. The partial `all_results` is - # already visible to the caller via the output_container kwarg. - await clearweb_checker.close() - await tor_checker.close() - await i2p_checker.close() + # Re-raise so the caller's `except CancelledError` runs. The + # partial `all_results` is already visible to the caller via the + # output_container kwarg. query_notify.finish() raise @@ -1288,11 +1275,6 @@ async def maigret( f'Restarting checks for {len(sites)} sites... ({attempts} attempts left)' ) - # closing http client session - await clearweb_checker.close() - await tor_checker.close() - await i2p_checker.close() - # notify caller that all queries are finished query_notify.finish() diff --git a/tests/test_activation.py b/tests/test_activation.py index 76e60f47..82c2cb7d 100644 --- a/tests/test_activation.py +++ b/tests/test_activation.py @@ -1,6 +1,6 @@ """Maigret activation test functions""" -import json +import inspect import yarl import aiohttp @@ -27,11 +27,12 @@ localhost FALSE / FALSE 0 a b @pytest.mark.skip("captcha") @pytest.mark.slow -def test_vimeo_activation(default_db): +@pytest.mark.asyncio +async def test_vimeo_activation(default_db): vimeo_site = default_db.sites_dict['Vimeo'] token1 = vimeo_site.headers['Authorization'] - ParsingActivator.vimeo(vimeo_site, Mock()) + await ParsingActivator.vimeo(vimeo_site, Mock()) token2 = vimeo_site.headers['Authorization'] assert token1 != token2 @@ -60,6 +61,7 @@ async def test_import_aiohttp_cookies(cookie_test_server): # ---- OnlyFans signing tests (pure-compute, no network) ---- + class _FakeSite: """Minimal stand-in for MaigretSite with the attributes onlyfans() touches.""" @@ -75,23 +77,79 @@ class _FakeSite: class _FakeResponse: - def __init__(self, cookies=None): - self.cookies = cookies or {} + def __init__(self, cookies=None, json_data=None): + self.cookies = { + key: type("Cookie", (), {"value": value})() + for key, value in (cookies or {}).items() + } + self._json_data = json_data or {} + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return False + + async def json(self, content_type=None): + return self._json_data -def test_onlyfans_sets_xbc_when_zero(monkeypatch): +@pytest.mark.parametrize("method", ["twitter", "vimeo", "onlyfans", "weibo"]) +def test_activation_methods_are_coroutines(method): + assert inspect.iscoroutinefunction(getattr(ParsingActivator, method)) + + +@pytest.mark.asyncio +async def test_vimeo_activation_uses_aiohttp(monkeypatch): + site = _FakeSite( + headers={"Authorization": "old-token", "User-Agent": "test"}, + activation={"url": "https://vimeo.test/viewer"}, + ) + captured = {} + + class FakeSession: + def __init__(self, **kwargs): + captured["session_kwargs"] = kwargs + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return False + + def get(self, url, headers=None, timeout=None): + captured["url"] = url + captured["headers"] = dict(headers or {}) + captured["timeout"] = timeout + return _FakeResponse(json_data={"jwt": "fresh"}) + + monkeypatch.setattr("maigret.activation.ClientSession", FakeSession) + + await ParsingActivator.vimeo(site, Mock(), timeout=7) + + assert captured["url"] == "https://vimeo.test/viewer" + assert captured["headers"] == {"User-Agent": "test"} + assert captured["timeout"] == 7 + assert captured["session_kwargs"] == {"trust_env": True} + assert site.headers["Authorization"] == "jwt fresh" + + +@pytest.mark.asyncio +async def test_onlyfans_sets_xbc_when_zero(monkeypatch): site = _FakeSite(headers={"x-bc": "0", "cookie": "existing=1"}) - # Prevent any real network. If _sign path still fires requests.get, fail loudly. - import maigret.activation as act_mod - + # Prevent any real network. If _sign path still opens a session, fail loudly. def boom(*a, **kw): # pragma: no cover - sanity - raise AssertionError("requests.get should not run when cookie is present") + raise AssertionError("ClientSession should not open when cookie is present") - monkeypatch.setattr(act_mod.__dict__.get("requests", None) or __import__("requests"), "get", boom, raising=False) + monkeypatch.setattr("maigret.activation.ClientSession", boom) logger = Mock() - ParsingActivator.onlyfans(site, logger, url="https://onlyfans.com/api2/v2/users/adam") + await ParsingActivator.onlyfans( + site, + logger, + url="https://onlyfans.com/api2/v2/users/adam", + ) # x-bc must be rewritten to a non-zero hex token assert site.headers["x-bc"] != "0" @@ -101,26 +159,42 @@ def test_onlyfans_sets_xbc_when_zero(monkeypatch): assert site.headers["sign"].startswith("57203:") -def test_onlyfans_fetches_init_cookie_when_missing(monkeypatch): +@pytest.mark.asyncio +async def test_onlyfans_fetches_init_cookie_when_missing(monkeypatch): """When cookie header is absent, init endpoint is called and its cookies stored.""" site = _FakeSite(headers={"x-bc": "already_set_token", "user-id": "0"}) - import requests - captured = {} - def fake_get(url, headers=None, timeout=15): - captured["url"] = url - captured["headers"] = dict(headers or {}) - return _FakeResponse(cookies={"sess": "abc123", "csrf": "xyz"}) + class FakeSession: + def __init__(self, **kwargs): + captured["session_kwargs"] = kwargs - monkeypatch.setattr(requests, "get", fake_get) + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return False + + def get(self, url, headers=None, timeout=15): + captured["url"] = url + captured["headers"] = dict(headers or {}) + captured["timeout"] = timeout + return _FakeResponse(cookies={"sess": "abc123", "csrf": "xyz"}) + + monkeypatch.setattr("maigret.activation.ClientSession", FakeSession) logger = Mock() - ParsingActivator.onlyfans(site, logger, url="https://onlyfans.com/api2/v2/users/adam") + await ParsingActivator.onlyfans( + site, + logger, + url="https://onlyfans.com/api2/v2/users/adam", + ) # init request made assert captured["url"] == site.activation["url"] + assert captured["timeout"] == 15 + assert captured["session_kwargs"] == {"trust_env": True} # headers passed to init include freshly generated time/sign assert "time" in captured["headers"] assert captured["headers"]["sign"].startswith("57203:") @@ -128,38 +202,59 @@ def test_onlyfans_fetches_init_cookie_when_missing(monkeypatch): assert site.headers["cookie"] == "sess=abc123; csrf=xyz" -def test_onlyfans_signature_is_deterministic_for_same_time(monkeypatch): +@pytest.mark.asyncio +async def test_onlyfans_signature_is_deterministic_for_same_time(monkeypatch): """Two calls with patched time produce identical signatures.""" site1 = _FakeSite(headers={"x-bc": "token", "cookie": "c=1"}) site2 = _FakeSite(headers={"x-bc": "token", "cookie": "c=1"}) import maigret.activation + monkeypatch.setattr(maigret.activation, "_time", __import__("time"), raising=False) fixed = 1_700_000_000.123 import time as time_mod + monkeypatch.setattr(time_mod, "time", lambda: fixed) logger = Mock() - ParsingActivator.onlyfans(site1, logger, url="https://onlyfans.com/api2/v2/users/adam") - ParsingActivator.onlyfans(site2, logger, url="https://onlyfans.com/api2/v2/users/adam") + await ParsingActivator.onlyfans( + site1, + logger, + url="https://onlyfans.com/api2/v2/users/adam", + ) + await ParsingActivator.onlyfans( + site2, + logger, + url="https://onlyfans.com/api2/v2/users/adam", + ) assert site1.headers["time"] == site2.headers["time"] assert site1.headers["sign"] == site2.headers["sign"] -def test_onlyfans_sign_differs_per_path(monkeypatch): +@pytest.mark.asyncio +async def test_onlyfans_sign_differs_per_path(monkeypatch): """Different target URLs must yield different signatures.""" site = _FakeSite(headers={"x-bc": "token", "cookie": "c=1"}) import time as time_mod + monkeypatch.setattr(time_mod, "time", lambda: 1_700_000_000.0) logger = Mock() - ParsingActivator.onlyfans(site, logger, url="https://onlyfans.com/api2/v2/users/adam") + await ParsingActivator.onlyfans( + site, + logger, + url="https://onlyfans.com/api2/v2/users/adam", + ) sig_adam = site.headers["sign"] - ParsingActivator.onlyfans(site, logger, url="https://onlyfans.com/api2/v2/users/bob") + await ParsingActivator.onlyfans( + site, + logger, + url="https://onlyfans.com/api2/v2/users/bob", + ) sig_bob = site.headers["sign"] assert sig_adam != sig_bob diff --git a/tests/test_checking.py b/tests/test_checking.py index 92eaa954..f7db4f70 100644 --- a/tests/test_checking.py +++ b/tests/test_checking.py @@ -1,9 +1,11 @@ +import asyncio from argparse import ArgumentTypeError from mock import Mock import pytest from maigret import search +from maigret.activation import ParsingActivator from maigret.checking import ( extract_ids_data, parse_usernames, @@ -12,6 +14,7 @@ from maigret.checking import ( timeout_check, debug_response_logging, process_site_result, + check_site_for_username, ) from maigret.error_detection import ErrorPageDetector from maigret.errors import CheckError @@ -390,6 +393,178 @@ def test_process_site_result_error_context_uses_instance(): assert "class" not in out["status"].context +@pytest.mark.asyncio +async def test_check_site_for_username_awaits_activation_before_retry(monkeypatch): + site = _make_site({ + "checkType": "status_code", + "headers": {"X-Initial": "1"}, + "activation": { + "method": "test_async", + "marks": ["NEEDS_ACTIVATION"], + }, + "protocol": "https", + }) + + class FakeChecker: + def __init__(self): + self.calls = 0 + self.prepared_headers = [] + self.url = None + self.headers = None + self.allow_redirects = True + self.timeout = 0 + self.method = "get" + self.payload = None + + def prepare( + self, + url, + headers=None, + allow_redirects=True, + timeout=0, + method="get", + payload=None, + ): + self.url = url + self.headers = headers + self.allow_redirects = allow_redirects + self.timeout = timeout + self.method = method + self.payload = payload + self.prepared_headers.append(dict(headers or {})) + return None + + async def check(self): + self.calls += 1 + if self.calls == 1: + return "NEEDS_ACTIVATION", 200, None + return "activated", 200, None + + async def activate(site, logger, **kwargs): + await asyncio.sleep(0) + site.headers["X-Activated"] = "yes" + + checker = FakeChecker() + monkeypatch.setattr( + ParsingActivator, + "test_async", + staticmethod(activate), + raising=False, + ) + + options = { + "parsing": False, + "cookie_jar": None, + "forced": True, + "id_type": "username", + "timeout": 3, + "proxy": None, + "checkers": {"https": checker}, + } + + _, result = await check_site_for_username( + site, + "a", + options, + Mock(), + Mock(), + ) + + assert checker.calls == 2 + assert checker.prepared_headers[-1]["X-Activated"] == "yes" + assert result["status"].status == MaigretCheckStatus.CLAIMED + + +@pytest.mark.asyncio +async def test_concurrent_activation_uses_independent_checkers(monkeypatch): + instances = [] + + class FakeChecker: + def __init__(self): + self.calls = 0 + self.prepared_urls = [] + self.url = None + self.headers = None + self.allow_redirects = True + self.timeout = 0 + self.method = "get" + self.payload = None + instances.append(self) + + def prepare( + self, + url, + headers=None, + allow_redirects=True, + timeout=0, + method="get", + payload=None, + ): + self.url = url + self.headers = headers + self.allow_redirects = allow_redirects + self.timeout = timeout + self.method = method + self.payload = payload + self.prepared_urls.append(url) + return None + + async def check(self): + await asyncio.sleep(0) + self.calls += 1 + if self.calls == 1: + return "NEEDS_ACTIVATION", 200, None + return "activated", 200, None + + async def activate(site, logger, **kwargs): + await asyncio.sleep(0) + site.headers["X-Activated"] = site.name + + monkeypatch.setattr( + ParsingActivator, + "test_async", + staticmethod(activate), + raising=False, + ) + + options = { + "parsing": False, + "cookie_jar": None, + "forced": True, + "id_type": "username", + "timeout": 3, + "proxy": None, + "checkers": {"https": FakeChecker}, + } + first = _make_site({ + "url": "https://x/one/{username}", + "urlMain": "https://x", + "checkType": "status_code", + "activation": {"method": "test_async", "marks": ["NEEDS_ACTIVATION"]}, + "protocol": "https", + }) + first.name = "First" + second = _make_site({ + "url": "https://x/two/{username}", + "urlMain": "https://x", + "checkType": "status_code", + "activation": {"method": "test_async", "marks": ["NEEDS_ACTIVATION"]}, + "protocol": "https", + }) + second.name = "Second" + + await asyncio.gather( + check_site_for_username(first, "a", options, Mock(), Mock()), + check_site_for_username(second, "a", options, Mock(), Mock()), + ) + + assert len(instances) == 2 + assert [checker.prepared_urls for checker in instances] == [ + ["https://x/one/a", "https://x/one/a"], + ["https://x/two/a", "https://x/two/a"], + ] + + # ---- CurlCffiChecker: TLS impersonation header sanitisation ----