From fbda9cdfcce9e342af4282a8f2b82f3def4df397 Mon Sep 17 00:00:00 2001 From: Soxoj <31013580+soxoj@users.noreply.github.com> Date: Tue, 11 Aug 2026 13:16:13 +0200 Subject: [PATCH] fix: retry transient network/proxy errors in site checkers (#2954) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rotating/residential proxies often drop a connection mid-request (truncated body, closed socket, failed handshake) instead of failing outright. These were previously lumped into "Unexpected" and never retried, silently discarding otherwise-successful checks. - Retry once on ClientPayloadError, ServerDisconnectedError, and aiohttp_socks' ProxyConnectionError/ProxyTimeoutError (aiohttp checker), and on CurlError (curl_cffi checker) — covers truncated responses, dropped connections, CONNECT-tunnel 502s, and TLS handshake failures. - Fix the proxy-error except clause: it caught python_socks' ProxyError, but aiohttp_socks (the connector actually in use) raises its own unrelated same-named exceptions, so it never fired. - Generic ProxyError (e.g. bad credentials) is classified but NOT retried — it fails identically every attempt. --- maigret/checking.py | 229 ++++++++++++++++++------------- maigret/errors.py | 1 + tests/test_checking.py | 301 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 438 insertions(+), 93 deletions(-) diff --git a/maigret/checking.py b/maigret/checking.py index 2de0c87..ef9a6af 100644 --- a/maigret/checking.py +++ b/maigret/checking.py @@ -19,9 +19,10 @@ from aiohttp.resolver import ThreadedResolver from aiohttp.client_exceptions import ( ClientConnectorDNSError, ClientConnectorError, + ClientPayloadError, ServerDisconnectedError, ) -from python_socks import _errors as proxy_errors +from aiohttp_socks import ProxyConnectionError, ProxyError, ProxyTimeoutError from socid_extractor import extract, mutate_url # type: ignore[import-not-found] # Local imports @@ -169,60 +170,87 @@ class SimpleAiohttpChecker(CheckerBase): async def _make_request( self, session, url, headers, allow_redirects, timeout, method, logger, payload=None ) -> Tuple[Optional[str], int, Optional[CheckError]]: - try: - if method.lower() == 'get': - request_method = session.get - elif method.lower() == 'post': - request_method = session.post - elif method.lower() == 'head': - request_method = session.head - else: - request_method = session.get + if method.lower() == 'get': + request_method = session.get + elif method.lower() == 'post': + request_method = session.post + elif method.lower() == 'head': + request_method = session.head + else: + request_method = session.get - kwargs = { - 'url': url, - 'headers': headers, - 'allow_redirects': allow_redirects, - 'timeout': timeout, - } - if payload and method.lower() == 'post': - if headers and headers.get('Content-Type') == 'application/x-www-form-urlencoded': - kwargs['data'] = payload + kwargs = { + 'url': url, + 'headers': headers, + 'allow_redirects': allow_redirects, + 'timeout': timeout, + } + if payload and method.lower() == 'post': + if headers and headers.get('Content-Type') == 'application/x-www-form-urlencoded': + kwargs['data'] = payload + else: + kwargs['json'] = payload + + # A rotating (residential) proxy occasionally switches exit node + # mid-request: aiohttp surfaces this as a truncated body + # (ClientPayloadError, wrapping the underlying TransferEncodingError + # or ContentLengthError), a dropped connection (ServerDisconnectedError), + # or a failure to reach/handshake with the picked proxy node + # (aiohttp_socks' ProxyConnectionError/ProxyTimeoutError — NOT + # python_socks' same-named classes, which this connector never + # raises). One retry on a fresh connection is enough — the pooled + # connection is already discarded, so the retry goes out through a + # new exit node. A generic ProxyError (e.g. bad credentials) is + # deliberately NOT retried: it fails identically every time, so + # retrying it would just double the cost of every check for no gain. + transient_retries = 1 + for attempt in range(transient_retries + 1): + try: + async with request_method(**kwargs) as response: + status_code = response.status + response_content = await response.content.read() + charset = self.encoding or response.charset or "utf-8" + decoded_content = response_content.decode(charset, "ignore") + + error = CheckError("Connection lost") if status_code == 0 else None + logger.debug(decoded_content) + + return decoded_content, status_code, error + + except asyncio.TimeoutError as e: + return None, 0, CheckError("Request timeout", str(e)) + except ClientConnectorError as e: + err_type = "Connecting failure (DNS)" if _is_dns_error(e) else "Connecting failure" + return None, 0, CheckError(err_type, str(e)) + except ServerDisconnectedError as e: + if attempt < transient_retries: + logger.debug(f"Server disconnected, retrying: {e}") + continue + return None, 0, CheckError("Server disconnected", str(e)) + except (ProxyConnectionError, ProxyTimeoutError) as e: + if attempt < transient_retries: + logger.debug(f"Proxy connection error, retrying: {e}") + continue + return None, 0, CheckError("Proxy", str(e)) + except http_exceptions.BadHttpMessage as e: + return None, 0, CheckError("HTTP", str(e)) + except ProxyError as e: + return None, 0, CheckError("Proxy", str(e)) + except ClientPayloadError as e: + if attempt < transient_retries: + logger.debug(f"Payload error, retrying: {e}") + continue + return None, 0, CheckError("Payload", str(e)) + except KeyboardInterrupt: + return None, 0, CheckError("Interrupted") + except Exception as e: + if sys.version_info.minor > 6 and ( + isinstance(e, ssl.SSLCertVerificationError) + or isinstance(e, ssl.SSLError) + ): + return None, 0, CheckError("SSL", str(e)) else: - kwargs['json'] = payload - - async with request_method(**kwargs) as response: - status_code = response.status - response_content = await response.content.read() - charset = self.encoding or response.charset or "utf-8" - decoded_content = response_content.decode(charset, "ignore") - - error = CheckError("Connection lost") if status_code == 0 else None - logger.debug(decoded_content) - - return decoded_content, status_code, error - - except asyncio.TimeoutError as e: - return None, 0, CheckError("Request timeout", str(e)) - except ClientConnectorError as e: - err_type = "Connecting failure (DNS)" if _is_dns_error(e) else "Connecting failure" - return None, 0, CheckError(err_type, str(e)) - except ServerDisconnectedError as e: - return None, 0, CheckError("Server disconnected", str(e)) - except http_exceptions.BadHttpMessage as e: - return None, 0, CheckError("HTTP", str(e)) - except proxy_errors.ProxyError as e: - return None, 0, CheckError("Proxy", str(e)) - except KeyboardInterrupt: - return None, 0, CheckError("Interrupted") - except Exception as e: - if sys.version_info.minor > 6 and ( - isinstance(e, ssl.SSLCertVerificationError) - or isinstance(e, ssl.SSLError) - ): - return None, 0, CheckError("SSL", str(e)) - else: - logger.debug(e, exc_info=True) + logger.debug(e, exc_info=True) return None, 0, CheckError("Unexpected", str(e)) async def check(self) -> Tuple[Optional[str], int, Optional[CheckError]]: @@ -301,6 +329,7 @@ class AiodnsDomainResolver(CheckerBase): return text, status, error +from curl_cffi import CurlError from curl_cffi.requests import AsyncSession as CurlCffiAsyncSession @@ -326,51 +355,65 @@ class CurlCffiChecker(CheckerBase): pass async def check(self) -> Tuple[Optional[str], int, Optional[CheckError]]: - try: - session_kwargs = {} - if self.proxy: - session_kwargs['proxies'] = {'http': self.proxy, 'https': self.proxy} - async with CurlCffiAsyncSession(**session_kwargs) as session: - # Strip the User-Agent so curl_cffi can use the impersonated browser's - # matching UA. Mixing a random UA with a Chrome TLS fingerprint trips - # composite bot scoring (e.g. Cloudflare returns a JS challenge for - # "Chrome 91 UA + Chrome 131 TLS"). Keep any site-specific custom headers. - headers = {k: v for k, v in (self.headers or {}).items() - if k.lower() not in ('user-agent', 'connection')} - kwargs = { - 'url': self.url, - 'headers': headers or None, - 'allow_redirects': self.allow_redirects, - 'timeout': self.timeout if self.timeout else 10, - 'impersonate': self.browser_emulate, - } - if self.payload and self.method.lower() == 'post': - kwargs['json'] = self.payload + session_kwargs = {} + if self.proxy: + session_kwargs['proxies'] = {'http': self.proxy, 'https': self.proxy} - if self.method.lower() == 'post': - response = await session.post(**kwargs) - elif self.method.lower() == 'head': - response = await session.head(**kwargs) - else: - response = await session.get(**kwargs) + # Mirrors SimpleAiohttpChecker's payload-error retry: a rotating + # proxy's CONNECT tunnel occasionally 502s or drops the TLS + # handshake mid-way (curl_cffi surfaces both as CurlError — e.g. + # "curl: (56) CONNECT tunnel failed, response 502" or + # "curl: (35) TLS connect error"). One retry on a fresh connection + # is enough to usually land on a working exit node. + connect_retries = 1 + for attempt in range(connect_retries + 1): + try: + async with CurlCffiAsyncSession(**session_kwargs) as session: + # Strip the User-Agent so curl_cffi can use the impersonated browser's + # matching UA. Mixing a random UA with a Chrome TLS fingerprint trips + # composite bot scoring (e.g. Cloudflare returns a JS challenge for + # "Chrome 91 UA + Chrome 131 TLS"). Keep any site-specific custom headers. + headers = {k: v for k, v in (self.headers or {}).items() + if k.lower() not in ('user-agent', 'connection')} + kwargs = { + 'url': self.url, + 'headers': headers or None, + 'allow_redirects': self.allow_redirects, + 'timeout': self.timeout if self.timeout else 10, + 'impersonate': self.browser_emulate, + } + if self.payload and self.method.lower() == 'post': + kwargs['json'] = self.payload - status_code = response.status_code - if self.encoding: - response.encoding = self.encoding - decoded_content = response.text + if self.method.lower() == 'post': + response = await session.post(**kwargs) + elif self.method.lower() == 'head': + response = await session.head(**kwargs) + else: + response = await session.get(**kwargs) - self.logger.debug(decoded_content) + status_code = response.status_code + if self.encoding: + response.encoding = self.encoding + decoded_content = response.text - error = CheckError("Connection lost") if status_code == 0 else None - return decoded_content, status_code, error + self.logger.debug(decoded_content) - except asyncio.TimeoutError as e: - return None, 0, CheckError("Request timeout", str(e)) - except KeyboardInterrupt: - return None, 0, CheckError("Interrupted") - except Exception as e: - self.logger.debug(e, exc_info=True) - return None, 0, CheckError("Unexpected", str(e)) + error = CheckError("Connection lost") if status_code == 0 else None + return decoded_content, status_code, error + + except asyncio.TimeoutError as e: + return None, 0, CheckError("Request timeout", str(e)) + except KeyboardInterrupt: + return None, 0, CheckError("Interrupted") + except CurlError as e: + if attempt < connect_retries: + self.logger.debug(f"curl_cffi connection error, retrying: {e}") + continue + return None, 0, CheckError("Connecting failure", str(e)) + except Exception as e: + self.logger.debug(e, exc_info=True) + return None, 0, CheckError("Unexpected", str(e)) class CloudflareWebgateChecker(CheckerBase): diff --git a/maigret/errors.py b/maigret/errors.py index 37a23f6..bee6c00 100644 --- a/maigret/errors.py +++ b/maigret/errors.py @@ -118,6 +118,7 @@ TEMPORARY_ERRORS_TYPES = [ 'Proxy', 'Interrupted', 'Connection lost', + 'Payload', ] THRESHOLD = 3 # percent — default threshold above which an error type is "important" diff --git a/tests/test_checking.py b/tests/test_checking.py index ce4c96b..89be565 100644 --- a/tests/test_checking.py +++ b/tests/test_checking.py @@ -3,6 +3,8 @@ from argparse import ArgumentTypeError from unittest.mock import Mock import pytest +from aiohttp.client_exceptions import ServerDisconnectedError +from curl_cffi import CurlError from maigret import search from maigret.activation import ParsingActivator @@ -978,6 +980,96 @@ async def test_curl_cffi_no_proxy_omits_proxies_kwarg(fake_curl_cffi): assert 'proxies' not in init +class _RaisingThenOkCurlSession: + """First .get() call raises CurlError, simulating a rotating proxy's + CONNECT tunnel 502ing or dropping the TLS handshake mid-way; the retry + goes out through a fresh connection and succeeds.""" + + calls = 0 + + def __init__(self, **kwargs): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return False + + async def get(self, **kwargs): + type(self).calls += 1 + if type(self).calls == 1: + raise CurlError( + "Failed to perform, curl: (56) CONNECT tunnel failed, response 502.", 56 + ) + return _FakeCurlResponse() + + +class _AlwaysRaisingCurlSession: + calls = 0 + + def __init__(self, **kwargs): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return False + + async def get(self, **kwargs): + type(self).calls += 1 + raise CurlError( + "Failed to perform, curl: (56) CONNECT tunnel failed, response 502.", 56 + ) + + +@pytest.mark.asyncio +async def test_curl_cffi_retries_once_on_connection_error(monkeypatch): + from maigret import checking + from maigret.checking import CurlCffiChecker + + _RaisingThenOkCurlSession.calls = 0 + monkeypatch.setattr(checking, 'CurlCffiAsyncSession', _RaisingThenOkCurlSession) + + checker = CurlCffiChecker(logger=Mock(), browser_emulate='chrome') + checker.prepare( + url='https://example.com/u/test', + headers=None, + allow_redirects=True, + timeout=10, + method='get', + ) + text, status, error = await checker.check() + + assert _RaisingThenOkCurlSession.calls == 2 + assert error is None + assert status == 200 + assert text == 'ok' + + +@pytest.mark.asyncio +async def test_curl_cffi_gives_up_as_connecting_failure_after_retry(monkeypatch): + from maigret import checking + from maigret.checking import CurlCffiChecker + + _AlwaysRaisingCurlSession.calls = 0 + monkeypatch.setattr(checking, 'CurlCffiAsyncSession', _AlwaysRaisingCurlSession) + + checker = CurlCffiChecker(logger=Mock(), browser_emulate='chrome') + checker.prepare( + url='https://example.com/u/test', + headers=None, + allow_redirects=True, + timeout=10, + method='get', + ) + text, status, error = await checker.check() + + assert _AlwaysRaisingCurlSession.calls == 2 # initial attempt + one retry, then gives up + assert error.type == 'Connecting failure' + + # ----------------------------------------------------------------------------- # DNS-resolver selection (issue #2688). When --dns-resolver=threaded is passed, # SimpleAiohttpChecker must build the TCPConnector with an explicit @@ -1550,3 +1642,212 @@ async def test_enrich_disabled_skips_mutations(monkeypatch): } await check_site_for_username(site, "a", options, Mock(), Mock()) assert called["n"] == 0 + + +class _RaisingPayloadCM: + """Async context manager simulating a rotating proxy dropping the + connection mid-body: aiohttp surfaces this as ClientPayloadError.""" + + async def __aenter__(self): + from aiohttp.client_exceptions import ClientPayloadError + + raise ClientPayloadError("Response payload is not completed") + + async def __aexit__(self, *exc): + return False + + +class _OkResponseCM: + status = 200 + charset = 'utf-8' + + class _Content: + async def read(self): + return b'ok' + + content = _Content() + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + +@pytest.mark.asyncio +async def test_simple_aiohttp_checker_retries_once_on_payload_error(): + from maigret.checking import SimpleAiohttpChecker + + calls = [] + + def fake_get(**kwargs): + calls.append(kwargs) + return _RaisingPayloadCM() if len(calls) == 1 else _OkResponseCM() + + session = Mock() + session.get = fake_get + + checker = SimpleAiohttpChecker(logger=Mock()) + text, status, error = await checker._make_request( + session, 'http://example.com', {}, True, 5, 'get', Mock() + ) + + assert len(calls) == 2 + assert error is None + assert status == 200 + assert text == 'ok' + + +@pytest.mark.asyncio +async def test_simple_aiohttp_checker_gives_up_as_payload_error_after_retry(): + from maigret.checking import SimpleAiohttpChecker + + calls = [] + + def fake_get(**kwargs): + calls.append(kwargs) + return _RaisingPayloadCM() + + session = Mock() + session.get = fake_get + + checker = SimpleAiohttpChecker(logger=Mock()) + text, status, error = await checker._make_request( + session, 'http://example.com', {}, True, 5, 'get', Mock() + ) + + assert len(calls) == 2 # initial attempt + one retry, then gives up + assert error.type == 'Payload' + + +class _RaisingCM: + """Async context manager that raises the given exception on entry.""" + + def __init__(self, exc): + self._exc = exc + + async def __aenter__(self): + raise self._exc + + async def __aexit__(self, *exc): + return False + + +def _fake_get_raising_then_ok(exc_factory): + calls = [] + + def fake_get(**kwargs): + calls.append(kwargs) + return _RaisingCM(exc_factory()) if len(calls) == 1 else _OkResponseCM() + + return calls, fake_get + + +def _fake_get_always_raising(exc_factory): + calls = [] + + def fake_get(**kwargs): + calls.append(kwargs) + return _RaisingCM(exc_factory()) + + return calls, fake_get + + +@pytest.mark.asyncio +async def test_simple_aiohttp_checker_retries_server_disconnected(): + from maigret.checking import SimpleAiohttpChecker + + calls, fake_get = _fake_get_raising_then_ok( + lambda: ServerDisconnectedError("Server disconnected") + ) + session = Mock() + session.get = fake_get + + checker = SimpleAiohttpChecker(logger=Mock()) + text, status, error = await checker._make_request( + session, 'http://example.com', {}, True, 5, 'get', Mock() + ) + + assert len(calls) == 2 + assert error is None + assert status == 200 + + +@pytest.mark.asyncio +async def test_simple_aiohttp_checker_gives_up_as_server_disconnected_after_retry(): + from maigret.checking import SimpleAiohttpChecker + + calls, fake_get = _fake_get_always_raising( + lambda: ServerDisconnectedError("Server disconnected") + ) + session = Mock() + session.get = fake_get + + checker = SimpleAiohttpChecker(logger=Mock()) + text, status, error = await checker._make_request( + session, 'http://example.com', {}, True, 5, 'get', Mock() + ) + + assert len(calls) == 2 + assert error.type == 'Server disconnected' + + +@pytest.mark.asyncio +async def test_simple_aiohttp_checker_retries_proxy_connection_error(): + from maigret.checking import SimpleAiohttpChecker, ProxyConnectionError + + calls, fake_get = _fake_get_raising_then_ok( + lambda: ProxyConnectionError("Couldn't connect to proxy") + ) + session = Mock() + session.get = fake_get + + checker = SimpleAiohttpChecker(logger=Mock()) + text, status, error = await checker._make_request( + session, 'http://example.com', {}, True, 5, 'get', Mock() + ) + + assert len(calls) == 2 + assert error is None + assert status == 200 + + +@pytest.mark.asyncio +async def test_simple_aiohttp_checker_gives_up_as_proxy_after_retry_on_proxy_timeout(): + from maigret.checking import SimpleAiohttpChecker, ProxyTimeoutError + + calls, fake_get = _fake_get_always_raising( + lambda: ProxyTimeoutError("Proxy connection timed out") + ) + session = Mock() + session.get = fake_get + + checker = SimpleAiohttpChecker(logger=Mock()) + text, status, error = await checker._make_request( + session, 'http://example.com', {}, True, 5, 'get', Mock() + ) + + assert len(calls) == 2 + assert error.type == 'Proxy' + + +@pytest.mark.asyncio +async def test_simple_aiohttp_checker_does_not_retry_generic_proxy_error(): + """Unlike ProxyConnectionError/ProxyTimeoutError, a generic ProxyError + (e.g. bad credentials) fails identically every time — retrying it would + double the cost of every check for zero chance of success.""" + from maigret.checking import SimpleAiohttpChecker, ProxyError + + calls, fake_get = _fake_get_always_raising( + lambda: ProxyError("Unsupported proxy response") + ) + session = Mock() + session.get = fake_get + + checker = SimpleAiohttpChecker(logger=Mock()) + text, status, error = await checker._make_request( + session, 'http://example.com', {}, True, 5, 'get', Mock() + ) + + assert len(calls) == 1 # no retry + assert error.type == 'Proxy'