From 77b565cfd5ad3118c2931f2f879e26c181053222 Mon Sep 17 00:00:00 2001 From: felipe <35511790+felicabrera@users.noreply.github.com> Date: Wed, 12 Aug 2026 12:04:32 -0300 Subject: [PATCH] fix: normalize SOCKS5 proxy scheme per HTTP transport (#2955) (#2966) --- docs/source/command-line-options.rst | 4 ++ maigret/checking.py | 57 ++++++++++++++++- maigret/maigret.py | 3 +- maigret/submit.py | 11 +++- tests/test_checking.py | 96 ++++++++++++++++++++++++++++ tests/test_submit.py | 49 ++++++++++++++ 6 files changed, 215 insertions(+), 5 deletions(-) diff --git a/docs/source/command-line-options.rst b/docs/source/command-line-options.rst index 40aac5a..e08f063 100644 --- a/docs/source/command-line-options.rst +++ b/docs/source/command-line-options.rst @@ -72,6 +72,10 @@ the given HTTP or SOCKS proxy. Example: ``socks5://127.0.0.1:1080``, routing the whole run through Tor (``--proxy socks5://127.0.0.1:9050``), a residential proxy, or any corporate gateway. No default. +``socks5://`` and ``socks5h://`` are interchangeable: Maigret rewrites the +scheme to the spelling expected by the transport handling each site, so +either one resolves hostnames **at the proxy** for the whole database. + ``--tor-proxy TOR_PROXY_URL`` - Gateway used **only** for ``.onion`` sites in the database **(default: socks5://127.0.0.1:9050)**. Clearweb sites are unaffected — for them Maigret uses your direct connection or diff --git a/maigret/checking.py b/maigret/checking.py index ef9a6af..0106dec 100644 --- a/maigret/checking.py +++ b/maigret/checking.py @@ -59,6 +59,59 @@ def _is_dns_error(exc: Exception) -> bool: return any(m in text for m in _DNS_ERROR_MARKERS) +# The two HTTP transports disagree about what a SOCKS5 proxy URL means. +# +# python_socks (via aiohttp_socks, used by SimpleAiohttpChecker) +# accepts exactly socks5/socks4/http and raises +# ValueError('Invalid scheme component: socks5h') on anything else, so +# socks5h:// is a hard crash before a single request is made. Its rdns +# flag defaults to True for SOCKS5, so socks5:// there already means +# proxy-side DNS. +# +# libcurl (via curl_cffi, used by CurlCffiChecker for tls_fingerprint sites) +# keeps the classic distinction: socks5:// resolves the hostname on the +# client and passes an address to the proxy, socks5h:// passes the +# hostname and lets the proxy resolve it. +# +# So a single `--proxy socks5://...` resolves most of the database at the +# proxy but the tls_fingerprint sites locally: their hostnames leak to the +# local resolver, and geo-balanced hosts get resolved for the wrong network. +# See issue #2955. +# +# Normalizing the scheme per transport makes both spellings mean proxy-side +# DNS everywhere, so users need not know which transport handles which site. +# Only SOCKS5 is remapped: python_socks defaults rdns to False for SOCKS4, +# so rewriting socks4 would change behavior instead of aligning it. +PYTHON_SOCKS_TRANSPORT = 'python_socks' +LIBCURL_TRANSPORT = 'libcurl' + +_PROXY_SCHEME_ALIASES = { + PYTHON_SOCKS_TRANSPORT: {'socks5h': 'socks5'}, + LIBCURL_TRANSPORT: {'socks5': 'socks5h'}, +} + + +def normalize_proxy_scheme(proxy: Optional[str], transport: str) -> Optional[str]: + """Rewrite a proxy URL's scheme to the spelling `transport` understands. + + Only the scheme is rewritten; host, port, credentials and path are passed + through as given, as are non-SOCKS5 schemes, schemeless values and empty + values. + """ + if not proxy: + return proxy + + scheme, separator, remainder = proxy.partition('://') + if not separator: + return proxy + + replacement = _PROXY_SCHEME_ALIASES[transport].get(scheme.lower()) + if replacement is None: + return proxy + + return f'{replacement}://{remainder}' + + SUPPORTED_IDS = ( "username", "yandex_public_id", @@ -142,7 +195,7 @@ class CheckerBase: class SimpleAiohttpChecker(CheckerBase): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.proxy = kwargs.get('proxy') + self.proxy = normalize_proxy_scheme(kwargs.get('proxy'), PYTHON_SOCKS_TRANSPORT) self.cookie_jar = kwargs.get('cookie_jar') # 'async' (default) uses aiohttp's DefaultResolver, which is AsyncResolver # (powered by aiodns / c-ares) when aiodns is installed. 'threaded' uses @@ -339,7 +392,7 @@ class CurlCffiChecker(CheckerBase): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.browser_emulate = kwargs.get('browser_emulate', 'chrome') - self.proxy = kwargs.get('proxy') + self.proxy = normalize_proxy_scheme(kwargs.get('proxy'), LIBCURL_TRANSPORT) def prepare(self, url, headers=None, allow_redirects=True, timeout=0, method='get', payload=None, encoding=None): self.url = url diff --git a/maigret/maigret.py b/maigret/maigret.py index 607811d..6672651 100755 --- a/maigret/maigret.py +++ b/maigret/maigret.py @@ -272,7 +272,8 @@ def setup_arguments_parser(settings: Settings): action="store", dest="proxy", default=settings.proxy_url, - help="Make requests over a proxy. e.g. socks5://127.0.0.1:1080", + help="Make requests over a proxy. e.g. socks5://127.0.0.1:1080 " + "(socks5:// and socks5h:// are equivalent, both resolve at the proxy)", ) parser.add_argument( "--tor-proxy", diff --git a/maigret/submit.py b/maigret/submit.py index 2893c47..44c09ee 100644 --- a/maigret/submit.py +++ b/maigret/submit.py @@ -14,7 +14,11 @@ from .result import MaigretCheckResult from .settings import Settings from .sites import MaigretDatabase, MaigretEngine, MaigretSite from .utils import get_random_user_agent -from .checking import site_self_check +from .checking import ( + site_self_check, + normalize_proxy_scheme, + PYTHON_SOCKS_TRANSPORT, +) from .utils import get_match_ratio, generate_random_username @@ -37,7 +41,10 @@ class Submitter: from aiohttp_socks import ProxyConnector - proxy = self.args.proxy + # Same python_socks scheme constraint as SimpleAiohttpChecker: socks5h + # is rejected outright, so --submit through a SOCKS proxy would crash + # without this. See issue #2955. + proxy = normalize_proxy_scheme(self.args.proxy, PYTHON_SOCKS_TRANSPORT) cookie_jar = None if args.cookie_file: if not os.path.exists(args.cookie_file): diff --git a/tests/test_checking.py b/tests/test_checking.py index 89be565..bf57632 100644 --- a/tests/test_checking.py +++ b/tests/test_checking.py @@ -1851,3 +1851,99 @@ async def test_simple_aiohttp_checker_does_not_retry_generic_proxy_error(): assert len(calls) == 1 # no retry assert error.type == 'Proxy' + + +# --- SOCKS proxy scheme normalization tests (issue #2955) --- + + +@pytest.mark.parametrize( + 'given, expected', + [ + # socks5h is rejected outright by python_socks, so it must be rewritten + ('socks5h://127.0.0.1:1080', 'socks5://127.0.0.1:1080'), + # socks5 already means proxy-side DNS there: pass through untouched + ('socks5://127.0.0.1:1080', 'socks5://127.0.0.1:1080'), + # schemes python_socks handles natively must not be touched + ('http://127.0.0.1:8080', 'http://127.0.0.1:8080'), + ('socks4://127.0.0.1:1080', 'socks4://127.0.0.1:1080'), + ], +) +def test_aiohttp_checker_normalizes_proxy_scheme(given, expected): + from maigret.checking import SimpleAiohttpChecker + + assert SimpleAiohttpChecker(logger=Mock(), proxy=given).proxy == expected + + +@pytest.mark.parametrize( + 'given, expected', + [ + # libcurl resolves locally for socks5, leaking hostnames past the proxy + ('socks5://127.0.0.1:1080', 'socks5h://127.0.0.1:1080'), + ('socks5h://127.0.0.1:1080', 'socks5h://127.0.0.1:1080'), + ('http://127.0.0.1:8080', 'http://127.0.0.1:8080'), + ('socks4://127.0.0.1:1080', 'socks4://127.0.0.1:1080'), + ], +) +def test_curl_cffi_checker_normalizes_proxy_scheme(given, expected): + from maigret.checking import CurlCffiChecker + + assert CurlCffiChecker(logger=Mock(), proxy=given).proxy == expected + + +def test_proxied_aiohttp_checker_normalizes_proxy_scheme(): + """--tor-proxy / --i2p-proxy go through the subclass, and .onion / .i2p + names only resolve at the proxy, so the same normalization must apply.""" + from maigret.checking import ProxiedAiohttpChecker + + checker = ProxiedAiohttpChecker(logger=Mock(), proxy='socks5h://127.0.0.1:9050') + assert checker.proxy == 'socks5://127.0.0.1:9050' + + +def test_both_checkers_agree_on_proxy_side_dns(): + """Whichever spelling the user passes, both transports end up resolving + at the proxy.""" + from maigret.checking import SimpleAiohttpChecker, CurlCffiChecker + + for spelling in ('socks5://127.0.0.1:1080', 'socks5h://127.0.0.1:1080'): + # socks5 + python_socks rdns default of True == socks5h + libcurl + assert SimpleAiohttpChecker(logger=Mock(), proxy=spelling).proxy == ( + 'socks5://127.0.0.1:1080' + ) + assert CurlCffiChecker(logger=Mock(), proxy=spelling).proxy == ( + 'socks5h://127.0.0.1:1080' + ) + + +@pytest.mark.parametrize( + 'transport, given, expected', + [ + # credentials, IPv6 literals and paths must survive untouched + ( + 'python_socks', + 'socks5h://user:p%40ss@proxy.example:1080', + 'socks5://user:p%40ss@proxy.example:1080', + ), + ( + 'libcurl', + 'socks5://user:p%40ss@[::1]:1080', + 'socks5h://user:p%40ss@[::1]:1080', + ), + # the scheme match is case-insensitive, like every other URL scheme + ('python_socks', 'SOCKS5H://127.0.0.1:1080', 'socks5://127.0.0.1:1080'), + ('libcurl', 'Socks5://127.0.0.1:1080', 'socks5h://127.0.0.1:1080'), + # only the leading scheme is rewritten, never a match inside the URL + ( + 'libcurl', + 'http://user:socks5://@127.0.0.1:8080', + 'http://user:socks5://@127.0.0.1:8080', + ), + # no proxy configured, or a value with no scheme at all + ('python_socks', None, None), + ('libcurl', '', ''), + ('libcurl', '127.0.0.1:1080', '127.0.0.1:1080'), + ], +) +def test_normalize_proxy_scheme(transport, given, expected): + from maigret.checking import normalize_proxy_scheme + + assert normalize_proxy_scheme(given, transport) == expected diff --git a/tests/test_submit.py b/tests/test_submit.py index 97a9895..07ce8ec 100644 --- a/tests/test_submit.py +++ b/tests/test_submit.py @@ -358,3 +358,52 @@ def test_dialog_nonexistent_site_name_no_crash(): ) assert old_site is not None assert old_site.name == "ValidActive" + + +# --- SOCKS proxy scheme normalization tests (issue #2955) --- + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + 'given, expected', + [ + # python_socks rejects socks5h outright, so --submit through a SOCKS + # proxy used to crash with "Invalid scheme component: socks5h" + ('socks5h://127.0.0.1:1080', 'socks5://127.0.0.1:1080'), + ('socks5://127.0.0.1:1080', 'socks5://127.0.0.1:1080'), + ('http://127.0.0.1:8080', 'http://127.0.0.1:8080'), + ], +) +async def test_submitter_normalizes_proxy_scheme(test_db, given, expected): + args = MagicMock() + args.cookie_file = "" + args.proxy = given + + captured = [] + + class _DummyConnector: + def __init__(self, *args, **kwargs): + # aiohttp's ClientSession expects these on its connector + self._loop = None + self.closed = False + + async def close(self): + pass + + @property + def force_close(self): + return False + + def fake_from_url(url, **kwargs): + captured.append(url) + return _DummyConnector() + + # Only the URL handed to python_socks matters here; whether ClientSession + # then accepts the dummy connector is irrelevant to this assertion. + with patch('aiohttp_socks.ProxyConnector.from_url', side_effect=fake_from_url): + try: + Submitter(test_db, MagicMock(), logging.getLogger(), args) + except Exception: + pass + + assert captured == [expected]