fix: normalize SOCKS5 proxy scheme per HTTP transport (#2955) (#2966)

docs/source/command-line-options.rst

@@ -72,6 +72,10 @@ the given HTTP or SOCKS proxy. Example: ``socks5://127.0.0.1:1080``,
 routing the whole run through Tor (``--proxy socks5://127.0.0.1:9050``),
 a residential proxy, or any corporate gateway. No default.
 
+``socks5://`` and ``socks5h://`` are interchangeable: Maigret rewrites the
+scheme to the spelling expected by the transport handling each site, so
+either one resolves hostnames **at the proxy** for the whole database.
+
 ``--tor-proxy TOR_PROXY_URL`` - Gateway used **only** for ``.onion``
 sites in the database **(default: socks5://127.0.0.1:9050)**. Clearweb
 sites are unaffected — for them Maigret uses your direct connection or

maigret/checking.py

@@ -59,6 +59,59 @@ def _is_dns_error(exc: Exception) -> bool:
     return any(m in text for m in _DNS_ERROR_MARKERS)
 
 
+# The two HTTP transports disagree about what a SOCKS5 proxy URL means.
+#
+#   python_socks (via aiohttp_socks, used by SimpleAiohttpChecker)
+#     accepts exactly socks5/socks4/http and raises
+#     ValueError('Invalid scheme component: socks5h') on anything else, so
+#     socks5h:// is a hard crash before a single request is made. Its rdns
+#     flag defaults to True for SOCKS5, so socks5:// there already means
+#     proxy-side DNS.
+#
+#   libcurl (via curl_cffi, used by CurlCffiChecker for tls_fingerprint sites)
+#     keeps the classic distinction: socks5:// resolves the hostname on the
+#     client and passes an address to the proxy, socks5h:// passes the
+#     hostname and lets the proxy resolve it.
+#
+# So a single `--proxy socks5://...` resolves most of the database at the
+# proxy but the tls_fingerprint sites locally: their hostnames leak to the
+# local resolver, and geo-balanced hosts get resolved for the wrong network.
+# See issue #2955.
+#
+# Normalizing the scheme per transport makes both spellings mean proxy-side
+# DNS everywhere, so users need not know which transport handles which site.
+# Only SOCKS5 is remapped: python_socks defaults rdns to False for SOCKS4,
+# so rewriting socks4 would change behavior instead of aligning it.
+PYTHON_SOCKS_TRANSPORT = 'python_socks'
+LIBCURL_TRANSPORT = 'libcurl'
+
+_PROXY_SCHEME_ALIASES = {
+    PYTHON_SOCKS_TRANSPORT: {'socks5h': 'socks5'},
+    LIBCURL_TRANSPORT: {'socks5': 'socks5h'},
+}
+
+
+def normalize_proxy_scheme(proxy: Optional[str], transport: str) -> Optional[str]:
+    """Rewrite a proxy URL's scheme to the spelling `transport` understands.
+
+    Only the scheme is rewritten; host, port, credentials and path are passed
+    through as given, as are non-SOCKS5 schemes, schemeless values and empty
+    values.
+    """
+    if not proxy:
+        return proxy
+
+    scheme, separator, remainder = proxy.partition('://')
+    if not separator:
+        return proxy
+
+    replacement = _PROXY_SCHEME_ALIASES[transport].get(scheme.lower())
+    if replacement is None:
+        return proxy
+
+    return f'{replacement}://{remainder}'
+
+
 SUPPORTED_IDS = (
     "username",
     "yandex_public_id",
@@ -142,7 +195,7 @@ class CheckerBase:
 class SimpleAiohttpChecker(CheckerBase):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.proxy = kwargs.get('proxy')
+        self.proxy = normalize_proxy_scheme(kwargs.get('proxy'), PYTHON_SOCKS_TRANSPORT)
         self.cookie_jar = kwargs.get('cookie_jar')
         # 'async' (default) uses aiohttp's DefaultResolver, which is AsyncResolver
         # (powered by aiodns / c-ares) when aiodns is installed. 'threaded' uses
@@ -339,7 +392,7 @@ class CurlCffiChecker(CheckerBase):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.browser_emulate = kwargs.get('browser_emulate', 'chrome')
-        self.proxy = kwargs.get('proxy')
+        self.proxy = normalize_proxy_scheme(kwargs.get('proxy'), LIBCURL_TRANSPORT)
 
     def prepare(self, url, headers=None, allow_redirects=True, timeout=0, method='get', payload=None, encoding=None):
         self.url = url

maigret/maigret.py

@@ -272,7 +272,8 @@ def setup_arguments_parser(settings: Settings):
         action="store",
         dest="proxy",
         default=settings.proxy_url,
-        help="Make requests over a proxy. e.g. socks5://127.0.0.1:1080",
+        help="Make requests over a proxy. e.g. socks5://127.0.0.1:1080 "
+        "(socks5:// and socks5h:// are equivalent, both resolve at the proxy)",
     )
     parser.add_argument(
         "--tor-proxy",

maigret/submit.py

@@ -14,7 +14,11 @@ from .result import MaigretCheckResult
 from .settings import Settings
 from .sites import MaigretDatabase, MaigretEngine, MaigretSite
 from .utils import get_random_user_agent
-from .checking import site_self_check
+from .checking import (
+    site_self_check,
+    normalize_proxy_scheme,
+    PYTHON_SOCKS_TRANSPORT,
+)
 from .utils import get_match_ratio, generate_random_username
 
 
@@ -37,7 +41,10 @@ class Submitter:
 
         from aiohttp_socks import ProxyConnector
 
-        proxy = self.args.proxy
+        # Same python_socks scheme constraint as SimpleAiohttpChecker: socks5h
+        # is rejected outright, so --submit through a SOCKS proxy would crash
+        # without this. See issue #2955.
+        proxy = normalize_proxy_scheme(self.args.proxy, PYTHON_SOCKS_TRANSPORT)
         cookie_jar = None
         if args.cookie_file:
             if not os.path.exists(args.cookie_file):

tests/test_checking.py

@@ -1851,3 +1851,99 @@ async def test_simple_aiohttp_checker_does_not_retry_generic_proxy_error():
 
     assert len(calls) == 1  # no retry
     assert error.type == 'Proxy'
+
+
+# --- SOCKS proxy scheme normalization tests (issue #2955) ---
+
+
+@pytest.mark.parametrize(
+    'given, expected',
+    [
+        # socks5h is rejected outright by python_socks, so it must be rewritten
+        ('socks5h://127.0.0.1:1080', 'socks5://127.0.0.1:1080'),
+        # socks5 already means proxy-side DNS there: pass through untouched
+        ('socks5://127.0.0.1:1080', 'socks5://127.0.0.1:1080'),
+        # schemes python_socks handles natively must not be touched
+        ('http://127.0.0.1:8080', 'http://127.0.0.1:8080'),
+        ('socks4://127.0.0.1:1080', 'socks4://127.0.0.1:1080'),
+    ],
+)
+def test_aiohttp_checker_normalizes_proxy_scheme(given, expected):
+    from maigret.checking import SimpleAiohttpChecker
+
+    assert SimpleAiohttpChecker(logger=Mock(), proxy=given).proxy == expected
+
+
+@pytest.mark.parametrize(
+    'given, expected',
+    [
+        # libcurl resolves locally for socks5, leaking hostnames past the proxy
+        ('socks5://127.0.0.1:1080', 'socks5h://127.0.0.1:1080'),
+        ('socks5h://127.0.0.1:1080', 'socks5h://127.0.0.1:1080'),
+        ('http://127.0.0.1:8080', 'http://127.0.0.1:8080'),
+        ('socks4://127.0.0.1:1080', 'socks4://127.0.0.1:1080'),
+    ],
+)
+def test_curl_cffi_checker_normalizes_proxy_scheme(given, expected):
+    from maigret.checking import CurlCffiChecker
+
+    assert CurlCffiChecker(logger=Mock(), proxy=given).proxy == expected
+
+
+def test_proxied_aiohttp_checker_normalizes_proxy_scheme():
+    """--tor-proxy / --i2p-proxy go through the subclass, and .onion / .i2p
+    names only resolve at the proxy, so the same normalization must apply."""
+    from maigret.checking import ProxiedAiohttpChecker
+
+    checker = ProxiedAiohttpChecker(logger=Mock(), proxy='socks5h://127.0.0.1:9050')
+    assert checker.proxy == 'socks5://127.0.0.1:9050'
+
+
+def test_both_checkers_agree_on_proxy_side_dns():
+    """Whichever spelling the user passes, both transports end up resolving
+    at the proxy."""
+    from maigret.checking import SimpleAiohttpChecker, CurlCffiChecker
+
+    for spelling in ('socks5://127.0.0.1:1080', 'socks5h://127.0.0.1:1080'):
+        # socks5 + python_socks rdns default of True == socks5h + libcurl
+        assert SimpleAiohttpChecker(logger=Mock(), proxy=spelling).proxy == (
+            'socks5://127.0.0.1:1080'
+        )
+        assert CurlCffiChecker(logger=Mock(), proxy=spelling).proxy == (
+            'socks5h://127.0.0.1:1080'
+        )
+
+
+@pytest.mark.parametrize(
+    'transport, given, expected',
+    [
+        # credentials, IPv6 literals and paths must survive untouched
+        (
+            'python_socks',
+            'socks5h://user:p%40ss@proxy.example:1080',
+            'socks5://user:p%40ss@proxy.example:1080',
+        ),
+        (
+            'libcurl',
+            'socks5://user:p%40ss@[::1]:1080',
+            'socks5h://user:p%40ss@[::1]:1080',
+        ),
+        # the scheme match is case-insensitive, like every other URL scheme
+        ('python_socks', 'SOCKS5H://127.0.0.1:1080', 'socks5://127.0.0.1:1080'),
+        ('libcurl', 'Socks5://127.0.0.1:1080', 'socks5h://127.0.0.1:1080'),
+        # only the leading scheme is rewritten, never a match inside the URL
+        (
+            'libcurl',
+            'http://user:socks5://@127.0.0.1:8080',
+            'http://user:socks5://@127.0.0.1:8080',
+        ),
+        # no proxy configured, or a value with no scheme at all
+        ('python_socks', None, None),
+        ('libcurl', '', ''),
+        ('libcurl', '127.0.0.1:1080', '127.0.0.1:1080'),
+    ],
+)
+def test_normalize_proxy_scheme(transport, given, expected):
+    from maigret.checking import normalize_proxy_scheme
+
+    assert normalize_proxy_scheme(given, transport) == expected

tests/test_submit.py

@@ -358,3 +358,52 @@ def test_dialog_nonexistent_site_name_no_crash():
     )
     assert old_site is not None
     assert old_site.name == "ValidActive"
+
+
+# --- SOCKS proxy scheme normalization tests (issue #2955) ---
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    'given, expected',
+    [
+        # python_socks rejects socks5h outright, so --submit through a SOCKS
+        # proxy used to crash with "Invalid scheme component: socks5h"
+        ('socks5h://127.0.0.1:1080', 'socks5://127.0.0.1:1080'),
+        ('socks5://127.0.0.1:1080', 'socks5://127.0.0.1:1080'),
+        ('http://127.0.0.1:8080', 'http://127.0.0.1:8080'),
+    ],
+)
+async def test_submitter_normalizes_proxy_scheme(test_db, given, expected):
+    args = MagicMock()
+    args.cookie_file = ""
+    args.proxy = given
+
+    captured = []
+
+    class _DummyConnector:
+        def __init__(self, *args, **kwargs):
+            # aiohttp's ClientSession expects these on its connector
+            self._loop = None
+            self.closed = False
+
+        async def close(self):
+            pass
+
+        @property
+        def force_close(self):
+            return False
+
+    def fake_from_url(url, **kwargs):
+        captured.append(url)
+        return _DummyConnector()
+
+    # Only the URL handed to python_socks matters here; whether ClientSession
+    # then accepts the dummy connector is irrelevant to this assertion.
+    with patch('aiohttp_socks.ProxyConnector.from_url', side_effect=fake_from_url):
+        try:
+            Submitter(test_db, MagicMock(), logging.getLogger(), args)
+        except Exception:
+            pass
+
+    assert captured == [expected]