diff --git a/pyproject.toml b/pyproject.toml index 490a5a22..ee937f2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ dependencies = [ "aiodns==3.6.1", "aiofiles==25.1.0", "aiohttp==3.13.3", + "aiohttp-socks==0.11.0", "aiomultiprocess==0.9.1", "aiosqlite==0.22.1", "beautifulsoup4==4.14.3", diff --git a/theHarvester/data/proxies.yaml b/theHarvester/data/proxies.yaml index a5b35624..67faf78f 100644 --- a/theHarvester/data/proxies.yaml +++ b/theHarvester/data/proxies.yaml @@ -1,2 +1,4 @@ http: - ip:port +socks5: + - ip:port diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index 98414efc..89a27187 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -13,6 +13,7 @@ import certifi # need to import as different name as to not shadow already existing json var in post_fetch import ujson as json_loader import yaml +from aiohttp_socks import ProxyConnector from .version import version @@ -187,10 +188,11 @@ class Core: return Core.api_keys()['zoomeye']['key'] @staticmethod - def proxy_list() -> list: + def proxy_list() -> dict: keys = yaml.safe_load(Core._read_config('proxies.yaml')) http_list = [f'http://{proxy}' for proxy in keys['http']] if keys['http'] is not None else [] - return http_list + socks5_list = [f'socks5://{proxy}' for proxy in keys['socks5']] if keys.get('socks5') is not None else [] + return {'http': http_list, 'socks5': socks5_list} @staticmethod def banner() -> None: @@ -344,6 +346,40 @@ class Core: class AsyncFetcher: proxy_list = Core.proxy_list() + @staticmethod + def _get_random_proxy(proxy_dict: dict) -> tuple[str | None, str | None]: + """ + Get a random proxy from the proxy dictionary. + Returns (proxy_url, proxy_type) where proxy_type is 'http' or 'socks5' + """ + all_proxies = [] + for proxy_type, proxies in proxy_dict.items(): + if proxies: + for proxy in proxies: + all_proxies.append((proxy, proxy_type)) + + if not all_proxies: + return None, None + + return random.choice(all_proxies) + + @staticmethod + async def _create_connector( + proxy_url: str | None, proxy_type: str | None, ssl_context: ssl.SSLContext | bool | None = None + ) -> aiohttp.BaseConnector: + """ + Create an appropriate connector for the given proxy type. + Returns a connector that can be used with aiohttp.ClientSession. + """ + if proxy_url and proxy_type == 'socks5': + # Create SOCKS5 proxy connector using aiohttp-socks + # ProxyConnector.from_url can handle socks5://host:port URLs + connector = ProxyConnector.from_url(proxy_url, ssl=ssl_context) + return connector + else: + # Use default TCP connector for HTTP proxies or no proxy + return aiohttp.TCPConnector(ssl=ssl_context if ssl_context else ssl.create_default_context(cafile=certifi.where())) + @classmethod async def post_fetch( cls, @@ -363,15 +399,21 @@ class AsyncFetcher: # results are well worth the wait try: if proxy: - proxy = random.choice(cls().proxy_list) + proxy_url, proxy_type = cls._get_random_proxy(cls().proxy_list) + sslcontext = ssl.create_default_context(cafile=certifi.where()) + connector = await cls._create_connector(proxy_url, proxy_type, sslcontext) + if params != '': - async with aiohttp.ClientSession(headers=headers, timeout=timeout) as session: - async with session.get(url, params=params, proxy=str(proxy) if proxy else None) as response: + async with aiohttp.ClientSession(headers=headers, timeout=timeout, connector=connector) as session: + # For HTTP proxies, pass proxy parameter; for SOCKS5, connector handles it + proxy_param = proxy_url if proxy_type == 'http' else None + async with session.get(url, params=params, proxy=proxy_param) as response: await asyncio.sleep(5) return await response.text() if json is False else await response.json() else: - async with aiohttp.ClientSession(headers=headers, timeout=timeout) as session: - async with session.get(url, proxy=str(proxy) if proxy else None) as response: + async with aiohttp.ClientSession(headers=headers, timeout=timeout, connector=connector) as session: + proxy_param = proxy_url if proxy_type == 'http' else None + async with session.get(url, proxy=proxy_param) as response: await asyncio.sleep(5) return await response.text() if json is False else await response.json() elif params == '': @@ -423,14 +465,16 @@ class AsyncFetcher: # Resolve proxy parameter proxy_url: str | None = None + proxy_type: str | None = None if isinstance(proxy, str) and proxy != '': proxy_url = proxy + proxy_type = 'socks5' if proxy_url.startswith('socks5://') else 'http' elif isinstance(proxy, bool) and proxy: try: - proxy_choice = random.choice(cls().proxy_list) - proxy_url = str(proxy_choice) if proxy_choice else None + proxy_url, proxy_type = cls._get_random_proxy(cls().proxy_list) except Exception: proxy_url = None + proxy_type = None # Prepare timeout client_timeout = aiohttp.ClientTimeout(total=request_timeout) if request_timeout else None @@ -441,14 +485,17 @@ class AsyncFetcher: # Decide whether we need to manage the session owns_session = session is None if owns_session: - session = aiohttp.ClientSession(headers=req_headers, timeout=client_timeout) + # Create connector based on proxy type + connector = await cls._create_connector(proxy_url, proxy_type, ssl_arg) if proxy_url else None + session = aiohttp.ClientSession(headers=req_headers, timeout=client_timeout, connector=connector) assert session is not None try: request_kwargs: dict[str, Any] = { 'ssl': ssl_arg, } - if proxy_url: + # For HTTP proxies, pass the proxy parameter; for SOCKS5, the connector handles it + if proxy_url and proxy_type == 'http': request_kwargs['proxy'] = proxy_url if follow_redirects is not None: request_kwargs['allow_redirects'] = follow_redirects @@ -521,8 +568,10 @@ class AsyncFetcher: if takeover: async with aiohttp.ClientSession(headers=headers, timeout=aiohttp.ClientTimeout(total=15)) as session: if proxy: + # Get random proxy for each URL + proxy_urls = [cls._get_random_proxy(cls().proxy_list)[0] for _ in urls] return await asyncio.gather( - *[AsyncFetcher.takeover_fetch(session, url, proxy=random.choice(cls().proxy_list)) for url in urls] + *[AsyncFetcher.takeover_fetch(session, url, proxy=proxy_url) for url, proxy_url in zip(urls, proxy_urls)] ) else: return await asyncio.gather(*[AsyncFetcher.takeover_fetch(session, url) for url in urls]) @@ -530,15 +579,17 @@ class AsyncFetcher: if len(params) == 0: async with aiohttp.ClientSession(headers=headers, timeout=timeout) as session: if proxy: + # Get random proxy for each URL (returns tuple of proxy_url and proxy_type) + proxy_data = [cls._get_random_proxy(cls().proxy_list) for _ in urls] return await asyncio.gather( *[ AsyncFetcher.fetch( session, url, json=json, - proxy=random.choice(cls().proxy_list), + proxy=proxy_url, ) - for url in urls + for url, (proxy_url, proxy_type) in zip(urls, proxy_data) ] ) else: @@ -547,6 +598,7 @@ class AsyncFetcher: # Indicates the request has certain params async with aiohttp.ClientSession(headers=headers, timeout=timeout) as session: if proxy: + proxy_data = [cls._get_random_proxy(cls().proxy_list) for _ in urls] return await asyncio.gather( *[ AsyncFetcher.fetch( @@ -554,9 +606,9 @@ class AsyncFetcher: url, params, json, - proxy=random.choice(cls().proxy_list), + proxy=proxy_url, ) - for url in urls + for url, (proxy_url, proxy_type) in zip(urls, proxy_data) ] ) else: diff --git a/theHarvester/screenshot/screenshot.py b/theHarvester/screenshot/screenshot.py index 61eba683..771830f6 100644 --- a/theHarvester/screenshot/screenshot.py +++ b/theHarvester/screenshot/screenshot.py @@ -11,6 +11,7 @@ from datetime import datetime import aiohttp import certifi +from aiohttp_socks import ProxyConnector from playwright.async_api import async_playwright @@ -51,7 +52,7 @@ class ScreenShotter: return [list(items)[i : i + chunk_size] for i in range(0, len(items), chunk_size)] @staticmethod - async def visit(url: str) -> tuple[str, str]: + async def visit(url: str, proxy: str | None = None) -> tuple[str, str]: try: timeout = aiohttp.ClientTimeout(total=35) headers = { @@ -61,12 +62,26 @@ class ScreenShotter: url = f'http://{url}' if not url.startswith('http') else url url = url.replace('www.', '') sslcontext = ssl.create_default_context(cafile=certifi.where()) + + # Create connector based on proxy type + connector = None + proxy_param = None + if proxy: + if proxy.startswith('socks5://'): + connector = ProxyConnector.from_url(proxy, ssl=sslcontext) + else: + # HTTP proxy + connector = aiohttp.TCPConnector(ssl=sslcontext) + proxy_param = proxy + else: + connector = aiohttp.TCPConnector(ssl=sslcontext) + async with aiohttp.ClientSession( timeout=timeout, headers=headers, - connector=aiohttp.TCPConnector(ssl=sslcontext), + connector=connector, ) as session: - async with session.get(url, ssl=False) as resp: + async with session.get(url, ssl=False, proxy=proxy_param) as resp: text = await resp.text('UTF-8') return f'http://{url}' if not url.startswith('http') else url, text except Exception as e: diff --git a/uv.lock b/uv.lock index ba710ac4..de2402f2 100644 --- a/uv.lock +++ b/uv.lock @@ -121,6 +121,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b4/63/278a98c715ae467624eafe375542d8ba9b4383a016df8fdefe0ae28382a7/aiohttp-3.13.3-cp314-cp314t-win_amd64.whl", hash = "sha256:44531a36aa2264a1860089ffd4dce7baf875ee5a6079d5fb42e261c704ef7344", size = 499694, upload-time = "2026-01-03T17:32:24.546Z" }, ] +[[package]] +name = "aiohttp-socks" +version = "0.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "python-socks" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1f/cc/e5bbd54f76bd56291522251e47267b645dac76327b2657ade9545e30522c/aiohttp_socks-0.11.0.tar.gz", hash = "sha256:0afe51638527c79077e4bd6e57052c87c4824233d6e20bb061c53766421b10f0", size = 11196, upload-time = "2025-12-09T13:35:52.564Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bf/7d/4b633d709b8901d59444d2e512b93e72fe62d2b492a040097c3f7ba017bb/aiohttp_socks-0.11.0-py3-none-any.whl", hash = "sha256:9aacce57c931b8fbf8f6d333cf3cafe4c35b971b35430309e167a35a8aab9ec1", size = 10556, upload-time = "2025-12-09T13:35:50.18Z" }, +] + [[package]] name = "aiomultiprocess" version = "0.9.1" @@ -1290,6 +1303,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] +[[package]] +name = "python-socks" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/07/cfdd6a846ac859e513b4e68bb6c669a90a74d89d8d405516fba7fc9c6f0c/python_socks-2.8.0.tar.gz", hash = "sha256:340f82778b20a290bdd538ee47492978d603dff7826aaf2ce362d21ad9ee6f1b", size = 273130, upload-time = "2025-12-09T12:17:05.433Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/10/e2b575faa32d1d32e5e6041fc64794fa9f09526852a06b25353b66f52cae/python_socks-2.8.0-py3-none-any.whl", hash = "sha256:57c24b416569ccea493a101d38b0c82ed54be603aa50b6afbe64c46e4a4e4315", size = 55075, upload-time = "2025-12-09T12:17:03.269Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -1475,6 +1497,7 @@ dependencies = [ { name = "aiodns" }, { name = "aiofiles" }, { name = "aiohttp" }, + { name = "aiohttp-socks" }, { name = "aiomultiprocess" }, { name = "aiosqlite" }, { name = "beautifulsoup4" }, @@ -1518,6 +1541,7 @@ requires-dist = [ { name = "aiodns", specifier = "==3.6.1" }, { name = "aiofiles", specifier = "==25.1.0" }, { name = "aiohttp", specifier = "==3.13.3" }, + { name = "aiohttp-socks", specifier = "==0.11.0" }, { name = "aiomultiprocess", specifier = "==0.9.1" }, { name = "aiosqlite", specifier = "==0.22.1" }, { name = "beautifulsoup4", specifier = "==4.14.3" },