WEBDIENSTE/theHarvester
1
0
Fork 0
mirror of https://github.com/laramies/theHarvester.git synced 2026-09-23 18:14:57 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fix issue #469

Browse Source
This commit is contained in:
L1ghtn1ng
2026-01-06 21:37:49 +00:00
parent da8a2c8561
commit e38bb8fbb4
5 changed files with 113 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pyproject.toml
+1
View File
@@ -22,6 +22,7 @@ dependencies = [
"aiodns==3.6.1",
"aiofiles==25.1.0",
"aiohttp==3.13.3",
"aiohttp-socks==0.11.0",
"aiomultiprocess==0.9.1",
"aiosqlite==0.22.1",
"beautifulsoup4==4.14.3",
theHarvester/data/proxies.yaml
+2
View File
@@ -1,2 +1,4 @@
http:
- ip:port
socks5:
- ip:port
theHarvester/lib/core.py
+68 -16
View File
@@ -13,6 +13,7 @@ import certifi
# need to import as different name as to not shadow already existing json var in post_fetch
import ujson as json_loader
import yaml
from aiohttp_socks import ProxyConnector
from .version import version
@@ -187,10 +188,11 @@ class Core:
return Core.api_keys()['zoomeye']['key']
@staticmethod
def proxy_list() -> list:
def proxy_list() -> dict:
keys = yaml.safe_load(Core._read_config('proxies.yaml'))
http_list = [f'http://{proxy}' for proxy in keys['http']] if keys['http'] is not None else []
return http_list
socks5_list = [f'socks5://{proxy}' for proxy in keys['socks5']] if keys.get('socks5') is not None else []
return {'http': http_list, 'socks5': socks5_list}
@staticmethod
def banner() -> None:
@@ -344,6 +346,40 @@ class Core:
class AsyncFetcher:
proxy_list = Core.proxy_list()
@staticmethod
def _get_random_proxy(proxy_dict: dict) -> tuple[str | None, str | None]:
"""
Get a random proxy from the proxy dictionary.
Returns (proxy_url, proxy_type) where proxy_type is 'http' or 'socks5'
"""
all_proxies = []
for proxy_type, proxies in proxy_dict.items():
if proxies:
for proxy in proxies:
all_proxies.append((proxy, proxy_type))
if not all_proxies:
return None, None
return random.choice(all_proxies)
@staticmethod
async def _create_connector(
proxy_url: str | None, proxy_type: str | None, ssl_context: ssl.SSLContext | bool | None = None
) -> aiohttp.BaseConnector:
"""
Create an appropriate connector for the given proxy type.
Returns a connector that can be used with aiohttp.ClientSession.
"""
if proxy_url and proxy_type == 'socks5':
# Create SOCKS5 proxy connector using aiohttp-socks
# ProxyConnector.from_url can handle socks5://host:port URLs
connector = ProxyConnector.from_url(proxy_url, ssl=ssl_context)
return connector
else:
# Use default TCP connector for HTTP proxies or no proxy
return aiohttp.TCPConnector(ssl=ssl_context if ssl_context else ssl.create_default_context(cafile=certifi.where()))
@classmethod
async def post_fetch(
cls,
@@ -363,15 +399,21 @@ class AsyncFetcher:
# results are well worth the wait
try:
if proxy:
proxy = random.choice(cls().proxy_list)
proxy_url, proxy_type = cls._get_random_proxy(cls().proxy_list)
sslcontext = ssl.create_default_context(cafile=certifi.where())
connector = await cls._create_connector(proxy_url, proxy_type, sslcontext)
if params != '':
async with aiohttp.ClientSession(headers=headers, timeout=timeout) as session:
async with session.get(url, params=params, proxy=str(proxy) if proxy else None) as response:
async with aiohttp.ClientSession(headers=headers, timeout=timeout, connector=connector) as session:
# For HTTP proxies, pass proxy parameter; for SOCKS5, connector handles it
proxy_param = proxy_url if proxy_type == 'http' else None
async with session.get(url, params=params, proxy=proxy_param) as response:
await asyncio.sleep(5)
return await response.text() if json is False else await response.json()
else:
async with aiohttp.ClientSession(headers=headers, timeout=timeout) as session:
async with session.get(url, proxy=str(proxy) if proxy else None) as response:
async with aiohttp.ClientSession(headers=headers, timeout=timeout, connector=connector) as session:
proxy_param = proxy_url if proxy_type == 'http' else None
async with session.get(url, proxy=proxy_param) as response:
await asyncio.sleep(5)
return await response.text() if json is False else await response.json()
elif params == '':
@@ -423,14 +465,16 @@ class AsyncFetcher:
# Resolve proxy parameter
proxy_url: str | None = None
proxy_type: str | None = None
if isinstance(proxy, str) and proxy != '':
proxy_url = proxy
proxy_type = 'socks5' if proxy_url.startswith('socks5://') else 'http'
elif isinstance(proxy, bool) and proxy:
try:
proxy_choice = random.choice(cls().proxy_list)
proxy_url = str(proxy_choice) if proxy_choice else None
proxy_url, proxy_type = cls._get_random_proxy(cls().proxy_list)
except Exception:
proxy_url = None
proxy_type = None
# Prepare timeout
client_timeout = aiohttp.ClientTimeout(total=request_timeout) if request_timeout else None
@@ -441,14 +485,17 @@ class AsyncFetcher:
# Decide whether we need to manage the session
owns_session = session is None
if owns_session:
session = aiohttp.ClientSession(headers=req_headers, timeout=client_timeout)
# Create connector based on proxy type
connector = await cls._create_connector(proxy_url, proxy_type, ssl_arg) if proxy_url else None
session = aiohttp.ClientSession(headers=req_headers, timeout=client_timeout, connector=connector)
assert session is not None
try:
request_kwargs: dict[str, Any] = {
'ssl': ssl_arg,
}
if proxy_url:
# For HTTP proxies, pass the proxy parameter; for SOCKS5, the connector handles it
if proxy_url and proxy_type == 'http':
request_kwargs['proxy'] = proxy_url
if follow_redirects is not None:
request_kwargs['allow_redirects'] = follow_redirects
@@ -521,8 +568,10 @@ class AsyncFetcher:
if takeover:
async with aiohttp.ClientSession(headers=headers, timeout=aiohttp.ClientTimeout(total=15)) as session:
if proxy:
# Get random proxy for each URL
proxy_urls = [cls._get_random_proxy(cls().proxy_list)[0] for _ in urls]
return await asyncio.gather(
*[AsyncFetcher.takeover_fetch(session, url, proxy=random.choice(cls().proxy_list)) for url in urls]
*[AsyncFetcher.takeover_fetch(session, url, proxy=proxy_url) for url, proxy_url in zip(urls, proxy_urls)]
)
else:
return await asyncio.gather(*[AsyncFetcher.takeover_fetch(session, url) for url in urls])
@@ -530,15 +579,17 @@ class AsyncFetcher:
if len(params) == 0:
async with aiohttp.ClientSession(headers=headers, timeout=timeout) as session:
if proxy:
# Get random proxy for each URL (returns tuple of proxy_url and proxy_type)
proxy_data = [cls._get_random_proxy(cls().proxy_list) for _ in urls]
return await asyncio.gather(
*[
AsyncFetcher.fetch(
session,
url,
json=json,
proxy=random.choice(cls().proxy_list),
proxy=proxy_url,
)
for url in urls
for url, (proxy_url, proxy_type) in zip(urls, proxy_data)
]
)
else:
@@ -547,6 +598,7 @@ class AsyncFetcher:
# Indicates the request has certain params
async with aiohttp.ClientSession(headers=headers, timeout=timeout) as session:
if proxy:
proxy_data = [cls._get_random_proxy(cls().proxy_list) for _ in urls]
return await asyncio.gather(
*[
AsyncFetcher.fetch(
@@ -554,9 +606,9 @@ class AsyncFetcher:
url,
params,
json,
proxy=random.choice(cls().proxy_list),
proxy=proxy_url,
)
for url in urls
for url, (proxy_url, proxy_type) in zip(urls, proxy_data)
]
)
else:
theHarvester/screenshot/screenshot.py
+18 -3
View File
@@ -11,6 +11,7 @@ from datetime import datetime
import aiohttp
import certifi
from aiohttp_socks import ProxyConnector
from playwright.async_api import async_playwright
@@ -51,7 +52,7 @@ class ScreenShotter:
return [list(items)[i : i + chunk_size] for i in range(0, len(items), chunk_size)]
@staticmethod
async def visit(url: str) -> tuple[str, str]:
async def visit(url: str, proxy: str | None = None) -> tuple[str, str]:
try:
timeout = aiohttp.ClientTimeout(total=35)
headers = {
@@ -61,12 +62,26 @@ class ScreenShotter:
url = f'http://{url}' if not url.startswith('http') else url
url = url.replace('www.', '')
sslcontext = ssl.create_default_context(cafile=certifi.where())
# Create connector based on proxy type
connector = None
proxy_param = None
if proxy:
if proxy.startswith('socks5://'):
connector = ProxyConnector.from_url(proxy, ssl=sslcontext)
else:
# HTTP proxy
connector = aiohttp.TCPConnector(ssl=sslcontext)
proxy_param = proxy
else:
connector = aiohttp.TCPConnector(ssl=sslcontext)
async with aiohttp.ClientSession(
timeout=timeout,
headers=headers,
connector=aiohttp.TCPConnector(ssl=sslcontext),
connector=connector,
) as session:
async with session.get(url, ssl=False) as resp:
async with session.get(url, ssl=False, proxy=proxy_param) as resp:
text = await resp.text('UTF-8')
return f'http://{url}' if not url.startswith('http') else url, text
except Exception as e:
uv.lock
Generated
+24
View File
@@ -121,6 +121,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/b4/63/278a98c715ae467624eafe375542d8ba9b4383a016df8fdefe0ae28382a7/aiohttp-3.13.3-cp314-cp314t-win_amd64.whl", hash = "sha256:44531a36aa2264a1860089ffd4dce7baf875ee5a6079d5fb42e261c704ef7344", size = 499694, upload-time = "2026-01-03T17:32:24.546Z" },
]
[[package]]
name = "aiohttp-socks"
version = "0.11.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "aiohttp" },
{ name = "python-socks" },
]
sdist = { url = "https://files.pythonhosted.org/packages/1f/cc/e5bbd54f76bd56291522251e47267b645dac76327b2657ade9545e30522c/aiohttp_socks-0.11.0.tar.gz", hash = "sha256:0afe51638527c79077e4bd6e57052c87c4824233d6e20bb061c53766421b10f0", size = 11196, upload-time = "2025-12-09T13:35:52.564Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/bf/7d/4b633d709b8901d59444d2e512b93e72fe62d2b492a040097c3f7ba017bb/aiohttp_socks-0.11.0-py3-none-any.whl", hash = "sha256:9aacce57c931b8fbf8f6d333cf3cafe4c35b971b35430309e167a35a8aab9ec1", size = 10556, upload-time = "2025-12-09T13:35:50.18Z" },
]
[[package]]
name = "aiomultiprocess"
version = "0.9.1"
@@ -1290,6 +1303,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" },
]
[[package]]
name = "python-socks"
version = "2.8.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/6c/07/cfdd6a846ac859e513b4e68bb6c669a90a74d89d8d405516fba7fc9c6f0c/python_socks-2.8.0.tar.gz", hash = "sha256:340f82778b20a290bdd538ee47492978d603dff7826aaf2ce362d21ad9ee6f1b", size = 273130, upload-time = "2025-12-09T12:17:05.433Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/13/10/e2b575faa32d1d32e5e6041fc64794fa9f09526852a06b25353b66f52cae/python_socks-2.8.0-py3-none-any.whl", hash = "sha256:57c24b416569ccea493a101d38b0c82ed54be603aa50b6afbe64c46e4a4e4315", size = 55075, upload-time = "2025-12-09T12:17:03.269Z" },
]
[[package]]
name = "pyyaml"
version = "6.0.3"
@@ -1475,6 +1497,7 @@ dependencies = [
{ name = "aiodns" },
{ name = "aiofiles" },
{ name = "aiohttp" },
{ name = "aiohttp-socks" },
{ name = "aiomultiprocess" },
{ name = "aiosqlite" },
{ name = "beautifulsoup4" },
@@ -1518,6 +1541,7 @@ requires-dist = [
{ name = "aiodns", specifier = "==3.6.1" },
{ name = "aiofiles", specifier = "==25.1.0" },
{ name = "aiohttp", specifier = "==3.13.3" },
{ name = "aiohttp-socks", specifier = "==0.11.0" },
{ name = "aiomultiprocess", specifier = "==0.9.1" },
{ name = "aiosqlite", specifier = "==0.22.1" },
{ name = "beautifulsoup4", specifier = "==4.14.3" },