diff --git a/CHANGELOG.md b/CHANGELOG.md index c332b205..635895e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,6 +62,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Removed the nonfunctional ThreatCrowd source because its service hostnames terminate at deleted AWS load balancers and return NXDOMAIN; OTX remains available through its separate adapter. ### Fixed +- Sent a stable, versioned theHarvester identity with provider and API requests while preserving explicit browser identities for sources that require them. - Kept API endpoint scan URLs canonical instead of prefixing targets onto already complete URLs. - Made DeHashed pagination honor the CLI limit, retain only normalized email and IP evidence, and discard raw breach rows; aligned LeakIX with its authenticated subdomain endpoint and documented rate-limit retry. - Added offline contracts for explicitly selected DNS and direct sources, retained normalized Pentest-Tools host and IP results, and hardened Shodan InternetDB, SubdomainFinder C99, and Windvane evidence boundaries. diff --git a/tests/discovery/test_baidusearch.py b/tests/discovery/test_baidusearch.py index 1fe0a4ce..39c27f92 100644 --- a/tests/discovery/test_baidusearch.py +++ b/tests/discovery/test_baidusearch.py @@ -54,7 +54,7 @@ class TestBaiduSearch: response('Visit sub.a.example.com. baz@example.com'), ], ) - monkeypatch.setattr(baidusearch.Core, 'get_user_agent', staticmethod(lambda: 'UA')) + monkeypatch.setattr(baidusearch.Core, 'get_browser_user_agent', staticmethod(lambda: 'UA')) search = baidusearch.SearchBaidu(word='example.com', limit=21) await search.process(proxy=True) diff --git a/tests/discovery/test_dnsdb.py b/tests/discovery/test_dnsdb.py index 1c14da9b..677b9268 100644 --- a/tests/discovery/test_dnsdb.py +++ b/tests/discovery/test_dnsdb.py @@ -74,7 +74,7 @@ async def test_process_collects_normalized_in_scope_rrset_owners(monkeypatch: py assert isinstance(stream_options, dict) assert stream_options['headers'] == { 'Accept': 'application/x-ndjson', - 'User-Agent': f'theHarvester/{dnsdb.__version__}', + 'User-Agent': dnsdb.Core.get_user_agent(), 'X-API-Key': 'dnsdb-test-key', } assert stream_options['framing'] == 'ndjson' diff --git a/tests/discovery/test_sourcegraph.py b/tests/discovery/test_sourcegraph.py index 77dec9e8..cb6ee6dd 100644 --- a/tests/discovery/test_sourcegraph.py +++ b/tests/discovery/test_sourcegraph.py @@ -95,7 +95,6 @@ async def test_sourcegraph_uses_fixed_chunk_query_and_collects_descendants( event('done', {}), ) calls = install_stream(monkeypatch, records) - monkeypatch.setattr(sourcegraph.Core, 'get_user_agent', staticmethod(lambda: 'test-agent')) search = sourcegraph.SearchSourcegraph(' Scope.TEST. ', limit=1) await search.process(proxy=True) @@ -104,7 +103,7 @@ async def test_sourcegraph_uses_fixed_chunk_query_and_collects_descendants( { 'url': 'https://sourcegraph.com/.api/search/stream', 'framing': 'sse', - 'headers': {'Accept': 'text/event-stream', 'User-Agent': 'test-agent'}, + 'headers': {'Accept': 'text/event-stream'}, 'params': { 'q': '"scope.test" type:file count:5000 timeout:10s patternType:keyword', 'v': 'V3', diff --git a/tests/discovery/test_takeover.py b/tests/discovery/test_takeover.py index 9771571e..c10d3363 100644 --- a/tests/discovery/test_takeover.py +++ b/tests/discovery/test_takeover.py @@ -13,6 +13,7 @@ async def test_takeover_distinguishes_transport_failure_from_successful_empty_bo async def fake_fetch_all(urls, **kwargs): assert kwargs['include_metadata'] is True + assert kwargs['headers'] == {'User-Agent': takeover.Core.get_browser_user_agent()} assert set(urls) == { 'https://api.example.com', 'http://api.example.com', diff --git a/tests/discovery/test_yahoosearch.py b/tests/discovery/test_yahoosearch.py index 4513fdc9..e6438218 100644 --- a/tests/discovery/test_yahoosearch.py +++ b/tests/discovery/test_yahoosearch.py @@ -21,7 +21,7 @@ async def test_yahoo_uses_exact_pages_and_normalizes_evidence(monkeypatch: pytes 'Ignore outsider@example.net and api.example.net', ] - monkeypatch.setattr(yahoosearch.Core, 'get_user_agent', staticmethod(lambda: 'UA')) + monkeypatch.setattr(yahoosearch.Core, 'get_browser_user_agent', staticmethod(lambda: 'UA')) monkeypatch.setattr(yahoosearch.AsyncFetcher, 'fetch_all', fake_fetch_all) search = yahoosearch.SearchYahoo('example.com', 20) @@ -33,7 +33,7 @@ async def test_yahoo_uses_exact_pages_and_normalizes_evidence(monkeypatch: pytes 'https://search.yahoo.com/search?p=%40example.com&b=0&pz=10', 'https://search.yahoo.com/search?p=%40example.com&b=10&pz=10', ], - 'headers': {'Host': 'search.yahoo.com', 'User-agent': 'UA'}, + 'headers': {'Host': 'search.yahoo.com', 'User-Agent': 'UA'}, 'proxy': True, } ] diff --git a/tests/lib/test_core.py b/tests/lib/test_core.py index 005ba66b..6313f776 100644 --- a/tests/lib/test_core.py +++ b/tests/lib/test_core.py @@ -301,6 +301,17 @@ def test_api_keys_yaml_is_in_sync_with_core_accessors(): assert not missing_fields, f"Missing fields in api-keys.yaml: {missing_fields}" +def test_user_agent_policy_separates_provider_and_browser_identities() -> None: + assert Core.get_user_agent() == ( + f'theHarvester/{core_module.__version__} (+https://github.com/laramies/theHarvester)' + ) + assert Core.get_browser_user_agent() == ( + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' + 'AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/151.0.0.0 Safari/537.36' + ) + + @pytest.mark.parametrize( ("accessor_name", "expected"), [ @@ -352,6 +363,25 @@ async def test_fetch_creates_session_with_default_headers(monkeypatch) -> None: ] +def test_default_headers_add_project_identity_without_mutating_caller_headers(monkeypatch) -> None: + monkeypatch.setattr(Core, 'get_user_agent', staticmethod(lambda: 'test-agent')) + supplied = {'Accept': 'application/json'} + + headers = AsyncFetcher._default_headers(supplied) + + assert headers == {'Accept': 'application/json', 'User-Agent': 'test-agent'} + assert supplied == {'Accept': 'application/json'} + + +@pytest.mark.parametrize('header_name', ['User-Agent', 'User-agent', 'user-agent']) +def test_default_headers_preserve_explicit_user_agent_case_insensitively(monkeypatch, header_name: str) -> None: + monkeypatch.setattr(Core, 'get_user_agent', staticmethod(lambda: 'default-agent')) + + headers = AsyncFetcher._default_headers({header_name: 'caller-agent', 'Accept': 'application/json'}) + + assert headers == {header_name: 'caller-agent', 'Accept': 'application/json'} + + @pytest.mark.asyncio async def test_fetch_can_include_buffered_response_metadata(monkeypatch) -> None: reset_dummy_sessions() @@ -882,6 +912,7 @@ async def test_takeover_fetch_uses_the_shared_transport( session, url, proxy=proxy, + headers={'User-Agent': 'browser-agent'}, ) assert result == (url, 'response-text') @@ -890,6 +921,7 @@ async def test_takeover_fetch_uses_the_shared_transport( 'session': session if uses_shared_session else None, 'url': url, 'proxy': proxy, + 'headers': {'User-Agent': 'browser-agent'}, 'request_timeout': 15, 'include_metadata': False, } @@ -913,7 +945,11 @@ async def test_takeover_fetch_all_falls_back_to_direct_when_proxy_pool_is_empty( assert result == [('http://example.com', 'direct response')] assert len(calls) == 1 - assert calls[0][1] == {'proxy': None, 'include_metadata': False} + assert calls[0][1] == { + 'proxy': None, + 'headers': {'User-Agent': Core.get_user_agent()}, + 'include_metadata': False, + } @pytest.mark.asyncio diff --git a/tests/test_mojeek.py b/tests/test_mojeek.py index df11a2d9..dddbb5b0 100644 --- a/tests/test_mojeek.py +++ b/tests/test_mojeek.py @@ -37,7 +37,7 @@ class TestMojeekSearch: raise AssertionError('keyless Mojeek pages must be requested sequentially') monkeypatch.setattr(mojeek.Core, 'mojeek_key', staticmethod(lambda: '')) - monkeypatch.setattr(mojeek.Core, 'get_user_agent', staticmethod(lambda: 'UA')) + monkeypatch.setattr(mojeek.Core, 'get_browser_user_agent', staticmethod(lambda: 'UA')) monkeypatch.setattr(mojeek.AsyncFetcher, 'fetch', fake_fetch) monkeypatch.setattr(mojeek.AsyncFetcher, 'fetch_all', reject_fetch_all) monkeypatch.setattr(mojeek.asyncio, 'sleep', fake_sleep) @@ -50,6 +50,7 @@ class TestMojeekSearch: 'https://www.mojeek.com/search?q=example.com&s=10', ] assert all(call['include_metadata'] is True for call in calls) + assert all(call['headers'] == {'User-Agent': 'UA'} for call in calls) assert all(call['follow_redirects'] is False for call in calls) assert all(call['proxy'] is True for call in calls) assert delays == [1.0] diff --git a/theHarvester/discovery/api_endpoints.py b/theHarvester/discovery/api_endpoints.py index 6a50fd85..0f86d5dc 100644 --- a/theHarvester/discovery/api_endpoints.py +++ b/theHarvester/discovery/api_endpoints.py @@ -65,7 +65,7 @@ class SearchApiEndpoints: concurrency: Maximum number of requests in flight. timeout: Timeout for each request, in seconds. proxy: Optional HTTP proxy URL. - user_agent: HTTP User-Agent value. The default comes from ``Core``. + user_agent: HTTP User-Agent value. The default is the shared Chrome identity. follow_redirects: Whether requests follow redirects. verify_ssl: Whether to verify TLS certificates. additional_headers: Extra HTTP headers to send. @@ -91,7 +91,7 @@ class SearchApiEndpoints: self.follow_redirects = follow_redirects self.verify_ssl = verify_ssl self.semaphore = asyncio.Semaphore(concurrency) - self.user_agent = user_agent or Core.get_user_agent() + self.user_agent = user_agent or Core.get_browser_user_agent() self.additional_headers = additional_headers or {} self._session: aiohttp.ClientSession | None = None self.scan_error_type: str | None = None diff --git a/theHarvester/discovery/baidusearch.py b/theHarvester/discovery/baidusearch.py index 5bd9593a..dd620b09 100644 --- a/theHarvester/discovery/baidusearch.py +++ b/theHarvester/discovery/baidusearch.py @@ -21,7 +21,7 @@ class SearchBaidu: async def do_search(self) -> None: self.execution_status = None self.stop_reason = None - headers = {'Host': self.hostname, 'User-agent': Core.get_user_agent()} + headers = {'Host': self.hostname, 'User-Agent': Core.get_browser_user_agent()} base_url = f'https://{self.server}/s' urls = [ f'{base_url}?{urlencode({"wd": f"site:{self.word}", "pn": num})}' diff --git a/theHarvester/discovery/builtwith.py b/theHarvester/discovery/builtwith.py index d2710ff0..f062d934 100644 --- a/theHarvester/discovery/builtwith.py +++ b/theHarvester/discovery/builtwith.py @@ -16,7 +16,11 @@ class SearchBuiltWith: if self.api_key is None: raise MissingKey('BuiltWith') self.base_url = 'https://api.builtwith.com/v21/api.json' - self.headers = {'Authorization': f'Bearer {self.api_key}', 'Content-Type': 'application/json'} + self.headers = { + 'Authorization': f'Bearer {self.api_key}', + 'Content-Type': 'application/json', + 'User-Agent': Core.get_user_agent(), + } self.hosts: set[str] = set() self.tech_stack: dict[str, Any] = {} self.urls: set[str] = set() diff --git a/theHarvester/discovery/constants.py b/theHarvester/discovery/constants.py index ae6b95d4..d4e04a7b 100644 --- a/theHarvester/discovery/constants.py +++ b/theHarvester/discovery/constants.py @@ -78,7 +78,7 @@ async def google_workaround(visit_url: str) -> bool | str: 'type': 'GET&http=1.1', 'uak': str(random.randint(4, 8)), # select random UA to send to Google } - returned_html = await AsyncFetcher.post_fetch(url, headers={'User-Agent': Core.get_user_agent()}, data=data) + returned_html = await AsyncFetcher.post_fetch(url, headers={'User-Agent': Core.get_browser_user_agent()}, data=data) returned_html = ( 'This page appears when Google automatically detects requests coming from your computer network' if returned_html == '' diff --git a/theHarvester/discovery/dnsdb.py b/theHarvester/discovery/dnsdb.py index 4dbb99ce..52794cc9 100644 --- a/theHarvester/discovery/dnsdb.py +++ b/theHarvester/discovery/dnsdb.py @@ -4,7 +4,6 @@ import json import logging from urllib.parse import quote -from theHarvester import __version__ from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import AsyncFetcher, Core, ResponseStreamError @@ -48,7 +47,7 @@ class SearchDNSDB: url = f'{self.BASE_URL}/{query}?limit=0' headers = { 'Accept': 'application/x-ndjson', - 'User-Agent': f'theHarvester/{__version__}', + 'User-Agent': Core.get_user_agent(), 'X-API-Key': self.key, } async with AsyncFetcher.stream_records( diff --git a/theHarvester/discovery/haveibeenpwned.py b/theHarvester/discovery/haveibeenpwned.py index e616c05d..8203a79e 100644 --- a/theHarvester/discovery/haveibeenpwned.py +++ b/theHarvester/discovery/haveibeenpwned.py @@ -1,6 +1,6 @@ import logging -from theHarvester.lib.core import AsyncFetcher, FetcherResponse +from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse logger = logging.getLogger(__name__) @@ -9,7 +9,7 @@ class SearchHaveIBeenPwned: def __init__(self, word: str): self.word = word self.base_url = 'https://haveibeenpwned.com/api/v3' - self.headers = {'user-agent': 'theHarvester', 'Content-Type': 'application/json'} + self.headers = {'User-Agent': Core.get_user_agent(), 'Content-Type': 'application/json'} self.hosts: set[str] = set() self.emails: set[str] = set() self.breaches: list[dict] = [] diff --git a/theHarvester/discovery/hibpverified.py b/theHarvester/discovery/hibpverified.py index 00336e07..d7cb39ad 100644 --- a/theHarvester/discovery/hibpverified.py +++ b/theHarvester/discovery/hibpverified.py @@ -13,7 +13,7 @@ class SearchHibpVerified: if not self.api_key: raise MissingKey('HIBP verified domain') self.base_url = 'https://haveibeenpwned.com/api/v3' - self.headers = {'hibp-api-key': self.api_key, 'user-agent': 'theHarvester'} + self.headers = {'hibp-api-key': self.api_key, 'User-Agent': Core.get_user_agent()} self.emails: set[str] = set() self.breach_names: set[str] = set() diff --git a/theHarvester/discovery/intelxsearch.py b/theHarvester/discovery/intelxsearch.py index 03fd5341..11b45cde 100644 --- a/theHarvester/discovery/intelxsearch.py +++ b/theHarvester/discovery/intelxsearch.py @@ -39,7 +39,7 @@ class SearchIntelx: try: headers = { 'x-key': self.key, - 'User-Agent': f'{Core.get_user_agent()}-theHarvester', + 'User-Agent': Core.get_user_agent(), 'Content-Type': 'application/json', } data = { diff --git a/theHarvester/discovery/mojeek.py b/theHarvester/discovery/mojeek.py index c01f7283..37bf5ca1 100644 --- a/theHarvester/discovery/mojeek.py +++ b/theHarvester/discovery/mojeek.py @@ -143,7 +143,8 @@ class SearchMojeek: async def do_search(self) -> None: self.execution_status = None self.stop_reason = None - headers = {'User-Agent': Core.get_user_agent()} + user_agent = Core.get_user_agent() if self.api_key else Core.get_browser_user_agent() + headers = {'User-Agent': user_agent} if self.api_key: await self._search_api(headers) else: diff --git a/theHarvester/discovery/rapiddns.py b/theHarvester/discovery/rapiddns.py index d13df279..90fdb4a8 100644 --- a/theHarvester/discovery/rapiddns.py +++ b/theHarvester/discovery/rapiddns.py @@ -19,7 +19,7 @@ class SearchRapidDns: async def do_search(self): try: - headers = {'User-agent': Core.get_user_agent()} + headers = {'User-Agent': Core.get_browser_user_agent()} # TODO see if it's worth adding sameip searches # f'{self.hostname}/sameip/{self.word}?full=1#result' urls = [f'https://rapiddns.io/subdomain/{self.word}?full=1#result'] diff --git a/theHarvester/discovery/search_dnsdumpster.py b/theHarvester/discovery/search_dnsdumpster.py index d50ddc5b..e174ab7a 100644 --- a/theHarvester/discovery/search_dnsdumpster.py +++ b/theHarvester/discovery/search_dnsdumpster.py @@ -24,7 +24,7 @@ class SearchDNSDumpster: async def do_search(self) -> None: url = f'{self.base_url}/domain/{self.word}' - headers = {'User-Agent': 'Mozilla/5.0 (theHarvester)', 'X-API-Key': self.key} + headers = {'User-Agent': Core.get_user_agent(), 'X-API-Key': self.key} try: response = await AsyncFetcher.fetch_all( [url], diff --git a/theHarvester/discovery/securityscorecard.py b/theHarvester/discovery/securityscorecard.py index 4b9f030b..f785ba4d 100644 --- a/theHarvester/discovery/securityscorecard.py +++ b/theHarvester/discovery/securityscorecard.py @@ -15,7 +15,11 @@ class SearchSecurityScorecard: if self.api_key is None: raise MissingKey('SecurityScorecard') self.base_url = 'https://api.securityscorecard.io' - self.headers = {'Authorization': f'Token {self.api_key}', 'Content-Type': 'application/json'} + self.headers = { + 'Authorization': f'Token {self.api_key}', + 'Content-Type': 'application/json', + 'User-Agent': Core.get_user_agent(), + } self.hosts: set[str] = set() self.score: int = 0 self.grades: dict = {} diff --git a/theHarvester/discovery/sourcegraph.py b/theHarvester/discovery/sourcegraph.py index 83a15ebb..21153454 100644 --- a/theHarvester/discovery/sourcegraph.py +++ b/theHarvester/discovery/sourcegraph.py @@ -4,7 +4,7 @@ import json import re from typing import Any -from theHarvester.lib.core import AsyncFetcher, Core, ResponseStreamError +from theHarvester.lib.core import AsyncFetcher, ResponseStreamError _HOST_TOKEN = re.compile( r'(? None: # Based on https://gist.github.com/th3gundy/bc83580cbe04031e9164362b33600962 - headers = {'User-Agent': Core.get_user_agent()} + headers = {'User-Agent': Core.get_browser_user_agent()} resp = await AsyncFetcher.fetch_all([self.server], headers=headers, proxy=self.proxy) if not resp or not isinstance(resp[0], str): return diff --git a/theHarvester/discovery/takeover.py b/theHarvester/discovery/takeover.py index 47ee3953..b7ae4080 100644 --- a/theHarvester/discovery/takeover.py +++ b/theHarvester/discovery/takeover.py @@ -97,6 +97,7 @@ class TakeOver: shuffle(all_hosts) responses: list[tuple[str, FetcherResponse | None]] = await AsyncFetcher.fetch_all( all_hosts, + headers={'User-Agent': Core.get_browser_user_agent()}, takeover=True, proxy=self.proxy, include_metadata=True, diff --git a/theHarvester/discovery/yahoosearch.py b/theHarvester/discovery/yahoosearch.py index dbe65ab5..84dd469e 100644 --- a/theHarvester/discovery/yahoosearch.py +++ b/theHarvester/discovery/yahoosearch.py @@ -12,7 +12,7 @@ class SearchYahoo: async def do_search(self) -> None: base_url = f'https://{self.server}/search?p=%40{self.word}&b=xx&pz=10' - headers = {'Host': self.server, 'User-agent': Core.get_user_agent()} + headers = {'Host': self.server, 'User-Agent': Core.get_browser_user_agent()} urls = [base_url.replace('xx', str(num)) for num in range(0, self.limit, 10) if num <= self.limit] responses = await AsyncFetcher.fetch_all(urls, headers=headers, proxy=self.proxy) for response in responses: diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index abb4ff66..74c5c258 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -472,70 +472,13 @@ class Core: @staticmethod def get_user_agent() -> str: - # User-Agents from https://techblog.willshouse.com/2012/01/03/most-common-user-agents/ - # Lasted updated 21-12-25 - user_agents = [ - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:145.0) Gecko/20100101 Firefox/145.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64; rv:145.0) Gecko/20100101 Firefox/145.0', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36 Edg/142.0.0.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:146.0) Gecko/20100101 Firefox/146.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:145.0) Gecko/20100101 Firefox/145.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36 Edg/143.0.0.0', - 'Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.1 Safari/605.1.15', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64; rv:146.0) Gecko/20100101 Firefox/146.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:146.0) Gecko/20100101 Firefox/146.0', - 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:145.0) Gecko/20100101 Firefox/145.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64; rv:144.0) Gecko/20100101 Firefox/144.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.2 Safari/605.1.15', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36 Edg/142.0.0.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0', - 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:146.0) Gecko/20100101 Firefox/146.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0', - 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:136.0) Gecko/20100101 Firefox/136.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:144.0) Gecko/20100101 Firefox/144.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36 OPR/124.0.0.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko)', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36 OPR/123.0.0.0', - 'Mozilla/5.0 (X11; Linux x86_64; rv:143.0) Gecko/20100101 Firefox/143.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36; Manus-User/1.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36 Edg/143.0.0.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.0.1 Safari/605.1.15', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/119.0', - 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.5 Safari/605.1.15', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36 Edg/141.0.0.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/109.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:136.0) Gecko/20100101 Firefox/136.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:144.0) Gecko/20100101 Firefox/144.0', - 'Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64; rv:139.0) Gecko/20100101 Firefox/139.0', - 'Mozilla/5.0 (X11; Linux x86_64; rv:142.0) Gecko/20100101 Firefox/142.0', - ] - return random.choice(user_agents) + """Return the stable identity used for provider and API requests.""" + return f'theHarvester/{__version__} (+https://github.com/laramies/theHarvester)' + + @staticmethod + def get_browser_user_agent() -> str: + """Return the Chrome identity used only for browser-oriented sources.""" + return 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36' class AsyncFetcher: @@ -553,7 +496,10 @@ class AsyncFetcher: @staticmethod def _default_headers(headers: dict[str, str] | None = None) -> dict[str, str]: - return headers or {'User-Agent': Core.get_user_agent()} + request_headers = dict(headers or {}) + if not any(name.lower() == 'user-agent' for name in request_headers): + request_headers['User-Agent'] = Core.get_user_agent() + return request_headers @staticmethod def _ssl_context(verify: bool | None = True) -> ssl.SSLContext | bool: @@ -934,6 +880,7 @@ class AsyncFetcher: session, url: str, proxy: str | None = None, + headers: dict[str, str] | None = None, include_metadata: bool = False, ) -> tuple[Any, Any] | str: _, proxy_type = AsyncFetcher._resolve_proxy(proxy) @@ -941,6 +888,7 @@ class AsyncFetcher: session=None if proxy_type == 'socks5' else session, url=url, proxy=proxy, + headers=headers, request_timeout=15, include_metadata=include_metadata, ) @@ -975,6 +923,7 @@ class AsyncFetcher: session, url, proxy=proxy_url, + headers=headers, include_metadata=include_metadata, ) for url, proxy_url in zip(urls, proxy_urls, strict=False) @@ -984,7 +933,15 @@ class AsyncFetcher: else: return list( await asyncio.gather( - *[AsyncFetcher.takeover_fetch(session, url, include_metadata=include_metadata) for url in urls] + *[ + AsyncFetcher.takeover_fetch( + session, + url, + headers=headers, + include_metadata=include_metadata, + ) + for url in urls + ] ) )