fix: use explicit provider and browser user agents (#2530)

CHANGELOG.md

@@ -62,6 +62,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Removed the nonfunctional ThreatCrowd source because its service hostnames terminate at deleted AWS load balancers and return NXDOMAIN; OTX remains available through its separate adapter.
 
 ### Fixed
+- Sent a stable, versioned theHarvester identity with provider and API requests while preserving explicit browser identities for sources that require them.
 - Kept API endpoint scan URLs canonical instead of prefixing targets onto already complete URLs.
 - Made DeHashed pagination honor the CLI limit, retain only normalized email and IP evidence, and discard raw breach rows; aligned LeakIX with its authenticated subdomain endpoint and documented rate-limit retry.
 - Added offline contracts for explicitly selected DNS and direct sources, retained normalized Pentest-Tools host and IP results, and hardened Shodan InternetDB, SubdomainFinder C99, and Windvane evidence boundaries.

tests/discovery/test_baidusearch.py

@@ -54,7 +54,7 @@ class TestBaiduSearch:
                 response('Visit sub.a.example.com. baz@example.com'),
             ],
         )
-        monkeypatch.setattr(baidusearch.Core, 'get_user_agent', staticmethod(lambda: 'UA'))
+        monkeypatch.setattr(baidusearch.Core, 'get_browser_user_agent', staticmethod(lambda: 'UA'))
 
         search = baidusearch.SearchBaidu(word='example.com', limit=21)
         await search.process(proxy=True)

tests/discovery/test_dnsdb.py

@@ -74,7 +74,7 @@ async def test_process_collects_normalized_in_scope_rrset_owners(monkeypatch: py
     assert isinstance(stream_options, dict)
     assert stream_options['headers'] == {
         'Accept': 'application/x-ndjson',
-        'User-Agent': f'theHarvester/{dnsdb.__version__}',
+        'User-Agent': dnsdb.Core.get_user_agent(),
         'X-API-Key': 'dnsdb-test-key',
     }
     assert stream_options['framing'] == 'ndjson'

tests/discovery/test_sourcegraph.py

@@ -95,7 +95,6 @@ async def test_sourcegraph_uses_fixed_chunk_query_and_collects_descendants(
         event('done', {}),
     )
     calls = install_stream(monkeypatch, records)
-    monkeypatch.setattr(sourcegraph.Core, 'get_user_agent', staticmethod(lambda: 'test-agent'))
     search = sourcegraph.SearchSourcegraph(' Scope.TEST. ', limit=1)
 
     await search.process(proxy=True)
@@ -104,7 +103,7 @@ async def test_sourcegraph_uses_fixed_chunk_query_and_collects_descendants(
         {
             'url': 'https://sourcegraph.com/.api/search/stream',
             'framing': 'sse',
-            'headers': {'Accept': 'text/event-stream', 'User-Agent': 'test-agent'},
+            'headers': {'Accept': 'text/event-stream'},
             'params': {
                 'q': '"scope.test" type:file count:5000 timeout:10s patternType:keyword',
                 'v': 'V3',

tests/discovery/test_takeover.py

@@ -13,6 +13,7 @@ async def test_takeover_distinguishes_transport_failure_from_successful_empty_bo
 
     async def fake_fetch_all(urls, **kwargs):
         assert kwargs['include_metadata'] is True
+        assert kwargs['headers'] == {'User-Agent': takeover.Core.get_browser_user_agent()}
         assert set(urls) == {
             'https://api.example.com',
             'http://api.example.com',

tests/discovery/test_yahoosearch.py

@@ -21,7 +21,7 @@ async def test_yahoo_uses_exact_pages_and_normalizes_evidence(monkeypatch: pytes
             'Ignore outsider@example.net and api.example.net',
         ]
 
-    monkeypatch.setattr(yahoosearch.Core, 'get_user_agent', staticmethod(lambda: 'UA'))
+    monkeypatch.setattr(yahoosearch.Core, 'get_browser_user_agent', staticmethod(lambda: 'UA'))
     monkeypatch.setattr(yahoosearch.AsyncFetcher, 'fetch_all', fake_fetch_all)
 
     search = yahoosearch.SearchYahoo('example.com', 20)
@@ -33,7 +33,7 @@ async def test_yahoo_uses_exact_pages_and_normalizes_evidence(monkeypatch: pytes
                 'https://search.yahoo.com/search?p=%40example.com&b=0&pz=10',
                 'https://search.yahoo.com/search?p=%40example.com&b=10&pz=10',
             ],
-            'headers': {'Host': 'search.yahoo.com', 'User-agent': 'UA'},
+            'headers': {'Host': 'search.yahoo.com', 'User-Agent': 'UA'},
             'proxy': True,
         }
     ]

tests/lib/test_core.py

@@ -301,6 +301,17 @@ def test_api_keys_yaml_is_in_sync_with_core_accessors():
     assert not missing_fields, f"Missing fields in api-keys.yaml: {missing_fields}"
 
 
+def test_user_agent_policy_separates_provider_and_browser_identities() -> None:
+    assert Core.get_user_agent() == (
+        f'theHarvester/{core_module.__version__} (+https://github.com/laramies/theHarvester)'
+    )
+    assert Core.get_browser_user_agent() == (
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
+        'AppleWebKit/537.36 (KHTML, like Gecko) '
+        'Chrome/151.0.0.0 Safari/537.36'
+    )
+
+
 @pytest.mark.parametrize(
     ("accessor_name", "expected"),
     [
@@ -352,6 +363,25 @@ async def test_fetch_creates_session_with_default_headers(monkeypatch) -> None:
     ]
 
 
+def test_default_headers_add_project_identity_without_mutating_caller_headers(monkeypatch) -> None:
+    monkeypatch.setattr(Core, 'get_user_agent', staticmethod(lambda: 'test-agent'))
+    supplied = {'Accept': 'application/json'}
+
+    headers = AsyncFetcher._default_headers(supplied)
+
+    assert headers == {'Accept': 'application/json', 'User-Agent': 'test-agent'}
+    assert supplied == {'Accept': 'application/json'}
+
+
+@pytest.mark.parametrize('header_name', ['User-Agent', 'User-agent', 'user-agent'])
+def test_default_headers_preserve_explicit_user_agent_case_insensitively(monkeypatch, header_name: str) -> None:
+    monkeypatch.setattr(Core, 'get_user_agent', staticmethod(lambda: 'default-agent'))
+
+    headers = AsyncFetcher._default_headers({header_name: 'caller-agent', 'Accept': 'application/json'})
+
+    assert headers == {header_name: 'caller-agent', 'Accept': 'application/json'}
+
+
 @pytest.mark.asyncio
 async def test_fetch_can_include_buffered_response_metadata(monkeypatch) -> None:
     reset_dummy_sessions()
@@ -882,6 +912,7 @@ async def test_takeover_fetch_uses_the_shared_transport(
         session,
         url,
         proxy=proxy,
+        headers={'User-Agent': 'browser-agent'},
     )
 
     assert result == (url, 'response-text')
@@ -890,6 +921,7 @@ async def test_takeover_fetch_uses_the_shared_transport(
             'session': session if uses_shared_session else None,
             'url': url,
             'proxy': proxy,
+            'headers': {'User-Agent': 'browser-agent'},
             'request_timeout': 15,
             'include_metadata': False,
         }
@@ -913,7 +945,11 @@ async def test_takeover_fetch_all_falls_back_to_direct_when_proxy_pool_is_empty(
 
     assert result == [('http://example.com', 'direct response')]
     assert len(calls) == 1
-    assert calls[0][1] == {'proxy': None, 'include_metadata': False}
+    assert calls[0][1] == {
+        'proxy': None,
+        'headers': {'User-Agent': Core.get_user_agent()},
+        'include_metadata': False,
+    }
 
 
 @pytest.mark.asyncio

tests/test_mojeek.py

@@ -37,7 +37,7 @@ class TestMojeekSearch:
             raise AssertionError('keyless Mojeek pages must be requested sequentially')
 
         monkeypatch.setattr(mojeek.Core, 'mojeek_key', staticmethod(lambda: ''))
-        monkeypatch.setattr(mojeek.Core, 'get_user_agent', staticmethod(lambda: 'UA'))
+        monkeypatch.setattr(mojeek.Core, 'get_browser_user_agent', staticmethod(lambda: 'UA'))
         monkeypatch.setattr(mojeek.AsyncFetcher, 'fetch', fake_fetch)
         monkeypatch.setattr(mojeek.AsyncFetcher, 'fetch_all', reject_fetch_all)
         monkeypatch.setattr(mojeek.asyncio, 'sleep', fake_sleep)
@@ -50,6 +50,7 @@ class TestMojeekSearch:
             'https://www.mojeek.com/search?q=example.com&s=10',
         ]
         assert all(call['include_metadata'] is True for call in calls)
+        assert all(call['headers'] == {'User-Agent': 'UA'} for call in calls)
         assert all(call['follow_redirects'] is False for call in calls)
         assert all(call['proxy'] is True for call in calls)
         assert delays == [1.0]

theHarvester/discovery/api_endpoints.py

@@ -65,7 +65,7 @@ class SearchApiEndpoints:
             concurrency: Maximum number of requests in flight.
             timeout: Timeout for each request, in seconds.
             proxy: Optional HTTP proxy URL.
-            user_agent: HTTP User-Agent value. The default comes from ``Core``.
+            user_agent: HTTP User-Agent value. The default is the shared Chrome identity.
             follow_redirects: Whether requests follow redirects.
             verify_ssl: Whether to verify TLS certificates.
             additional_headers: Extra HTTP headers to send.
@@ -91,7 +91,7 @@ class SearchApiEndpoints:
         self.follow_redirects = follow_redirects
         self.verify_ssl = verify_ssl
         self.semaphore = asyncio.Semaphore(concurrency)
-        self.user_agent = user_agent or Core.get_user_agent()
+        self.user_agent = user_agent or Core.get_browser_user_agent()
         self.additional_headers = additional_headers or {}
         self._session: aiohttp.ClientSession | None = None
         self.scan_error_type: str | None = None

theHarvester/discovery/baidusearch.py

@@ -21,7 +21,7 @@ class SearchBaidu:
     async def do_search(self) -> None:
         self.execution_status = None
         self.stop_reason = None
-        headers = {'Host': self.hostname, 'User-agent': Core.get_user_agent()}
+        headers = {'Host': self.hostname, 'User-Agent': Core.get_browser_user_agent()}
         base_url = f'https://{self.server}/s'
         urls = [
             f'{base_url}?{urlencode({"wd": f"site:{self.word}", "pn": num})}'

theHarvester/discovery/builtwith.py

@@ -16,7 +16,11 @@ class SearchBuiltWith:
         if self.api_key is None:
             raise MissingKey('BuiltWith')
         self.base_url = 'https://api.builtwith.com/v21/api.json'
-        self.headers = {'Authorization': f'Bearer {self.api_key}', 'Content-Type': 'application/json'}
+        self.headers = {
+            'Authorization': f'Bearer {self.api_key}',
+            'Content-Type': 'application/json',
+            'User-Agent': Core.get_user_agent(),
+        }
         self.hosts: set[str] = set()
         self.tech_stack: dict[str, Any] = {}
         self.urls: set[str] = set()

theHarvester/discovery/constants.py

@@ -78,7 +78,7 @@ async def google_workaround(visit_url: str) -> bool | str:
         'type': 'GET&http=1.1',
         'uak': str(random.randint(4, 8)),  # select random UA to send to Google
     }
-    returned_html = await AsyncFetcher.post_fetch(url, headers={'User-Agent': Core.get_user_agent()}, data=data)
+    returned_html = await AsyncFetcher.post_fetch(url, headers={'User-Agent': Core.get_browser_user_agent()}, data=data)
     returned_html = (
         'This page appears when Google automatically detects requests coming from your computer network'
         if returned_html == ''

theHarvester/discovery/dnsdb.py

@@ -4,7 +4,6 @@ import json
 import logging
 from urllib.parse import quote
 
-from theHarvester import __version__
 from theHarvester.discovery.constants import MissingKey
 from theHarvester.lib.core import AsyncFetcher, Core, ResponseStreamError
 
@@ -48,7 +47,7 @@ class SearchDNSDB:
         url = f'{self.BASE_URL}/{query}?limit=0'
         headers = {
             'Accept': 'application/x-ndjson',
-            'User-Agent': f'theHarvester/{__version__}',
+            'User-Agent': Core.get_user_agent(),
             'X-API-Key': self.key,
         }
         async with AsyncFetcher.stream_records(

theHarvester/discovery/haveibeenpwned.py

@@ -1,6 +1,6 @@
 import logging
 
-from theHarvester.lib.core import AsyncFetcher, FetcherResponse
+from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse
 
 logger = logging.getLogger(__name__)
 
@@ -9,7 +9,7 @@ class SearchHaveIBeenPwned:
     def __init__(self, word: str):
         self.word = word
         self.base_url = 'https://haveibeenpwned.com/api/v3'
-        self.headers = {'user-agent': 'theHarvester', 'Content-Type': 'application/json'}
+        self.headers = {'User-Agent': Core.get_user_agent(), 'Content-Type': 'application/json'}
         self.hosts: set[str] = set()
         self.emails: set[str] = set()
         self.breaches: list[dict] = []

theHarvester/discovery/hibpverified.py

@@ -13,7 +13,7 @@ class SearchHibpVerified:
         if not self.api_key:
             raise MissingKey('HIBP verified domain')
         self.base_url = 'https://haveibeenpwned.com/api/v3'
-        self.headers = {'hibp-api-key': self.api_key, 'user-agent': 'theHarvester'}
+        self.headers = {'hibp-api-key': self.api_key, 'User-Agent': Core.get_user_agent()}
         self.emails: set[str] = set()
         self.breach_names: set[str] = set()
 

theHarvester/discovery/intelxsearch.py

@@ -39,7 +39,7 @@ class SearchIntelx:
         try:
             headers = {
                 'x-key': self.key,
-                'User-Agent': f'{Core.get_user_agent()}-theHarvester',
+                'User-Agent': Core.get_user_agent(),
                 'Content-Type': 'application/json',
             }
             data = {

theHarvester/discovery/mojeek.py

@@ -143,7 +143,8 @@ class SearchMojeek:
     async def do_search(self) -> None:
         self.execution_status = None
         self.stop_reason = None
-        headers = {'User-Agent': Core.get_user_agent()}
+        user_agent = Core.get_user_agent() if self.api_key else Core.get_browser_user_agent()
+        headers = {'User-Agent': user_agent}
         if self.api_key:
             await self._search_api(headers)
         else:

theHarvester/discovery/rapiddns.py

@@ -19,7 +19,7 @@ class SearchRapidDns:
 
     async def do_search(self):
         try:
-            headers = {'User-agent': Core.get_user_agent()}
+            headers = {'User-Agent': Core.get_browser_user_agent()}
             # TODO see if it's worth adding sameip searches
             # f'{self.hostname}/sameip/{self.word}?full=1#result'
             urls = [f'https://rapiddns.io/subdomain/{self.word}?full=1#result']

theHarvester/discovery/search_dnsdumpster.py

@@ -24,7 +24,7 @@ class SearchDNSDumpster:
 
     async def do_search(self) -> None:
         url = f'{self.base_url}/domain/{self.word}'
-        headers = {'User-Agent': 'Mozilla/5.0 (theHarvester)', 'X-API-Key': self.key}
+        headers = {'User-Agent': Core.get_user_agent(), 'X-API-Key': self.key}
         try:
             response = await AsyncFetcher.fetch_all(
                 [url],

theHarvester/discovery/securityscorecard.py

@@ -15,7 +15,11 @@ class SearchSecurityScorecard:
         if self.api_key is None:
             raise MissingKey('SecurityScorecard')
         self.base_url = 'https://api.securityscorecard.io'
-        self.headers = {'Authorization': f'Token {self.api_key}', 'Content-Type': 'application/json'}
+        self.headers = {
+            'Authorization': f'Token {self.api_key}',
+            'Content-Type': 'application/json',
+            'User-Agent': Core.get_user_agent(),
+        }
         self.hosts: set[str] = set()
         self.score: int = 0
         self.grades: dict = {}

theHarvester/discovery/sourcegraph.py

@@ -4,7 +4,7 @@ import json
 import re
 from typing import Any
 
-from theHarvester.lib.core import AsyncFetcher, Core, ResponseStreamError
+from theHarvester.lib.core import AsyncFetcher, ResponseStreamError
 
 _HOST_TOKEN = re.compile(
     r'(?<![\w*.-])(?:\*|[a-z0-9-]+)(?:\.(?:\*|[a-z0-9-]+))+\.?(?![\w*.-])',
@@ -174,7 +174,7 @@ class SearchSourcegraph:
             async with AsyncFetcher.stream_records(
                 self.ENDPOINT,
                 framing='sse',
-                headers={'Accept': 'text/event-stream', 'User-Agent': Core.get_user_agent()},
+                headers={'Accept': 'text/event-stream'},
                 params=params,
                 proxy=self.proxy,
                 follow_redirects=False,

theHarvester/discovery/subdomainfinderc99.py

@@ -20,7 +20,7 @@ class SearchSubdomainfinderc99:
 
     async def do_search(self) -> None:
         # Based on https://gist.github.com/th3gundy/bc83580cbe04031e9164362b33600962
-        headers = {'User-Agent': Core.get_user_agent()}
+        headers = {'User-Agent': Core.get_browser_user_agent()}
         resp = await AsyncFetcher.fetch_all([self.server], headers=headers, proxy=self.proxy)
         if not resp or not isinstance(resp[0], str):
             return

theHarvester/discovery/takeover.py

@@ -97,6 +97,7 @@ class TakeOver:
                 shuffle(all_hosts)
                 responses: list[tuple[str, FetcherResponse | None]] = await AsyncFetcher.fetch_all(
                     all_hosts,
+                    headers={'User-Agent': Core.get_browser_user_agent()},
                     takeover=True,
                     proxy=self.proxy,
                     include_metadata=True,

theHarvester/discovery/yahoosearch.py

@@ -12,7 +12,7 @@ class SearchYahoo:
 
     async def do_search(self) -> None:
         base_url = f'https://{self.server}/search?p=%40{self.word}&b=xx&pz=10'
-        headers = {'Host': self.server, 'User-agent': Core.get_user_agent()}
+        headers = {'Host': self.server, 'User-Agent': Core.get_browser_user_agent()}
         urls = [base_url.replace('xx', str(num)) for num in range(0, self.limit, 10) if num <= self.limit]
         responses = await AsyncFetcher.fetch_all(urls, headers=headers, proxy=self.proxy)
         for response in responses:

theHarvester/lib/core.py

@@ -472,70 +472,13 @@ class Core:
 
     @staticmethod
     def get_user_agent() -> str:
-        # User-Agents from https://techblog.willshouse.com/2012/01/03/most-common-user-agents/
+        """Return the stable identity used for provider and API requests."""
-        # Lasted updated 21-12-25
+        return f'theHarvester/{__version__} (+https://github.com/laramies/theHarvester)'
-        user_agents = [
+
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36',
+    @staticmethod
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36',
+    def get_browser_user_agent() -> str:
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36',
+        """Return the Chrome identity used only for browser-oriented sources."""
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:145.0) Gecko/20100101 Firefox/145.0',
+        return 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36'
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36',
-            'Mozilla/5.0 (X11; Linux x86_64; rv:145.0) Gecko/20100101 Firefox/145.0',
-            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36',
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36 Edg/142.0.0.0',
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:146.0) Gecko/20100101 Firefox/146.0',
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:145.0) Gecko/20100101 Firefox/145.0',
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36',
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36 Edg/143.0.0.0',
-            'Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0',
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36',
-            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36',
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.1 Safari/605.1.15',
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
-            'Mozilla/5.0 (X11; Linux x86_64; rv:146.0) Gecko/20100101 Firefox/146.0',
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15',
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:146.0) Gecko/20100101 Firefox/146.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:145.0) Gecko/20100101 Firefox/145.0',
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36',
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0',
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36',
-            'Mozilla/5.0 (X11; Linux x86_64; rv:144.0) Gecko/20100101 Firefox/144.0',
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.2 Safari/605.1.15',
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36 Edg/142.0.0.0',
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36',
-            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
-            'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:146.0) Gecko/20100101 Firefox/146.0',
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
-            'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:136.0) Gecko/20100101 Firefox/136.0',
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:144.0) Gecko/20100101 Firefox/144.0',
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36 OPR/124.0.0.0',
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0',
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko)',
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36 OPR/123.0.0.0',
-            'Mozilla/5.0 (X11; Linux x86_64; rv:143.0) Gecko/20100101 Firefox/143.0',
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0',
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36; Manus-User/1.0',
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36 Edg/143.0.0.0',
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15',
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.0.1 Safari/605.1.15',
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
-            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36',
-            'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/119.0',
-            'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Mobile Safari/537.36',
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36',
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.5 Safari/605.1.15',
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36 Edg/141.0.0.0',
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/109.0',
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:136.0) Gecko/20100101 Firefox/136.0',
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:144.0) Gecko/20100101 Firefox/144.0',
-            'Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36',
-            'Mozilla/5.0 (X11; Linux x86_64; rv:139.0) Gecko/20100101 Firefox/139.0',
-            'Mozilla/5.0 (X11; Linux x86_64; rv:142.0) Gecko/20100101 Firefox/142.0',
-        ]
-        return random.choice(user_agents)
 
 
 class AsyncFetcher:
@@ -553,7 +496,10 @@ class AsyncFetcher:
 
     @staticmethod
     def _default_headers(headers: dict[str, str] | None = None) -> dict[str, str]:
-        return headers or {'User-Agent': Core.get_user_agent()}
+        request_headers = dict(headers or {})
+        if not any(name.lower() == 'user-agent' for name in request_headers):
+            request_headers['User-Agent'] = Core.get_user_agent()
+        return request_headers
 
     @staticmethod
     def _ssl_context(verify: bool | None = True) -> ssl.SSLContext | bool:
@@ -934,6 +880,7 @@ class AsyncFetcher:
         session,
         url: str,
         proxy: str | None = None,
+        headers: dict[str, str] | None = None,
         include_metadata: bool = False,
     ) -> tuple[Any, Any] | str:
         _, proxy_type = AsyncFetcher._resolve_proxy(proxy)
@@ -941,6 +888,7 @@ class AsyncFetcher:
             session=None if proxy_type == 'socks5' else session,
             url=url,
             proxy=proxy,
+            headers=headers,
             request_timeout=15,
             include_metadata=include_metadata,
         )
@@ -975,6 +923,7 @@ class AsyncFetcher:
                                     session,
                                     url,
                                     proxy=proxy_url,
+                                    headers=headers,
                                     include_metadata=include_metadata,
                                 )
                                 for url, proxy_url in zip(urls, proxy_urls, strict=False)
@@ -984,7 +933,15 @@ class AsyncFetcher:
                 else:
                     return list(
                         await asyncio.gather(
-                            *[AsyncFetcher.takeover_fetch(session, url, include_metadata=include_metadata) for url in urls]
+                            *[
+                                AsyncFetcher.takeover_fetch(
+                                    session,
+                                    url,
+                                    headers=headers,
+                                    include_metadata=include_metadata,
+                                )
+                                for url in urls
+                            ]
                         )
                     )