fix: harden web search provider contracts (#99)

CHANGELOG.md

@@ -30,6 +30,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Expanded offline regression coverage for discovery providers, configuration contracts, logging, output, documentation, workflow policy, and scope boundaries.
 
 ### Fixed
+- Fixed Baidu, Mojeek, and Yahoo page-response separation and Brave missing-credential reporting, with offline web-search provider contract coverage.
 - Fixed DNS candidate validation to omit names without usable A, AAAA, or CNAME evidence, normalize and deduplicate IPv4, IPv6, and canonical-name records, and preserve the existing `Checker.check()` and `DnsForce.run()` return shape.
 - Fixed REST `/query` requests with a filename so they no longer fail with an unbound local value and HTTP 500 response ([c358df80](https://github.com/laramies/theHarvester/commit/c358df80)).
 - Fixed Brave result limits, malformed Mojeek responses, DuckDuckGo provider and parser boundaries, Baidu verification status reporting, invalid Robtex reverse lookups, and transient crt.sh failures ([72c0d9eb](https://github.com/laramies/theHarvester/commit/72c0d9eb), [48be3ccd](https://github.com/laramies/theHarvester/commit/48be3ccd), [6e7945b5](https://github.com/laramies/theHarvester/commit/6e7945b5), [48f959cc](https://github.com/laramies/theHarvester/commit/48f959cc), [1d56dc78](https://github.com/laramies/theHarvester/commit/1d56dc78), [e8d5278b](https://github.com/laramies/theHarvester/commit/e8d5278b), [26adbc49](https://github.com/laramies/theHarvester/commit/26adbc49)).

tests/discovery/test_baidusearch.py

@@ -4,6 +4,19 @@ from theHarvester.discovery import baidusearch
 
 
 class TestBaiduSearch:
+    @pytest.mark.asyncio
+    async def test_page_responses_are_separated_before_normalizing_evidence(self, monkeypatch):
+        async def fake_fetch_all(urls, headers=None, proxy=False):
+            return ['Contact Admin@Example.COM. at Blog.Example.COM.', 'Ignore outsider@example.net']
+
+        monkeypatch.setattr(baidusearch.AsyncFetcher, 'fetch_all', fake_fetch_all)
+        search = baidusearch.SearchBaidu(word='example.com', limit=20)
+
+        await search.process()
+
+        assert await search.get_emails() == {'admin@example.com'}
+        assert await search.get_hostnames() == ['blog.example.com', 'example.com']
+
     @pytest.mark.asyncio
     async def test_empty_fetch_response_is_reported(self, monkeypatch):
         async def fake_fetch_all(urls, headers=None, proxy=False):

tests/discovery/test_brave.py

@@ -4,6 +4,7 @@ from urllib.parse import parse_qs, urlparse
 import pytest
 
 from theHarvester.discovery import bravesearch
+from theHarvester.discovery.constants import MissingKey
 from theHarvester.lib.configuration import InMemoryCredentialAdapter
 
 
@@ -35,6 +36,67 @@ def no_brave_sleep(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.setattr(bravesearch.asyncio, 'sleep', no_sleep)
 
 
+def test_brave_requires_an_api_key() -> None:
+    with pytest.raises(MissingKey, match='Brave Search'):
+        bravesearch.SearchBrave('example.com', 10, credential_adapter=InMemoryCredentialAdapter({}))
+
+
+@pytest.mark.asyncio
+async def test_brave_normalizes_in_scope_evidence(
+    monkeypatch: pytest.MonkeyPatch,
+    brave_credentials: InMemoryCredentialAdapter,
+) -> None:
+    responses = iter(
+        [
+            _response(
+                [
+                    {
+                        'title': 'Contact Admin@Example.COM.',
+                        'description': 'Ignore outsider@example.net and api.example.net',
+                        'url': 'https://Blog.Example.COM./contact',
+                    }
+                ],
+                more=False,
+            ),
+            {'error': {'message': 'Access denied', 'code': 'forbidden'}},
+        ]
+    )
+
+    proxies: list[bool] = []
+
+    async def fake_fetch(*, url: str, **kwargs: Any) -> dict[str, Any]:
+        proxies.append(kwargs['proxy'])
+        return next(responses)
+
+    monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch', fake_fetch)
+    search = bravesearch.SearchBrave('example.com', 10, credential_adapter=brave_credentials)
+
+    await search.process(proxy=True)
+
+    assert proxies == [True, True]
+    assert await search.get_emails() == {'admin@example.com'}
+    assert await search.get_hostnames() == ['blog.example.com', 'example.com']
+
+
+@pytest.mark.parametrize('response', [None, []], ids=['empty', 'malformed'])
+@pytest.mark.asyncio
+async def test_brave_unusable_response_returns_no_evidence(
+    monkeypatch: pytest.MonkeyPatch,
+    brave_credentials: InMemoryCredentialAdapter,
+    response: list[Any] | None,
+) -> None:
+    async def fake_fetch(*, url: str, **_kwargs: Any) -> list[Any] | None:
+        return response
+
+    monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch', fake_fetch)
+    search = bravesearch.SearchBrave('example.com', 10, credential_adapter=brave_credentials)
+
+    await search.process()
+
+    assert await search.get_emails() == set()
+    assert await search.get_hostnames() == []
+
+
 @pytest.mark.asyncio
 async def test_brave_uses_page_offsets_and_one_global_limit(
     monkeypatch: pytest.MonkeyPatch,

tests/discovery/test_duckduckgo.py

@@ -33,3 +33,29 @@ async def test_duckduckgo_does_not_fetch_provider_returned_urls(monkeypatch: pyt
     assert requests == [(['https://api.duckduckgo.com/?q=example.com&format=json&pretty=1'], True)]
     assert await search.get_hostnames() == ['api.example.com', 'example.com']
     assert await search.get_emails() == {'admin@example.com'}
+
+
+@pytest.mark.parametrize(
+    'payload',
+    [
+        '',
+        '{"broken": ',
+        '{"error": "Access denied", "url": "https://api.example.net"}',
+    ],
+    ids=['empty', 'malformed', 'blocked'],
+)
+@pytest.mark.asyncio
+async def test_duckduckgo_unusable_response_returns_no_evidence(
+    monkeypatch: pytest.MonkeyPatch,
+    payload: str,
+) -> None:
+    async def fake_fetch_all(urls: list[str] | set[str], **_kwargs: Any) -> list[str]:
+        return [payload]
+
+    monkeypatch.setattr(duckduckgosearch.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    search = duckduckgosearch.SearchDuckDuckGo('example.com', 100)
+
+    await search.process()
+
+    assert await search.get_hostnames() == []
+    assert await search.get_emails() == set()

tests/discovery/test_yahoosearch.py

@@ -0,0 +1,63 @@
+from typing import Any
+
+import pytest
+
+from theHarvester.discovery import yahoosearch
+
+
+@pytest.mark.asyncio
+async def test_yahoo_uses_exact_pages_and_normalizes_evidence(monkeypatch: pytest.MonkeyPatch) -> None:
+    requests: list[dict[str, Any]] = []
+
+    async def fake_fetch_all(
+        urls: list[str] | set[str],
+        headers: dict[str, str] | None = None,
+        proxy: bool = False,
+        **_kwargs: Any,
+    ) -> list[str]:
+        requests.append({'urls': list(urls), 'headers': headers, 'proxy': proxy})
+        return [
+            'Contact Admin@Example.COM. at Blog.Example.COM.',
+            'Ignore outsider@example.net and api.example.net',
+        ]
+
+    monkeypatch.setattr(yahoosearch.Core, 'get_user_agent', staticmethod(lambda: 'UA'))
+    monkeypatch.setattr(yahoosearch.AsyncFetcher, 'fetch_all', fake_fetch_all)
+
+    search = yahoosearch.SearchYahoo('example.com', 20)
+    await search.process(proxy=True)
+
+    assert requests == [
+        {
+            'urls': [
+                'https://search.yahoo.com/search?p=%40example.com&b=0&pz=10',
+                'https://search.yahoo.com/search?p=%40example.com&b=10&pz=10',
+            ],
+            'headers': {'Host': 'search.yahoo.com', 'User-agent': 'UA'},
+            'proxy': True,
+        }
+    ]
+    assert set(await search.get_emails()) == {'admin@example.com'}
+    assert await search.get_hostnames() == ['blog.example.com', 'example.com']
+
+
+@pytest.mark.parametrize(
+    'response',
+    ['', None, '<html>Access denied at api.example.net</html>'],
+    ids=['empty', 'malformed', 'blocked'],
+)
+@pytest.mark.asyncio
+async def test_yahoo_unusable_responses_return_no_evidence(
+    monkeypatch: pytest.MonkeyPatch,
+    response: str | None,
+) -> None:
+    async def fake_fetch_all(urls: list[str] | set[str], **_kwargs: Any) -> list[str | None]:
+        return [response] * len(urls)
+
+    monkeypatch.setattr(yahoosearch.AsyncFetcher, 'fetch_all', fake_fetch_all)
+
+    search = yahoosearch.SearchYahoo('example.com', 20)
+    await search.process()
+
+    assert await search.get_emails() == []
+    assert await search.get_hostnames() == []

tests/test_mojeek.py

@@ -38,6 +38,26 @@ def _patch_mojeek(
 
 
 class TestMojeekSearch:
+    @pytest.mark.asyncio
+    async def test_scraped_pages_are_separated_before_normalizing_evidence(
+        self,
+        monkeypatch: pytest.MonkeyPatch,
+    ) -> None:
+        _patch_mojeek(
+            monkeypatch,
+            api_key='',
+            scrape_responses=[
+                'Contact Admin@Example.COM. at Blog.Example.COM.',
+                'Ignore outsider@example.net',
+            ],
+        )
+        search = mojeek.SearchMojeek(word='example.com', limit=20)
+
+        await search.process()
+
+        assert await search.get_emails() == {'admin@example.com'}
+        assert await search.get_hostnames() == ['blog.example.com', 'example.com']
+
     @pytest.mark.asyncio
     async def test_keyless_mode_uses_scraping_without_json(self, monkeypatch: pytest.MonkeyPatch) -> None:
         requests = _patch_mojeek(
@@ -73,9 +93,9 @@ class TestMojeekSearch:
                     'response': {
                         'results': [
                             {
-                                'url': 'https:\\/\\/blog.example.com\\/contact',
-                                'title': 'Contact admin@example.com',
-                                'desc': 'API docs at api.example.com',
+                                'url': 'https:\\/\\/Blog.Example.COM.\\/contact',
+                                'title': 'Contact Admin@Example.COM.',
+                                'desc': 'API docs at api.example.com; ignore outsider@example.net',
                             }
                         ]
                     }

theHarvester/discovery/baidusearch.py

@@ -29,7 +29,7 @@ class SearchBaidu:
                 continue
             if '百度安全验证' in response or 'wappass.baidu.com/static/captcha' in response:
                 raise RuntimeError('Baidu returned a security verification page')
-            self.total_results += response
+            self.total_results += f' {response}'
 
     async def process(self, proxy: bool = False) -> None:
         self.proxy = proxy

theHarvester/discovery/bravesearch.py

@@ -26,7 +26,10 @@ class SearchBrave:
         self.results: list[dict[str, Any]] = []
         self.totalresults = ''
         credentials = credential_adapter if credential_adapter is not None else FileSystemCredentialAdapter()
-        self.api_key = credentials.get('brave')
+        try:
+            self.api_key = credentials.get('brave')
+        except KeyError:
+            raise MissingKey('Brave Search') from None
         if self.api_key is None or self.api_key == '':
             raise MissingKey('Brave Search')
         self.server = 'https://api.search.brave.com/res/v1/web/search'

theHarvester/discovery/mojeek.py

@@ -90,7 +90,7 @@ class SearchMojeek:
 
         responses = await AsyncFetcher.fetch_all(urls, headers=headers, proxy=self.proxy)
         for response in responses:
-            self.total_results += str(response)
+            self.total_results += f' {response}'
 
     async def process(self, proxy: bool = False) -> None:
         self.proxy = proxy

theHarvester/discovery/yahoosearch.py

@@ -16,7 +16,7 @@ class SearchYahoo:
         urls = [base_url.replace('xx', str(num)) for num in range(0, self.limit, 10) if num <= self.limit]
         responses = await AsyncFetcher.fetch_all(urls, headers=headers, proxy=self.proxy)
         for response in responses:
-            self.total_results += response
+            self.total_results += f' {response}'
 
     async def process(self, proxy: bool = False) -> None:
         self.proxy = proxy