From ec4a8d920ff4d7b6034afb01eb24eeed321d5085 Mon Sep 17 00:00:00 2001 From: NotoriousRebel <36310667+NotoriousRebel@users.noreply.github.com> Date: Wed, 5 Aug 2026 13:16:19 -0400 Subject: [PATCH] fix: harden web search provider contracts (#99) --- CHANGELOG.md | 1 + tests/discovery/test_baidusearch.py | 13 ++++++ tests/discovery/test_brave.py | 62 ++++++++++++++++++++++++++ tests/discovery/test_duckduckgo.py | 26 +++++++++++ tests/discovery/test_yahoosearch.py | 63 +++++++++++++++++++++++++++ tests/test_mojeek.py | 26 +++++++++-- theHarvester/discovery/baidusearch.py | 2 +- theHarvester/discovery/bravesearch.py | 5 ++- theHarvester/discovery/mojeek.py | 2 +- theHarvester/discovery/yahoosearch.py | 2 +- 10 files changed, 195 insertions(+), 7 deletions(-) create mode 100644 tests/discovery/test_yahoosearch.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 797a0315..eb42f10a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Expanded offline regression coverage for discovery providers, configuration contracts, logging, output, documentation, workflow policy, and scope boundaries. ### Fixed +- Fixed Baidu, Mojeek, and Yahoo page-response separation and Brave missing-credential reporting, with offline web-search provider contract coverage. - Fixed DNS candidate validation to omit names without usable A, AAAA, or CNAME evidence, normalize and deduplicate IPv4, IPv6, and canonical-name records, and preserve the existing `Checker.check()` and `DnsForce.run()` return shape. - Fixed REST `/query` requests with a filename so they no longer fail with an unbound local value and HTTP 500 response ([c358df80](https://github.com/laramies/theHarvester/commit/c358df80)). - Fixed Brave result limits, malformed Mojeek responses, DuckDuckGo provider and parser boundaries, Baidu verification status reporting, invalid Robtex reverse lookups, and transient crt.sh failures ([72c0d9eb](https://github.com/laramies/theHarvester/commit/72c0d9eb), [48be3ccd](https://github.com/laramies/theHarvester/commit/48be3ccd), [6e7945b5](https://github.com/laramies/theHarvester/commit/6e7945b5), [48f959cc](https://github.com/laramies/theHarvester/commit/48f959cc), [1d56dc78](https://github.com/laramies/theHarvester/commit/1d56dc78), [e8d5278b](https://github.com/laramies/theHarvester/commit/e8d5278b), [26adbc49](https://github.com/laramies/theHarvester/commit/26adbc49)). diff --git a/tests/discovery/test_baidusearch.py b/tests/discovery/test_baidusearch.py index 13c7c2a1..412bd0bb 100644 --- a/tests/discovery/test_baidusearch.py +++ b/tests/discovery/test_baidusearch.py @@ -4,6 +4,19 @@ from theHarvester.discovery import baidusearch class TestBaiduSearch: + @pytest.mark.asyncio + async def test_page_responses_are_separated_before_normalizing_evidence(self, monkeypatch): + async def fake_fetch_all(urls, headers=None, proxy=False): + return ['Contact Admin@Example.COM. at Blog.Example.COM.', 'Ignore outsider@example.net'] + + monkeypatch.setattr(baidusearch.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = baidusearch.SearchBaidu(word='example.com', limit=20) + + await search.process() + + assert await search.get_emails() == {'admin@example.com'} + assert await search.get_hostnames() == ['blog.example.com', 'example.com'] + @pytest.mark.asyncio async def test_empty_fetch_response_is_reported(self, monkeypatch): async def fake_fetch_all(urls, headers=None, proxy=False): diff --git a/tests/discovery/test_brave.py b/tests/discovery/test_brave.py index fe19a8db..e41ee6d1 100644 --- a/tests/discovery/test_brave.py +++ b/tests/discovery/test_brave.py @@ -4,6 +4,7 @@ from urllib.parse import parse_qs, urlparse import pytest from theHarvester.discovery import bravesearch +from theHarvester.discovery.constants import MissingKey from theHarvester.lib.configuration import InMemoryCredentialAdapter @@ -35,6 +36,67 @@ def no_brave_sleep(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(bravesearch.asyncio, 'sleep', no_sleep) +def test_brave_requires_an_api_key() -> None: + with pytest.raises(MissingKey, match='Brave Search'): + bravesearch.SearchBrave('example.com', 10, credential_adapter=InMemoryCredentialAdapter({})) + + +@pytest.mark.asyncio +async def test_brave_normalizes_in_scope_evidence( + monkeypatch: pytest.MonkeyPatch, + brave_credentials: InMemoryCredentialAdapter, +) -> None: + responses = iter( + [ + _response( + [ + { + 'title': 'Contact Admin@Example.COM.', + 'description': 'Ignore outsider@example.net and api.example.net', + 'url': 'https://Blog.Example.COM./contact', + } + ], + more=False, + ), + {'error': {'message': 'Access denied', 'code': 'forbidden'}}, + ] + ) + + proxies: list[bool] = [] + + async def fake_fetch(*, url: str, **kwargs: Any) -> dict[str, Any]: + proxies.append(kwargs['proxy']) + return next(responses) + + monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch', fake_fetch) + search = bravesearch.SearchBrave('example.com', 10, credential_adapter=brave_credentials) + + await search.process(proxy=True) + + assert proxies == [True, True] + assert await search.get_emails() == {'admin@example.com'} + assert await search.get_hostnames() == ['blog.example.com', 'example.com'] + + +@pytest.mark.parametrize('response', [None, []], ids=['empty', 'malformed']) +@pytest.mark.asyncio +async def test_brave_unusable_response_returns_no_evidence( + monkeypatch: pytest.MonkeyPatch, + brave_credentials: InMemoryCredentialAdapter, + response: list[Any] | None, +) -> None: + async def fake_fetch(*, url: str, **_kwargs: Any) -> list[Any] | None: + return response + + monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch', fake_fetch) + search = bravesearch.SearchBrave('example.com', 10, credential_adapter=brave_credentials) + + await search.process() + + assert await search.get_emails() == set() + assert await search.get_hostnames() == [] + + @pytest.mark.asyncio async def test_brave_uses_page_offsets_and_one_global_limit( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/discovery/test_duckduckgo.py b/tests/discovery/test_duckduckgo.py index fb797b5d..b7d7af29 100644 --- a/tests/discovery/test_duckduckgo.py +++ b/tests/discovery/test_duckduckgo.py @@ -33,3 +33,29 @@ async def test_duckduckgo_does_not_fetch_provider_returned_urls(monkeypatch: pyt assert requests == [(['https://api.duckduckgo.com/?q=example.com&format=json&pretty=1'], True)] assert await search.get_hostnames() == ['api.example.com', 'example.com'] assert await search.get_emails() == {'admin@example.com'} + + +@pytest.mark.parametrize( + 'payload', + [ + '', + '{"broken": ', + '{"error": "Access denied", "url": "https://api.example.net"}', + ], + ids=['empty', 'malformed', 'blocked'], +) +@pytest.mark.asyncio +async def test_duckduckgo_unusable_response_returns_no_evidence( + monkeypatch: pytest.MonkeyPatch, + payload: str, +) -> None: + async def fake_fetch_all(urls: list[str] | set[str], **_kwargs: Any) -> list[str]: + return [payload] + + monkeypatch.setattr(duckduckgosearch.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = duckduckgosearch.SearchDuckDuckGo('example.com', 100) + + await search.process() + + assert await search.get_hostnames() == [] + assert await search.get_emails() == set() diff --git a/tests/discovery/test_yahoosearch.py b/tests/discovery/test_yahoosearch.py new file mode 100644 index 00000000..4513fdc9 --- /dev/null +++ b/tests/discovery/test_yahoosearch.py @@ -0,0 +1,63 @@ +from typing import Any + +import pytest + +from theHarvester.discovery import yahoosearch + + +@pytest.mark.asyncio +async def test_yahoo_uses_exact_pages_and_normalizes_evidence(monkeypatch: pytest.MonkeyPatch) -> None: + requests: list[dict[str, Any]] = [] + + async def fake_fetch_all( + urls: list[str] | set[str], + headers: dict[str, str] | None = None, + proxy: bool = False, + **_kwargs: Any, + ) -> list[str]: + requests.append({'urls': list(urls), 'headers': headers, 'proxy': proxy}) + return [ + 'Contact Admin@Example.COM. at Blog.Example.COM.', + 'Ignore outsider@example.net and api.example.net', + ] + + monkeypatch.setattr(yahoosearch.Core, 'get_user_agent', staticmethod(lambda: 'UA')) + monkeypatch.setattr(yahoosearch.AsyncFetcher, 'fetch_all', fake_fetch_all) + + search = yahoosearch.SearchYahoo('example.com', 20) + await search.process(proxy=True) + + assert requests == [ + { + 'urls': [ + 'https://search.yahoo.com/search?p=%40example.com&b=0&pz=10', + 'https://search.yahoo.com/search?p=%40example.com&b=10&pz=10', + ], + 'headers': {'Host': 'search.yahoo.com', 'User-agent': 'UA'}, + 'proxy': True, + } + ] + assert set(await search.get_emails()) == {'admin@example.com'} + assert await search.get_hostnames() == ['blog.example.com', 'example.com'] + + +@pytest.mark.parametrize( + 'response', + ['', None, 'Access denied at api.example.net'], + ids=['empty', 'malformed', 'blocked'], +) +@pytest.mark.asyncio +async def test_yahoo_unusable_responses_return_no_evidence( + monkeypatch: pytest.MonkeyPatch, + response: str | None, +) -> None: + async def fake_fetch_all(urls: list[str] | set[str], **_kwargs: Any) -> list[str | None]: + return [response] * len(urls) + + monkeypatch.setattr(yahoosearch.AsyncFetcher, 'fetch_all', fake_fetch_all) + + search = yahoosearch.SearchYahoo('example.com', 20) + await search.process() + + assert await search.get_emails() == [] + assert await search.get_hostnames() == [] diff --git a/tests/test_mojeek.py b/tests/test_mojeek.py index 00ed71af..00f3d26b 100644 --- a/tests/test_mojeek.py +++ b/tests/test_mojeek.py @@ -38,6 +38,26 @@ def _patch_mojeek( class TestMojeekSearch: + @pytest.mark.asyncio + async def test_scraped_pages_are_separated_before_normalizing_evidence( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + _patch_mojeek( + monkeypatch, + api_key='', + scrape_responses=[ + 'Contact Admin@Example.COM. at Blog.Example.COM.', + 'Ignore outsider@example.net', + ], + ) + search = mojeek.SearchMojeek(word='example.com', limit=20) + + await search.process() + + assert await search.get_emails() == {'admin@example.com'} + assert await search.get_hostnames() == ['blog.example.com', 'example.com'] + @pytest.mark.asyncio async def test_keyless_mode_uses_scraping_without_json(self, monkeypatch: pytest.MonkeyPatch) -> None: requests = _patch_mojeek( @@ -73,9 +93,9 @@ class TestMojeekSearch: 'response': { 'results': [ { - 'url': 'https:\\/\\/blog.example.com\\/contact', - 'title': 'Contact admin@example.com', - 'desc': 'API docs at api.example.com', + 'url': 'https:\\/\\/Blog.Example.COM.\\/contact', + 'title': 'Contact Admin@Example.COM.', + 'desc': 'API docs at api.example.com; ignore outsider@example.net', } ] } diff --git a/theHarvester/discovery/baidusearch.py b/theHarvester/discovery/baidusearch.py index 07457efd..cdc20c18 100644 --- a/theHarvester/discovery/baidusearch.py +++ b/theHarvester/discovery/baidusearch.py @@ -29,7 +29,7 @@ class SearchBaidu: continue if '百度安全验证' in response or 'wappass.baidu.com/static/captcha' in response: raise RuntimeError('Baidu returned a security verification page') - self.total_results += response + self.total_results += f' {response}' async def process(self, proxy: bool = False) -> None: self.proxy = proxy diff --git a/theHarvester/discovery/bravesearch.py b/theHarvester/discovery/bravesearch.py index c7375425..e689c011 100644 --- a/theHarvester/discovery/bravesearch.py +++ b/theHarvester/discovery/bravesearch.py @@ -26,7 +26,10 @@ class SearchBrave: self.results: list[dict[str, Any]] = [] self.totalresults = '' credentials = credential_adapter if credential_adapter is not None else FileSystemCredentialAdapter() - self.api_key = credentials.get('brave') + try: + self.api_key = credentials.get('brave') + except KeyError: + raise MissingKey('Brave Search') from None if self.api_key is None or self.api_key == '': raise MissingKey('Brave Search') self.server = 'https://api.search.brave.com/res/v1/web/search' diff --git a/theHarvester/discovery/mojeek.py b/theHarvester/discovery/mojeek.py index 85fe0e69..f952ec75 100644 --- a/theHarvester/discovery/mojeek.py +++ b/theHarvester/discovery/mojeek.py @@ -90,7 +90,7 @@ class SearchMojeek: responses = await AsyncFetcher.fetch_all(urls, headers=headers, proxy=self.proxy) for response in responses: - self.total_results += str(response) + self.total_results += f' {response}' async def process(self, proxy: bool = False) -> None: self.proxy = proxy diff --git a/theHarvester/discovery/yahoosearch.py b/theHarvester/discovery/yahoosearch.py index ea7f5b93..dbe65ab5 100644 --- a/theHarvester/discovery/yahoosearch.py +++ b/theHarvester/discovery/yahoosearch.py @@ -16,7 +16,7 @@ class SearchYahoo: urls = [base_url.replace('xx', str(num)) for num in range(0, self.limit, 10) if num <= self.limit] responses = await AsyncFetcher.fetch_all(urls, headers=headers, proxy=self.proxy) for response in responses: - self.total_results += response + self.total_results += f' {response}' async def process(self, proxy: bool = False) -> None: self.proxy = proxy