diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c5f3f73..c47b94fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Made no-filename REST `/query` executions reach completed-result construction and SQLite persistence without changing the legacy response fields. - Made Chaos reject empty credentials, report HTTP and malformed-response failures, and preserve supported subdomain response shapes. - Made Fofa reject incomplete credentials, report HTTP and malformed-response failures, normalize scoped hosts, and discard invalid IP values. +- Made FullHunt reject empty credentials, report HTTP and malformed-response failures, and isolate malformed host records. - Made Hudson Rock HTTP failures status-aware, bounded rate-limit retries, removed trailing request delays, isolated malformed provider items, and retained infostealer details in completed JSONL and SQLite results. - Made the public Have I Been Pwned breach catalogue keyless, added offline response contracts, and retained stable breach names in completed JSONL and SQLite results. - Fixed THC rate-limit exhaustion so terminal failures are reported without sleeping after the final attempt, with offline recovery, non-success, and malformed-response contracts. diff --git a/tests/discovery/test_fullhuntsearch.py b/tests/discovery/test_fullhuntsearch.py new file mode 100644 index 00000000..85ddc10f --- /dev/null +++ b/tests/discovery/test_fullhuntsearch.py @@ -0,0 +1,194 @@ +import logging +from typing import Any + +import pytest + +from theHarvester.discovery import fullhuntsearch +from theHarvester.discovery.constants import MissingKey +from theHarvester.lib.core import FetcherResponse + + +@pytest.mark.asyncio +async def test_http_failure_is_reported_without_results( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + monkeypatch.setattr(fullhuntsearch.Core, 'fullhunt_key', lambda: 'test-key') + requests: list[str] = [] + + async def fake_fetch_all(urls: list[str], **kwargs: Any) -> list[FetcherResponse]: + requests.extend(urls) + assert kwargs['json'] is True + assert kwargs['include_metadata'] is True + return [FetcherResponse(body={'error': 'forbidden'}, status=403, headers={})] + + monkeypatch.setattr(fullhuntsearch.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = fullhuntsearch.SearchFullHunt('example.com') + + with caplog.at_level(logging.INFO, logger=fullhuntsearch.__name__): + await search.process() + + assert await search.get_hostnames() == [] + assert await search.get_ips() == [] + assert requests == ['https://fullhunt.io/api/v1/domain/example.com/details'] + assert 'FullHunt request failed with HTTP 403' in caplog.text + + +@pytest.mark.parametrize('key', ['', ' ']) +def test_empty_api_key_is_rejected(monkeypatch: pytest.MonkeyPatch, key: str) -> None: + monkeypatch.setattr(fullhuntsearch.Core, 'fullhunt_key', lambda: key) + + with pytest.raises(MissingKey): + fullhuntsearch.SearchFullHunt('example.com') + + +def test_missing_api_key_configuration_is_normalized(monkeypatch: pytest.MonkeyPatch) -> None: + def missing_key() -> str: + raise KeyError('fullhunt') + + monkeypatch.setattr(fullhuntsearch.Core, 'fullhunt_key', missing_key) + + with pytest.raises(MissingKey): + fullhuntsearch.SearchFullHunt('example.com') + + +@pytest.mark.asyncio +async def test_malformed_domain_details_are_reported_without_fallback( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + monkeypatch.setattr(fullhuntsearch.Core, 'fullhunt_key', lambda: 'test-key') + requests: list[str] = [] + + async def fake_fetch_all(urls: list[str], **_kwargs: Any) -> list[FetcherResponse]: + requests.extend(urls) + return [FetcherResponse(body={'unexpected': []}, status=200, headers={})] + + monkeypatch.setattr(fullhuntsearch.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = fullhuntsearch.SearchFullHunt('example.com') + + with caplog.at_level(logging.INFO, logger=fullhuntsearch.__name__): + await search.process() + + assert await search.get_hostnames() == [] + assert requests == ['https://fullhunt.io/api/v1/domain/example.com/details'] + assert 'FullHunt returned malformed domain details' in caplog.text + + +@pytest.mark.asyncio +async def test_malformed_host_does_not_hide_later_valid_results( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + monkeypatch.setattr(fullhuntsearch.Core, 'fullhunt_key', lambda: 'test-key') + + async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: + return [ + FetcherResponse( + body={ + 'hosts': [ + 'not-an-object', + {'host': 7}, + {'host': 'ports.example.com', 'network_ports': [{}]}, + {'host': 'products.example.com', 'products': [{}]}, + {'host': 'api.example.com', 'ip_address': '192.0.2.20'}, + ] + }, + status=200, + headers={}, + ) + ] + + monkeypatch.setattr(fullhuntsearch.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = fullhuntsearch.SearchFullHunt('example.com') + + with caplog.at_level(logging.INFO, logger=fullhuntsearch.__name__): + await search.process() + + assert await search.get_hostnames() == ['api.example.com'] + assert await search.get_ips() == ['192.0.2.20'] + assert caplog.text.count('FullHunt ignored a malformed host item') == 4 + + +@pytest.mark.asyncio +async def test_malformed_subdomain_fallback_is_reported( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + monkeypatch.setattr(fullhuntsearch.Core, 'fullhunt_key', lambda: 'test-key') + responses = [ + FetcherResponse(body={'hosts': []}, status=200, headers={}), + FetcherResponse(body={'hosts': 7}, status=200, headers={}), + ] + + async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: + return [responses.pop(0)] + + monkeypatch.setattr(fullhuntsearch.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = fullhuntsearch.SearchFullHunt('example.com') + + with caplog.at_level(logging.INFO, logger=fullhuntsearch.__name__): + await search.process() + + assert await search.get_hostnames() == [] + assert 'FullHunt returned malformed subdomains' in caplog.text + + +@pytest.mark.asyncio +async def test_fallback_ignores_malformed_and_out_of_scope_hosts( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + monkeypatch.setattr(fullhuntsearch.Core, 'fullhunt_key', lambda: 'test-key') + responses = [ + FetcherResponse(body={'hosts': []}, status=200, headers={}), + FetcherResponse(body={'hosts': [7, 'outside.test', 'API.Example.COM']}, status=200, headers={}), + ] + + async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: + return [responses.pop(0)] + + monkeypatch.setattr(fullhuntsearch.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = fullhuntsearch.SearchFullHunt('example.com') + + with caplog.at_level(logging.INFO, logger=fullhuntsearch.__name__): + await search.process() + + assert await search.get_hostnames() == ['api.example.com'] + assert caplog.text.count('FullHunt ignored a malformed subdomain item') == 2 + + +@pytest.mark.asyncio +async def test_nested_results_use_normalized_hostname(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(fullhuntsearch.Core, 'fullhunt_key', lambda: 'test-key') + + async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: + return [ + FetcherResponse( + body={ + 'hosts': [ + { + 'host': 'API.Example.COM', + 'dns_records': {'a': ['192.0.2.20']}, + 'http_response': {'status': 200}, + 'geo': {'country': 'US'}, + 'cloud': {'provider': 'example'}, + 'certificate': {'issuer': 'Example CA'}, + } + ] + }, + status=200, + headers={}, + ) + ] + + monkeypatch.setattr(fullhuntsearch.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = fullhuntsearch.SearchFullHunt('example.com') + + await search.process() + + assert await search.get_dns_records() == {'api.example.com': {'a': ['192.0.2.20']}} + assert await search.get_http_info() == {'api.example.com': {'status': 200}} + assert await search.get_geo_info() == {'api.example.com': {'country': 'US'}} + assert await search.get_cloud_info() == {'api.example.com': {'provider': 'example'}} + assert await search.get_certificate_info() == [{'issuer': 'Example CA', 'hostname': 'api.example.com'}] diff --git a/theHarvester/discovery/fullhuntsearch.py b/theHarvester/discovery/fullhuntsearch.py index 7abf2db5..a8d706d9 100644 --- a/theHarvester/discovery/fullhuntsearch.py +++ b/theHarvester/discovery/fullhuntsearch.py @@ -1,9 +1,11 @@ import logging +from ipaddress import ip_address from typing import Any, ClassVar from urllib.parse import quote from theHarvester.discovery.constants import MissingKey -from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse +from theHarvester.lib.hostnames import normalize_scoped_hostname logger = logging.getLogger(__name__) @@ -115,8 +117,11 @@ class SearchFullHunt: def __init__(self, word) -> None: self.word = word - self.key = Core.fullhunt_key() - if self.key is None: + try: + self.key = Core.fullhunt_key() + except Exception as error: + raise MissingKey('fullhunt') from error + if not isinstance(self.key, str) or not self.key.strip(): raise MissingKey('fullhunt') self.total_results: dict[str, Any] = { 'hosts': [], # List of subdomains @@ -148,8 +153,16 @@ class SearchFullHunt: json=True, headers=self._get_headers(), proxy=self.proxy, + include_metadata=True, ) - return response[0] + metadata = response[0] if response and isinstance(response[0], FetcherResponse) else None + if metadata is None: + raise RuntimeError('FullHunt request failed') + if not 200 <= metadata.status < 300: + raise RuntimeError(f'FullHunt request failed with HTTP {metadata.status}') + if not isinstance(metadata.body, dict): + raise ValueError('FullHunt returned malformed data') + return metadata.body def add_filter(self, filter_name: str, filter_value: str) -> None: """Add a search filter to be used in advanced searches @@ -287,13 +300,46 @@ class SearchFullHunt: hosts = details['hosts'] for host_data in hosts: + if not isinstance(host_data, dict): + logger.info('FullHunt ignored a malformed host item') + continue + hostname = normalize_scoped_hostname(host_data.get('host'), self.word) + address = host_data.get('ip_address') + try: + normalized_address = str(ip_address(address)) if isinstance(address, str) else None + except ValueError: + normalized_address = None + if ( + hostname is None + or (address is not None and normalized_address is None) + or ( + 'network_ports' in host_data + and ( + not isinstance(host_data['network_ports'], (list, set, tuple)) + or any( + not isinstance(port, int) or isinstance(port, bool) or not 0 <= port <= 65535 + for port in host_data['network_ports'] + ) + ) + ) + or any( + field in host_data + and (not isinstance(host_data[field], list) or any(not isinstance(value, str) for value in host_data[field])) + for field in ('products', 'tags') + ) + or any( + field in host_data and not isinstance(host_data[field], dict) + for field in ('dns_records', 'http_response', 'geo', 'cloud', 'certificate') + ) + ): + logger.info('FullHunt ignored a malformed host item') + continue # Extract subdomains - if host_data.get('host'): - self.total_results['hosts'].append(host_data['host']) + self.total_results['hosts'].append(hostname) # Extract IPs - if host_data.get('ip_address'): - self.total_results['ips'].append(host_data['ip_address']) + if normalized_address: + self.total_results['ips'].append(normalized_address) # Extract ports if host_data.get('network_ports'): @@ -310,7 +356,6 @@ class SearchFullHunt: # Extract DNS information if 'dns_records' in host_data: dns_records = host_data['dns_records'] - hostname = host_data.get('host', '') if hostname not in self.total_results['dns_records']: self.total_results['dns_records'][hostname] = {} @@ -321,7 +366,6 @@ class SearchFullHunt: # Extract HTTP information if 'http_response' in host_data: http_info = host_data['http_response'] - hostname = host_data.get('host', '') if hostname not in self.total_results['http_info']: self.total_results['http_info'][hostname] = {} @@ -332,7 +376,6 @@ class SearchFullHunt: # Extract geographic information if 'geo' in host_data: geo_info = host_data['geo'] - hostname = host_data.get('host', '') if hostname not in self.total_results['geo_info']: self.total_results['geo_info'][hostname] = {} @@ -343,7 +386,6 @@ class SearchFullHunt: # Extract cloud information if 'cloud' in host_data: cloud_info = host_data['cloud'] - hostname = host_data.get('host', '') if hostname not in self.total_results['cloud_info']: self.total_results['cloud_info'][hostname] = {} @@ -353,8 +395,7 @@ class SearchFullHunt: # Extract certificate information if 'certificate' in host_data: - cert_info = host_data['certificate'] - cert_info['hostname'] = host_data.get('host', '') + cert_info = {**host_data['certificate'], 'hostname': hostname} self.total_results['cert_info'].append(cert_info) # Deduplicate results @@ -379,14 +420,22 @@ class SearchFullHunt: try: # First get domain details which includes most information domain_details = await self.get_domain_details() + if not isinstance(domain_details.get('hosts'), list): + raise ValueError('FullHunt returned malformed domain details') self.total_results['domain_details'] = domain_details await self.extract_data_from_domain_details(domain_details) # If no hosts found in domain details, try the dedicated subdomains endpoint if not self.total_results['hosts']: subdomains_response = await self.get_subdomains() - if 'hosts' in subdomains_response: - self.total_results['hosts'] = subdomains_response['hosts'] + hosts = subdomains_response.get('hosts') + if not isinstance(hosts, list): + raise ValueError('FullHunt returned malformed subdomains') + for host in hosts: + if normalized_host := normalize_scoped_hostname(host, self.word): + self.total_results['hosts'].append(normalized_host) + else: + logger.info('FullHunt ignored a malformed subdomain item') # If filters are set, perform an advanced search if self.filters: