diff --git a/CHANGELOG.md b/CHANGELOG.md index fd167da5..8c5f3f73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Made no-filename REST `/query` executions reach completed-result construction and SQLite persistence without changing the legacy response fields. - Made Chaos reject empty credentials, report HTTP and malformed-response failures, and preserve supported subdomain response shapes. +- Made Fofa reject incomplete credentials, report HTTP and malformed-response failures, normalize scoped hosts, and discard invalid IP values. - Made Hudson Rock HTTP failures status-aware, bounded rate-limit retries, removed trailing request delays, isolated malformed provider items, and retained infostealer details in completed JSONL and SQLite results. - Made the public Have I Been Pwned breach catalogue keyless, added offline response contracts, and retained stable breach names in completed JSONL and SQLite results. - Fixed THC rate-limit exhaustion so terminal failures are reported without sleeping after the final attempt, with offline recovery, non-success, and malformed-response contracts. diff --git a/tests/discovery/test_fofa.py b/tests/discovery/test_fofa.py new file mode 100644 index 00000000..cbf6a709 --- /dev/null +++ b/tests/discovery/test_fofa.py @@ -0,0 +1,138 @@ +import logging +from typing import Any + +import pytest + +from theHarvester.discovery import fofa +from theHarvester.discovery.constants import MissingKey +from theHarvester.lib.core import FetcherResponse + + +@pytest.mark.asyncio +async def test_http_failure_is_reported_without_results( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: ('test-key', 'operator@example.com')) + + async def fake_fetch_all(urls: list[str], **kwargs: Any) -> list[FetcherResponse]: + assert len(urls) == 1 + assert urls[0].startswith('https://fofa.info/api/v1/search/all?') + assert 'key=test-key' in urls[0] + assert kwargs['json'] is True + assert kwargs['include_metadata'] is True + return [FetcherResponse(body={'error': True}, status=429, headers={})] + + monkeypatch.setattr(fofa.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = fofa.SearchFofa('example.com') + + with caplog.at_level(logging.INFO, logger=fofa.__name__): + await search.process() + + assert await search.get_hostnames() == set() + assert await search.get_ips() == set() + assert 'Fofa request failed with HTTP 429' in caplog.text + + +@pytest.mark.asyncio +@pytest.mark.parametrize('error_message', ['Invalid credentials', '账号无效']) +async def test_provider_body_authentication_failure_is_actionable( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + error_message: str, +) -> None: + monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: ('test-key', 'operator@example.com')) + + async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: + return [FetcherResponse(body={'error': True, 'errmsg': error_message}, status=200, headers={})] + + monkeypatch.setattr(fofa.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = fofa.SearchFofa('example.com') + + with caplog.at_level(logging.INFO, logger=fofa.__name__): + await search.process() + + assert 'Fofa API rejected the configured credentials' in caplog.text + + +@pytest.mark.asyncio +async def test_provider_body_quota_failure_is_actionable( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: ('test-key', 'operator@example.com')) + + async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: + return [FetcherResponse(body={'error': True, 'errmsg': 'Query quota exhausted'}, status=200, headers={})] + + monkeypatch.setattr(fofa.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = fofa.SearchFofa('example.com') + + with caplog.at_level(logging.INFO, logger=fofa.__name__): + await search.process() + + assert 'Fofa API quota or plan limit was reached' in caplog.text + + +@pytest.mark.parametrize( + 'credentials', + [('', 'operator@example.com'), ('test-key', ' ')], +) +def test_empty_credentials_are_rejected( + monkeypatch: pytest.MonkeyPatch, + credentials: tuple[str, str], +) -> None: + monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: credentials) + + with pytest.raises(MissingKey): + fofa.SearchFofa('example.com') + + +@pytest.mark.asyncio +async def test_malformed_results_are_reported( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: ('test-key', 'operator@example.com')) + + async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: + return [FetcherResponse(body={'error': False, 'results': 7}, status=200, headers={})] + + monkeypatch.setattr(fofa.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = fofa.SearchFofa('example.com') + + with caplog.at_level(logging.INFO, logger=fofa.__name__): + await search.process() + + assert await search.get_hostnames() == set() + assert await search.get_ips() == set() + assert 'Fofa returned malformed results' in caplog.text + + +@pytest.mark.asyncio +async def test_success_preserves_scoped_hosts_and_valid_ips(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: ('test-key', 'operator@example.com')) + + async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: + return [ + FetcherResponse( + body={ + 'error': False, + 'results': [ + ['https://API.Example.COM:443', '192.0.2.10'], + ['https://outside.test', 'not-an-ip'], + ['malformed'], + ], + }, + status=200, + headers={}, + ) + ] + + monkeypatch.setattr(fofa.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = fofa.SearchFofa('example.com') + + await search.process() + + assert await search.get_hostnames() == {'api.example.com'} + assert await search.get_ips() == {'192.0.2.10'} diff --git a/theHarvester/discovery/fofa.py b/theHarvester/discovery/fofa.py index 0eb7640a..b890ada0 100644 --- a/theHarvester/discovery/fofa.py +++ b/theHarvester/discovery/fofa.py @@ -1,23 +1,14 @@ import base64 -import json as _stdlib_json import logging -from types import ModuleType +from ipaddress import ip_address +from urllib.parse import urlparse from theHarvester.discovery.constants import MissingKey -from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse +from theHarvester.lib.hostnames import normalize_scoped_hostname logger = logging.getLogger(__name__) -json: ModuleType = _stdlib_json -try: - import ujson as _ujson - - json = _ujson -except ImportError: - pass -except Exception: - pass - class SearchFofa: """Class uses Fofa API to search for domain and host intelligence @@ -35,20 +26,12 @@ class SearchFofa: def _get_api_credentials(self) -> tuple[str, str]: """Get Fofa API credentials""" try: - return Core.fofa_key() - except Exception: + api_key, email = Core.fofa_key() + except Exception as error: + raise MissingKey('Fofa API (key and email required)') from error + if not all(isinstance(value, str) and value.strip() for value in (api_key, email)): raise MissingKey('Fofa API (key and email required)') - - @staticmethod - def _safe_parse_json(payload: object) -> dict: - if isinstance(payload, dict): - return payload - if isinstance(payload, str): - try: - return json.loads(payload) - except Exception: - return {} - return {} + return api_key, email async def do_search(self) -> None: try: @@ -72,42 +55,62 @@ class SearchFofa: param_string = '&'.join([f'{k}={v}' for k, v in params.items()]) full_url = f'{url}?{param_string}' - response = await AsyncFetcher.fetch_all([full_url], headers=headers, proxy=self.proxy) + response = await AsyncFetcher.fetch_all( + [full_url], + headers=headers, + proxy=self.proxy, + json=True, + include_metadata=True, + ) - if not response or not isinstance(response, list) or not response[0]: + metadata = response[0] if response and isinstance(response[0], FetcherResponse) else None + if metadata is None: logger.info(f'No response from Fofa API for: {self.word}') return + if not 200 <= metadata.status < 300: + logger.info(f'Fofa request failed with HTTP {metadata.status}') + return try: - data = self._safe_parse_json(response[0]) + data = metadata.body + if not isinstance(data, dict): + logger.info('Fofa returned malformed data') + return - if isinstance(data, dict): - # Check for errors - if data.get('error', False): - error_msg = data.get('errmsg', 'Unknown error') + # Check for errors + if data.get('error', False): + error_message = data.get('errmsg') + normalized_error = error_message.casefold() if isinstance(error_message, str) else '' + if 'invalid' in normalized_error or '账号无效' in normalized_error: + logger.info('Fofa API rejected the configured credentials') + elif any(term in normalized_error for term in ('quota', 'limit', 'plan')): + logger.info('Fofa API quota or plan limit was reached') + else: logger.info('Fofa API returned an error') - if '账号无效' in error_msg or 'invalid' in error_msg.lower(): - raise MissingKey('Fofa API (Invalid credentials)') - return + return - # Extract results - results = data.get('results', []) - if isinstance(results, list): - for result in results: - if isinstance(result, list) and len(result) >= 2: - host = result[0] # host field - ip = result[1] # ip field + # Extract results + results = data.get('results', []) + if not isinstance(results, list): + logger.info('Fofa returned malformed results') + return + for result in results: + if isinstance(result, list) and len(result) >= 2: + host = result[0] # host field + ip = result[1] # ip field - # Add host if it's related to our domain - if isinstance(host, str) and self.word in host: - # Extract clean hostname - clean_host = host.replace('http://', '').replace('https://', '').split(':')[0] - if clean_host.endswith(f'.{self.word}') or clean_host == self.word: - self.totalhosts.add(clean_host.lower()) + # Add host if it's related to our domain + if isinstance(host, str): + parsed = urlparse(host if '://' in host else f'//{host}') + if clean_host := normalize_scoped_hostname(parsed.hostname, self.word): + self.totalhosts.add(clean_host) - # Add IP - if isinstance(ip, str) and ip: - self.totalips.add(ip) + # Add IP + if isinstance(ip, str) and ip: + try: + self.totalips.add(str(ip_address(ip))) + except ValueError: + continue except Exception as e: logger.info(f'Failed to parse Fofa response: {e}')