Make Fofa provider outcomes explicit (#2485)

* fix: make Fofa outcomes explicit

* fix: classify Fofa provider errors

CHANGELOG.md

@@ -39,6 +39,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 - Made no-filename REST `/query` executions reach completed-result construction and SQLite persistence without changing the legacy response fields.
 - Made Chaos reject empty credentials, report HTTP and malformed-response failures, and preserve supported subdomain response shapes.
+- Made Fofa reject incomplete credentials, report HTTP and malformed-response failures, normalize scoped hosts, and discard invalid IP values.
 - Made Hudson Rock HTTP failures status-aware, bounded rate-limit retries, removed trailing request delays, isolated malformed provider items, and retained infostealer details in completed JSONL and SQLite results.
 - Made the public Have I Been Pwned breach catalogue keyless, added offline response contracts, and retained stable breach names in completed JSONL and SQLite results.
 - Fixed THC rate-limit exhaustion so terminal failures are reported without sleeping after the final attempt, with offline recovery, non-success, and malformed-response contracts.

tests/discovery/test_fofa.py

@@ -0,0 +1,138 @@
+import logging
+from typing import Any
+
+import pytest
+
+from theHarvester.discovery import fofa
+from theHarvester.discovery.constants import MissingKey
+from theHarvester.lib.core import FetcherResponse
+
+
+@pytest.mark.asyncio
+async def test_http_failure_is_reported_without_results(
+    monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: ('test-key', 'operator@example.com'))
+
+    async def fake_fetch_all(urls: list[str], **kwargs: Any) -> list[FetcherResponse]:
+        assert len(urls) == 1
+        assert urls[0].startswith('https://fofa.info/api/v1/search/all?')
+        assert 'key=test-key' in urls[0]
+        assert kwargs['json'] is True
+        assert kwargs['include_metadata'] is True
+        return [FetcherResponse(body={'error': True}, status=429, headers={})]
+
+    monkeypatch.setattr(fofa.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    search = fofa.SearchFofa('example.com')
+
+    with caplog.at_level(logging.INFO, logger=fofa.__name__):
+        await search.process()
+
+    assert await search.get_hostnames() == set()
+    assert await search.get_ips() == set()
+    assert 'Fofa request failed with HTTP 429' in caplog.text
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize('error_message', ['Invalid credentials', '账号无效'])
+async def test_provider_body_authentication_failure_is_actionable(
+    monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+    error_message: str,
+) -> None:
+    monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: ('test-key', 'operator@example.com'))
+
+    async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]:
+        return [FetcherResponse(body={'error': True, 'errmsg': error_message}, status=200, headers={})]
+
+    monkeypatch.setattr(fofa.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    search = fofa.SearchFofa('example.com')
+
+    with caplog.at_level(logging.INFO, logger=fofa.__name__):
+        await search.process()
+
+    assert 'Fofa API rejected the configured credentials' in caplog.text
+
+
+@pytest.mark.asyncio
+async def test_provider_body_quota_failure_is_actionable(
+    monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: ('test-key', 'operator@example.com'))
+
+    async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]:
+        return [FetcherResponse(body={'error': True, 'errmsg': 'Query quota exhausted'}, status=200, headers={})]
+
+    monkeypatch.setattr(fofa.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    search = fofa.SearchFofa('example.com')
+
+    with caplog.at_level(logging.INFO, logger=fofa.__name__):
+        await search.process()
+
+    assert 'Fofa API quota or plan limit was reached' in caplog.text
+
+
+@pytest.mark.parametrize(
+    'credentials',
+    [('', 'operator@example.com'), ('test-key', '   ')],
+)
+def test_empty_credentials_are_rejected(
+    monkeypatch: pytest.MonkeyPatch,
+    credentials: tuple[str, str],
+) -> None:
+    monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: credentials)
+
+    with pytest.raises(MissingKey):
+        fofa.SearchFofa('example.com')
+
+
+@pytest.mark.asyncio
+async def test_malformed_results_are_reported(
+    monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: ('test-key', 'operator@example.com'))
+
+    async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]:
+        return [FetcherResponse(body={'error': False, 'results': 7}, status=200, headers={})]
+
+    monkeypatch.setattr(fofa.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    search = fofa.SearchFofa('example.com')
+
+    with caplog.at_level(logging.INFO, logger=fofa.__name__):
+        await search.process()
+
+    assert await search.get_hostnames() == set()
+    assert await search.get_ips() == set()
+    assert 'Fofa returned malformed results' in caplog.text
+
+
+@pytest.mark.asyncio
+async def test_success_preserves_scoped_hosts_and_valid_ips(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: ('test-key', 'operator@example.com'))
+
+    async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]:
+        return [
+            FetcherResponse(
+                body={
+                    'error': False,
+                    'results': [
+                        ['https://API.Example.COM:443', '192.0.2.10'],
+                        ['https://outside.test', 'not-an-ip'],
+                        ['malformed'],
+                    ],
+                },
+                status=200,
+                headers={},
+            )
+        ]
+
+    monkeypatch.setattr(fofa.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    search = fofa.SearchFofa('example.com')
+
+    await search.process()
+
+    assert await search.get_hostnames() == {'api.example.com'}
+    assert await search.get_ips() == {'192.0.2.10'}

theHarvester/discovery/fofa.py

@@ -1,23 +1,14 @@
 import base64
-import json as _stdlib_json
 import logging
-from types import ModuleType
+from ipaddress import ip_address
+from urllib.parse import urlparse
 
 from theHarvester.discovery.constants import MissingKey
-from theHarvester.lib.core import AsyncFetcher, Core
+from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse
+from theHarvester.lib.hostnames import normalize_scoped_hostname
 
 logger = logging.getLogger(__name__)
 
-json: ModuleType = _stdlib_json
-try:
-    import ujson as _ujson
-
-    json = _ujson
-except ImportError:
-    pass
-except Exception:
-    pass
-
 
 class SearchFofa:
     """Class uses Fofa API to search for domain and host intelligence
@@ -35,20 +26,12 @@ class SearchFofa:
     def _get_api_credentials(self) -> tuple[str, str]:
         """Get Fofa API credentials"""
         try:
-            return Core.fofa_key()
-        except Exception:
+            api_key, email = Core.fofa_key()
+        except Exception as error:
+            raise MissingKey('Fofa API (key and email required)') from error
+        if not all(isinstance(value, str) and value.strip() for value in (api_key, email)):
             raise MissingKey('Fofa API (key and email required)')
-
-    @staticmethod
-    def _safe_parse_json(payload: object) -> dict:
-        if isinstance(payload, dict):
-            return payload
-        if isinstance(payload, str):
-            try:
-                return json.loads(payload)
-            except Exception:
-                return {}
-        return {}
+        return api_key, email
 
     async def do_search(self) -> None:
         try:
@@ -72,42 +55,62 @@ class SearchFofa:
             param_string = '&'.join([f'{k}={v}' for k, v in params.items()])
             full_url = f'{url}?{param_string}'
 
-            response = await AsyncFetcher.fetch_all([full_url], headers=headers, proxy=self.proxy)
+            response = await AsyncFetcher.fetch_all(
+                [full_url],
+                headers=headers,
+                proxy=self.proxy,
+                json=True,
+                include_metadata=True,
+            )
 
-            if not response or not isinstance(response, list) or not response[0]:
+            metadata = response[0] if response and isinstance(response[0], FetcherResponse) else None
+            if metadata is None:
                 logger.info(f'No response from Fofa API for: {self.word}')
                 return
+            if not 200 <= metadata.status < 300:
+                logger.info(f'Fofa request failed with HTTP {metadata.status}')
+                return
 
             try:
-                data = self._safe_parse_json(response[0])
+                data = metadata.body
+                if not isinstance(data, dict):
+                    logger.info('Fofa returned malformed data')
+                    return
 
-                if isinstance(data, dict):
-                    # Check for errors
-                    if data.get('error', False):
-                        error_msg = data.get('errmsg', 'Unknown error')
+                # Check for errors
+                if data.get('error', False):
+                    error_message = data.get('errmsg')
+                    normalized_error = error_message.casefold() if isinstance(error_message, str) else ''
+                    if 'invalid' in normalized_error or '账号无效' in normalized_error:
+                        logger.info('Fofa API rejected the configured credentials')
+                    elif any(term in normalized_error for term in ('quota', 'limit', 'plan')):
+                        logger.info('Fofa API quota or plan limit was reached')
+                    else:
                         logger.info('Fofa API returned an error')
-                        if '账号无效' in error_msg or 'invalid' in error_msg.lower():
-                            raise MissingKey('Fofa API (Invalid credentials)')
-                        return
+                    return
 
-                    # Extract results
-                    results = data.get('results', [])
-                    if isinstance(results, list):
-                        for result in results:
-                            if isinstance(result, list) and len(result) >= 2:
-                                host = result[0]  # host field
-                                ip = result[1]  # ip field
+                # Extract results
+                results = data.get('results', [])
+                if not isinstance(results, list):
+                    logger.info('Fofa returned malformed results')
+                    return
+                for result in results:
+                    if isinstance(result, list) and len(result) >= 2:
+                        host = result[0]  # host field
+                        ip = result[1]  # ip field
 
-                                # Add host if it's related to our domain
-                                if isinstance(host, str) and self.word in host:
-                                    # Extract clean hostname
-                                    clean_host = host.replace('http://', '').replace('https://', '').split(':')[0]
-                                    if clean_host.endswith(f'.{self.word}') or clean_host == self.word:
-                                        self.totalhosts.add(clean_host.lower())
+                        # Add host if it's related to our domain
+                        if isinstance(host, str):
+                            parsed = urlparse(host if '://' in host else f'//{host}')
+                            if clean_host := normalize_scoped_hostname(parsed.hostname, self.word):
+                                self.totalhosts.add(clean_host)
 
-                                # Add IP
-                                if isinstance(ip, str) and ip:
-                                    self.totalips.add(ip)
+                        # Add IP
+                        if isinstance(ip, str) and ip:
+                            try:
+                                self.totalips.add(str(ip_address(ip)))
+                            except ValueError:
+                                continue
 
             except Exception as e:
                 logger.info(f'Failed to parse Fofa response: {e}')