diff --git a/tests/discovery/test_criminalip.py b/tests/discovery/test_criminalip.py index a9c7feba..5c4da84a 100644 --- a/tests/discovery/test_criminalip.py +++ b/tests/discovery/test_criminalip.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 import asyncio +import json import logging import pytest from theHarvester.discovery import criminalip from theHarvester.lib import core as core_module +from theHarvester.lib.core import FetcherResponse from theHarvester.lib.source_execution import SourceExecutionReport @@ -15,7 +17,9 @@ async def test_failed_response_body_is_not_logged(monkeypatch, caplog) -> None: monkeypatch.setattr(criminalip.Core, 'get_user_agent', lambda: 'test-agent') async def fake_post_fetch(*args, **kwargs): - return {'status': 500, 'secret': 'provider-secret-payload'} + assert kwargs['include_metadata'] is True + assert kwargs['json_body'] == {'query': 'example.com'} + return FetcherResponse(body={'status': 500, 'secret': 'provider-secret-payload'}, status=200, headers={}) monkeypatch.setattr(criminalip.AsyncFetcher, 'post_fetch', fake_post_fetch) caplog.set_level(logging.INFO, logger=criminalip.__name__) @@ -89,15 +93,15 @@ async def test_do_search_uses_v2_report_endpoint(monkeypatch) -> None: async def fake_post_fetch(url, **kwargs): assert url == 'https://api.criminalip.io/v1/domain/scan' - return {'status': 200, 'data': {'scan_id': 12345}} + return FetcherResponse(body={'status': 200, 'data': {'scan_id': 12345}}, status=200, headers={}) async def fake_fetch(*_args, url, **_kwargs): called_urls.append(url) if '/v1/domain/status/' in url: - return {'status': 200, 'data': {'scan_percentage': 100}} + return FetcherResponse(body={'status': 200, 'data': {'scan_percentage': 100}}, status=200, headers={}) if '/v2/domain/report/' in url: - return {'status': 200, 'data': {}} - return {'status': 500} + return FetcherResponse(body={'status': 200, 'data': {}}, status=200, headers={}) + return FetcherResponse(body={'status': 500}, status=200, headers={}) monkeypatch.setattr(criminalip.AsyncFetcher, 'post_fetch', fake_post_fetch) monkeypatch.setattr(criminalip.AsyncFetcher, 'fetch', fake_fetch) @@ -132,6 +136,9 @@ async def test_provider_conversation_uses_one_session_and_proxy(monkeypatch: pyt async def __aexit__(self, *_args) -> None: return None + async def text(self) -> str: + return json.dumps(self.body) + async def json(self): return self.body @@ -177,12 +184,12 @@ async def test_waiting_scan_reports_runtime_limit(monkeypatch) -> None: status_calls = 0 async def fake_post_fetch(*_args, **_kwargs): - return {'status': 200, 'data': {'scan_id': 12345}} + return FetcherResponse(body={'status': 200, 'data': {'scan_id': 12345}}, status=200, headers={}) async def fake_fetch(*_args, **_kwargs): nonlocal status_calls status_calls += 1 - return {'status': 200, 'data': {'scan_percentage': 50}} + return FetcherResponse(body={'status': 200, 'data': {'scan_percentage': 50}}, status=200, headers={}) async def no_sleep(*_args, **_kwargs): return None @@ -203,10 +210,10 @@ async def test_polling_cancellation_propagates(monkeypatch) -> None: monkeypatch.setattr(criminalip.Core, 'get_user_agent', lambda: 'test-agent') async def fake_post_fetch(*_args, **_kwargs): - return {'status': 200, 'data': {'scan_id': 12345}} + return FetcherResponse(body={'status': 200, 'data': {'scan_id': 12345}}, status=200, headers={}) async def fake_fetch(*_args, **_kwargs): - return {'status': 200, 'data': {'scan_percentage': 50}} + return FetcherResponse(body={'status': 200, 'data': {'scan_percentage': 50}}, status=200, headers={}) async def cancel(*_args, **_kwargs): raise asyncio.CancelledError @@ -232,4 +239,40 @@ async def test_provider_timeout_returns_explicit_transport_error(monkeypatch) -> assert await criminalip.SearchCriminalIP('example.com').process() == SourceExecutionReport('failed', 'transport-error') +@pytest.mark.asyncio +@pytest.mark.parametrize( + ('status', 'expected_report'), + [ + (401, SourceExecutionReport('failed', 'access-denied')), + (403, SourceExecutionReport('failed', 'access-denied')), + (429, SourceExecutionReport('rate-limited', 'http-429')), + (500, SourceExecutionReport('failed', 'http-500')), + ], +) +async def test_http_failures_are_classified_before_body_inspection(monkeypatch, status, expected_report) -> None: + monkeypatch.setattr(criminalip.Core, 'criminalip_key', lambda: 'test-key') + monkeypatch.setattr(criminalip.Core, 'get_user_agent', lambda: 'test-agent') + + async def fake_post_fetch(*_args, **kwargs): + assert kwargs['include_metadata'] is True + return FetcherResponse(body='provider detail', status=status, headers={}) + + monkeypatch.setattr(criminalip.AsyncFetcher, 'post_fetch', fake_post_fetch) + + assert await criminalip.SearchCriminalIP('example.com').process() == expected_report + + +@pytest.mark.asyncio +async def test_transport_failure_without_a_response(monkeypatch) -> None: + monkeypatch.setattr(criminalip.Core, 'criminalip_key', lambda: 'test-key') + monkeypatch.setattr(criminalip.Core, 'get_user_agent', lambda: 'test-agent') + + async def fake_post_fetch(*_args, **_kwargs): + return None + + monkeypatch.setattr(criminalip.AsyncFetcher, 'post_fetch', fake_post_fetch) + + assert await criminalip.SearchCriminalIP('example.com').process() == SourceExecutionReport('failed', 'transport-error') + + pytestmark = pytest.mark.provider_contract('criminalip') diff --git a/tests/discovery/test_githubcode_contract.py b/tests/discovery/test_githubcode_contract.py index 1d09e498..b9b95d6d 100644 --- a/tests/discovery/test_githubcode_contract.py +++ b/tests/discovery/test_githubcode_contract.py @@ -323,4 +323,40 @@ async def test_github_code_cancellation_propagates(monkeypatch: pytest.MonkeyPat await search.process() +@pytest.mark.asyncio +async def test_github_code_forbidden_fails_immediately_as_access_denied(install_github_responses) -> None: + class ForbiddenResponse(FakeResponse): + status = 403 + + requested_urls = install_github_responses(ForbiddenResponse({}, {})) + search = githubcode.SearchGithubCode('example.com', limit=None) + + report = await search.process() + + assert len(requested_urls) == 1 + assert report == githubcode.SourceExecutionReport('failed', 'access-denied') + + +@pytest.mark.asyncio +async def test_github_code_rate_limited_retries_then_reports( + install_github_responses, + monkeypatch: pytest.MonkeyPatch, +) -> None: + class TooManyRequestsResponse(FakeResponse): + status = 429 + + requested_urls = install_github_responses(*(TooManyRequestsResponse({}, {}) for _ in range(4))) + + async def no_sleep(_delay: float) -> None: + return None + + monkeypatch.setattr(githubcode.asyncio, 'sleep', no_sleep) + search = githubcode.SearchGithubCode('example.com', limit=None) + + report = await search.process() + + assert len(requested_urls) == 4 + assert report == githubcode.SourceExecutionReport('rate-limited', 'rate-limited') + + pytestmark = pytest.mark.provider_contract('github-code') diff --git a/tests/test_mojeek.py b/tests/test_mojeek.py index 22841a60..3d68f87c 100644 --- a/tests/test_mojeek.py +++ b/tests/test_mojeek.py @@ -269,10 +269,8 @@ class TestMojeekSearch: monkeypatch: pytest.MonkeyPatch, ) -> None: requests: list[dict[str, Any]] = [] - - async def fake_fetch_all(urls: list[str], **kwargs: Any) -> list[FetcherResponse]: - requests.append({'urls': urls, **kwargs}) - return [ + responses = iter( + [ FetcherResponse( status=200, headers={}, @@ -290,6 +288,11 @@ class TestMojeekSearch: ), FetcherResponse(body={'response': {'results': []}}, status=200, headers={}), ] + ) + + async def fake_fetch_all(urls: list[str], **kwargs: Any) -> list[FetcherResponse]: + requests.append({'urls': urls, **kwargs}) + return [next(responses)] async def reject_scrape(**_kwargs: Any) -> FetcherResponse: raise AssertionError('successful keyed API calls must not scrape') @@ -304,15 +307,19 @@ class TestMojeekSearch: assert requests == [ { - 'urls': [ - 'https://api.mojeek.com/search?api_key=test-key&q=example.com&fmt=json&s=1', - 'https://api.mojeek.com/search?api_key=test-key&q=example.com&fmt=json&s=11', - ], + 'urls': ['https://api.mojeek.com/search?api_key=test-key&q=example.com&fmt=json&s=1'], 'headers': {'User-Agent': 'UA'}, 'proxy': True, 'json': True, 'include_metadata': True, - } + }, + { + 'urls': ['https://api.mojeek.com/search?api_key=test-key&q=example.com&fmt=json&s=11'], + 'headers': {'User-Agent': 'UA'}, + 'proxy': True, + 'json': True, + 'include_metadata': True, + }, ] assert await search.get_emails() == {'admin@example.com'} assert set(await search.get_hostnames()) - {'example.com'} == { @@ -321,5 +328,44 @@ class TestMojeekSearch: } assert report is None + @pytest.mark.asyncio + @pytest.mark.parametrize( + ('limit', 'expected_offsets'), + [ + (1, ['s=1']), + (10, ['s=1']), + (11, ['s=1', 's=11']), + (25, ['s=1', 's=11', 's=21']), + ], + ) + async def test_keyed_api_enumerates_every_page_covering_the_limit( + self, + monkeypatch: pytest.MonkeyPatch, + limit: int, + expected_offsets: list[str], + ) -> None: + requested_urls: list[str] = [] + + async def fake_fetch_all(urls: list[str], **_kwargs: Any) -> list[FetcherResponse]: + requested_urls.extend(urls) + offset = urls[0].rsplit('s=', 1)[1] + return [ + FetcherResponse( + body={'response': {'results': [{'url': f'https://page-{offset}.example.com'}]}}, + status=200, + headers={}, + ) + ] + + monkeypatch.setattr(mojeek.Core, 'mojeek_key', staticmethod(lambda: 'test-key')) + monkeypatch.setattr(mojeek.AsyncFetcher, 'fetch_all', fake_fetch_all) + + search = mojeek.SearchMojeek(word='example.com', limit=limit) + await search.process() + + assert requested_urls == [ + f'https://api.mojeek.com/search?api_key=test-key&q=example.com&fmt=json&{offset}' for offset in expected_offsets + ] + pytestmark = pytest.mark.provider_contract('mojeek') diff --git a/theHarvester/discovery/criminalip.py b/theHarvester/discovery/criminalip.py index 5f3cb972..7ea30767 100644 --- a/theHarvester/discovery/criminalip.py +++ b/theHarvester/discovery/criminalip.py @@ -4,7 +4,8 @@ from typing import TYPE_CHECKING, Any from urllib.parse import urlparse from theHarvester.discovery.constants import MissingKey, get_delay -from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.discovery.provider_response import provider_http_error +from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse from theHarvester.lib.source_execution import SourceExecutionReport if TYPE_CHECKING: @@ -88,32 +89,53 @@ class SearchCriminalIP: for nested_value in value.values(): self._collect_hosts_from_value(nested_value) + @staticmethod + def _transport_report(response: object, stage: str) -> SourceExecutionReport | None: + """Classify transport and HTTP failures for one CriminalIP response.""" + if not isinstance(response, FetcherResponse): + logger.info(f'CriminalIP {stage} request failed without a response') + return SourceExecutionReport('failed', 'transport-error') + if error := provider_http_error(response): + logger.info(f'CriminalIP {stage} request failed with HTTP {response.status}') + return SourceExecutionReport(*error) + return None + + @staticmethod + def _provider_status_report(payload: dict, stage: str) -> SourceExecutionReport | None: + """Classify application-level status codes reported inside a 200 response.""" + status = payload.get('status') + if status == 200: + return None + logger.info(f'CriminalIP {stage} request failed with status {status}') + if status == 429: + return SourceExecutionReport('rate-limited', 'http-429') + if status in {401, 403}: + return SourceExecutionReport('failed', 'access-denied') + return SourceExecutionReport('failed', 'provider-error') + async def do_search(self, session: aiohttp.ClientSession) -> SourceExecutionReport | None: # https://www.criminalip.io/developer/api/post-domain-scan # https://www.criminalip.io/developer/api/get-domain-status-id # https://www.criminalip.io/developer/api/get-v2-domain-report-id url = 'https://api.criminalip.io/v1/domain/scan' - data = f'{{"query": "{self.word}"}}' response = await AsyncFetcher.post_fetch( url, json=True, - data=data, + json_body={'query': self.word}, session=session, + include_metadata=True, ) # Expected response format: # {'data': {'scan_id': scan_id}, 'message': 'api success', 'status': 200} - if not isinstance(response, dict): - logger.info(f'CriminalIP scan response has unexpected type: {type(response).__name__}') + if report := self._transport_report(response, 'scan'): + return report + if not isinstance(response.body, dict): + logger.info(f'CriminalIP scan response has unexpected type: {type(response.body).__name__}') return SourceExecutionReport('failed', 'invalid-response') - if response.get('status') != 200: - logger.info(f'CriminalIP scan request failed with status {response.get("status")}') - if response.get('status') == 429: - return SourceExecutionReport('rate-limited', 'http-429') - if response.get('status') in {401, 403}: - return SourceExecutionReport('failed', 'access-denied') - return SourceExecutionReport('failed', 'provider-error') + if report := self._provider_status_report(response.body, 'scan'): + return report - scan_id = response.get('data', {}).get('scan_id') + scan_id = response.body.get('data', {}).get('scan_id') if scan_id is None: logger.info('CriminalIP scan response did not include a scan_id') return SourceExecutionReport('failed', 'invalid-response') @@ -123,22 +145,21 @@ class SearchCriminalIP: status: dict[str, Any] = {} while scan_percentage != 100: status_url = f'https://api.criminalip.io/v1/domain/status/{scan_id}' - status = await AsyncFetcher.fetch( + poll = await AsyncFetcher.fetch( session=session, url=status_url, json=True, request_timeout=60, + include_metadata=True, ) - if not isinstance(status, dict): - logger.info(f'CriminalIP status response has unexpected type: {type(status).__name__}') + if report := self._transport_report(poll, 'status'): + return report + if not isinstance(poll.body, dict): + logger.info(f'CriminalIP status response has unexpected type: {type(poll.body).__name__}') return SourceExecutionReport('failed', 'invalid-response') - if status.get('status') != 200: - logger.info(f'CriminalIP status request failed with status {status.get("status")}') - if status.get('status') == 429: - return SourceExecutionReport('rate-limited', 'http-429') - if status.get('status') in {401, 403}: - return SourceExecutionReport('failed', 'access-denied') - return SourceExecutionReport('failed', 'provider-error') + status = poll.body + if report := self._provider_status_report(status, 'status'): + return report # Expected format: # {"data": {"scan_percentage": 100}, "message": "api success", "status": 200} @@ -172,20 +193,18 @@ class SearchCriminalIP: url=report_url, json=True, request_timeout=60, + include_metadata=True, ) - if not isinstance(scan, dict): - logger.info(f'CriminalIP report response has unexpected type: {type(scan).__name__}') + if report := self._transport_report(scan, 'report'): + return report + if not isinstance(scan.body, dict): + logger.info(f'CriminalIP report response has unexpected type: {type(scan.body).__name__}') return SourceExecutionReport('failed', 'invalid-response') - if scan.get('status') != 200: - logger.info(f'CriminalIP report request failed with status {scan.get("status")}') - if scan.get('status') == 429: - return SourceExecutionReport('rate-limited', 'http-429') - if scan.get('status') in {401, 403}: - return SourceExecutionReport('failed', 'access-denied') - return SourceExecutionReport('failed', 'provider-error') + if report := self._provider_status_report(scan.body, 'report'): + return report try: - await self.parser(scan) + await self.parser(scan.body) except Exception as e: logger.info(f'CriminalIP report parsing failed with {type(e).__name__}') return SourceExecutionReport('failed', 'invalid-response') diff --git a/theHarvester/discovery/githubcode.py b/theHarvester/discovery/githubcode.py index 77581a55..f63a5720 100644 --- a/theHarvester/discovery/githubcode.py +++ b/theHarvester/discovery/githubcode.py @@ -98,7 +98,7 @@ class SearchGithubCode: next_page = await self.page_from_response('next', links) or 0 last_page = await self.page_from_response('last', links) or 0 return SuccessResult(results, next_page, last_page) - if status in (429, 403): + if status == 429: return RetryResult(60) return ErrorResult(status, json_data if isinstance(json_data, dict) else text) except Exception as e: diff --git a/theHarvester/discovery/hudsonrocksearch.py b/theHarvester/discovery/hudsonrocksearch.py index 1747b839..c5e15445 100644 --- a/theHarvester/discovery/hudsonrocksearch.py +++ b/theHarvester/discovery/hudsonrocksearch.py @@ -99,7 +99,7 @@ class SearchHudsonRock: return bool(re.match(pattern, email)) async def _search_domain(self, domain: str, session: ClientSession) -> SourceExecutionReport | None: - """Search Hudson Rock by domain with retry logic. + """Search Hudson Rock by domain, retrying rate-limited responses. Args: domain: Domain to search. @@ -115,7 +115,7 @@ class SearchHudsonRock: return None async def _search_email(self, email: str, session: ClientSession) -> SourceExecutionReport | None: - """Search Hudson Rock by email with retry logic. + """Search Hudson Rock by email, retrying rate-limited responses. Args: email: Email address to search. @@ -133,43 +133,37 @@ class SearchHudsonRock: async def _fetch_response( self, url: str, search_type: str, target: str, session: ClientSession ) -> tuple[dict | None, SourceExecutionReport | None]: + # The shared transport swallows transport errors and returns None elements, + # so only HTTP 429 responses are retried here; transport failures are terminal. for attempt in range(self.max_retries): - try: - self.logger.debug(f'Searching {search_type}: {target} (attempt {attempt + 1})') - responses = await AsyncFetcher.fetch_all([url], session=session, json=True, include_metadata=True) - response = responses[0] if responses and isinstance(responses[0], FetcherResponse) else None - if response is None: - self.logger.warning(f'Invalid response format for {search_type} search: {target}') - return None, SourceExecutionReport('failed', 'transport-error') - if isinstance(response, FetcherResponse) and response.status == 429: - if attempt == self.max_retries - 1: - self.logger.info(f'Hudson Rock {search_type} search returned HTTP 429 after {self.max_retries} attempts') - return None, SourceExecutionReport('rate-limited', 'http-429') - retry_after = response.headers.get('retry-after') - try: - delay = int(retry_after) if retry_after is not None else 2**attempt - except ValueError: - delay = 2**attempt - await asyncio.sleep(max(0, min(delay, 60))) - continue - if error := provider_http_error(response): - self.logger.info(f'Hudson Rock {search_type} search failed with HTTP {response.status}') - return None, SourceExecutionReport(*error) - if not isinstance(response.body, dict): - self.logger.warning(f'Invalid response format for {search_type} search: {target}') - return None, SourceExecutionReport('failed', 'invalid-response') - if response.body.get('error'): - self.logger.info(f'Hudson Rock {search_type} search returned a provider error') - return None, SourceExecutionReport('failed', 'provider-error') - return response.body, None - - except OSError, RuntimeError, ValueError: - self.logger.error(f'Hudson Rock {search_type} search attempt {attempt + 1} failed') - if attempt < self.max_retries - 1: - await asyncio.sleep(2**attempt) - else: - return None, SourceExecutionReport('failed', 'transport-error') - return None, SourceExecutionReport('failed', 'transport-error') + self.logger.debug(f'Searching {search_type}: {target} (attempt {attempt + 1})') + responses = await AsyncFetcher.fetch_all([url], session=session, json=True, include_metadata=True) + response = responses[0] if responses and isinstance(responses[0], FetcherResponse) else None + if response is None: + self.logger.warning(f'Invalid response format for {search_type} search: {target}') + return None, SourceExecutionReport('failed', 'transport-error') + if response.status == 429: + if attempt == self.max_retries - 1: + self.logger.info(f'Hudson Rock {search_type} search returned HTTP 429 after {self.max_retries} attempts') + return None, SourceExecutionReport('rate-limited', 'http-429') + retry_after = response.headers.get('retry-after') + try: + delay = int(retry_after) if retry_after is not None else 2**attempt + except ValueError: + delay = 2**attempt + await asyncio.sleep(max(0, min(delay, 60))) + continue + if error := provider_http_error(response): + self.logger.info(f'Hudson Rock {search_type} search failed with HTTP {response.status}') + return None, SourceExecutionReport(*error) + if not isinstance(response.body, dict): + self.logger.warning(f'Invalid response format for {search_type} search: {target}') + return None, SourceExecutionReport('failed', 'invalid-response') + if response.body.get('error'): + self.logger.info(f'Hudson Rock {search_type} search returned a provider error') + return None, SourceExecutionReport('failed', 'provider-error') + return response.body, None + return None, SourceExecutionReport('rate-limited', 'http-429') def _process_domain_response(self, response: dict) -> bool: """Process domain search response from Hudson Rock API. diff --git a/theHarvester/discovery/mojeek.py b/theHarvester/discovery/mojeek.py index ce56c0f9..b7d08a86 100644 --- a/theHarvester/discovery/mojeek.py +++ b/theHarvester/discovery/mojeek.py @@ -102,23 +102,15 @@ class SearchMojeek: return result_limit = self.limit - urls = [ - f'https://{self.api_server}/search?api_key={self.api_key}&q={self.word}&fmt=json&s={num}' - for num in range(1, result_limit, 10) - ] - responses = await AsyncFetcher.fetch_all( - urls, - headers=headers, - proxy=self.proxy, - json=True, - include_metadata=True, - ) seen_finite_pages: set[tuple[str, ...]] = set() - for response in responses: - if not isinstance(response, FetcherResponse): + offset = 1 + while offset <= result_limit: + url = f'https://{self.api_server}/search?api_key={self.api_key}&q={self.word}&fmt=json&s={offset}' + responses = await AsyncFetcher.fetch_all([url], headers=headers, proxy=self.proxy, json=True, include_metadata=True) + if len(responses) != 1 or not isinstance(responses[0], FetcherResponse): self._stop('failed', 'transport-error') return - parsed_results = self._api_page_results(response) + parsed_results = self._api_page_results(responses[0]) if parsed_results is None: return if not parsed_results: @@ -129,6 +121,7 @@ class SearchMojeek: return seen_finite_pages.add(signature) self.total_results += f' {" ".join(parsed_results)} ' + offset += 10 logger.info('[*] Mojeek: API search completed successfully.') async def _search_keyless(self, headers: dict[str, str]) -> None: