From 9b982db7b68aae3b9ff3c52f3008aff51dd48367 Mon Sep 17 00:00:00 2001 From: NotoriousRebel <36310667+NotoriousRebel@users.noreply.github.com> Date: Sun, 16 Aug 2026 16:47:59 -0400 Subject: [PATCH] fix: report provider collection outcomes truthfully --- tests/discovery/test_brave.py | 276 +++++++++++++++--- tests/discovery/test_dnsdb.py | 103 ++++++- tests/discovery/test_duckduckgo.py | 118 +++++++- .../test_provider_session_lifecycle.py | 85 +++++- tests/lib/test_core.py | 21 ++ tests/test_bravesearch.py | 33 ++- theHarvester/discovery/bravesearch.py | 155 +++++----- theHarvester/discovery/dnsdb.py | 45 +-- theHarvester/discovery/duckduckgosearch.py | 105 ++++++- theHarvester/lib/core.py | 2 + 10 files changed, 770 insertions(+), 173 deletions(-) diff --git a/tests/discovery/test_brave.py b/tests/discovery/test_brave.py index 0eb2d7d7..129b5001 100644 --- a/tests/discovery/test_brave.py +++ b/tests/discovery/test_brave.py @@ -1,3 +1,5 @@ +import contextlib +from collections.abc import AsyncIterator from typing import Any from urllib.parse import parse_qs, urlparse @@ -6,6 +8,8 @@ import pytest from theHarvester.discovery import bravesearch from theHarvester.discovery.constants import MissingKey from theHarvester.lib.configuration import InMemoryCredentialAdapter +from theHarvester.lib.core import FetcherResponse, ResponseStreamError +from theHarvester.lib.source_execution import SourceExecutionReport def _result(index: int) -> dict[str, str]: @@ -16,11 +20,15 @@ def _result(index: int) -> dict[str, str]: } -def _response(results: list[dict[str, str]], *, more: bool) -> dict[str, Any]: - return { - 'query': {'more_results_available': more}, - 'web': {'results': results}, - } +def _response(results: list[dict[str, str]], *, more: bool) -> FetcherResponse: + return FetcherResponse( + { + 'query': {'more_results_available': more}, + 'web': {'results': results}, + }, + 200, + {}, + ) @pytest.fixture @@ -58,43 +66,210 @@ async def test_brave_normalizes_in_scope_evidence( ], more=False, ), - {'error': {'message': 'Access denied', 'code': 'forbidden'}}, + FetcherResponse({'error': {'message': 'Access denied', 'code': 'forbidden'}}, 200, {}), ] ) - proxies: list[bool] = [] + request_proxies: list[bool | None] = [] + session_options: list[dict[str, Any]] = [] - async def fake_fetch(*, url: str, **kwargs: Any) -> dict[str, Any]: - proxies.append(kwargs['proxy']) + @contextlib.asynccontextmanager + async def fake_open_session(**kwargs: Any) -> AsyncIterator[object]: + session_options.append(kwargs) + yield object() + + async def fake_fetch(*_args: Any, **kwargs: Any) -> FetcherResponse: + request_proxies.append(kwargs.get('proxy')) return next(responses) - monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch', fake_fetch) + monkeypatch.setattr(bravesearch.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch) search = bravesearch.SearchBrave('example.com', 10, credential_adapter=brave_credentials) await search.process(proxy=True) - assert proxies == [True, True] + assert session_options == [{'headers': search_headers(search), 'proxy': True, 'request_timeout': 60}] + assert request_proxies == [None, None] assert await search.get_emails() == {'admin@example.com'} assert await search.get_hostnames() == ['blog.example.com', 'example.com'] -@pytest.mark.parametrize('response', [None, []], ids=['empty', 'malformed']) +def search_headers(search: bravesearch.SearchBrave) -> dict[str, str]: + return { + 'Accept': 'application/json', + 'Accept-Encoding': 'gzip', + 'X-Subscription-Token': search.api_key, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize('failure_point', ['open', 'close']) +async def test_brave_reports_session_lifecycle_failures( + monkeypatch: pytest.MonkeyPatch, + brave_credentials: InMemoryCredentialAdapter, + failure_point: str, +) -> None: + @contextlib.asynccontextmanager + async def failed_open_session(**_kwargs: Any) -> AsyncIterator[object]: + if failure_point == 'open': + raise OSError('session creation failed') + yield object() + raise OSError('session close failed') + + async def fake_fetch_json(*_args: Any, **_kwargs: Any) -> FetcherResponse: + return _response([], more=False) + + monkeypatch.setattr(bravesearch.AsyncFetcher, 'open_session', failed_open_session) + monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch_json) + search = bravesearch.SearchBrave('example.com', 10, credential_adapter=brave_credentials) + + assert await search.process() == SourceExecutionReport('failed', 'transport-error') + + +@pytest.mark.asyncio +async def test_brave_does_not_misclassify_value_errors_as_transport_failures( + monkeypatch: pytest.MonkeyPatch, + brave_credentials: InMemoryCredentialAdapter, +) -> None: + @contextlib.asynccontextmanager + async def failed_open_session(**_kwargs: Any) -> AsyncIterator[object]: + raise ValueError('programming defect') + yield object() + + monkeypatch.setattr(bravesearch.AsyncFetcher, 'open_session', failed_open_session) + search = bravesearch.SearchBrave('example.com', 10, credential_adapter=brave_credentials) + + with pytest.raises(ValueError, match='programming defect'): + await search.process() + + +@pytest.mark.parametrize('body', [None, []], ids=['empty', 'malformed']) @pytest.mark.asyncio async def test_brave_unusable_response_returns_no_evidence( monkeypatch: pytest.MonkeyPatch, brave_credentials: InMemoryCredentialAdapter, - response: list[Any] | None, + body: list[Any] | None, ) -> None: - async def fake_fetch(*, url: str, **_kwargs: Any) -> list[Any] | None: - return response + async def fake_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse: + return FetcherResponse(body, 200, {}) - monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch', fake_fetch) + monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch) search = bravesearch.SearchBrave('example.com', 10, credential_adapter=brave_credentials) - await search.process() + report = await search.process() assert await search.get_emails() == set() assert await search.get_hostnames() == [] + assert report == SourceExecutionReport('failed', 'invalid-response') + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ('response', 'expected_report'), + [ + (FetcherResponse(None, 401, {}), SourceExecutionReport('failed', 'access-denied')), + (FetcherResponse(None, 429, {}), SourceExecutionReport('rate-limited', 'http-429')), + (FetcherResponse(None, 503, {}), SourceExecutionReport('failed', 'http-503')), + ], +) +async def test_brave_reports_http_failures( + monkeypatch: pytest.MonkeyPatch, + brave_credentials: InMemoryCredentialAdapter, + response: FetcherResponse, + expected_report: SourceExecutionReport, +) -> None: + async def fake_fetch_json(*_args: Any, **_kwargs: Any) -> FetcherResponse: + return response + + async def legacy_fetch(*_args: Any, **_kwargs: Any) -> None: + raise AssertionError('Brave must use the bounded fetch_json seam') + + monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch_json) + monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch', legacy_fetch) + + search = bravesearch.SearchBrave('example.com', 10, credential_adapter=brave_credentials) + + assert await search.process() == expected_report + + +@pytest.mark.asyncio +@pytest.mark.parametrize('reason', ['transport-error', 'invalid-response', 'response-limit']) +async def test_brave_reports_bounded_fetch_failures( + monkeypatch: pytest.MonkeyPatch, + brave_credentials: InMemoryCredentialAdapter, + reason: str, +) -> None: + async def fake_fetch_json(*_args: Any, **_kwargs: Any) -> FetcherResponse: + raise ResponseStreamError(reason) # type: ignore[arg-type] + + monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch_json) + search = bravesearch.SearchBrave('example.com', 10, credential_adapter=brave_credentials) + + assert await search.process() == SourceExecutionReport('failed', reason) + + +@pytest.mark.asyncio +async def test_brave_propagates_cancellation( + monkeypatch: pytest.MonkeyPatch, + brave_credentials: InMemoryCredentialAdapter, +) -> None: + async def fake_fetch_json(*_args: Any, **_kwargs: Any) -> FetcherResponse: + raise bravesearch.asyncio.CancelledError + + monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch_json) + search = bravesearch.SearchBrave('example.com', 10, credential_adapter=brave_credentials) + + with pytest.raises(bravesearch.asyncio.CancelledError): + await search.process() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + 'body', + [ + {}, + {'web': []}, + {'web': {'results': 'invalid'}, 'query': {'more_results_available': False}}, + {'web': {'results': []}, 'query': {}}, + {'web': {'results': ['invalid']}, 'query': {'more_results_available': False}}, + ], +) +async def test_brave_reports_malformed_payloads( + monkeypatch: pytest.MonkeyPatch, + brave_credentials: InMemoryCredentialAdapter, + body: dict[str, Any], +) -> None: + async def fake_fetch_json(*_args: Any, **_kwargs: Any) -> FetcherResponse: + return FetcherResponse(body, 200, {}) + + monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch_json) + search = bravesearch.SearchBrave('example.com', 10, credential_adapter=brave_credentials) + + assert await search.process() == SourceExecutionReport('failed', 'invalid-response') + + +@pytest.mark.asyncio +@pytest.mark.parametrize('field', ['title', 'description', 'url']) +async def test_brave_rejects_non_string_evidence_fields( + monkeypatch: pytest.MonkeyPatch, + brave_credentials: InMemoryCredentialAdapter, + field: str, +) -> None: + result: dict[str, object] = _result(1) + result[field] = {'unexpected': 'host.example.com'} + + async def fake_fetch_json(*_args: Any, **_kwargs: Any) -> FetcherResponse: + return FetcherResponse( + {'web': {'results': [result]}, 'query': {'more_results_available': False}}, + 200, + {}, + ) + + monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch_json) + search = bravesearch.SearchBrave('example.com', 10, credential_adapter=brave_credentials) + + assert await search.process() == SourceExecutionReport('failed', 'invalid-response') + assert search.totalresults == '' @pytest.mark.asyncio @@ -110,11 +285,12 @@ async def test_brave_uses_page_offsets_and_one_global_limit( ) requests: list[dict[str, list[str]]] = [] - async def fake_fetch(*, url: str, **_kwargs: Any) -> dict[str, Any]: + async def fake_fetch(*_args: Any, **kwargs: Any) -> FetcherResponse: + url = kwargs.get('url', _args[0] if _args else '') requests.append(parse_qs(urlparse(url).query)) return next(responses) - monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch', fake_fetch) + monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch) search = bravesearch.SearchBrave('example.com', 25, credential_adapter=brave_credentials) await search.process() @@ -156,11 +332,12 @@ async def test_brave_requests_another_page_only_when_available( ) requests: list[dict[str, list[str]]] = [] - async def fake_fetch(*, url: str, **_kwargs: Any) -> dict[str, Any]: + async def fake_fetch(*_args: Any, **kwargs: Any) -> FetcherResponse: + url = kwargs.get('url', _args[0] if _args else '') requests.append(parse_qs(urlparse(url).query)) return next(responses, _response([], more=False)) - monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch', fake_fetch) + monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch) search = bravesearch.SearchBrave('example.com', 10, credential_adapter=brave_credentials) await search.process() @@ -183,11 +360,12 @@ async def test_brave_continues_sparse_pages_while_more_results_are_available( ) requests: list[dict[str, list[str]]] = [] - async def fake_fetch(*, url: str, **_kwargs: Any) -> dict[str, Any]: + async def fake_fetch(*_args: Any, **kwargs: Any) -> FetcherResponse: + url = kwargs.get('url', _args[0] if _args else '') requests.append(parse_qs(urlparse(url).query)) return next(responses) - monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch', fake_fetch) + monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch) search = bravesearch.SearchBrave('example.com', 20, credential_adapter=brave_credentials) await search.process() @@ -198,6 +376,23 @@ async def test_brave_continues_sparse_pages_while_more_results_are_available( assert len(search.results) == 20 +@pytest.mark.asyncio +async def test_brave_rejects_empty_page_that_claims_more_results( + monkeypatch: pytest.MonkeyPatch, + brave_credentials: InMemoryCredentialAdapter, +) -> None: + async def fake_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse: + return _response([], more=True) + + monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch) + search = bravesearch.SearchBrave('example.com', 20, credential_adapter=brave_credentials) + + report = await search.process() + + assert report == SourceExecutionReport('failed', 'invalid-response') + assert search.results == [] + + @pytest.mark.asyncio async def test_brave_stops_after_an_exact_full_page( monkeypatch: pytest.MonkeyPatch, @@ -205,15 +400,17 @@ async def test_brave_stops_after_an_exact_full_page( ) -> None: requests: list[dict[str, list[str]]] = [] - async def fake_fetch(*, url: str, **_kwargs: Any) -> dict[str, Any]: + async def fake_fetch(*_args: Any, **kwargs: Any) -> FetcherResponse: + url = kwargs.get('url', _args[0] if _args else '') requests.append(parse_qs(urlparse(url).query)) return _response([_result(index) for index in range(20)], more=True) - monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch', fake_fetch) + monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch) search = bravesearch.SearchBrave('example.com', 20, credential_adapter=brave_credentials) - await search.process() + report = await search.process() assert [(request['offset'], request['count']) for request in requests] == [(['0'], ['20'])] + assert report == SourceExecutionReport('completed', 'result-limit') @pytest.mark.asyncio @@ -223,42 +420,43 @@ async def test_brave_rate_limit_does_not_skip_to_the_next_page( ) -> None: responses = iter( [ - {'error': {'message': 'Rate limit exceeded', 'code': 'rate_limit_exceeded'}}, + FetcherResponse({'error': {'message': 'Rate limit exceeded', 'code': 'rate_limit_exceeded'}}, 200, {}), _response([], more=False), ] ) requests: list[dict[str, list[str]]] = [] - async def fake_fetch(*, url: str, **_kwargs: Any) -> dict[str, Any]: + async def fake_fetch(*_args: Any, **kwargs: Any) -> FetcherResponse: + url = kwargs.get('url', _args[0] if _args else '') requests.append(parse_qs(urlparse(url).query)) return next(responses) - monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch', fake_fetch) + monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch) search = bravesearch.SearchBrave('example.com', 40, credential_adapter=brave_credentials) - await search.process() + report = await search.process() - assert [(request['q'], request['offset']) for request in requests] == [ - (['"example.com"'], ['0']), - (['site:example.com'], ['0']), - ] + assert [(request['q'], request['offset']) for request in requests] == [(['"example.com"'], ['0'])] + assert report == SourceExecutionReport('rate-limited', 'provider-rate-limit') @pytest.mark.asyncio -async def test_brave_never_exceeds_maximum_page_offset( +async def test_brave_reports_maximum_page_offset_as_truncation( monkeypatch: pytest.MonkeyPatch, brave_credentials: InMemoryCredentialAdapter, ) -> None: requests: list[dict[str, list[str]]] = [] - async def fake_fetch(*, url: str, **_kwargs: Any) -> dict[str, Any]: + async def fake_fetch(*_args: Any, **kwargs: Any) -> FetcherResponse: + url = kwargs.get('url', _args[0] if _args else '') requests.append(parse_qs(urlparse(url).query)) return _response([_result(len(requests))], more=True) - monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch', fake_fetch) + monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch) search = bravesearch.SearchBrave('example.com', 1_000, credential_adapter=brave_credentials) - await search.process() + report = await search.process() - assert [request['offset'] for request in requests] == [[str(offset)] for offset in range(10)] * 2 + assert [request['offset'] for request in requests] == [[str(offset)] for offset in range(10)] + assert report == SourceExecutionReport('partial', 'pagination-limit') pytestmark = pytest.mark.provider_contract('brave') diff --git a/tests/discovery/test_dnsdb.py b/tests/discovery/test_dnsdb.py index 54608948..8fc2d161 100644 --- a/tests/discovery/test_dnsdb.py +++ b/tests/discovery/test_dnsdb.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import contextlib import logging @@ -7,6 +8,7 @@ import pytest from theHarvester.discovery import dnsdb from theHarvester.discovery.constants import MissingKey +from theHarvester.lib.source_execution import SourceExecutionReport def _install_response( @@ -108,10 +110,14 @@ async def test_process_uses_configured_proxy_when_enabled(monkeypatch: pytest.Mo @pytest.mark.asyncio @pytest.mark.parametrize( - ('last_line', 'expected_message'), + ('last_line', 'expected_message', 'expected_report'), [ - (b'{"cond":"limited"}\n', 'ended with limited'), - (b'not-json\n', 'malformed NDJSON'), + ( + b'{"cond":"limited"}\n', + 'ended with limited', + SourceExecutionReport('rate-limited', 'provider-limited'), + ), + (b'not-json\n', 'malformed NDJSON', SourceExecutionReport('failed', 'invalid-response')), ], ) async def test_process_preserves_partial_results( @@ -119,6 +125,7 @@ async def test_process_preserves_partial_results( caplog: pytest.LogCaptureFixture, last_line: bytes, expected_message: str, + expected_report: SourceExecutionReport, ) -> None: caplog.set_level(logging.INFO, logger=dnsdb.__name__) monkeypatch.setattr(dnsdb.Core, 'dnsdb_key', lambda: 'dnsdb-test-key') @@ -132,12 +139,53 @@ async def test_process_preserves_partial_results( ) search = dnsdb.SearchDNSDB('example.com') - await search.process() + report = await search.process() assert await search.get_hostnames() == {'first.example.com'} + assert report == expected_report assert any(expected_message in message for message in caplog.messages) +@pytest.mark.asyncio +@pytest.mark.parametrize( + ('lines', 'expected_report'), + [ + ((b'[]\n',), SourceExecutionReport('failed', 'invalid-response')), + ((b'{"cond":"wrong"}\n',), SourceExecutionReport('failed', 'invalid-response')), + ( + (b'{"cond":"begin"}\n', b'{"cond":"failed"}\n'), + SourceExecutionReport('failed', 'provider-failed'), + ), + ((b'{"cond":"begin"}\n',), SourceExecutionReport('failed', 'invalid-response')), + ( + (b'{"cond":"begin"}\n', b'{"cond":"wrong"}\n', b'{"cond":"succeeded"}\n'), + SourceExecutionReport('failed', 'invalid-response'), + ), + ( + (b'{"cond":"begin"}\n', b'{"obj":[]}\n', b'{"cond":"succeeded"}\n'), + SourceExecutionReport('failed', 'invalid-response'), + ), + ( + (b'{"cond":"begin"}\n', b'{"obj":{}}\n', b'{"cond":"succeeded"}\n'), + SourceExecutionReport('failed', 'invalid-response'), + ), + ( + (b'{"cond":"begin"}\n', b'{"obj":{"rrname":7}}\n', b'{"cond":"succeeded"}\n'), + SourceExecutionReport('failed', 'invalid-response'), + ), + ], +) +async def test_process_reports_abnormal_stream_termination( + monkeypatch: pytest.MonkeyPatch, + lines: tuple[bytes, ...], + expected_report: SourceExecutionReport, +) -> None: + monkeypatch.setattr(dnsdb.Core, 'dnsdb_key', lambda: 'dnsdb-test-key') + _install_response(monkeypatch, lines) + + assert await dnsdb.SearchDNSDB('example.com').process() == expected_report + + @pytest.mark.asyncio async def test_process_preserves_partial_results_on_midstream_timeout( monkeypatch: pytest.MonkeyPatch, @@ -155,31 +203,60 @@ async def test_process_preserves_partial_results_on_midstream_timeout( ) search = dnsdb.SearchDNSDB('example.com') - await search.process() + report = await search.process() assert await search.get_hostnames() == {'first.example.com'} + assert report == SourceExecutionReport('failed', 'transport-error') assert any('request failed' in message for message in caplog.messages) +@pytest.mark.asyncio +async def test_process_reports_deeply_nested_json_as_invalid_response(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(dnsdb.Core, 'dnsdb_key', lambda: 'dnsdb-test-key') + _install_response(monkeypatch, (b'{"cond":"begin"}\n', b'deeply-nested-json\n')) + json_loads = dnsdb.json.loads + + def parse_record(line: str) -> object: + if line.strip() == 'deeply-nested-json': + raise RecursionError + return json_loads(line) + + monkeypatch.setattr(dnsdb.json, 'loads', parse_record) + + assert await dnsdb.SearchDNSDB('example.com').process() == SourceExecutionReport('failed', 'invalid-response') + + +@pytest.mark.asyncio +async def test_process_propagates_cancellation(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(dnsdb.Core, 'dnsdb_key', lambda: 'dnsdb-test-key') + _install_response( + monkeypatch, + (b'{"cond":"begin"}\n',), + stream_error=asyncio.CancelledError(), + ) + + with pytest.raises(asyncio.CancelledError): + await dnsdb.SearchDNSDB('example.com').process() + + @pytest.mark.asyncio @pytest.mark.parametrize( - ('status', 'expected_error'), + ('status', 'expected_report'), [ - (401, PermissionError), - (429, ConnectionError), - (503, ConnectionError), + (401, SourceExecutionReport('failed', 'access-denied')), + (429, SourceExecutionReport('rate-limited', 'http-429')), + (503, SourceExecutionReport('failed', 'http-503')), ], ) -async def test_process_exposes_http_failures( +async def test_process_reports_http_failures( monkeypatch: pytest.MonkeyPatch, status: int, - expected_error: type[Exception], + expected_report: SourceExecutionReport, ) -> None: monkeypatch.setattr(dnsdb.Core, 'dnsdb_key', lambda: 'dnsdb-test-key') _install_response(monkeypatch, (), status=status) - with pytest.raises(expected_error): - await dnsdb.SearchDNSDB('example.com').process() + assert await dnsdb.SearchDNSDB('example.com').process() == expected_report pytestmark = pytest.mark.provider_contract('dnsdb') diff --git a/tests/discovery/test_duckduckgo.py b/tests/discovery/test_duckduckgo.py index d48588d3..14d7183f 100644 --- a/tests/discovery/test_duckduckgo.py +++ b/tests/discovery/test_duckduckgo.py @@ -1,13 +1,45 @@ +import asyncio +import json from typing import Any import pytest from theHarvester.discovery import duckduckgosearch +from theHarvester.lib.core import FetcherResponse, ResponseStreamError +from theHarvester.lib.source_execution import SourceExecutionReport + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ('response', 'expected_report'), + [ + (FetcherResponse(None, 401, {}), SourceExecutionReport('failed', 'access-denied')), + (FetcherResponse(None, 429, {}), SourceExecutionReport('rate-limited', 'http-429')), + (FetcherResponse(None, 503, {}), SourceExecutionReport('failed', 'http-503')), + ], +) +async def test_duckduckgo_reports_http_failures( + monkeypatch: pytest.MonkeyPatch, + response: FetcherResponse, + expected_report: SourceExecutionReport, +) -> None: + async def fake_fetch_json(*_args: Any, **_kwargs: Any) -> FetcherResponse: + return response + + async def legacy_fetch_all(*_args: Any, **_kwargs: Any) -> None: + raise AssertionError('DuckDuckGo must use the bounded fetch_json seam') + + monkeypatch.setattr(duckduckgosearch.AsyncFetcher, 'fetch_json', fake_fetch_json) + monkeypatch.setattr(duckduckgosearch.AsyncFetcher, 'fetch_all', legacy_fetch_all) + + search = duckduckgosearch.SearchDuckDuckGo('example.com', 100) + + assert await search.process() == expected_report @pytest.mark.asyncio async def test_duckduckgo_does_not_fetch_provider_returned_urls(monkeypatch: pytest.MonkeyPatch) -> None: - requests: list[tuple[list[str], bool]] = [] + requests: list[tuple[str, bool]] = [] payload = """ { "AbstractURL": "https://api.example.com", @@ -16,21 +48,21 @@ async def test_duckduckgo_does_not_fetch_provider_returned_urls(monkeypatch: pyt } """ - async def fake_fetch_all( - urls: list[str] | set[str], + async def fake_fetch_json( + url: str, *, headers: dict[str, str] | None = None, proxy: bool = False, **_kwargs: Any, - ) -> list[str]: - requests.append((list(urls), proxy)) - return [payload] + ) -> FetcherResponse: + requests.append((url, proxy)) + return FetcherResponse(json.loads(payload), 200, {}) - monkeypatch.setattr(duckduckgosearch.AsyncFetcher, 'fetch_all', fake_fetch_all) + monkeypatch.setattr(duckduckgosearch.AsyncFetcher, 'fetch_json', fake_fetch_json) search = duckduckgosearch.SearchDuckDuckGo('example.com', 100) await search.process(proxy=True) - assert requests == [(['https://api.duckduckgo.com/?q=example.com&format=json&pretty=1'], True)] + assert requests == [('https://api.duckduckgo.com/?q=example.com&format=json&pretty=1', True)] assert await search.get_hostnames() == ['api.example.com', 'example.com'] assert await search.get_emails() == {'admin@example.com'} @@ -41,24 +73,84 @@ async def test_duckduckgo_does_not_fetch_provider_returned_urls(monkeypatch: pyt '', '{"broken": ', '{"error": "Access denied", "url": "https://api.example.net"}', + '{"unexpected": "host.example.com"}', + '{"AbstractText": {"unexpected": "host.example.com"}}', + '{"Results": [{"FirstURL": {"unexpected": "host.example.com"}}]}', ], - ids=['empty', 'malformed', 'blocked'], + ids=['empty', 'malformed', 'blocked', 'unknown-schema', 'invalid-text-field', 'invalid-result-field'], ) @pytest.mark.asyncio async def test_duckduckgo_unusable_response_returns_no_evidence( monkeypatch: pytest.MonkeyPatch, payload: str, ) -> None: - async def fake_fetch_all(urls: list[str] | set[str], **_kwargs: Any) -> list[str]: - return [payload] + async def fake_fetch_json(*_args: Any, **_kwargs: Any) -> FetcherResponse: + if payload == '{"broken": ': + raise ResponseStreamError('invalid-response') + return FetcherResponse(json.loads(payload) if payload else None, 200, {}) - monkeypatch.setattr(duckduckgosearch.AsyncFetcher, 'fetch_all', fake_fetch_all) + monkeypatch.setattr(duckduckgosearch.AsyncFetcher, 'fetch_json', fake_fetch_json) search = duckduckgosearch.SearchDuckDuckGo('example.com', 100) - await search.process() + report = await search.process() assert await search.get_hostnames() == [] assert await search.get_emails() == set() + assert report == SourceExecutionReport( + 'failed', + 'access-denied' if 'Access denied' in payload else 'invalid-response', + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize('reason', ['transport-error', 'invalid-response', 'response-limit']) +async def test_duckduckgo_reports_bounded_fetch_failures( + monkeypatch: pytest.MonkeyPatch, + reason: str, +) -> None: + async def fake_fetch_json(*_args: Any, **_kwargs: Any) -> FetcherResponse: + raise ResponseStreamError(reason) # type: ignore[arg-type] + + monkeypatch.setattr(duckduckgosearch.AsyncFetcher, 'fetch_json', fake_fetch_json) + + report = await duckduckgosearch.SearchDuckDuckGo('example.com', 100).process() + + assert report == SourceExecutionReport('failed', reason) + + +@pytest.mark.asyncio +async def test_duckduckgo_propagates_cancellation(monkeypatch: pytest.MonkeyPatch) -> None: + async def fake_fetch_json(*_args: Any, **_kwargs: Any) -> FetcherResponse: + raise asyncio.CancelledError + + monkeypatch.setattr(duckduckgosearch.AsyncFetcher, 'fetch_json', fake_fetch_json) + + with pytest.raises(asyncio.CancelledError): + await duckduckgosearch.SearchDuckDuckGo('example.com', 100).process() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ('body', 'expected_report'), + [ + ({}, None), + ({'error': 'Rate limit exceeded'}, SourceExecutionReport('rate-limited', 'provider-rate-limit')), + ({'error': 'Unknown provider failure'}, SourceExecutionReport('failed', 'provider-error')), + ], +) +async def test_duckduckgo_classifies_valid_empty_and_provider_error_payloads( + monkeypatch: pytest.MonkeyPatch, + body: dict[str, str], + expected_report: SourceExecutionReport | None, +) -> None: + async def fake_fetch_json(*_args: Any, **_kwargs: Any) -> FetcherResponse: + return FetcherResponse(body, 200, {}) + + monkeypatch.setattr(duckduckgosearch.AsyncFetcher, 'fetch_json', fake_fetch_json) + + report = await duckduckgosearch.SearchDuckDuckGo('example.com', 100).process() + + assert report == expected_report pytestmark = pytest.mark.provider_contract('duckduckgo') diff --git a/tests/discovery/test_provider_session_lifecycle.py b/tests/discovery/test_provider_session_lifecycle.py index 00764d1a..9f5121fd 100644 --- a/tests/discovery/test_provider_session_lifecycle.py +++ b/tests/discovery/test_provider_session_lifecycle.py @@ -1,7 +1,88 @@ import pytest -from aiohttp import web +from aiohttp import ClientSession, web -from theHarvester.discovery import censysearch, githubcode +from theHarvester.discovery import bravesearch, censysearch, githubcode +from theHarvester.lib.configuration import InMemoryCredentialAdapter +from theHarvester.lib.source_execution import SourceExecutionReport + + +@pytest.mark.asyncio +async def test_brave_pagination_preserves_provider_cookies( + monkeypatch: pytest.MonkeyPatch, + unused_tcp_port: int, +) -> None: + requests: list[tuple[str | None, str | None]] = [] + sessions: list[ClientSession] = [] + original_build_session = bravesearch.AsyncFetcher._build_session + + async def tracked_build_session(*args: object, **kwargs: object) -> ClientSession: + session = await original_build_session(*args, **kwargs) # type: ignore[arg-type] + sessions.append(session) + return session + + monkeypatch.setattr(bravesearch.AsyncFetcher, '_build_session', tracked_build_session) + + async def search(request: web.Request) -> web.Response: + offset = request.query.get('offset') + requests.append((offset, request.cookies.get('provider-session'))) + if offset == '0': + response = web.json_response( + { + 'query': {'more_results_available': True}, + 'web': { + 'results': [ + { + 'title': 'First', + 'description': 'one.example.com', + 'url': 'https://one.example.com', + } + ] + }, + } + ) + response.set_cookie('provider-session', 'ready') + return response + if request.cookies.get('provider-session') != 'ready': + return web.json_response({'error': 'missing provider session'}, status=403) + return web.json_response( + { + 'query': {'more_results_available': False}, + 'web': { + 'results': [ + { + 'title': 'Second', + 'description': 'two.example.com', + 'url': 'https://two.example.com', + } + ] + }, + } + ) + + app = web.Application() + app.router.add_get('/search', search) + runner = web.AppRunner(app) + await runner.setup() + site = web.TCPSite(runner, '127.0.0.1', unused_tcp_port) + await site.start() + monkeypatch.setattr(bravesearch, 'get_delay', lambda: 0) + + try: + source = bravesearch.SearchBrave( + 'example.com', + limit=2, + credential_adapter=InMemoryCredentialAdapter({'brave': {'key': 'test-token'}}), + ) + source.server = f'http://localhost:{unused_tcp_port}/search' + report = await source.process() + finally: + await runner.cleanup() + + assert requests == [('0', None), ('1', 'ready')] + assert await source.get_hostnames() == ['one.example.com', 'two.example.com'] + assert report == SourceExecutionReport('completed', 'result-limit') + assert len(sessions) == 1 + assert sessions[0].closed is True @pytest.mark.asyncio diff --git a/tests/lib/test_core.py b/tests/lib/test_core.py index a7a0e6ba..7e6159f0 100644 --- a/tests/lib/test_core.py +++ b/tests/lib/test_core.py @@ -961,6 +961,27 @@ async def test_fetch_text_reuses_caller_owned_session_without_closing_it( assert len(DummySession.instances) == 1 +@pytest.mark.asyncio +async def test_fetch_json_reuses_caller_owned_session_without_closing_it( + monkeypatch: pytest.MonkeyPatch, +) -> None: + install_stream_response(monkeypatch, chunks=(b'{"provider":"evidence"}',)) + session = DummySession( + headers={'User-Agent': 'shared'}, + timeout=core_module.aiohttp.ClientTimeout(total=None), + ) + + result = await AsyncFetcher.fetch_json( + 'https://provider.example/data', + session=session, + request_timeout=None, + ) + + assert result.body == {'provider': 'evidence'} + assert session.closed is False + assert len(DummySession.instances) == 1 + + @pytest.mark.asyncio async def test_fetch_json_accepts_body_at_shared_limit(monkeypatch) -> None: install_stream_response(monkeypatch, chunks=(b'{"a":1}',)) diff --git a/tests/test_bravesearch.py b/tests/test_bravesearch.py index 35a8495c..6f775657 100644 --- a/tests/test_bravesearch.py +++ b/tests/test_bravesearch.py @@ -7,28 +7,33 @@ import pytest from theHarvester.discovery.bravesearch import SearchBrave from theHarvester.discovery.constants import MissingKey from theHarvester.lib.configuration import InMemoryCredentialAdapter -from theHarvester.lib.core import AsyncFetcher +from theHarvester.lib.core import AsyncFetcher, FetcherResponse @pytest.mark.asyncio async def test_brave_collects_with_in_memory_credentials(monkeypatch: pytest.MonkeyPatch) -> None: request_headers: list[dict[str, str]] = [] - async def fetch(*, headers: dict[str, str], **_kwargs: Any) -> dict[str, Any]: + async def fetch(*_args: Any, headers: dict[str, str], **_kwargs: Any) -> FetcherResponse: request_headers.append(headers) - return { - 'web': { - 'results': [ - { - 'title': 'Documentation', - 'description': 'Example documentation', - 'url': 'https://docs.example.com', - } - ] - } - } + return FetcherResponse( + { + 'query': {'more_results_available': False}, + 'web': { + 'results': [ + { + 'title': 'Documentation', + 'description': 'Example documentation', + 'url': 'https://docs.example.com', + } + ] + }, + }, + 200, + {}, + ) - monkeypatch.setattr(AsyncFetcher, 'fetch', fetch) + monkeypatch.setattr(AsyncFetcher, 'fetch_json', fetch) search = SearchBrave( 'example.com', 1, diff --git a/theHarvester/discovery/bravesearch.py b/theHarvester/discovery/bravesearch.py index e689c011..0ff7c1b1 100644 --- a/theHarvester/discovery/bravesearch.py +++ b/theHarvester/discovery/bravesearch.py @@ -1,14 +1,16 @@ import asyncio -import logging +import ssl from typing import Any from urllib.parse import quote -from theHarvester.discovery.constants import MissingKey, get_delay -from theHarvester.lib.configuration import CredentialAdapter, FileSystemCredentialAdapter -from theHarvester.lib.core import AsyncFetcher -from theHarvester.parsers import myparser +import aiohttp -logger = logging.getLogger(__name__) +from theHarvester.discovery.constants import MissingKey, get_delay +from theHarvester.discovery.provider_response import provider_http_error +from theHarvester.lib.configuration import CredentialAdapter, FileSystemCredentialAdapter +from theHarvester.lib.core import AsyncFetcher, ResponseStreamError +from theHarvester.lib.source_execution import SourceExecutionReport +from theHarvester.parsers import myparser class SearchBrave: @@ -34,11 +36,20 @@ class SearchBrave: raise MissingKey('Brave Search') self.server = 'https://api.search.brave.com/res/v1/web/search' self.limit = limit - self.proxy = False - self.rate_limit_delay = 1 # Initial delay for rate limiting + self.proxy: bool | str = False - async def do_search(self): + async def do_search(self, session: Any | None = None) -> SourceExecutionReport | None: headers = {'Accept': 'application/json', 'Accept-Encoding': 'gzip', 'X-Subscription-Token': self.api_key} + if session is None: + try: + async with AsyncFetcher.open_session( + headers=headers, + proxy=self.proxy, + request_timeout=60, + ) as owned_session: + return await self.do_search(owned_session) + except ResponseStreamError as error: + return SourceExecutionReport('failed', error.reason) # Search queries: exact match and site-specific queries = [f'"{self.word}"', f'site:{self.word}'] @@ -66,77 +77,80 @@ class SearchBrave: param_string = '&'.join([f'{k}={quote(str(v))}' for k, v in params.items()]) url = f'{self.server}?{param_string}' - resp = await AsyncFetcher.fetch(url=url, headers=headers, proxy=self.proxy, json=True) + response = await AsyncFetcher.fetch_json( + url, + session=session, + headers=headers, + ) + if failure := provider_http_error(response): + return SourceExecutionReport(*failure) + resp = response.body # Handle API response if resp is None: - logger.info('No response received from Brave Search API') - break + return SourceExecutionReport('failed', 'invalid-response') + if not isinstance(resp, dict): + return SourceExecutionReport('failed', 'invalid-response') # Check for API errors (rate limit, quota exceeded, etc.) if 'error' in resp: - error_msg = resp.get('error', {}).get('message', 'Unknown API error') - error_code = resp.get('error', {}).get('code', 'unknown') + provider_error = resp['error'] + if not isinstance(provider_error, dict): + return SourceExecutionReport('failed', 'invalid-response') + error_message = str(provider_error.get('message', '')).lower() + error_code = str(provider_error.get('code', '')).lower() + if 'rate limit' in error_message or error_code == 'rate_limit_exceeded': + return SourceExecutionReport('rate-limited', 'provider-rate-limit') + if 'quota' in error_message or error_code == 'quota_exceeded': + return SourceExecutionReport('failed', 'quota-exhausted') + return SourceExecutionReport('failed', 'provider-error') - if 'rate limit' in error_msg.lower() or error_code == 'rate_limit_exceeded': - logger.info(f'Rate limit exceeded. Increasing delay to {self.rate_limit_delay * 2} seconds') - self.rate_limit_delay *= 2 - await asyncio.sleep(self.rate_limit_delay) - break - elif 'quota' in error_msg.lower() or error_code == 'quota_exceeded': - logger.info('Brave Search API quota exceeded') - break - else: - break + web = resp.get('web') + query_data = resp.get('query') + if not isinstance(web, dict) or not isinstance(web.get('results'), list): + return SourceExecutionReport('failed', 'invalid-response') + if not isinstance(query_data, dict): + return SourceExecutionReport('failed', 'invalid-response') + more_results_available = query_data.get('more_results_available') + if not isinstance(more_results_available, bool): + return SourceExecutionReport('failed', 'invalid-response') - if 'web' in resp and 'results' in resp['web']: - results = resp['web']['results'][:remaining] - if not results: - break + results = web['results'][:remaining] + if any(not isinstance(result, dict) for result in results): + return SourceExecutionReport('failed', 'invalid-response') + if not results: + if more_results_available: + return SourceExecutionReport('failed', 'invalid-response') + break - # Extract text content from results for parsing (including extra snippets) - for result in results: - result_text = f'{result.get("title", "")} {result.get("description", "")}' + for result in results: + snippets = result.get('extra_snippets', []) + if not isinstance(snippets, list) or any(not isinstance(snippet, str) for snippet in snippets): + return SourceExecutionReport('failed', 'invalid-response') + title = result.get('title', '') + description = result.get('description', '') + result_url = result.get('url', '') + if not all(isinstance(value, str) for value in (title, description, result_url)): + return SourceExecutionReport('failed', 'invalid-response') + result_text = f'{title} {description}' + for snippet in snippets: + result_text += f' {snippet}' + result_text += f' {result_url}' + self.totalresults += result_text + '\n' - # Add extra snippets if available - if 'extra_snippets' in result: - for snippet in result['extra_snippets']: - result_text += f' {snippet}' - - result_text += f' {result.get("url", "")}' - self.totalresults += result_text + '\n' - - self.results.extend(results) - - # Stop if we've reached our limit - if len(self.results) >= self.limit: - break - if not resp.get('query', {}).get('more_results_available', False): - break - else: - logger.info('Unexpected response format from Brave Search API') + self.results.extend(results) + if len(self.results) >= self.limit: + return SourceExecutionReport('completed', 'result-limit') + if not more_results_available: break await asyncio.sleep(get_delay()) - - except Exception as e: - error_msg = str(e).lower() - - # Handle specific API-related exceptions - if 'rate limit' in error_msg or '429' in error_msg: - logger.info(f'Rate limit detected in exception. Increasing delay to {self.rate_limit_delay * 2} seconds') - self.rate_limit_delay *= 2 - await asyncio.sleep(self.rate_limit_delay) - elif 'quota' in error_msg or '403' in error_msg: - logger.info(f'Quota exceeded or access denied: {e}') - break - elif 'timeout' in error_msg: - logger.info(f'Request timeout occurred: {e}') - await asyncio.sleep(get_delay() + 2) else: - logger.info(f'An exception has occurred in bravesearch: {e}') - await asyncio.sleep(get_delay() + 5) - continue + return SourceExecutionReport('partial', 'pagination-limit') + + except ResponseStreamError as error: + return SourceExecutionReport('failed', error.reason) + return None async def get_emails(self): rawres = myparser.Parser(self.totalresults, self.word) @@ -146,6 +160,11 @@ class SearchBrave: rawres = myparser.Parser(self.totalresults, self.word) return await rawres.hostnames() - async def process(self, proxy=False): + async def process(self, proxy: bool | str = False) -> SourceExecutionReport | None: self.proxy = proxy - await self.do_search() + try: + return await self.do_search() + except asyncio.CancelledError: + raise + except aiohttp.ClientError, TimeoutError, OSError, ssl.SSLError: + return SourceExecutionReport('failed', 'transport-error') diff --git a/theHarvester/discovery/dnsdb.py b/theHarvester/discovery/dnsdb.py index 52794cc9..00ba10d6 100644 --- a/theHarvester/discovery/dnsdb.py +++ b/theHarvester/discovery/dnsdb.py @@ -5,7 +5,9 @@ import logging from urllib.parse import quote from theHarvester.discovery.constants import MissingKey -from theHarvester.lib.core import AsyncFetcher, Core, ResponseStreamError +from theHarvester.discovery.provider_response import provider_http_error +from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse, ResponseStreamError +from theHarvester.lib.source_execution import SourceExecutionReport logger = logging.getLogger(__name__) @@ -42,7 +44,7 @@ class SearchDNSDB: return hostname return None - async def do_search(self) -> None: + async def do_search(self) -> SourceExecutionReport | None: query = quote(f'*.{self.target_domain}', safe='*.') url = f'{self.BASE_URL}/{query}?limit=0' headers = { @@ -59,49 +61,56 @@ class SearchDNSDB: follow_redirects=False, request_timeout=120, ) as response: - if response.status == 429: - raise ConnectionError('DNSDB rate limit reached') - if response.status in {401, 403}: - raise PermissionError('DNSDB authentication failed') - if response.status == 503: - raise ConnectionError('DNSDB concurrent connection limit exceeded') - if response.status != 200: - raise ConnectionError(f'DNSDB returned HTTP {response.status}') + if failure := provider_http_error(FetcherResponse(None, response.status, response.headers)): + return SourceExecutionReport(*failure) first_record = True async for line in response: try: record = json.loads(line) - except json.JSONDecodeError: + except json.JSONDecodeError, RecursionError: logger.info('DNSDB returned malformed NDJSON; partial results were preserved.') - return + return SourceExecutionReport('failed', 'invalid-response') if not isinstance(record, dict): logger.info('DNSDB returned an invalid stream record; partial results were preserved.') - return + return SourceExecutionReport('failed', 'invalid-response') if first_record: first_record = False if record.get('cond') != 'begin': logger.info('DNSDB stream did not begin correctly; no results were accepted.') - return + return SourceExecutionReport('failed', 'invalid-response') continue condition = record.get('cond') if condition in {'succeeded', 'limited', 'failed'}: if condition != 'succeeded': logger.info(f'DNSDB stream ended with {condition}; partial results were preserved.') - return + if condition == 'limited': + return SourceExecutionReport('rate-limited', 'provider-limited') + if condition == 'failed': + return SourceExecutionReport('failed', 'provider-failed') + return None + if condition is not None: + return SourceExecutionReport('failed', 'invalid-response') obj = record.get('obj') - if isinstance(obj, dict) and (hostname := self._hostname(obj.get('rrname'))): + if not isinstance(obj, dict): + return SourceExecutionReport('failed', 'invalid-response') + rrname = obj.get('rrname') + if not isinstance(rrname, str) or not rrname.strip(): + return SourceExecutionReport('failed', 'invalid-response') + if hostname := self._hostname(rrname): self.totalhosts.add(hostname) logger.info('DNSDB stream ended without a terminal condition; partial results were preserved.') + return SourceExecutionReport('failed', 'invalid-response') async def get_hostnames(self) -> set[str]: return self.totalhosts - async def process(self, proxy: bool | str = False) -> None: + async def process(self, proxy: bool | str = False) -> SourceExecutionReport | None: self.proxy = proxy try: - await self.do_search() + return await self.do_search() except ResponseStreamError as error: logger.info(f'DNSDB request failed with {type(error).__name__}; partial results were preserved.') + return SourceExecutionReport('failed', error.reason) diff --git a/theHarvester/discovery/duckduckgosearch.py b/theHarvester/discovery/duckduckgosearch.py index a7af0b44..fb53a477 100644 --- a/theHarvester/discovery/duckduckgosearch.py +++ b/theHarvester/discovery/duckduckgosearch.py @@ -1,6 +1,81 @@ -from theHarvester.lib.core import AsyncFetcher, Core +from typing import Any + +from theHarvester.discovery.provider_response import provider_http_error +from theHarvester.lib.core import AsyncFetcher, Core, ResponseStreamError +from theHarvester.lib.source_execution import SourceExecutionReport from theHarvester.parsers import myparser +_TEXT_FIELDS = ( + 'Abstract', + 'AbstractSource', + 'AbstractText', + 'AbstractURL', + 'Answer', + 'AnswerType', + 'Definition', + 'DefinitionSource', + 'DefinitionURL', + 'Entity', + 'Heading', + 'Image', + 'Redirect', + 'Type', +) +_TOPIC_TEXT_FIELDS = ('FirstURL', 'Name', 'Result', 'Text') + + +def _topic_text(items: object) -> list[str] | None: + if not isinstance(items, list): + return None + text: list[str] = [] + for item in items: + if not isinstance(item, dict): + return None + recognized = False + for field in _TOPIC_TEXT_FIELDS: + if field not in item: + continue + recognized = True + value = item[field] + if not isinstance(value, str): + return None + text.append(value) + if 'Topics' in item: + recognized = True + nested = _topic_text(item['Topics']) + if nested is None: + return None + text.extend(nested) + if item and not recognized: + return None + return text + + +def _provider_text(body: dict[str, Any]) -> str | None: + if not body: + return '' + recognized = False + text: list[str] = [] + for field in _TEXT_FIELDS: + if field not in body: + continue + recognized = True + value = body[field] + if not isinstance(value, str): + return None + text.append(value) + for field in ('RelatedTopics', 'Results'): + if field not in body: + continue + recognized = True + topic_text = _topic_text(body[field]) + if topic_text is None: + return None + text.extend(topic_text) + if not recognized: + return None + return '\n'.join(text) + class SearchDuckDuckGo: def __init__(self, word, limit) -> None: @@ -15,13 +90,31 @@ class SearchDuckDuckGo: self.limit = limit self.proxy: bool = False - async def do_search(self) -> None: + async def do_search(self) -> SourceExecutionReport | None: # Query only the provider; URLs in the response are evidence, not crawl targets. url = self.api.replace('x', self.word) headers = {'User-Agent': Core.get_user_agent()} - first_resp = await AsyncFetcher.fetch_all([url], headers=headers, proxy=self.proxy) - self.results = first_resp[0] + try: + response = await AsyncFetcher.fetch_json(url, headers=headers, proxy=self.proxy) + except ResponseStreamError as error: + return SourceExecutionReport('failed', error.reason) + if failure := provider_http_error(response): + return SourceExecutionReport(*failure) + if not isinstance(response.body, dict): + return SourceExecutionReport('failed', 'invalid-response') + if 'error' in response.body: + provider_error = str(response.body['error']).lower() + if 'rate limit' in provider_error: + return SourceExecutionReport('rate-limited', 'provider-rate-limit') + if 'access denied' in provider_error or 'forbidden' in provider_error: + return SourceExecutionReport('failed', 'access-denied') + return SourceExecutionReport('failed', 'provider-error') + provider_text = _provider_text(response.body) + if provider_text is None: + return SourceExecutionReport('failed', 'invalid-response') + self.results = provider_text self.totalresults += self.results + return None async def get_emails(self): rawres = myparser.Parser(self.totalresults, self.word) @@ -31,6 +124,6 @@ class SearchDuckDuckGo: rawres = myparser.Parser(self.totalresults, self.word) return await rawres.hostnames() - async def process(self, proxy: bool = False) -> None: + async def process(self, proxy: bool = False) -> SourceExecutionReport | None: self.proxy = proxy - await self.do_search() # Only need to search once since using API. + return await self.do_search() # Only need to search once since using API. diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index f3402651..ee548569 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -827,6 +827,7 @@ class AsyncFetcher: cls, url: str, *, + session: aiohttp.ClientSession | None = None, params: Sized = '', proxy: str | bool | None = '', headers: dict[str, str] | None = None, @@ -835,6 +836,7 @@ class AsyncFetcher: """Fetch one bounded JSON response without following redirects.""" async with cls._open_get_response( url, + session=session, params=params, proxy=proxy, headers=headers,