fix: report provider collection outcomes truthfully

tests/discovery/test_brave.py

@@ -1,3 +1,5 @@
+import contextlib
+from collections.abc import AsyncIterator
 from typing import Any
 from urllib.parse import parse_qs, urlparse
 
@@ -6,6 +8,8 @@ import pytest
 from theHarvester.discovery import bravesearch
 from theHarvester.discovery.constants import MissingKey
 from theHarvester.lib.configuration import InMemoryCredentialAdapter
+from theHarvester.lib.core import FetcherResponse, ResponseStreamError
+from theHarvester.lib.source_execution import SourceExecutionReport
 
 
 def _result(index: int) -> dict[str, str]:
@@ -16,11 +20,15 @@ def _result(index: int) -> dict[str, str]:
     }
 
 
-def _response(results: list[dict[str, str]], *, more: bool) -> dict[str, Any]:
-    return {
+def _response(results: list[dict[str, str]], *, more: bool) -> FetcherResponse:
+    return FetcherResponse(
+        {
             'query': {'more_results_available': more},
             'web': {'results': results},
-    }
+        },
+        200,
+        {},
+    )
 
 
 @pytest.fixture
@@ -58,43 +66,210 @@ async def test_brave_normalizes_in_scope_evidence(
                 ],
                 more=False,
             ),
-            {'error': {'message': 'Access denied', 'code': 'forbidden'}},
+            FetcherResponse({'error': {'message': 'Access denied', 'code': 'forbidden'}}, 200, {}),
         ]
     )
 
-    proxies: list[bool] = []
+    request_proxies: list[bool | None] = []
+    session_options: list[dict[str, Any]] = []
 
-    async def fake_fetch(*, url: str, **kwargs: Any) -> dict[str, Any]:
-        proxies.append(kwargs['proxy'])
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**kwargs: Any) -> AsyncIterator[object]:
+        session_options.append(kwargs)
+        yield object()
+
+    async def fake_fetch(*_args: Any, **kwargs: Any) -> FetcherResponse:
+        request_proxies.append(kwargs.get('proxy'))
         return next(responses)
 
-    monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch', fake_fetch)
+    monkeypatch.setattr(bravesearch.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch)
     search = bravesearch.SearchBrave('example.com', 10, credential_adapter=brave_credentials)
 
     await search.process(proxy=True)
 
-    assert proxies == [True, True]
+    assert session_options == [{'headers': search_headers(search), 'proxy': True, 'request_timeout': 60}]
+    assert request_proxies == [None, None]
     assert await search.get_emails() == {'admin@example.com'}
     assert await search.get_hostnames() == ['blog.example.com', 'example.com']
 
 
-@pytest.mark.parametrize('response', [None, []], ids=['empty', 'malformed'])
+def search_headers(search: bravesearch.SearchBrave) -> dict[str, str]:
+    return {
+        'Accept': 'application/json',
+        'Accept-Encoding': 'gzip',
+        'X-Subscription-Token': search.api_key,
+    }
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize('failure_point', ['open', 'close'])
+async def test_brave_reports_session_lifecycle_failures(
+    monkeypatch: pytest.MonkeyPatch,
+    brave_credentials: InMemoryCredentialAdapter,
+    failure_point: str,
+) -> None:
+    @contextlib.asynccontextmanager
+    async def failed_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        if failure_point == 'open':
+            raise OSError('session creation failed')
+        yield object()
+        raise OSError('session close failed')
+
+    async def fake_fetch_json(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        return _response([], more=False)
+
+    monkeypatch.setattr(bravesearch.AsyncFetcher, 'open_session', failed_open_session)
+    monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch_json)
+    search = bravesearch.SearchBrave('example.com', 10, credential_adapter=brave_credentials)
+
+    assert await search.process() == SourceExecutionReport('failed', 'transport-error')
+
+
+@pytest.mark.asyncio
+async def test_brave_does_not_misclassify_value_errors_as_transport_failures(
+    monkeypatch: pytest.MonkeyPatch,
+    brave_credentials: InMemoryCredentialAdapter,
+) -> None:
+    @contextlib.asynccontextmanager
+    async def failed_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        raise ValueError('programming defect')
+        yield object()
+
+    monkeypatch.setattr(bravesearch.AsyncFetcher, 'open_session', failed_open_session)
+    search = bravesearch.SearchBrave('example.com', 10, credential_adapter=brave_credentials)
+
+    with pytest.raises(ValueError, match='programming defect'):
+        await search.process()
+
+
+@pytest.mark.parametrize('body', [None, []], ids=['empty', 'malformed'])
 @pytest.mark.asyncio
 async def test_brave_unusable_response_returns_no_evidence(
     monkeypatch: pytest.MonkeyPatch,
     brave_credentials: InMemoryCredentialAdapter,
-    response: list[Any] | None,
+    body: list[Any] | None,
 ) -> None:
-    async def fake_fetch(*, url: str, **_kwargs: Any) -> list[Any] | None:
-        return response
+    async def fake_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        return FetcherResponse(body, 200, {})
 
-    monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch', fake_fetch)
+    monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch)
     search = bravesearch.SearchBrave('example.com', 10, credential_adapter=brave_credentials)
 
-    await search.process()
+    report = await search.process()
 
     assert await search.get_emails() == set()
     assert await search.get_hostnames() == []
+    assert report == SourceExecutionReport('failed', 'invalid-response')
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ('response', 'expected_report'),
+    [
+        (FetcherResponse(None, 401, {}), SourceExecutionReport('failed', 'access-denied')),
+        (FetcherResponse(None, 429, {}), SourceExecutionReport('rate-limited', 'http-429')),
+        (FetcherResponse(None, 503, {}), SourceExecutionReport('failed', 'http-503')),
+    ],
+)
+async def test_brave_reports_http_failures(
+    monkeypatch: pytest.MonkeyPatch,
+    brave_credentials: InMemoryCredentialAdapter,
+    response: FetcherResponse,
+    expected_report: SourceExecutionReport,
+) -> None:
+    async def fake_fetch_json(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        return response
+
+    async def legacy_fetch(*_args: Any, **_kwargs: Any) -> None:
+        raise AssertionError('Brave must use the bounded fetch_json seam')
+
+    monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch_json)
+    monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch', legacy_fetch)
+
+    search = bravesearch.SearchBrave('example.com', 10, credential_adapter=brave_credentials)
+
+    assert await search.process() == expected_report
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize('reason', ['transport-error', 'invalid-response', 'response-limit'])
+async def test_brave_reports_bounded_fetch_failures(
+    monkeypatch: pytest.MonkeyPatch,
+    brave_credentials: InMemoryCredentialAdapter,
+    reason: str,
+) -> None:
+    async def fake_fetch_json(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        raise ResponseStreamError(reason)  # type: ignore[arg-type]
+
+    monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch_json)
+    search = bravesearch.SearchBrave('example.com', 10, credential_adapter=brave_credentials)
+
+    assert await search.process() == SourceExecutionReport('failed', reason)
+
+
+@pytest.mark.asyncio
+async def test_brave_propagates_cancellation(
+    monkeypatch: pytest.MonkeyPatch,
+    brave_credentials: InMemoryCredentialAdapter,
+) -> None:
+    async def fake_fetch_json(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        raise bravesearch.asyncio.CancelledError
+
+    monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch_json)
+    search = bravesearch.SearchBrave('example.com', 10, credential_adapter=brave_credentials)
+
+    with pytest.raises(bravesearch.asyncio.CancelledError):
+        await search.process()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    'body',
+    [
+        {},
+        {'web': []},
+        {'web': {'results': 'invalid'}, 'query': {'more_results_available': False}},
+        {'web': {'results': []}, 'query': {}},
+        {'web': {'results': ['invalid']}, 'query': {'more_results_available': False}},
+    ],
+)
+async def test_brave_reports_malformed_payloads(
+    monkeypatch: pytest.MonkeyPatch,
+    brave_credentials: InMemoryCredentialAdapter,
+    body: dict[str, Any],
+) -> None:
+    async def fake_fetch_json(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        return FetcherResponse(body, 200, {})
+
+    monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch_json)
+    search = bravesearch.SearchBrave('example.com', 10, credential_adapter=brave_credentials)
+
+    assert await search.process() == SourceExecutionReport('failed', 'invalid-response')
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize('field', ['title', 'description', 'url'])
+async def test_brave_rejects_non_string_evidence_fields(
+    monkeypatch: pytest.MonkeyPatch,
+    brave_credentials: InMemoryCredentialAdapter,
+    field: str,
+) -> None:
+    result: dict[str, object] = _result(1)
+    result[field] = {'unexpected': 'host.example.com'}
+
+    async def fake_fetch_json(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        return FetcherResponse(
+            {'web': {'results': [result]}, 'query': {'more_results_available': False}},
+            200,
+            {},
+        )
+
+    monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch_json)
+    search = bravesearch.SearchBrave('example.com', 10, credential_adapter=brave_credentials)
+
+    assert await search.process() == SourceExecutionReport('failed', 'invalid-response')
+    assert search.totalresults == ''
 
 
 @pytest.mark.asyncio
@@ -110,11 +285,12 @@ async def test_brave_uses_page_offsets_and_one_global_limit(
     )
     requests: list[dict[str, list[str]]] = []
 
-    async def fake_fetch(*, url: str, **_kwargs: Any) -> dict[str, Any]:
+    async def fake_fetch(*_args: Any, **kwargs: Any) -> FetcherResponse:
+        url = kwargs.get('url', _args[0] if _args else '')
         requests.append(parse_qs(urlparse(url).query))
         return next(responses)
 
-    monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch', fake_fetch)
+    monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch)
     search = bravesearch.SearchBrave('example.com', 25, credential_adapter=brave_credentials)
     await search.process()
 
@@ -156,11 +332,12 @@ async def test_brave_requests_another_page_only_when_available(
     )
     requests: list[dict[str, list[str]]] = []
 
-    async def fake_fetch(*, url: str, **_kwargs: Any) -> dict[str, Any]:
+    async def fake_fetch(*_args: Any, **kwargs: Any) -> FetcherResponse:
+        url = kwargs.get('url', _args[0] if _args else '')
         requests.append(parse_qs(urlparse(url).query))
         return next(responses, _response([], more=False))
 
-    monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch', fake_fetch)
+    monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch)
     search = bravesearch.SearchBrave('example.com', 10, credential_adapter=brave_credentials)
     await search.process()
 
@@ -183,11 +360,12 @@ async def test_brave_continues_sparse_pages_while_more_results_are_available(
     )
     requests: list[dict[str, list[str]]] = []
 
-    async def fake_fetch(*, url: str, **_kwargs: Any) -> dict[str, Any]:
+    async def fake_fetch(*_args: Any, **kwargs: Any) -> FetcherResponse:
+        url = kwargs.get('url', _args[0] if _args else '')
         requests.append(parse_qs(urlparse(url).query))
         return next(responses)
 
-    monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch', fake_fetch)
+    monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch)
     search = bravesearch.SearchBrave('example.com', 20, credential_adapter=brave_credentials)
     await search.process()
 
@@ -198,6 +376,23 @@ async def test_brave_continues_sparse_pages_while_more_results_are_available(
     assert len(search.results) == 20
 
 
+@pytest.mark.asyncio
+async def test_brave_rejects_empty_page_that_claims_more_results(
+    monkeypatch: pytest.MonkeyPatch,
+    brave_credentials: InMemoryCredentialAdapter,
+) -> None:
+    async def fake_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        return _response([], more=True)
+
+    monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch)
+    search = bravesearch.SearchBrave('example.com', 20, credential_adapter=brave_credentials)
+
+    report = await search.process()
+
+    assert report == SourceExecutionReport('failed', 'invalid-response')
+    assert search.results == []
+
+
 @pytest.mark.asyncio
 async def test_brave_stops_after_an_exact_full_page(
     monkeypatch: pytest.MonkeyPatch,
@@ -205,15 +400,17 @@ async def test_brave_stops_after_an_exact_full_page(
 ) -> None:
     requests: list[dict[str, list[str]]] = []
 
-    async def fake_fetch(*, url: str, **_kwargs: Any) -> dict[str, Any]:
+    async def fake_fetch(*_args: Any, **kwargs: Any) -> FetcherResponse:
+        url = kwargs.get('url', _args[0] if _args else '')
         requests.append(parse_qs(urlparse(url).query))
         return _response([_result(index) for index in range(20)], more=True)
 
-    monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch', fake_fetch)
+    monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch)
     search = bravesearch.SearchBrave('example.com', 20, credential_adapter=brave_credentials)
-    await search.process()
+    report = await search.process()
 
     assert [(request['offset'], request['count']) for request in requests] == [(['0'], ['20'])]
+    assert report == SourceExecutionReport('completed', 'result-limit')
 
 
 @pytest.mark.asyncio
@@ -223,42 +420,43 @@ async def test_brave_rate_limit_does_not_skip_to_the_next_page(
 ) -> None:
     responses = iter(
         [
-            {'error': {'message': 'Rate limit exceeded', 'code': 'rate_limit_exceeded'}},
+            FetcherResponse({'error': {'message': 'Rate limit exceeded', 'code': 'rate_limit_exceeded'}}, 200, {}),
             _response([], more=False),
         ]
     )
     requests: list[dict[str, list[str]]] = []
 
-    async def fake_fetch(*, url: str, **_kwargs: Any) -> dict[str, Any]:
+    async def fake_fetch(*_args: Any, **kwargs: Any) -> FetcherResponse:
+        url = kwargs.get('url', _args[0] if _args else '')
         requests.append(parse_qs(urlparse(url).query))
         return next(responses)
 
-    monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch', fake_fetch)
+    monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch)
     search = bravesearch.SearchBrave('example.com', 40, credential_adapter=brave_credentials)
-    await search.process()
+    report = await search.process()
 
-    assert [(request['q'], request['offset']) for request in requests] == [
-        (['"example.com"'], ['0']),
-        (['site:example.com'], ['0']),
-    ]
+    assert [(request['q'], request['offset']) for request in requests] == [(['"example.com"'], ['0'])]
+    assert report == SourceExecutionReport('rate-limited', 'provider-rate-limit')
 
 
 @pytest.mark.asyncio
-async def test_brave_never_exceeds_maximum_page_offset(
+async def test_brave_reports_maximum_page_offset_as_truncation(
     monkeypatch: pytest.MonkeyPatch,
     brave_credentials: InMemoryCredentialAdapter,
 ) -> None:
     requests: list[dict[str, list[str]]] = []
 
-    async def fake_fetch(*, url: str, **_kwargs: Any) -> dict[str, Any]:
+    async def fake_fetch(*_args: Any, **kwargs: Any) -> FetcherResponse:
+        url = kwargs.get('url', _args[0] if _args else '')
         requests.append(parse_qs(urlparse(url).query))
         return _response([_result(len(requests))], more=True)
 
-    monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch', fake_fetch)
+    monkeypatch.setattr(bravesearch.AsyncFetcher, 'fetch_json', fake_fetch)
     search = bravesearch.SearchBrave('example.com', 1_000, credential_adapter=brave_credentials)
-    await search.process()
+    report = await search.process()
 
-    assert [request['offset'] for request in requests] == [[str(offset)] for offset in range(10)] * 2
+    assert [request['offset'] for request in requests] == [[str(offset)] for offset in range(10)]
+    assert report == SourceExecutionReport('partial', 'pagination-limit')
 
 
 pytestmark = pytest.mark.provider_contract('brave')

tests/discovery/test_dnsdb.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import asyncio
 import contextlib
 import logging
 
@@ -7,6 +8,7 @@ import pytest
 
 from theHarvester.discovery import dnsdb
 from theHarvester.discovery.constants import MissingKey
+from theHarvester.lib.source_execution import SourceExecutionReport
 
 
 def _install_response(
@@ -108,10 +110,14 @@ async def test_process_uses_configured_proxy_when_enabled(monkeypatch: pytest.Mo
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
-    ('last_line', 'expected_message'),
+    ('last_line', 'expected_message', 'expected_report'),
     [
-        (b'{"cond":"limited"}\n', 'ended with limited'),
-        (b'not-json\n', 'malformed NDJSON'),
+        (
+            b'{"cond":"limited"}\n',
+            'ended with limited',
+            SourceExecutionReport('rate-limited', 'provider-limited'),
+        ),
+        (b'not-json\n', 'malformed NDJSON', SourceExecutionReport('failed', 'invalid-response')),
     ],
 )
 async def test_process_preserves_partial_results(
@@ -119,6 +125,7 @@ async def test_process_preserves_partial_results(
     caplog: pytest.LogCaptureFixture,
     last_line: bytes,
     expected_message: str,
+    expected_report: SourceExecutionReport,
 ) -> None:
     caplog.set_level(logging.INFO, logger=dnsdb.__name__)
     monkeypatch.setattr(dnsdb.Core, 'dnsdb_key', lambda: 'dnsdb-test-key')
@@ -132,12 +139,53 @@ async def test_process_preserves_partial_results(
     )
 
     search = dnsdb.SearchDNSDB('example.com')
-    await search.process()
+    report = await search.process()
 
     assert await search.get_hostnames() == {'first.example.com'}
+    assert report == expected_report
     assert any(expected_message in message for message in caplog.messages)
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ('lines', 'expected_report'),
+    [
+        ((b'[]\n',), SourceExecutionReport('failed', 'invalid-response')),
+        ((b'{"cond":"wrong"}\n',), SourceExecutionReport('failed', 'invalid-response')),
+        (
+            (b'{"cond":"begin"}\n', b'{"cond":"failed"}\n'),
+            SourceExecutionReport('failed', 'provider-failed'),
+        ),
+        ((b'{"cond":"begin"}\n',), SourceExecutionReport('failed', 'invalid-response')),
+        (
+            (b'{"cond":"begin"}\n', b'{"cond":"wrong"}\n', b'{"cond":"succeeded"}\n'),
+            SourceExecutionReport('failed', 'invalid-response'),
+        ),
+        (
+            (b'{"cond":"begin"}\n', b'{"obj":[]}\n', b'{"cond":"succeeded"}\n'),
+            SourceExecutionReport('failed', 'invalid-response'),
+        ),
+        (
+            (b'{"cond":"begin"}\n', b'{"obj":{}}\n', b'{"cond":"succeeded"}\n'),
+            SourceExecutionReport('failed', 'invalid-response'),
+        ),
+        (
+            (b'{"cond":"begin"}\n', b'{"obj":{"rrname":7}}\n', b'{"cond":"succeeded"}\n'),
+            SourceExecutionReport('failed', 'invalid-response'),
+        ),
+    ],
+)
+async def test_process_reports_abnormal_stream_termination(
+    monkeypatch: pytest.MonkeyPatch,
+    lines: tuple[bytes, ...],
+    expected_report: SourceExecutionReport,
+) -> None:
+    monkeypatch.setattr(dnsdb.Core, 'dnsdb_key', lambda: 'dnsdb-test-key')
+    _install_response(monkeypatch, lines)
+
+    assert await dnsdb.SearchDNSDB('example.com').process() == expected_report
+
+
 @pytest.mark.asyncio
 async def test_process_preserves_partial_results_on_midstream_timeout(
     monkeypatch: pytest.MonkeyPatch,
@@ -155,31 +203,60 @@ async def test_process_preserves_partial_results_on_midstream_timeout(
     )
 
     search = dnsdb.SearchDNSDB('example.com')
-    await search.process()
+    report = await search.process()
 
     assert await search.get_hostnames() == {'first.example.com'}
+    assert report == SourceExecutionReport('failed', 'transport-error')
     assert any('request failed' in message for message in caplog.messages)
 
 
+@pytest.mark.asyncio
+async def test_process_reports_deeply_nested_json_as_invalid_response(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(dnsdb.Core, 'dnsdb_key', lambda: 'dnsdb-test-key')
+    _install_response(monkeypatch, (b'{"cond":"begin"}\n', b'deeply-nested-json\n'))
+    json_loads = dnsdb.json.loads
+
+    def parse_record(line: str) -> object:
+        if line.strip() == 'deeply-nested-json':
+            raise RecursionError
+        return json_loads(line)
+
+    monkeypatch.setattr(dnsdb.json, 'loads', parse_record)
+
+    assert await dnsdb.SearchDNSDB('example.com').process() == SourceExecutionReport('failed', 'invalid-response')
+
+
+@pytest.mark.asyncio
+async def test_process_propagates_cancellation(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(dnsdb.Core, 'dnsdb_key', lambda: 'dnsdb-test-key')
+    _install_response(
+        monkeypatch,
+        (b'{"cond":"begin"}\n',),
+        stream_error=asyncio.CancelledError(),
+    )
+
+    with pytest.raises(asyncio.CancelledError):
+        await dnsdb.SearchDNSDB('example.com').process()
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
-    ('status', 'expected_error'),
+    ('status', 'expected_report'),
     [
-        (401, PermissionError),
-        (429, ConnectionError),
-        (503, ConnectionError),
+        (401, SourceExecutionReport('failed', 'access-denied')),
+        (429, SourceExecutionReport('rate-limited', 'http-429')),
+        (503, SourceExecutionReport('failed', 'http-503')),
     ],
 )
-async def test_process_exposes_http_failures(
+async def test_process_reports_http_failures(
     monkeypatch: pytest.MonkeyPatch,
     status: int,
-    expected_error: type[Exception],
+    expected_report: SourceExecutionReport,
 ) -> None:
     monkeypatch.setattr(dnsdb.Core, 'dnsdb_key', lambda: 'dnsdb-test-key')
     _install_response(monkeypatch, (), status=status)
 
-    with pytest.raises(expected_error):
-        await dnsdb.SearchDNSDB('example.com').process()
+    assert await dnsdb.SearchDNSDB('example.com').process() == expected_report
 
 
 pytestmark = pytest.mark.provider_contract('dnsdb')

tests/discovery/test_duckduckgo.py

@@ -1,13 +1,45 @@
+import asyncio
+import json
 from typing import Any
 
 import pytest
 
 from theHarvester.discovery import duckduckgosearch
+from theHarvester.lib.core import FetcherResponse, ResponseStreamError
+from theHarvester.lib.source_execution import SourceExecutionReport
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ('response', 'expected_report'),
+    [
+        (FetcherResponse(None, 401, {}), SourceExecutionReport('failed', 'access-denied')),
+        (FetcherResponse(None, 429, {}), SourceExecutionReport('rate-limited', 'http-429')),
+        (FetcherResponse(None, 503, {}), SourceExecutionReport('failed', 'http-503')),
+    ],
+)
+async def test_duckduckgo_reports_http_failures(
+    monkeypatch: pytest.MonkeyPatch,
+    response: FetcherResponse,
+    expected_report: SourceExecutionReport,
+) -> None:
+    async def fake_fetch_json(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        return response
+
+    async def legacy_fetch_all(*_args: Any, **_kwargs: Any) -> None:
+        raise AssertionError('DuckDuckGo must use the bounded fetch_json seam')
+
+    monkeypatch.setattr(duckduckgosearch.AsyncFetcher, 'fetch_json', fake_fetch_json)
+    monkeypatch.setattr(duckduckgosearch.AsyncFetcher, 'fetch_all', legacy_fetch_all)
+
+    search = duckduckgosearch.SearchDuckDuckGo('example.com', 100)
+
+    assert await search.process() == expected_report
 
 
 @pytest.mark.asyncio
 async def test_duckduckgo_does_not_fetch_provider_returned_urls(monkeypatch: pytest.MonkeyPatch) -> None:
-    requests: list[tuple[list[str], bool]] = []
+    requests: list[tuple[str, bool]] = []
     payload = """
     {
       "AbstractURL": "https://api.example.com",
@@ -16,21 +48,21 @@ async def test_duckduckgo_does_not_fetch_provider_returned_urls(monkeypatch: pyt
     }
     """
 
-    async def fake_fetch_all(
-        urls: list[str] | set[str],
+    async def fake_fetch_json(
+        url: str,
         *,
         headers: dict[str, str] | None = None,
         proxy: bool = False,
         **_kwargs: Any,
-    ) -> list[str]:
-        requests.append((list(urls), proxy))
-        return [payload]
+    ) -> FetcherResponse:
+        requests.append((url, proxy))
+        return FetcherResponse(json.loads(payload), 200, {})
 
-    monkeypatch.setattr(duckduckgosearch.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    monkeypatch.setattr(duckduckgosearch.AsyncFetcher, 'fetch_json', fake_fetch_json)
     search = duckduckgosearch.SearchDuckDuckGo('example.com', 100)
     await search.process(proxy=True)
 
-    assert requests == [(['https://api.duckduckgo.com/?q=example.com&format=json&pretty=1'], True)]
+    assert requests == [('https://api.duckduckgo.com/?q=example.com&format=json&pretty=1', True)]
     assert await search.get_hostnames() == ['api.example.com', 'example.com']
     assert await search.get_emails() == {'admin@example.com'}
 
@@ -41,24 +73,84 @@ async def test_duckduckgo_does_not_fetch_provider_returned_urls(monkeypatch: pyt
         '',
         '{"broken": ',
         '{"error": "Access denied", "url": "https://api.example.net"}',
+        '{"unexpected": "host.example.com"}',
+        '{"AbstractText": {"unexpected": "host.example.com"}}',
+        '{"Results": [{"FirstURL": {"unexpected": "host.example.com"}}]}',
     ],
-    ids=['empty', 'malformed', 'blocked'],
+    ids=['empty', 'malformed', 'blocked', 'unknown-schema', 'invalid-text-field', 'invalid-result-field'],
 )
 @pytest.mark.asyncio
 async def test_duckduckgo_unusable_response_returns_no_evidence(
     monkeypatch: pytest.MonkeyPatch,
     payload: str,
 ) -> None:
-    async def fake_fetch_all(urls: list[str] | set[str], **_kwargs: Any) -> list[str]:
-        return [payload]
+    async def fake_fetch_json(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        if payload == '{"broken": ':
+            raise ResponseStreamError('invalid-response')
+        return FetcherResponse(json.loads(payload) if payload else None, 200, {})
 
-    monkeypatch.setattr(duckduckgosearch.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    monkeypatch.setattr(duckduckgosearch.AsyncFetcher, 'fetch_json', fake_fetch_json)
     search = duckduckgosearch.SearchDuckDuckGo('example.com', 100)
 
-    await search.process()
+    report = await search.process()
 
     assert await search.get_hostnames() == []
     assert await search.get_emails() == set()
+    assert report == SourceExecutionReport(
+        'failed',
+        'access-denied' if 'Access denied' in payload else 'invalid-response',
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize('reason', ['transport-error', 'invalid-response', 'response-limit'])
+async def test_duckduckgo_reports_bounded_fetch_failures(
+    monkeypatch: pytest.MonkeyPatch,
+    reason: str,
+) -> None:
+    async def fake_fetch_json(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        raise ResponseStreamError(reason)  # type: ignore[arg-type]
+
+    monkeypatch.setattr(duckduckgosearch.AsyncFetcher, 'fetch_json', fake_fetch_json)
+
+    report = await duckduckgosearch.SearchDuckDuckGo('example.com', 100).process()
+
+    assert report == SourceExecutionReport('failed', reason)
+
+
+@pytest.mark.asyncio
+async def test_duckduckgo_propagates_cancellation(monkeypatch: pytest.MonkeyPatch) -> None:
+    async def fake_fetch_json(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        raise asyncio.CancelledError
+
+    monkeypatch.setattr(duckduckgosearch.AsyncFetcher, 'fetch_json', fake_fetch_json)
+
+    with pytest.raises(asyncio.CancelledError):
+        await duckduckgosearch.SearchDuckDuckGo('example.com', 100).process()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ('body', 'expected_report'),
+    [
+        ({}, None),
+        ({'error': 'Rate limit exceeded'}, SourceExecutionReport('rate-limited', 'provider-rate-limit')),
+        ({'error': 'Unknown provider failure'}, SourceExecutionReport('failed', 'provider-error')),
+    ],
+)
+async def test_duckduckgo_classifies_valid_empty_and_provider_error_payloads(
+    monkeypatch: pytest.MonkeyPatch,
+    body: dict[str, str],
+    expected_report: SourceExecutionReport | None,
+) -> None:
+    async def fake_fetch_json(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        return FetcherResponse(body, 200, {})
+
+    monkeypatch.setattr(duckduckgosearch.AsyncFetcher, 'fetch_json', fake_fetch_json)
+
+    report = await duckduckgosearch.SearchDuckDuckGo('example.com', 100).process()
+
+    assert report == expected_report
 
 
 pytestmark = pytest.mark.provider_contract('duckduckgo')

tests/discovery/test_provider_session_lifecycle.py

@@ -1,7 +1,88 @@
 import pytest
-from aiohttp import web
+from aiohttp import ClientSession, web
 
-from theHarvester.discovery import censysearch, githubcode
+from theHarvester.discovery import bravesearch, censysearch, githubcode
+from theHarvester.lib.configuration import InMemoryCredentialAdapter
+from theHarvester.lib.source_execution import SourceExecutionReport
+
+
+@pytest.mark.asyncio
+async def test_brave_pagination_preserves_provider_cookies(
+    monkeypatch: pytest.MonkeyPatch,
+    unused_tcp_port: int,
+) -> None:
+    requests: list[tuple[str | None, str | None]] = []
+    sessions: list[ClientSession] = []
+    original_build_session = bravesearch.AsyncFetcher._build_session
+
+    async def tracked_build_session(*args: object, **kwargs: object) -> ClientSession:
+        session = await original_build_session(*args, **kwargs)  # type: ignore[arg-type]
+        sessions.append(session)
+        return session
+
+    monkeypatch.setattr(bravesearch.AsyncFetcher, '_build_session', tracked_build_session)
+
+    async def search(request: web.Request) -> web.Response:
+        offset = request.query.get('offset')
+        requests.append((offset, request.cookies.get('provider-session')))
+        if offset == '0':
+            response = web.json_response(
+                {
+                    'query': {'more_results_available': True},
+                    'web': {
+                        'results': [
+                            {
+                                'title': 'First',
+                                'description': 'one.example.com',
+                                'url': 'https://one.example.com',
+                            }
+                        ]
+                    },
+                }
+            )
+            response.set_cookie('provider-session', 'ready')
+            return response
+        if request.cookies.get('provider-session') != 'ready':
+            return web.json_response({'error': 'missing provider session'}, status=403)
+        return web.json_response(
+            {
+                'query': {'more_results_available': False},
+                'web': {
+                    'results': [
+                        {
+                            'title': 'Second',
+                            'description': 'two.example.com',
+                            'url': 'https://two.example.com',
+                        }
+                    ]
+                },
+            }
+        )
+
+    app = web.Application()
+    app.router.add_get('/search', search)
+    runner = web.AppRunner(app)
+    await runner.setup()
+    site = web.TCPSite(runner, '127.0.0.1', unused_tcp_port)
+    await site.start()
+    monkeypatch.setattr(bravesearch, 'get_delay', lambda: 0)
+
+    try:
+        source = bravesearch.SearchBrave(
+            'example.com',
+            limit=2,
+            credential_adapter=InMemoryCredentialAdapter({'brave': {'key': 'test-token'}}),
+        )
+        source.server = f'http://localhost:{unused_tcp_port}/search'
+        report = await source.process()
+    finally:
+        await runner.cleanup()
+
+    assert requests == [('0', None), ('1', 'ready')]
+    assert await source.get_hostnames() == ['one.example.com', 'two.example.com']
+    assert report == SourceExecutionReport('completed', 'result-limit')
+    assert len(sessions) == 1
+    assert sessions[0].closed is True
 
 
 @pytest.mark.asyncio

tests/lib/test_core.py

@@ -961,6 +961,27 @@ async def test_fetch_text_reuses_caller_owned_session_without_closing_it(
     assert len(DummySession.instances) == 1
 
 
+@pytest.mark.asyncio
+async def test_fetch_json_reuses_caller_owned_session_without_closing_it(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    install_stream_response(monkeypatch, chunks=(b'{"provider":"evidence"}',))
+    session = DummySession(
+        headers={'User-Agent': 'shared'},
+        timeout=core_module.aiohttp.ClientTimeout(total=None),
+    )
+
+    result = await AsyncFetcher.fetch_json(
+        'https://provider.example/data',
+        session=session,
+        request_timeout=None,
+    )
+
+    assert result.body == {'provider': 'evidence'}
+    assert session.closed is False
+    assert len(DummySession.instances) == 1
+
+
 @pytest.mark.asyncio
 async def test_fetch_json_accepts_body_at_shared_limit(monkeypatch) -> None:
     install_stream_response(monkeypatch, chunks=(b'{"a":1}',))

tests/test_bravesearch.py

@@ -7,16 +7,18 @@ import pytest
 from theHarvester.discovery.bravesearch import SearchBrave
 from theHarvester.discovery.constants import MissingKey
 from theHarvester.lib.configuration import InMemoryCredentialAdapter
-from theHarvester.lib.core import AsyncFetcher
+from theHarvester.lib.core import AsyncFetcher, FetcherResponse
 
 
 @pytest.mark.asyncio
 async def test_brave_collects_with_in_memory_credentials(monkeypatch: pytest.MonkeyPatch) -> None:
     request_headers: list[dict[str, str]] = []
 
-    async def fetch(*, headers: dict[str, str], **_kwargs: Any) -> dict[str, Any]:
+    async def fetch(*_args: Any, headers: dict[str, str], **_kwargs: Any) -> FetcherResponse:
         request_headers.append(headers)
-        return {
+        return FetcherResponse(
+            {
+                'query': {'more_results_available': False},
                 'web': {
                     'results': [
                         {
@@ -25,10 +27,13 @@ async def test_brave_collects_with_in_memory_credentials(monkeypatch: pytest.Mon
                             'url': 'https://docs.example.com',
                         }
                     ]
-            }
-        }
+                },
+            },
+            200,
+            {},
+        )
 
-    monkeypatch.setattr(AsyncFetcher, 'fetch', fetch)
+    monkeypatch.setattr(AsyncFetcher, 'fetch_json', fetch)
     search = SearchBrave(
         'example.com',
         1,

theHarvester/discovery/bravesearch.py

@@ -1,14 +1,16 @@
 import asyncio
-import logging
+import ssl
 from typing import Any
 from urllib.parse import quote
 
-from theHarvester.discovery.constants import MissingKey, get_delay
-from theHarvester.lib.configuration import CredentialAdapter, FileSystemCredentialAdapter
-from theHarvester.lib.core import AsyncFetcher
-from theHarvester.parsers import myparser
+import aiohttp
 
-logger = logging.getLogger(__name__)
+from theHarvester.discovery.constants import MissingKey, get_delay
+from theHarvester.discovery.provider_response import provider_http_error
+from theHarvester.lib.configuration import CredentialAdapter, FileSystemCredentialAdapter
+from theHarvester.lib.core import AsyncFetcher, ResponseStreamError
+from theHarvester.lib.source_execution import SourceExecutionReport
+from theHarvester.parsers import myparser
 
 
 class SearchBrave:
@@ -34,11 +36,20 @@ class SearchBrave:
             raise MissingKey('Brave Search')
         self.server = 'https://api.search.brave.com/res/v1/web/search'
         self.limit = limit
-        self.proxy = False
-        self.rate_limit_delay = 1  # Initial delay for rate limiting
+        self.proxy: bool | str = False
 
-    async def do_search(self):
+    async def do_search(self, session: Any | None = None) -> SourceExecutionReport | None:
         headers = {'Accept': 'application/json', 'Accept-Encoding': 'gzip', 'X-Subscription-Token': self.api_key}
+        if session is None:
+            try:
+                async with AsyncFetcher.open_session(
+                    headers=headers,
+                    proxy=self.proxy,
+                    request_timeout=60,
+                ) as owned_session:
+                    return await self.do_search(owned_session)
+            except ResponseStreamError as error:
+                return SourceExecutionReport('failed', error.reason)
 
         # Search queries: exact match and site-specific
         queries = [f'"{self.word}"', f'site:{self.word}']
@@ -66,77 +77,80 @@ class SearchBrave:
                     param_string = '&'.join([f'{k}={quote(str(v))}' for k, v in params.items()])
                     url = f'{self.server}?{param_string}'
 
-                    resp = await AsyncFetcher.fetch(url=url, headers=headers, proxy=self.proxy, json=True)
+                    response = await AsyncFetcher.fetch_json(
+                        url,
+                        session=session,
+                        headers=headers,
+                    )
+                    if failure := provider_http_error(response):
+                        return SourceExecutionReport(*failure)
+                    resp = response.body
 
                     # Handle API response
                     if resp is None:
-                        logger.info('No response received from Brave Search API')
-                        break
+                        return SourceExecutionReport('failed', 'invalid-response')
+                    if not isinstance(resp, dict):
+                        return SourceExecutionReport('failed', 'invalid-response')
 
                     # Check for API errors (rate limit, quota exceeded, etc.)
                     if 'error' in resp:
-                        error_msg = resp.get('error', {}).get('message', 'Unknown API error')
-                        error_code = resp.get('error', {}).get('code', 'unknown')
+                        provider_error = resp['error']
+                        if not isinstance(provider_error, dict):
+                            return SourceExecutionReport('failed', 'invalid-response')
+                        error_message = str(provider_error.get('message', '')).lower()
+                        error_code = str(provider_error.get('code', '')).lower()
+                        if 'rate limit' in error_message or error_code == 'rate_limit_exceeded':
+                            return SourceExecutionReport('rate-limited', 'provider-rate-limit')
+                        if 'quota' in error_message or error_code == 'quota_exceeded':
+                            return SourceExecutionReport('failed', 'quota-exhausted')
+                        return SourceExecutionReport('failed', 'provider-error')
 
-                        if 'rate limit' in error_msg.lower() or error_code == 'rate_limit_exceeded':
-                            logger.info(f'Rate limit exceeded. Increasing delay to {self.rate_limit_delay * 2} seconds')
-                            self.rate_limit_delay *= 2
-                            await asyncio.sleep(self.rate_limit_delay)
-                            break
-                        elif 'quota' in error_msg.lower() or error_code == 'quota_exceeded':
-                            logger.info('Brave Search API quota exceeded')
-                            break
-                        else:
-                            break
+                    web = resp.get('web')
+                    query_data = resp.get('query')
+                    if not isinstance(web, dict) or not isinstance(web.get('results'), list):
+                        return SourceExecutionReport('failed', 'invalid-response')
+                    if not isinstance(query_data, dict):
+                        return SourceExecutionReport('failed', 'invalid-response')
+                    more_results_available = query_data.get('more_results_available')
+                    if not isinstance(more_results_available, bool):
+                        return SourceExecutionReport('failed', 'invalid-response')
 
-                    if 'web' in resp and 'results' in resp['web']:
-                        results = resp['web']['results'][:remaining]
+                    results = web['results'][:remaining]
+                    if any(not isinstance(result, dict) for result in results):
+                        return SourceExecutionReport('failed', 'invalid-response')
                     if not results:
+                        if more_results_available:
+                            return SourceExecutionReport('failed', 'invalid-response')
                         break
 
-                        # Extract text content from results for parsing (including extra snippets)
                     for result in results:
-                            result_text = f'{result.get("title", "")} {result.get("description", "")}'
-
-                            # Add extra snippets if available
-                            if 'extra_snippets' in result:
-                                for snippet in result['extra_snippets']:
+                        snippets = result.get('extra_snippets', [])
+                        if not isinstance(snippets, list) or any(not isinstance(snippet, str) for snippet in snippets):
+                            return SourceExecutionReport('failed', 'invalid-response')
+                        title = result.get('title', '')
+                        description = result.get('description', '')
+                        result_url = result.get('url', '')
+                        if not all(isinstance(value, str) for value in (title, description, result_url)):
+                            return SourceExecutionReport('failed', 'invalid-response')
+                        result_text = f'{title} {description}'
+                        for snippet in snippets:
                             result_text += f' {snippet}'
-
-                            result_text += f' {result.get("url", "")}'
+                        result_text += f' {result_url}'
                         self.totalresults += result_text + '\n'
 
                     self.results.extend(results)
-
-                        # Stop if we've reached our limit
                     if len(self.results) >= self.limit:
-                            break
-                        if not resp.get('query', {}).get('more_results_available', False):
-                            break
-                    else:
-                        logger.info('Unexpected response format from Brave Search API')
+                        return SourceExecutionReport('completed', 'result-limit')
+                    if not more_results_available:
                         break
 
                     await asyncio.sleep(get_delay())
-
-            except Exception as e:
-                error_msg = str(e).lower()
-
-                # Handle specific API-related exceptions
-                if 'rate limit' in error_msg or '429' in error_msg:
-                    logger.info(f'Rate limit detected in exception. Increasing delay to {self.rate_limit_delay * 2} seconds')
-                    self.rate_limit_delay *= 2
-                    await asyncio.sleep(self.rate_limit_delay)
-                elif 'quota' in error_msg or '403' in error_msg:
-                    logger.info(f'Quota exceeded or access denied: {e}')
-                    break
-                elif 'timeout' in error_msg:
-                    logger.info(f'Request timeout occurred: {e}')
-                    await asyncio.sleep(get_delay() + 2)
                 else:
-                    logger.info(f'An exception has occurred in bravesearch: {e}')
-                    await asyncio.sleep(get_delay() + 5)
-                continue
+                    return SourceExecutionReport('partial', 'pagination-limit')
+
+            except ResponseStreamError as error:
+                return SourceExecutionReport('failed', error.reason)
+        return None
 
     async def get_emails(self):
         rawres = myparser.Parser(self.totalresults, self.word)
@@ -146,6 +160,11 @@ class SearchBrave:
         rawres = myparser.Parser(self.totalresults, self.word)
         return await rawres.hostnames()
 
-    async def process(self, proxy=False):
+    async def process(self, proxy: bool | str = False) -> SourceExecutionReport | None:
         self.proxy = proxy
-        await self.do_search()
+        try:
+            return await self.do_search()
+        except asyncio.CancelledError:
+            raise
+        except aiohttp.ClientError, TimeoutError, OSError, ssl.SSLError:
+            return SourceExecutionReport('failed', 'transport-error')

theHarvester/discovery/dnsdb.py

@@ -5,7 +5,9 @@ import logging
 from urllib.parse import quote
 
 from theHarvester.discovery.constants import MissingKey
-from theHarvester.lib.core import AsyncFetcher, Core, ResponseStreamError
+from theHarvester.discovery.provider_response import provider_http_error
+from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse, ResponseStreamError
+from theHarvester.lib.source_execution import SourceExecutionReport
 
 logger = logging.getLogger(__name__)
 
@@ -42,7 +44,7 @@ class SearchDNSDB:
             return hostname
         return None
 
-    async def do_search(self) -> None:
+    async def do_search(self) -> SourceExecutionReport | None:
         query = quote(f'*.{self.target_domain}', safe='*.')
         url = f'{self.BASE_URL}/{query}?limit=0'
         headers = {
@@ -59,49 +61,56 @@ class SearchDNSDB:
             follow_redirects=False,
             request_timeout=120,
         ) as response:
-            if response.status == 429:
-                raise ConnectionError('DNSDB rate limit reached')
-            if response.status in {401, 403}:
-                raise PermissionError('DNSDB authentication failed')
-            if response.status == 503:
-                raise ConnectionError('DNSDB concurrent connection limit exceeded')
-            if response.status != 200:
-                raise ConnectionError(f'DNSDB returned HTTP {response.status}')
+            if failure := provider_http_error(FetcherResponse(None, response.status, response.headers)):
+                return SourceExecutionReport(*failure)
 
             first_record = True
             async for line in response:
                 try:
                     record = json.loads(line)
-                except json.JSONDecodeError:
+                except json.JSONDecodeError, RecursionError:
                     logger.info('DNSDB returned malformed NDJSON; partial results were preserved.')
-                    return
+                    return SourceExecutionReport('failed', 'invalid-response')
                 if not isinstance(record, dict):
                     logger.info('DNSDB returned an invalid stream record; partial results were preserved.')
-                    return
+                    return SourceExecutionReport('failed', 'invalid-response')
                 if first_record:
                     first_record = False
                     if record.get('cond') != 'begin':
                         logger.info('DNSDB stream did not begin correctly; no results were accepted.')
-                        return
+                        return SourceExecutionReport('failed', 'invalid-response')
                     continue
 
                 condition = record.get('cond')
                 if condition in {'succeeded', 'limited', 'failed'}:
                     if condition != 'succeeded':
                         logger.info(f'DNSDB stream ended with {condition}; partial results were preserved.')
-                    return
+                    if condition == 'limited':
+                        return SourceExecutionReport('rate-limited', 'provider-limited')
+                    if condition == 'failed':
+                        return SourceExecutionReport('failed', 'provider-failed')
+                    return None
+                if condition is not None:
+                    return SourceExecutionReport('failed', 'invalid-response')
                 obj = record.get('obj')
-                if isinstance(obj, dict) and (hostname := self._hostname(obj.get('rrname'))):
+                if not isinstance(obj, dict):
+                    return SourceExecutionReport('failed', 'invalid-response')
+                rrname = obj.get('rrname')
+                if not isinstance(rrname, str) or not rrname.strip():
+                    return SourceExecutionReport('failed', 'invalid-response')
+                if hostname := self._hostname(rrname):
                     self.totalhosts.add(hostname)
 
             logger.info('DNSDB stream ended without a terminal condition; partial results were preserved.')
+            return SourceExecutionReport('failed', 'invalid-response')
 
     async def get_hostnames(self) -> set[str]:
         return self.totalhosts
 
-    async def process(self, proxy: bool | str = False) -> None:
+    async def process(self, proxy: bool | str = False) -> SourceExecutionReport | None:
         self.proxy = proxy
         try:
-            await self.do_search()
+            return await self.do_search()
         except ResponseStreamError as error:
             logger.info(f'DNSDB request failed with {type(error).__name__}; partial results were preserved.')
+            return SourceExecutionReport('failed', error.reason)

theHarvester/discovery/duckduckgosearch.py

@@ -1,6 +1,81 @@
-from theHarvester.lib.core import AsyncFetcher, Core
+from typing import Any
+
+from theHarvester.discovery.provider_response import provider_http_error
+from theHarvester.lib.core import AsyncFetcher, Core, ResponseStreamError
+from theHarvester.lib.source_execution import SourceExecutionReport
 from theHarvester.parsers import myparser
 
+_TEXT_FIELDS = (
+    'Abstract',
+    'AbstractSource',
+    'AbstractText',
+    'AbstractURL',
+    'Answer',
+    'AnswerType',
+    'Definition',
+    'DefinitionSource',
+    'DefinitionURL',
+    'Entity',
+    'Heading',
+    'Image',
+    'Redirect',
+    'Type',
+)
+_TOPIC_TEXT_FIELDS = ('FirstURL', 'Name', 'Result', 'Text')
+
+
+def _topic_text(items: object) -> list[str] | None:
+    if not isinstance(items, list):
+        return None
+    text: list[str] = []
+    for item in items:
+        if not isinstance(item, dict):
+            return None
+        recognized = False
+        for field in _TOPIC_TEXT_FIELDS:
+            if field not in item:
+                continue
+            recognized = True
+            value = item[field]
+            if not isinstance(value, str):
+                return None
+            text.append(value)
+        if 'Topics' in item:
+            recognized = True
+            nested = _topic_text(item['Topics'])
+            if nested is None:
+                return None
+            text.extend(nested)
+        if item and not recognized:
+            return None
+    return text
+
+
+def _provider_text(body: dict[str, Any]) -> str | None:
+    if not body:
+        return ''
+    recognized = False
+    text: list[str] = []
+    for field in _TEXT_FIELDS:
+        if field not in body:
+            continue
+        recognized = True
+        value = body[field]
+        if not isinstance(value, str):
+            return None
+        text.append(value)
+    for field in ('RelatedTopics', 'Results'):
+        if field not in body:
+            continue
+        recognized = True
+        topic_text = _topic_text(body[field])
+        if topic_text is None:
+            return None
+        text.extend(topic_text)
+    if not recognized:
+        return None
+    return '\n'.join(text)
+
 
 class SearchDuckDuckGo:
     def __init__(self, word, limit) -> None:
@@ -15,13 +90,31 @@ class SearchDuckDuckGo:
         self.limit = limit
         self.proxy: bool = False
 
-    async def do_search(self) -> None:
+    async def do_search(self) -> SourceExecutionReport | None:
         # Query only the provider; URLs in the response are evidence, not crawl targets.
         url = self.api.replace('x', self.word)
         headers = {'User-Agent': Core.get_user_agent()}
-        first_resp = await AsyncFetcher.fetch_all([url], headers=headers, proxy=self.proxy)
-        self.results = first_resp[0]
+        try:
+            response = await AsyncFetcher.fetch_json(url, headers=headers, proxy=self.proxy)
+        except ResponseStreamError as error:
+            return SourceExecutionReport('failed', error.reason)
+        if failure := provider_http_error(response):
+            return SourceExecutionReport(*failure)
+        if not isinstance(response.body, dict):
+            return SourceExecutionReport('failed', 'invalid-response')
+        if 'error' in response.body:
+            provider_error = str(response.body['error']).lower()
+            if 'rate limit' in provider_error:
+                return SourceExecutionReport('rate-limited', 'provider-rate-limit')
+            if 'access denied' in provider_error or 'forbidden' in provider_error:
+                return SourceExecutionReport('failed', 'access-denied')
+            return SourceExecutionReport('failed', 'provider-error')
+        provider_text = _provider_text(response.body)
+        if provider_text is None:
+            return SourceExecutionReport('failed', 'invalid-response')
+        self.results = provider_text
         self.totalresults += self.results
+        return None
 
     async def get_emails(self):
         rawres = myparser.Parser(self.totalresults, self.word)
@@ -31,6 +124,6 @@ class SearchDuckDuckGo:
         rawres = myparser.Parser(self.totalresults, self.word)
         return await rawres.hostnames()
 
-    async def process(self, proxy: bool = False) -> None:
+    async def process(self, proxy: bool = False) -> SourceExecutionReport | None:
         self.proxy = proxy
-        await self.do_search()  # Only need to search once since using API.
+        return await self.do_search()  # Only need to search once since using API.

theHarvester/lib/core.py

@@ -827,6 +827,7 @@ class AsyncFetcher:
         cls,
         url: str,
         *,
+        session: aiohttp.ClientSession | None = None,
         params: Sized = '',
         proxy: str | bool | None = '',
         headers: dict[str, str] | None = None,
@@ -835,6 +836,7 @@ class AsyncFetcher:
         """Fetch one bounded JSON response without following redirects."""
         async with cls._open_get_response(
             url,
+            session=session,
             params=params,
             proxy=proxy,
             headers=headers,