Merge pull request #2550 from NotoriousRebel/codex/provider-contract-release-gate

Harden provider contracts and enforce offline coverage

.github/workflows/provider-smoke.yml

@@ -43,34 +43,35 @@ jobs:
         timeout-minutes: 5
         run: pytest --run-live-network -m live_network
 
+      # These are bounded CLI crash smokes, not provider conformance tests.
       - name: Run theHarvester module CertSpotter
         timeout-minutes: 5
-        run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b certspotter
+        run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b certspotter -l 10 -q
 
       - name: Run theHarvester module Crtsh
         timeout-minutes: 5
-        run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b crtsh
+        run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b crtsh -l 10 -q
 
       - name: Run theHarvester module DuckDuckGo
         timeout-minutes: 5
-        run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b duckduckgo
+        run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b duckduckgo -l 10 -q
 
       - name: Run theHarvester module HackerTarget
         timeout-minutes: 5
-        run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b hackertarget
+        run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b hackertarget -l 10 -q
 
       - name: Run theHarvester module Otx
         timeout-minutes: 5
-        run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b otx
+        run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b otx -l 10 -q
 
       - name: Run theHarvester module RapidDns
         timeout-minutes: 5
-        run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b rapiddns
+        run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b rapiddns -l 10 -q
 
       - name: Run theHarvester module Urlscan
         timeout-minutes: 5
-        run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b urlscan
+        run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b urlscan -l 10 -q
 
       - name: Run theHarvester module Yahoo
         timeout-minutes: 5
-        run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b yahoo
+        run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b yahoo -l 10 -q

.github/workflows/theHarvester.yml

@@ -56,5 +56,7 @@ jobs:
           args: format --check
 
       - name: Test with pytest
-        run: |
-          pytest
+        run: pytest
+
+      - name: Type check with mypy
+        run: mypy theHarvester

CHANGELOG.md

@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Added
+- Added a catalog-derived offline provider-contract gate that fails on missing, unknown, or duplicate source coverage while keeping live checks outside routine CI.
 - Added sourced ASN organization attribution from URLScan, ONYPHE, and Shodan, linked to the exact hostname or IP evidence and retained in SQLite, JSONL, the API, CLI output, and HarvestView without claiming ownership or scope.
 - Added bounded RouteViews routing enrichment for exact discovered IPs with sourced ASN attribution, or explicit ASN, IP, and CIDR targets, retaining typed origin, BGP-route, and RPKI evidence as external relationships without expanding active scope.
 - Added route-aware `--no-hosts` and REST/HarvestView `no_hosts` filtering that skips hostname-only sources while retaining non-host evidence from mixed sources.
@@ -35,6 +36,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added root contributor and security policies, structured issue forms, repository agent guidance, discovery terminology, and an operator-focused documentation wiki ([d090a29a](https://github.com/laramies/theHarvester/commit/d090a29a), [7c491ef5](https://github.com/laramies/theHarvester/commit/7c491ef5), [8b9d420b](https://github.com/laramies/theHarvester/commit/8b9d420b)).
 
 ### Changed
+- Updated BeVigil, Dymo, FOFA, FullHunt, Hunter.how, Netlas, ONYPHE, SecurityScorecard, SecurityTrails, SherlockEye, SubdomainFinder C99, VirusTotal, WhoisXML, and ZoomEye provider contracts to retain scoped partial evidence and report authentication, quota, transport, HTTP, and malformed-response outcomes truthfully. FOFA and ONYPHE now honor the operator result limit across documented pagination, while ZoomEye uses the current `POST /v2/search` API and no longer stops after five empty result pages.
 - Replaced runtime takeover fingerprint downloads and global body-substring matches with pinned provider-gated DNS, wildcard controls, and compound HTTP rules. Every checked hostname is now stored as an indicator, no-indicator, or inconclusive outcome with typed DNS, HTTP, rule, and error details in JSONL, SQLite, the API, and HarvestView. Direct checks share one cookie-free HTTP session, keep bounded response bodies, and rely on the whole-run deadline instead of silently inheriting aiohttp's default timeout.
 - HarvestView now summarizes retained evidence and producer health at a glance, links directly to execution outcomes that need attention, and keeps evidence values ahead of optional actions on mobile. Its source picker reports credential readiness without exposing values, prevents unavailable source selections, and replaces the mobile nested-scroll catalog with collapsible activity groups.
 - Routed discovery sources through immutable source jobs with bounded `TaskGroup` ownership, typed outcomes, and native cancellation propagation instead of queuing live coroutine objects.

CONTRIBUTING.md

@@ -49,6 +49,13 @@ Useful cases include:
 - pagination and retry termination;
 - normalized, deduplicated results.
 
+Every canonical provider has one offline contract module marked with
+`pytest.mark.provider_contract("source-name")`. The catalog-derived coverage
+gate fails when a catalog entry has no contract, when a contract names an
+unknown source, or when two modules claim the same source. Add the marker to
+the provider's deterministic contract module; do not maintain a second source
+list in tests.
+
 Use the shared transport when it can represent the request. If a provider requires behavior the shared transport does not support, keep the exception local and explain it in the pull request. Never log credentials, account information, private target data, or raw API responses.
 
 ## Test safely
@@ -65,15 +72,12 @@ Before submitting, run the non-mutating quality checks and full test suite:
 uv run ruff check .
 uv run ruff format --check .
 uv run pytest
-```
-
-Changes to typed interfaces should also pass:
-
-```bash
 uv run mypy theHarvester
 ```
 
-Routine verification must use mocks, local fixtures, and reserved example domains. Do not run broad or active reconnaissance against third-party targets. If live verification is essential, use only a target you own or are explicitly authorized to test, limit the request scope, and keep collected data out of commits, issues, and pull requests.
+Routine verification must use mocks, local fixtures, and reserved example domains. The test harness blocks external Python socket traffic unless a test is marked `live_network` and pytest is invoked with both `--run-live-network` and `-m live_network`. A live-marked test never satisfies the provider-contract coverage gate.
+
+Do not run broad or active reconnaissance against third-party targets. If live verification is essential, use only a target you own or are explicitly authorized to test, limit the request scope, and keep collected data out of commits, issues, and pull requests. The manually dispatched provider workflow uses `mozilla.org` for small passive CLI crash smokes. Those runs can detect packaging, credential, or provider drift; they are not conformance tests and should not be retried merely to obtain more results.
 
 CI runs additional source smoke tests, CodeQL, dependency review, and container checks. Contributors do not need to reproduce broad live-source checks locally.
 

pyproject.toml

@@ -70,6 +70,7 @@ asyncio_default_fixture_loop_scope = "function"
 addopts = "--no-header --strict-markers -m 'not harvestview_e2e'"
 markers = [
     "live_network: contacts an external service and runs only with --run-live-network",
+    "provider_contract(source): deterministic offline contract for one canonical discovery source",
     "harvestview_e2e: real-browser tests against an isolated local HarvestView server",
 ]
 testpaths = ["tests"]

tests/conftest.py

@@ -8,6 +8,7 @@ from typing import Any
 import pytest
 
 NETWORK_GUARD = pytest.StashKey[pytest.MonkeyPatch]()
+PROVIDER_CONTRACT_SOURCES = pytest.StashKey[tuple[str, ...]]()
 
 _getaddrinfo = socket.getaddrinfo
 _gethostbyaddr = socket.gethostbyaddr
@@ -39,6 +40,11 @@ def live_test_domain() -> str:
     return os.environ.get('SMOKE_TEST_DOMAIN', 'mozilla.org')
 
 
+@pytest.fixture
+def provider_contract_sources(request: pytest.FixtureRequest) -> tuple[str, ...]:
+    return request.config.stash.get(PROVIDER_CONTRACT_SOURCES, ())
+
+
 def pytest_sessionstart(session: pytest.Session) -> None:
     guard = pytest.MonkeyPatch()
     guard.setattr(socket, 'getaddrinfo', _guarded_getaddrinfo)
@@ -53,6 +59,16 @@ def pytest_sessionstart(session: pytest.Session) -> None:
 
 
 def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
+    contract_modules: set[tuple[str, str]] = set()
+    for item in items:
+        if item.get_closest_marker('live_network') is not None:
+            continue
+        for marker in item.iter_markers('provider_contract'):
+            if len(marker.args) != 1 or marker.kwargs or not isinstance(marker.args[0], str):
+                raise pytest.UsageError(f'{item.nodeid}: provider_contract requires one canonical source name')
+            contract_modules.add((marker.args[0], str(item.path)))
+    config.stash[PROVIDER_CONTRACT_SOURCES] = tuple(source for source, _path in sorted(contract_modules))
+
     if config.getoption('--run-live-network'):
         if config.getoption('markexpr') != 'live_network':
             raise pytest.UsageError('--run-live-network requires -m live_network')

tests/discovery/test_apisguru.py

@@ -681,6 +681,8 @@ async def test_directory_failures_are_attributed(
     assert search.stop_reason == stop_reason
 
 
+
+
 @pytest.mark.asyncio
 async def test_missing_provider_is_completed_with_no_results(monkeypatch: pytest.MonkeyPatch) -> None:
     async def fake_fetch(**_kwargs: Any) -> FetcherResponse:
@@ -971,3 +973,6 @@ async def test_preferred_spec_failures_are_attributed(
 
     assert search.execution_status == execution_status
     assert search.stop_reason == stop_reason
+
+
+pytestmark = pytest.mark.provider_contract('apis-guru')

tests/discovery/test_arquivo.py

@@ -91,3 +91,6 @@ async def test_process_reports_http_and_malformed_responses(
     assert await second.get_hostnames() == set()
     assert 'Arquivo.pt request failed with HTTP 429' in caplog.text
     assert 'Arquivo.pt returned malformed CDX data' in caplog.text
+
+
+pytestmark = pytest.mark.provider_contract('arquivo')

tests/discovery/test_baidusearch.py

@@ -175,3 +175,6 @@ class TestBaiduSearch:
             'https://www.baidu.com/s?wd=site%3Aexample.com&pn=0',
             'https://www.baidu.com/s?wd=site%3Aexample.com&pn=10',
         ]
+
+
+pytestmark = pytest.mark.provider_contract('baidu')

tests/discovery/test_bevigil.py

@@ -0,0 +1,182 @@
+import asyncio
+import contextlib
+from collections.abc import AsyncIterator
+from typing import Any
+
+import pytest
+
+from theHarvester.discovery import bevigil
+from theHarvester.discovery.constants import MissingKey
+from theHarvester.lib.core import FetcherResponse
+
+
+@pytest.mark.provider_contract('bevigil')
+@pytest.mark.asyncio
+async def test_process_collects_scoped_hostnames_and_urls(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(bevigil.Core, 'bevigil_key', staticmethod(lambda: 'test-key'))
+    session = object()
+    session_exited = False
+    open_calls: list[dict[str, Any]] = []
+    calls: list[tuple[list[str], dict[str, Any]]] = []
+    responses = [
+        FetcherResponse(
+            body={'subdomains': ['API.Example.COM.', 'outside.test']},
+            status=200,
+            headers={},
+        ),
+        FetcherResponse(
+            body={'urls': ['https://portal.example.com/path', 'https://outside.test/example.com']},
+            status=200,
+            headers={},
+        ),
+    ]
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**kwargs: Any) -> AsyncIterator[object]:
+        nonlocal session_exited
+        open_calls.append(kwargs)
+        try:
+            yield session
+        finally:
+            session_exited = True
+
+    async def fake_fetch_all(urls: list[str], **kwargs: Any) -> list[FetcherResponse]:
+        calls.append((urls, kwargs))
+        return [responses.pop(0)]
+
+    monkeypatch.setattr(bevigil.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(bevigil.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    search = bevigil.SearchBeVigil('example.com')
+
+    await search.process(proxy=True)
+
+    assert await search.get_hostnames() == {'api.example.com'}
+    assert await search.get_urls() == {'https://portal.example.com/path'}
+    assert [urls for urls, _kwargs in calls] == [
+        ['https://osint.bevigil.com/api/example.com/subdomains/'],
+        ['https://osint.bevigil.com/api/example.com/urls/'],
+    ]
+    assert all(kwargs['headers'] == {'X-Access-Token': 'test-key'} for _urls, kwargs in calls)
+    assert all(kwargs['proxy'] is True for _urls, kwargs in calls)
+    assert all(kwargs['json'] is True for _urls, kwargs in calls)
+    assert all(kwargs['include_metadata'] is True for _urls, kwargs in calls)
+    assert all(kwargs['session'] is session for _urls, kwargs in calls)
+    assert open_calls == [
+        {
+            'headers': {'X-Access-Token': 'test-key'},
+            'proxy': True,
+            'request_timeout': 60,
+        }
+    ]
+    assert session_exited is True
+    assert search.execution_status == 'completed'
+    assert search.stop_reason is None
+
+
+@pytest.mark.parametrize('key', [None, '', '   '])
+def test_missing_or_empty_key_fails_before_transport(monkeypatch: pytest.MonkeyPatch, key: str | None) -> None:
+    monkeypatch.setattr(bevigil.Core, 'bevigil_key', staticmethod(lambda: key))
+
+    with pytest.raises(MissingKey):
+        bevigil.SearchBeVigil('example.com')
+
+
+@pytest.mark.parametrize(
+    ('response', 'execution_status', 'stop_reason'),
+    [
+        (None, 'failed', 'transport-error'),
+        (FetcherResponse(body={}, status=401, headers={}), 'failed', 'access-denied'),
+        (FetcherResponse(body={}, status=403, headers={}), 'failed', 'access-denied'),
+        (FetcherResponse(body={}, status=429, headers={}), 'rate-limited', 'http-429'),
+        (FetcherResponse(body={}, status=503, headers={}), 'failed', 'http-503'),
+        (FetcherResponse(body={'subdomains': {}}, status=200, headers={}), 'failed', 'invalid-response'),
+    ],
+)
+@pytest.mark.asyncio
+async def test_failed_first_response_is_attributed(
+    monkeypatch: pytest.MonkeyPatch,
+    response: FetcherResponse | None,
+    execution_status: str,
+    stop_reason: str,
+) -> None:
+    monkeypatch.setattr(bevigil.Core, 'bevigil_key', staticmethod(lambda: 'test-key'))
+
+    async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse | None]:
+        return [response]
+
+    monkeypatch.setattr(bevigil.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    search = bevigil.SearchBeVigil('example.com')
+
+    await search.process()
+
+    assert await search.get_hostnames() == set()
+    assert await search.get_urls() == set()
+    assert search.execution_status == execution_status
+    assert search.stop_reason == stop_reason
+
+
+@pytest.mark.asyncio
+async def test_later_malformed_response_preserves_partial_results(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(bevigil.Core, 'bevigil_key', staticmethod(lambda: 'test-key'))
+    responses = [
+        FetcherResponse(body={'subdomains': ['api.example.com']}, status=200, headers={}),
+        FetcherResponse(body={'urls': {}}, status=200, headers={}),
+    ]
+
+    async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]:
+        return [responses.pop(0)]
+
+    monkeypatch.setattr(bevigil.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    search = bevigil.SearchBeVigil('example.com')
+
+    await search.process()
+
+    assert await search.get_hostnames() == {'api.example.com'}
+    assert await search.get_urls() == set()
+    assert search.execution_status == 'partial'
+    assert search.stop_reason == 'invalid-response'
+
+
+@pytest.mark.asyncio
+async def test_cancellation_closes_provider_session(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(bevigil.Core, 'bevigil_key', staticmethod(lambda: 'test-key'))
+    session_exited = False
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        nonlocal session_exited
+        try:
+            yield object()
+        finally:
+            session_exited = True
+
+    async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]:
+        raise asyncio.CancelledError('operator-stop')
+
+    monkeypatch.setattr(bevigil.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(bevigil.AsyncFetcher, 'fetch_all', fake_fetch_all)
+
+    with pytest.raises(asyncio.CancelledError, match='operator-stop'):
+        await bevigil.SearchBeVigil('example.com').process()
+    assert session_exited is True
+
+
+@pytest.mark.asyncio
+async def test_early_malformed_rows_and_later_valid_evidence_are_partial(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(bevigil.Core, 'bevigil_key', staticmethod(lambda: 'test-key'))
+    responses = [
+        FetcherResponse(body={'subdomains': [7]}, status=200, headers={}),
+        FetcherResponse(body={'urls': ['https://portal.example.com/path']}, status=200, headers={}),
+    ]
+
+    async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]:
+        return [responses.pop(0)]
+
+    monkeypatch.setattr(bevigil.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    search = bevigil.SearchBeVigil('example.com')
+
+    await search.process()
+
+    assert await search.get_urls() == {'https://portal.example.com/path'}
+    assert search.execution_status == 'partial'
+    assert search.stop_reason == 'invalid-response'

tests/discovery/test_brave.py

@@ -259,3 +259,6 @@ async def test_brave_never_exceeds_maximum_page_offset(
     await search.process()
 
     assert [request['offset'] for request in requests] == [[str(offset)] for offset in range(10)] * 2
+
+
+pytestmark = pytest.mark.provider_contract('brave')

tests/discovery/test_bufferoverun.py

@@ -39,6 +39,8 @@ async def test_process_parses_historical_four_column_rows(monkeypatch: pytest.Mo
     assert search.stop_reason == 'invalid-response'
 
 
+
+
 @pytest.mark.asyncio
 async def test_valid_empty_response_is_completed(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.setattr(bufferoverun.Core, 'bufferoverun_key', staticmethod(lambda: 'test-key'))
@@ -137,3 +139,6 @@ async def test_non_string_row_preserves_valid_partial_results(monkeypatch: pytes
     assert await search.get_ips() == {'192.0.2.10'}
     assert search.execution_status == 'partial'
     assert search.stop_reason == 'invalid-response'
+
+
+pytestmark = pytest.mark.provider_contract('bufferoverun')

tests/discovery/test_builtwith.py

@@ -353,3 +353,6 @@ async def test_normalized_builtwith_results_reach_completed_jsonl(
     assert {'type': 'server', 'value': 'nginx', 'sources': ['builtwith']} in records
     assert {'type': 'cms', 'value': 'WordPress', 'sources': ['builtwith']} in records
     assert {'type': 'analytics', 'value': 'Google Analytics', 'sources': ['builtwith']} in records
+
+
+pytestmark = pytest.mark.provider_contract('builtwith')

tests/discovery/test_censys.py

@@ -310,3 +310,6 @@ async def test_search_classifies_transport_exceptions(monkeypatch) -> None:
 
 def test_deprecated_censys_sdk_is_not_a_runtime_dependency() -> None:
     assert '"censys==' not in Path('pyproject.toml').read_text()
+
+
+pytestmark = pytest.mark.provider_contract('censys')

tests/discovery/test_certspotter.py

@@ -337,3 +337,6 @@ class TestCertspotterSearch(object):
 
 if __name__ == '__main__':
     pytest.main()
+
+
+pytestmark = pytest.mark.provider_contract('certspotter')

tests/discovery/test_commoncrawl.py

@@ -416,6 +416,8 @@ async def test_process_retains_partial_results_at_the_runtime_limit(monkeypatch:
     await asyncio.wait_for(search.process(), timeout=0.1)
 
     assert await search.get_hostnames() == {'api.example.com'}
+
+
     assert search.execution_status == 'partial'
     assert search.stop_reason == 'runtime-limit'
 
@@ -548,3 +550,6 @@ async def test_process_keeps_results_when_another_query_fails(monkeypatch: pytes
     await search.process()
 
     assert await search.get_hostnames() == {'api.example.com'}
+
+
+pytestmark = pytest.mark.provider_contract('commoncrawl')

tests/discovery/test_criminalip.py

@@ -120,3 +120,6 @@ async def test_do_search_uses_v2_report_endpoint(monkeypatch) -> None:
 
     assert any('/v2/domain/report/12345' in url for url in called_urls)
     assert all('/v1/domain/report/' not in url for url in called_urls)
+
+
+pytestmark = pytest.mark.provider_contract('criminalip')

tests/discovery/test_crtname.py

@@ -319,3 +319,6 @@ async def test_crt_name_and_crtsh_share_one_result_with_both_sources(
     findings = {(record['type'], record['value']): record for record in records[1:]}
     assert findings[('hostname', 'shared.example.com')]['sources'] == ['crt-name', 'crtsh']
     assert findings[('hostname', 'only-crt-name.example.com')]['sources'] == ['crt-name']
+
+
+pytestmark = pytest.mark.provider_contract('crt-name')

tests/discovery/test_crtsh.py

@@ -169,3 +169,6 @@ class TestCrtshIntegration:
         from theHarvester.lib.source_catalog import SOURCE_SPECS
 
         assert 'crtsh' in SOURCE_SPECS
+
+
+pytestmark = pytest.mark.provider_contract('crtsh')

tests/discovery/test_dnsdb.py

@@ -180,3 +180,6 @@ async def test_process_exposes_http_failures(
 
     with pytest.raises(expected_error):
         await dnsdb.SearchDNSDB('example.com').process()
+
+
+pytestmark = pytest.mark.provider_contract('dnsdb')

tests/discovery/test_dnsdumpster.py

@@ -140,3 +140,6 @@ async def test_fetch_exception_is_transport_failure(monkeypatch: pytest.MonkeyPa
 
     assert search.execution_status == 'failed'
     assert search.stop_reason == 'transport-error'
+
+
+pytestmark = pytest.mark.provider_contract('dnsdumpster')

tests/discovery/test_duckduckgo.py

@@ -59,3 +59,6 @@ async def test_duckduckgo_unusable_response_returns_no_evidence(
 
     assert await search.get_hostnames() == []
     assert await search.get_emails() == set()
+
+
+pytestmark = pytest.mark.provider_contract('duckduckgo')

tests/discovery/test_dymosearch.py

@@ -1,145 +1,104 @@
+import asyncio
+from typing import Any
+
 import pytest
 
 from theHarvester.discovery import dymosearch
 from theHarvester.discovery.constants import MissingKey
+from theHarvester.lib.core import FetcherResponse
 
 
-def _patch_dymo_key(monkeypatch, value):
-    import theHarvester.lib.core as core_module
+@pytest.mark.provider_contract('dymo')
+@pytest.mark.asyncio
+async def test_process_extracts_scoped_canonical_and_suggested_domains(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(dymosearch.Core, 'dymo_key', lambda: 'token-xyz')
+    captured: dict[str, Any] = {}
 
-    monkeypatch.setattr(core_module.Core, 'dymo_key', staticmethod(lambda: value), raising=True)
-    monkeypatch.setattr(core_module.Core, 'get_user_agent', staticmethod(lambda: 'UA'), raising=True)
+    async def fake_post_fetch(url: str, **kwargs: Any) -> FetcherResponse:
+        captured.update({'url': url, **kwargs})
+        return FetcherResponse(
+            {
+                'domain': {'domain': 'example.com', 'didYouMean': 'www.example.com'},
+                'url': {'domain': 'outside.test', 'didYouMean': 'notexample.com'},
+            },
+            200,
+            {},
+        )
+
+    monkeypatch.setattr(dymosearch.AsyncFetcher, 'post_fetch', fake_post_fetch)
+    search = dymosearch.SearchDymo('example.com')
+    await search.process(proxy=True)
+
+    assert await search.get_hostnames() == {'example.com', 'www.example.com'}
+    assert (await search.get_results())['domain']['domain'] == 'example.com'
+    assert captured['url'] == dymosearch.SearchDymo.VERIFY_URL
+    assert captured['json_body'] == {'domain': 'example.com', 'url': 'https://example.com'}
+    assert captured['include_metadata'] is True
+    assert captured['proxy'] is True
+    assert search.execution_status == 'completed'
+    assert search.stop_reason is None
 
 
-class TestDymoSearch:
-    def test_missing_key_raises(self, monkeypatch):
-        _patch_dymo_key(monkeypatch, None)
-        with pytest.raises(MissingKey):
-            dymosearch.SearchDymo('example.com')
-
-    def test_init_sets_state(self, monkeypatch):
-        _patch_dymo_key(monkeypatch, 'token-123')
-        search = dymosearch.SearchDymo('example.com')
-        assert search.word == 'example.com'
-        assert search.key == 'token-123'
-        assert search.proxy is False
-        assert search.totalhosts == set()
-        assert search.results == {}
-
-    def test_headers_use_bearer(self, monkeypatch):
-        _patch_dymo_key(monkeypatch, 'token-abc')
-        search = dymosearch.SearchDymo('example.com')
-        headers = search._headers()
-        assert headers['Authorization'] == 'Bearer token-abc'
-        assert headers['Content-Type'] == 'application/json'
-        assert headers['User-Agent'] == 'UA'
-
-    @pytest.mark.asyncio
-    async def test_process_extracts_canonical_and_suggestion(self, monkeypatch):
-        _patch_dymo_key(monkeypatch, 'token-xyz')
-        captured = {}
-
-        async def fake_post_fetch(url, headers=None, data='', params='', json=False, proxy=False):
-            captured['url'] = url
-            captured['headers'] = headers
-            captured['data'] = data
-            captured['proxy'] = proxy
-            return {
-                'domain': {
-                    'valid': True,
-                    'fraud': False,
-                    'freeSubdomain': False,
-                    'domain': 'exemple.com',
-                    'didYouMean': 'www.exemple.com',
-                },
-                'url': {
-                    'valid': True,
-                    'domain': 'exemple.com',
-                    'didYouMean': None,
-                },
-            }
-
-        import theHarvester.lib.core as core_module
-
-        monkeypatch.setattr(core_module.AsyncFetcher, 'post_fetch', classmethod(lambda cls, *a, **kw: fake_post_fetch(*a, **kw)))
-
-        search = dymosearch.SearchDymo('exemple.com')
-        await search.process(proxy=True)
-
-        assert captured['url'] == dymosearch.SearchDymo.VERIFY_URL
-        assert captured['proxy'] is True
-        assert captured['data'] == {'domain': 'exemple.com', 'url': 'https://exemple.com'}
-        assert captured['headers']['Authorization'] == 'Bearer token-xyz'
-
-        hosts = await search.get_hostnames()
-        results = await search.get_results()
-
-        assert 'exemple.com' in hosts
-        assert 'www.exemple.com' in hosts
-        assert results['domain']['valid'] is True
-
-    @pytest.mark.asyncio
-    async def test_process_handles_empty_payload(self, monkeypatch):
-        _patch_dymo_key(monkeypatch, 'token')
-
-        async def fake_post_fetch(url, headers=None, data='', params='', json=False, proxy=False):
-            return {}
-
-        import theHarvester.lib.core as core_module
-
-        monkeypatch.setattr(core_module.AsyncFetcher, 'post_fetch', classmethod(lambda cls, *a, **kw: fake_post_fetch(*a, **kw)))
-
-        search = dymosearch.SearchDymo('example.com')
-        await search.process()
-        assert await search.get_hostnames() == set()
-        assert await search.get_results() == {}
-
-    @pytest.mark.asyncio
-    async def test_process_ignores_unrelated_suggestion(self, monkeypatch):
-        _patch_dymo_key(monkeypatch, 'token')
-
-        async def fake_post_fetch(url, headers=None, data='', params='', json=False, proxy=False):
-            return {
-                'domain': {
-                    'valid': True,
-                    'domain': 'totally-different.org',
-                    'didYouMean': 'somewhere-else.net',
-                },
-            }
-
-        import theHarvester.lib.core as core_module
-
-        monkeypatch.setattr(core_module.AsyncFetcher, 'post_fetch', classmethod(lambda cls, *a, **kw: fake_post_fetch(*a, **kw)))
-
-        search = dymosearch.SearchDymo('example.com')
-        await search.process()
-        # Neither contains 'example.com', so neither should be added.
-        assert await search.get_hostnames() == set()
-
-    @pytest.mark.asyncio
-    async def test_process_handles_non_dict_response(self, monkeypatch):
-        _patch_dymo_key(monkeypatch, 'token')
-
-        async def fake_post_fetch(url, headers=None, data='', params='', json=False, proxy=False):
-            return '<html>error</html>'
-
-        import theHarvester.lib.core as core_module
-
-        monkeypatch.setattr(core_module.AsyncFetcher, 'post_fetch', classmethod(lambda cls, *a, **kw: fake_post_fetch(*a, **kw)))
-
-        search = dymosearch.SearchDymo('example.com')
-        await search.process()
-        assert await search.get_hostnames() == set()
-        assert search.results == {}
+@pytest.mark.parametrize('key', [None, '', '   '])
+def test_missing_or_blank_key_fails_closed(monkeypatch: pytest.MonkeyPatch, key: str | None) -> None:
+    monkeypatch.setattr(dymosearch.Core, 'dymo_key', lambda: key)
+    with pytest.raises(MissingKey):
+        dymosearch.SearchDymo('example.com')
 
 
-class TestDymoIntegration:
-    def test_module_exposes_class(self, monkeypatch):
-        from theHarvester.discovery import dymosearch as mod
+@pytest.mark.parametrize(
+    ('response', 'status', 'reason'),
+    [
+        (None, 'failed', 'transport-error'),
+        (FetcherResponse({}, 401, {}), 'failed', 'access-denied'),
+        (FetcherResponse({}, 429, {}), 'rate-limited', 'http-429'),
+        (FetcherResponse({}, 503, {}), 'failed', 'http-503'),
+        (FetcherResponse([], 200, {}), 'failed', 'invalid-response'),
+        (FetcherResponse({'domain': []}, 200, {}), 'failed', 'invalid-response'),
+    ],
+)
+@pytest.mark.asyncio
+async def test_failures_are_structured(
+    monkeypatch: pytest.MonkeyPatch,
+    response: FetcherResponse | None,
+    status: str,
+    reason: str,
+) -> None:
+    monkeypatch.setattr(dymosearch.Core, 'dymo_key', lambda: 'token')
 
-        assert hasattr(mod, 'SearchDymo')
+    async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse | None:
+        return response
 
-    def test_source_catalog_lists_dymo(self):
-        from theHarvester.lib.source_catalog import SOURCE_SPECS
+    monkeypatch.setattr(dymosearch.AsyncFetcher, 'post_fetch', fake_post_fetch)
+    search = dymosearch.SearchDymo('example.com')
+    await search.process()
 
-        assert 'dymo' in SOURCE_SPECS
+    assert search.execution_status == status
+    assert search.stop_reason == reason
+
+
+@pytest.mark.asyncio
+async def test_empty_object_is_completed_without_evidence(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(dymosearch.Core, 'dymo_key', lambda: 'token')
+
+    async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        return FetcherResponse({}, 200, {})
+
+    monkeypatch.setattr(dymosearch.AsyncFetcher, 'post_fetch', fake_post_fetch)
+    search = dymosearch.SearchDymo('example.com')
+    await search.process()
+
+    assert search.execution_status == 'completed'
+    assert search.stop_reason == 'no-results'
+
+
+@pytest.mark.asyncio
+async def test_cancellation_propagates(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(dymosearch.Core, 'dymo_key', lambda: 'token')
+
+    async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        raise asyncio.CancelledError
+
+    monkeypatch.setattr(dymosearch.AsyncFetcher, 'post_fetch', fake_post_fetch)
+    with pytest.raises(asyncio.CancelledError):
+        await dymosearch.SearchDymo('example.com').process()

tests/discovery/test_fofa.py

@@ -1,4 +1,7 @@
-import logging
+import asyncio
+import base64
+import contextlib
+from collections.abc import AsyncIterator
 from typing import Any
 
 import pytest
@@ -8,131 +11,194 @@ from theHarvester.discovery.constants import MissingKey
 from theHarvester.lib.core import FetcherResponse
 
 
+@pytest.mark.provider_contract('fofa')
 @pytest.mark.asyncio
-async def test_http_failure_is_reported_without_results(
-    monkeypatch: pytest.MonkeyPatch,
-    caplog: pytest.LogCaptureFixture,
-) -> None:
+async def test_process_uses_cursor_api_to_limit_and_retains_scoped_results(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: ('test-key', 'operator@example.com'))
+    session = object()
+    session_exited = False
+    calls: list[dict[str, Any]] = []
+    responses = [
+        FetcherResponse(
+            {'error': False, 'results': [['https://API.Example.COM:443', '192.0.2.10']], 'next': 'cursor-2'},
+            200,
+            {},
+        ),
+        FetcherResponse(
+            {
+                'error': False,
+                'results': [['https://outside.test', 'not-an-ip'], ['mail.example.com', '2001:db8::10']],
+            },
+            200,
+            {},
+        ),
+    ]
 
-    async def fake_fetch_all(urls: list[str], **kwargs: Any) -> list[FetcherResponse]:
-        assert len(urls) == 1
-        assert urls[0].startswith('https://fofa.info/api/v1/search/all?')
-        assert 'key=test-key' in urls[0]
-        assert kwargs['json'] is True
-        assert kwargs['include_metadata'] is True
-        return [FetcherResponse(body={'error': True}, status=429, headers={})]
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**kwargs: Any) -> AsyncIterator[object]:
+        nonlocal session_exited
+        assert kwargs['proxy'] is True
+        try:
+            yield session
+        finally:
+            session_exited = True
 
-    monkeypatch.setattr(fofa.AsyncFetcher, 'fetch_all', fake_fetch_all)
-    search = fofa.SearchFofa('example.com')
+    async def fake_fetch(**kwargs: Any) -> FetcherResponse:
+        calls.append(kwargs)
+        return responses.pop(0)
 
-    with caplog.at_level(logging.INFO, logger=fofa.__name__):
-        await search.process()
+    monkeypatch.setattr(fofa.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(fofa.AsyncFetcher, 'fetch', fake_fetch)
+    search = fofa.SearchFofa('example.com', limit=3)
 
-    assert await search.get_hostnames() == set()
-    assert await search.get_ips() == set()
-    assert 'Fofa request failed with HTTP 429' in caplog.text
+    await search.process(proxy=True)
 
-
-@pytest.mark.asyncio
-@pytest.mark.parametrize('error_message', ['Invalid credentials', '账号无效'])
-async def test_provider_body_authentication_failure_is_actionable(
-    monkeypatch: pytest.MonkeyPatch,
-    caplog: pytest.LogCaptureFixture,
-    error_message: str,
-) -> None:
-    monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: ('test-key', 'operator@example.com'))
-
-    async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]:
-        return [FetcherResponse(body={'error': True, 'errmsg': error_message}, status=200, headers={})]
-
-    monkeypatch.setattr(fofa.AsyncFetcher, 'fetch_all', fake_fetch_all)
-    search = fofa.SearchFofa('example.com')
-
-    with caplog.at_level(logging.INFO, logger=fofa.__name__):
-        await search.process()
-
-    assert 'Fofa API rejected the configured credentials' in caplog.text
-
-
-@pytest.mark.asyncio
-async def test_provider_body_quota_failure_is_actionable(
-    monkeypatch: pytest.MonkeyPatch,
-    caplog: pytest.LogCaptureFixture,
-) -> None:
-    monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: ('test-key', 'operator@example.com'))
-
-    async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]:
-        return [FetcherResponse(body={'error': True, 'errmsg': 'Query quota exhausted'}, status=200, headers={})]
-
-    monkeypatch.setattr(fofa.AsyncFetcher, 'fetch_all', fake_fetch_all)
-    search = fofa.SearchFofa('example.com')
-
-    with caplog.at_level(logging.INFO, logger=fofa.__name__):
-        await search.process()
-
-    assert 'Fofa API quota or plan limit was reached' in caplog.text
+    assert await search.get_hostnames() == {'api.example.com', 'mail.example.com'}
+    assert await search.get_ips() == {'192.0.2.10', '2001:db8::10'}
+    assert search.execution_status == 'partial'
+    assert search.stop_reason == 'invalid-response'
+    assert [call['params']['size'] for call in calls] == [3, 2]
+    assert [call['params'].get('next') for call in calls] == [None, 'cursor-2']
+    assert all(call['url'] == 'https://fofa.info/api/v1/search/next' for call in calls)
+    assert all(call['session'] is session for call in calls)
+    assert session_exited is True
+    assert base64.b64decode(calls[0]['params']['qbase64']).decode() == 'domain="example.com"'
 
 
 @pytest.mark.parametrize(
     'credentials',
-    [('', 'operator@example.com'), ('test-key', '   ')],
+    [(None, 'operator@example.com'), ('', 'operator@example.com'), ('test-key', '   ')],
 )
-def test_empty_credentials_are_rejected(
+def test_missing_or_blank_credentials_fail_closed(
     monkeypatch: pytest.MonkeyPatch,
-    credentials: tuple[str, str],
+    credentials: tuple[str | None, str],
 ) -> None:
     monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: credentials)
-
     with pytest.raises(MissingKey):
-        fofa.SearchFofa('example.com')
+        fofa.SearchFofa('example.com', 10)
 
 
+@pytest.mark.parametrize('limit', [0, -1, True, 1.5])
+def test_limit_must_be_positive(monkeypatch: pytest.MonkeyPatch, limit: Any) -> None:
+    monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: ('test-key', 'operator@example.com'))
+    with pytest.raises(ValueError, match='positive integer'):
+        fofa.SearchFofa('example.com', limit)
+
+
+@pytest.mark.parametrize(
+    ('response', 'status', 'reason'),
+    [
+        (None, 'failed', 'transport-error'),
+        (FetcherResponse({}, 401, {}), 'failed', 'access-denied'),
+        (FetcherResponse({}, 429, {}), 'rate-limited', 'http-429'),
+        (FetcherResponse({}, 503, {}), 'failed', 'http-503'),
+        (FetcherResponse([], 200, {}), 'failed', 'invalid-response'),
+        (FetcherResponse({'error': True, 'errmsg': 'Invalid credentials'}, 200, {}), 'failed', 'access-denied'),
+        (FetcherResponse({'error': True, 'errmsg': 'Quota limit reached'}, 200, {}), 'failed', 'quota-exhausted'),
+        (FetcherResponse({'error': False, 'results': {}}, 200, {}), 'failed', 'invalid-response'),
+    ],
+)
 @pytest.mark.asyncio
-async def test_malformed_results_are_reported(
+async def test_failures_are_structured(
     monkeypatch: pytest.MonkeyPatch,
-    caplog: pytest.LogCaptureFixture,
+    response: FetcherResponse | None,
+    status: str,
+    reason: str,
 ) -> None:
     monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: ('test-key', 'operator@example.com'))
 
-    async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]:
-        return [FetcherResponse(body={'error': False, 'results': 7}, status=200, headers={})]
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        yield object()
 
-    monkeypatch.setattr(fofa.AsyncFetcher, 'fetch_all', fake_fetch_all)
-    search = fofa.SearchFofa('example.com')
+    async def fake_fetch(**_kwargs: Any) -> FetcherResponse | None:
+        return response
 
-    with caplog.at_level(logging.INFO, logger=fofa.__name__):
-        await search.process()
+    monkeypatch.setattr(fofa.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(fofa.AsyncFetcher, 'fetch', fake_fetch)
+    search = fofa.SearchFofa('example.com', 10)
+    await search.process()
 
-    assert await search.get_hostnames() == set()
-    assert await search.get_ips() == set()
-    assert 'Fofa returned malformed results' in caplog.text
+    assert search.execution_status == status
+    assert search.stop_reason == reason
 
 
 @pytest.mark.asyncio
-async def test_success_preserves_scoped_hosts_and_valid_ips(monkeypatch: pytest.MonkeyPatch) -> None:
+async def test_repeated_cursor_stops_without_a_third_request(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: ('test-key', 'operator@example.com'))
+    responses = [
+        FetcherResponse({'error': False, 'results': [['api.example.com', '192.0.2.1']], 'next': 'same'}, 200, {}),
+        FetcherResponse({'error': False, 'results': [['mail.example.com', '192.0.2.2']], 'next': 'same'}, 200, {}),
+    ]
 
-    async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]:
-        return [
-            FetcherResponse(
-                body={
-                    'error': False,
-                    'results': [
-                        ['https://API.Example.COM:443', '192.0.2.10'],
-                        ['https://outside.test', 'not-an-ip'],
-                        ['malformed'],
-                    ],
-                },
-                status=200,
-                headers={},
-            )
-        ]
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        yield object()
 
-    monkeypatch.setattr(fofa.AsyncFetcher, 'fetch_all', fake_fetch_all)
-    search = fofa.SearchFofa('example.com')
+    async def fake_fetch(**_kwargs: Any) -> FetcherResponse:
+        return responses.pop(0)
 
+    monkeypatch.setattr(fofa.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(fofa.AsyncFetcher, 'fetch', fake_fetch)
+    search = fofa.SearchFofa('example.com', 10)
+    await search.process()
+
+    assert search.execution_status == 'partial'
+    assert search.stop_reason == 'repeated-cursor'
+    assert responses == []
+
+
+@pytest.mark.asyncio
+async def test_malformed_url_does_not_discard_later_valid_rows(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: ('test-key', 'operator@example.com'))
+    response = FetcherResponse(
+        {
+            'error': False,
+            'results': [
+                ['https://[invalid', 'not-an-ip'],
+                ['https://api.example.com', '192.0.2.10'],
+            ],
+        },
+        200,
+        {},
+    )
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        yield object()
+
+    async def fake_fetch(**_kwargs: Any) -> FetcherResponse:
+        return response
+
+    monkeypatch.setattr(fofa.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(fofa.AsyncFetcher, 'fetch', fake_fetch)
+    search = fofa.SearchFofa('example.com', 2)
     await search.process()
 
     assert await search.get_hostnames() == {'api.example.com'}
     assert await search.get_ips() == {'192.0.2.10'}
+    assert search.execution_status == 'partial'
+    assert search.stop_reason == 'invalid-response'
+
+
+@pytest.mark.asyncio
+async def test_cancellation_propagates(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: ('test-key', 'operator@example.com'))
+    session_exited = False
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        nonlocal session_exited
+        try:
+            yield object()
+        finally:
+            session_exited = True
+
+    async def fake_fetch(**_kwargs: Any) -> FetcherResponse:
+        raise asyncio.CancelledError
+
+    monkeypatch.setattr(fofa.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(fofa.AsyncFetcher, 'fetch', fake_fetch)
+    with pytest.raises(asyncio.CancelledError):
+        await fofa.SearchFofa('example.com', 10).process()
+    assert session_exited is True

tests/discovery/test_fullhuntsearch.py

@@ -1,4 +1,7 @@
+import asyncio
+import contextlib
 import logging
+from collections.abc import AsyncIterator
 from typing import Any
 
 import pytest
@@ -8,6 +11,55 @@ from theHarvester.discovery.constants import MissingKey
 from theHarvester.lib.core import FetcherResponse
 
 
+@pytest.mark.asyncio
+async def test_process_reuses_one_session_for_fallback_requests(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(fullhuntsearch.Core, 'fullhunt_key', lambda: 'test-key')
+    session = object()
+    session_exited = False
+    open_calls: list[dict[str, Any]] = []
+    calls: list[tuple[list[str], dict[str, Any]]] = []
+    responses = [
+        FetcherResponse({'hosts': []}, 200, {}),
+        FetcherResponse({'hosts': ['api.example.com']}, 200, {}),
+    ]
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**kwargs: Any) -> AsyncIterator[object]:
+        nonlocal session_exited
+        open_calls.append(kwargs)
+        try:
+            yield session
+        finally:
+            session_exited = True
+
+    async def fake_fetch_all(urls: list[str], **kwargs: Any) -> list[FetcherResponse]:
+        calls.append((urls, kwargs))
+        return [responses.pop(0)]
+
+    monkeypatch.setattr(fullhuntsearch.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(fullhuntsearch.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    search = fullhuntsearch.SearchFullHunt('example.com')
+
+    await search.process(proxy=True)
+
+    assert await search.get_hostnames() == ['api.example.com']
+    assert [urls for urls, _kwargs in calls] == [
+        ['https://fullhunt.io/api/v1/domain/example.com/details'],
+        ['https://fullhunt.io/api/v1/domain/example.com/subdomains'],
+    ]
+    assert all(kwargs['session'] is session for _urls, kwargs in calls)
+    assert open_calls == [
+        {
+            'headers': {'User-Agent': fullhuntsearch.Core.get_user_agent(), 'X-API-KEY': 'test-key'},
+            'proxy': True,
+            'request_timeout': 60,
+        }
+    ]
+    assert session_exited is True
+    assert search.execution_status == 'completed'
+    assert search.stop_reason is None
+
+
 @pytest.mark.asyncio
 async def test_http_failure_is_reported_without_results(
     monkeypatch: pytest.MonkeyPatch,
@@ -31,7 +83,8 @@ async def test_http_failure_is_reported_without_results(
     assert await search.get_hostnames() == []
     assert await search.get_ips() == []
     assert requests == ['https://fullhunt.io/api/v1/domain/example.com/details']
-    assert 'FullHunt request failed with HTTP 403' in caplog.text
+    assert search.execution_status == 'failed'
+    assert search.stop_reason == 'access-denied'
 
 
 @pytest.mark.parametrize('key', ['', '   '])
@@ -72,7 +125,8 @@ async def test_malformed_domain_details_are_reported_without_fallback(
 
     assert await search.get_hostnames() == []
     assert requests == ['https://fullhunt.io/api/v1/domain/example.com/details']
-    assert 'FullHunt returned malformed domain details' in caplog.text
+    assert search.execution_status == 'failed'
+    assert search.stop_reason == 'invalid-response'
 
 
 @pytest.mark.asyncio
@@ -108,6 +162,8 @@ async def test_malformed_host_does_not_hide_later_valid_results(
     assert await search.get_hostnames() == ['api.example.com']
     assert await search.get_ips() == ['192.0.2.20']
     assert caplog.text.count('FullHunt ignored a malformed host item') == 4
+    assert search.execution_status == 'partial'
+    assert search.stop_reason == 'invalid-response'
 
 
 @pytest.mark.asyncio
@@ -131,7 +187,8 @@ async def test_malformed_subdomain_fallback_is_reported(
         await search.process()
 
     assert await search.get_hostnames() == []
-    assert 'FullHunt returned malformed subdomains' in caplog.text
+    assert search.execution_status == 'failed'
+    assert search.stop_reason == 'invalid-response'
 
 
 @pytest.mark.asyncio
@@ -156,6 +213,8 @@ async def test_fallback_ignores_malformed_and_out_of_scope_hosts(
 
     assert await search.get_hostnames() == ['api.example.com']
     assert caplog.text.count('FullHunt ignored a malformed subdomain item') == 2
+    assert search.execution_status == 'partial'
+    assert search.stop_reason == 'invalid-response'
 
 
 @pytest.mark.asyncio
@@ -192,3 +251,60 @@ async def test_nested_results_use_normalized_hostname(monkeypatch: pytest.Monkey
     assert await search.get_geo_info() == {'api.example.com': {'country': 'US'}}
     assert await search.get_cloud_info() == {'api.example.com': {'provider': 'example'}}
     assert await search.get_certificate_info() == [{'issuer': 'Example CA', 'hostname': 'api.example.com'}]
+    assert search.execution_status == 'completed'
+    assert search.stop_reason is None
+
+
+@pytest.mark.parametrize(
+    ('response', 'status', 'reason'),
+    [
+        (None, 'failed', 'transport-error'),
+        (FetcherResponse({}, 429, {}), 'rate-limited', 'http-429'),
+        (FetcherResponse({}, 503, {}), 'failed', 'http-503'),
+        (FetcherResponse([], 200, {}), 'failed', 'invalid-response'),
+    ],
+)
+@pytest.mark.asyncio
+async def test_failures_are_structured(
+    monkeypatch: pytest.MonkeyPatch,
+    response: FetcherResponse | None,
+    status: str,
+    reason: str,
+) -> None:
+    monkeypatch.setattr(fullhuntsearch.Core, 'fullhunt_key', lambda: 'test-key')
+
+    async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse | None]:
+        return [response]
+
+    monkeypatch.setattr(fullhuntsearch.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    search = fullhuntsearch.SearchFullHunt('example.com')
+    await search.process()
+
+    assert search.execution_status == status
+    assert search.stop_reason == reason
+
+
+@pytest.mark.asyncio
+async def test_cancellation_propagates(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(fullhuntsearch.Core, 'fullhunt_key', lambda: 'test-key')
+    session_exited = False
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        nonlocal session_exited
+        try:
+            yield object()
+        finally:
+            session_exited = True
+
+    async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]:
+        raise asyncio.CancelledError
+
+    monkeypatch.setattr(fullhuntsearch.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    monkeypatch.setattr(fullhuntsearch.AsyncFetcher, 'open_session', fake_open_session)
+    with pytest.raises(asyncio.CancelledError):
+        await fullhuntsearch.SearchFullHunt('example.com').process()
+    assert session_exited is True
+
+
+pytestmark = pytest.mark.provider_contract('fullhunt')

tests/discovery/test_githubcode_contract.py

@@ -206,3 +206,6 @@ async def test_github_code_malformed_page_terminates_without_following_paginatio
     assert requested_urls == ['https://api.github.com/search/code?q="example.com"&page=1']
     assert await search.get_emails() == set()
     assert await search.get_hostnames() == []
+
+
+pytestmark = pytest.mark.provider_contract('github-code')

tests/discovery/test_gitlabsearch.py

@@ -174,3 +174,6 @@ async def test_gitlab_urls_reach_completed_jsonl(
     assert completed_results[0].results == (('url', 'https://gitlab.com/group/project'),)
     records = [json.loads(line) for line in report.with_suffix('.jsonl').read_text().splitlines()]
     assert {'type': 'url', 'value': 'https://gitlab.com/group/project', 'sources': ['gitlab']} in records
+
+
+pytestmark = pytest.mark.provider_contract('gitlab')

tests/discovery/test_haveibeenpwned.py

@@ -211,3 +211,6 @@ async def test_public_breach_names_reach_completed_result_and_jsonl(
     records = [json.loads(line) for line in report.with_suffix('.jsonl').read_text().splitlines()]
     assert {'type': 'breach', 'value': 'Adobe', 'sources': ['haveibeenpwned']} in records
     assert {'type': 'breach', 'value': 'ExampleBreach', 'sources': ['haveibeenpwned']} in records
+
+
+pytestmark = pytest.mark.provider_contract('haveibeenpwned')

tests/discovery/test_hibpverified.py

@@ -190,3 +190,6 @@ async def test_verified_domain_results_reach_completed_jsonl_and_sqlite_handoff(
     records = [json.loads(line) for line in report.with_suffix('.jsonl').read_text().splitlines()]
     assert {'type': 'breach', 'value': 'ExampleBreach', 'sources': ['hibpverified']} in records
     assert {'type': 'email', 'value': 'alice@example.com', 'sources': ['hibpverified']} in records
+
+
+pytestmark = pytest.mark.provider_contract('hibpverified')

tests/discovery/test_hudsonrock.py

@@ -259,3 +259,6 @@ async def test_infostealer_data_reaches_completed_result_and_jsonl(
     assert ('infostealer', stealer) in completed_results[0].results
     records = [json.loads(line) for line in report.with_suffix('.jsonl').read_text().splitlines()]
     assert {'type': 'infostealer', 'value': stealer, 'sources': ['hudsonrock']} in records
+
+
+pytestmark = pytest.mark.provider_contract('hudsonrock')

tests/discovery/test_hunterhow.py

@@ -0,0 +1,185 @@
+import asyncio
+import base64
+import contextlib
+from collections.abc import AsyncIterator
+from typing import Any
+
+import pytest
+
+from theHarvester.discovery import searchhunterhow
+from theHarvester.discovery.constants import MissingKey
+from theHarvester.lib.core import FetcherResponse
+
+
+@pytest.mark.provider_contract('hunterhow')
+@pytest.mark.asyncio
+async def test_process_paginates_to_limit_and_keeps_scoped_hostnames(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(searchhunterhow.Core, 'hunterhow_key', staticmethod(lambda: 'test-key'))
+    session = object()
+    session_exited = False
+    session_options: list[dict[str, Any]] = []
+    calls: list[dict[str, Any]] = []
+    delays: list[float] = []
+    responses = [
+        FetcherResponse(
+            body={
+                'code': 200,
+                'data': {
+                    'total': 3,
+                    'list': [{'domain': 'API.Example.COM.'}, {'domain': 'outside.test'}],
+                },
+            },
+            status=200,
+            headers={},
+        ),
+        FetcherResponse(
+            body={'code': 200, 'data': {'total': 3, 'list': [{'domain': 'www.example.com'}]}},
+            status=200,
+            headers={},
+        ),
+    ]
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**kwargs: Any) -> AsyncIterator[object]:
+        nonlocal session_exited
+        session_options.append(kwargs)
+        try:
+            yield session
+        finally:
+            session_exited = True
+
+    async def fake_fetch(**kwargs: Any) -> FetcherResponse:
+        calls.append(kwargs)
+        return responses.pop(0)
+
+    async def fake_sleep(delay: float) -> None:
+        delays.append(delay)
+
+    monkeypatch.setattr(searchhunterhow.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(searchhunterhow.AsyncFetcher, 'fetch', fake_fetch)
+    monkeypatch.setattr(searchhunterhow.asyncio, 'sleep', fake_sleep)
+    search = searchhunterhow.SearchHunterHow('example.com', limit=3)
+
+    await search.process(proxy=True)
+
+    assert await search.get_hostnames() == {'api.example.com', 'www.example.com'}
+    assert session_options == [{'headers': {'User-Agent': searchhunterhow.Core.get_user_agent()}, 'proxy': True}]
+    assert [call['params']['page'] for call in calls] == [1, 2]
+    assert all(call['params']['page_size'] == 10 for call in calls)
+    assert all(call['params']['api-key'] == 'test-key' for call in calls)
+    assert all(base64.urlsafe_b64decode(call['params']['query']).decode() == 'domain.suffix="example.com"' for call in calls)
+    assert all(call['session'] is session for call in calls)
+    assert all(call['url'] == 'https://api.hunter.how/search' for call in calls)
+    assert all(call['include_metadata'] is True for call in calls)
+    assert delays == [2.0]
+    assert search.execution_status == 'completed'
+    assert search.stop_reason is None
+    assert session_exited is True
+
+
+@pytest.mark.parametrize('key', [None, '', '   '])
+def test_missing_or_empty_key_fails_before_transport(monkeypatch: pytest.MonkeyPatch, key: str | None) -> None:
+    monkeypatch.setattr(searchhunterhow.Core, 'hunterhow_key', staticmethod(lambda: key))
+
+    with pytest.raises(MissingKey):
+        searchhunterhow.SearchHunterHow('example.com', limit=10)
+
+
+@pytest.mark.parametrize(
+    ('response', 'execution_status', 'stop_reason'),
+    [
+        (None, 'failed', 'transport-error'),
+        (FetcherResponse({}, 401, {}), 'failed', 'access-denied'),
+        (FetcherResponse({}, 403, {}), 'failed', 'access-denied'),
+        (FetcherResponse({}, 429, {}), 'rate-limited', 'http-429'),
+        (FetcherResponse({}, 503, {}), 'failed', 'http-503'),
+        (FetcherResponse({'code': 40001}, 200, {}), 'failed', 'access-denied'),
+        (FetcherResponse({'code': 200, 'data': {'total': 'three', 'list': {}}}, 200, {}), 'failed', 'invalid-response'),
+    ],
+)
+@pytest.mark.asyncio
+async def test_failed_response_is_attributed(
+    monkeypatch: pytest.MonkeyPatch,
+    response: FetcherResponse | None,
+    execution_status: str,
+    stop_reason: str,
+) -> None:
+    monkeypatch.setattr(searchhunterhow.Core, 'hunterhow_key', staticmethod(lambda: 'test-key'))
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        yield object()
+
+    async def fake_fetch(**_kwargs: Any) -> FetcherResponse | None:
+        return response
+
+    monkeypatch.setattr(searchhunterhow.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(searchhunterhow.AsyncFetcher, 'fetch', fake_fetch)
+    search = searchhunterhow.SearchHunterHow('example.com', limit=10)
+
+    await search.process()
+
+    assert await search.get_hostnames() == set()
+    assert search.execution_status == execution_status
+    assert search.stop_reason == stop_reason
+
+
+def test_limit_must_be_positive(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(searchhunterhow.Core, 'hunterhow_key', staticmethod(lambda: 'test-key'))
+
+    with pytest.raises(ValueError, match='positive integer'):
+        searchhunterhow.SearchHunterHow('example.com', limit=0)
+
+
+@pytest.mark.asyncio
+async def test_early_malformed_rows_and_later_valid_evidence_are_partial(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(searchhunterhow.Core, 'hunterhow_key', staticmethod(lambda: 'test-key'))
+    responses = [
+        FetcherResponse({'code': 200, 'data': {'total': 2, 'list': [7]}}, 200, {}),
+        FetcherResponse({'code': 200, 'data': {'total': 2, 'list': [{'domain': 'api.example.com'}]}}, 200, {}),
+    ]
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        yield object()
+
+    async def fake_fetch(**_kwargs: Any) -> FetcherResponse:
+        return responses.pop(0)
+
+    async def fake_sleep(_delay: float) -> None:
+        return None
+
+    monkeypatch.setattr(searchhunterhow.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(searchhunterhow.AsyncFetcher, 'fetch', fake_fetch)
+    monkeypatch.setattr(searchhunterhow.asyncio, 'sleep', fake_sleep)
+    search = searchhunterhow.SearchHunterHow('example.com', limit=2)
+
+    await search.process()
+
+    assert await search.get_hostnames() == {'api.example.com'}
+    assert search.execution_status == 'partial'
+    assert search.stop_reason == 'invalid-response'
+
+
+@pytest.mark.asyncio
+async def test_cancellation_closes_provider_session(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(searchhunterhow.Core, 'hunterhow_key', staticmethod(lambda: 'test-key'))
+    session_exited = False
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        nonlocal session_exited
+        try:
+            yield object()
+        finally:
+            session_exited = True
+
+    async def fake_fetch(**_kwargs: Any) -> FetcherResponse:
+        raise asyncio.CancelledError('operator-stop')
+
+    monkeypatch.setattr(searchhunterhow.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(searchhunterhow.AsyncFetcher, 'fetch', fake_fetch)
+
+    with pytest.raises(asyncio.CancelledError, match='operator-stop'):
+        await searchhunterhow.SearchHunterHow('example.com', limit=10).process()
+    assert session_exited is True

tests/discovery/test_huntersearch.py

@@ -33,6 +33,8 @@ async def test_hunter_http_failures_return_no_results(monkeypatch, caplog, statu
 
     assert await search.get_emails() == []
     assert await search.get_hostnames() == []
+
+
     assert f'Hunter request failed with HTTP {status}' in caplog.text
     assert 'provider detail' not in caplog.text
 
@@ -221,3 +223,6 @@ async def test_paid_hunter_search_stops_before_exceeding_quota(monkeypatch) -> N
     ]
     assert await search.get_emails() == []
     assert await search.get_hostnames() == []
+
+
+pytestmark = pytest.mark.provider_contract('hunter')

tests/discovery/test_intelxsearch.py

@@ -169,3 +169,6 @@ async def test_orchestrator_stores_intelx_subdomains_without_dns(monkeypatch: py
 
     assert results[-1] == ['api.example.com']
     assert completed_results[0].observations == (ResultObservation('intelx', 'hostname', 'api.example.com'),)
+
+
+pytestmark = pytest.mark.provider_contract('intelx')

tests/discovery/test_leakix.py

@@ -124,3 +124,6 @@ async def test_unusable_responses_fail_closed_without_logging_provider_detail(
     assert 'provider-secret-auth-detail' not in caplog.text
     if expected_log is not None:
         assert expected_log in caplog.text
+
+
+pytestmark = pytest.mark.provider_contract('leakix')

tests/discovery/test_leaklookup.py

@@ -196,3 +196,6 @@ async def test_leaklookup_emails_and_breaches_reach_completed_result_and_jsonl(m
     records = [json.loads(line) for line in report.with_suffix('.jsonl').read_text().splitlines()]
     assert {'type': 'breach', 'value': 'Example Breach', 'sources': ['leaklookup']} in records
     assert {'type': 'email', 'value': 'alice@example.com', 'sources': ['leaklookup']} in records
+
+
+pytestmark = pytest.mark.provider_contract('leaklookup')

tests/discovery/test_netlas.py

@@ -0,0 +1,169 @@
+from __future__ import annotations
+
+import asyncio
+import contextlib
+from typing import TYPE_CHECKING, Any
+
+import pytest
+
+from theHarvester.discovery import netlas
+from theHarvester.discovery.constants import MissingKey
+from theHarvester.lib.core import FetcherResponse
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+
+
+@pytest.mark.provider_contract('netlas')
+@pytest.mark.asyncio
+async def test_process_uses_current_api_contract_and_keeps_scoped_results(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(netlas.Core, 'netlas_key', staticmethod(lambda: 'test-key'))
+    session = object()
+    session_exited = False
+    session_options: list[dict[str, Any]] = []
+    calls: list[dict[str, Any]] = []
+    responses = [
+        FetcherResponse(
+            [
+                {'data': {'domain': 'API.Example.COM.'}},
+                {'data': {'domain': 'outside.test'}},
+                {'data': {'domain': 'www.example.com'}},
+            ],
+            200,
+            {},
+        ),
+    ]
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**kwargs: Any) -> AsyncIterator[object]:
+        nonlocal session_exited
+        session_options.append(kwargs)
+        try:
+            yield session
+        finally:
+            session_exited = True
+
+    async def fake_post_fetch(*args: Any, **kwargs: Any) -> FetcherResponse:
+        calls.append({'url': args[0], **kwargs})
+        return responses.pop(0)
+
+    monkeypatch.setattr(netlas.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(netlas.AsyncFetcher, 'post_fetch', fake_post_fetch)
+
+    search = netlas.SearchNetlas('example.com', limit=2)
+    await search.process(proxy=True)
+
+    assert await search.get_hostnames() == {'api.example.com'}
+    assert search.execution_status == 'completed'
+    assert search.stop_reason is None
+    assert session_options == [{'headers': {'Authorization': 'Bearer test-key'}, 'proxy': True}]
+    assert calls[0]['url'] == 'https://app.netlas.io/api/domains/download/'
+    assert calls[0]['session'] is session
+    assert calls[0]['json_body'] == {
+        'q': '*.example.com',
+        'size': 2,
+        'fields': ['domain'],
+        'source_type': 'include',
+    }
+    assert session_exited is True
+
+
+@pytest.mark.parametrize('key', [None, '', '   '])
+def test_missing_or_blank_key_fails_closed(monkeypatch: pytest.MonkeyPatch, key: str | None) -> None:
+    monkeypatch.setattr(netlas.Core, 'netlas_key', staticmethod(lambda: key))
+    with pytest.raises(MissingKey):
+        netlas.SearchNetlas('example.com', limit=10)
+
+
+@pytest.mark.parametrize('limit', [0, -1, True, 1.5])
+def test_limit_must_be_a_positive_integer(monkeypatch: pytest.MonkeyPatch, limit: Any) -> None:
+    monkeypatch.setattr(netlas.Core, 'netlas_key', staticmethod(lambda: 'test-key'))
+    with pytest.raises(ValueError, match='positive integer'):
+        netlas.SearchNetlas('example.com', limit=limit)
+
+
+@pytest.mark.parametrize(
+    ('response', 'status', 'reason'),
+    [
+        (None, 'failed', 'transport-error'),
+        (FetcherResponse({}, 401, {}), 'failed', 'access-denied'),
+        (FetcherResponse({}, 402, {}), 'failed', 'quota-exhausted'),
+        (FetcherResponse({}, 429, {}), 'rate-limited', 'http-429'),
+        (FetcherResponse({}, 503, {}), 'failed', 'http-503'),
+        (FetcherResponse(None, 200, {}), 'failed', 'invalid-response'),
+        (FetcherResponse({'count': 'many'}, 200, {}), 'failed', 'invalid-response'),
+    ],
+)
+@pytest.mark.asyncio
+async def test_download_failures_are_truthful(
+    monkeypatch: pytest.MonkeyPatch,
+    response: FetcherResponse | None,
+    status: str,
+    reason: str,
+) -> None:
+    monkeypatch.setattr(netlas.Core, 'netlas_key', staticmethod(lambda: 'test-key'))
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        yield object()
+
+    async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse | None:
+        return response
+
+    monkeypatch.setattr(netlas.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(netlas.AsyncFetcher, 'post_fetch', fake_post_fetch)
+    search = netlas.SearchNetlas('example.com', limit=10)
+
+    await search.process()
+
+    assert search.execution_status == status
+    assert search.stop_reason == reason
+
+
+@pytest.mark.asyncio
+async def test_malformed_download_rows_preserve_valid_partial_results(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(netlas.Core, 'netlas_key', staticmethod(lambda: 'test-key'))
+    responses = [
+        FetcherResponse([{'data': {'domain': 'ok.example.com'}}, {'data': {'domain': 7}}], 200, {}),
+    ]
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        yield object()
+
+    async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        return responses.pop(0)
+
+    monkeypatch.setattr(netlas.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(netlas.AsyncFetcher, 'post_fetch', fake_post_fetch)
+    search = netlas.SearchNetlas('example.com', limit=10)
+
+    await search.process()
+
+    assert await search.get_hostnames() == {'ok.example.com'}
+    assert search.execution_status == 'partial'
+    assert search.stop_reason == 'invalid-response'
+
+
+@pytest.mark.asyncio
+async def test_cancellation_closes_provider_session(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(netlas.Core, 'netlas_key', staticmethod(lambda: 'test-key'))
+    session_exited = False
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        nonlocal session_exited
+        try:
+            yield object()
+        finally:
+            session_exited = True
+
+    async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        raise asyncio.CancelledError('operator-stop')
+
+    monkeypatch.setattr(netlas.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(netlas.AsyncFetcher, 'post_fetch', fake_post_fetch)
+
+    with pytest.raises(asyncio.CancelledError, match='operator-stop'):
+        await netlas.SearchNetlas('example.com', limit=10).process()
+    assert session_exited is True

tests/discovery/test_onyphe.py

@@ -1,162 +1,259 @@
 import asyncio
-import logging
+import contextlib
+from collections.abc import AsyncIterator
 from typing import Any
 
 import pytest
 
 from theHarvester.discovery import onyphe
+from theHarvester.discovery.constants import MissingKey
 from theHarvester.lib.core import FetcherResponse
 
 
+@pytest.mark.provider_contract('onyphe')
 @pytest.mark.asyncio
-async def test_process_keeps_only_canonical_individual_ips_and_preserves_routes(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
+async def test_process_paginates_to_limit_and_preserves_all_routes(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.setattr(onyphe.Core, 'onyphe_key', lambda: 'test-key')
-    monkeypatch.setattr(onyphe.Core, 'get_user_agent', lambda: 'test-agent')
-    captured: dict[str, Any] = {}
+    session = object()
+    session_exited = False
+    calls: list[dict[str, Any]] = []
+    responses = [
+        FetcherResponse(
+            {
+                'text': 'Success',
+                'total': 3,
+                'max_page': 2,
+                'results': [
+                    {
+                        'ip': '192.0.2.10',
+                        'alternativeip': ['2001:0db8::10'],
+                        'url': ['https://www.example.com/path'],
+                        'asn': 'AS64496',
+                        'organization': 'Example Physical Network',
+                        'geolocus': {
+                            'asn': 'AS64497',
+                            'organization': 'Example Logical Network',
+                            'domain': ['geo.example.com'],
+                        },
+                        'hostname': ['api.example.com'],
+                    },
+                    {'hostname': ['outside.test']},
+                ],
+            },
+            200,
+            {},
+        ),
+        FetcherResponse(
+            {'text': 'Success', 'total': 3, 'max_page': 2, 'results': [{'subdomains': ['mail.example.com']}]},
+            200,
+            {},
+        ),
+    ]
 
-    async def fake_fetch_all(urls: list[str], **kwargs: Any) -> list[FetcherResponse]:
-        assert urls == ['https://www.onyphe.io/api/v2/search/?q=domain:example.com']
-        captured.update(kwargs)
-        return [
-            FetcherResponse(
-                body={
-                    'text': 'Success',
-                    'results': [
-                        {
-                            'ip': '192.0.2.10',
-                            'alternativeip': ['2001:0db8::10'],
-                            'subnet': '192.0.2.0/24',
-                            'url': ['https://www.example.com/path'],
-                            'asn': 'AS64496',
-                            'organization': 'Example Physical Network',
-                            'geolocus': {
-                                'asn': 'AS64497',
-                                'organization': 'Example Logical Network',
-                                'subnet': '198.51.100.0/24',
-                                'domain': ['geo.example.com'],
-                            },
-                            'hostname': ['api.example.com'],
-                        }
-                    ],
-                },
-                status=200,
-                headers={},
-            )
-        ]
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**kwargs: Any) -> AsyncIterator[object]:
+        nonlocal session_exited
+        assert kwargs['proxy'] is True
+        assert kwargs['headers']['Authorization'] == 'bearer test-key'
+        try:
+            yield session
+        finally:
+            session_exited = True
 
-    monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch_all', fake_fetch_all)
-    search = onyphe.SearchOnyphe('example.com')
+    async def fake_fetch(**kwargs: Any) -> FetcherResponse:
+        calls.append(kwargs)
+        return responses.pop(0)
 
+    monkeypatch.setattr(onyphe.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch', fake_fetch)
+    search = onyphe.SearchOnyphe('example.com', limit=3)
     await search.process(proxy=True)
 
-    assert captured['proxy'] is True
-    assert captured['json'] is True
-    assert captured['include_metadata'] is True
-    assert captured['headers']['Authorization'] == 'bearer test-key'
     assert await search.get_ips() == {'192.0.2.10', '2001:db8::10'}
-    assert await search.get_hostnames() == {'api.example.com', 'geo.example.com', 'www.example.com'}
+    assert await search.get_hostnames() == {'api.example.com', 'geo.example.com', 'mail.example.com', 'www.example.com'}
     assert await search.get_asns() == {'AS64496', 'AS64497'}
     assert {
-        (
-            observation.asn,
-            observation.organization_label,
-            observation.subject_kind,
-            observation.subject_value,
-        )
-        for observation in await search.get_asn_attributions()
+        (item.asn, item.organization_label, item.subject_kind, item.subject_value) for item in await search.get_asn_attributions()
     } == {
-        (asn, organization, subject_kind, subject_value)
-        for asn, organization in {
-            ('AS64496', 'Example Physical Network'),
-            ('AS64497', 'Example Logical Network'),
-        }
-        for subject_kind, subject_value in {('ip', '192.0.2.10')}
+        ('AS64496', 'Example Physical Network', 'ip', '192.0.2.10'),
+        ('AS64497', 'Example Logical Network', 'ip', '192.0.2.10'),
     }
+    assert [call['params']['page'] for call in calls] == [1, 2]
+    assert [call['params']['size'] for call in calls] == [3, 3]
+    assert all(call['session'] is session for call in calls)
+    assert search.execution_status == 'completed'
+    assert search.stop_reason is None
+    assert session_exited is True
+
+
+@pytest.mark.parametrize('key', [None, '', '   '])
+def test_missing_or_blank_key_fails_closed(monkeypatch: pytest.MonkeyPatch, key: str | None) -> None:
+    monkeypatch.setattr(onyphe.Core, 'onyphe_key', lambda: key)
+    with pytest.raises(MissingKey):
+        onyphe.SearchOnyphe('example.com', 10)
+
+
+@pytest.mark.parametrize('limit', [0, -1, True, 1.5])
+def test_limit_must_be_positive(monkeypatch: pytest.MonkeyPatch, limit: Any) -> None:
+    monkeypatch.setattr(onyphe.Core, 'onyphe_key', lambda: 'test-key')
+    with pytest.raises(ValueError, match='positive integer'):
+        onyphe.SearchOnyphe('example.com', limit)
+
+
+@pytest.mark.parametrize(
+    ('response', 'status', 'reason'),
+    [
+        (None, 'failed', 'transport-error'),
+        (FetcherResponse({}, 401, {}), 'failed', 'access-denied'),
+        (FetcherResponse({}, 429, {}), 'rate-limited', 'http-429'),
+        (FetcherResponse({}, 503, {}), 'failed', 'http-503'),
+        (FetcherResponse([], 200, {}), 'failed', 'invalid-response'),
+        (FetcherResponse({'text': 'Denied'}, 200, {}), 'failed', 'provider-error'),
+        (FetcherResponse({'text': 'Success', 'results': {}}, 200, {}), 'failed', 'invalid-response'),
+    ],
+)
+@pytest.mark.asyncio
+async def test_failures_are_structured(
+    monkeypatch: pytest.MonkeyPatch,
+    response: FetcherResponse | None,
+    status: str,
+    reason: str,
+) -> None:
+    monkeypatch.setattr(onyphe.Core, 'onyphe_key', lambda: 'test-key')
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        yield object()
+
+    async def fake_fetch(**_kwargs: Any) -> FetcherResponse | None:
+        return response
+
+    monkeypatch.setattr(onyphe.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch', fake_fetch)
+    search = onyphe.SearchOnyphe('example.com', 10)
+    await search.process()
+
+    assert search.execution_status == status
+    assert search.stop_reason == reason
+
+
+@pytest.mark.asyncio
+async def test_later_page_failure_preserves_partial_evidence(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(onyphe.Core, 'onyphe_key', lambda: 'test-key')
+    responses = [
+        FetcherResponse(
+            {'text': 'Success', 'total': 2, 'max_page': 2, 'results': [{'ip': '192.0.2.10'}]},
+            200,
+            {},
+        ),
+        FetcherResponse({}, 429, {}),
+    ]
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        yield object()
+
+    async def fake_fetch(**_kwargs: Any) -> FetcherResponse:
+        return responses.pop(0)
+
+    monkeypatch.setattr(onyphe.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch', fake_fetch)
+    search = onyphe.SearchOnyphe('example.com', 2)
+    await search.process()
+
+    assert await search.get_ips() == {'192.0.2.10'}
+    assert search.execution_status == 'partial'
+    assert search.stop_reason == 'http-429'
+
+
+@pytest.mark.asyncio
+async def test_operator_limit_is_not_reported_as_a_provider_limit(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(onyphe.Core, 'onyphe_key', lambda: 'test-key')
+    response = FetcherResponse(
+        {
+            'text': 'Success',
+            'total': 3,
+            'max_page': 2,
+            'results': [{'ip': '192.0.2.10'}, {'ip': '192.0.2.11'}],
+        },
+        200,
+        {},
+    )
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        yield object()
+
+    async def fake_fetch(**_kwargs: Any) -> FetcherResponse:
+        return response
+
+    monkeypatch.setattr(onyphe.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch', fake_fetch)
+    search = onyphe.SearchOnyphe('example.com', 2)
+    await search.process()
+
+    assert await search.get_ips() == {'192.0.2.10', '192.0.2.11'}
     assert search.execution_status == 'completed'
     assert search.stop_reason is None
 
 
 @pytest.mark.asyncio
-async def test_valid_empty_response_is_completed(monkeypatch: pytest.MonkeyPatch) -> None:
+async def test_search_api_reports_its_documented_total_boundary(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.setattr(onyphe.Core, 'onyphe_key', lambda: 'test-key')
+    calls: list[dict[str, Any]] = []
+    response = FetcherResponse(
+        {
+            'text': 'Success',
+            'total': 10_001,
+            'max_page': 2,
+            'results': [{'hostname': ['api.example.com']}, *({} for _ in range(9_999))],
+        },
+        200,
+        {},
+    )
 
-    async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]:
-        return [FetcherResponse(body={'text': 'Success', 'results': []}, status=200, headers={})]
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        yield object()
 
-    monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch_all', fake_fetch_all)
-    search = onyphe.SearchOnyphe('example.com')
+    async def fake_fetch(**kwargs: Any) -> FetcherResponse:
+        calls.append(kwargs)
+        return response
 
+    monkeypatch.setattr(onyphe.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch', fake_fetch)
+    search = onyphe.SearchOnyphe('example.com', 10_001)
     await search.process()
 
-    assert await search.get_ips() == set()
-    assert await search.get_hostnames() == set()
-    assert await search.get_asns() == set()
-    assert search.execution_status == 'completed'
-    assert search.stop_reason == 'no-results'
-
-
-@pytest.mark.parametrize(
-    ('response', 'execution_status', 'stop_reason'),
-    [
-        (None, 'failed', 'transport-error'),
-        (FetcherResponse(body={}, status=401, headers={}), 'failed', 'access-denied'),
-        (FetcherResponse(body={}, status=403, headers={}), 'failed', 'access-denied'),
-        (FetcherResponse(body={}, status=429, headers={}), 'rate-limited', 'http-429'),
-        (FetcherResponse(body={}, status=503, headers={}), 'failed', 'http-503'),
-        (FetcherResponse(body=['provider-secret-payload'], status=200, headers={}), 'failed', 'invalid-response'),
-        (
-            FetcherResponse(body={'text': 'Success', 'results': {}}, status=200, headers={}),
-            'failed',
-            'invalid-response',
-        ),
-    ],
-)
-@pytest.mark.asyncio
-async def test_failed_responses_are_attributed(
-    monkeypatch: pytest.MonkeyPatch,
-    response: FetcherResponse | None,
-    execution_status: str,
-    stop_reason: str,
-) -> None:
-    monkeypatch.setattr(onyphe.Core, 'onyphe_key', lambda: 'test-key')
-
-    async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse | None]:
-        return [response]
-
-    monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch_all', fake_fetch_all)
-    search = onyphe.SearchOnyphe('example.com')
-
-    await search.process()
-
-    assert await search.get_ips() == set()
-    assert search.execution_status == execution_status
-    assert search.stop_reason == stop_reason
+    assert [call['params']['page'] for call in calls] == [1]
+    assert [call['params']['size'] for call in calls] == [10_000]
+    assert await search.get_hostnames() == {'api.example.com'}
+    assert search.execution_status == 'partial'
+    assert search.stop_reason == 'provider-limit'
 
 
 @pytest.mark.asyncio
 async def test_malformed_items_preserve_valid_partial_results(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.setattr(onyphe.Core, 'onyphe_key', lambda: 'test-key')
+    response = FetcherResponse(
+        {
+            'text': 'Success',
+            'results': [{'ip': '192.0.2.10', 'alternativeip': ['not-an-ip', None]}, 'malformed-record'],
+        },
+        200,
+        {},
+    )
 
-    async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]:
-        return [
-            FetcherResponse(
-                body={
-                    'text': 'Success',
-                    'results': [
-                        {'ip': '192.0.2.10', 'alternativeip': ['not-an-ip', None]},
-                        'malformed-record',
-                    ],
-                },
-                status=200,
-                headers={},
-            )
-        ]
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        yield object()
 
-    monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch_all', fake_fetch_all)
-    search = onyphe.SearchOnyphe('example.com')
+    async def fake_fetch(**_kwargs: Any) -> FetcherResponse:
+        return response
 
+    monkeypatch.setattr(onyphe.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch', fake_fetch)
+    search = onyphe.SearchOnyphe('example.com', 10)
     await search.process()
 
     assert await search.get_ips() == {'192.0.2.10'}
@@ -164,85 +261,27 @@ async def test_malformed_items_preserve_valid_partial_results(monkeypatch: pytes
     assert search.stop_reason == 'invalid-response'
 
 
-@pytest.mark.asyncio
-async def test_malformed_url_preserves_valid_partial_results(monkeypatch: pytest.MonkeyPatch) -> None:
-    monkeypatch.setattr(onyphe.Core, 'onyphe_key', lambda: 'test-key')
-
-    async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]:
-        return [
-            FetcherResponse(
-                body={
-                    'text': 'Success',
-                    'results': [{'ip': '192.0.2.10', 'url': ['http://[malformed']}],
-                },
-                status=200,
-                headers={},
-            )
-        ]
-
-    monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch_all', fake_fetch_all)
-    search = onyphe.SearchOnyphe('example.com')
-
-    await search.process()
-
-    assert await search.get_ips() == {'192.0.2.10'}
-    assert search.execution_status == 'partial'
-    assert search.stop_reason == 'invalid-response'
-
-
-@pytest.mark.asyncio
-async def test_failed_response_body_is_not_logged(monkeypatch, caplog) -> None:
-    monkeypatch.setattr(onyphe.Core, 'onyphe_key', lambda: 'test-key')
-    monkeypatch.setattr(onyphe.Core, 'get_user_agent', lambda: 'test-agent')
-
-    async def fake_fetch_all(*args, **kwargs):
-        return [
-            FetcherResponse(
-                body={'text': 'Failed', 'secret': 'provider-secret-payload'},
-                status=200,
-                headers={},
-            )
-        ]
-
-    monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch_all', fake_fetch_all)
-    caplog.set_level(logging.INFO, logger=onyphe.__name__)
-
-    search = onyphe.SearchOnyphe('example.com')
-    await search.process()
-
-    assert 'provider-secret-payload' not in caplog.text
-    assert 'did not succeed' in caplog.text
-    assert search.execution_status == 'failed'
-    assert search.stop_reason == 'provider-error'
-
-
-@pytest.mark.asyncio
-async def test_unexpected_response_body_is_not_logged(monkeypatch, caplog) -> None:
-    monkeypatch.setattr(onyphe.Core, 'onyphe_key', lambda: 'test-key')
-    monkeypatch.setattr(onyphe.Core, 'get_user_agent', lambda: 'test-agent')
-
-    async def fake_fetch_all(*args, **kwargs):
-        return [FetcherResponse(body='provider-secret-payload', status=200, headers={})]
-
-    monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch_all', fake_fetch_all)
-    caplog.set_level(logging.INFO, logger=onyphe.__name__)
-    search = onyphe.SearchOnyphe('example.com')
-
-    await search.process()
-
-    assert 'provider-secret-payload' not in caplog.text
-    assert search.execution_status == 'failed'
-    assert search.stop_reason == 'invalid-response'
-
-
 @pytest.mark.asyncio
 async def test_cancellation_propagates(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.setattr(onyphe.Core, 'onyphe_key', lambda: 'test-key')
+    session_exited = False
 
-    async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]:
-        raise asyncio.CancelledError
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        nonlocal session_exited
+        try:
+            yield object()
+        finally:
+            session_exited = True
 
-    monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    monkeypatch.setattr(onyphe.AsyncFetcher, 'open_session', fake_open_session)
+    cancellation = asyncio.CancelledError('operator-stop')
 
-    with pytest.raises(asyncio.CancelledError):
-        await onyphe.SearchOnyphe('example.com').process()
+    async def fake_fetch(**_kwargs: Any) -> FetcherResponse:
+        raise cancellation
+
+    monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch', fake_fetch)
+    with pytest.raises(asyncio.CancelledError) as caught:
+        await onyphe.SearchOnyphe('example.com', 10).process()
+    assert caught.value is cancellation
+    assert session_exited is True

tests/discovery/test_otx.py

@@ -219,3 +219,6 @@ class TestOtx:
 
 if __name__ == '__main__':
     pytest.main()
+
+
+pytestmark = pytest.mark.provider_contract('otx')

tests/discovery/test_pentesttools.py

@@ -133,6 +133,8 @@ async def test_malformed_start_response_completes_without_evidence(monkeypatch,
 
     assert not await search.get_hostnames()
     assert not await search.get_ips()
+
+
     assert 'malformed' in caplog.text
 
 
@@ -252,3 +254,6 @@ async def test_non_subdomain_output_is_not_collected(monkeypatch) -> None:
 
     assert not await search.get_hostnames()
     assert not await search.get_ips()
+
+
+pytestmark = pytest.mark.provider_contract('pentesttools')

tests/discovery/test_projectdiscovery.py

@@ -205,3 +205,6 @@ async def test_parser_exception_preserves_valid_partial_results(
     assert search.execution_status == 'partial'
     assert search.stop_reason == 'invalid-response'
     assert 'private provider payload' not in caplog.text
+
+
+pytestmark = pytest.mark.provider_contract('projectdiscovery')

tests/discovery/test_rapiddns.py

@@ -186,7 +186,8 @@ async def test_rapiddns_evidence_reaches_existing_outputs(
     class FakeSecurityScorecard:
         created = 0
 
-        def __init__(self, _domain: str) -> None:
+        def __init__(self, _domain: str, limit: int) -> None:
+            assert limit == 500
             type(self).created += 1
 
         async def process(self, _proxy: bool) -> None:
@@ -377,3 +378,6 @@ async def test_rapiddns_evidence_reaches_existing_outputs(
     assert failed_write_exit.value.code == 0
     assert len(completed_results) == 3
     assert 'forced completed-result failure' in capsys.readouterr().out
+
+
+pytestmark = pytest.mark.provider_contract('rapiddns')

tests/discovery/test_robtex.py

@@ -153,3 +153,6 @@ async def test_robtex_cancellation_propagates(monkeypatch: pytest.MonkeyPatch) -
 
     with pytest.raises(asyncio.CancelledError):
         await search.process(proxy=True)
+
+
+pytestmark = pytest.mark.provider_contract('robtex')

tests/discovery/test_rocketreach.py

@@ -117,3 +117,6 @@ async def test_do_search_stops_on_throttling_message(monkeypatch) -> None:
     await search.process()
 
     assert len(calls) == 1
+
+
+pytestmark = pytest.mark.provider_contract('rocketreach')

tests/discovery/test_search_dehashed.py

@@ -142,3 +142,6 @@ async def test_rate_limit_retries_once_and_preserves_earlier_page(monkeypatch, c
     assert sleeps == [0.0]
     assert await search.get_emails() == {'first@example.com', 'second@example.com'}
     assert 'provider-secret-limit-detail' not in caplog.text
+
+
+pytestmark = pytest.mark.provider_contract('dehashed')

tests/discovery/test_securityscorecard.py

@@ -0,0 +1,180 @@
+from __future__ import annotations
+
+import asyncio
+import contextlib
+from collections.abc import AsyncIterator
+from typing import Any
+
+import pytest
+
+from theHarvester.discovery import securityscorecard
+from theHarvester.discovery.constants import MissingKey
+from theHarvester.lib.core import FetcherResponse
+
+
+@pytest.mark.provider_contract('securityscorecard')
+@pytest.mark.asyncio
+async def test_process_paginates_documented_domain_and_ip_asset_routes(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(securityscorecard.Core, 'securityscorecard_key', staticmethod(lambda: 'test-key'))
+    monkeypatch.setattr(securityscorecard.SearchSecurityScorecard, 'PAGE_SIZE', 2)
+    session = object()
+    session_exited = False
+    calls: list[dict[str, Any]] = []
+    post_responses = [
+        FetcherResponse(
+            {'entries': [{'domain': 'API.Example.COM.'}, {'domain': 'outside.test'}], 'size': 3},
+            200,
+            {},
+        ),
+        FetcherResponse({'entries': [{'domain': 'www.example.com'}], 'size': 3}, 200, {}),
+        FetcherResponse({'entries': [{'ip': '192.0.2.1'}, {'ip': '2001:db8::1'}], 'size': 3}, 200, {}),
+        FetcherResponse({'entries': [{'ip': '198.51.100.2'}], 'size': 3}, 200, {}),
+    ]
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**kwargs: Any) -> AsyncIterator[object]:
+        nonlocal session_exited
+        assert kwargs == {'headers': search.headers, 'proxy': True}
+        try:
+            yield session
+        finally:
+            session_exited = True
+
+    async def fake_fetch(**kwargs: Any) -> FetcherResponse:
+        calls.append(kwargs)
+        return FetcherResponse({'score': 92, 'grade': 'A', 'factor_grades': {'network_security': 'A'}}, 200, {})
+
+    async def fake_post_fetch(*args: Any, **kwargs: Any) -> FetcherResponse:
+        calls.append({'url': args[0], **kwargs})
+        return post_responses.pop(0)
+
+    monkeypatch.setattr(securityscorecard.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(securityscorecard.AsyncFetcher, 'fetch', fake_fetch)
+    monkeypatch.setattr(securityscorecard.AsyncFetcher, 'post_fetch', fake_post_fetch)
+    search = securityscorecard.SearchSecurityScorecard('example.com', 3)
+    await search.process(proxy=True)
+
+    assert await search.get_hostnames() == {'api.example.com', 'www.example.com'}
+    assert await search.get_ips() == {'192.0.2.1', '198.51.100.2', '2001:db8::1'}
+    assert await search.get_score() == 92
+    assert await search.get_grades() == {'overall': 'A', 'network_security': 'A'}
+    assert search.execution_status == 'completed'
+    assert search.stop_reason is None
+    assert calls[0] == {
+        'session': session,
+        'url': 'https://api.securityscorecard.io/companies/example.com',
+        'json': True,
+        'include_metadata': True,
+    }
+    assert [(call['url'], call['json_body']) for call in calls[1:]] == [
+        ('https://api.securityscorecard.io/parent-domains/example.com/domains', {'page': 0, 'page_size': 2}),
+        ('https://api.securityscorecard.io/parent-domains/example.com/domains', {'page': 1, 'page_size': 2}),
+        ('https://api.securityscorecard.io/parent-domains/example.com/ips', {'page': 0, 'page_size': 2}),
+        ('https://api.securityscorecard.io/parent-domains/example.com/ips', {'page': 1, 'page_size': 2}),
+    ]
+    assert all(call['session'] is session for call in calls[1:])
+    assert session_exited is True
+
+
+@pytest.mark.parametrize('key', [None, '', '  '])
+def test_missing_or_blank_key_fails_closed(monkeypatch: pytest.MonkeyPatch, key: str | None) -> None:
+    monkeypatch.setattr(securityscorecard.Core, 'securityscorecard_key', staticmethod(lambda: key))
+    with pytest.raises(MissingKey):
+        securityscorecard.SearchSecurityScorecard('example.com', 10)
+
+
+@pytest.mark.parametrize('limit', [0, -1, True, 1.5])
+def test_limit_must_be_a_positive_integer(monkeypatch: pytest.MonkeyPatch, limit: Any) -> None:
+    monkeypatch.setattr(securityscorecard.Core, 'securityscorecard_key', staticmethod(lambda: 'test-key'))
+    with pytest.raises(ValueError, match='positive integer'):
+        securityscorecard.SearchSecurityScorecard('example.com', limit)
+
+
+@pytest.mark.parametrize(
+    ('response', 'status', 'reason'),
+    [
+        (None, 'failed', 'transport-error'),
+        (FetcherResponse({}, 401, {}), 'failed', 'access-denied'),
+        (FetcherResponse({}, 403, {}), 'failed', 'access-denied'),
+        (FetcherResponse({}, 429, {}), 'rate-limited', 'http-429'),
+        (FetcherResponse({}, 503, {}), 'failed', 'http-503'),
+        (FetcherResponse([], 200, {}), 'failed', 'invalid-response'),
+    ],
+)
+@pytest.mark.asyncio
+async def test_provider_failures_are_truthful(
+    monkeypatch: pytest.MonkeyPatch,
+    response: FetcherResponse | None,
+    status: str,
+    reason: str,
+) -> None:
+    monkeypatch.setattr(securityscorecard.Core, 'securityscorecard_key', staticmethod(lambda: 'test-key'))
+
+    async def fake_fetch(**_kwargs: Any) -> FetcherResponse | None:
+        return response
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        yield object()
+
+    monkeypatch.setattr(securityscorecard.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(securityscorecard.AsyncFetcher, 'fetch', fake_fetch)
+    search = securityscorecard.SearchSecurityScorecard('example.com', 10)
+    await search.process()
+
+    assert search.execution_status == status
+    assert search.stop_reason == reason
+
+
+@pytest.mark.asyncio
+async def test_malformed_asset_rows_preserve_valid_partial_results(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(securityscorecard.Core, 'securityscorecard_key', staticmethod(lambda: 'test-key'))
+    responses = [
+        FetcherResponse({'entries': [{'domain': 7}, {'domain': 'api.example.com'}], 'size': 2}, 200, {}),
+        FetcherResponse({'entries': [{'ip': 'not-an-ip'}, {'ip': '192.0.2.1'}], 'size': 2}, 200, {}),
+    ]
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        yield object()
+
+    async def fake_fetch(**_kwargs: Any) -> FetcherResponse:
+        return FetcherResponse({'score': 90, 'grade': 'A'}, 200, {})
+
+    async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        return responses.pop(0)
+
+    monkeypatch.setattr(securityscorecard.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(securityscorecard.AsyncFetcher, 'fetch', fake_fetch)
+    monkeypatch.setattr(securityscorecard.AsyncFetcher, 'post_fetch', fake_post_fetch)
+    search = securityscorecard.SearchSecurityScorecard('example.com', 2)
+    await search.process()
+
+    assert await search.get_hostnames() == {'api.example.com'}
+    assert await search.get_ips() == {'192.0.2.1'}
+    assert search.execution_status == 'partial'
+    assert search.stop_reason == 'invalid-response'
+
+
+@pytest.mark.asyncio
+async def test_cancellation_closes_provider_session(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(securityscorecard.Core, 'securityscorecard_key', staticmethod(lambda: 'test-key'))
+    session_exited = False
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        nonlocal session_exited
+        try:
+            yield object()
+        finally:
+            session_exited = True
+
+    async def fake_fetch(**_kwargs: Any) -> FetcherResponse:
+        raise asyncio.CancelledError('operator-stop')
+
+    monkeypatch.setattr(securityscorecard.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(securityscorecard.AsyncFetcher, 'fetch', fake_fetch)
+
+    with pytest.raises(asyncio.CancelledError, match='operator-stop'):
+        await securityscorecard.SearchSecurityScorecard('example.com', 10).process()
+    assert session_exited is True

tests/discovery/test_securitytrails.py

@@ -0,0 +1,132 @@
+from __future__ import annotations
+
+import asyncio
+import contextlib
+from typing import TYPE_CHECKING, Any
+
+import pytest
+
+from theHarvester.discovery import securitytrailssearch
+from theHarvester.discovery.constants import MissingKey
+from theHarvester.lib.core import FetcherResponse
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+
+
+@pytest.mark.provider_contract('securityTrails')
+@pytest.mark.asyncio
+async def test_process_reuses_session_and_parses_scoped_evidence(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(securitytrailssearch.Core, 'security_trails_key', staticmethod(lambda: 'test-key'))
+    session = object()
+    session_exited = False
+    calls: list[dict[str, Any]] = []
+    responses = [
+        FetcherResponse(
+            {
+                'hostname': 'example.com',
+                'current_dns': {
+                    'a': {'values': [{'ip': '192.0.2.1'}]},
+                    'aaaa': {'values': [{'ipv6': '2001:db8::1'}]},
+                },
+            },
+            200,
+            {},
+        ),
+        FetcherResponse({'subdomains': ['API', 'www', 7]}, 200, {}),
+    ]
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**kwargs: Any) -> AsyncIterator[object]:
+        nonlocal session_exited
+        assert kwargs == {'headers': {'APIKEY': 'test-key', 'Accept': 'application/json'}, 'proxy': True}
+        try:
+            yield session
+        finally:
+            session_exited = True
+
+    async def fake_fetch(**kwargs: Any) -> FetcherResponse:
+        calls.append(kwargs)
+        return responses.pop(0)
+
+    monkeypatch.setattr(securitytrailssearch.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(securitytrailssearch.AsyncFetcher, 'fetch', fake_fetch)
+    search = securitytrailssearch.SearchSecuritytrail('example.com')
+    await search.process(proxy=True)
+
+    assert await search.get_hostnames() == {'api.example.com', 'www.example.com'}
+    assert await search.get_ips() == {'192.0.2.1', '2001:db8::1'}
+    assert search.execution_status == 'partial'
+    assert search.stop_reason == 'invalid-response'
+    assert [call['url'] for call in calls] == [
+        'https://api.securitytrails.com/v1/domain/example.com',
+        'https://api.securitytrails.com/v1/domain/example.com/subdomains',
+    ]
+    assert all(call['session'] is session for call in calls)
+    assert session_exited is True
+
+
+@pytest.mark.parametrize('key', [None, '', '  '])
+def test_missing_or_blank_key_fails_closed(monkeypatch: pytest.MonkeyPatch, key: str | None) -> None:
+    monkeypatch.setattr(securitytrailssearch.Core, 'security_trails_key', staticmethod(lambda: key))
+    with pytest.raises(MissingKey):
+        securitytrailssearch.SearchSecuritytrail('example.com')
+
+
+@pytest.mark.parametrize(
+    ('response', 'status', 'reason'),
+    [
+        (None, 'failed', 'transport-error'),
+        (FetcherResponse({}, 401, {}), 'failed', 'access-denied'),
+        (FetcherResponse({}, 429, {}), 'rate-limited', 'http-429'),
+        (FetcherResponse({}, 503, {}), 'failed', 'http-503'),
+        (FetcherResponse([], 200, {}), 'failed', 'invalid-response'),
+    ],
+)
+@pytest.mark.asyncio
+async def test_first_request_failures_are_truthful(
+    monkeypatch: pytest.MonkeyPatch,
+    response: FetcherResponse | None,
+    status: str,
+    reason: str,
+) -> None:
+    monkeypatch.setattr(securitytrailssearch.Core, 'security_trails_key', staticmethod(lambda: 'test-key'))
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        yield object()
+
+    async def fake_fetch(**_kwargs: Any) -> FetcherResponse | None:
+        return response
+
+    monkeypatch.setattr(securitytrailssearch.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(securitytrailssearch.AsyncFetcher, 'fetch', fake_fetch)
+    search = securitytrailssearch.SearchSecuritytrail('example.com')
+    await search.process()
+
+    assert search.execution_status == status
+    assert search.stop_reason == reason
+
+
+@pytest.mark.asyncio
+async def test_cancellation_closes_provider_session(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(securitytrailssearch.Core, 'security_trails_key', staticmethod(lambda: 'test-key'))
+    session_exited = False
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        nonlocal session_exited
+        try:
+            yield object()
+        finally:
+            session_exited = True
+
+    async def fake_fetch(**_kwargs: Any) -> FetcherResponse:
+        raise asyncio.CancelledError('operator-stop')
+
+    monkeypatch.setattr(securitytrailssearch.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(securitytrailssearch.AsyncFetcher, 'fetch', fake_fetch)
+
+    with pytest.raises(asyncio.CancelledError, match='operator-stop'):
+        await securitytrailssearch.SearchSecuritytrail('example.com').process()
+    assert session_exited is True

tests/discovery/test_sherlockeye.py

@@ -1,6 +1,10 @@
+import asyncio
+import contextlib
 import logging
 import sys
 import types
+from collections.abc import AsyncIterator
+from typing import Any
 
 import pytest
 
@@ -12,21 +16,83 @@ if 'aiohttp_socks' not in sys.modules:
         def from_url(*_args, **_kwargs):
             return None
 
-    setattr(aiohttp_socks_stub, 'ProxyConnector', _ProxyConnector)
+    aiohttp_socks_stub.ProxyConnector = _ProxyConnector  # type: ignore[attr-defined]
     sys.modules['aiohttp_socks'] = aiohttp_socks_stub
 
 from theHarvester.discovery import sherlockeye
 from theHarvester.discovery.constants import MissingKey
+from theHarvester.lib.core import FetcherResponse
 
 
-@pytest.mark.asyncio
-async def test_missing_key_raises(monkeypatch) -> None:
-    monkeypatch.setattr(sherlockeye.Core, 'sherlockeye_key', lambda: None)
+@pytest.fixture(autouse=True)
+def provider_session(monkeypatch: pytest.MonkeyPatch) -> None:
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        yield object()
+
+    monkeypatch.setattr(sherlockeye.AsyncFetcher, 'open_session', fake_open_session)
+
+
+@pytest.mark.parametrize('key', [None, '', '   '])
+def test_missing_or_blank_key_raises(monkeypatch: pytest.MonkeyPatch, key: str | None) -> None:
+    monkeypatch.setattr(sherlockeye.Core, 'sherlockeye_key', lambda: key)
 
     with pytest.raises(MissingKey):
         sherlockeye.SearchSherlockeye('example.com')
 
 
+@pytest.mark.asyncio
+async def test_process_uses_one_shared_provider_session(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(sherlockeye.Core, 'sherlockeye_key', lambda: 'dummy-key')
+    session = object()
+    exited = False
+    calls: list[dict[str, Any]] = []
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**kwargs: Any) -> AsyncIterator[object]:
+        nonlocal exited
+        assert kwargs == {
+            'headers': {
+                'User-Agent': sherlockeye.Core.get_user_agent(),
+                'Authorization': 'Bearer dummy-key',
+                'Content-Type': 'application/json',
+            },
+            'proxy': True,
+            'request_timeout': 90,
+        }
+        try:
+            yield session
+        finally:
+            exited = True
+
+    async def fake_post_fetch(*args: Any, **kwargs: Any) -> FetcherResponse:
+        calls.append({'url': args[0], **kwargs})
+        return FetcherResponse({'success': True, 'data': {'results': []}}, 200, {})
+
+    monkeypatch.setattr(sherlockeye.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(sherlockeye.AsyncFetcher, 'post_fetch', fake_post_fetch)
+    search = sherlockeye.SearchSherlockeye('example.com')
+
+    await search.process(proxy=True)
+
+    assert calls == [
+        {
+            'url': search.SYNC_SEARCH_URL,
+            'session': session,
+            'json': True,
+            'include_metadata': True,
+            'json_body': {
+                'type': 'domain',
+                'value': 'example.com',
+                'timeoutSeconds': 60,
+            },
+        }
+    ]
+    assert exited is True
+    assert search.execution_status == 'completed'
+    assert search.stop_reason == 'no-results'
+
+
 @pytest.mark.asyncio
 async def test_process_extracts_domain_intelligence(monkeypatch) -> None:
     monkeypatch.setattr(sherlockeye.Core, 'sherlockeye_key', lambda: 'dummy-key')
@@ -59,37 +125,17 @@ async def test_process_extracts_domain_intelligence(monkeypatch) -> None:
                         'link': 'https://api.example.com/docs',
                     },
                 },
+                {'attributes': {'email': 'user@notexample.com'}},
+                {'attributes': {'email': 'user@example.com.evil'}},
             ],
         },
         'balance': {'credits': 10},
     }
 
-    class _FakeResponse:
-        status = 200
+    async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        return FetcherResponse(api_payload, 200, {})
 
-        async def json(self):
-            return api_payload
-
-        async def __aenter__(self):
-            return self
-
-        async def __aexit__(self, *_args):
-            pass
-
-    class _FakeSession:
-        def __init__(self, **_kwargs):
-            pass
-
-        def post(self, *_args, **_kwargs):
-            return _FakeResponse()
-
-        async def __aenter__(self):
-            return self
-
-        async def __aexit__(self, *_args):
-            pass
-
-    monkeypatch.setattr(sherlockeye.aiohttp, 'ClientSession', _FakeSession)
+    monkeypatch.setattr(sherlockeye.AsyncFetcher, 'post_fetch', fake_post_fetch)
 
     search = sherlockeye.SearchSherlockeye('example.com')
     await search.process()
@@ -97,38 +143,18 @@ async def test_process_extracts_domain_intelligence(monkeypatch) -> None:
     assert await search.get_hostnames() == {'sub.example.com', 'www.example.com', 'api.example.com'}
     assert await search.get_emails() == {'user@example.com'}
     assert await search.get_ips() == {'203.0.113.10'}
+    assert search.execution_status == 'completed'
+    assert search.stop_reason is None
 
 
 @pytest.mark.asyncio
 async def test_process_handles_api_error(monkeypatch, caplog) -> None:
     monkeypatch.setattr(sherlockeye.Core, 'sherlockeye_key', lambda: 'dummy-key')
 
-    class _FakeResponse:
-        status = 401
+    async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        return FetcherResponse({'secret': 'provider-secret-payload'}, 401, {})
 
-        async def text(self):
-            return 'provider-secret-payload'
-
-        async def __aenter__(self):
-            return self
-
-        async def __aexit__(self, *_args):
-            pass
-
-    class _FakeSession:
-        def __init__(self, **_kwargs):
-            pass
-
-        def post(self, *_args, **_kwargs):
-            return _FakeResponse()
-
-        async def __aenter__(self):
-            return self
-
-        async def __aexit__(self, *_args):
-            pass
-
-    monkeypatch.setattr(sherlockeye.aiohttp, 'ClientSession', _FakeSession)
+    monkeypatch.setattr(sherlockeye.AsyncFetcher, 'post_fetch', fake_post_fetch)
     caplog.set_level(logging.INFO, logger=sherlockeye.__name__)
 
     search = sherlockeye.SearchSherlockeye('example.com')
@@ -139,41 +165,121 @@ async def test_process_handles_api_error(monkeypatch, caplog) -> None:
     assert await search.get_ips() == set()
     assert 'provider-secret-payload' not in caplog.text
     assert '401' in caplog.text
+    assert search.execution_status == 'failed'
+    assert search.stop_reason == 'access-denied'
 
 
 @pytest.mark.asyncio
 async def test_process_does_not_log_provider_error_message(monkeypatch, caplog) -> None:
     monkeypatch.setattr(sherlockeye.Core, 'sherlockeye_key', lambda: 'dummy-key')
 
-    class _FakeResponse:
-        status = 200
+    async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        return FetcherResponse({'success': False, 'message': 'provider-secret-payload'}, 200, {})
 
-        async def json(self):
-            return {'success': False, 'message': 'provider-secret-payload'}
-
-        async def __aenter__(self):
-            return self
-
-        async def __aexit__(self, *_args):
-            pass
-
-    class _FakeSession:
-        def __init__(self, **_kwargs):
-            pass
-
-        def post(self, *_args, **_kwargs):
-            return _FakeResponse()
-
-        async def __aenter__(self):
-            return self
-
-        async def __aexit__(self, *_args):
-            pass
-
-    monkeypatch.setattr(sherlockeye.aiohttp, 'ClientSession', _FakeSession)
+    monkeypatch.setattr(sherlockeye.AsyncFetcher, 'post_fetch', fake_post_fetch)
     caplog.set_level(logging.INFO, logger=sherlockeye.__name__)
 
-    await sherlockeye.SearchSherlockeye('example.com').process()
+    search = sherlockeye.SearchSherlockeye('example.com')
+    await search.process()
 
     assert 'provider-secret-payload' not in caplog.text
     assert 'API error' in caplog.text
+    assert search.execution_status == 'failed'
+    assert search.stop_reason == 'provider-error'
+
+
+@pytest.mark.parametrize(
+    ('status', 'execution_status', 'stop_reason'),
+    [(429, 'rate-limited', 'http-429'), (503, 'failed', 'http-503')],
+)
+@pytest.mark.asyncio
+async def test_http_failures_are_structured(
+    monkeypatch: pytest.MonkeyPatch,
+    status: int,
+    execution_status: str,
+    stop_reason: str,
+) -> None:
+    monkeypatch.setattr(sherlockeye.Core, 'sherlockeye_key', lambda: 'dummy-key')
+
+    async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        return FetcherResponse({}, status, {})
+
+    monkeypatch.setattr(sherlockeye.AsyncFetcher, 'post_fetch', fake_post_fetch)
+    search = sherlockeye.SearchSherlockeye('example.com')
+    await search.process()
+
+    assert search.execution_status == execution_status
+    assert search.stop_reason == stop_reason
+
+
+@pytest.mark.asyncio
+async def test_malformed_response_is_structured(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(sherlockeye.Core, 'sherlockeye_key', lambda: 'dummy-key')
+
+    async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        return FetcherResponse([], 200, {})
+
+    monkeypatch.setattr(sherlockeye.AsyncFetcher, 'post_fetch', fake_post_fetch)
+    search = sherlockeye.SearchSherlockeye('example.com')
+    await search.process()
+
+    assert search.execution_status == 'failed'
+    assert search.stop_reason == 'invalid-response'
+
+
+def test_malformed_link_does_not_discard_later_valid_results(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(sherlockeye.Core, 'sherlockeye_key', lambda: 'dummy-key')
+    search = sherlockeye.SearchSherlockeye('example.com')
+
+    search._extract_response(
+        {
+            'success': True,
+            'data': {
+                'results': [
+                    {'attributes': {'link': 'https://[invalid'}},
+                    {'attributes': {'link': 'https://api.example.com/path'}},
+                ]
+            },
+        }
+    )
+
+    assert search.totalhosts == {'api.example.com'}
+    assert search.execution_status == 'partial'
+    assert search.stop_reason == 'invalid-response'
+
+
+@pytest.mark.asyncio
+async def test_transport_failure_and_cancellation_are_distinct(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(sherlockeye.Core, 'sherlockeye_key', lambda: 'dummy-key')
+    session_exit_count = 0
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        nonlocal session_exit_count
+        try:
+            yield object()
+        finally:
+            session_exit_count += 1
+
+    monkeypatch.setattr(sherlockeye.AsyncFetcher, 'open_session', fake_open_session)
+
+    async def failed_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        raise RuntimeError('provider-secret')
+
+    monkeypatch.setattr(sherlockeye.AsyncFetcher, 'post_fetch', failed_post_fetch)
+    search = sherlockeye.SearchSherlockeye('example.com')
+    await search.process()
+    assert search.execution_status == 'failed'
+    assert search.stop_reason == 'transport-error'
+    assert session_exit_count == 1
+
+    async def cancelled_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        raise asyncio.CancelledError
+
+    monkeypatch.setattr(sherlockeye.AsyncFetcher, 'post_fetch', cancelled_post_fetch)
+    with pytest.raises(asyncio.CancelledError):
+        await sherlockeye.SearchSherlockeye('example.com').process()
+    assert session_exit_count == 2
+
+
+pytestmark = pytest.mark.provider_contract('sherlockeye')

tests/discovery/test_shodan_engine.py

@@ -697,3 +697,9 @@ class TestShodanEngine:
         assert not await search.get_ips()
         assert 'Shodan InternetDB request failed' in caplog.text
         assert 'provider-secret-payload' not in caplog.text
+
+
+pytestmark = [
+    pytest.mark.provider_contract('shodan'),
+    pytest.mark.provider_contract('shodanInternetDB'),
+]

tests/discovery/test_shodanct.py

@@ -99,3 +99,6 @@ async def test_process_reports_transport_and_malformed_responses(
 
     assert await search.get_hostnames() == set()
     assert message in caplog.text
+
+
+pytestmark = pytest.mark.provider_contract('shodanct')

tests/discovery/test_sourcegraph.py

@@ -483,3 +483,6 @@ async def test_sourcegraph_partial_outcome_reaches_completed_result(
     assert execution.result_count == 1
     summary = json.loads(report.with_suffix('.jsonl').read_text().splitlines()[0])
     assert summary['source_executions'][0]['stop_reason'] == 'provider-limited'
+
+
+pytestmark = pytest.mark.provider_contract('sourcegraph')

tests/discovery/test_subdomaincenter.py

@@ -82,3 +82,6 @@ async def test_process_attributes_http_failures(
 
     assert await search.get_hostnames() == set()
     assert 'SubdomainCenter request failed with HTTP 429' in caplog.text
+
+
+pytestmark = pytest.mark.provider_contract('subdomaincenter')

tests/discovery/test_subdomainfinderc99.py

@@ -1,20 +1,41 @@
+import asyncio
+import contextlib
+from collections.abc import AsyncIterator
+from typing import Any
+
 import pytest
 
 from theHarvester.discovery import subdomainfinderc99
+from theHarvester.lib.core import FetcherResponse
 
 
 @pytest.mark.asyncio
 async def test_successful_response_returns_only_scoped_hostnames(monkeypatch) -> None:
-    async def fake_fetch_all(*_args, **_kwargs):
-        return ['<div class="input-group"><input name="token" value="abc"></div>']
+    session = object()
+    session_exited = False
+    calls: list[tuple[str, object]] = []
 
-    async def fake_post_fetch(*_args, **_kwargs):
-        return 'api.example.test www.notexample.test'
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        nonlocal session_exited
+        try:
+            yield session
+        finally:
+            session_exited = True
+
+    async def fake_fetch(*_args, **kwargs):
+        calls.append(('get', kwargs['session']))
+        return FetcherResponse('<div class="input-group"><input name="token" value="abc"></div>', 200, {})
+
+    async def fake_post_fetch(*_args, **kwargs):
+        calls.append(('post', kwargs['session']))
+        return FetcherResponse('api.example.test www.notexample.test', 200, {})
 
     async def no_sleep(*_args, **_kwargs):
         return None
 
-    monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'fetch', fake_fetch)
     monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'post_fetch', fake_post_fetch)
     monkeypatch.setattr(subdomainfinderc99.asyncio, 'sleep', no_sleep)
 
@@ -22,33 +43,61 @@ async def test_successful_response_returns_only_scoped_hostnames(monkeypatch) ->
     await search.process()
 
     assert set(await search.get_hostnames()) == {'api.example.test'}
+    assert search.execution_status == 'completed'
+    assert search.stop_reason is None
+    assert calls == [('get', session), ('post', session)]
+    assert session_exited is True
 
 
 @pytest.mark.asyncio
-async def test_empty_initial_response_completes_without_evidence(monkeypatch) -> None:
-    async def fake_fetch_all(*_args, **_kwargs):
-        return []
+async def test_empty_initial_response_is_transport_failure(monkeypatch) -> None:
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        yield object()
 
-    monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    async def fake_fetch(*_args, **_kwargs):
+        return None
+
+    monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'fetch', fake_fetch)
 
     search = subdomainfinderc99.SearchSubdomainfinderc99('example.test')
     await search.process()
 
     assert not await search.get_hostnames()
+    assert search.execution_status == 'failed'
+    assert search.stop_reason == 'transport-error'
 
 
+@pytest.mark.parametrize(
+    ('response', 'status', 'reason'),
+    [
+        (FetcherResponse('', 403, {}), 'failed', 'access-denied'),
+        (FetcherResponse(None, 200, {}), 'failed', 'invalid-response'),
+    ],
+)
 @pytest.mark.asyncio
-async def test_malformed_scan_response_completes_without_evidence(monkeypatch) -> None:
-    async def fake_fetch_all(*_args, **_kwargs):
-        return ['<div class="input-group"><input name="token" value="abc"></div>']
+async def test_scan_failures_are_structured(
+    monkeypatch: pytest.MonkeyPatch,
+    response: FetcherResponse,
+    status: str,
+    reason: str,
+) -> None:
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        yield object()
+
+    async def fake_fetch(*_args, **_kwargs):
+        return FetcherResponse('<div class="input-group"><input name="token" value="abc"></div>', 200, {})
 
     async def fake_post_fetch(*_args, **_kwargs):
-        return None
+        return response
 
     async def no_sleep(*_args, **_kwargs):
         return None
 
-    monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'fetch', fake_fetch)
     monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'post_fetch', fake_post_fetch)
     monkeypatch.setattr(subdomainfinderc99.asyncio, 'sleep', no_sleep)
 
@@ -56,3 +105,63 @@ async def test_malformed_scan_response_completes_without_evidence(monkeypatch) -
     await search.process()
 
     assert not await search.get_hostnames()
+    assert search.execution_status == status
+    assert search.stop_reason == reason
+
+
+@pytest.mark.parametrize(
+    ('response', 'status', 'reason'),
+    [
+        (FetcherResponse('', 401, {}), 'failed', 'access-denied'),
+        (FetcherResponse('', 403, {}), 'failed', 'access-denied'),
+        (FetcherResponse('', 429, {}), 'rate-limited', 'http-429'),
+        (FetcherResponse('', 503, {}), 'failed', 'http-503'),
+        (FetcherResponse({}, 200, {}), 'failed', 'invalid-response'),
+    ],
+)
+@pytest.mark.asyncio
+async def test_initial_failures_are_structured(
+    monkeypatch: pytest.MonkeyPatch,
+    response: FetcherResponse,
+    status: str,
+    reason: str,
+) -> None:
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        yield object()
+
+    async def fake_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        return response
+
+    monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'fetch', fake_fetch)
+    search = subdomainfinderc99.SearchSubdomainfinderc99('example.test')
+    await search.process()
+
+    assert search.execution_status == status
+    assert search.stop_reason == reason
+
+
+@pytest.mark.asyncio
+async def test_cancellation_propagates(monkeypatch: pytest.MonkeyPatch) -> None:
+    session_exited = False
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        nonlocal session_exited
+        try:
+            yield object()
+        finally:
+            session_exited = True
+
+    async def fake_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        raise asyncio.CancelledError
+
+    monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'fetch', fake_fetch)
+    with pytest.raises(asyncio.CancelledError):
+        await subdomainfinderc99.SearchSubdomainfinderc99('example.test').process()
+    assert session_exited is True
+
+
+pytestmark = pytest.mark.provider_contract('subdomainfinderc99')

tests/discovery/test_thc.py

@@ -444,3 +444,6 @@ class TestThcIntegration:
 
 if __name__ == '__main__':
     pytest.main()
+
+
+pytestmark = pytest.mark.provider_contract('thc')

tests/discovery/test_tombasearch.py

@@ -55,6 +55,8 @@ async def test_tomba_http_failures_return_no_results(monkeypatch, caplog, status
 
     assert await search.get_emails() == []
     assert await search.get_hostnames() == []
+
+
     assert f'Tomba request failed with HTTP {status}' in caplog.text
     assert 'provider detail' not in caplog.text
 
@@ -294,3 +296,6 @@ async def test_paid_tomba_search_stops_before_exceeding_quota(monkeypatch) -> No
     ]
     assert await search.get_emails() == []
     assert await search.get_hostnames() == []
+
+
+pytestmark = pytest.mark.provider_contract('tomba')

tests/discovery/test_urlscan.py

@@ -1,4 +1,6 @@
 import asyncio
+import contextlib
+from collections.abc import AsyncIterator
 from datetime import UTC, datetime, timedelta
 from typing import Any
 
@@ -8,9 +10,30 @@ from theHarvester.discovery import urlscan
 from theHarvester.lib.core import FetcherResponse
 
 
+class ProviderSession:
+    def __init__(self) -> None:
+        self.exited = False
+
+
+@pytest.fixture(autouse=True)
+def provider_session(monkeypatch: pytest.MonkeyPatch) -> ProviderSession:
+    session = ProviderSession()
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        try:
+            yield session
+        finally:
+            session.exited = True
+
+    monkeypatch.setattr(urlscan.AsyncFetcher, 'open_session', fake_open_session)
+    return session
+
+
 @pytest.mark.asyncio
 async def test_process_collects_sequential_pages_and_preserves_all_routes(
     monkeypatch: pytest.MonkeyPatch,
+    provider_session: ProviderSession,
 ) -> None:
     responses = [
         FetcherResponse(
@@ -49,7 +72,6 @@ async def test_process_collects_sequential_pages_and_preserves_all_routes(
             status=200,
             headers={},
         ),
-        FetcherResponse(body={'results': []}, status=200, headers={}),
     ]
     calls: list[dict[str, Any]] = []
 
@@ -58,7 +80,7 @@ async def test_process_collects_sequential_pages_and_preserves_all_routes(
         return responses.pop(0)
 
     monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch)
-    search = urlscan.SearchUrlscan('example.com')
+    search = urlscan.SearchUrlscan('example.com', 2)
 
     await search.process(proxy=True)
 
@@ -84,15 +106,15 @@ async def test_process_collects_sequential_pages_and_preserves_all_routes(
         ('AS64497', 'Example Transit Two', 'ip', '2001:db8::10'),
     }
     assert [call['params'] for call in calls] == [
-        {'q': 'domain:example.com'},
-        {'q': 'domain:example.com', 'search_after': '200,first'},
-        {'q': 'domain:example.com', 'search_after': '100,second'},
+        {'q': 'domain:example.com', 'size': 2},
+        {'q': 'domain:example.com', 'size': 1, 'search_after': '200,first'},
     ]
     assert all(call['url'] == 'https://urlscan.io/api/v1/search/' for call in calls)
+    assert all(call['session'] is provider_session for call in calls)
     assert all(call['json'] is True for call in calls)
     assert all(call['include_metadata'] is True for call in calls)
-    assert all(call['proxy'] is True for call in calls)
-    assert all(call['request_timeout'] == 60 for call in calls)
+    assert all('request_timeout' not in call for call in calls)
+    assert provider_session.exited is True
     assert search.execution_status == 'completed'
     assert search.stop_reason is None
 
@@ -125,7 +147,7 @@ async def test_repeated_asn_relationship_is_retained_once_per_source_run(monkeyp
 
     monkeypatch.setattr(urlscan, 'datetime', TickingDateTime)
     monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch)
-    search = urlscan.SearchUrlscan('example.com')
+    search = urlscan.SearchUrlscan('example.com', 10)
 
     await search.process()
 
@@ -138,7 +160,7 @@ async def test_valid_empty_response_is_completed(monkeypatch: pytest.MonkeyPatch
         return FetcherResponse(body={'results': []}, status=200, headers={})
 
     monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch)
-    search = urlscan.SearchUrlscan('example.com')
+    search = urlscan.SearchUrlscan('example.com', 10)
 
     await search.process()
 
@@ -165,7 +187,7 @@ async def test_missing_optional_fields_are_skipped(monkeypatch: pytest.MonkeyPat
         return responses.pop(0)
 
     monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch)
-    search = urlscan.SearchUrlscan('example.com')
+    search = urlscan.SearchUrlscan('example.com', 10)
 
     await search.process()
 
@@ -194,7 +216,7 @@ async def test_malformed_nested_fields_preserve_valid_partial_results(monkeypatc
         return responses.pop(0)
 
     monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch)
-    search = urlscan.SearchUrlscan('example.com')
+    search = urlscan.SearchUrlscan('example.com', 10)
 
     await search.process()
 
@@ -240,7 +262,7 @@ async def test_results_are_typed_and_scoped_before_insertion(monkeypatch: pytest
         return responses.pop(0)
 
     monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch)
-    search = urlscan.SearchUrlscan('example.com')
+    search = urlscan.SearchUrlscan('example.com', 10)
 
     await search.process()
 
@@ -275,7 +297,7 @@ async def test_failed_first_page_is_attributed(
         return response
 
     monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch)
-    search = urlscan.SearchUrlscan('example.com')
+    search = urlscan.SearchUrlscan('example.com', 10)
 
     await search.process()
 
@@ -312,7 +334,7 @@ async def test_later_failure_preserves_partial_results(
         return responses.pop(0)
 
     monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch)
-    search = urlscan.SearchUrlscan('example.com')
+    search = urlscan.SearchUrlscan('example.com', 10)
 
     await search.process()
 
@@ -327,7 +349,7 @@ async def test_fetch_exception_is_transport_failure(monkeypatch: pytest.MonkeyPa
         raise OSError('private transport details')
 
     monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch)
-    search = urlscan.SearchUrlscan('example.com')
+    search = urlscan.SearchUrlscan('example.com', 10)
 
     await search.process()
 
@@ -349,7 +371,7 @@ async def test_missing_cursor_stops_after_first_page(monkeypatch: pytest.MonkeyP
         )
 
     monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch)
-    search = urlscan.SearchUrlscan('example.com')
+    search = urlscan.SearchUrlscan('example.com', 10)
 
     await search.process()
 
@@ -381,7 +403,7 @@ async def test_repeated_cursor_stops_without_a_third_request(monkeypatch: pytest
         return responses.pop(0)
 
     monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch)
-    search = urlscan.SearchUrlscan('example.com')
+    search = urlscan.SearchUrlscan('example.com', 10)
 
     await search.process()
 
@@ -392,43 +414,89 @@ async def test_repeated_cursor_stops_without_a_third_request(monkeypatch: pytest
 
 
 @pytest.mark.asyncio
-async def test_page_limit_preserves_results(monkeypatch: pytest.MonkeyPatch) -> None:
-    calls = 0
-
-    async def fake_fetch(**_kwargs: Any) -> FetcherResponse:
-        nonlocal calls
-        calls += 1
-        return FetcherResponse(
+async def test_pagination_continues_beyond_the_removed_local_page_ceiling(monkeypatch: pytest.MonkeyPatch) -> None:
+    calls: list[dict[str, Any]] = []
+    first_page = [
+        {'page': {'domain': f'page-{index}.example.com'}, 'sort': [10_001 - index, f'cursor-{index}']}
+        for index in range(1, 10_001)
+    ]
+    responses = [
+        FetcherResponse(body={'results': first_page}, status=200, headers={}),
+        FetcherResponse(
             body={
                 'results': [
                     {
-                        'page': {'domain': f'page-{calls}.example.com'},
-                        'sort': [calls, f'cursor-{calls}'],
+                        'page': {'domain': 'page-10001.example.com'},
+                        'sort': [0, 'cursor-10001'],
                     }
                 ]
             },
             status=200,
             headers={},
-        )
+        ),
+    ]
+
+    async def fake_fetch(**kwargs: Any) -> FetcherResponse:
+        calls.append(kwargs['params'])
+        return responses.pop(0)
 
     monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch)
-    monkeypatch.setattr(urlscan.SearchUrlscan, 'MAX_PAGES', 2)
-    search = urlscan.SearchUrlscan('example.com')
+    search = urlscan.SearchUrlscan('example.com', 10_001)
 
     await search.process()
 
-    assert calls == 2
-    assert await search.get_hostnames() == {'page-1.example.com', 'page-2.example.com'}
-    assert search.execution_status == 'partial'
-    assert search.stop_reason == 'page-limit'
+    assert calls == [
+        {'q': 'domain:example.com', 'size': 10_000},
+        {'q': 'domain:example.com', 'size': 1, 'search_after': '1,cursor-10000'},
+    ]
+    assert await search.get_hostnames() == {f'page-{page}.example.com' for page in range(1, 10_002)}
+    assert search.execution_status == 'completed'
+    assert search.stop_reason is None
 
 
 @pytest.mark.asyncio
-async def test_cancellation_propagates(monkeypatch: pytest.MonkeyPatch) -> None:
+async def test_operator_limit_sets_page_size_and_stops_without_an_extra_request(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    calls: list[dict[str, Any]] = []
+    results = [
+        {'page': {'domain': f'result-{index}.example.com'}, 'sort': [10 - index, f'cursor-{index}']} for index in range(10)
+    ]
+
+    async def fake_fetch(**kwargs: Any) -> FetcherResponse:
+        calls.append(kwargs['params'])
+        return FetcherResponse(body={'results': results}, status=200, headers={})
+
+    monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch)
+    search = urlscan.SearchUrlscan('example.com', 10)
+
+    await search.process()
+
+    assert calls == [{'q': 'domain:example.com', 'size': 10}]
+    assert len(await search.get_hostnames()) == 10
+    assert search.execution_status == 'completed'
+    assert search.stop_reason is None
+
+
+@pytest.mark.parametrize('limit', [0, -1, True, 1.5])
+def test_limit_must_be_a_positive_integer(limit: Any) -> None:
+    with pytest.raises(ValueError, match='positive integer'):
+        urlscan.SearchUrlscan('example.com', limit)
+
+
+@pytest.mark.asyncio
+async def test_cancellation_propagates(
+    monkeypatch: pytest.MonkeyPatch,
+    provider_session: ProviderSession,
+) -> None:
     async def fake_fetch(**_kwargs: Any) -> FetcherResponse:
         raise asyncio.CancelledError
 
     monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch)
 
     with pytest.raises(asyncio.CancelledError):
-        await urlscan.SearchUrlscan('example.com').process()
+        await urlscan.SearchUrlscan('example.com', 10).process()
+    assert provider_session.exited is True
+
+
+pytestmark = pytest.mark.provider_contract('urlscan')

tests/discovery/test_virustotal.py

@@ -1,6 +1,89 @@
+from __future__ import annotations
+
+import asyncio
+import contextlib
+from typing import TYPE_CHECKING, Any
+
 import pytest
 
 from theHarvester.discovery import virustotal
+from theHarvester.discovery.constants import MissingKey
+from theHarvester.lib.core import FetcherResponse
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+
+
+@pytest.mark.provider_contract('virustotal')
+@pytest.mark.asyncio
+async def test_process_paginates_without_fixed_sleeps_and_keeps_scoped_evidence(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(virustotal.Core, 'virustotal_key', staticmethod(lambda: 'test-key'))
+    session = object()
+    session_exited = False
+    calls: list[dict[str, Any]] = []
+    responses = [
+        FetcherResponse(
+            {
+                'data': [
+                    {
+                        'id': 'API.Example.COM.',
+                        'attributes': {
+                            'last_dns_records': [{'value': 'dns.example.com'}],
+                            'last_https_certificate': {
+                                'extensions': {'subject_alternative_name': ['tls.example.com', 'outside.test']}
+                            },
+                        },
+                    }
+                ],
+                'meta': {'cursor': 'next-page'},
+            },
+            200,
+            {},
+        ),
+        FetcherResponse(
+            {'data': [{'id': 'mail.example.com', 'attributes': {'last_dns_records': []}}], 'meta': {}},
+            200,
+            {},
+        ),
+    ]
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**kwargs: Any) -> AsyncIterator[object]:
+        nonlocal session_exited
+        assert kwargs == {
+            'headers': {
+                'Accept': 'application/json',
+                'x-apikey': 'test-key',
+            },
+            'proxy': True,
+        }
+        try:
+            yield session
+        finally:
+            session_exited = True
+
+    async def fake_fetch(**kwargs: Any) -> FetcherResponse:
+        calls.append(kwargs)
+        return responses.pop(0)
+
+    monkeypatch.setattr(virustotal.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(virustotal.AsyncFetcher, 'fetch', fake_fetch)
+    search = virustotal.SearchVirustotal('example.com', limit=10)
+    await search.process(proxy=True)
+
+    assert await search.get_hostnames() == {
+        'api.example.com',
+        'dns.example.com',
+        'mail.example.com',
+        'tls.example.com',
+    }
+    assert search.execution_status == 'completed'
+    assert search.stop_reason is None
+    assert [call['params'] for call in calls] == [{'limit': 10}, {'limit': 9, 'cursor': 'next-page'}]
+    assert all(call['session'] is session for call in calls)
+    assert session_exited is True
 
 
 @pytest.mark.asyncio
@@ -10,15 +93,140 @@ async def test_parse_hostnames_preserves_www_evidence() -> None:
             'id': 'www.example.com',
             'attributes': {
                 'last_dns_records': [{'value': 'www.api.example.com'}],
-                'last_https_certificate': {
-                    'extensions': {'subject_alternative_name': ['www.mail.example.com']}
-                },
+                'last_https_certificate': {'extensions': {'subject_alternative_name': ['www.mail.example.com']}},
             },
         }
     ]
 
-    assert await virustotal.SearchVirustotal.parse_hostnames(data, 'example.com') == [
-        'www.api.example.com',
-        'www.example.com',
-        'www.mail.example.com',
+    hostnames, malformed = virustotal.SearchVirustotal.parse_hostnames(data, 'example.com')
+    assert hostnames == {'www.api.example.com', 'www.example.com', 'www.mail.example.com'}
+    assert malformed is False
+
+
+@pytest.mark.parametrize('key', [None, '', '  '])
+def test_missing_or_blank_key_fails_closed(monkeypatch: pytest.MonkeyPatch, key: str | None) -> None:
+    monkeypatch.setattr(virustotal.Core, 'virustotal_key', staticmethod(lambda: key))
+    with pytest.raises(MissingKey):
+        virustotal.SearchVirustotal('example.com', limit=10)
+
+
+@pytest.mark.parametrize('limit', [0, -1, True, 1.5])
+def test_limit_must_be_a_positive_integer(monkeypatch: pytest.MonkeyPatch, limit: Any) -> None:
+    monkeypatch.setattr(virustotal.Core, 'virustotal_key', staticmethod(lambda: 'test-key'))
+    with pytest.raises(ValueError, match='positive integer'):
+        virustotal.SearchVirustotal('example.com', limit=limit)
+
+
+@pytest.mark.parametrize(
+    ('response', 'status', 'reason'),
+    [
+        (None, 'failed', 'transport-error'),
+        (FetcherResponse({}, 401, {}), 'failed', 'access-denied'),
+        (FetcherResponse({}, 429, {}), 'rate-limited', 'http-429'),
+        (FetcherResponse({}, 503, {}), 'failed', 'http-503'),
+        (FetcherResponse([], 200, {}), 'failed', 'invalid-response'),
+    ],
+)
+@pytest.mark.asyncio
+async def test_provider_failures_are_truthful(
+    monkeypatch: pytest.MonkeyPatch,
+    response: FetcherResponse | None,
+    status: str,
+    reason: str,
+) -> None:
+    monkeypatch.setattr(virustotal.Core, 'virustotal_key', staticmethod(lambda: 'test-key'))
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        yield object()
+
+    async def fake_fetch(**_kwargs: Any) -> FetcherResponse | None:
+        return response
+
+    monkeypatch.setattr(virustotal.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(virustotal.AsyncFetcher, 'fetch', fake_fetch)
+    search = virustotal.SearchVirustotal('example.com', limit=10)
+    await search.process()
+
+    assert search.execution_status == status
+    assert search.stop_reason == reason
+
+
+@pytest.mark.asyncio
+async def test_later_rate_limit_preserves_partial_results(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(virustotal.Core, 'virustotal_key', staticmethod(lambda: 'test-key'))
+    responses = [
+        FetcherResponse(
+            {'data': [{'id': 'api.example.com', 'attributes': {}}], 'meta': {'cursor': 'next'}},
+            200,
+            {},
+        ),
+        FetcherResponse({}, 429, {}),
     ]
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        yield object()
+
+    async def fake_fetch(**_kwargs: Any) -> FetcherResponse:
+        return responses.pop(0)
+
+    monkeypatch.setattr(virustotal.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(virustotal.AsyncFetcher, 'fetch', fake_fetch)
+    search = virustotal.SearchVirustotal('example.com', limit=10)
+    await search.process()
+
+    assert await search.get_hostnames() == {'api.example.com'}
+    assert search.execution_status == 'partial'
+    assert search.stop_reason == 'http-429'
+
+
+@pytest.mark.asyncio
+async def test_repeated_cursor_stops_without_spending_more_quota(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(virustotal.Core, 'virustotal_key', staticmethod(lambda: 'test-key'))
+    responses = [
+        FetcherResponse({'data': [{'id': 'outside.test', 'attributes': {}}], 'meta': {'cursor': 'same'}}, 200, {}),
+        FetcherResponse({'data': [{'id': 'api.example.com', 'attributes': {}}], 'meta': {'cursor': 'same'}}, 200, {}),
+    ]
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        yield object()
+
+    async def fake_fetch(**_kwargs: Any) -> FetcherResponse:
+        return responses.pop(0)
+
+    monkeypatch.setattr(virustotal.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(virustotal.AsyncFetcher, 'fetch', fake_fetch)
+    search = virustotal.SearchVirustotal('example.com', limit=10)
+
+    await search.process()
+
+    assert await search.get_hostnames() == {'api.example.com'}
+    assert search.execution_status == 'partial'
+    assert search.stop_reason == 'repeated-cursor'
+    assert responses == []
+
+
+@pytest.mark.asyncio
+async def test_cancellation_closes_provider_session(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(virustotal.Core, 'virustotal_key', staticmethod(lambda: 'test-key'))
+    session_exited = False
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        nonlocal session_exited
+        try:
+            yield object()
+        finally:
+            session_exited = True
+
+    async def fake_fetch(**_kwargs: Any) -> FetcherResponse:
+        raise asyncio.CancelledError('operator-stop')
+
+    monkeypatch.setattr(virustotal.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(virustotal.AsyncFetcher, 'fetch', fake_fetch)
+
+    with pytest.raises(asyncio.CancelledError, match='operator-stop'):
+        await virustotal.SearchVirustotal('example.com', limit=10).process()
+    assert session_exited is True

tests/discovery/test_waybackarchive.py

@@ -250,3 +250,6 @@ async def test_process_keeps_an_earlier_failure_when_a_later_pattern_reaches_the
     assert await search.get_hostnames() == {'example.com'}
     assert search.execution_status == 'partial'
     assert search.stop_reason == 'invalid-response'
+
+
+pytestmark = pytest.mark.provider_contract('waybackarchive')

tests/discovery/test_whoisxml.py

@@ -1,28 +1,210 @@
-import logging
+from __future__ import annotations
+
+import asyncio
+import contextlib
+from typing import TYPE_CHECKING, Any
 
 import pytest
 
 from theHarvester.discovery import whoisxml
+from theHarvester.discovery.constants import MissingKey
+from theHarvester.lib.core import FetcherResponse
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+
+
+class ProviderSession:
+    def __init__(self) -> None:
+        self.exited = False
+
+
+@pytest.fixture(autouse=True)
+def provider_session(monkeypatch: pytest.MonkeyPatch) -> ProviderSession:
+    session = ProviderSession()
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        try:
+            yield session
+        finally:
+            session.exited = True
+
+    monkeypatch.setattr(whoisxml.AsyncFetcher, 'open_session', fake_open_session)
+    return session
+
+
+@pytest.mark.provider_contract('whoisxml')
+@pytest.mark.asyncio
+async def test_response_body_is_not_logged_and_scoped_records_are_returned(
+    monkeypatch: pytest.MonkeyPatch,
+    provider_session: ProviderSession,
+) -> None:
+    monkeypatch.setattr(whoisxml.Core, 'whoisxml_key', staticmethod(lambda: 'test-key'))
+    calls: list[dict[str, Any]] = []
+    responses = [
+        FetcherResponse(
+            {
+                'secret': 'provider-secret-payload',
+                'result': {
+                    'count': 2,
+                    'nextPageSearchAfter': 'www.example.com',
+                    'records': [
+                        {'domain': 'API.Example.COM.'},
+                        {'domain': 'outside.test'},
+                    ],
+                },
+            },
+            200,
+            {},
+        ),
+        FetcherResponse(
+            {
+                'result': {
+                    'count': 1,
+                    'nextPageSearchAfter': '',
+                    'records': [{'domain': 'www.example.com'}],
+                },
+            },
+            200,
+            {},
+        ),
+    ]
+
+    async def fake_fetch(**kwargs: Any) -> FetcherResponse:
+        calls.append(kwargs)
+        return responses.pop(0)
+
+    monkeypatch.setattr(whoisxml.AsyncFetcher, 'fetch', fake_fetch)
+    search = whoisxml.SearchWhoisXML('example.com', 3)
+    await search.process(proxy=True)
+
+    assert await search.get_hostnames() == {'api.example.com', 'www.example.com'}
+    assert search.execution_status == 'completed'
+    assert search.stop_reason is None
+    assert [call['url'] for call in calls] == ['https://subdomains.whoisxmlapi.com/api/v2'] * 2
+    assert all(call['session'] is provider_session for call in calls)
+    assert [call['params'] for call in calls] == [
+        {'apiKey': 'test-key', 'domainName': 'example.com'},
+        {'apiKey': 'test-key', 'domainName': 'example.com', 'searchAfter': 'www.example.com'},
+    ]
+    assert all(call['json'] is True and call['include_metadata'] is True for call in calls)
+    assert provider_session.exited is True
+
+
+@pytest.mark.parametrize('key', [None, '', '  '])
+def test_missing_or_blank_key_fails_closed(monkeypatch: pytest.MonkeyPatch, key: str | None) -> None:
+    monkeypatch.setattr(whoisxml.Core, 'whoisxml_key', staticmethod(lambda: key))
+    with pytest.raises(MissingKey):
+        whoisxml.SearchWhoisXML('example.com', 10)
+
+
+@pytest.mark.parametrize('limit', [0, -1, True, 1.5])
+def test_limit_must_be_positive(monkeypatch: pytest.MonkeyPatch, limit: Any) -> None:
+    monkeypatch.setattr(whoisxml.Core, 'whoisxml_key', staticmethod(lambda: 'test-key'))
+    with pytest.raises(ValueError, match='positive integer'):
+        whoisxml.SearchWhoisXML('example.com', limit)
+
+
+@pytest.mark.parametrize(
+    ('response', 'status', 'reason'),
+    [
+        (None, 'failed', 'transport-error'),
+        (FetcherResponse({}, 401, {}), 'failed', 'access-denied'),
+        (FetcherResponse({}, 429, {}), 'rate-limited', 'http-429'),
+        (FetcherResponse({}, 503, {}), 'failed', 'http-503'),
+        (FetcherResponse([], 200, {}), 'failed', 'invalid-response'),
+        (FetcherResponse({'result': {'records': 'many'}}, 200, {}), 'failed', 'invalid-response'),
+    ],
+)
+@pytest.mark.asyncio
+async def test_provider_failures_are_truthful(
+    monkeypatch: pytest.MonkeyPatch,
+    response: FetcherResponse | None,
+    status: str,
+    reason: str,
+) -> None:
+    monkeypatch.setattr(whoisxml.Core, 'whoisxml_key', staticmethod(lambda: 'test-key'))
+
+    async def fake_fetch(**_kwargs: Any) -> FetcherResponse | None:
+        return response
+
+    monkeypatch.setattr(whoisxml.AsyncFetcher, 'fetch', fake_fetch)
+    search = whoisxml.SearchWhoisXML('example.com', 10)
+    await search.process()
+
+    assert search.execution_status == status
+    assert search.stop_reason == reason
 
 
 @pytest.mark.asyncio
-async def test_response_body_is_not_logged_and_records_are_returned(monkeypatch, caplog) -> None:
-    monkeypatch.setattr(whoisxml.Core, 'whoisxml_key', lambda: 'test-key')
-    monkeypatch.setattr(whoisxml.Core, 'get_user_agent', lambda: 'test-agent')
+async def test_malformed_rows_preserve_valid_partial_results(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(whoisxml.Core, 'whoisxml_key', staticmethod(lambda: 'test-key'))
 
-    async def fake_fetch_all(*args, **kwargs):
-        return [
+    async def fake_fetch(**_kwargs: Any) -> FetcherResponse:
+        return FetcherResponse(
             {
-                'secret': 'provider-secret-payload',
-                'result': {'records': [{'domain': 'www.example.com'}]},
-            }
-        ]
+                'result': {
+                    'count': 2,
+                    'nextPageSearchAfter': '',
+                    'records': [{'domain': 'ok.example.com'}, {'domain': 7}],
+                }
+            },
+            200,
+            {},
+        )
 
-    monkeypatch.setattr(whoisxml.AsyncFetcher, 'fetch_all', fake_fetch_all)
-    caplog.set_level(logging.INFO, logger=whoisxml.__name__)
-
-    search = whoisxml.SearchWhoisXML('example.com')
+    monkeypatch.setattr(whoisxml.AsyncFetcher, 'fetch', fake_fetch)
+    search = whoisxml.SearchWhoisXML('example.com', 10)
     await search.process()
 
-    assert await search.get_hostnames() == ['www.example.com']
-    assert 'provider-secret-payload' not in caplog.text
+    assert await search.get_hostnames() == {'ok.example.com'}
+    assert search.execution_status == 'partial'
+    assert search.stop_reason == 'invalid-response'
+
+
+@pytest.mark.asyncio
+async def test_later_page_failure_preserves_partial_results(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(whoisxml.Core, 'whoisxml_key', staticmethod(lambda: 'test-key'))
+    responses = [
+        FetcherResponse(
+            {
+                'result': {
+                    'count': 1,
+                    'nextPageSearchAfter': 'next.example.com',
+                    'records': [{'domain': 'api.example.com'}],
+                }
+            },
+            200,
+            {},
+        ),
+        FetcherResponse({}, 429, {}),
+    ]
+
+    async def fake_fetch(**_kwargs: Any) -> FetcherResponse:
+        return responses.pop(0)
+
+    monkeypatch.setattr(whoisxml.AsyncFetcher, 'fetch', fake_fetch)
+    search = whoisxml.SearchWhoisXML('example.com', 10)
+    await search.process()
+
+    assert await search.get_hostnames() == {'api.example.com'}
+    assert search.execution_status == 'partial'
+    assert search.stop_reason == 'http-429'
+
+
+@pytest.mark.asyncio
+async def test_cancellation_closes_provider_session(
+    monkeypatch: pytest.MonkeyPatch,
+    provider_session: ProviderSession,
+) -> None:
+    monkeypatch.setattr(whoisxml.Core, 'whoisxml_key', staticmethod(lambda: 'test-key'))
+
+    async def fake_fetch(**_kwargs: Any) -> FetcherResponse:
+        raise asyncio.CancelledError('operator-stop')
+
+    monkeypatch.setattr(whoisxml.AsyncFetcher, 'fetch', fake_fetch)
+
+    with pytest.raises(asyncio.CancelledError, match='operator-stop'):
+        await whoisxml.SearchWhoisXML('example.com', 10).process()
+    assert provider_session.exited is True

tests/discovery/test_windvane.py

@@ -79,3 +79,6 @@ async def test_keyless_provider_failure_does_not_guess_dns_names(monkeypatch) ->
     assert requests == 1
     assert await search.get_hostnames() == set()
     assert await search.get_ips() == set()
+
+
+pytestmark = pytest.mark.provider_contract('windvane')

tests/discovery/test_yahoosearch.py

@@ -61,3 +61,6 @@ async def test_yahoo_unusable_responses_return_no_evidence(
 
     assert await search.get_emails() == []
     assert await search.get_hostnames() == []
+
+
+pytestmark = pytest.mark.provider_contract('yahoo')

tests/discovery/test_zoomeyesearch.py

@@ -1,6 +1,210 @@
+import asyncio
+import base64
+import contextlib
+from collections.abc import AsyncIterator
+from typing import Any
+
 import pytest
 
 from theHarvester.discovery import zoomeyesearch
+from theHarvester.discovery.constants import MissingKey
+from theHarvester.lib.core import FetcherResponse
+
+
+@pytest.mark.provider_contract('zoomeye')
+@pytest.mark.asyncio
+async def test_process_reuses_session_and_collects_all_capabilities(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(zoomeyesearch.Core, 'zoomeye_key', staticmethod(lambda: 'test-key'))
+    session = object()
+    session_exited = False
+    calls: list[dict[str, Any]] = []
+    responses = [
+        FetcherResponse(
+            {
+                'code': 60000,
+                'total': 1,
+                'data': [
+                    {
+                        'ip': '192.0.2.1',
+                        'domain': 'api.example.com',
+                        'hostname': 'host.example.com',
+                        'asn': 64500,
+                        'url': 'https://portal.example.com/v1',
+                        'banner': 'admin@example.com',
+                    }
+                ],
+            },
+            200,
+            {},
+        )
+    ]
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**kwargs: Any) -> AsyncIterator[object]:
+        nonlocal session_exited
+        assert kwargs == {
+            'headers': {'API-KEY': 'test-key', 'Content-Type': 'application/json'},
+            'proxy': True,
+        }
+        try:
+            yield session
+        finally:
+            session_exited = True
+
+    async def fake_post_fetch(*args: Any, **kwargs: Any) -> FetcherResponse:
+        calls.append({'url': args[0], **kwargs})
+        return responses.pop(0)
+
+    monkeypatch.setattr(zoomeyesearch.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(zoomeyesearch.AsyncFetcher, 'post_fetch', fake_post_fetch)
+    search = zoomeyesearch.SearchZoomEye('example.com', 2)
+    await search.process(proxy=True)
+
+    assert await search.get_hostnames() == {'api.example.com', 'host.example.com', 'portal.example.com'}
+    assert await search.get_ips() == {'192.0.2.1'}
+    assert await search.get_asns() == {'AS64500'}
+    assert await search.get_urls() == {'https://portal.example.com/v1'}
+    assert await search.get_emails() == {'admin@example.com'}
+    assert search.execution_status == 'completed'
+    assert search.stop_reason is None
+    assert [call['url'] for call in calls] == [search.baseurl]
+    assert all(call['session'] is session for call in calls)
+    assert session_exited is True
+    assert base64.b64decode(calls[0]['json_body']['qbase64']).decode() == 'domain="example.com"'
+    assert calls[0]['json_body']['page'] == 1
+    assert calls[0]['json_body']['pagesize'] == 2
+
+
+@pytest.mark.parametrize('key', [None, '', '  '])
+def test_missing_or_blank_key_fails_closed(monkeypatch: pytest.MonkeyPatch, key: str | None) -> None:
+    monkeypatch.setattr(zoomeyesearch.Core, 'zoomeye_key', staticmethod(lambda: key))
+    with pytest.raises(MissingKey):
+        zoomeyesearch.SearchZoomEye('example.com', 1)
+
+
+@pytest.mark.parametrize('limit', [0, -1, True, 1.5])
+def test_limit_must_be_a_positive_integer(monkeypatch: pytest.MonkeyPatch, limit: Any) -> None:
+    monkeypatch.setattr(zoomeyesearch.Core, 'zoomeye_key', staticmethod(lambda: 'test-key'))
+    with pytest.raises(ValueError, match='positive integer'):
+        zoomeyesearch.SearchZoomEye('example.com', limit)
+
+
+@pytest.mark.parametrize(
+    ('response', 'status', 'reason'),
+    [
+        (None, 'failed', 'transport-error'),
+        (FetcherResponse({}, 401, {}), 'failed', 'access-denied'),
+        (FetcherResponse({}, 429, {}), 'rate-limited', 'http-429'),
+        (FetcherResponse({}, 503, {}), 'failed', 'http-503'),
+        (FetcherResponse([], 200, {}), 'failed', 'invalid-response'),
+        (FetcherResponse({'code': 60001}, 200, {}), 'failed', 'provider-error'),
+    ],
+)
+@pytest.mark.asyncio
+async def test_provider_failures_are_truthful(
+    monkeypatch: pytest.MonkeyPatch,
+    response: FetcherResponse | None,
+    status: str,
+    reason: str,
+) -> None:
+    monkeypatch.setattr(zoomeyesearch.Core, 'zoomeye_key', staticmethod(lambda: 'test-key'))
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        yield object()
+
+    async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse | None:
+        return response
+
+    monkeypatch.setattr(zoomeyesearch.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(zoomeyesearch.AsyncFetcher, 'post_fetch', fake_post_fetch)
+    search = zoomeyesearch.SearchZoomEye('example.com', 1)
+    await search.process()
+
+    assert search.execution_status == status
+    assert search.stop_reason == reason
+
+
+@pytest.mark.asyncio
+async def test_empty_pages_do_not_hide_later_provider_results(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(zoomeyesearch.Core, 'zoomeye_key', staticmethod(lambda: 'test-key'))
+    responses = [
+        FetcherResponse(
+            {
+                'code': 60000,
+                'total': 70_000,
+                'data': ([{'hostname': 'late.example.com'}] if page == 6 else []),
+            },
+            200,
+            {},
+        )
+        for page in range(1, 8)
+    ]
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        yield object()
+
+    async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        return responses.pop(0)
+
+    monkeypatch.setattr(zoomeyesearch.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(zoomeyesearch.AsyncFetcher, 'post_fetch', fake_post_fetch)
+    search = zoomeyesearch.SearchZoomEye('example.com', 70_000)
+    await search.process()
+
+    assert await search.get_hostnames() == {'late.example.com'}
+    assert responses == []
+    assert search.execution_status == 'completed'
+
+
+@pytest.mark.asyncio
+async def test_numbered_pages_keep_a_stable_size_and_slice_the_final_page(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(zoomeyesearch.Core, 'zoomeye_key', staticmethod(lambda: 'test-key'))
+    calls: list[dict[str, Any]] = []
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        yield object()
+
+    async def fake_post_fetch(*_args: Any, **kwargs: Any) -> FetcherResponse:
+        calls.append(kwargs['json_body'])
+        return FetcherResponse({'code': 60000, 'total': 10_005, 'data': []}, 200, {})
+
+    monkeypatch.setattr(zoomeyesearch.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(zoomeyesearch.AsyncFetcher, 'post_fetch', fake_post_fetch)
+    search = zoomeyesearch.SearchZoomEye('example.com', 10_005)
+
+    await search.process()
+
+    assert [(call['page'], call['pagesize']) for call in calls] == [(1, 10_000), (2, 10_000)]
+
+
+@pytest.mark.asyncio
+async def test_early_malformed_rows_and_later_valid_evidence_are_partial(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(zoomeyesearch.Core, 'zoomeye_key', staticmethod(lambda: 'test-key'))
+    monkeypatch.setattr(zoomeyesearch.SearchZoomEye, 'PAGE_SIZE', 1)
+    responses = [
+        FetcherResponse({'code': 60000, 'total': 2, 'data': [7]}, 200, {}),
+        FetcherResponse({'code': 60000, 'total': 2, 'data': [{'hostname': 'api.example.com'}]}, 200, {}),
+    ]
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        yield object()
+
+    async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        return responses.pop(0)
+
+    monkeypatch.setattr(zoomeyesearch.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(zoomeyesearch.AsyncFetcher, 'post_fetch', fake_post_fetch)
+    search = zoomeyesearch.SearchZoomEye('example.com', 2)
+
+    await search.process()
+
+    assert await search.get_hostnames() == {'api.example.com'}
+    assert search.execution_status == 'partial'
+    assert search.stop_reason == 'invalid-response'
 
 
 @pytest.mark.asyncio
@@ -18,6 +222,31 @@ async def test_banner_urls_are_absolute_http_and_scoped(monkeypatch: pytest.Monk
         )
     )
 
-    _hostnames, _emails, _ips, _asns, urls = await search.parse_matches([{'service': {'banner': banner}}])
+    _hostnames, _emails, _ips, _asns, urls, malformed = await search.parse_matches([{'banner': banner}])
 
     assert urls == {'https://api.example.com/v1'}
+    assert malformed is False
+
+
+@pytest.mark.asyncio
+async def test_cancellation_closes_provider_session(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(zoomeyesearch.Core, 'zoomeye_key', staticmethod(lambda: 'test-key'))
+    session_exited = False
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]:
+        nonlocal session_exited
+        try:
+            yield object()
+        finally:
+            session_exited = True
+
+    async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse:
+        raise asyncio.CancelledError('operator-stop')
+
+    monkeypatch.setattr(zoomeyesearch.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(zoomeyesearch.AsyncFetcher, 'post_fetch', fake_post_fetch)
+
+    with pytest.raises(asyncio.CancelledError, match='operator-stop'):
+        await zoomeyesearch.SearchZoomEye('example.com', 10).process()
+    assert session_exited is True

tests/lib/test_core.py

@@ -1127,6 +1127,26 @@ async def test_fetch_all_propagates_metadata_opt_in(monkeypatch) -> None:
     assert [result.status for result in results] == [429, 429]
 
 
+@pytest.mark.asyncio
+async def test_fetch_all_reuses_a_caller_owned_session(monkeypatch: pytest.MonkeyPatch) -> None:
+    session = object()
+    seen_sessions: list[object] = []
+
+    async def fake_fetch(*_args: Any, session: object, **_kwargs: Any) -> str:
+        seen_sessions.append(session)
+        return 'ok'
+
+    monkeypatch.setattr(AsyncFetcher, 'fetch', fake_fetch)
+
+    results = await AsyncFetcher.fetch_all(
+        ['https://one.example', 'https://two.example'],
+        session=session,
+    )
+
+    assert results == ['ok', 'ok']
+    assert seen_sessions == [session, session]
+
+
 @pytest.mark.asyncio
 async def test_fetch_uses_http_proxy_when_enabled(monkeypatch) -> None:
     reset_dummy_sessions()

tests/lib/test_source_runner.py

@@ -74,7 +74,7 @@ def test_source_factories_match_the_catalog() -> None:
         ),
         ('duckduckgo', 'theHarvester.lib.source_runner.duckduckgosearch.SearchDuckDuckGo', ('example.test', 25), {}),
         ('dymo', 'theHarvester.lib.source_runner.dymosearch.SearchDymo', ('example.test',), {}),
-        ('fofa', 'theHarvester.lib.source_runner.fofa.SearchFofa', ('example.test',), {}),
+        ('fofa', 'theHarvester.lib.source_runner.fofa.SearchFofa', ('example.test', 25), {}),
         ('fullhunt', 'theHarvester.lib.source_runner.fullhuntsearch.SearchFullHunt', ('example.test',), {}),
         ('github-code', 'theHarvester.lib.source_runner.githubcode.SearchGithubCode', ('example.test', 25), {}),
         ('gitlab', 'theHarvester.lib.source_runner.gitlabsearch.SearchGitlab', ('example.test',), {}),
@@ -98,13 +98,13 @@ def test_source_factories_match_the_catalog() -> None:
         ),
         ('hudsonrock', 'theHarvester.lib.source_runner.hudsonrocksearch.SearchHudsonRock', ('example.test',), {}),
         ('hunter', 'theHarvester.lib.source_runner.huntersearch.SearchHunter', ('example.test', 25, 5), {}),
-        ('hunterhow', 'theHarvester.lib.source_runner.searchhunterhow.SearchHunterHow', ('example.test',), {}),
+        ('hunterhow', 'theHarvester.lib.source_runner.searchhunterhow.SearchHunterHow', ('example.test', 25), {}),
         ('intelx', 'theHarvester.lib.source_runner.intelxsearch.SearchIntelx', ('example.test',), {}),
         ('leakix', 'theHarvester.lib.source_runner.leakix.SearchLeakix', ('example.test',), {}),
         ('leaklookup', 'theHarvester.lib.source_runner.leaklookup.SearchLeakLookup', ('example.test',), {}),
         ('mojeek', 'theHarvester.lib.source_runner.mojeek.SearchMojeek', ('example.test', 25), {}),
         ('netlas', 'theHarvester.lib.source_runner.netlas.SearchNetlas', ('example.test', 25), {}),
-        ('onyphe', 'theHarvester.lib.source_runner.onyphe.SearchOnyphe', ('example.test',), {}),
+        ('onyphe', 'theHarvester.lib.source_runner.onyphe.SearchOnyphe', ('example.test', 25), {}),
         ('otx', 'theHarvester.lib.source_runner.otxsearch.SearchOtx', ('example.test',), {}),
         (
             'pentesttools',
@@ -130,7 +130,7 @@ def test_source_factories_match_the_catalog() -> None:
         (
             'securityscorecard',
             'theHarvester.lib.source_runner.securityscorecard.SearchSecurityScorecard',
-            ('example.test',),
+            ('example.test', 25),
             {},
         ),
         (
@@ -162,15 +162,15 @@ def test_source_factories_match_the_catalog() -> None:
         ),
         ('thc', 'theHarvester.lib.source_runner.thc.SearchThc', ('example.test',), {}),
         ('tomba', 'theHarvester.lib.source_runner.tombasearch.SearchTomba', ('example.test', 25, 5), {}),
-        ('urlscan', 'theHarvester.lib.source_runner.urlscan.SearchUrlscan', ('example.test',), {}),
-        ('virustotal', 'theHarvester.lib.source_runner.virustotal.SearchVirustotal', ('example.test',), {}),
+        ('urlscan', 'theHarvester.lib.source_runner.urlscan.SearchUrlscan', ('example.test', 25), {}),
+        ('virustotal', 'theHarvester.lib.source_runner.virustotal.SearchVirustotal', ('example.test', 25), {}),
         (
             'waybackarchive',
             'theHarvester.lib.source_runner.waybackarchive.SearchWaybackarchive',
             ('example.test', 25),
             {},
         ),
-        ('whoisxml', 'theHarvester.lib.source_runner.whoisxml.SearchWhoisXML', ('example.test',), {}),
+        ('whoisxml', 'theHarvester.lib.source_runner.whoisxml.SearchWhoisXML', ('example.test', 25), {}),
         ('windvane', 'theHarvester.lib.source_runner.windvane.SearchWindvane', ('example.test',), {}),
         ('yahoo', 'theHarvester.lib.source_runner.yahoosearch.SearchYahoo', ('example.test', 25), {}),
         ('zoomeye', 'theHarvester.lib.source_runner.zoomeyesearch.SearchZoomEye', ('example.test', 25), {}),

tests/test_hackertarget_apikey.py

@@ -291,3 +291,6 @@ class TestHackerTargetApiKey:
 
         with pytest.raises(asyncio.CancelledError):
             await search.process(proxy=True)
+
+
+pytestmark = pytest.mark.provider_contract('hackertarget')

tests/test_main.py

@@ -1231,8 +1231,9 @@ async def test_rest_dns_lookup_runs_before_return_and_retains_action_evidence(mo
             completed.append(result)
 
     class FakeSecurityScorecard:
-        def __init__(self, domain: str) -> None:
+        def __init__(self, domain: str, limit: int) -> None:
             assert domain == 'example.com'
+            assert limit == 500
 
         async def process(self, _proxy: bool) -> None:
             return None
@@ -1315,8 +1316,8 @@ async def test_dns_lookup_cancellation_persists_partial_evidence(
             completed.append(result)
 
     class FakeSecurityScorecard:
-        def __init__(self, _domain: str) -> None:
-            pass
+        def __init__(self, _domain: str, limit: int) -> None:
+            assert limit == 500
 
         async def process(self, _proxy: bool) -> None:
             return None
@@ -3830,8 +3831,8 @@ async def test_routeviews_pivots_from_attributed_ips_without_expanding_discovere
     calls: list[tuple[tuple[object, ...], tuple[str, ...]]] = []
 
     class FakeUrlscan:
-        def __init__(self, _word: str) -> None:
-            pass
+        def __init__(self, _word: str, limit: int) -> None:
+            assert limit == 500
 
         async def process(self, _proxy: bool) -> None:
             return None

tests/test_mojeek.py

@@ -207,3 +207,6 @@ class TestMojeekSearch:
         }
         assert search.execution_status == 'completed'
         assert search.stop_reason is None
+
+
+pytestmark = pytest.mark.provider_contract('mojeek')

tests/test_provider_contract_coverage.py

@@ -0,0 +1,34 @@
+from collections import Counter
+
+from theHarvester.lib.source_catalog import SOURCE_SPECS
+
+
+def _provider_contract_problems(
+    sources: tuple[str, ...],
+    canonical_sources: set[str],
+) -> list[str]:
+    counts = Counter(sources)
+    marked_sources = set(counts)
+    problems: list[str] = []
+    if unknown := sorted(marked_sources - canonical_sources):
+        problems.append(f'unknown provider contracts: {", ".join(unknown)}')
+    if duplicates := sorted(source for source, count in counts.items() if count > 1):
+        problems.append(f'duplicate provider contracts: {", ".join(duplicates)}')
+    if missing := sorted(canonical_sources - marked_sources):
+        problems.append(f'missing provider contracts: {", ".join(missing)}')
+    return problems
+
+
+def test_every_canonical_source_has_one_offline_provider_contract(
+    provider_contract_sources: tuple[str, ...],
+) -> None:
+    problems = _provider_contract_problems(provider_contract_sources, set(SOURCE_SPECS))
+    assert not problems, '; '.join(problems)
+
+
+def test_provider_contract_failures_name_unknown_duplicate_and_missing_sources() -> None:
+    assert _provider_contract_problems(('known', 'known', 'unknown'), {'known', 'missing'}) == [
+        'unknown provider contracts: unknown',
+        'duplicate provider contracts: known',
+        'missing provider contracts: missing',
+    ]

tests/test_workflow_contract.py

@@ -25,6 +25,8 @@ def test_routine_ci_is_read_only_and_offline() -> None:
     assert 'git push' not in commands
     assert 'theHarvester -d' not in commands
     assert '\npytest\n' in f'\n{commands.strip()}\n'
+    assert 'mypy theHarvester' in commands
+    assert routine_job['strategy']['matrix']['python-version'] == ['3.12', '3.13', '3.14']
 
 
 def test_live_provider_smoke_requires_manual_dispatch() -> None:
@@ -36,6 +38,9 @@ def test_live_provider_smoke_requires_manual_dispatch() -> None:
     assert workflow['permissions'] == {'contents': 'read'}
     assert smoke_job['env']['SMOKE_TEST_DOMAIN'] == 'mozilla.org'
     assert 'pytest --run-live-network -m live_network' in commands
+    cli_smokes = [line for line in commands.splitlines() if line.startswith('theHarvester -d')]
+    assert cli_smokes
+    assert all('-l 10 -q' in command for command in cli_smokes)
 
 
 def test_harvestview_browser_failures_keep_only_targeted_diagnostics() -> None:

theHarvester/discovery/bevigil.py

@@ -1,37 +1,106 @@
+from urllib.parse import urlsplit, urlunsplit
+
 from theHarvester.discovery.constants import MissingKey
-from theHarvester.lib.core import AsyncFetcher, Core
+from theHarvester.discovery.provider_response import provider_http_error
+from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse
+from theHarvester.lib.hostnames import normalize_scoped_hostname
 
 
 class SearchBeVigil:
-    def __init__(self, word) -> None:
+    def __init__(self, word: str) -> None:
         self.word = word
-        self.totalhosts: set = set()
-        self.urls: set = set()
+        self.totalhosts: set[str] = set()
+        self.urls: set[str] = set()
         self.key = Core.bevigil_key()
-        if self.key is None:
-            self.key = ''
+        if not isinstance(self.key, str) or not self.key.strip():
             raise MissingKey('bevigil')
         self.proxy = False
+        self.execution_status: str | None = None
+        self.stop_reason: str | None = None
+
+    def _has_results(self) -> bool:
+        return bool(self.totalhosts or self.urls)
+
+    def _stop(self, status: str, reason: str) -> None:
+        self.execution_status = 'partial' if self._has_results() else status
+        self.stop_reason = reason
+
+    def _scoped_url(self, value: object) -> str | None:
+        if not isinstance(value, str):
+            return None
+        try:
+            parsed = urlsplit(value.strip())
+            hostname = normalize_scoped_hostname(parsed.hostname, self.word)
+            port = parsed.port
+        except ValueError:
+            return None
+        if parsed.scheme.casefold() not in {'http', 'https'} or hostname is None or parsed.username or parsed.password:
+            return None
+        netloc = f'{hostname}:{port}' if port is not None else hostname
+        return urlunsplit((parsed.scheme.casefold(), netloc, parsed.path, parsed.query, ''))
 
     async def do_search(self) -> None:
+        self.execution_status = None
+        self.stop_reason = None
         subdomain_endpoint = f'https://osint.bevigil.com/api/{self.word}/subdomains/'
         url_endpoint = f'https://osint.bevigil.com/api/{self.word}/urls/'
         headers = {'X-Access-Token': self.key}
+        requests = (
+            (subdomain_endpoint, 'subdomains'),
+            (url_endpoint, 'urls'),
+        )
 
-        responses = await AsyncFetcher.fetch_all([subdomain_endpoint], json=True, proxy=self.proxy, headers=headers)
-        response = responses[0]
-        for subdomain in response['subdomains']:
-            self.totalhosts.add(subdomain)
+        try:
+            async with AsyncFetcher.open_session(
+                headers=headers,
+                proxy=self.proxy,
+                request_timeout=60,
+            ) as session:
+                for endpoint, field in requests:
+                    responses = await AsyncFetcher.fetch_all(
+                        [endpoint],
+                        json=True,
+                        proxy=self.proxy,
+                        headers=headers,
+                        include_metadata=True,
+                        session=session,
+                    )
+                    response = responses[0] if responses else None
+                    if error := provider_http_error(response):
+                        self._stop(*error)
+                        return
+                    assert isinstance(response, FetcherResponse)
+                    if not isinstance(response.body, dict) or not isinstance(response.body.get(field), list):
+                        self._stop('failed', 'invalid-response')
+                        return
 
-        responses = await AsyncFetcher.fetch_all([url_endpoint], json=True, proxy=self.proxy, headers=headers)
-        response = responses[0]
-        for url in response['urls']:
-            self.urls.add(url)
+                    malformed = False
+                    for value in response.body[field]:
+                        if field == 'subdomains':
+                            if not isinstance(value, str):
+                                malformed = True
+                            elif hostname := normalize_scoped_hostname(value, self.word):
+                                self.totalhosts.add(hostname)
+                        elif url := self._scoped_url(value):
+                            self.urls.add(url)
+                        elif not isinstance(value, str):
+                            malformed = True
+                    if malformed:
+                        self._stop('failed', 'invalid-response')
+        except Exception:
+            self._stop('failed', 'transport-error')
+            return
 
-    async def get_hostnames(self) -> set:
+        if self.execution_status is not None and self._has_results():
+            self.execution_status = 'partial'
+        elif self.execution_status is None:
+            self.execution_status = 'completed'
+            self.stop_reason = None if self._has_results() else 'no-results'
+
+    async def get_hostnames(self) -> set[str]:
         return self.totalhosts
 
-    async def get_urls(self) -> set:
+    async def get_urls(self) -> set[str]:
         return self.urls
 
     async def process(self, proxy: bool = False) -> None:

theHarvester/discovery/dymosearch.py

@@ -1,7 +1,9 @@
 from typing import Any
 
 from theHarvester.discovery.constants import MissingKey
-from theHarvester.lib.core import AsyncFetcher, Core
+from theHarvester.discovery.provider_response import provider_http_error
+from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse
+from theHarvester.lib.hostnames import normalize_scoped_hostname
 
 
 class SearchDymo:
@@ -24,9 +26,15 @@ class SearchDymo:
         self.totalhosts: set[str] = set()
         self.results: dict[str, Any] = {}
         self.key = Core.dymo_key()
-        if self.key is None:
+        if not isinstance(self.key, str) or not self.key.strip():
             raise MissingKey('dymo')
         self.proxy = False
+        self.execution_status: str | None = None
+        self.stop_reason: str | None = None
+
+    def _stop(self, status: str, reason: str) -> None:
+        self.execution_status = 'partial' if self.totalhosts else status
+        self.stop_reason = reason
 
     def _headers(self) -> dict[str, str]:
         return {
@@ -43,26 +51,45 @@ class SearchDymo:
         response = await AsyncFetcher.post_fetch(
             self.VERIFY_URL,
             headers=self._headers(),
-            data=payload,
             json=True,
+            json_body=payload,
             proxy=self.proxy,
+            include_metadata=True,
         )
-        if not isinstance(response, dict):
+        if error := provider_http_error(response):
+            self._stop(*error)
+            return
+        assert isinstance(response, FetcherResponse)
+        if not isinstance(response.body, dict):
+            self._stop('failed', 'invalid-response')
             return
 
-        self.results = response
+        self.results = response.body
 
-        domain_block = response.get('domain') if isinstance(response.get('domain'), dict) else {}
-        url_block = response.get('url') if isinstance(response.get('url'), dict) else {}
+        raw_domain_block = response.body.get('domain')
+        raw_url_block = response.body.get('url')
+        malformed = any(block is not None and not isinstance(block, dict) for block in (raw_domain_block, raw_url_block))
+        domain_block = raw_domain_block if isinstance(raw_domain_block, dict) else {}
+        url_block = raw_url_block if isinstance(raw_url_block, dict) else {}
 
         for block in (domain_block, url_block):
             candidate = block.get('domain') if isinstance(block, dict) else None
-            if isinstance(candidate, str) and self.word in candidate:
-                self.totalhosts.add(candidate)
+            if normalized := normalize_scoped_hostname(candidate, self.word):
+                self.totalhosts.add(normalized)
+            elif candidate is not None and not isinstance(candidate, str):
+                malformed = True
 
             suggestion = block.get('didYouMean') if isinstance(block, dict) else None
-            if isinstance(suggestion, str) and self.word in suggestion:
-                self.totalhosts.add(suggestion)
+            if normalized := normalize_scoped_hostname(suggestion, self.word):
+                self.totalhosts.add(normalized)
+            elif suggestion is not None and not isinstance(suggestion, str):
+                malformed = True
+
+        if malformed:
+            self._stop('failed', 'invalid-response')
+        else:
+            self.execution_status = 'completed'
+            self.stop_reason = None if self.totalhosts else 'no-results'
 
     async def get_hostnames(self) -> set:
         return self.totalhosts
@@ -72,4 +99,9 @@ class SearchDymo:
 
     async def process(self, proxy: bool = False) -> None:
         self.proxy = proxy
-        await self.do_search()
+        self.execution_status = None
+        self.stop_reason = None
+        try:
+            await self.do_search()
+        except Exception:
+            self._stop('failed', 'transport-error')

theHarvester/discovery/fofa.py

@@ -1,30 +1,33 @@
 import base64
-import logging
 from ipaddress import ip_address
+from typing import Any
 from urllib.parse import urlparse
 
 from theHarvester.discovery.constants import MissingKey
+from theHarvester.discovery.provider_response import provider_http_error
 from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse
 from theHarvester.lib.hostnames import normalize_scoped_hostname
 
-logger = logging.getLogger(__name__)
-
 
 class SearchFofa:
-    """Class uses Fofa API to search for domain and host intelligence
-    Fofa is a Chinese search engine for network-connected devices
-    """
+    """Collect scoped domain assets through FOFA's cursor search API."""
 
-    def __init__(self, word) -> None:
+    MAX_PAGE_SIZE = 10_000
+
+    def __init__(self, word: str, limit: int) -> None:
+        if isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0:
+            raise ValueError('FOFA limit must be a positive integer')
         self.word = word
-        self.totalhosts: set = set()
-        self.totalips: set = set()
+        self.limit = limit
+        self.totalhosts: set[str] = set()
+        self.totalips: set[str] = set()
         self.proxy = False
         self.hostname = 'https://fofa.info'
         self.api_key, self.email = self._get_api_credentials()
+        self.execution_status: str | None = None
+        self.stop_reason: str | None = None
 
     def _get_api_credentials(self) -> tuple[str, str]:
-        """Get Fofa API credentials"""
         try:
             api_key, email = Core.fofa_key()
         except Exception as error:
@@ -33,99 +36,122 @@ class SearchFofa:
             raise MissingKey('Fofa API (key and email required)')
         return api_key, email
 
+    def _has_results(self) -> bool:
+        return bool(self.totalhosts or self.totalips)
+
+    def _stop(self, status: str, reason: str) -> None:
+        self.execution_status = 'partial' if self._has_results() else status
+        self.stop_reason = reason
+
+    def _store_results(self, results: list[Any]) -> bool:
+        malformed = False
+        for result in results:
+            if not isinstance(result, list) or len(result) < 2:
+                malformed = True
+                continue
+            host, address = result[:2]
+            if isinstance(host, str):
+                try:
+                    parsed_hostname = urlparse(host if '://' in host else f'//{host}').hostname
+                except ValueError:
+                    malformed = True
+                else:
+                    if clean_host := normalize_scoped_hostname(parsed_hostname, self.word):
+                        self.totalhosts.add(clean_host)
+            else:
+                malformed = True
+            if isinstance(address, str):
+                try:
+                    self.totalips.add(str(ip_address(address)))
+                except ValueError:
+                    malformed = True
+            else:
+                malformed = True
+        return malformed
+
+    def _provider_error(self, body: dict[str, Any]) -> None:
+        message = body.get('errmsg')
+        normalized = message.casefold() if isinstance(message, str) else ''
+        if 'invalid' in normalized or '账号无效' in normalized:
+            self._stop('failed', 'access-denied')
+        elif any(term in normalized for term in ('quota', 'limit', 'plan')):
+            self._stop('failed', 'quota-exhausted')
+        else:
+            self._stop('failed', 'provider-error')
+
     async def do_search(self) -> None:
+        query = base64.b64encode(f'domain="{self.word}"'.encode()).decode()
+        cursor: str | None = None
+        seen_cursors: set[str] = set()
+        records_seen = 0
         try:
-            headers = {'User-agent': Core.get_user_agent()}
-
-            # Fofa search query - encode in base64
-            query = f'domain="{self.word}"'
-            query_encoded = base64.b64encode(query.encode()).decode()
-
-            # Fofa API endpoint
-            url = f'{self.hostname}/api/v1/search/all'
-            params = {
-                'email': self.email,
-                'key': self.api_key,
-                'qbase64': query_encoded,
-                'fields': 'host,ip,port,protocol,title',
-                'size': 100,  # Limit results
-            }
-
-            # Build URL with parameters
-            param_string = '&'.join([f'{k}={v}' for k, v in params.items()])
-            full_url = f'{url}?{param_string}'
-
-            response = await AsyncFetcher.fetch_all(
-                [full_url],
-                headers=headers,
+            async with AsyncFetcher.open_session(
+                headers={'User-Agent': Core.get_user_agent()},
                 proxy=self.proxy,
-                json=True,
-                include_metadata=True,
-            )
+            ) as session:
+                while records_seen < self.limit:
+                    remaining = self.limit - records_seen
+                    params: dict[str, str | int] = {
+                        'email': self.email,
+                        'key': self.api_key,
+                        'qbase64': query,
+                        'fields': 'host,ip',
+                        'size': min(self.MAX_PAGE_SIZE, remaining),
+                    }
+                    if cursor is not None:
+                        params['next'] = cursor
+                    response = await AsyncFetcher.fetch(
+                        session=session,
+                        url=f'{self.hostname}/api/v1/search/next',
+                        params=params,
+                        json=True,
+                        include_metadata=True,
+                    )
+                    if error := provider_http_error(response):
+                        self._stop(*error)
+                        return
+                    assert isinstance(response, FetcherResponse)
+                    if not isinstance(response.body, dict):
+                        self._stop('failed', 'invalid-response')
+                        return
+                    if response.body.get('error') is True:
+                        self._provider_error(response.body)
+                        return
+                    results = response.body.get('results')
+                    if not isinstance(results, list):
+                        self._stop('failed', 'invalid-response')
+                        return
 
-            metadata = response[0] if response and isinstance(response[0], FetcherResponse) else None
-            if metadata is None:
-                logger.info(f'No response from Fofa API for: {self.word}')
-                return
-            if not 200 <= metadata.status < 300:
-                logger.info(f'Fofa request failed with HTTP {metadata.status}')
-                return
+                    page_results = results[:remaining]
+                    records_seen += len(page_results)
+                    if self._store_results(page_results):
+                        self._stop('failed', 'invalid-response')
+                    next_cursor = response.body.get('next')
+                    if not results or not isinstance(next_cursor, str) or not next_cursor:
+                        break
+                    if next_cursor in seen_cursors:
+                        self._stop('failed', 'repeated-cursor')
+                        break
+                    seen_cursors.add(next_cursor)
+                    cursor = next_cursor
+        except Exception:
+            self._stop('failed', 'transport-error')
+            return
 
-            try:
-                data = metadata.body
-                if not isinstance(data, dict):
-                    logger.info('Fofa returned malformed data')
-                    return
+        if self.execution_status is not None and self._has_results():
+            self.execution_status = 'partial'
+        elif self.execution_status is None:
+            self.execution_status = 'completed'
+            self.stop_reason = None if self._has_results() else 'no-results'
 
-                # Check for errors
-                if data.get('error', False):
-                    error_message = data.get('errmsg')
-                    normalized_error = error_message.casefold() if isinstance(error_message, str) else ''
-                    if 'invalid' in normalized_error or '账号无效' in normalized_error:
-                        logger.info('Fofa API rejected the configured credentials')
-                    elif any(term in normalized_error for term in ('quota', 'limit', 'plan')):
-                        logger.info('Fofa API quota or plan limit was reached')
-                    else:
-                        logger.info('Fofa API returned an error')
-                    return
-
-                # Extract results
-                results = data.get('results', [])
-                if not isinstance(results, list):
-                    logger.info('Fofa returned malformed results')
-                    return
-                for result in results:
-                    if isinstance(result, list) and len(result) >= 2:
-                        host = result[0]  # host field
-                        ip = result[1]  # ip field
-
-                        # Add host if it's related to our domain
-                        if isinstance(host, str):
-                            parsed = urlparse(host if '://' in host else f'//{host}')
-                            if clean_host := normalize_scoped_hostname(parsed.hostname, self.word):
-                                self.totalhosts.add(clean_host)
-
-                        # Add IP
-                        if isinstance(ip, str) and ip:
-                            try:
-                                self.totalips.add(str(ip_address(ip)))
-                            except ValueError:
-                                continue
-
-            except Exception as e:
-                logger.info(f'Failed to parse Fofa response: {e}')
-
-        except MissingKey:
-            raise
-        except Exception as e:
-            logger.info(f'Fofa API error: {e}')
-
-    async def get_hostnames(self) -> set:
+    async def get_hostnames(self) -> set[str]:
         return self.totalhosts
 
-    async def get_ips(self) -> set:
+    async def get_ips(self) -> set[str]:
         return self.totalips
 
     async def process(self, proxy: bool = False) -> None:
         self.proxy = proxy
+        self.execution_status = None
+        self.stop_reason = None
         await self.do_search()

theHarvester/discovery/fullhuntsearch.py

@@ -4,6 +4,7 @@ from typing import Any, ClassVar
 from urllib.parse import quote
 
 from theHarvester.discovery.constants import MissingKey
+from theHarvester.discovery.provider_response import provider_http_error
 from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse
 from theHarvester.lib.hostnames import normalize_scoped_hostname
 
@@ -140,12 +141,21 @@ class SearchFullHunt:
         }
         self.proxy = False
         self.filters: dict[str, str] = {}  # Store filters for advanced searches
+        self.execution_status: str | None = None
+        self.stop_reason: str | None = None
+
+    def _has_results(self) -> bool:
+        return bool(self.total_results['hosts'] or self.total_results['ips'])
+
+    def _stop(self, status: str, reason: str) -> None:
+        self.execution_status = 'partial' if self._has_results() else status
+        self.stop_reason = reason
 
     def _get_headers(self) -> dict[str, str]:
         """Returns the headers needed for API requests"""
         return {'User-Agent': Core.get_user_agent(), 'X-API-KEY': self.key}
 
-    async def _fetch_data(self, endpoint: str) -> dict[str, Any]:
+    async def _fetch_data(self, endpoint: str, session: Any | None = None) -> dict[str, Any]:
         """Generic method to fetch data from a specific endpoint"""
         url = f'{self.BASE_URL}/{endpoint}'
         response = await AsyncFetcher.fetch_all(
@@ -154,13 +164,15 @@ class SearchFullHunt:
             headers=self._get_headers(),
             proxy=self.proxy,
             include_metadata=True,
+            session=session,
         )
-        metadata = response[0] if response and isinstance(response[0], FetcherResponse) else None
-        if metadata is None:
-            raise RuntimeError('FullHunt request failed')
-        if not 200 <= metadata.status < 300:
-            raise RuntimeError(f'FullHunt request failed with HTTP {metadata.status}')
+        metadata = response[0] if response else None
+        if error := provider_http_error(metadata):
+            self._stop(*error)
+            raise RuntimeError(f'FullHunt request failed: {error[1]}')
+        assert isinstance(metadata, FetcherResponse)
         if not isinstance(metadata.body, dict):
+            self._stop('failed', 'invalid-response')
             raise ValueError('FullHunt returned malformed data')
         return metadata.body
 
@@ -213,7 +225,7 @@ class SearchFullHunt:
 
         return ' '.join(query_parts)
 
-    async def advanced_search(self) -> dict[str, Any]:
+    async def advanced_search(self, session: Any | None = None) -> dict[str, Any]:
         """Perform an advanced search using the configured filters
 
         This method uses the search endpoint with the filters configured via add_filter
@@ -226,17 +238,17 @@ class SearchFullHunt:
         query = self._build_query_string()
         encoded_query = quote(query)
         endpoint = f'search?query={encoded_query}'
-        return await self._fetch_data(endpoint)
+        return await self._fetch_data(endpoint, session)
 
-    async def get_domain_details(self) -> dict[str, Any]:
+    async def get_domain_details(self, session: Any | None = None) -> dict[str, Any]:
         """Get comprehensive details about a domain"""
         endpoint = f'domain/{self.word}/details'
-        return await self._fetch_data(endpoint)
+        return await self._fetch_data(endpoint, session)
 
-    async def get_subdomains(self) -> dict[str, Any]:
+    async def get_subdomains(self, session: Any | None = None) -> dict[str, Any]:
         """Get subdomains for a domain"""
         endpoint = f'domain/{self.word}/subdomains'
-        return await self._fetch_data(endpoint)
+        return await self._fetch_data(endpoint, session)
 
     async def get_host_details(self, host: str) -> dict[str, Any]:
         """Get detailed information about a specific host"""
@@ -301,6 +313,7 @@ class SearchFullHunt:
         hosts = details['hosts']
         for host_data in hosts:
             if not isinstance(host_data, dict):
+                self._stop('failed', 'invalid-response')
                 logger.info('FullHunt ignored a malformed host item')
                 continue
             hostname = normalize_scoped_hostname(host_data.get('host'), self.word)
@@ -332,6 +345,7 @@ class SearchFullHunt:
                     for field in ('dns_records', 'http_response', 'geo', 'cloud', 'certificate')
                 )
             ):
+                self._stop('failed', 'invalid-response')
                 logger.info('FullHunt ignored a malformed host item')
                 continue
             # Extract subdomains
@@ -418,32 +432,48 @@ class SearchFullHunt:
     async def do_search(self) -> None:
         """Main search method that calls the various endpoints"""
         try:
-            # First get domain details which includes most information
-            domain_details = await self.get_domain_details()
-            if not isinstance(domain_details.get('hosts'), list):
-                raise ValueError('FullHunt returned malformed domain details')
-            self.total_results['domain_details'] = domain_details
-            await self.extract_data_from_domain_details(domain_details)
+            async with AsyncFetcher.open_session(
+                headers=self._get_headers(),
+                proxy=self.proxy,
+                request_timeout=60,
+            ) as session:
+                # First get domain details which includes most information
+                domain_details = await self.get_domain_details(session)
+                if not isinstance(domain_details.get('hosts'), list):
+                    raise ValueError('FullHunt returned malformed domain details')
+                self.total_results['domain_details'] = domain_details
+                await self.extract_data_from_domain_details(domain_details)
 
-            # If no hosts found in domain details, try the dedicated subdomains endpoint
-            if not self.total_results['hosts']:
-                subdomains_response = await self.get_subdomains()
-                hosts = subdomains_response.get('hosts')
-                if not isinstance(hosts, list):
-                    raise ValueError('FullHunt returned malformed subdomains')
-                for host in hosts:
-                    if normalized_host := normalize_scoped_hostname(host, self.word):
-                        self.total_results['hosts'].append(normalized_host)
-                    else:
-                        logger.info('FullHunt ignored a malformed subdomain item')
+                # If no hosts found in domain details, try the dedicated subdomains endpoint
+                if not self.total_results['hosts']:
+                    subdomains_response = await self.get_subdomains(session)
+                    hosts = subdomains_response.get('hosts')
+                    if not isinstance(hosts, list):
+                        raise ValueError('FullHunt returned malformed subdomains')
+                    for host in hosts:
+                        if normalized_host := normalize_scoped_hostname(host, self.word):
+                            self.total_results['hosts'].append(normalized_host)
+                        else:
+                            self._stop('failed', 'invalid-response')
+                            logger.info('FullHunt ignored a malformed subdomain item')
 
-            # If filters are set, perform an advanced search
-            if self.filters:
-                search_results = await self.advanced_search()
-                await self.extract_data_from_search_results(search_results)
+                # If filters are set, perform an advanced search
+                if self.filters:
+                    search_results = await self.advanced_search(session)
+                    await self.extract_data_from_search_results(search_results)
 
-        except Exception as e:
-            logger.info(f'Error during FullHunt search: {e}')
+        except Exception as error:
+            if self.execution_status is None:
+                reason = 'invalid-response' if isinstance(error, ValueError) else 'transport-error'
+                self._stop('failed', reason)
+            logger.info('Error during FullHunt search: %s', type(error).__name__)
+            return
+
+        if self.execution_status is not None and self._has_results():
+            self.execution_status = 'partial'
+        elif self.execution_status is None:
+            self.execution_status = 'completed'
+            self.stop_reason = None if self._has_results() else 'no-results'
 
     async def get_hostnames(self) -> list[str]:
         """Return list of discovered subdomains"""
@@ -498,6 +528,8 @@ class SearchFullHunt:
 
         """
         self.proxy = proxy
+        self.execution_status = None
+        self.stop_reason = None
 
         # Apply filters if provided
         if filters:

theHarvester/discovery/netlas.py

@@ -1,63 +1,94 @@
-import json
+from __future__ import annotations
+
+from typing import Any
 
 from theHarvester.discovery.constants import MissingKey
-from theHarvester.lib.core import AsyncFetcher, Core
+from theHarvester.discovery.provider_response import provider_http_error
+from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse
+from theHarvester.lib.hostnames import normalize_scoped_hostname
 
 
 class SearchNetlas:
-    def __init__(self, word, limit: int) -> None:
+    def __init__(self, word: str, limit: int) -> None:
+        if isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0:
+            raise ValueError('Netlas limit must be a positive integer')
         self.word = word
-        self.totalhosts: list = []
-        self.totalips: list = []
-        self.key = Core.netlas_key()
         self.limit = limit
-        if self.key is None:
+        self.totalhosts: set[str] = set()
+        self.key = Core.netlas_key()
+        if not isinstance(self.key, str) or not self.key.strip():
             raise MissingKey('netlas')
         self.proxy = False
+        self.execution_status: str | None = None
+        self.stop_reason: str | None = None
 
-    async def do_count(self) -> None:
-        """Counts the total number of subdomains
+    def _stop(self, status: str, reason: str) -> None:
+        self.execution_status = 'partial' if self.totalhosts else status
+        self.stop_reason = reason
 
-        :return: None
-        """
-        api = f'https://app.netlas.io/api/domains_count/?q=*.{self.word}'
-        headers = {'X-API-Key': self.key}
-        response = await AsyncFetcher.fetch_all([api], json=True, headers=headers, proxy=self.proxy)
-        amount_size = response[0]['count']
-        self.limit = min(self.limit, amount_size)
+    def _response_body(self, response: Any) -> Any | None:
+        if isinstance(response, FetcherResponse) and response.status == 402:
+            self._stop('failed', 'quota-exhausted')
+            return None
+        if error := provider_http_error(response):
+            self._stop(*error)
+            return None
+        assert isinstance(response, FetcherResponse)
+        if response.body is None:
+            self._stop('failed', 'invalid-response')
+            return None
+        return response.body
 
-    async def do_search(self) -> None:
-        """Download domains for query 'q' size of 'limit'
+    async def do_search(self, session: Any, size: int) -> None:
+        response = await AsyncFetcher.post_fetch(
+            'https://app.netlas.io/api/domains/download/',
+            session=session,
+            json=True,
+            include_metadata=True,
+            json_body={
+                'q': f'*.{self.word}',
+                'size': size,
+                'fields': ['domain'],
+                'source_type': 'include',
+            },
+        )
+        body = self._response_body(response)
+        if body is None:
+            return
+        if not isinstance(body, list):
+            self._stop('failed', 'invalid-response')
+            return
 
-        :return: None
-        """
-        user_agent = Core.get_user_agent()
-        url = 'https://app.netlas.io/api/domains/download/'
+        malformed = False
+        for row in body[:size]:
+            if not isinstance(row, dict) or not isinstance(row.get('data'), dict):
+                malformed = True
+                continue
+            domain = row['data'].get('domain')
+            if not isinstance(domain, str):
+                malformed = True
+                continue
+            if hostname := normalize_scoped_hostname(domain, self.word):
+                self.totalhosts.add(hostname)
+        if malformed:
+            self._stop('failed', 'invalid-response')
 
-        payload = {
-            'q': f'*.{self.word}',
-            'fields': json.dumps(['domain']),  # Convert the list to a JSON string
-            'source_type': 'include',
-            'size': str(self.limit),  # Convert integer to string
-            'type': 'json',
-            'indice': json.dumps([0]),  # Convert the list to a JSON string
-        }
-
-        headers = {
-            'X-API-Key': self.key,
-            'User-Agent': user_agent,
-        }
-        response = await AsyncFetcher.post_fetch(url, data=payload, headers=headers, proxy=self.proxy)
-        resp_json = json.loads(response)
-
-        for data in resp_json:
-            domain = data['data']['domain']
-            self.totalhosts.append(domain)
-
-    async def get_hostnames(self) -> list:
+    async def get_hostnames(self) -> set[str]:
         return self.totalhosts
 
     async def process(self, proxy: bool = False) -> None:
         self.proxy = proxy
-        await self.do_count()
-        await self.do_search()
+        self.execution_status = None
+        self.stop_reason = None
+        try:
+            async with AsyncFetcher.open_session(
+                headers={'Authorization': f'Bearer {self.key}'},
+                proxy=proxy,
+            ) as session:
+                await self.do_search(session, self.limit)
+        except Exception:
+            self._stop('failed', 'transport-error')
+            return
+        if self.execution_status is None:
+            self.execution_status = 'completed'
+            self.stop_reason = None if self.totalhosts else 'no-results'

theHarvester/discovery/onyphe.py

@@ -4,6 +4,7 @@ from ipaddress import ip_address
 from urllib.parse import urlparse
 
 from theHarvester.discovery.constants import MissingKey
+from theHarvester.discovery.provider_response import provider_http_error
 from theHarvester.lib.asn_attribution import AsnAttributionObservation, SubjectKind
 from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse
 from theHarvester.lib.hostnames import normalize_scoped_hostname
@@ -22,81 +23,113 @@ class SearchOnyphe:
     WHOIS data. Both stay separate and are linked to the record's primary IP.
     """
 
-    def __init__(self, word) -> None:
+    MAX_RESULTS = 10_000
+
+    def __init__(self, word: str, limit: int) -> None:
+        if isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0:
+            raise ValueError('ONYPHE limit must be a positive integer')
         self.word = word
-        self.response = ''
+        self.limit = limit
+        self.response: object = {}
         self.totalhosts: set = set()
         self.totalips: set = set()
         self.asns: set = set()
         self.asn_attributions: set[AsnAttributionObservation] = set()
         self.key = Core.onyphe_key()
-        if self.key is None:
+        if not isinstance(self.key, str) or not self.key.strip():
             raise MissingKey('onyphe')
         self.proxy = False
         self.execution_status: str | None = None
         self.stop_reason: str | None = None
 
+    def _has_results(self) -> bool:
+        return bool(self.totalhosts or self.totalips or self.asns)
+
+    def _stop(self, status: str, reason: str) -> None:
+        self.execution_status = 'partial' if self._has_results() else status
+        self.stop_reason = reason
+
     async def do_search(self) -> None:
-        # https://www.onyphe.io/docs/apis/search
-        # https://www.onyphe.io/search?q=domain%3Acharter.com&captcharesponse=j5cGT
-        # base_url = f'https://www.onyphe.io/api/v2/search/?q=domain:domain:{self.word}'
-        base_url = f'https://www.onyphe.io/api/v2/search/?q=domain:{self.word}'
+        base_url = 'https://www.onyphe.io/api/v2/search/'
         headers = {
             'User-Agent': Core.get_user_agent(),
             'Content-Type': 'application/json',
             'Authorization': f'bearer {self.key}',
         }
+        page = 1
+        records_seen = 0
+        result_limit = min(self.limit, self.MAX_RESULTS)
+        page_size = min(result_limit, self.MAX_RESULTS)
+        last_total = 0
         try:
-            response = await AsyncFetcher.fetch_all(
-                [base_url],
-                json=True,
-                headers=headers,
-                proxy=self.proxy,
-                include_metadata=True,
-            )
+            async with AsyncFetcher.open_session(headers=headers, proxy=self.proxy) as session:
+                while records_seen < result_limit:
+                    remaining = result_limit - records_seen
+                    metadata = await AsyncFetcher.fetch(
+                        session=session,
+                        url=base_url,
+                        params={'q': f'domain:{self.word}', 'page': page, 'size': page_size},
+                        json=True,
+                        include_metadata=True,
+                    )
+                    if error := provider_http_error(metadata):
+                        self._stop(*error)
+                        return
+                    assert isinstance(metadata, FetcherResponse)
+                    if not isinstance(metadata.body, dict):
+                        self._stop('failed', 'invalid-response')
+                        return
+                    response_text = metadata.body.get('text')
+                    if response_text != 'Success':
+                        self._stop('failed', 'provider-error' if isinstance(response_text, str) else 'invalid-response')
+                        return
+                    results = metadata.body.get('results')
+                    max_page = metadata.body.get('max_page', 1)
+                    total = metadata.body.get('total', len(results) if isinstance(results, list) else None)
+                    if (
+                        not isinstance(results, list)
+                        or isinstance(max_page, bool)
+                        or not isinstance(max_page, int)
+                        or max_page < 1
+                        or isinstance(total, bool)
+                        or not isinstance(total, int)
+                        or total < 0
+                    ):
+                        self._stop('failed', 'invalid-response')
+                        return
+
+                    page_results = results[:remaining]
+                    records_seen += len(page_results)
+                    last_total = total
+                    self.response = {**metadata.body, 'results': page_results}
+                    if await self.parse_onyphe_resp_json():
+                        self._stop('failed', 'invalid-response')
+                    expected_records = min(total, result_limit)
+                    if (not results or page >= max_page) and records_seen < expected_records:
+                        self._stop('failed', 'invalid-response')
+                        break
+                    if not results or page >= max_page or records_seen >= expected_records:
+                        break
+                    page += 1
         except Exception as error:
-            self.execution_status = 'failed'
-            self.stop_reason = 'transport-error'
+            self._stop('failed', 'transport-error')
             logger.info('Onyphe request failed: %s', type(error).__name__)
             return
 
-        metadata = response[0] if response and isinstance(response[0], FetcherResponse) else None
-        if metadata is None:
-            self.execution_status = 'failed'
-            self.stop_reason = 'transport-error'
-            return
-        if metadata.status == 429:
-            self.execution_status = 'rate-limited'
-            self.stop_reason = 'http-429'
-            return
-        if metadata.status in {401, 403}:
-            self.execution_status = 'failed'
-            self.stop_reason = 'access-denied'
-            return
-        if not 200 <= metadata.status < 300:
-            self.execution_status = 'failed'
-            self.stop_reason = f'http-{metadata.status}'
-            return
+        if self.limit > self.MAX_RESULTS and last_total > self.MAX_RESULTS and records_seen >= self.MAX_RESULTS:
+            self._stop('failed', 'provider-limit')
+        if self.execution_status is not None and self._has_results():
+            self.execution_status = 'partial'
+        elif self.execution_status is None:
+            self.execution_status = 'completed'
+            self.stop_reason = None if self._has_results() else 'no-results'
 
-        self.response = metadata.body
-        await self.parse_onyphe_resp_json()
-
-    async def parse_onyphe_resp_json(self):
+    async def parse_onyphe_resp_json(self) -> bool:
         if not isinstance(self.response, dict):
-            self.execution_status = 'failed'
-            self.stop_reason = 'invalid-response'
-            return
-        response_text = self.response.get('text')
-        if response_text != 'Success':
-            self.execution_status = 'failed'
-            self.stop_reason = 'provider-error' if isinstance(response_text, str) else 'invalid-response'
-            logger.info('Onyphe API query did not succeed')
-            return
+            return True
         results = self.response.get('results')
         if not isinstance(results, list):
-            self.execution_status = 'failed'
-            self.stop_reason = 'invalid-response'
-            return
+            return True
 
         malformed = False
         for result in results:
@@ -212,12 +245,7 @@ class SearchOnyphe:
                     except ValueError:
                         malformed = True
 
-        if malformed:
-            self.execution_status = 'partial' if self.totalhosts or self.totalips or self.asns else 'failed'
-            self.stop_reason = 'invalid-response'
-        else:
-            self.execution_status = 'completed'
-            self.stop_reason = None if self.totalhosts or self.totalips or self.asns else 'no-results'
+        return malformed
 
     async def get_asns(self) -> set:
         return self.asns
@@ -233,4 +261,6 @@ class SearchOnyphe:
 
     async def process(self, proxy: bool = False) -> None:
         self.proxy = proxy
+        self.execution_status = None
+        self.stop_reason = None
         await self.do_search()

theHarvester/discovery/provider_response.py

@@ -0,0 +1,14 @@
+from theHarvester.lib.core import FetcherResponse
+
+
+def provider_http_error(response: object) -> tuple[str, str] | None:
+    """Classify transport and HTTP failures shared by provider adapters."""
+    if not isinstance(response, FetcherResponse):
+        return 'failed', 'transport-error'
+    if response.status in {401, 403}:
+        return 'failed', 'access-denied'
+    if response.status == 429:
+        return 'rate-limited', 'http-429'
+    if not 200 <= response.status < 300:
+        return 'failed', f'http-{response.status}'
+    return None

theHarvester/discovery/searchhunterhow.py

@@ -1,58 +1,126 @@
+import asyncio
 import base64
-import logging
-from datetime import datetime
+from datetime import UTC, datetime
+from typing import Any
 
 from dateutil.relativedelta import relativedelta
 
 from theHarvester.discovery.constants import MissingKey
-from theHarvester.lib.core import AsyncFetcher, Core
-
-logger = logging.getLogger(__name__)
+from theHarvester.discovery.provider_response import provider_http_error
+from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse
+from theHarvester.lib.hostnames import normalize_scoped_hostname
 
 
 class SearchHunterHow:
-    def __init__(self, word) -> None:
+    REQUEST_DELAY_SECONDS = 2.0
+
+    def __init__(self, word: str, limit: int = 500) -> None:
+        if isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0:
+            raise ValueError('Hunter.how limit must be a positive integer')
         self.word = word
-        self.total_hostnames: set = set()
+        self.limit = limit
+        self.total_hostnames: set[str] = set()
         self.key = Core.hunterhow_key()
-        if self.key is None:
+        if not isinstance(self.key, str) or not self.key.strip():
             raise MissingKey('hunterhow')
         self.proxy = False
+        self.execution_status: str | None = None
+        self.stop_reason: str | None = None
+
+    def _stop(self, status: str, reason: str) -> None:
+        self.execution_status = 'partial' if self.total_hostnames else status
+        self.stop_reason = reason
+
+    @staticmethod
+    def _page_size(remaining: int) -> int:
+        for size in (10, 20, 50, 100, 1000):
+            if remaining <= size:
+                return size
+        return 1000
 
     async def do_search(self) -> None:
-        # https://hunter.how/search-api
-        query = f'domain.suffix="{self.word}"'
-        # second_query = f'domain="{self.word}"'
-        encoded_query = base64.urlsafe_b64encode(query.encode('utf-8')).decode('ascii')
+        self.execution_status = None
+        self.stop_reason = None
+        query = base64.urlsafe_b64encode(f'domain.suffix="{self.word}"'.encode()).decode('ascii')
+        end = datetime.now(UTC).date()
+        start = end - relativedelta(days=364)
         page = 1
-        page_size = 100  # can be either: 10,20,50,100)
-        # The interval between the start time and the end time cannot exceed one year
-        # Can not exceed one year, but years=1 does not work due to their backend, 364 will suffice
-        today = datetime.today()
-        one_year_ago = today - relativedelta(days=364)
-        start_time = one_year_ago.strftime('%Y-%m-%d')
-        end_time = today.strftime('%Y-%m-%d')
-        # two_years_ago = one_year_ago - relativedelta(days=364)
-        # start_time = two_years_ago.strftime('%Y-%m-%d')
-        # end_time = one_year_ago.strftime('%Y-%m-%d')
-        url = f'https://api.hunter.how/search?api-key={self.key}&query={encoded_query}&page={page}&page_size={page_size}&start_time={start_time}&end_time={end_time}'
-        response = await AsyncFetcher.fetch_all(
-            [url],
-            json=True,
-            headers={'User-Agent': Core.get_user_agent(), 'x-api-key': f'{self.key}'},
-            proxy=self.proxy,
-        )
-        dct = response[0]
-        if 'code' in dct.keys():
-            if dct['code'] == 40001:
-                logger.info('SearchHunterHow API returned code 40001')
-                return
-        # total = dct['data']['total']
-        # TODO determine if total is ever 100 how to get more subdomains?
-        for sub in dct['data']['list']:
-            self.total_hostnames.add(sub['domain'])
+        returned = 0
+        params: dict[str, Any] = {
+            'api-key': self.key,
+            'query': query,
+            'start_time': start.isoformat(),
+            'end_time': end.isoformat(),
+            'fields': 'domain',
+        }
+        try:
+            async with AsyncFetcher.open_session(
+                headers={'User-Agent': Core.get_user_agent()},
+                proxy=self.proxy,
+            ) as session:
+                while returned < self.limit:
+                    request_params = {
+                        **params,
+                        'page': page,
+                        'page_size': self._page_size(self.limit - returned),
+                    }
+                    response = await AsyncFetcher.fetch(
+                        session=session,
+                        url='https://api.hunter.how/search',
+                        params=request_params,
+                        include_metadata=True,
+                    )
+                    if error := provider_http_error(response):
+                        self._stop(*error)
+                        return
+                    assert isinstance(response, FetcherResponse)
+                    if not isinstance(response.body, dict):
+                        self._stop('failed', 'invalid-response')
+                        return
+                    code = response.body.get('code')
+                    if code == 40001:
+                        self._stop('failed', 'access-denied')
+                        return
+                    if code != 200:
+                        self._stop('failed', 'provider-error')
+                        return
+                    data = response.body.get('data')
+                    if not isinstance(data, dict):
+                        self._stop('failed', 'invalid-response')
+                        return
+                    total = data.get('total')
+                    rows = data.get('list')
+                    if isinstance(total, bool) or not isinstance(total, int) or total < 0 or not isinstance(rows, list):
+                        self._stop('failed', 'invalid-response')
+                        return
 
-    async def get_hostnames(self) -> set:
+                    remaining = self.limit - returned
+                    malformed = False
+                    for row in rows[:remaining]:
+                        if not isinstance(row, dict) or not isinstance(row.get('domain'), str):
+                            malformed = True
+                            continue
+                        if hostname := normalize_scoped_hostname(row['domain'], self.word):
+                            self.total_hostnames.add(hostname)
+                    if malformed:
+                        self._stop('failed', 'invalid-response')
+
+                    returned += len(rows)
+                    if not rows or returned >= min(total, self.limit):
+                        break
+                    page += 1
+                    await asyncio.sleep(self.REQUEST_DELAY_SECONDS)
+        except Exception:
+            self._stop('failed', 'transport-error')
+            return
+
+        if self.execution_status is not None and self.total_hostnames:
+            self.execution_status = 'partial'
+        elif self.execution_status is None:
+            self.execution_status = 'completed'
+            self.stop_reason = None if self.total_hostnames else 'no-results'
+
+    async def get_hostnames(self) -> set[str]:
         return self.total_hostnames
 
     async def process(self, proxy: bool = False) -> None:

theHarvester/discovery/securityscorecard.py

@@ -1,18 +1,24 @@
-import logging
+from __future__ import annotations
 
-import aiohttp
+from ipaddress import ip_address
+from typing import Any
 
 from theHarvester.discovery.constants import MissingKey
-from theHarvester.lib.core import AsyncFetcher, Core
-
-logger = logging.getLogger(__name__)
+from theHarvester.discovery.provider_response import provider_http_error
+from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse
+from theHarvester.lib.hostnames import normalize_scoped_hostname
 
 
 class SearchSecurityScorecard:
-    def __init__(self, word: str):
+    PAGE_SIZE = 50
+
+    def __init__(self, word: str, limit: int) -> None:
+        if isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0:
+            raise ValueError('SecurityScorecard limit must be a positive integer')
         self.word = word
+        self.limit = limit
         self.api_key = Core.securityscorecard_key()
-        if self.api_key is None:
+        if not isinstance(self.api_key, str) or not self.api_key.strip():
             raise MissingKey('SecurityScorecard')
         self.base_url = 'https://api.securityscorecard.io'
         self.headers = {
@@ -22,79 +28,143 @@ class SearchSecurityScorecard:
         }
         self.hosts: set[str] = set()
         self.score: int = 0
-        self.grades: dict = {}
-        self.issues: list[dict] = []
-        self.recommendations: list[dict] = []
-        self.history: list[dict] = []
-        self.ips: list[str] = []
+        self.grades: dict[str, Any] = {}
+        self.issues: list[dict[str, Any]] = []
+        self.recommendations: list[dict[str, Any]] = []
+        self.history: list[dict[str, Any]] = []
+        self.ips: set[str] = set()
+        self.execution_status: str | None = None
+        self.stop_reason: str | None = None
+
+    def _stop(self, status: str, reason: str) -> None:
+        self.execution_status = 'partial' if self.hosts or self.ips else status
+        self.stop_reason = reason
+
+    def _response_body(self, response: Any) -> dict[str, Any] | None:
+        if error := provider_http_error(response):
+            self._stop(*error)
+            return None
+        assert isinstance(response, FetcherResponse)
+        if not isinstance(response.body, dict):
+            self._stop('failed', 'invalid-response')
+            return None
+        return response.body
+
+    def _extract_summary(self, data: dict[str, Any]) -> bool:
+        malformed = False
+        score = data.get('score')
+        if score is not None:
+            if isinstance(score, bool) or not isinstance(score, int):
+                malformed = True
+            else:
+                self.score = score
+
+        grade = data.get('grade')
+        if grade is not None:
+            if isinstance(grade, str) and grade.strip():
+                self.grades['overall'] = grade.strip()
+            else:
+                malformed = True
+        factor_grades = data.get('factor_grades')
+        if factor_grades is not None:
+            if isinstance(factor_grades, dict):
+                self.grades.update(factor_grades)
+            else:
+                malformed = True
+        return malformed
+
+    async def _collect_assets(self, session: Any, route: str, field: str) -> bool:
+        page = 0
+        records_seen = 0
+        page_size = min(self.PAGE_SIZE, self.limit)
+        while records_seen < self.limit:
+            response = await AsyncFetcher.post_fetch(
+                f'{self.base_url}/parent-domains/{self.word}/{route}',
+                session=session,
+                json=True,
+                include_metadata=True,
+                json_body={'page': page, 'page_size': page_size},
+            )
+            body = self._response_body(response)
+            if body is None:
+                return False
+            entries = body.get('entries')
+            size = body.get('size')
+            if not isinstance(entries, list) or isinstance(size, bool) or not isinstance(size, int | float) or size < 0:
+                self._stop('failed', 'invalid-response')
+                return False
+
+            remaining = self.limit - records_seen
+            page_entries = entries[:remaining]
+            records_seen += len(page_entries)
+            malformed = False
+            for entry in page_entries:
+                if not isinstance(entry, dict) or not isinstance(entry.get(field), str):
+                    malformed = True
+                    continue
+                value = entry[field]
+                if field == 'domain':
+                    if hostname := normalize_scoped_hostname(value, self.word):
+                        self.hosts.add(hostname)
+                else:
+                    try:
+                        self.ips.add(str(ip_address(value.strip())))
+                    except ValueError:
+                        malformed = True
+            if malformed:
+                self._stop('failed', 'invalid-response')
+            if records_seen >= self.limit or len(entries) < page_size:
+                return True
+            page += 1
+        return True
 
     async def process(self, proxy: bool = False) -> None:
-        """Get security scorecard information for a domain."""
+        self.execution_status = None
+        self.stop_reason = None
         try:
-            if proxy:
+            async with AsyncFetcher.open_session(headers=self.headers, proxy=proxy) as session:
                 response = await AsyncFetcher.fetch(
-                    session=None, url=f'{self.base_url}/companies/{self.word}', headers=self.headers, proxy=proxy
+                    session=session,
+                    url=f'{self.base_url}/companies/{self.word}',
+                    json=True,
+                    include_metadata=True,
                 )
-                if response:
-                    self._extract_data(response)
-            else:
-                async with aiohttp.ClientSession(headers=self.headers) as session:
-                    async with session.get(f'{self.base_url}/companies/{self.word}') as response:
-                        if response.status == 200:
-                            data = await response.json()
-                            self._extract_data(data)
-        except Exception as e:
-            logger.info(f'Error in SecurityScorecard search: {e}')
+                body = self._response_body(response)
+                if body is None:
+                    return
+                if self._extract_summary(body):
+                    self._stop('failed', 'invalid-response')
+                if not await self._collect_assets(session, 'domains', 'domain'):
+                    return
+                if not await self._collect_assets(session, 'ips', 'ip'):
+                    return
+        except Exception:
+            self._stop('failed', 'transport-error')
+            return
 
-    def _extract_data(self, data: dict) -> None:
-        """Extract and categorize security scorecard information."""
-        if 'grade' in data:
-            self.score = data.get('grade', 0)
-
-        if 'factor_grades' in data:
-            self.grades = data['factor_grades']
-
-        if 'issues' in data:
-            self.issues = data['issues']
-
-        if 'recommendations' in data:
-            self.recommendations = data['recommendations']
-
-        if 'history' in data:
-            self.history = data['history']
-
-        if 'domains' in data:
-            self.hosts.update(data['domains'])
-
-        # Some responses may include IP addresses under different keys
-        ips = []
-        if isinstance(data.get('ips'), list):
-            ips = [str(ip) for ip in data.get('ips', []) if isinstance(ip, str | int)]
-        elif isinstance(data.get('ip_addresses'), list):
-            ips = [str(ip) for ip in data.get('ip_addresses', []) if isinstance(ip, str | int)]
-        elif isinstance(data.get('associated_ips'), list):
-            ips = [str(ip) for ip in data.get('associated_ips', []) if isinstance(ip, str | int)]
-        if ips:
-            # Deduplicate while preserving already stored entries
-            self.ips = list({*self.ips, *ips})
+        if self.execution_status is not None and (self.hosts or self.ips):
+            self.execution_status = 'partial'
+        elif self.execution_status is None:
+            self.execution_status = 'completed'
+            self.stop_reason = None if self.hosts or self.ips else 'no-results'
 
     async def get_hostnames(self) -> set[str]:
         return self.hosts
 
-    async def get_ips(self) -> list[str]:
+    async def get_ips(self) -> set[str]:
         return self.ips
 
     async def get_score(self) -> int:
         return self.score
 
-    async def get_grades(self) -> dict:
+    async def get_grades(self) -> dict[str, Any]:
         return self.grades
 
-    async def get_issues(self) -> list[dict]:
+    async def get_issues(self) -> list[dict[str, Any]]:
         return self.issues
 
-    async def get_recommendations(self) -> list[dict]:
+    async def get_recommendations(self) -> list[dict[str, Any]]:
         return self.recommendations
 
-    async def get_history(self) -> list[dict]:
+    async def get_history(self) -> list[dict[str, Any]]:
         return self.history

theHarvester/discovery/securitytrailssearch.py

@@ -1,95 +1,118 @@
-import asyncio
-import logging
+from __future__ import annotations
+
+from ipaddress import ip_address
+from typing import Any
 
 from theHarvester.discovery.constants import MissingKey
-from theHarvester.lib.core import AsyncFetcher, Core
-from theHarvester.parsers import securitytrailsparser
-
-logger = logging.getLogger(__name__)
+from theHarvester.discovery.provider_response import provider_http_error
+from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse
+from theHarvester.lib.hostnames import normalize_scoped_hostname
 
 
 class SearchSecuritytrail:
-    def __init__(self, word) -> None:
+    def __init__(self, word: str) -> None:
         self.word = word
         self.key = Core.security_trails_key()
-        if self.key is None:
+        if not isinstance(self.key, str) or not self.key.strip():
             raise MissingKey('Securitytrail')
-        self.results = ''
-        self.totalresults = ''
         self.api = 'https://api.securitytrails.com/v1/'
-        self.info: tuple[set, set] = (set(), set())
+        self.info: tuple[set[str], set[str]] = (set(), set())
         self.proxy = False
-        # Hold structured responses for robust parsing
-        self.domain_data: dict = {}
-        self.subdomains_data: dict = {}
+        self.domain_data: dict[str, Any] = {}
+        self.subdomains_data: dict[str, Any] = {}
+        self.execution_status: str | None = None
+        self.stop_reason: str | None = None
 
-    async def authenticate(self) -> None:
-        # Method to authenticate API key before sending requests.
-        headers = {'APIKEY': self.key}
-        url = f'{self.api}ping'
-        auth_responses = await AsyncFetcher.fetch_all([url], headers=headers, proxy=self.proxy)
-        auth_responses = auth_responses[0]
-        if 'False' in auth_responses or 'Invalid authentication' in auth_responses:
-            logger.info('\tKey could not be authenticated exiting program.')
-        await asyncio.sleep(5)
+    def _stop(self, status: str, reason: str) -> None:
+        self.execution_status = 'partial' if self.info[0] or self.info[1] else status
+        self.stop_reason = reason
 
-    async def do_search(self) -> None:
-        try:
-            # https://api.securitytrails.com/v1/domain/domain.com
-            domain_url = f'{self.api}domain/{self.word}'
-            headers = {'APIKEY': self.key, 'Accept': 'application/json'}
-            # Request JSON payloads for robust parsing
-            domain_response = await AsyncFetcher.fetch_all([domain_url], headers=headers, json=True, proxy=self.proxy)
-            await asyncio.sleep(5)  # 2+ seconds is required due to rate limit.
+    def _body(self, response: Any) -> dict[str, Any] | None:
+        if error := provider_http_error(response):
+            self._stop(*error)
+            return None
+        assert isinstance(response, FetcherResponse)
+        if not isinstance(response.body, dict):
+            self._stop('failed', 'invalid-response')
+            return None
+        return response.body
 
-            if domain_response and isinstance(domain_response[0], dict | list):
-                self.domain_data = domain_response[0] if isinstance(domain_response[0], dict) else {}
-            else:
-                logger.info('SecurityTrails: No JSON response received for domain query')
-                # keep legacy string totalresults for any downstream reliance
-                if domain_response and domain_response[0]:
-                    self.results = str(domain_response[0])
-                    self.totalresults += self.results
-                return
+    def _parse_domain(self, data: dict[str, Any]) -> bool:
+        malformed = False
+        current_dns = data.get('current_dns', {})
+        if not isinstance(current_dns, dict):
+            return True
+        ips = self.info[0]
+        for record_type, key in (('a', 'ip'), ('aaaa', 'ipv6')):
+            records = current_dns.get(record_type, {})
+            if not isinstance(records, dict) or not isinstance(records.get('values', []), list):
+                malformed = True
+                continue
+            for record in records.get('values', []):
+                if not isinstance(record, dict) or not isinstance(record.get(key), str):
+                    malformed = True
+                    continue
+                try:
+                    ips.add(str(ip_address(record[key].strip())))
+                except ValueError:
+                    malformed = True
+        return malformed
 
-            # Get subdomains now.
-            subdomains_url = f'{domain_url}/subdomains'
-            subdomain_response = await AsyncFetcher.fetch_all([subdomains_url], headers=headers, json=True, proxy=self.proxy)
-            await asyncio.sleep(5)
-
-            if subdomain_response and isinstance(subdomain_response[0], dict | list):
-                self.subdomains_data = subdomain_response[0] if isinstance(subdomain_response[0], dict) else {}
-            else:
-                logger.info('SecurityTrails: No JSON response received for subdomain query')
-                if subdomain_response and subdomain_response[0]:
-                    self.results = str(subdomain_response[0])
-                    self.totalresults += self.results
-        except Exception as e:
-            logger.info(f'SecurityTrails API error: {e}')
-            return
+    def _parse_subdomains(self, data: dict[str, Any]) -> bool:
+        values = data.get('subdomains')
+        if not isinstance(values, list):
+            return True
+        malformed = False
+        hostnames = self.info[1]
+        for value in values:
+            if not isinstance(value, str) or not value.strip():
+                malformed = True
+                continue
+            if hostname := normalize_scoped_hostname(f'{value}.{self.word}', self.word):
+                hostnames.add(hostname)
+        return malformed
 
     async def process(self, proxy: bool = False) -> None:
         self.proxy = proxy
-        await self.authenticate()
-        await self.do_search()
-        # Prefer structured JSON if available; fallback to legacy text
-        combined_payload = None
-        if isinstance(self.domain_data, dict) or isinstance(self.subdomains_data, dict):
-            combined_payload = {
-                'domain': self.domain_data if isinstance(self.domain_data, dict) else {},
-                'subdomains': self.subdomains_data if isinstance(self.subdomains_data, dict) else {},
-            }
-        parser_input = (
-            combined_payload
-            if combined_payload and (combined_payload['domain'] or combined_payload['subdomains'])
-            else self.totalresults
-        )
-        parser = securitytrailsparser.Parser(word=self.word, text=parser_input)
-        self.info = await parser.parse_text()
-        # Create parser and set self.info to tuple returned from parsing text.
+        self.execution_status = None
+        self.stop_reason = None
+        headers = {'APIKEY': self.key, 'Accept': 'application/json'}
+        try:
+            async with AsyncFetcher.open_session(headers=headers, proxy=proxy) as session:
+                domain_response = await AsyncFetcher.fetch(
+                    session=session,
+                    url=f'{self.api}domain/{self.word}',
+                    json=True,
+                    include_metadata=True,
+                )
+                domain_body = self._body(domain_response)
+                if domain_body is None:
+                    return
+                self.domain_data = domain_body
+                malformed = self._parse_domain(domain_body)
 
-    async def get_ips(self) -> set:
+                subdomain_response = await AsyncFetcher.fetch(
+                    session=session,
+                    url=f'{self.api}domain/{self.word}/subdomains',
+                    json=True,
+                    include_metadata=True,
+                )
+                subdomain_body = self._body(subdomain_response)
+                if subdomain_body is None:
+                    return
+                self.subdomains_data = subdomain_body
+                malformed = self._parse_subdomains(subdomain_body) or malformed
+        except Exception:
+            self._stop('failed', 'transport-error')
+            return
+        if malformed:
+            self._stop('failed', 'invalid-response')
+        if self.execution_status is None:
+            self.execution_status = 'completed'
+            self.stop_reason = None if self.info[0] or self.info[1] else 'no-results'
+
+    async def get_ips(self) -> set[str]:
         return self.info[0]
 
-    async def get_hostnames(self) -> set:
+    async def get_hostnames(self) -> set[str]:
         return self.info[1]

theHarvester/discovery/sherlockeye.py

@@ -1,12 +1,12 @@
 import logging
-import random
+from ipaddress import ip_address as normalize_ip_address
 from typing import Any
 from urllib.parse import urlparse
 
-import aiohttp
-
 from theHarvester.discovery.constants import MissingKey
-from theHarvester.lib.core import Core
+from theHarvester.discovery.provider_response import provider_http_error
+from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse
+from theHarvester.lib.hostnames import normalize_scoped_hostname
 
 logger = logging.getLogger(__name__)
 
@@ -26,13 +26,22 @@ class SearchSherlockeye:
     def __init__(self, word: str) -> None:
         self.word = word
         self.key = Core.sherlockeye_key()
-        if self.key is None:
+        if not isinstance(self.key, str) or not self.key.strip():
             raise MissingKey('sherlockeye')
         self.totalhosts: set[str] = set()
         self.totalemails: set[str] = set()
         self.totalips: set[str] = set()
         self.results: list[dict[str, Any]] = []
         self.proxy: bool | str = False
+        self.execution_status: str | None = None
+        self.stop_reason: str | None = None
+
+    def _has_results(self) -> bool:
+        return bool(self.totalhosts or self.totalemails or self.totalips)
+
+    def _stop(self, status: str, reason: str) -> None:
+        self.execution_status = 'partial' if self._has_results() else status
+        self.stop_reason = reason
 
     def _headers(self) -> dict[str, str]:
         return {
@@ -41,77 +50,91 @@ class SearchSherlockeye:
             'Content-Type': 'application/json',
         }
 
-    def _proxy_url(self) -> str | None:
-        if isinstance(self.proxy, str) and self.proxy:
-            return self.proxy
-        if isinstance(self.proxy, bool) and self.proxy:
-            try:
-                proxy_list = Core.proxy_list()
-                proxy_urls = [*proxy_list.get('http', []), *proxy_list.get('socks5', [])]
-                if proxy_urls:
-                    return random.choice(proxy_urls)
-            except Exception:
-                return None
-        return None
-
     def _add_hostname(self, hostname: str) -> None:
-        hostname = hostname.strip().lower()
-        if hostname.endswith(f'.{self.word}') or hostname == self.word:
-            self.totalhosts.add(hostname)
+        if normalized := normalize_scoped_hostname(hostname, self.word):
+            self.totalhosts.add(normalized)
 
     def _add_email(self, email: str) -> None:
-        email = email.strip().lower()
-        if '@' in email and self.word in email:
-            self.totalemails.add(email)
+        normalized_email = email.strip().lower()
+        local_part, separator, domain = normalized_email.rpartition('@')
+        if local_part and separator and (normalized_domain := normalize_scoped_hostname(domain, self.word)):
+            self.totalemails.add(f'{local_part}@{normalized_domain}')
 
     def _add_ip(self, ip_address: str) -> None:
-        ip_address = ip_address.strip()
-        if ip_address:
-            self.totalips.add(ip_address)
+        try:
+            self.totalips.add(str(normalize_ip_address(ip_address.strip())))
+        except ValueError:
+            return
 
-    def _extract_from_link(self, link: str) -> None:
-        parsed = urlparse(link.strip())
+    def _extract_from_link(self, link: str) -> bool:
+        try:
+            parsed = urlparse(link.strip())
+        except ValueError:
+            return True
         if parsed.hostname:
             self._add_hostname(parsed.hostname)
+        return False
 
-    def _extract_result(self, result: dict[str, Any]) -> None:
+    def _extract_result(self, result: dict[str, Any]) -> bool:
         attributes = result.get('attributes')
         if not isinstance(attributes, dict):
-            return
+            return True
+
+        malformed = False
 
         domain = attributes.get('domain')
         if isinstance(domain, str):
             self._add_hostname(domain)
+        elif domain is not None:
+            malformed = True
 
         email = attributes.get('email')
         if isinstance(email, str):
             self._add_email(email)
+        elif email is not None:
+            malformed = True
 
         ip_address = attributes.get('ip')
         if isinstance(ip_address, str):
             self._add_ip(ip_address)
+        elif ip_address is not None:
+            malformed = True
 
         link = attributes.get('link')
         if isinstance(link, str):
-            self._extract_from_link(link)
+            malformed |= self._extract_from_link(link)
+        elif link is not None:
+            malformed = True
+        return malformed
 
     def _extract_response(self, response: dict[str, Any]) -> None:
         if response.get('success') is False:
             logger.info('Sherlockeye API error')
+            self._stop('failed', 'provider-error')
             return
 
         data = response.get('data')
         if not isinstance(data, dict):
+            self._stop('failed', 'invalid-response')
             return
 
         search_results = data.get('results')
         if not isinstance(search_results, list):
+            self._stop('failed', 'invalid-response')
             return
 
         self.results = search_results
+        malformed = False
         for result in search_results:
             if isinstance(result, dict):
-                self._extract_result(result)
+                malformed |= self._extract_result(result)
+            else:
+                malformed = True
+        if malformed:
+            self._stop('failed', 'invalid-response')
+        elif self.execution_status is None:
+            self.execution_status = 'completed'
+            self.stop_reason = None if self._has_results() else 'no-results'
 
     async def do_search(self) -> None:
         payload = {
@@ -119,24 +142,36 @@ class SearchSherlockeye:
             'value': self.word,
             'timeoutSeconds': self.DEFAULT_TIMEOUT_SECONDS,
         }
-        timeout = aiohttp.ClientTimeout(total=self.DEFAULT_TIMEOUT_SECONDS + 30)
-
         try:
-            async with aiohttp.ClientSession(headers=self._headers(), timeout=timeout) as session:
-                async with session.post(
+            async with AsyncFetcher.open_session(
+                headers=self._headers(),
+                proxy=self.proxy,
+                request_timeout=self.DEFAULT_TIMEOUT_SECONDS + 30,
+            ) as session:
+                response = await AsyncFetcher.post_fetch(
                     self.SYNC_SEARCH_URL,
-                    json=payload,
-                    proxy=self._proxy_url(),
-                ) as response:
-                    if response.status != 200:
-                        logger.info(f'Sherlockeye API request failed with status {response.status}')
-                        return
-
-                    response_data = await response.json()
-                    if isinstance(response_data, dict):
-                        self._extract_response(response_data)
+                    session=session,
+                    json=True,
+                    include_metadata=True,
+                    json_body=payload,
+                )
+                if error := provider_http_error(response):
+                    self._stop(*error)
+                    status = response.status if isinstance(response, FetcherResponse) else 'transport'
+                    logger.info('Sherlockeye API request failed with status %s: %s', status, error[1])
+                    return
+                assert isinstance(response, FetcherResponse)
+                if response.status != 200:
+                    self._stop('failed', f'http-{response.status}')
+                    logger.info('Sherlockeye API request failed with status %s', response.status)
+                    return
+                if isinstance(response.body, dict):
+                    self._extract_response(response.body)
+                else:
+                    self._stop('failed', 'invalid-response')
         except Exception as error:
-            logger.info(f'Sherlockeye API error: {error}')
+            self._stop('failed', 'transport-error')
+            logger.info('Sherlockeye API error: %s', type(error).__name__)
 
     async def get_hostnames(self) -> set[str]:
         return self.totalhosts
@@ -152,4 +187,6 @@ class SearchSherlockeye:
 
     async def process(self, proxy: bool = False) -> None:
         self.proxy = proxy
+        self.execution_status = None
+        self.stop_reason = None
         await self.do_search()

theHarvester/discovery/subdomainfinderc99.py

@@ -5,7 +5,8 @@ from bs4 import BeautifulSoup
 from bs4.element import Tag
 
 from theHarvester.discovery.constants import get_delay
-from theHarvester.lib.core import AsyncFetcher, Core
+from theHarvester.discovery.provider_response import provider_http_error
+from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse
 from theHarvester.parsers import myparser
 
 
@@ -17,22 +18,54 @@ class SearchSubdomainfinderc99:
         # TODO add api support
         self.server = 'https://subdomainfinder.c99.nl/'
         self.totalresults = ''
+        self.execution_status: str | None = None
+        self.stop_reason: str | None = None
+
+    def _stop(self, status: str, reason: str) -> None:
+        self.execution_status = status
+        self.stop_reason = reason
 
     async def do_search(self) -> None:
         # Based on https://gist.github.com/th3gundy/bc83580cbe04031e9164362b33600962
         headers = {'User-Agent': Core.get_browser_user_agent()}
-        resp = await AsyncFetcher.fetch_all([self.server], headers=headers, proxy=self.proxy)
-        if not resp or not isinstance(resp[0], str):
-            return
-        data = await self.get_csrf_params(resp[0])
+        async with AsyncFetcher.open_session(headers=headers, proxy=self.proxy) as session:
+            metadata = await AsyncFetcher.fetch(
+                session=session,
+                url=self.server,
+                include_metadata=True,
+            )
+            if error := provider_http_error(metadata):
+                self._stop(*error)
+                return
+            assert isinstance(metadata, FetcherResponse)
+            if not isinstance(metadata.body, str):
+                self._stop('failed', 'invalid-response')
+                return
+            data = await self.get_csrf_params(metadata.body)
+            if not data:
+                self._stop('failed', 'invalid-response')
+                return
 
-        data['scan_subdomains'] = ''
-        data['domain'] = self.word
-        data['privatequery'] = 'on'
-        await asyncio.sleep(get_delay())
-        second_resp = await AsyncFetcher.post_fetch(self.server, headers=headers, proxy=self.proxy, data=ujson.dumps(data))
-        if isinstance(second_resp, str):
-            self.totalresults += second_resp
+            data['scan_subdomains'] = ''
+            data['domain'] = self.word
+            data['privatequery'] = 'on'
+            await asyncio.sleep(get_delay())
+            second_resp = await AsyncFetcher.post_fetch(
+                self.server,
+                session=session,
+                data=ujson.dumps(data),
+                include_metadata=True,
+            )
+            if error := provider_http_error(second_resp):
+                self._stop(*error)
+                return
+            assert isinstance(second_resp, FetcherResponse)
+            if not isinstance(second_resp.body, str):
+                self._stop('failed', 'invalid-response')
+                return
+            self.totalresults += second_resp.body
+        self.execution_status = 'completed'
+        self.stop_reason = None if await self.get_hostnames() else 'no-results'
 
     async def get_hostnames(self):
         rawres = myparser.Parser(self.totalresults, self.word)
@@ -40,7 +73,12 @@ class SearchSubdomainfinderc99:
 
     async def process(self, proxy: bool = False) -> None:
         self.proxy = proxy
-        await self.do_search()
+        self.execution_status = None
+        self.stop_reason = None
+        try:
+            await self.do_search()
+        except Exception:
+            self._stop('failed', 'transport-error')
 
     @staticmethod
     async def get_csrf_params(data):

theHarvester/discovery/urlscan.py

@@ -3,6 +3,7 @@ from datetime import UTC, datetime
 from ipaddress import ip_address
 from urllib.parse import urlsplit
 
+from theHarvester.discovery.provider_response import provider_http_error
 from theHarvester.lib.asn_attribution import AsnAttributionObservation, SubjectKind
 from theHarvester.lib.core import AsyncFetcher, FetcherResponse
 from theHarvester.lib.hostnames import normalize_scoped_hostname
@@ -12,11 +13,13 @@ logger = logging.getLogger(__name__)
 
 
 class SearchUrlscan:
-    # ponytail: hard cap protects against endless unique cursors; raise only if real targets exceed 1,000 pages.
-    MAX_PAGES = 1000
+    MAX_PAGE_SIZE = 10_000
 
-    def __init__(self, word) -> None:
+    def __init__(self, word: str, limit: int) -> None:
+        if isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0:
+            raise ValueError('URLScan limit must be a positive integer')
         self.word = word
+        self.limit = limit
         self.totalhosts: set = set()
         self.totalips: set = set()
         self.urls: set = set()
@@ -144,63 +147,67 @@ class SearchUrlscan:
         collected_at = datetime.now(UTC)
         cursor = None
         seen_cursors: set[str] = set()
+        records_seen = 0
         malformed = False
-        for _ in range(self.MAX_PAGES):
-            params = {'q': f'domain:{self.word}'}
-            if cursor is not None:
-                params['search_after'] = cursor
-            try:
-                response = await AsyncFetcher.fetch(
-                    url=url,
-                    params=params,
-                    json=True,
-                    proxy=self.proxy,
-                    request_timeout=60,
-                    include_metadata=True,
-                )
-            except Exception as error:
-                self._stop('failed', 'transport-error')
-                logger.info('URLScan request failed: %s', type(error).__name__)
-                return
+        try:
+            async with AsyncFetcher.open_session(proxy=self.proxy) as session:
+                while records_seen < self.limit:
+                    remaining = self.limit - records_seen
+                    params: dict[str, str | int] = {
+                        'q': f'domain:{self.word}',
+                        'size': min(self.MAX_PAGE_SIZE, remaining),
+                    }
+                    if cursor is not None:
+                        params['search_after'] = cursor
+                    response = await AsyncFetcher.fetch(
+                        session=session,
+                        url=url,
+                        params=params,
+                        json=True,
+                        include_metadata=True,
+                    )
 
-            if not isinstance(response, FetcherResponse):
-                self._stop('failed', 'transport-error')
-                return
-            if response.status == 429:
-                self._stop('rate-limited', 'http-429')
-                return
-            if response.status in {401, 403}:
-                self._stop('failed', 'access-denied')
-                return
-            if not 200 <= response.status < 300:
-                self._stop('failed', f'http-{response.status}')
-                return
-            if not isinstance(response.body, dict) or not isinstance(response.body.get('results'), list):
-                self._stop('failed', 'invalid-response')
-                return
+                    if error := provider_http_error(response):
+                        self._stop(*error)
+                        return
+                    assert isinstance(response, FetcherResponse)
+                    if not isinstance(response.body, dict) or not isinstance(response.body.get('results'), list):
+                        self._stop('failed', 'invalid-response')
+                        return
 
-            results = response.body['results']
-            if not results:
-                if malformed:
-                    self._stop('failed', 'invalid-response')
-                else:
-                    self.execution_status = 'completed'
-                    self.stop_reason = None if self._has_results() else 'no-results'
-                return
+                    results = response.body['results']
+                    if not results:
+                        if malformed:
+                            self._stop('failed', 'invalid-response')
+                        else:
+                            self.execution_status = 'completed'
+                            self.stop_reason = None if self._has_results() else 'no-results'
+                        return
 
-            malformed = self._parse_results(results, collected_at) or malformed
-            next_cursor = self._cursor(results[-1])
-            if next_cursor is None:
-                self._stop('failed', 'invalid-cursor')
-                return
-            if next_cursor in seen_cursors:
-                self._stop('failed', 'repeated-cursor')
-                return
-            seen_cursors.add(next_cursor)
-            cursor = next_cursor
+                    page_results = results[:remaining]
+                    records_seen += len(page_results)
+                    malformed = self._parse_results(page_results, collected_at) or malformed
+                    if records_seen >= self.limit:
+                        break
+                    next_cursor = self._cursor(page_results[-1])
+                    if next_cursor is None:
+                        self._stop('failed', 'invalid-cursor')
+                        return
+                    if next_cursor in seen_cursors:
+                        self._stop('failed', 'repeated-cursor')
+                        return
+                    seen_cursors.add(next_cursor)
+                    cursor = next_cursor
+        except Exception as error:
+            self._stop('failed', 'transport-error')
+            logger.info('URLScan request failed: %s', type(error).__name__)
+            return
 
-        self.execution_status = 'partial'
-        self.stop_reason = 'page-limit'
+        if self.execution_status is not None and self._has_results():
+            self.execution_status = 'partial'
+        elif self.execution_status is None:
+            self.execution_status = 'completed'
+            self.stop_reason = None if self._has_results() else 'no-results'
 
     async def get_hostnames(self) -> set:
         return self.totalhosts

theHarvester/discovery/virustotal.py

@@ -1,98 +1,139 @@
-import asyncio
+from __future__ import annotations
+
+from typing import Any
 
 from theHarvester.discovery.constants import MissingKey
-from theHarvester.lib.core import AsyncFetcher, Core
+from theHarvester.discovery.provider_response import provider_http_error
+from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse
+from theHarvester.lib.hostnames import normalize_scoped_hostname
 
 
 class SearchVirustotal:
-    def __init__(self, word) -> None:
+    def __init__(self, word: str, limit: int) -> None:
+        if isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0:
+            raise ValueError('VirusTotal limit must be a positive integer')
         self.key = Core.virustotal_key()
-        if self.key is None:
+        if not isinstance(self.key, str) or not self.key.strip():
             raise MissingKey('virustotal')
         self.word = word
+        self.limit = limit
         self.proxy = False
-        self.hostnames: list = []
+        self.hostnames: set[str] = set()
+        self.execution_status: str | None = None
+        self.stop_reason: str | None = None
+
+    def _stop(self, status: str, reason: str) -> None:
+        self.execution_status = 'partial' if self.hostnames else status
+        self.stop_reason = reason
 
     async def do_search(self) -> None:
-        # TODO determine if more endpoints can yield useful info given a domain
-        # based on: https://developers.virustotal.com/reference/domains-relationships
-        # base_url = "https://www.virustotal.com/api/v3/domains/domain/subdomains?limit=40"
-        headers = {
-            'User-Agent': Core.get_user_agent(),
-            'Accept': 'application/json',
-            'x-apikey': self.key,
-        }
-        base_url = f'https://www.virustotal.com/api/v3/domains/{self.word}/subdomains?limit=40'
-        cursor = ''
-        count = 0
-        fail_counter = 0
-        counter = 0
-        breakcon = False
-        while True:
-            if breakcon:
-                break
-            # rate limit is 4 per minute
-            # TODO add timer logic if proven to be needed
-            # in the meantime sleeping 16 seconds should eliminate hitting the rate limit
-            # in case rate limit is hit, fail counter exists and sleep for 65 seconds
-            send_url = base_url + '&cursor=' + cursor if cursor != '' and len(cursor) > 2 else base_url
-            responses = await AsyncFetcher.fetch_all([send_url], headers=headers, proxy=self.proxy, json=True)
-            jdata = responses[0]
-            if 'data' not in jdata:
-                await asyncio.sleep(60 + 5)
-                fail_counter += 1
-            if 'meta' in jdata:
-                cursor = jdata['meta']['cursor'] if 'cursor' in jdata['meta'] else ''
-                if len(cursor) == 0 and 'data' in jdata:
-                    # if cursor no longer is within the meta field have hit last entry
-                    breakcon = True
-            count += jdata['meta']['count']
-            if count == 0 or fail_counter >= 2:
-                break
-            if 'data' in jdata:
-                data = jdata['data']
-                self.hostnames.extend(await self.parse_hostnames(data, self.word))
-                counter += 1
-            await asyncio.sleep(16)
-        self.hostnames = list(sorted(set(self.hostnames)))
-        # verify domains such as x.x.com.multicdn.x.com are parsed properly
-        self.hostnames = [
-            host for host in self.hostnames if ((len(host.split('.')) >= 3) and host.split('.')[-2] == self.word.split('.')[-2])
-        ]
+        headers = {'Accept': 'application/json', 'x-apikey': self.key}
+        cursor: str | None = None
+        seen_cursors: set[str] = set()
+        records_seen = 0
+        try:
+            async with AsyncFetcher.open_session(headers=headers, proxy=self.proxy) as session:
+                while records_seen < self.limit:
+                    remaining = self.limit - records_seen
+                    params: dict[str, int | str] = {'limit': min(40, remaining)}
+                    if cursor:
+                        params['cursor'] = cursor
+                    response = await AsyncFetcher.fetch(
+                        session=session,
+                        url=f'https://www.virustotal.com/api/v3/domains/{self.word}/subdomains',
+                        params=params,
+                        json=True,
+                        include_metadata=True,
+                    )
+                    if error := provider_http_error(response):
+                        self._stop(*error)
+                        return
+                    assert isinstance(response, FetcherResponse)
+                    if not isinstance(response.body, dict):
+                        self._stop('failed', 'invalid-response')
+                        return
+                    data = response.body.get('data')
+                    meta = response.body.get('meta', {})
+                    if not isinstance(data, list) or not isinstance(meta, dict):
+                        self._stop('failed', 'invalid-response')
+                        return
+                    page_data = data[:remaining]
+                    records_seen += len(page_data)
+                    hostnames, malformed = self.parse_hostnames(page_data, self.word)
+                    for hostname in sorted(hostnames):
+                        if len(self.hostnames) >= self.limit:
+                            break
+                        self.hostnames.add(hostname)
+                    if malformed:
+                        self._stop('failed', 'invalid-response')
+                    next_cursor = meta.get('cursor')
+                    if not data or not isinstance(next_cursor, str) or not next_cursor:
+                        break
+                    if next_cursor in seen_cursors:
+                        self._stop('failed', 'repeated-cursor')
+                        break
+                    seen_cursors.add(next_cursor)
+                    cursor = next_cursor
+        except Exception:
+            self._stop('failed', 'transport-error')
+            return
 
-    async def get_hostnames(self) -> list:
+        if self.execution_status is not None and self.hostnames:
+            self.execution_status = 'partial'
+        elif self.execution_status is None:
+            self.execution_status = 'completed'
+            self.stop_reason = None if self.hostnames else 'no-results'
+
+    async def get_hostnames(self) -> set[str]:
         return self.hostnames
 
     @staticmethod
-    async def parse_hostnames(data, word):
-        total_subdomains: set[str] = set()
-        for attribute in data:
-            total_subdomains.add(attribute['id'].replace('"', ''))
-            attributes = attribute['attributes']
-            total_subdomains.update(
-                {value['value'].replace('"', '') for value in attributes['last_dns_records'] if word in value['value']}
-            )
-            if 'last_https_certificate' in attributes:
-                total_subdomains.update(
-                    {
-                        value.replace('"', '')
-                        for value in attributes['last_https_certificate']['extensions']['subject_alternative_name']
-                        if word in value
-                    }
-                )
-        # Convert to list for further processing without changing variable type mid-function
-        subdomains_list: list[str] = list(sorted(total_subdomains))
-        # Other false positives may occur over time and yes there are other ways to parse this, feel free to implement
-        # them and submit a PR or raise an issue if you run into this filtering not being enough
-        # TODO determine if parsing 'v=spf1 include:_spf-x.acme.com include:_spf-x.acme.com' is worth parsing
-        subdomains_list = [
-            x
-            for x in subdomains_list
-            if 'edgekey.net' not in str(x) and 'akadns.net' not in str(x) and 'include:_spf' not in str(x)
-        ]
-        subdomains_list.sort()
-        return subdomains_list
+    def parse_hostnames(data: list[Any], word: str) -> tuple[set[str], bool]:
+        hostnames: set[str] = set()
+        malformed = False
+
+        def add(value: Any) -> None:
+            nonlocal malformed
+            if not isinstance(value, str):
+                malformed = True
+                return
+            if hostname := normalize_scoped_hostname(value.replace('"', ''), word):
+                hostnames.add(hostname)
+
+        for item in data:
+            if not isinstance(item, dict):
+                malformed = True
+                continue
+            add(item.get('id'))
+            attributes = item.get('attributes', {})
+            if not isinstance(attributes, dict):
+                malformed = True
+                continue
+            records = attributes.get('last_dns_records', [])
+            if not isinstance(records, list):
+                malformed = True
+            else:
+                for record in records:
+                    if not isinstance(record, dict):
+                        malformed = True
+                    else:
+                        add(record.get('value'))
+            certificate = attributes.get('last_https_certificate')
+            if certificate is None:
+                continue
+            if not isinstance(certificate, dict) or not isinstance(certificate.get('extensions'), dict):
+                malformed = True
+                continue
+            names = certificate['extensions'].get('subject_alternative_name', [])
+            if not isinstance(names, list):
+                malformed = True
+                continue
+            for name in names:
+                add(name)
+        return hostnames, malformed
 
     async def process(self, proxy: bool = False) -> None:
         self.proxy = proxy
+        self.execution_status = None
+        self.stop_reason = None
         await self.do_search()

theHarvester/discovery/whoisxml.py

@@ -1,38 +1,92 @@
+from __future__ import annotations
+
 from theHarvester.discovery.constants import MissingKey
-from theHarvester.lib.core import AsyncFetcher, Core
+from theHarvester.discovery.provider_response import provider_http_error
+from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse
+from theHarvester.lib.hostnames import normalize_scoped_hostname
 
 
 class SearchWhoisXML:
-    def __init__(self, word) -> None:
+    def __init__(self, word: str, limit: int) -> None:
+        if isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0:
+            raise ValueError('WhoisXML limit must be a positive integer')
         self.word = word
+        self.limit = limit
         self.key = Core.whoisxml_key()
-        if self.key is None:
+        if not isinstance(self.key, str) or not self.key.strip():
             raise MissingKey('whoisxml')
-        self.total_results: list[str] = []
-        self.proxy: bool = False
+        self.total_results: set[str] = set()
+        self.proxy = False
+        self.execution_status: str | None = None
+        self.stop_reason: str | None = None
 
-    async def do_search(self):
-        # https://subdomains.whoisxmlapi.com/api/documentation/making-requests
-        url = 'https://subdomains.whoisxmlapi.com/api/v1'
-        params = {'apiKey': self.key, 'domainName': self.word}
-        response = await AsyncFetcher.fetch_all(
-            [url],
-            json=True,
-            params=params,
-            headers={'User-Agent': Core.get_user_agent()},
-            proxy=self.proxy,
-        )
-        # Parse the response according to the example JSON structure:
-        # {"search":"example.com.com","result":{"count":10000,"records":[{"domain":"test.example.com","firstSeen":1678169400,"lastSeen":1678169400}]}}
-        self.total_results = []
-        if response and response[0]:
-            # Extract domains from the records array
-            if 'result' in response[0] and 'records' in response[0]['result']:
-                self.total_results = [record['domain'] for record in response[0]['result']['records']]
+    def _stop(self, status: str, reason: str) -> None:
+        self.execution_status = 'partial' if self.total_results else status
+        self.stop_reason = reason
 
-    async def get_hostnames(self):
+    async def do_search(self) -> None:
+        cursor: str | None = None
+        seen_cursors: set[str] = set()
+        records_seen = 0
+        async with AsyncFetcher.open_session(proxy=self.proxy) as session:
+            while records_seen < self.limit:
+                params = {'apiKey': self.key, 'domainName': self.word}
+                if cursor is not None:
+                    params['searchAfter'] = cursor
+                response = await AsyncFetcher.fetch(
+                    session=session,
+                    url='https://subdomains.whoisxmlapi.com/api/v2',
+                    params=params,
+                    json=True,
+                    include_metadata=True,
+                )
+                if error := provider_http_error(response):
+                    self._stop(*error)
+                    return
+                assert isinstance(response, FetcherResponse)
+                if not isinstance(response.body, dict):
+                    self._stop('failed', 'invalid-response')
+                    return
+                result = response.body.get('result')
+                if not isinstance(result, dict) or not isinstance(result.get('records'), list):
+                    self._stop('failed', 'invalid-response')
+                    return
+                next_cursor = result.get('nextPageSearchAfter')
+                if not isinstance(next_cursor, str):
+                    self._stop('failed', 'invalid-response')
+                    return
+
+                remaining = self.limit - records_seen
+                records = result['records'][:remaining]
+                records_seen += len(records)
+                malformed = False
+                for record in records:
+                    if not isinstance(record, dict) or not isinstance(record.get('domain'), str):
+                        malformed = True
+                        continue
+                    if hostname := normalize_scoped_hostname(record['domain'], self.word):
+                        self.total_results.add(hostname)
+                if malformed:
+                    self._stop('failed', 'invalid-response')
+                if records_seen >= self.limit or not next_cursor:
+                    break
+                if next_cursor in seen_cursors:
+                    self._stop('failed', 'repeated-cursor')
+                    break
+                seen_cursors.add(next_cursor)
+                cursor = next_cursor
+        if self.execution_status is None:
+            self.execution_status = 'completed'
+            self.stop_reason = None if self.total_results else 'no-results'
+
+    async def get_hostnames(self) -> set[str]:
         return self.total_results
 
     async def process(self, proxy: bool = False) -> None:
         self.proxy = proxy
-        await self.do_search()
+        self.execution_status = None
+        self.stop_reason = None
+        try:
+            await self.do_search()
+        except Exception:
+            self._stop('failed', 'transport-error')

theHarvester/discovery/zoomeyesearch.py

@@ -1,390 +1,236 @@
-import asyncio
-import logging
+from __future__ import annotations
+
+import base64
 import math
 import re
-from collections.abc import Iterable
+from ipaddress import ip_address
 from typing import Any
-from urllib.parse import urlparse
+from urllib.parse import urlsplit, urlunsplit
 
-from theHarvester.discovery.constants import MissingKey, get_delay
-from theHarvester.lib.core import AsyncFetcher, Core
+from theHarvester.discovery.constants import MissingKey
+from theHarvester.discovery.provider_response import provider_http_error
+from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse
 from theHarvester.lib.hostnames import normalize_scoped_hostname
 from theHarvester.parsers import myparser
 
-logger = logging.getLogger(__name__)
-
 
 class SearchZoomEye:
-    def __init__(self, word, limit) -> None:
-        self.word = word
-        self.limit = limit
-        self.key = Core.zoomeye_key()
-        # NOTE for ZoomEye you get a system recharge on the 1st of every month
-        # Which resets your balance to 10000 requests
-        # If you wish to extract as many subdomains as possible visit the fetch_subdomains
-        # To see how
-        if self.key is None:
+    PAGE_SIZE = 10_000
+    RESPONSE_FIELDS = ','.join(
+        (
+            'ip',
+            'domain',
+            'hostname',
+            'rdns',
+            'asn',
+            'url',
+            'banner',
+            'header',
+            'body',
+            'ssl',
+        )
+    )
+    URL_PATTERN = re.compile(r'https?://[^\s"\'<>]+')
+
+    def __init__(self, word: str, limit: int) -> None:
+        if isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0:
+            raise ValueError('ZoomEye limit must be a positive integer')
+        key = Core.zoomeye_key()
+        if not isinstance(key, str) or not key.strip():
             raise MissingKey('zoomeye')
-        # API v2 base
-        self.baseurl = 'https://api.zoomeye.ai/host/search'
-        self.domain_url = 'https://api.zoomeye.ai/domain/search'
+        self.word = word
+        self.target = word.strip().lower().removeprefix('www.').rstrip('.')
+        self.limit = limit
+        self.key = key
+        self.baseurl = 'https://api.zoomeye.ai/v2/search'
         self.proxy = False
-        self.totalasns: list = list()
-        self.totalhosts: list = list()
-        self.urls: list = list()
-        self.totalips: list = list()
-        self.totalemails: list = list()
-        # Regex used is directly from: https://github.com/GerbenJavado/LinkFinder/blob/master/linkfinder.py#L29
-        # Maybe one day it will be a pip package
-        # Regardless LinkFinder is an amazing tool!
-        regex_str = r"""
-          (?:"|')                               # Start newline delimiter
-          (
-            ((?:[a-zA-Z]{1,10}://|//)           # Match a scheme [a-Z]*1-10 or //
-            [^"'/]{1,}\.                        # Match a domainname (any character + dot)
-            [a-zA-Z]{2,}[^"']{0,})              # The domainextension and/or path
-            |
-            ((?:/|\.\./|\./)                    # Start with /,../,./
-            [^"'><,;| *()(%%$^/\\\[\]]          # Next character can't be...
-            [^"'><,;|()]{1,})                   # Rest of the characters can't be
-            |
-            ([a-zA-Z0-9_\-/]{1,}/               # Relative endpoint with /
-            [a-zA-Z0-9_\-/]{1,}                 # Resource name
-            \.(?:[a-zA-Z]{1,4}|action)          # Rest + extension (length 1-4 or action)
-            (?:[\?|#][^"|']{0,}|))              # ? or # mark with parameters
-            |
-            ([a-zA-Z0-9_\-/]{1,}/               # REST API (no extension) with /
-            [a-zA-Z0-9_\-/]{3,}                 # Proper REST endpoints usually have 3+ chars
-            (?:[\?|#][^"|']{0,}|))              # ? or # mark with parameters
-            |
-            ([a-zA-Z0-9_\-]{1,}                 # filename
-            \.(?:php|asp|aspx|jsp|json|
-                 action|html|js|txt|xml)        # . + extension
-            (?:[\?|#][^"|']{0,}|))              # ? or # mark with parameters
-          )
-          (?:"|')                               # End newline delimiter
-        """
-        self.url_regex = re.compile(regex_str, re.VERBOSE)
+        self.totalasns: set[str] = set()
+        self.totalhosts: set[str] = set()
+        self.urls: set[str] = set()
+        self.totalips: set[str] = set()
+        self.totalemails: set[str] = set()
+        self.execution_status: str | None = None
+        self.stop_reason: str | None = None
 
-    def _build_headers(self) -> dict[str, str]:
-        # API v2 uses API-KEY header
-        return {'API-KEY': self.key, 'User-Agent': Core.get_user_agent()}
+    def _stop(self, status: str, reason: str) -> None:
+        has_results = any((self.totalhosts, self.totalemails, self.totalips, self.totalasns, self.urls))
+        self.execution_status = 'partial' if has_results else status
+        self.stop_reason = reason
 
-    @staticmethod
-    def _is_success(resp: dict[str, Any]) -> bool:
-        # Accept multiple success indicators across versions
+    def _normalize_url(self, value: Any) -> str | None:
+        if not isinstance(value, str):
+            return None
         try:
-            if 'code' in resp:
-                # v2 style often uses code==0 for success
-                return resp.get('code') in (0, 200)
-            if 'status' in resp and isinstance(resp.get('status'), int):
-                # some responses put HTTP-like code here
-                return resp.get('status') in (0, 200)
-        except Exception as e:
-            logger.info(f'ZoomEye response status parsing failed with {type(e).__name__}')
-            return False
+            parsed = urlsplit(value.rstrip('),.;'))
+        except ValueError:
+            return None
+        hostname = normalize_scoped_hostname(parsed.hostname, self.target)
+        if parsed.scheme not in {'http', 'https'} or not parsed.netloc or hostname is None:
+            return None
+        if parsed.username is not None or parsed.password is not None:
+            return None
+        try:
+            port = f':{parsed.port}' if parsed.port is not None else ''
+        except ValueError:
+            return None
+        return urlunsplit((parsed.scheme, f'{hostname}{port}', parsed.path, parsed.query, ''))
 
-        # If no explicit status, assume success and let parsing validate
-        return True
-
-    @staticmethod
-    def _unwrap_data(resp: dict[str, Any]) -> dict[str, Any]:
-        # Many v2 endpoints return {'code':0,'data':{...}}
-        data = resp.get('data')
-        return data if isinstance(data, dict) else resp
-
-    @staticmethod
-    def _page_total_from_payload(payload: dict[str, Any], page_size: int) -> int:
-        # Prefer explicit page total if provided
-        if 'available' in payload:
-            try:
-                return int(payload['available'])
-            except ValueError:
-                logger.info('Payload availablity is not a integer')
-            except Exception as e:
-                logger.info(f'An error occurred in page_total_from_payload : {e}')
-        total_results = payload.get('total') or payload.get('count') or payload.get('total_count')
-        if isinstance(total_results, int) and total_results >= 0:
-            size = payload.get('size') or page_size
-            try:
-                size_int = int(size)
-                size_int = size_int if size_int > 0 else page_size
-            except Exception:
-                size_int = page_size
-            return max(1, math.ceil(total_results / size_int))
-        # Fallback: if a list/matches is present, consider at least one page
-        if any(k in payload for k in ('matches', 'list', 'results', 'items')):
-            return 1
-        return 1
-
-    @staticmethod
-    def _safe_add_hostname(container: set, value: str | None) -> None:
-        if not value or not isinstance(value, str):
-            return
-        v = value.strip()
-        if not v:
-            return
-        v = v.removesuffix('.')
-        container.add(v)
-
-    async def fetch_subdomains(self) -> None:
-        headers = self._build_headers()
-        # type=0 for subdomain search per docs
-        size = 30
-        params = (('q', self.word), ('type', '0'), ('page', '1'), ('size', str(size)))
-        response = await AsyncFetcher.fetch_all(
-            [self.domain_url],
+    async def _fetch_page(self, session: Any, page: int, page_size: int) -> dict[str, Any] | None:
+        query = base64.b64encode(f'domain="{self.target}"'.encode()).decode()
+        response = await AsyncFetcher.post_fetch(
+            self.baseurl,
+            session=session,
             json=True,
-            proxy=self.proxy,
-            headers=headers,
-            params=params,
+            include_metadata=True,
+            json_body={
+                'qbase64': query,
+                'sub_type': 'all',
+                'page': page,
+                'pagesize': page_size,
+                'fields': self.RESPONSE_FIELDS,
+            },
         )
-        if not response:
+        if error := provider_http_error(response):
+            self._stop(*error)
+            return None
+        assert isinstance(response, FetcherResponse)
+        if not isinstance(response.body, dict):
+            self._stop('failed', 'invalid-response')
+            return None
+        if response.body.get('code') != 60000:
+            self._stop('failed', 'provider-error')
+            return None
+        data = response.body.get('data')
+        total = response.body.get('total')
+        if not isinstance(data, list) or isinstance(total, bool) or not isinstance(total, int) or total < 0:
+            self._stop('failed', 'invalid-response')
+            return None
+        return response.body
+
+    async def do_search(self, session: Any) -> None:
+        page_size = min(self.PAGE_SIZE, self.limit)
+        first = await self._fetch_page(session, 1, page_size)
+        if first is None:
             return
-        raw = response[0] or {}
-        if not self._is_success(raw):
-            return
-        payload = self._unwrap_data(raw)
-        total_pages = self._page_total_from_payload(payload, size)
-        # If user requested more pages than available, clamp to available
-        self.limit = min(self.limit, total_pages) if total_pages >= 1 else self.limit
+        await self._store_matches(first['data'][:page_size])
+        page_limit = math.ceil(min(first['total'], self.limit) / page_size) if first['total'] else 1
+        for page in range(2, page_limit + 1):
+            remaining = self.limit - ((page - 1) * page_size)
+            response = await self._fetch_page(session, page, page_size)
+            if response is None:
+                return
+            await self._store_matches(response['data'][:remaining])
 
-        # Parse first page
-        first_list = payload.get('list') or payload.get('results') or []
-        self.totalhosts.extend(
-            [item.get('name') or item.get('domain') or item.get('host') for item in first_list if isinstance(item, dict)]
-        )
+    async def _store_matches(self, matches: list[Any]) -> None:
+        hostnames, emails, ips, asns, urls, malformed = await self.parse_matches(matches)
+        self.totalhosts.update(hostnames)
+        self.totalemails.update(emails)
+        self.totalips.update(ips)
+        self.totalasns.update(asns)
+        self.urls.update(urls)
+        if malformed:
+            self._stop('failed', 'invalid-response')
 
-        # Iterate remaining pages
-        for i in range(2, self.limit + 1):
-            params = (('q', self.word), ('type', '0'), ('page', str(i)), ('size', str(size)))
-            response = await AsyncFetcher.fetch_all(
-                [self.domain_url],
-                json=True,
-                proxy=self.proxy,
-                headers=headers,
-                params=params,
-            )
-            if not response:
-                break
-            raw = response[0] or {}
-            if not self._is_success(raw):
-                break
-            payload = self._unwrap_data(raw)
-            page_list = payload.get('list') or payload.get('results') or []
-            found_subdomains = [
-                item.get('name') or item.get('domain') or item.get('host') for item in page_list if isinstance(item, dict)
-            ]
-            found_subdomains = [x for x in found_subdomains if x]
-            if not found_subdomains:
-                break
-            self.totalhosts.extend(found_subdomains)
-            if i % 10 == 0:
-                await asyncio.sleep(get_delay() + 1)
-
-    async def do_search(self) -> None:
-        headers = self._build_headers()
-        # Fetch subdomains first
-        await self.fetch_subdomains()
-
-        size = 20
-        params = (('query', f'site:{self.word}'), ('page', '1'), ('size', str(size)))
-        response = await AsyncFetcher.fetch_all([self.baseurl], json=True, proxy=self.proxy, headers=headers, params=params)
-        if not response:
-            return
-        raw = response[0] or {}
-        payload = self._unwrap_data(raw)
-        total_pages = self._page_total_from_payload(payload, size)
-        self.limit = min(self.limit, total_pages) if total_pages >= 1 else self.limit
-        cur_page = 2 if self.limit >= 2 else -1
-
-        nomatches_counter = 0
-
-        def extract_matches(p: dict[str, Any]) -> Iterable[dict]:
-            return p.get('matches') or p.get('list') or p.get('results') or []
-
-        if cur_page == -1:
-            if isinstance(payload, dict):
-                matches = extract_matches(payload)
-                if matches:
-                    hostnames, emails, ips, asns, urls = await self.parse_matches(matches)
-                    self.totalhosts.extend(hostnames)
-                    self.totalemails.extend(emails)
-                    self.totalips.extend(ips)
-                    self.totalasns.extend(asns)
-                    self.urls.extend(urls)
-            return
-
-        # Parse first page then loop
-        if isinstance(payload, dict):
-            matches = extract_matches(payload)
-            if matches:
-                hostnames, emails, ips, asns, urls = await self.parse_matches(matches)
-                self.totalhosts.extend(hostnames)
-                self.totalemails.extend(emails)
-                self.totalips.extend(ips)
-                self.totalasns.extend(asns)
-                self.urls.extend(urls)
-
-        for num in range(2, self.limit + 1):
-            params = (('query', f'site:{self.word}'), ('page', str(num)), ('size', str(size)))
-            response = await AsyncFetcher.fetch_all(
-                [self.baseurl],
-                json=True,
-                proxy=self.proxy,
-                headers=headers,
-                params=params,
-            )
-            if not response:
-                break
-            raw = response[0] or {}
-            payload = self._unwrap_data(raw)
-            matches = extract_matches(payload)
-            if not matches:
-                nomatches_counter += 1
-                if nomatches_counter >= 5:
-                    break
-                continue
-
-            hostnames, emails, ips, asns, urls = await self.parse_matches(matches)
-
-            if len(hostnames) == 0 and len(emails) == 0 and len(ips) == 0 and len(asns) == 0 and len(urls) == 0:
-                nomatches_counter += 1
-                if nomatches_counter >= 5:
-                    break
-
-            self.totalhosts.extend(hostnames)
-            self.totalemails.extend(emails)
-            self.totalips.extend(ips)
-            self.totalasns.extend(asns)
-            self.urls.extend(urls)
-
-            if num % 10 == 0:
-                await asyncio.sleep(get_delay() + 1)
-
-    async def parse_matches(self, matches):
-        # Helper function to parse items from match json
+    async def parse_matches(
+        self,
+        matches: list[Any],
+    ) -> tuple[set[str], set[str], set[str], set[str], set[str], bool]:
         ips: set[str] = set()
         urls: set[str] = set()
         hostnames: set[str] = set()
         asns: set[str] = set()
         emails: set[str] = set()
+        malformed = False
 
         for match in matches:
             if not isinstance(match, dict):
+                malformed = True
                 continue
-            try:
-                # IPs
-                ip = match.get('ip') or match.get('ip_str') or match.get('ip_str_v4') or match.get('address')
-                if isinstance(ip, str):
-                    ips.add(ip)
+            raw_ip = match.get('ip')
+            if raw_ip is not None:
+                try:
+                    ips.add(str(ip_address(str(raw_ip).strip())))
+                except ValueError:
+                    malformed = True
 
-                # ASNs
-                asn_val = None
-                if isinstance(match.get('geoinfo'), dict):
-                    asn_val = match['geoinfo'].get('asn')
-                asn_val = asn_val or match.get('asn')
-                if asn_val:
-                    try:
-                        asns.add(f'AS{int(asn_val)}')
-                    except Exception:
-                        # if already a string like 'AS12345'
-                        asns.add(str(asn_val) if str(asn_val).startswith('AS') else f'AS{asn_val!s}')
+            raw_asn = match.get('asn')
+            if raw_asn is not None:
+                try:
+                    asns.add(f'AS{int(str(raw_asn).removeprefix("AS"))}')
+                except ValueError:
+                    malformed = True
 
-                # Reverse DNS and hostnames
-                rdns_new = match.get('rdns_new')
-                if isinstance(rdns_new, str) and rdns_new:
-                    if ',' in rdns_new:
-                        parts = str(rdns_new).split(',')
-                        primary = parts[0]
-                        secondary = parts[1] if len(parts) == 2 else None
-                        if primary:
-                            self._safe_add_hostname(hostnames, primary)
-                        if secondary:
-                            self._safe_add_hostname(hostnames, secondary)
-                    else:
-                        self._safe_add_hostname(hostnames, rdns_new)
+            for field in ('domain', 'hostname', 'rdns'):
+                value = match.get(field)
+                if value is None:
+                    continue
+                if not isinstance(value, str):
+                    malformed = True
+                elif (hostname := normalize_scoped_hostname(value, self.target)) and hostname != self.target:
+                    hostnames.add(hostname)
 
-                rdns = match.get('rdns')
-                if isinstance(rdns, str) and rdns:
-                    self._safe_add_hostname(hostnames, rdns)
+            if raw_url := match.get('url'):
+                if normalized_url := self._normalize_url(raw_url):
+                    urls.add(normalized_url)
+                    if url_hostname := normalize_scoped_hostname(urlsplit(normalized_url).hostname, self.target):
+                        if url_hostname != self.target:
+                            hostnames.add(url_hostname)
+                elif isinstance(raw_url, str):
+                    malformed = True
 
-                # Additional hostname-like fields
-                for f in ('hostname', 'host', 'domain', 'site', 'fqdn'):
-                    self._safe_add_hostname(hostnames, match.get(f))
-                for f in ('hostnames', 'domains', 'names'):
-                    vals = match.get(f)
-                    if isinstance(vals, list):
-                        for v in vals:
-                            if isinstance(v, str):
-                                self._safe_add_hostname(hostnames, v)
+            text_values: list[str] = []
+            for field in ('banner', 'header', 'body', 'ssl'):
+                value = match.get(field)
+                if value is None:
+                    continue
+                if isinstance(value, str):
+                    text_values.append(value)
+                else:
+                    malformed = True
+            if not text_values:
+                continue
+            content = '\n'.join(text_values)
+            parser = myparser.Parser(content, self.word)
+            emails.update(await parser.emails())
+            parser = myparser.Parser(content, self.word)
+            hostnames.update(hostname for hostname in await parser.hostnames() if hostname != self.target)
+            for candidate in self.URL_PATTERN.findall(content):
+                if normalized_url := self._normalize_url(candidate):
+                    urls.add(normalized_url)
 
-                # Banner/content extraction for emails, hostnames, and URLs
-                banners = []
-
-                portinfo = match.get('portinfo')
-                if isinstance(portinfo, dict):
-                    b = portinfo.get('banner')
-                    if isinstance(b, str) and b:
-                        banners.append(b)
-
-                service = match.get('service')
-                if isinstance(service, dict):
-                    for key in ('banner', 'data', 'raw'):
-                        v = service.get(key)
-                        if isinstance(v, str) and v:
-                            banners.append(v)
-                    http = service.get('http')
-                    if isinstance(http, dict):
-                        for key in ('title', 'html', 'body', 'server', 'raw'):
-                            v = http.get(key)
-                            if isinstance(v, str) and v:
-                                banners.append(v)
-
-                content_blob = '\n'.join(banners)
-                if content_blob:
-                    temp_emails = set(await self.parse_emails(content_blob))
-                    emails.update(temp_emails)
-                    hostnames.update(set(await self.parse_hostnames(content_blob)))
-                    for url_match in re.finditer(self.url_regex, content_blob):
-                        candidate = str(url_match.group(1)).replace('"', '')
-                        try:
-                            parsed = urlparse(candidate)
-                            hostname = normalize_scoped_hostname(parsed.hostname, self.word)
-                        except ValueError:
-                            continue
-                        if parsed.scheme in {'http', 'https'} and parsed.netloc and hostname:
-                            urls.add(candidate)
-
-            except Exception as e:
-                # Continue processing other matches instead of failing completely
-                logger.info(f'ZoomEye parsing error: {e}')
-
-        return hostnames, emails, ips, asns, urls
+        return hostnames, emails, ips, asns, urls, malformed
 
     async def process(self, proxy: bool = False) -> None:
         self.proxy = proxy
-        await self.do_search()  # Only need to do it once.
+        self.execution_status = None
+        self.stop_reason = None
+        try:
+            async with AsyncFetcher.open_session(
+                headers={'API-KEY': self.key, 'Content-Type': 'application/json'},
+                proxy=proxy,
+            ) as session:
+                await self.do_search(session)
+        except Exception:
+            self._stop('failed', 'transport-error')
+            return
+        has_results = any((self.totalhosts, self.totalemails, self.totalips, self.totalasns, self.urls))
+        if self.execution_status is not None and has_results:
+            self.execution_status = 'partial'
+        elif self.execution_status is None:
+            self.execution_status = 'completed'
+            self.stop_reason = None if has_results else 'no-results'
 
-    async def parse_emails(self, content):
-        rawres = myparser.Parser(content, self.word)
-        return await rawres.emails()
+    async def get_hostnames(self) -> set[str]:
+        return self.totalhosts
 
-    async def parse_hostnames(self, content):
-        rawres = myparser.Parser(content, self.word)
-        return await rawres.hostnames()
+    async def get_emails(self) -> set[str]:
+        return self.totalemails
 
-    async def get_hostnames(self):
-        return set(self.totalhosts)
+    async def get_ips(self) -> set[str]:
+        return self.totalips
 
-    async def get_emails(self):
-        return set(self.totalemails)
+    async def get_asns(self) -> set[str]:
+        return self.totalasns
 
-    async def get_ips(self):
-        return set(self.totalips)
-
-    async def get_asns(self):
-        return set(self.totalasns)
-
-    async def get_urls(self):
-        return set(self.urls)
+    async def get_urls(self) -> set[str]:
+        return self.urls

theHarvester/lib/core.py

@@ -945,13 +945,30 @@ class AsyncFetcher:
     @classmethod
     async def fetch_all(
         cls,
-        urls,
-        headers=None,
+        urls: list[str],
+        headers: dict[str, str] | None = None,
         params: Sized = '',
         json: bool = False,
-        proxy: bool = False,
+        proxy: str | bool | None = False,
         include_metadata: bool = False,
-    ) -> list:
+        *,
+        session: aiohttp.ClientSession | None = None,
+    ) -> list[Any]:
+        if session is not None:
+            return list(
+                await asyncio.gather(
+                    *[
+                        AsyncFetcher.fetch(
+                            session=session,
+                            url=url,
+                            params=params,
+                            json=json,
+                            include_metadata=include_metadata,
+                        )
+                        for url in urls
+                    ]
+                )
+            )
         # By default, timeout is 5 minutes; 60 seconds should suffice
         headers = cls._default_headers(headers)
         timeout = cls._request_timeout(60)

theHarvester/lib/source_runner.py

@@ -143,7 +143,7 @@ SOURCE_FACTORIES: dict[str, SourceFactory] = {
     'dnsdumpster': lambda request: search_dnsdumpster.SearchDNSDumpster(request.target),
     'duckduckgo': lambda request: duckduckgosearch.SearchDuckDuckGo(request.target, request.limit),
     'dymo': lambda request: dymosearch.SearchDymo(request.target),
-    'fofa': lambda request: fofa.SearchFofa(request.target),
+    'fofa': lambda request: fofa.SearchFofa(request.target, request.limit),
     'fullhunt': lambda request: fullhuntsearch.SearchFullHunt(request.target),
     'github-code': lambda request: githubcode.SearchGithubCode(request.target, request.limit),
     'gitlab': lambda request: gitlabsearch.SearchGitlab(request.target),
@@ -152,13 +152,13 @@ SOURCE_FACTORIES: dict[str, SourceFactory] = {
     'hibpverified': lambda request: hibpverified.SearchHibpVerified(request.target),
     'hudsonrock': lambda request: hudsonrocksearch.SearchHudsonRock(request.target),
     'hunter': lambda request: huntersearch.SearchHunter(request.target, request.limit, request.start),
-    'hunterhow': lambda request: searchhunterhow.SearchHunterHow(request.target),
+    'hunterhow': lambda request: searchhunterhow.SearchHunterHow(request.target, request.limit),
     'intelx': lambda request: intelxsearch.SearchIntelx(request.target),
     'leakix': lambda request: leakix.SearchLeakix(request.target),
     'leaklookup': lambda request: leaklookup.SearchLeakLookup(request.target),
     'mojeek': lambda request: mojeek.SearchMojeek(request.target, request.limit),
     'netlas': lambda request: netlas.SearchNetlas(request.target, request.limit),
-    'onyphe': lambda request: onyphe.SearchOnyphe(request.target),
+    'onyphe': lambda request: onyphe.SearchOnyphe(request.target, request.limit),
     'otx': lambda request: otxsearch.SearchOtx(request.target),
     'pentesttools': lambda request: pentesttools.SearchPentestTools(request.target),
     'projectdiscovery': lambda request: projectdiscovery.SearchDiscovery(request.target),
@@ -166,7 +166,7 @@ SOURCE_FACTORIES: dict[str, SourceFactory] = {
     'robtex': lambda request: robtex.SearchRobtex(request.target),
     'rocketreach': lambda request: rocketreach.SearchRocketReach(request.target, request.limit),
     'securityTrails': lambda request: securitytrailssearch.SearchSecuritytrail(request.target),
-    'securityscorecard': lambda request: securityscorecard.SearchSecurityScorecard(request.target),
+    'securityscorecard': lambda request: securityscorecard.SearchSecurityScorecard(request.target, request.limit),
     'sherlockeye': lambda request: sherlockeye.SearchSherlockeye(request.target),
     'shodan': lambda request: shodansearch.SearchShodan(request.target),
     'shodanInternetDB': lambda request: shodan_internetdb.SearchShodanInternetDB(request.target),
@@ -176,10 +176,10 @@ SOURCE_FACTORIES: dict[str, SourceFactory] = {
     'subdomainfinderc99': lambda request: subdomainfinderc99.SearchSubdomainfinderc99(request.target),
     'thc': lambda request: thc.SearchThc(request.target),
     'tomba': lambda request: tombasearch.SearchTomba(request.target, request.limit, request.start),
-    'urlscan': lambda request: urlscan.SearchUrlscan(request.target),
-    'virustotal': lambda request: virustotal.SearchVirustotal(request.target),
+    'urlscan': lambda request: urlscan.SearchUrlscan(request.target, request.limit),
+    'virustotal': lambda request: virustotal.SearchVirustotal(request.target, request.limit),
     'waybackarchive': lambda request: waybackarchive.SearchWaybackarchive(request.target, request.limit),
-    'whoisxml': lambda request: whoisxml.SearchWhoisXML(request.target),
+    'whoisxml': lambda request: whoisxml.SearchWhoisXML(request.target, request.limit),
     'windvane': lambda request: windvane.SearchWindvane(request.target),
     'yahoo': lambda request: yahoosearch.SearchYahoo(request.target, request.limit),
     'zoomeye': lambda request: zoomeyesearch.SearchZoomEye(request.target, request.limit),