diff --git a/.github/workflows/provider-smoke.yml b/.github/workflows/provider-smoke.yml index c33a356c..2f9dcdbc 100644 --- a/.github/workflows/provider-smoke.yml +++ b/.github/workflows/provider-smoke.yml @@ -43,34 +43,35 @@ jobs: timeout-minutes: 5 run: pytest --run-live-network -m live_network + # These are bounded CLI crash smokes, not provider conformance tests. - name: Run theHarvester module CertSpotter timeout-minutes: 5 - run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b certspotter + run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b certspotter -l 10 -q - name: Run theHarvester module Crtsh timeout-minutes: 5 - run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b crtsh + run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b crtsh -l 10 -q - name: Run theHarvester module DuckDuckGo timeout-minutes: 5 - run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b duckduckgo + run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b duckduckgo -l 10 -q - name: Run theHarvester module HackerTarget timeout-minutes: 5 - run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b hackertarget + run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b hackertarget -l 10 -q - name: Run theHarvester module Otx timeout-minutes: 5 - run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b otx + run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b otx -l 10 -q - name: Run theHarvester module RapidDns timeout-minutes: 5 - run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b rapiddns + run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b rapiddns -l 10 -q - name: Run theHarvester module Urlscan timeout-minutes: 5 - run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b urlscan + run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b urlscan -l 10 -q - name: Run theHarvester module Yahoo timeout-minutes: 5 - run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b yahoo + run: theHarvester -d "$SMOKE_TEST_DOMAIN" -b yahoo -l 10 -q diff --git a/.github/workflows/theHarvester.yml b/.github/workflows/theHarvester.yml index f997be5f..52cfbba9 100644 --- a/.github/workflows/theHarvester.yml +++ b/.github/workflows/theHarvester.yml @@ -56,5 +56,7 @@ jobs: args: format --check - name: Test with pytest - run: | - pytest + run: pytest + + - name: Type check with mypy + run: mypy theHarvester diff --git a/CHANGELOG.md b/CHANGELOG.md index e1d3acad..019429a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Added a catalog-derived offline provider-contract gate that fails on missing, unknown, or duplicate source coverage while keeping live checks outside routine CI. - Added sourced ASN organization attribution from URLScan, ONYPHE, and Shodan, linked to the exact hostname or IP evidence and retained in SQLite, JSONL, the API, CLI output, and HarvestView without claiming ownership or scope. - Added bounded RouteViews routing enrichment for exact discovered IPs with sourced ASN attribution, or explicit ASN, IP, and CIDR targets, retaining typed origin, BGP-route, and RPKI evidence as external relationships without expanding active scope. - Added route-aware `--no-hosts` and REST/HarvestView `no_hosts` filtering that skips hostname-only sources while retaining non-host evidence from mixed sources. @@ -35,6 +36,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added root contributor and security policies, structured issue forms, repository agent guidance, discovery terminology, and an operator-focused documentation wiki ([d090a29a](https://github.com/laramies/theHarvester/commit/d090a29a), [7c491ef5](https://github.com/laramies/theHarvester/commit/7c491ef5), [8b9d420b](https://github.com/laramies/theHarvester/commit/8b9d420b)). ### Changed +- Updated BeVigil, Dymo, FOFA, FullHunt, Hunter.how, Netlas, ONYPHE, SecurityScorecard, SecurityTrails, SherlockEye, SubdomainFinder C99, VirusTotal, WhoisXML, and ZoomEye provider contracts to retain scoped partial evidence and report authentication, quota, transport, HTTP, and malformed-response outcomes truthfully. FOFA and ONYPHE now honor the operator result limit across documented pagination, while ZoomEye uses the current `POST /v2/search` API and no longer stops after five empty result pages. - Replaced runtime takeover fingerprint downloads and global body-substring matches with pinned provider-gated DNS, wildcard controls, and compound HTTP rules. Every checked hostname is now stored as an indicator, no-indicator, or inconclusive outcome with typed DNS, HTTP, rule, and error details in JSONL, SQLite, the API, and HarvestView. Direct checks share one cookie-free HTTP session, keep bounded response bodies, and rely on the whole-run deadline instead of silently inheriting aiohttp's default timeout. - HarvestView now summarizes retained evidence and producer health at a glance, links directly to execution outcomes that need attention, and keeps evidence values ahead of optional actions on mobile. Its source picker reports credential readiness without exposing values, prevents unavailable source selections, and replaces the mobile nested-scroll catalog with collapsible activity groups. - Routed discovery sources through immutable source jobs with bounded `TaskGroup` ownership, typed outcomes, and native cancellation propagation instead of queuing live coroutine objects. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b123b857..91857ae6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -49,6 +49,13 @@ Useful cases include: - pagination and retry termination; - normalized, deduplicated results. +Every canonical provider has one offline contract module marked with +`pytest.mark.provider_contract("source-name")`. The catalog-derived coverage +gate fails when a catalog entry has no contract, when a contract names an +unknown source, or when two modules claim the same source. Add the marker to +the provider's deterministic contract module; do not maintain a second source +list in tests. + Use the shared transport when it can represent the request. If a provider requires behavior the shared transport does not support, keep the exception local and explain it in the pull request. Never log credentials, account information, private target data, or raw API responses. ## Test safely @@ -65,15 +72,12 @@ Before submitting, run the non-mutating quality checks and full test suite: uv run ruff check . uv run ruff format --check . uv run pytest -``` - -Changes to typed interfaces should also pass: - -```bash uv run mypy theHarvester ``` -Routine verification must use mocks, local fixtures, and reserved example domains. Do not run broad or active reconnaissance against third-party targets. If live verification is essential, use only a target you own or are explicitly authorized to test, limit the request scope, and keep collected data out of commits, issues, and pull requests. +Routine verification must use mocks, local fixtures, and reserved example domains. The test harness blocks external Python socket traffic unless a test is marked `live_network` and pytest is invoked with both `--run-live-network` and `-m live_network`. A live-marked test never satisfies the provider-contract coverage gate. + +Do not run broad or active reconnaissance against third-party targets. If live verification is essential, use only a target you own or are explicitly authorized to test, limit the request scope, and keep collected data out of commits, issues, and pull requests. The manually dispatched provider workflow uses `mozilla.org` for small passive CLI crash smokes. Those runs can detect packaging, credential, or provider drift; they are not conformance tests and should not be retried merely to obtain more results. CI runs additional source smoke tests, CodeQL, dependency review, and container checks. Contributors do not need to reproduce broad live-source checks locally. diff --git a/pyproject.toml b/pyproject.toml index 1136d0e5..302e7af4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,7 @@ asyncio_default_fixture_loop_scope = "function" addopts = "--no-header --strict-markers -m 'not harvestview_e2e'" markers = [ "live_network: contacts an external service and runs only with --run-live-network", + "provider_contract(source): deterministic offline contract for one canonical discovery source", "harvestview_e2e: real-browser tests against an isolated local HarvestView server", ] testpaths = ["tests"] diff --git a/tests/conftest.py b/tests/conftest.py index 9136d7da..f8147b10 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,6 +8,7 @@ from typing import Any import pytest NETWORK_GUARD = pytest.StashKey[pytest.MonkeyPatch]() +PROVIDER_CONTRACT_SOURCES = pytest.StashKey[tuple[str, ...]]() _getaddrinfo = socket.getaddrinfo _gethostbyaddr = socket.gethostbyaddr @@ -39,6 +40,11 @@ def live_test_domain() -> str: return os.environ.get('SMOKE_TEST_DOMAIN', 'mozilla.org') +@pytest.fixture +def provider_contract_sources(request: pytest.FixtureRequest) -> tuple[str, ...]: + return request.config.stash.get(PROVIDER_CONTRACT_SOURCES, ()) + + def pytest_sessionstart(session: pytest.Session) -> None: guard = pytest.MonkeyPatch() guard.setattr(socket, 'getaddrinfo', _guarded_getaddrinfo) @@ -53,6 +59,16 @@ def pytest_sessionstart(session: pytest.Session) -> None: def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + contract_modules: set[tuple[str, str]] = set() + for item in items: + if item.get_closest_marker('live_network') is not None: + continue + for marker in item.iter_markers('provider_contract'): + if len(marker.args) != 1 or marker.kwargs or not isinstance(marker.args[0], str): + raise pytest.UsageError(f'{item.nodeid}: provider_contract requires one canonical source name') + contract_modules.add((marker.args[0], str(item.path))) + config.stash[PROVIDER_CONTRACT_SOURCES] = tuple(source for source, _path in sorted(contract_modules)) + if config.getoption('--run-live-network'): if config.getoption('markexpr') != 'live_network': raise pytest.UsageError('--run-live-network requires -m live_network') diff --git a/tests/discovery/test_apisguru.py b/tests/discovery/test_apisguru.py index 9da36cdc..d38d78c2 100644 --- a/tests/discovery/test_apisguru.py +++ b/tests/discovery/test_apisguru.py @@ -681,6 +681,8 @@ async def test_directory_failures_are_attributed( assert search.stop_reason == stop_reason + + @pytest.mark.asyncio async def test_missing_provider_is_completed_with_no_results(monkeypatch: pytest.MonkeyPatch) -> None: async def fake_fetch(**_kwargs: Any) -> FetcherResponse: @@ -971,3 +973,6 @@ async def test_preferred_spec_failures_are_attributed( assert search.execution_status == execution_status assert search.stop_reason == stop_reason + + +pytestmark = pytest.mark.provider_contract('apis-guru') diff --git a/tests/discovery/test_arquivo.py b/tests/discovery/test_arquivo.py index bad45dc2..ceff39b5 100644 --- a/tests/discovery/test_arquivo.py +++ b/tests/discovery/test_arquivo.py @@ -91,3 +91,6 @@ async def test_process_reports_http_and_malformed_responses( assert await second.get_hostnames() == set() assert 'Arquivo.pt request failed with HTTP 429' in caplog.text assert 'Arquivo.pt returned malformed CDX data' in caplog.text + + +pytestmark = pytest.mark.provider_contract('arquivo') diff --git a/tests/discovery/test_baidusearch.py b/tests/discovery/test_baidusearch.py index 39c27f92..52e3f9d1 100644 --- a/tests/discovery/test_baidusearch.py +++ b/tests/discovery/test_baidusearch.py @@ -175,3 +175,6 @@ class TestBaiduSearch: 'https://www.baidu.com/s?wd=site%3Aexample.com&pn=0', 'https://www.baidu.com/s?wd=site%3Aexample.com&pn=10', ] + + +pytestmark = pytest.mark.provider_contract('baidu') diff --git a/tests/discovery/test_bevigil.py b/tests/discovery/test_bevigil.py new file mode 100644 index 00000000..02e11ead --- /dev/null +++ b/tests/discovery/test_bevigil.py @@ -0,0 +1,182 @@ +import asyncio +import contextlib +from collections.abc import AsyncIterator +from typing import Any + +import pytest + +from theHarvester.discovery import bevigil +from theHarvester.discovery.constants import MissingKey +from theHarvester.lib.core import FetcherResponse + + +@pytest.mark.provider_contract('bevigil') +@pytest.mark.asyncio +async def test_process_collects_scoped_hostnames_and_urls(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(bevigil.Core, 'bevigil_key', staticmethod(lambda: 'test-key')) + session = object() + session_exited = False + open_calls: list[dict[str, Any]] = [] + calls: list[tuple[list[str], dict[str, Any]]] = [] + responses = [ + FetcherResponse( + body={'subdomains': ['API.Example.COM.', 'outside.test']}, + status=200, + headers={}, + ), + FetcherResponse( + body={'urls': ['https://portal.example.com/path', 'https://outside.test/example.com']}, + status=200, + headers={}, + ), + ] + + @contextlib.asynccontextmanager + async def fake_open_session(**kwargs: Any) -> AsyncIterator[object]: + nonlocal session_exited + open_calls.append(kwargs) + try: + yield session + finally: + session_exited = True + + async def fake_fetch_all(urls: list[str], **kwargs: Any) -> list[FetcherResponse]: + calls.append((urls, kwargs)) + return [responses.pop(0)] + + monkeypatch.setattr(bevigil.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(bevigil.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = bevigil.SearchBeVigil('example.com') + + await search.process(proxy=True) + + assert await search.get_hostnames() == {'api.example.com'} + assert await search.get_urls() == {'https://portal.example.com/path'} + assert [urls for urls, _kwargs in calls] == [ + ['https://osint.bevigil.com/api/example.com/subdomains/'], + ['https://osint.bevigil.com/api/example.com/urls/'], + ] + assert all(kwargs['headers'] == {'X-Access-Token': 'test-key'} for _urls, kwargs in calls) + assert all(kwargs['proxy'] is True for _urls, kwargs in calls) + assert all(kwargs['json'] is True for _urls, kwargs in calls) + assert all(kwargs['include_metadata'] is True for _urls, kwargs in calls) + assert all(kwargs['session'] is session for _urls, kwargs in calls) + assert open_calls == [ + { + 'headers': {'X-Access-Token': 'test-key'}, + 'proxy': True, + 'request_timeout': 60, + } + ] + assert session_exited is True + assert search.execution_status == 'completed' + assert search.stop_reason is None + + +@pytest.mark.parametrize('key', [None, '', ' ']) +def test_missing_or_empty_key_fails_before_transport(monkeypatch: pytest.MonkeyPatch, key: str | None) -> None: + monkeypatch.setattr(bevigil.Core, 'bevigil_key', staticmethod(lambda: key)) + + with pytest.raises(MissingKey): + bevigil.SearchBeVigil('example.com') + + +@pytest.mark.parametrize( + ('response', 'execution_status', 'stop_reason'), + [ + (None, 'failed', 'transport-error'), + (FetcherResponse(body={}, status=401, headers={}), 'failed', 'access-denied'), + (FetcherResponse(body={}, status=403, headers={}), 'failed', 'access-denied'), + (FetcherResponse(body={}, status=429, headers={}), 'rate-limited', 'http-429'), + (FetcherResponse(body={}, status=503, headers={}), 'failed', 'http-503'), + (FetcherResponse(body={'subdomains': {}}, status=200, headers={}), 'failed', 'invalid-response'), + ], +) +@pytest.mark.asyncio +async def test_failed_first_response_is_attributed( + monkeypatch: pytest.MonkeyPatch, + response: FetcherResponse | None, + execution_status: str, + stop_reason: str, +) -> None: + monkeypatch.setattr(bevigil.Core, 'bevigil_key', staticmethod(lambda: 'test-key')) + + async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse | None]: + return [response] + + monkeypatch.setattr(bevigil.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = bevigil.SearchBeVigil('example.com') + + await search.process() + + assert await search.get_hostnames() == set() + assert await search.get_urls() == set() + assert search.execution_status == execution_status + assert search.stop_reason == stop_reason + + +@pytest.mark.asyncio +async def test_later_malformed_response_preserves_partial_results(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(bevigil.Core, 'bevigil_key', staticmethod(lambda: 'test-key')) + responses = [ + FetcherResponse(body={'subdomains': ['api.example.com']}, status=200, headers={}), + FetcherResponse(body={'urls': {}}, status=200, headers={}), + ] + + async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: + return [responses.pop(0)] + + monkeypatch.setattr(bevigil.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = bevigil.SearchBeVigil('example.com') + + await search.process() + + assert await search.get_hostnames() == {'api.example.com'} + assert await search.get_urls() == set() + assert search.execution_status == 'partial' + assert search.stop_reason == 'invalid-response' + + +@pytest.mark.asyncio +async def test_cancellation_closes_provider_session(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(bevigil.Core, 'bevigil_key', staticmethod(lambda: 'test-key')) + session_exited = False + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + nonlocal session_exited + try: + yield object() + finally: + session_exited = True + + async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: + raise asyncio.CancelledError('operator-stop') + + monkeypatch.setattr(bevigil.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(bevigil.AsyncFetcher, 'fetch_all', fake_fetch_all) + + with pytest.raises(asyncio.CancelledError, match='operator-stop'): + await bevigil.SearchBeVigil('example.com').process() + assert session_exited is True + + +@pytest.mark.asyncio +async def test_early_malformed_rows_and_later_valid_evidence_are_partial(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(bevigil.Core, 'bevigil_key', staticmethod(lambda: 'test-key')) + responses = [ + FetcherResponse(body={'subdomains': [7]}, status=200, headers={}), + FetcherResponse(body={'urls': ['https://portal.example.com/path']}, status=200, headers={}), + ] + + async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: + return [responses.pop(0)] + + monkeypatch.setattr(bevigil.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = bevigil.SearchBeVigil('example.com') + + await search.process() + + assert await search.get_urls() == {'https://portal.example.com/path'} + assert search.execution_status == 'partial' + assert search.stop_reason == 'invalid-response' diff --git a/tests/discovery/test_brave.py b/tests/discovery/test_brave.py index e41ee6d1..0eb2d7d7 100644 --- a/tests/discovery/test_brave.py +++ b/tests/discovery/test_brave.py @@ -259,3 +259,6 @@ async def test_brave_never_exceeds_maximum_page_offset( await search.process() assert [request['offset'] for request in requests] == [[str(offset)] for offset in range(10)] * 2 + + +pytestmark = pytest.mark.provider_contract('brave') diff --git a/tests/discovery/test_bufferoverun.py b/tests/discovery/test_bufferoverun.py index 627730b8..8dad5fb7 100644 --- a/tests/discovery/test_bufferoverun.py +++ b/tests/discovery/test_bufferoverun.py @@ -39,6 +39,8 @@ async def test_process_parses_historical_four_column_rows(monkeypatch: pytest.Mo assert search.stop_reason == 'invalid-response' + + @pytest.mark.asyncio async def test_valid_empty_response_is_completed(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(bufferoverun.Core, 'bufferoverun_key', staticmethod(lambda: 'test-key')) @@ -137,3 +139,6 @@ async def test_non_string_row_preserves_valid_partial_results(monkeypatch: pytes assert await search.get_ips() == {'192.0.2.10'} assert search.execution_status == 'partial' assert search.stop_reason == 'invalid-response' + + +pytestmark = pytest.mark.provider_contract('bufferoverun') diff --git a/tests/discovery/test_builtwith.py b/tests/discovery/test_builtwith.py index 95072e13..f2b9508c 100644 --- a/tests/discovery/test_builtwith.py +++ b/tests/discovery/test_builtwith.py @@ -353,3 +353,6 @@ async def test_normalized_builtwith_results_reach_completed_jsonl( assert {'type': 'server', 'value': 'nginx', 'sources': ['builtwith']} in records assert {'type': 'cms', 'value': 'WordPress', 'sources': ['builtwith']} in records assert {'type': 'analytics', 'value': 'Google Analytics', 'sources': ['builtwith']} in records + + +pytestmark = pytest.mark.provider_contract('builtwith') diff --git a/tests/discovery/test_censys.py b/tests/discovery/test_censys.py index 50e17bec..aef02150 100644 --- a/tests/discovery/test_censys.py +++ b/tests/discovery/test_censys.py @@ -310,3 +310,6 @@ async def test_search_classifies_transport_exceptions(monkeypatch) -> None: def test_deprecated_censys_sdk_is_not_a_runtime_dependency() -> None: assert '"censys==' not in Path('pyproject.toml').read_text() + + +pytestmark = pytest.mark.provider_contract('censys') diff --git a/tests/discovery/test_certspotter.py b/tests/discovery/test_certspotter.py index 6eff4a88..9ed17eb1 100644 --- a/tests/discovery/test_certspotter.py +++ b/tests/discovery/test_certspotter.py @@ -337,3 +337,6 @@ class TestCertspotterSearch(object): if __name__ == '__main__': pytest.main() + + +pytestmark = pytest.mark.provider_contract('certspotter') diff --git a/tests/discovery/test_commoncrawl.py b/tests/discovery/test_commoncrawl.py index 0938ae21..4cfe2c0d 100644 --- a/tests/discovery/test_commoncrawl.py +++ b/tests/discovery/test_commoncrawl.py @@ -416,6 +416,8 @@ async def test_process_retains_partial_results_at_the_runtime_limit(monkeypatch: await asyncio.wait_for(search.process(), timeout=0.1) assert await search.get_hostnames() == {'api.example.com'} + + assert search.execution_status == 'partial' assert search.stop_reason == 'runtime-limit' @@ -548,3 +550,6 @@ async def test_process_keeps_results_when_another_query_fails(monkeypatch: pytes await search.process() assert await search.get_hostnames() == {'api.example.com'} + + +pytestmark = pytest.mark.provider_contract('commoncrawl') diff --git a/tests/discovery/test_criminalip.py b/tests/discovery/test_criminalip.py index bb50752d..64bc8366 100644 --- a/tests/discovery/test_criminalip.py +++ b/tests/discovery/test_criminalip.py @@ -120,3 +120,6 @@ async def test_do_search_uses_v2_report_endpoint(monkeypatch) -> None: assert any('/v2/domain/report/12345' in url for url in called_urls) assert all('/v1/domain/report/' not in url for url in called_urls) + + +pytestmark = pytest.mark.provider_contract('criminalip') diff --git a/tests/discovery/test_crtname.py b/tests/discovery/test_crtname.py index f7e15fbb..68c3e41c 100644 --- a/tests/discovery/test_crtname.py +++ b/tests/discovery/test_crtname.py @@ -319,3 +319,6 @@ async def test_crt_name_and_crtsh_share_one_result_with_both_sources( findings = {(record['type'], record['value']): record for record in records[1:]} assert findings[('hostname', 'shared.example.com')]['sources'] == ['crt-name', 'crtsh'] assert findings[('hostname', 'only-crt-name.example.com')]['sources'] == ['crt-name'] + + +pytestmark = pytest.mark.provider_contract('crt-name') diff --git a/tests/discovery/test_crtsh.py b/tests/discovery/test_crtsh.py index 8ef5cea7..487beeb3 100644 --- a/tests/discovery/test_crtsh.py +++ b/tests/discovery/test_crtsh.py @@ -169,3 +169,6 @@ class TestCrtshIntegration: from theHarvester.lib.source_catalog import SOURCE_SPECS assert 'crtsh' in SOURCE_SPECS + + +pytestmark = pytest.mark.provider_contract('crtsh') diff --git a/tests/discovery/test_dnsdb.py b/tests/discovery/test_dnsdb.py index 677b9268..54608948 100644 --- a/tests/discovery/test_dnsdb.py +++ b/tests/discovery/test_dnsdb.py @@ -180,3 +180,6 @@ async def test_process_exposes_http_failures( with pytest.raises(expected_error): await dnsdb.SearchDNSDB('example.com').process() + + +pytestmark = pytest.mark.provider_contract('dnsdb') diff --git a/tests/discovery/test_dnsdumpster.py b/tests/discovery/test_dnsdumpster.py index 4b0805ed..97eb4b99 100644 --- a/tests/discovery/test_dnsdumpster.py +++ b/tests/discovery/test_dnsdumpster.py @@ -140,3 +140,6 @@ async def test_fetch_exception_is_transport_failure(monkeypatch: pytest.MonkeyPa assert search.execution_status == 'failed' assert search.stop_reason == 'transport-error' + + +pytestmark = pytest.mark.provider_contract('dnsdumpster') diff --git a/tests/discovery/test_duckduckgo.py b/tests/discovery/test_duckduckgo.py index b7d7af29..d48588d3 100644 --- a/tests/discovery/test_duckduckgo.py +++ b/tests/discovery/test_duckduckgo.py @@ -59,3 +59,6 @@ async def test_duckduckgo_unusable_response_returns_no_evidence( assert await search.get_hostnames() == [] assert await search.get_emails() == set() + + +pytestmark = pytest.mark.provider_contract('duckduckgo') diff --git a/tests/discovery/test_dymosearch.py b/tests/discovery/test_dymosearch.py index 5a86f3fc..3a2e3cce 100644 --- a/tests/discovery/test_dymosearch.py +++ b/tests/discovery/test_dymosearch.py @@ -1,145 +1,104 @@ +import asyncio +from typing import Any + import pytest from theHarvester.discovery import dymosearch from theHarvester.discovery.constants import MissingKey +from theHarvester.lib.core import FetcherResponse -def _patch_dymo_key(monkeypatch, value): - import theHarvester.lib.core as core_module +@pytest.mark.provider_contract('dymo') +@pytest.mark.asyncio +async def test_process_extracts_scoped_canonical_and_suggested_domains(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(dymosearch.Core, 'dymo_key', lambda: 'token-xyz') + captured: dict[str, Any] = {} - monkeypatch.setattr(core_module.Core, 'dymo_key', staticmethod(lambda: value), raising=True) - monkeypatch.setattr(core_module.Core, 'get_user_agent', staticmethod(lambda: 'UA'), raising=True) + async def fake_post_fetch(url: str, **kwargs: Any) -> FetcherResponse: + captured.update({'url': url, **kwargs}) + return FetcherResponse( + { + 'domain': {'domain': 'example.com', 'didYouMean': 'www.example.com'}, + 'url': {'domain': 'outside.test', 'didYouMean': 'notexample.com'}, + }, + 200, + {}, + ) + + monkeypatch.setattr(dymosearch.AsyncFetcher, 'post_fetch', fake_post_fetch) + search = dymosearch.SearchDymo('example.com') + await search.process(proxy=True) + + assert await search.get_hostnames() == {'example.com', 'www.example.com'} + assert (await search.get_results())['domain']['domain'] == 'example.com' + assert captured['url'] == dymosearch.SearchDymo.VERIFY_URL + assert captured['json_body'] == {'domain': 'example.com', 'url': 'https://example.com'} + assert captured['include_metadata'] is True + assert captured['proxy'] is True + assert search.execution_status == 'completed' + assert search.stop_reason is None -class TestDymoSearch: - def test_missing_key_raises(self, monkeypatch): - _patch_dymo_key(monkeypatch, None) - with pytest.raises(MissingKey): - dymosearch.SearchDymo('example.com') - - def test_init_sets_state(self, monkeypatch): - _patch_dymo_key(monkeypatch, 'token-123') - search = dymosearch.SearchDymo('example.com') - assert search.word == 'example.com' - assert search.key == 'token-123' - assert search.proxy is False - assert search.totalhosts == set() - assert search.results == {} - - def test_headers_use_bearer(self, monkeypatch): - _patch_dymo_key(monkeypatch, 'token-abc') - search = dymosearch.SearchDymo('example.com') - headers = search._headers() - assert headers['Authorization'] == 'Bearer token-abc' - assert headers['Content-Type'] == 'application/json' - assert headers['User-Agent'] == 'UA' - - @pytest.mark.asyncio - async def test_process_extracts_canonical_and_suggestion(self, monkeypatch): - _patch_dymo_key(monkeypatch, 'token-xyz') - captured = {} - - async def fake_post_fetch(url, headers=None, data='', params='', json=False, proxy=False): - captured['url'] = url - captured['headers'] = headers - captured['data'] = data - captured['proxy'] = proxy - return { - 'domain': { - 'valid': True, - 'fraud': False, - 'freeSubdomain': False, - 'domain': 'exemple.com', - 'didYouMean': 'www.exemple.com', - }, - 'url': { - 'valid': True, - 'domain': 'exemple.com', - 'didYouMean': None, - }, - } - - import theHarvester.lib.core as core_module - - monkeypatch.setattr(core_module.AsyncFetcher, 'post_fetch', classmethod(lambda cls, *a, **kw: fake_post_fetch(*a, **kw))) - - search = dymosearch.SearchDymo('exemple.com') - await search.process(proxy=True) - - assert captured['url'] == dymosearch.SearchDymo.VERIFY_URL - assert captured['proxy'] is True - assert captured['data'] == {'domain': 'exemple.com', 'url': 'https://exemple.com'} - assert captured['headers']['Authorization'] == 'Bearer token-xyz' - - hosts = await search.get_hostnames() - results = await search.get_results() - - assert 'exemple.com' in hosts - assert 'www.exemple.com' in hosts - assert results['domain']['valid'] is True - - @pytest.mark.asyncio - async def test_process_handles_empty_payload(self, monkeypatch): - _patch_dymo_key(monkeypatch, 'token') - - async def fake_post_fetch(url, headers=None, data='', params='', json=False, proxy=False): - return {} - - import theHarvester.lib.core as core_module - - monkeypatch.setattr(core_module.AsyncFetcher, 'post_fetch', classmethod(lambda cls, *a, **kw: fake_post_fetch(*a, **kw))) - - search = dymosearch.SearchDymo('example.com') - await search.process() - assert await search.get_hostnames() == set() - assert await search.get_results() == {} - - @pytest.mark.asyncio - async def test_process_ignores_unrelated_suggestion(self, monkeypatch): - _patch_dymo_key(monkeypatch, 'token') - - async def fake_post_fetch(url, headers=None, data='', params='', json=False, proxy=False): - return { - 'domain': { - 'valid': True, - 'domain': 'totally-different.org', - 'didYouMean': 'somewhere-else.net', - }, - } - - import theHarvester.lib.core as core_module - - monkeypatch.setattr(core_module.AsyncFetcher, 'post_fetch', classmethod(lambda cls, *a, **kw: fake_post_fetch(*a, **kw))) - - search = dymosearch.SearchDymo('example.com') - await search.process() - # Neither contains 'example.com', so neither should be added. - assert await search.get_hostnames() == set() - - @pytest.mark.asyncio - async def test_process_handles_non_dict_response(self, monkeypatch): - _patch_dymo_key(monkeypatch, 'token') - - async def fake_post_fetch(url, headers=None, data='', params='', json=False, proxy=False): - return 'error' - - import theHarvester.lib.core as core_module - - monkeypatch.setattr(core_module.AsyncFetcher, 'post_fetch', classmethod(lambda cls, *a, **kw: fake_post_fetch(*a, **kw))) - - search = dymosearch.SearchDymo('example.com') - await search.process() - assert await search.get_hostnames() == set() - assert search.results == {} +@pytest.mark.parametrize('key', [None, '', ' ']) +def test_missing_or_blank_key_fails_closed(monkeypatch: pytest.MonkeyPatch, key: str | None) -> None: + monkeypatch.setattr(dymosearch.Core, 'dymo_key', lambda: key) + with pytest.raises(MissingKey): + dymosearch.SearchDymo('example.com') -class TestDymoIntegration: - def test_module_exposes_class(self, monkeypatch): - from theHarvester.discovery import dymosearch as mod +@pytest.mark.parametrize( + ('response', 'status', 'reason'), + [ + (None, 'failed', 'transport-error'), + (FetcherResponse({}, 401, {}), 'failed', 'access-denied'), + (FetcherResponse({}, 429, {}), 'rate-limited', 'http-429'), + (FetcherResponse({}, 503, {}), 'failed', 'http-503'), + (FetcherResponse([], 200, {}), 'failed', 'invalid-response'), + (FetcherResponse({'domain': []}, 200, {}), 'failed', 'invalid-response'), + ], +) +@pytest.mark.asyncio +async def test_failures_are_structured( + monkeypatch: pytest.MonkeyPatch, + response: FetcherResponse | None, + status: str, + reason: str, +) -> None: + monkeypatch.setattr(dymosearch.Core, 'dymo_key', lambda: 'token') - assert hasattr(mod, 'SearchDymo') + async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse | None: + return response - def test_source_catalog_lists_dymo(self): - from theHarvester.lib.source_catalog import SOURCE_SPECS + monkeypatch.setattr(dymosearch.AsyncFetcher, 'post_fetch', fake_post_fetch) + search = dymosearch.SearchDymo('example.com') + await search.process() - assert 'dymo' in SOURCE_SPECS + assert search.execution_status == status + assert search.stop_reason == reason + + +@pytest.mark.asyncio +async def test_empty_object_is_completed_without_evidence(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(dymosearch.Core, 'dymo_key', lambda: 'token') + + async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse: + return FetcherResponse({}, 200, {}) + + monkeypatch.setattr(dymosearch.AsyncFetcher, 'post_fetch', fake_post_fetch) + search = dymosearch.SearchDymo('example.com') + await search.process() + + assert search.execution_status == 'completed' + assert search.stop_reason == 'no-results' + + +@pytest.mark.asyncio +async def test_cancellation_propagates(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(dymosearch.Core, 'dymo_key', lambda: 'token') + + async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse: + raise asyncio.CancelledError + + monkeypatch.setattr(dymosearch.AsyncFetcher, 'post_fetch', fake_post_fetch) + with pytest.raises(asyncio.CancelledError): + await dymosearch.SearchDymo('example.com').process() diff --git a/tests/discovery/test_fofa.py b/tests/discovery/test_fofa.py index cbf6a709..9c399066 100644 --- a/tests/discovery/test_fofa.py +++ b/tests/discovery/test_fofa.py @@ -1,4 +1,7 @@ -import logging +import asyncio +import base64 +import contextlib +from collections.abc import AsyncIterator from typing import Any import pytest @@ -8,131 +11,194 @@ from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import FetcherResponse +@pytest.mark.provider_contract('fofa') @pytest.mark.asyncio -async def test_http_failure_is_reported_without_results( - monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, -) -> None: +async def test_process_uses_cursor_api_to_limit_and_retains_scoped_results(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: ('test-key', 'operator@example.com')) + session = object() + session_exited = False + calls: list[dict[str, Any]] = [] + responses = [ + FetcherResponse( + {'error': False, 'results': [['https://API.Example.COM:443', '192.0.2.10']], 'next': 'cursor-2'}, + 200, + {}, + ), + FetcherResponse( + { + 'error': False, + 'results': [['https://outside.test', 'not-an-ip'], ['mail.example.com', '2001:db8::10']], + }, + 200, + {}, + ), + ] - async def fake_fetch_all(urls: list[str], **kwargs: Any) -> list[FetcherResponse]: - assert len(urls) == 1 - assert urls[0].startswith('https://fofa.info/api/v1/search/all?') - assert 'key=test-key' in urls[0] - assert kwargs['json'] is True - assert kwargs['include_metadata'] is True - return [FetcherResponse(body={'error': True}, status=429, headers={})] + @contextlib.asynccontextmanager + async def fake_open_session(**kwargs: Any) -> AsyncIterator[object]: + nonlocal session_exited + assert kwargs['proxy'] is True + try: + yield session + finally: + session_exited = True - monkeypatch.setattr(fofa.AsyncFetcher, 'fetch_all', fake_fetch_all) - search = fofa.SearchFofa('example.com') + async def fake_fetch(**kwargs: Any) -> FetcherResponse: + calls.append(kwargs) + return responses.pop(0) - with caplog.at_level(logging.INFO, logger=fofa.__name__): - await search.process() + monkeypatch.setattr(fofa.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(fofa.AsyncFetcher, 'fetch', fake_fetch) + search = fofa.SearchFofa('example.com', limit=3) - assert await search.get_hostnames() == set() - assert await search.get_ips() == set() - assert 'Fofa request failed with HTTP 429' in caplog.text + await search.process(proxy=True) - -@pytest.mark.asyncio -@pytest.mark.parametrize('error_message', ['Invalid credentials', '账号无效']) -async def test_provider_body_authentication_failure_is_actionable( - monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, - error_message: str, -) -> None: - monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: ('test-key', 'operator@example.com')) - - async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: - return [FetcherResponse(body={'error': True, 'errmsg': error_message}, status=200, headers={})] - - monkeypatch.setattr(fofa.AsyncFetcher, 'fetch_all', fake_fetch_all) - search = fofa.SearchFofa('example.com') - - with caplog.at_level(logging.INFO, logger=fofa.__name__): - await search.process() - - assert 'Fofa API rejected the configured credentials' in caplog.text - - -@pytest.mark.asyncio -async def test_provider_body_quota_failure_is_actionable( - monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, -) -> None: - monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: ('test-key', 'operator@example.com')) - - async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: - return [FetcherResponse(body={'error': True, 'errmsg': 'Query quota exhausted'}, status=200, headers={})] - - monkeypatch.setattr(fofa.AsyncFetcher, 'fetch_all', fake_fetch_all) - search = fofa.SearchFofa('example.com') - - with caplog.at_level(logging.INFO, logger=fofa.__name__): - await search.process() - - assert 'Fofa API quota or plan limit was reached' in caplog.text + assert await search.get_hostnames() == {'api.example.com', 'mail.example.com'} + assert await search.get_ips() == {'192.0.2.10', '2001:db8::10'} + assert search.execution_status == 'partial' + assert search.stop_reason == 'invalid-response' + assert [call['params']['size'] for call in calls] == [3, 2] + assert [call['params'].get('next') for call in calls] == [None, 'cursor-2'] + assert all(call['url'] == 'https://fofa.info/api/v1/search/next' for call in calls) + assert all(call['session'] is session for call in calls) + assert session_exited is True + assert base64.b64decode(calls[0]['params']['qbase64']).decode() == 'domain="example.com"' @pytest.mark.parametrize( 'credentials', - [('', 'operator@example.com'), ('test-key', ' ')], + [(None, 'operator@example.com'), ('', 'operator@example.com'), ('test-key', ' ')], ) -def test_empty_credentials_are_rejected( +def test_missing_or_blank_credentials_fail_closed( monkeypatch: pytest.MonkeyPatch, - credentials: tuple[str, str], + credentials: tuple[str | None, str], ) -> None: monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: credentials) - with pytest.raises(MissingKey): - fofa.SearchFofa('example.com') + fofa.SearchFofa('example.com', 10) +@pytest.mark.parametrize('limit', [0, -1, True, 1.5]) +def test_limit_must_be_positive(monkeypatch: pytest.MonkeyPatch, limit: Any) -> None: + monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: ('test-key', 'operator@example.com')) + with pytest.raises(ValueError, match='positive integer'): + fofa.SearchFofa('example.com', limit) + + +@pytest.mark.parametrize( + ('response', 'status', 'reason'), + [ + (None, 'failed', 'transport-error'), + (FetcherResponse({}, 401, {}), 'failed', 'access-denied'), + (FetcherResponse({}, 429, {}), 'rate-limited', 'http-429'), + (FetcherResponse({}, 503, {}), 'failed', 'http-503'), + (FetcherResponse([], 200, {}), 'failed', 'invalid-response'), + (FetcherResponse({'error': True, 'errmsg': 'Invalid credentials'}, 200, {}), 'failed', 'access-denied'), + (FetcherResponse({'error': True, 'errmsg': 'Quota limit reached'}, 200, {}), 'failed', 'quota-exhausted'), + (FetcherResponse({'error': False, 'results': {}}, 200, {}), 'failed', 'invalid-response'), + ], +) @pytest.mark.asyncio -async def test_malformed_results_are_reported( +async def test_failures_are_structured( monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, + response: FetcherResponse | None, + status: str, + reason: str, ) -> None: monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: ('test-key', 'operator@example.com')) - async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: - return [FetcherResponse(body={'error': False, 'results': 7}, status=200, headers={})] + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + yield object() - monkeypatch.setattr(fofa.AsyncFetcher, 'fetch_all', fake_fetch_all) - search = fofa.SearchFofa('example.com') + async def fake_fetch(**_kwargs: Any) -> FetcherResponse | None: + return response - with caplog.at_level(logging.INFO, logger=fofa.__name__): - await search.process() + monkeypatch.setattr(fofa.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(fofa.AsyncFetcher, 'fetch', fake_fetch) + search = fofa.SearchFofa('example.com', 10) + await search.process() - assert await search.get_hostnames() == set() - assert await search.get_ips() == set() - assert 'Fofa returned malformed results' in caplog.text + assert search.execution_status == status + assert search.stop_reason == reason @pytest.mark.asyncio -async def test_success_preserves_scoped_hosts_and_valid_ips(monkeypatch: pytest.MonkeyPatch) -> None: +async def test_repeated_cursor_stops_without_a_third_request(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: ('test-key', 'operator@example.com')) + responses = [ + FetcherResponse({'error': False, 'results': [['api.example.com', '192.0.2.1']], 'next': 'same'}, 200, {}), + FetcherResponse({'error': False, 'results': [['mail.example.com', '192.0.2.2']], 'next': 'same'}, 200, {}), + ] - async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: - return [ - FetcherResponse( - body={ - 'error': False, - 'results': [ - ['https://API.Example.COM:443', '192.0.2.10'], - ['https://outside.test', 'not-an-ip'], - ['malformed'], - ], - }, - status=200, - headers={}, - ) - ] + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + yield object() - monkeypatch.setattr(fofa.AsyncFetcher, 'fetch_all', fake_fetch_all) - search = fofa.SearchFofa('example.com') + async def fake_fetch(**_kwargs: Any) -> FetcherResponse: + return responses.pop(0) + monkeypatch.setattr(fofa.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(fofa.AsyncFetcher, 'fetch', fake_fetch) + search = fofa.SearchFofa('example.com', 10) + await search.process() + + assert search.execution_status == 'partial' + assert search.stop_reason == 'repeated-cursor' + assert responses == [] + + +@pytest.mark.asyncio +async def test_malformed_url_does_not_discard_later_valid_rows(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: ('test-key', 'operator@example.com')) + response = FetcherResponse( + { + 'error': False, + 'results': [ + ['https://[invalid', 'not-an-ip'], + ['https://api.example.com', '192.0.2.10'], + ], + }, + 200, + {}, + ) + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + yield object() + + async def fake_fetch(**_kwargs: Any) -> FetcherResponse: + return response + + monkeypatch.setattr(fofa.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(fofa.AsyncFetcher, 'fetch', fake_fetch) + search = fofa.SearchFofa('example.com', 2) await search.process() assert await search.get_hostnames() == {'api.example.com'} assert await search.get_ips() == {'192.0.2.10'} + assert search.execution_status == 'partial' + assert search.stop_reason == 'invalid-response' + + +@pytest.mark.asyncio +async def test_cancellation_propagates(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(fofa.Core, 'fofa_key', lambda: ('test-key', 'operator@example.com')) + session_exited = False + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + nonlocal session_exited + try: + yield object() + finally: + session_exited = True + + async def fake_fetch(**_kwargs: Any) -> FetcherResponse: + raise asyncio.CancelledError + + monkeypatch.setattr(fofa.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(fofa.AsyncFetcher, 'fetch', fake_fetch) + with pytest.raises(asyncio.CancelledError): + await fofa.SearchFofa('example.com', 10).process() + assert session_exited is True diff --git a/tests/discovery/test_fullhuntsearch.py b/tests/discovery/test_fullhuntsearch.py index 85ddc10f..d0570b64 100644 --- a/tests/discovery/test_fullhuntsearch.py +++ b/tests/discovery/test_fullhuntsearch.py @@ -1,4 +1,7 @@ +import asyncio +import contextlib import logging +from collections.abc import AsyncIterator from typing import Any import pytest @@ -8,6 +11,55 @@ from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import FetcherResponse +@pytest.mark.asyncio +async def test_process_reuses_one_session_for_fallback_requests(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(fullhuntsearch.Core, 'fullhunt_key', lambda: 'test-key') + session = object() + session_exited = False + open_calls: list[dict[str, Any]] = [] + calls: list[tuple[list[str], dict[str, Any]]] = [] + responses = [ + FetcherResponse({'hosts': []}, 200, {}), + FetcherResponse({'hosts': ['api.example.com']}, 200, {}), + ] + + @contextlib.asynccontextmanager + async def fake_open_session(**kwargs: Any) -> AsyncIterator[object]: + nonlocal session_exited + open_calls.append(kwargs) + try: + yield session + finally: + session_exited = True + + async def fake_fetch_all(urls: list[str], **kwargs: Any) -> list[FetcherResponse]: + calls.append((urls, kwargs)) + return [responses.pop(0)] + + monkeypatch.setattr(fullhuntsearch.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(fullhuntsearch.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = fullhuntsearch.SearchFullHunt('example.com') + + await search.process(proxy=True) + + assert await search.get_hostnames() == ['api.example.com'] + assert [urls for urls, _kwargs in calls] == [ + ['https://fullhunt.io/api/v1/domain/example.com/details'], + ['https://fullhunt.io/api/v1/domain/example.com/subdomains'], + ] + assert all(kwargs['session'] is session for _urls, kwargs in calls) + assert open_calls == [ + { + 'headers': {'User-Agent': fullhuntsearch.Core.get_user_agent(), 'X-API-KEY': 'test-key'}, + 'proxy': True, + 'request_timeout': 60, + } + ] + assert session_exited is True + assert search.execution_status == 'completed' + assert search.stop_reason is None + + @pytest.mark.asyncio async def test_http_failure_is_reported_without_results( monkeypatch: pytest.MonkeyPatch, @@ -31,7 +83,8 @@ async def test_http_failure_is_reported_without_results( assert await search.get_hostnames() == [] assert await search.get_ips() == [] assert requests == ['https://fullhunt.io/api/v1/domain/example.com/details'] - assert 'FullHunt request failed with HTTP 403' in caplog.text + assert search.execution_status == 'failed' + assert search.stop_reason == 'access-denied' @pytest.mark.parametrize('key', ['', ' ']) @@ -72,7 +125,8 @@ async def test_malformed_domain_details_are_reported_without_fallback( assert await search.get_hostnames() == [] assert requests == ['https://fullhunt.io/api/v1/domain/example.com/details'] - assert 'FullHunt returned malformed domain details' in caplog.text + assert search.execution_status == 'failed' + assert search.stop_reason == 'invalid-response' @pytest.mark.asyncio @@ -108,6 +162,8 @@ async def test_malformed_host_does_not_hide_later_valid_results( assert await search.get_hostnames() == ['api.example.com'] assert await search.get_ips() == ['192.0.2.20'] assert caplog.text.count('FullHunt ignored a malformed host item') == 4 + assert search.execution_status == 'partial' + assert search.stop_reason == 'invalid-response' @pytest.mark.asyncio @@ -131,7 +187,8 @@ async def test_malformed_subdomain_fallback_is_reported( await search.process() assert await search.get_hostnames() == [] - assert 'FullHunt returned malformed subdomains' in caplog.text + assert search.execution_status == 'failed' + assert search.stop_reason == 'invalid-response' @pytest.mark.asyncio @@ -156,6 +213,8 @@ async def test_fallback_ignores_malformed_and_out_of_scope_hosts( assert await search.get_hostnames() == ['api.example.com'] assert caplog.text.count('FullHunt ignored a malformed subdomain item') == 2 + assert search.execution_status == 'partial' + assert search.stop_reason == 'invalid-response' @pytest.mark.asyncio @@ -192,3 +251,60 @@ async def test_nested_results_use_normalized_hostname(monkeypatch: pytest.Monkey assert await search.get_geo_info() == {'api.example.com': {'country': 'US'}} assert await search.get_cloud_info() == {'api.example.com': {'provider': 'example'}} assert await search.get_certificate_info() == [{'issuer': 'Example CA', 'hostname': 'api.example.com'}] + assert search.execution_status == 'completed' + assert search.stop_reason is None + + +@pytest.mark.parametrize( + ('response', 'status', 'reason'), + [ + (None, 'failed', 'transport-error'), + (FetcherResponse({}, 429, {}), 'rate-limited', 'http-429'), + (FetcherResponse({}, 503, {}), 'failed', 'http-503'), + (FetcherResponse([], 200, {}), 'failed', 'invalid-response'), + ], +) +@pytest.mark.asyncio +async def test_failures_are_structured( + monkeypatch: pytest.MonkeyPatch, + response: FetcherResponse | None, + status: str, + reason: str, +) -> None: + monkeypatch.setattr(fullhuntsearch.Core, 'fullhunt_key', lambda: 'test-key') + + async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse | None]: + return [response] + + monkeypatch.setattr(fullhuntsearch.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = fullhuntsearch.SearchFullHunt('example.com') + await search.process() + + assert search.execution_status == status + assert search.stop_reason == reason + + +@pytest.mark.asyncio +async def test_cancellation_propagates(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(fullhuntsearch.Core, 'fullhunt_key', lambda: 'test-key') + session_exited = False + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + nonlocal session_exited + try: + yield object() + finally: + session_exited = True + + async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: + raise asyncio.CancelledError + + monkeypatch.setattr(fullhuntsearch.AsyncFetcher, 'fetch_all', fake_fetch_all) + monkeypatch.setattr(fullhuntsearch.AsyncFetcher, 'open_session', fake_open_session) + with pytest.raises(asyncio.CancelledError): + await fullhuntsearch.SearchFullHunt('example.com').process() + assert session_exited is True + + +pytestmark = pytest.mark.provider_contract('fullhunt') diff --git a/tests/discovery/test_githubcode_contract.py b/tests/discovery/test_githubcode_contract.py index 1e2e55e4..61a3a8f7 100644 --- a/tests/discovery/test_githubcode_contract.py +++ b/tests/discovery/test_githubcode_contract.py @@ -206,3 +206,6 @@ async def test_github_code_malformed_page_terminates_without_following_paginatio assert requested_urls == ['https://api.github.com/search/code?q="example.com"&page=1'] assert await search.get_emails() == set() assert await search.get_hostnames() == [] + + +pytestmark = pytest.mark.provider_contract('github-code') diff --git a/tests/discovery/test_gitlabsearch.py b/tests/discovery/test_gitlabsearch.py index 4d75c3cb..af4bda03 100644 --- a/tests/discovery/test_gitlabsearch.py +++ b/tests/discovery/test_gitlabsearch.py @@ -174,3 +174,6 @@ async def test_gitlab_urls_reach_completed_jsonl( assert completed_results[0].results == (('url', 'https://gitlab.com/group/project'),) records = [json.loads(line) for line in report.with_suffix('.jsonl').read_text().splitlines()] assert {'type': 'url', 'value': 'https://gitlab.com/group/project', 'sources': ['gitlab']} in records + + +pytestmark = pytest.mark.provider_contract('gitlab') diff --git a/tests/discovery/test_haveibeenpwned.py b/tests/discovery/test_haveibeenpwned.py index 0ba4b55a..b9c38eb6 100644 --- a/tests/discovery/test_haveibeenpwned.py +++ b/tests/discovery/test_haveibeenpwned.py @@ -211,3 +211,6 @@ async def test_public_breach_names_reach_completed_result_and_jsonl( records = [json.loads(line) for line in report.with_suffix('.jsonl').read_text().splitlines()] assert {'type': 'breach', 'value': 'Adobe', 'sources': ['haveibeenpwned']} in records assert {'type': 'breach', 'value': 'ExampleBreach', 'sources': ['haveibeenpwned']} in records + + +pytestmark = pytest.mark.provider_contract('haveibeenpwned') diff --git a/tests/discovery/test_hibpverified.py b/tests/discovery/test_hibpverified.py index da209efa..e21aa2d8 100644 --- a/tests/discovery/test_hibpverified.py +++ b/tests/discovery/test_hibpverified.py @@ -190,3 +190,6 @@ async def test_verified_domain_results_reach_completed_jsonl_and_sqlite_handoff( records = [json.loads(line) for line in report.with_suffix('.jsonl').read_text().splitlines()] assert {'type': 'breach', 'value': 'ExampleBreach', 'sources': ['hibpverified']} in records assert {'type': 'email', 'value': 'alice@example.com', 'sources': ['hibpverified']} in records + + +pytestmark = pytest.mark.provider_contract('hibpverified') diff --git a/tests/discovery/test_hudsonrock.py b/tests/discovery/test_hudsonrock.py index e69e7fdc..21139055 100644 --- a/tests/discovery/test_hudsonrock.py +++ b/tests/discovery/test_hudsonrock.py @@ -259,3 +259,6 @@ async def test_infostealer_data_reaches_completed_result_and_jsonl( assert ('infostealer', stealer) in completed_results[0].results records = [json.loads(line) for line in report.with_suffix('.jsonl').read_text().splitlines()] assert {'type': 'infostealer', 'value': stealer, 'sources': ['hudsonrock']} in records + + +pytestmark = pytest.mark.provider_contract('hudsonrock') diff --git a/tests/discovery/test_hunterhow.py b/tests/discovery/test_hunterhow.py new file mode 100644 index 00000000..8afd84c4 --- /dev/null +++ b/tests/discovery/test_hunterhow.py @@ -0,0 +1,185 @@ +import asyncio +import base64 +import contextlib +from collections.abc import AsyncIterator +from typing import Any + +import pytest + +from theHarvester.discovery import searchhunterhow +from theHarvester.discovery.constants import MissingKey +from theHarvester.lib.core import FetcherResponse + + +@pytest.mark.provider_contract('hunterhow') +@pytest.mark.asyncio +async def test_process_paginates_to_limit_and_keeps_scoped_hostnames(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(searchhunterhow.Core, 'hunterhow_key', staticmethod(lambda: 'test-key')) + session = object() + session_exited = False + session_options: list[dict[str, Any]] = [] + calls: list[dict[str, Any]] = [] + delays: list[float] = [] + responses = [ + FetcherResponse( + body={ + 'code': 200, + 'data': { + 'total': 3, + 'list': [{'domain': 'API.Example.COM.'}, {'domain': 'outside.test'}], + }, + }, + status=200, + headers={}, + ), + FetcherResponse( + body={'code': 200, 'data': {'total': 3, 'list': [{'domain': 'www.example.com'}]}}, + status=200, + headers={}, + ), + ] + + @contextlib.asynccontextmanager + async def fake_open_session(**kwargs: Any) -> AsyncIterator[object]: + nonlocal session_exited + session_options.append(kwargs) + try: + yield session + finally: + session_exited = True + + async def fake_fetch(**kwargs: Any) -> FetcherResponse: + calls.append(kwargs) + return responses.pop(0) + + async def fake_sleep(delay: float) -> None: + delays.append(delay) + + monkeypatch.setattr(searchhunterhow.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(searchhunterhow.AsyncFetcher, 'fetch', fake_fetch) + monkeypatch.setattr(searchhunterhow.asyncio, 'sleep', fake_sleep) + search = searchhunterhow.SearchHunterHow('example.com', limit=3) + + await search.process(proxy=True) + + assert await search.get_hostnames() == {'api.example.com', 'www.example.com'} + assert session_options == [{'headers': {'User-Agent': searchhunterhow.Core.get_user_agent()}, 'proxy': True}] + assert [call['params']['page'] for call in calls] == [1, 2] + assert all(call['params']['page_size'] == 10 for call in calls) + assert all(call['params']['api-key'] == 'test-key' for call in calls) + assert all(base64.urlsafe_b64decode(call['params']['query']).decode() == 'domain.suffix="example.com"' for call in calls) + assert all(call['session'] is session for call in calls) + assert all(call['url'] == 'https://api.hunter.how/search' for call in calls) + assert all(call['include_metadata'] is True for call in calls) + assert delays == [2.0] + assert search.execution_status == 'completed' + assert search.stop_reason is None + assert session_exited is True + + +@pytest.mark.parametrize('key', [None, '', ' ']) +def test_missing_or_empty_key_fails_before_transport(monkeypatch: pytest.MonkeyPatch, key: str | None) -> None: + monkeypatch.setattr(searchhunterhow.Core, 'hunterhow_key', staticmethod(lambda: key)) + + with pytest.raises(MissingKey): + searchhunterhow.SearchHunterHow('example.com', limit=10) + + +@pytest.mark.parametrize( + ('response', 'execution_status', 'stop_reason'), + [ + (None, 'failed', 'transport-error'), + (FetcherResponse({}, 401, {}), 'failed', 'access-denied'), + (FetcherResponse({}, 403, {}), 'failed', 'access-denied'), + (FetcherResponse({}, 429, {}), 'rate-limited', 'http-429'), + (FetcherResponse({}, 503, {}), 'failed', 'http-503'), + (FetcherResponse({'code': 40001}, 200, {}), 'failed', 'access-denied'), + (FetcherResponse({'code': 200, 'data': {'total': 'three', 'list': {}}}, 200, {}), 'failed', 'invalid-response'), + ], +) +@pytest.mark.asyncio +async def test_failed_response_is_attributed( + monkeypatch: pytest.MonkeyPatch, + response: FetcherResponse | None, + execution_status: str, + stop_reason: str, +) -> None: + monkeypatch.setattr(searchhunterhow.Core, 'hunterhow_key', staticmethod(lambda: 'test-key')) + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + yield object() + + async def fake_fetch(**_kwargs: Any) -> FetcherResponse | None: + return response + + monkeypatch.setattr(searchhunterhow.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(searchhunterhow.AsyncFetcher, 'fetch', fake_fetch) + search = searchhunterhow.SearchHunterHow('example.com', limit=10) + + await search.process() + + assert await search.get_hostnames() == set() + assert search.execution_status == execution_status + assert search.stop_reason == stop_reason + + +def test_limit_must_be_positive(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(searchhunterhow.Core, 'hunterhow_key', staticmethod(lambda: 'test-key')) + + with pytest.raises(ValueError, match='positive integer'): + searchhunterhow.SearchHunterHow('example.com', limit=0) + + +@pytest.mark.asyncio +async def test_early_malformed_rows_and_later_valid_evidence_are_partial(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(searchhunterhow.Core, 'hunterhow_key', staticmethod(lambda: 'test-key')) + responses = [ + FetcherResponse({'code': 200, 'data': {'total': 2, 'list': [7]}}, 200, {}), + FetcherResponse({'code': 200, 'data': {'total': 2, 'list': [{'domain': 'api.example.com'}]}}, 200, {}), + ] + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + yield object() + + async def fake_fetch(**_kwargs: Any) -> FetcherResponse: + return responses.pop(0) + + async def fake_sleep(_delay: float) -> None: + return None + + monkeypatch.setattr(searchhunterhow.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(searchhunterhow.AsyncFetcher, 'fetch', fake_fetch) + monkeypatch.setattr(searchhunterhow.asyncio, 'sleep', fake_sleep) + search = searchhunterhow.SearchHunterHow('example.com', limit=2) + + await search.process() + + assert await search.get_hostnames() == {'api.example.com'} + assert search.execution_status == 'partial' + assert search.stop_reason == 'invalid-response' + + +@pytest.mark.asyncio +async def test_cancellation_closes_provider_session(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(searchhunterhow.Core, 'hunterhow_key', staticmethod(lambda: 'test-key')) + session_exited = False + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + nonlocal session_exited + try: + yield object() + finally: + session_exited = True + + async def fake_fetch(**_kwargs: Any) -> FetcherResponse: + raise asyncio.CancelledError('operator-stop') + + monkeypatch.setattr(searchhunterhow.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(searchhunterhow.AsyncFetcher, 'fetch', fake_fetch) + + with pytest.raises(asyncio.CancelledError, match='operator-stop'): + await searchhunterhow.SearchHunterHow('example.com', limit=10).process() + assert session_exited is True diff --git a/tests/discovery/test_huntersearch.py b/tests/discovery/test_huntersearch.py index 7e0d5ffd..7c567295 100644 --- a/tests/discovery/test_huntersearch.py +++ b/tests/discovery/test_huntersearch.py @@ -33,6 +33,8 @@ async def test_hunter_http_failures_return_no_results(monkeypatch, caplog, statu assert await search.get_emails() == [] assert await search.get_hostnames() == [] + + assert f'Hunter request failed with HTTP {status}' in caplog.text assert 'provider detail' not in caplog.text @@ -221,3 +223,6 @@ async def test_paid_hunter_search_stops_before_exceeding_quota(monkeypatch) -> N ] assert await search.get_emails() == [] assert await search.get_hostnames() == [] + + +pytestmark = pytest.mark.provider_contract('hunter') diff --git a/tests/discovery/test_intelxsearch.py b/tests/discovery/test_intelxsearch.py index 902620fe..f45c400a 100644 --- a/tests/discovery/test_intelxsearch.py +++ b/tests/discovery/test_intelxsearch.py @@ -169,3 +169,6 @@ async def test_orchestrator_stores_intelx_subdomains_without_dns(monkeypatch: py assert results[-1] == ['api.example.com'] assert completed_results[0].observations == (ResultObservation('intelx', 'hostname', 'api.example.com'),) + + +pytestmark = pytest.mark.provider_contract('intelx') diff --git a/tests/discovery/test_leakix.py b/tests/discovery/test_leakix.py index 8fda2655..559915a2 100644 --- a/tests/discovery/test_leakix.py +++ b/tests/discovery/test_leakix.py @@ -124,3 +124,6 @@ async def test_unusable_responses_fail_closed_without_logging_provider_detail( assert 'provider-secret-auth-detail' not in caplog.text if expected_log is not None: assert expected_log in caplog.text + + +pytestmark = pytest.mark.provider_contract('leakix') diff --git a/tests/discovery/test_leaklookup.py b/tests/discovery/test_leaklookup.py index a8913081..9b9a9eb7 100644 --- a/tests/discovery/test_leaklookup.py +++ b/tests/discovery/test_leaklookup.py @@ -196,3 +196,6 @@ async def test_leaklookup_emails_and_breaches_reach_completed_result_and_jsonl(m records = [json.loads(line) for line in report.with_suffix('.jsonl').read_text().splitlines()] assert {'type': 'breach', 'value': 'Example Breach', 'sources': ['leaklookup']} in records assert {'type': 'email', 'value': 'alice@example.com', 'sources': ['leaklookup']} in records + + +pytestmark = pytest.mark.provider_contract('leaklookup') diff --git a/tests/discovery/test_netlas.py b/tests/discovery/test_netlas.py new file mode 100644 index 00000000..af9259b7 --- /dev/null +++ b/tests/discovery/test_netlas.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +import asyncio +import contextlib +from typing import TYPE_CHECKING, Any + +import pytest + +from theHarvester.discovery import netlas +from theHarvester.discovery.constants import MissingKey +from theHarvester.lib.core import FetcherResponse + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + +@pytest.mark.provider_contract('netlas') +@pytest.mark.asyncio +async def test_process_uses_current_api_contract_and_keeps_scoped_results(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(netlas.Core, 'netlas_key', staticmethod(lambda: 'test-key')) + session = object() + session_exited = False + session_options: list[dict[str, Any]] = [] + calls: list[dict[str, Any]] = [] + responses = [ + FetcherResponse( + [ + {'data': {'domain': 'API.Example.COM.'}}, + {'data': {'domain': 'outside.test'}}, + {'data': {'domain': 'www.example.com'}}, + ], + 200, + {}, + ), + ] + + @contextlib.asynccontextmanager + async def fake_open_session(**kwargs: Any) -> AsyncIterator[object]: + nonlocal session_exited + session_options.append(kwargs) + try: + yield session + finally: + session_exited = True + + async def fake_post_fetch(*args: Any, **kwargs: Any) -> FetcherResponse: + calls.append({'url': args[0], **kwargs}) + return responses.pop(0) + + monkeypatch.setattr(netlas.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(netlas.AsyncFetcher, 'post_fetch', fake_post_fetch) + + search = netlas.SearchNetlas('example.com', limit=2) + await search.process(proxy=True) + + assert await search.get_hostnames() == {'api.example.com'} + assert search.execution_status == 'completed' + assert search.stop_reason is None + assert session_options == [{'headers': {'Authorization': 'Bearer test-key'}, 'proxy': True}] + assert calls[0]['url'] == 'https://app.netlas.io/api/domains/download/' + assert calls[0]['session'] is session + assert calls[0]['json_body'] == { + 'q': '*.example.com', + 'size': 2, + 'fields': ['domain'], + 'source_type': 'include', + } + assert session_exited is True + + +@pytest.mark.parametrize('key', [None, '', ' ']) +def test_missing_or_blank_key_fails_closed(monkeypatch: pytest.MonkeyPatch, key: str | None) -> None: + monkeypatch.setattr(netlas.Core, 'netlas_key', staticmethod(lambda: key)) + with pytest.raises(MissingKey): + netlas.SearchNetlas('example.com', limit=10) + + +@pytest.mark.parametrize('limit', [0, -1, True, 1.5]) +def test_limit_must_be_a_positive_integer(monkeypatch: pytest.MonkeyPatch, limit: Any) -> None: + monkeypatch.setattr(netlas.Core, 'netlas_key', staticmethod(lambda: 'test-key')) + with pytest.raises(ValueError, match='positive integer'): + netlas.SearchNetlas('example.com', limit=limit) + + +@pytest.mark.parametrize( + ('response', 'status', 'reason'), + [ + (None, 'failed', 'transport-error'), + (FetcherResponse({}, 401, {}), 'failed', 'access-denied'), + (FetcherResponse({}, 402, {}), 'failed', 'quota-exhausted'), + (FetcherResponse({}, 429, {}), 'rate-limited', 'http-429'), + (FetcherResponse({}, 503, {}), 'failed', 'http-503'), + (FetcherResponse(None, 200, {}), 'failed', 'invalid-response'), + (FetcherResponse({'count': 'many'}, 200, {}), 'failed', 'invalid-response'), + ], +) +@pytest.mark.asyncio +async def test_download_failures_are_truthful( + monkeypatch: pytest.MonkeyPatch, + response: FetcherResponse | None, + status: str, + reason: str, +) -> None: + monkeypatch.setattr(netlas.Core, 'netlas_key', staticmethod(lambda: 'test-key')) + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + yield object() + + async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse | None: + return response + + monkeypatch.setattr(netlas.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(netlas.AsyncFetcher, 'post_fetch', fake_post_fetch) + search = netlas.SearchNetlas('example.com', limit=10) + + await search.process() + + assert search.execution_status == status + assert search.stop_reason == reason + + +@pytest.mark.asyncio +async def test_malformed_download_rows_preserve_valid_partial_results(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(netlas.Core, 'netlas_key', staticmethod(lambda: 'test-key')) + responses = [ + FetcherResponse([{'data': {'domain': 'ok.example.com'}}, {'data': {'domain': 7}}], 200, {}), + ] + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + yield object() + + async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse: + return responses.pop(0) + + monkeypatch.setattr(netlas.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(netlas.AsyncFetcher, 'post_fetch', fake_post_fetch) + search = netlas.SearchNetlas('example.com', limit=10) + + await search.process() + + assert await search.get_hostnames() == {'ok.example.com'} + assert search.execution_status == 'partial' + assert search.stop_reason == 'invalid-response' + + +@pytest.mark.asyncio +async def test_cancellation_closes_provider_session(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(netlas.Core, 'netlas_key', staticmethod(lambda: 'test-key')) + session_exited = False + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + nonlocal session_exited + try: + yield object() + finally: + session_exited = True + + async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse: + raise asyncio.CancelledError('operator-stop') + + monkeypatch.setattr(netlas.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(netlas.AsyncFetcher, 'post_fetch', fake_post_fetch) + + with pytest.raises(asyncio.CancelledError, match='operator-stop'): + await netlas.SearchNetlas('example.com', limit=10).process() + assert session_exited is True diff --git a/tests/discovery/test_onyphe.py b/tests/discovery/test_onyphe.py index 5f1b013f..1fda9773 100644 --- a/tests/discovery/test_onyphe.py +++ b/tests/discovery/test_onyphe.py @@ -1,162 +1,259 @@ import asyncio -import logging +import contextlib +from collections.abc import AsyncIterator from typing import Any import pytest from theHarvester.discovery import onyphe +from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import FetcherResponse +@pytest.mark.provider_contract('onyphe') @pytest.mark.asyncio -async def test_process_keeps_only_canonical_individual_ips_and_preserves_routes( - monkeypatch: pytest.MonkeyPatch, -) -> None: +async def test_process_paginates_to_limit_and_preserves_all_routes(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(onyphe.Core, 'onyphe_key', lambda: 'test-key') - monkeypatch.setattr(onyphe.Core, 'get_user_agent', lambda: 'test-agent') - captured: dict[str, Any] = {} + session = object() + session_exited = False + calls: list[dict[str, Any]] = [] + responses = [ + FetcherResponse( + { + 'text': 'Success', + 'total': 3, + 'max_page': 2, + 'results': [ + { + 'ip': '192.0.2.10', + 'alternativeip': ['2001:0db8::10'], + 'url': ['https://www.example.com/path'], + 'asn': 'AS64496', + 'organization': 'Example Physical Network', + 'geolocus': { + 'asn': 'AS64497', + 'organization': 'Example Logical Network', + 'domain': ['geo.example.com'], + }, + 'hostname': ['api.example.com'], + }, + {'hostname': ['outside.test']}, + ], + }, + 200, + {}, + ), + FetcherResponse( + {'text': 'Success', 'total': 3, 'max_page': 2, 'results': [{'subdomains': ['mail.example.com']}]}, + 200, + {}, + ), + ] - async def fake_fetch_all(urls: list[str], **kwargs: Any) -> list[FetcherResponse]: - assert urls == ['https://www.onyphe.io/api/v2/search/?q=domain:example.com'] - captured.update(kwargs) - return [ - FetcherResponse( - body={ - 'text': 'Success', - 'results': [ - { - 'ip': '192.0.2.10', - 'alternativeip': ['2001:0db8::10'], - 'subnet': '192.0.2.0/24', - 'url': ['https://www.example.com/path'], - 'asn': 'AS64496', - 'organization': 'Example Physical Network', - 'geolocus': { - 'asn': 'AS64497', - 'organization': 'Example Logical Network', - 'subnet': '198.51.100.0/24', - 'domain': ['geo.example.com'], - }, - 'hostname': ['api.example.com'], - } - ], - }, - status=200, - headers={}, - ) - ] + @contextlib.asynccontextmanager + async def fake_open_session(**kwargs: Any) -> AsyncIterator[object]: + nonlocal session_exited + assert kwargs['proxy'] is True + assert kwargs['headers']['Authorization'] == 'bearer test-key' + try: + yield session + finally: + session_exited = True - monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch_all', fake_fetch_all) - search = onyphe.SearchOnyphe('example.com') + async def fake_fetch(**kwargs: Any) -> FetcherResponse: + calls.append(kwargs) + return responses.pop(0) + monkeypatch.setattr(onyphe.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch', fake_fetch) + search = onyphe.SearchOnyphe('example.com', limit=3) await search.process(proxy=True) - assert captured['proxy'] is True - assert captured['json'] is True - assert captured['include_metadata'] is True - assert captured['headers']['Authorization'] == 'bearer test-key' assert await search.get_ips() == {'192.0.2.10', '2001:db8::10'} - assert await search.get_hostnames() == {'api.example.com', 'geo.example.com', 'www.example.com'} + assert await search.get_hostnames() == {'api.example.com', 'geo.example.com', 'mail.example.com', 'www.example.com'} assert await search.get_asns() == {'AS64496', 'AS64497'} assert { - ( - observation.asn, - observation.organization_label, - observation.subject_kind, - observation.subject_value, - ) - for observation in await search.get_asn_attributions() + (item.asn, item.organization_label, item.subject_kind, item.subject_value) for item in await search.get_asn_attributions() } == { - (asn, organization, subject_kind, subject_value) - for asn, organization in { - ('AS64496', 'Example Physical Network'), - ('AS64497', 'Example Logical Network'), - } - for subject_kind, subject_value in {('ip', '192.0.2.10')} + ('AS64496', 'Example Physical Network', 'ip', '192.0.2.10'), + ('AS64497', 'Example Logical Network', 'ip', '192.0.2.10'), } + assert [call['params']['page'] for call in calls] == [1, 2] + assert [call['params']['size'] for call in calls] == [3, 3] + assert all(call['session'] is session for call in calls) + assert search.execution_status == 'completed' + assert search.stop_reason is None + assert session_exited is True + + +@pytest.mark.parametrize('key', [None, '', ' ']) +def test_missing_or_blank_key_fails_closed(monkeypatch: pytest.MonkeyPatch, key: str | None) -> None: + monkeypatch.setattr(onyphe.Core, 'onyphe_key', lambda: key) + with pytest.raises(MissingKey): + onyphe.SearchOnyphe('example.com', 10) + + +@pytest.mark.parametrize('limit', [0, -1, True, 1.5]) +def test_limit_must_be_positive(monkeypatch: pytest.MonkeyPatch, limit: Any) -> None: + monkeypatch.setattr(onyphe.Core, 'onyphe_key', lambda: 'test-key') + with pytest.raises(ValueError, match='positive integer'): + onyphe.SearchOnyphe('example.com', limit) + + +@pytest.mark.parametrize( + ('response', 'status', 'reason'), + [ + (None, 'failed', 'transport-error'), + (FetcherResponse({}, 401, {}), 'failed', 'access-denied'), + (FetcherResponse({}, 429, {}), 'rate-limited', 'http-429'), + (FetcherResponse({}, 503, {}), 'failed', 'http-503'), + (FetcherResponse([], 200, {}), 'failed', 'invalid-response'), + (FetcherResponse({'text': 'Denied'}, 200, {}), 'failed', 'provider-error'), + (FetcherResponse({'text': 'Success', 'results': {}}, 200, {}), 'failed', 'invalid-response'), + ], +) +@pytest.mark.asyncio +async def test_failures_are_structured( + monkeypatch: pytest.MonkeyPatch, + response: FetcherResponse | None, + status: str, + reason: str, +) -> None: + monkeypatch.setattr(onyphe.Core, 'onyphe_key', lambda: 'test-key') + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + yield object() + + async def fake_fetch(**_kwargs: Any) -> FetcherResponse | None: + return response + + monkeypatch.setattr(onyphe.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch', fake_fetch) + search = onyphe.SearchOnyphe('example.com', 10) + await search.process() + + assert search.execution_status == status + assert search.stop_reason == reason + + +@pytest.mark.asyncio +async def test_later_page_failure_preserves_partial_evidence(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(onyphe.Core, 'onyphe_key', lambda: 'test-key') + responses = [ + FetcherResponse( + {'text': 'Success', 'total': 2, 'max_page': 2, 'results': [{'ip': '192.0.2.10'}]}, + 200, + {}, + ), + FetcherResponse({}, 429, {}), + ] + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + yield object() + + async def fake_fetch(**_kwargs: Any) -> FetcherResponse: + return responses.pop(0) + + monkeypatch.setattr(onyphe.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch', fake_fetch) + search = onyphe.SearchOnyphe('example.com', 2) + await search.process() + + assert await search.get_ips() == {'192.0.2.10'} + assert search.execution_status == 'partial' + assert search.stop_reason == 'http-429' + + +@pytest.mark.asyncio +async def test_operator_limit_is_not_reported_as_a_provider_limit(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(onyphe.Core, 'onyphe_key', lambda: 'test-key') + response = FetcherResponse( + { + 'text': 'Success', + 'total': 3, + 'max_page': 2, + 'results': [{'ip': '192.0.2.10'}, {'ip': '192.0.2.11'}], + }, + 200, + {}, + ) + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + yield object() + + async def fake_fetch(**_kwargs: Any) -> FetcherResponse: + return response + + monkeypatch.setattr(onyphe.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch', fake_fetch) + search = onyphe.SearchOnyphe('example.com', 2) + await search.process() + + assert await search.get_ips() == {'192.0.2.10', '192.0.2.11'} assert search.execution_status == 'completed' assert search.stop_reason is None @pytest.mark.asyncio -async def test_valid_empty_response_is_completed(monkeypatch: pytest.MonkeyPatch) -> None: +async def test_search_api_reports_its_documented_total_boundary(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(onyphe.Core, 'onyphe_key', lambda: 'test-key') + calls: list[dict[str, Any]] = [] + response = FetcherResponse( + { + 'text': 'Success', + 'total': 10_001, + 'max_page': 2, + 'results': [{'hostname': ['api.example.com']}, *({} for _ in range(9_999))], + }, + 200, + {}, + ) - async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: - return [FetcherResponse(body={'text': 'Success', 'results': []}, status=200, headers={})] + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + yield object() - monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch_all', fake_fetch_all) - search = onyphe.SearchOnyphe('example.com') + async def fake_fetch(**kwargs: Any) -> FetcherResponse: + calls.append(kwargs) + return response + monkeypatch.setattr(onyphe.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch', fake_fetch) + search = onyphe.SearchOnyphe('example.com', 10_001) await search.process() - assert await search.get_ips() == set() - assert await search.get_hostnames() == set() - assert await search.get_asns() == set() - assert search.execution_status == 'completed' - assert search.stop_reason == 'no-results' - - -@pytest.mark.parametrize( - ('response', 'execution_status', 'stop_reason'), - [ - (None, 'failed', 'transport-error'), - (FetcherResponse(body={}, status=401, headers={}), 'failed', 'access-denied'), - (FetcherResponse(body={}, status=403, headers={}), 'failed', 'access-denied'), - (FetcherResponse(body={}, status=429, headers={}), 'rate-limited', 'http-429'), - (FetcherResponse(body={}, status=503, headers={}), 'failed', 'http-503'), - (FetcherResponse(body=['provider-secret-payload'], status=200, headers={}), 'failed', 'invalid-response'), - ( - FetcherResponse(body={'text': 'Success', 'results': {}}, status=200, headers={}), - 'failed', - 'invalid-response', - ), - ], -) -@pytest.mark.asyncio -async def test_failed_responses_are_attributed( - monkeypatch: pytest.MonkeyPatch, - response: FetcherResponse | None, - execution_status: str, - stop_reason: str, -) -> None: - monkeypatch.setattr(onyphe.Core, 'onyphe_key', lambda: 'test-key') - - async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse | None]: - return [response] - - monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch_all', fake_fetch_all) - search = onyphe.SearchOnyphe('example.com') - - await search.process() - - assert await search.get_ips() == set() - assert search.execution_status == execution_status - assert search.stop_reason == stop_reason + assert [call['params']['page'] for call in calls] == [1] + assert [call['params']['size'] for call in calls] == [10_000] + assert await search.get_hostnames() == {'api.example.com'} + assert search.execution_status == 'partial' + assert search.stop_reason == 'provider-limit' @pytest.mark.asyncio async def test_malformed_items_preserve_valid_partial_results(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(onyphe.Core, 'onyphe_key', lambda: 'test-key') + response = FetcherResponse( + { + 'text': 'Success', + 'results': [{'ip': '192.0.2.10', 'alternativeip': ['not-an-ip', None]}, 'malformed-record'], + }, + 200, + {}, + ) - async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: - return [ - FetcherResponse( - body={ - 'text': 'Success', - 'results': [ - {'ip': '192.0.2.10', 'alternativeip': ['not-an-ip', None]}, - 'malformed-record', - ], - }, - status=200, - headers={}, - ) - ] + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + yield object() - monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch_all', fake_fetch_all) - search = onyphe.SearchOnyphe('example.com') + async def fake_fetch(**_kwargs: Any) -> FetcherResponse: + return response + monkeypatch.setattr(onyphe.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch', fake_fetch) + search = onyphe.SearchOnyphe('example.com', 10) await search.process() assert await search.get_ips() == {'192.0.2.10'} @@ -164,85 +261,27 @@ async def test_malformed_items_preserve_valid_partial_results(monkeypatch: pytes assert search.stop_reason == 'invalid-response' -@pytest.mark.asyncio -async def test_malformed_url_preserves_valid_partial_results(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(onyphe.Core, 'onyphe_key', lambda: 'test-key') - - async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: - return [ - FetcherResponse( - body={ - 'text': 'Success', - 'results': [{'ip': '192.0.2.10', 'url': ['http://[malformed']}], - }, - status=200, - headers={}, - ) - ] - - monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch_all', fake_fetch_all) - search = onyphe.SearchOnyphe('example.com') - - await search.process() - - assert await search.get_ips() == {'192.0.2.10'} - assert search.execution_status == 'partial' - assert search.stop_reason == 'invalid-response' - - -@pytest.mark.asyncio -async def test_failed_response_body_is_not_logged(monkeypatch, caplog) -> None: - monkeypatch.setattr(onyphe.Core, 'onyphe_key', lambda: 'test-key') - monkeypatch.setattr(onyphe.Core, 'get_user_agent', lambda: 'test-agent') - - async def fake_fetch_all(*args, **kwargs): - return [ - FetcherResponse( - body={'text': 'Failed', 'secret': 'provider-secret-payload'}, - status=200, - headers={}, - ) - ] - - monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch_all', fake_fetch_all) - caplog.set_level(logging.INFO, logger=onyphe.__name__) - - search = onyphe.SearchOnyphe('example.com') - await search.process() - - assert 'provider-secret-payload' not in caplog.text - assert 'did not succeed' in caplog.text - assert search.execution_status == 'failed' - assert search.stop_reason == 'provider-error' - - -@pytest.mark.asyncio -async def test_unexpected_response_body_is_not_logged(monkeypatch, caplog) -> None: - monkeypatch.setattr(onyphe.Core, 'onyphe_key', lambda: 'test-key') - monkeypatch.setattr(onyphe.Core, 'get_user_agent', lambda: 'test-agent') - - async def fake_fetch_all(*args, **kwargs): - return [FetcherResponse(body='provider-secret-payload', status=200, headers={})] - - monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch_all', fake_fetch_all) - caplog.set_level(logging.INFO, logger=onyphe.__name__) - search = onyphe.SearchOnyphe('example.com') - - await search.process() - - assert 'provider-secret-payload' not in caplog.text - assert search.execution_status == 'failed' - assert search.stop_reason == 'invalid-response' - - @pytest.mark.asyncio async def test_cancellation_propagates(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(onyphe.Core, 'onyphe_key', lambda: 'test-key') + session_exited = False - async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: - raise asyncio.CancelledError + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + nonlocal session_exited + try: + yield object() + finally: + session_exited = True - monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch_all', fake_fetch_all) + monkeypatch.setattr(onyphe.AsyncFetcher, 'open_session', fake_open_session) + cancellation = asyncio.CancelledError('operator-stop') - with pytest.raises(asyncio.CancelledError): - await onyphe.SearchOnyphe('example.com').process() + async def fake_fetch(**_kwargs: Any) -> FetcherResponse: + raise cancellation + + monkeypatch.setattr(onyphe.AsyncFetcher, 'fetch', fake_fetch) + with pytest.raises(asyncio.CancelledError) as caught: + await onyphe.SearchOnyphe('example.com', 10).process() + assert caught.value is cancellation + assert session_exited is True diff --git a/tests/discovery/test_otx.py b/tests/discovery/test_otx.py index c9849408..ff3244f0 100644 --- a/tests/discovery/test_otx.py +++ b/tests/discovery/test_otx.py @@ -219,3 +219,6 @@ class TestOtx: if __name__ == '__main__': pytest.main() + + +pytestmark = pytest.mark.provider_contract('otx') diff --git a/tests/discovery/test_pentesttools.py b/tests/discovery/test_pentesttools.py index 34654e39..38cfbf17 100644 --- a/tests/discovery/test_pentesttools.py +++ b/tests/discovery/test_pentesttools.py @@ -133,6 +133,8 @@ async def test_malformed_start_response_completes_without_evidence(monkeypatch, assert not await search.get_hostnames() assert not await search.get_ips() + + assert 'malformed' in caplog.text @@ -252,3 +254,6 @@ async def test_non_subdomain_output_is_not_collected(monkeypatch) -> None: assert not await search.get_hostnames() assert not await search.get_ips() + + +pytestmark = pytest.mark.provider_contract('pentesttools') diff --git a/tests/discovery/test_projectdiscovery.py b/tests/discovery/test_projectdiscovery.py index 380622fb..7240ea2d 100644 --- a/tests/discovery/test_projectdiscovery.py +++ b/tests/discovery/test_projectdiscovery.py @@ -205,3 +205,6 @@ async def test_parser_exception_preserves_valid_partial_results( assert search.execution_status == 'partial' assert search.stop_reason == 'invalid-response' assert 'private provider payload' not in caplog.text + + +pytestmark = pytest.mark.provider_contract('projectdiscovery') diff --git a/tests/discovery/test_rapiddns.py b/tests/discovery/test_rapiddns.py index c2ff47db..c7239663 100644 --- a/tests/discovery/test_rapiddns.py +++ b/tests/discovery/test_rapiddns.py @@ -186,7 +186,8 @@ async def test_rapiddns_evidence_reaches_existing_outputs( class FakeSecurityScorecard: created = 0 - def __init__(self, _domain: str) -> None: + def __init__(self, _domain: str, limit: int) -> None: + assert limit == 500 type(self).created += 1 async def process(self, _proxy: bool) -> None: @@ -377,3 +378,6 @@ async def test_rapiddns_evidence_reaches_existing_outputs( assert failed_write_exit.value.code == 0 assert len(completed_results) == 3 assert 'forced completed-result failure' in capsys.readouterr().out + + +pytestmark = pytest.mark.provider_contract('rapiddns') diff --git a/tests/discovery/test_robtex.py b/tests/discovery/test_robtex.py index 4148d626..fd9388b3 100644 --- a/tests/discovery/test_robtex.py +++ b/tests/discovery/test_robtex.py @@ -153,3 +153,6 @@ async def test_robtex_cancellation_propagates(monkeypatch: pytest.MonkeyPatch) - with pytest.raises(asyncio.CancelledError): await search.process(proxy=True) + + +pytestmark = pytest.mark.provider_contract('robtex') diff --git a/tests/discovery/test_rocketreach.py b/tests/discovery/test_rocketreach.py index e2d54202..dfccd114 100644 --- a/tests/discovery/test_rocketreach.py +++ b/tests/discovery/test_rocketreach.py @@ -117,3 +117,6 @@ async def test_do_search_stops_on_throttling_message(monkeypatch) -> None: await search.process() assert len(calls) == 1 + + +pytestmark = pytest.mark.provider_contract('rocketreach') diff --git a/tests/discovery/test_search_dehashed.py b/tests/discovery/test_search_dehashed.py index 8fef644b..438ea922 100644 --- a/tests/discovery/test_search_dehashed.py +++ b/tests/discovery/test_search_dehashed.py @@ -142,3 +142,6 @@ async def test_rate_limit_retries_once_and_preserves_earlier_page(monkeypatch, c assert sleeps == [0.0] assert await search.get_emails() == {'first@example.com', 'second@example.com'} assert 'provider-secret-limit-detail' not in caplog.text + + +pytestmark = pytest.mark.provider_contract('dehashed') diff --git a/tests/discovery/test_securityscorecard.py b/tests/discovery/test_securityscorecard.py new file mode 100644 index 00000000..243963e3 --- /dev/null +++ b/tests/discovery/test_securityscorecard.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +import asyncio +import contextlib +from collections.abc import AsyncIterator +from typing import Any + +import pytest + +from theHarvester.discovery import securityscorecard +from theHarvester.discovery.constants import MissingKey +from theHarvester.lib.core import FetcherResponse + + +@pytest.mark.provider_contract('securityscorecard') +@pytest.mark.asyncio +async def test_process_paginates_documented_domain_and_ip_asset_routes(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(securityscorecard.Core, 'securityscorecard_key', staticmethod(lambda: 'test-key')) + monkeypatch.setattr(securityscorecard.SearchSecurityScorecard, 'PAGE_SIZE', 2) + session = object() + session_exited = False + calls: list[dict[str, Any]] = [] + post_responses = [ + FetcherResponse( + {'entries': [{'domain': 'API.Example.COM.'}, {'domain': 'outside.test'}], 'size': 3}, + 200, + {}, + ), + FetcherResponse({'entries': [{'domain': 'www.example.com'}], 'size': 3}, 200, {}), + FetcherResponse({'entries': [{'ip': '192.0.2.1'}, {'ip': '2001:db8::1'}], 'size': 3}, 200, {}), + FetcherResponse({'entries': [{'ip': '198.51.100.2'}], 'size': 3}, 200, {}), + ] + + @contextlib.asynccontextmanager + async def fake_open_session(**kwargs: Any) -> AsyncIterator[object]: + nonlocal session_exited + assert kwargs == {'headers': search.headers, 'proxy': True} + try: + yield session + finally: + session_exited = True + + async def fake_fetch(**kwargs: Any) -> FetcherResponse: + calls.append(kwargs) + return FetcherResponse({'score': 92, 'grade': 'A', 'factor_grades': {'network_security': 'A'}}, 200, {}) + + async def fake_post_fetch(*args: Any, **kwargs: Any) -> FetcherResponse: + calls.append({'url': args[0], **kwargs}) + return post_responses.pop(0) + + monkeypatch.setattr(securityscorecard.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(securityscorecard.AsyncFetcher, 'fetch', fake_fetch) + monkeypatch.setattr(securityscorecard.AsyncFetcher, 'post_fetch', fake_post_fetch) + search = securityscorecard.SearchSecurityScorecard('example.com', 3) + await search.process(proxy=True) + + assert await search.get_hostnames() == {'api.example.com', 'www.example.com'} + assert await search.get_ips() == {'192.0.2.1', '198.51.100.2', '2001:db8::1'} + assert await search.get_score() == 92 + assert await search.get_grades() == {'overall': 'A', 'network_security': 'A'} + assert search.execution_status == 'completed' + assert search.stop_reason is None + assert calls[0] == { + 'session': session, + 'url': 'https://api.securityscorecard.io/companies/example.com', + 'json': True, + 'include_metadata': True, + } + assert [(call['url'], call['json_body']) for call in calls[1:]] == [ + ('https://api.securityscorecard.io/parent-domains/example.com/domains', {'page': 0, 'page_size': 2}), + ('https://api.securityscorecard.io/parent-domains/example.com/domains', {'page': 1, 'page_size': 2}), + ('https://api.securityscorecard.io/parent-domains/example.com/ips', {'page': 0, 'page_size': 2}), + ('https://api.securityscorecard.io/parent-domains/example.com/ips', {'page': 1, 'page_size': 2}), + ] + assert all(call['session'] is session for call in calls[1:]) + assert session_exited is True + + +@pytest.mark.parametrize('key', [None, '', ' ']) +def test_missing_or_blank_key_fails_closed(monkeypatch: pytest.MonkeyPatch, key: str | None) -> None: + monkeypatch.setattr(securityscorecard.Core, 'securityscorecard_key', staticmethod(lambda: key)) + with pytest.raises(MissingKey): + securityscorecard.SearchSecurityScorecard('example.com', 10) + + +@pytest.mark.parametrize('limit', [0, -1, True, 1.5]) +def test_limit_must_be_a_positive_integer(monkeypatch: pytest.MonkeyPatch, limit: Any) -> None: + monkeypatch.setattr(securityscorecard.Core, 'securityscorecard_key', staticmethod(lambda: 'test-key')) + with pytest.raises(ValueError, match='positive integer'): + securityscorecard.SearchSecurityScorecard('example.com', limit) + + +@pytest.mark.parametrize( + ('response', 'status', 'reason'), + [ + (None, 'failed', 'transport-error'), + (FetcherResponse({}, 401, {}), 'failed', 'access-denied'), + (FetcherResponse({}, 403, {}), 'failed', 'access-denied'), + (FetcherResponse({}, 429, {}), 'rate-limited', 'http-429'), + (FetcherResponse({}, 503, {}), 'failed', 'http-503'), + (FetcherResponse([], 200, {}), 'failed', 'invalid-response'), + ], +) +@pytest.mark.asyncio +async def test_provider_failures_are_truthful( + monkeypatch: pytest.MonkeyPatch, + response: FetcherResponse | None, + status: str, + reason: str, +) -> None: + monkeypatch.setattr(securityscorecard.Core, 'securityscorecard_key', staticmethod(lambda: 'test-key')) + + async def fake_fetch(**_kwargs: Any) -> FetcherResponse | None: + return response + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + yield object() + + monkeypatch.setattr(securityscorecard.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(securityscorecard.AsyncFetcher, 'fetch', fake_fetch) + search = securityscorecard.SearchSecurityScorecard('example.com', 10) + await search.process() + + assert search.execution_status == status + assert search.stop_reason == reason + + +@pytest.mark.asyncio +async def test_malformed_asset_rows_preserve_valid_partial_results(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(securityscorecard.Core, 'securityscorecard_key', staticmethod(lambda: 'test-key')) + responses = [ + FetcherResponse({'entries': [{'domain': 7}, {'domain': 'api.example.com'}], 'size': 2}, 200, {}), + FetcherResponse({'entries': [{'ip': 'not-an-ip'}, {'ip': '192.0.2.1'}], 'size': 2}, 200, {}), + ] + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + yield object() + + async def fake_fetch(**_kwargs: Any) -> FetcherResponse: + return FetcherResponse({'score': 90, 'grade': 'A'}, 200, {}) + + async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse: + return responses.pop(0) + + monkeypatch.setattr(securityscorecard.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(securityscorecard.AsyncFetcher, 'fetch', fake_fetch) + monkeypatch.setattr(securityscorecard.AsyncFetcher, 'post_fetch', fake_post_fetch) + search = securityscorecard.SearchSecurityScorecard('example.com', 2) + await search.process() + + assert await search.get_hostnames() == {'api.example.com'} + assert await search.get_ips() == {'192.0.2.1'} + assert search.execution_status == 'partial' + assert search.stop_reason == 'invalid-response' + + +@pytest.mark.asyncio +async def test_cancellation_closes_provider_session(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(securityscorecard.Core, 'securityscorecard_key', staticmethod(lambda: 'test-key')) + session_exited = False + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + nonlocal session_exited + try: + yield object() + finally: + session_exited = True + + async def fake_fetch(**_kwargs: Any) -> FetcherResponse: + raise asyncio.CancelledError('operator-stop') + + monkeypatch.setattr(securityscorecard.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(securityscorecard.AsyncFetcher, 'fetch', fake_fetch) + + with pytest.raises(asyncio.CancelledError, match='operator-stop'): + await securityscorecard.SearchSecurityScorecard('example.com', 10).process() + assert session_exited is True diff --git a/tests/discovery/test_securitytrails.py b/tests/discovery/test_securitytrails.py new file mode 100644 index 00000000..3ca5882b --- /dev/null +++ b/tests/discovery/test_securitytrails.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +import asyncio +import contextlib +from typing import TYPE_CHECKING, Any + +import pytest + +from theHarvester.discovery import securitytrailssearch +from theHarvester.discovery.constants import MissingKey +from theHarvester.lib.core import FetcherResponse + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + +@pytest.mark.provider_contract('securityTrails') +@pytest.mark.asyncio +async def test_process_reuses_session_and_parses_scoped_evidence(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(securitytrailssearch.Core, 'security_trails_key', staticmethod(lambda: 'test-key')) + session = object() + session_exited = False + calls: list[dict[str, Any]] = [] + responses = [ + FetcherResponse( + { + 'hostname': 'example.com', + 'current_dns': { + 'a': {'values': [{'ip': '192.0.2.1'}]}, + 'aaaa': {'values': [{'ipv6': '2001:db8::1'}]}, + }, + }, + 200, + {}, + ), + FetcherResponse({'subdomains': ['API', 'www', 7]}, 200, {}), + ] + + @contextlib.asynccontextmanager + async def fake_open_session(**kwargs: Any) -> AsyncIterator[object]: + nonlocal session_exited + assert kwargs == {'headers': {'APIKEY': 'test-key', 'Accept': 'application/json'}, 'proxy': True} + try: + yield session + finally: + session_exited = True + + async def fake_fetch(**kwargs: Any) -> FetcherResponse: + calls.append(kwargs) + return responses.pop(0) + + monkeypatch.setattr(securitytrailssearch.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(securitytrailssearch.AsyncFetcher, 'fetch', fake_fetch) + search = securitytrailssearch.SearchSecuritytrail('example.com') + await search.process(proxy=True) + + assert await search.get_hostnames() == {'api.example.com', 'www.example.com'} + assert await search.get_ips() == {'192.0.2.1', '2001:db8::1'} + assert search.execution_status == 'partial' + assert search.stop_reason == 'invalid-response' + assert [call['url'] for call in calls] == [ + 'https://api.securitytrails.com/v1/domain/example.com', + 'https://api.securitytrails.com/v1/domain/example.com/subdomains', + ] + assert all(call['session'] is session for call in calls) + assert session_exited is True + + +@pytest.mark.parametrize('key', [None, '', ' ']) +def test_missing_or_blank_key_fails_closed(monkeypatch: pytest.MonkeyPatch, key: str | None) -> None: + monkeypatch.setattr(securitytrailssearch.Core, 'security_trails_key', staticmethod(lambda: key)) + with pytest.raises(MissingKey): + securitytrailssearch.SearchSecuritytrail('example.com') + + +@pytest.mark.parametrize( + ('response', 'status', 'reason'), + [ + (None, 'failed', 'transport-error'), + (FetcherResponse({}, 401, {}), 'failed', 'access-denied'), + (FetcherResponse({}, 429, {}), 'rate-limited', 'http-429'), + (FetcherResponse({}, 503, {}), 'failed', 'http-503'), + (FetcherResponse([], 200, {}), 'failed', 'invalid-response'), + ], +) +@pytest.mark.asyncio +async def test_first_request_failures_are_truthful( + monkeypatch: pytest.MonkeyPatch, + response: FetcherResponse | None, + status: str, + reason: str, +) -> None: + monkeypatch.setattr(securitytrailssearch.Core, 'security_trails_key', staticmethod(lambda: 'test-key')) + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + yield object() + + async def fake_fetch(**_kwargs: Any) -> FetcherResponse | None: + return response + + monkeypatch.setattr(securitytrailssearch.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(securitytrailssearch.AsyncFetcher, 'fetch', fake_fetch) + search = securitytrailssearch.SearchSecuritytrail('example.com') + await search.process() + + assert search.execution_status == status + assert search.stop_reason == reason + + +@pytest.mark.asyncio +async def test_cancellation_closes_provider_session(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(securitytrailssearch.Core, 'security_trails_key', staticmethod(lambda: 'test-key')) + session_exited = False + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + nonlocal session_exited + try: + yield object() + finally: + session_exited = True + + async def fake_fetch(**_kwargs: Any) -> FetcherResponse: + raise asyncio.CancelledError('operator-stop') + + monkeypatch.setattr(securitytrailssearch.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(securitytrailssearch.AsyncFetcher, 'fetch', fake_fetch) + + with pytest.raises(asyncio.CancelledError, match='operator-stop'): + await securitytrailssearch.SearchSecuritytrail('example.com').process() + assert session_exited is True diff --git a/tests/discovery/test_sherlockeye.py b/tests/discovery/test_sherlockeye.py index 48fcc936..6a0b14a6 100644 --- a/tests/discovery/test_sherlockeye.py +++ b/tests/discovery/test_sherlockeye.py @@ -1,6 +1,10 @@ +import asyncio +import contextlib import logging import sys import types +from collections.abc import AsyncIterator +from typing import Any import pytest @@ -12,21 +16,83 @@ if 'aiohttp_socks' not in sys.modules: def from_url(*_args, **_kwargs): return None - setattr(aiohttp_socks_stub, 'ProxyConnector', _ProxyConnector) + aiohttp_socks_stub.ProxyConnector = _ProxyConnector # type: ignore[attr-defined] sys.modules['aiohttp_socks'] = aiohttp_socks_stub from theHarvester.discovery import sherlockeye from theHarvester.discovery.constants import MissingKey +from theHarvester.lib.core import FetcherResponse -@pytest.mark.asyncio -async def test_missing_key_raises(monkeypatch) -> None: - monkeypatch.setattr(sherlockeye.Core, 'sherlockeye_key', lambda: None) +@pytest.fixture(autouse=True) +def provider_session(monkeypatch: pytest.MonkeyPatch) -> None: + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + yield object() + + monkeypatch.setattr(sherlockeye.AsyncFetcher, 'open_session', fake_open_session) + + +@pytest.mark.parametrize('key', [None, '', ' ']) +def test_missing_or_blank_key_raises(monkeypatch: pytest.MonkeyPatch, key: str | None) -> None: + monkeypatch.setattr(sherlockeye.Core, 'sherlockeye_key', lambda: key) with pytest.raises(MissingKey): sherlockeye.SearchSherlockeye('example.com') +@pytest.mark.asyncio +async def test_process_uses_one_shared_provider_session(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sherlockeye.Core, 'sherlockeye_key', lambda: 'dummy-key') + session = object() + exited = False + calls: list[dict[str, Any]] = [] + + @contextlib.asynccontextmanager + async def fake_open_session(**kwargs: Any) -> AsyncIterator[object]: + nonlocal exited + assert kwargs == { + 'headers': { + 'User-Agent': sherlockeye.Core.get_user_agent(), + 'Authorization': 'Bearer dummy-key', + 'Content-Type': 'application/json', + }, + 'proxy': True, + 'request_timeout': 90, + } + try: + yield session + finally: + exited = True + + async def fake_post_fetch(*args: Any, **kwargs: Any) -> FetcherResponse: + calls.append({'url': args[0], **kwargs}) + return FetcherResponse({'success': True, 'data': {'results': []}}, 200, {}) + + monkeypatch.setattr(sherlockeye.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(sherlockeye.AsyncFetcher, 'post_fetch', fake_post_fetch) + search = sherlockeye.SearchSherlockeye('example.com') + + await search.process(proxy=True) + + assert calls == [ + { + 'url': search.SYNC_SEARCH_URL, + 'session': session, + 'json': True, + 'include_metadata': True, + 'json_body': { + 'type': 'domain', + 'value': 'example.com', + 'timeoutSeconds': 60, + }, + } + ] + assert exited is True + assert search.execution_status == 'completed' + assert search.stop_reason == 'no-results' + + @pytest.mark.asyncio async def test_process_extracts_domain_intelligence(monkeypatch) -> None: monkeypatch.setattr(sherlockeye.Core, 'sherlockeye_key', lambda: 'dummy-key') @@ -59,37 +125,17 @@ async def test_process_extracts_domain_intelligence(monkeypatch) -> None: 'link': 'https://api.example.com/docs', }, }, + {'attributes': {'email': 'user@notexample.com'}}, + {'attributes': {'email': 'user@example.com.evil'}}, ], }, 'balance': {'credits': 10}, } - class _FakeResponse: - status = 200 + async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse: + return FetcherResponse(api_payload, 200, {}) - async def json(self): - return api_payload - - async def __aenter__(self): - return self - - async def __aexit__(self, *_args): - pass - - class _FakeSession: - def __init__(self, **_kwargs): - pass - - def post(self, *_args, **_kwargs): - return _FakeResponse() - - async def __aenter__(self): - return self - - async def __aexit__(self, *_args): - pass - - monkeypatch.setattr(sherlockeye.aiohttp, 'ClientSession', _FakeSession) + monkeypatch.setattr(sherlockeye.AsyncFetcher, 'post_fetch', fake_post_fetch) search = sherlockeye.SearchSherlockeye('example.com') await search.process() @@ -97,38 +143,18 @@ async def test_process_extracts_domain_intelligence(monkeypatch) -> None: assert await search.get_hostnames() == {'sub.example.com', 'www.example.com', 'api.example.com'} assert await search.get_emails() == {'user@example.com'} assert await search.get_ips() == {'203.0.113.10'} + assert search.execution_status == 'completed' + assert search.stop_reason is None @pytest.mark.asyncio async def test_process_handles_api_error(monkeypatch, caplog) -> None: monkeypatch.setattr(sherlockeye.Core, 'sherlockeye_key', lambda: 'dummy-key') - class _FakeResponse: - status = 401 + async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse: + return FetcherResponse({'secret': 'provider-secret-payload'}, 401, {}) - async def text(self): - return 'provider-secret-payload' - - async def __aenter__(self): - return self - - async def __aexit__(self, *_args): - pass - - class _FakeSession: - def __init__(self, **_kwargs): - pass - - def post(self, *_args, **_kwargs): - return _FakeResponse() - - async def __aenter__(self): - return self - - async def __aexit__(self, *_args): - pass - - monkeypatch.setattr(sherlockeye.aiohttp, 'ClientSession', _FakeSession) + monkeypatch.setattr(sherlockeye.AsyncFetcher, 'post_fetch', fake_post_fetch) caplog.set_level(logging.INFO, logger=sherlockeye.__name__) search = sherlockeye.SearchSherlockeye('example.com') @@ -139,41 +165,121 @@ async def test_process_handles_api_error(monkeypatch, caplog) -> None: assert await search.get_ips() == set() assert 'provider-secret-payload' not in caplog.text assert '401' in caplog.text + assert search.execution_status == 'failed' + assert search.stop_reason == 'access-denied' @pytest.mark.asyncio async def test_process_does_not_log_provider_error_message(monkeypatch, caplog) -> None: monkeypatch.setattr(sherlockeye.Core, 'sherlockeye_key', lambda: 'dummy-key') - class _FakeResponse: - status = 200 + async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse: + return FetcherResponse({'success': False, 'message': 'provider-secret-payload'}, 200, {}) - async def json(self): - return {'success': False, 'message': 'provider-secret-payload'} - - async def __aenter__(self): - return self - - async def __aexit__(self, *_args): - pass - - class _FakeSession: - def __init__(self, **_kwargs): - pass - - def post(self, *_args, **_kwargs): - return _FakeResponse() - - async def __aenter__(self): - return self - - async def __aexit__(self, *_args): - pass - - monkeypatch.setattr(sherlockeye.aiohttp, 'ClientSession', _FakeSession) + monkeypatch.setattr(sherlockeye.AsyncFetcher, 'post_fetch', fake_post_fetch) caplog.set_level(logging.INFO, logger=sherlockeye.__name__) - await sherlockeye.SearchSherlockeye('example.com').process() + search = sherlockeye.SearchSherlockeye('example.com') + await search.process() assert 'provider-secret-payload' not in caplog.text assert 'API error' in caplog.text + assert search.execution_status == 'failed' + assert search.stop_reason == 'provider-error' + + +@pytest.mark.parametrize( + ('status', 'execution_status', 'stop_reason'), + [(429, 'rate-limited', 'http-429'), (503, 'failed', 'http-503')], +) +@pytest.mark.asyncio +async def test_http_failures_are_structured( + monkeypatch: pytest.MonkeyPatch, + status: int, + execution_status: str, + stop_reason: str, +) -> None: + monkeypatch.setattr(sherlockeye.Core, 'sherlockeye_key', lambda: 'dummy-key') + + async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse: + return FetcherResponse({}, status, {}) + + monkeypatch.setattr(sherlockeye.AsyncFetcher, 'post_fetch', fake_post_fetch) + search = sherlockeye.SearchSherlockeye('example.com') + await search.process() + + assert search.execution_status == execution_status + assert search.stop_reason == stop_reason + + +@pytest.mark.asyncio +async def test_malformed_response_is_structured(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sherlockeye.Core, 'sherlockeye_key', lambda: 'dummy-key') + + async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse: + return FetcherResponse([], 200, {}) + + monkeypatch.setattr(sherlockeye.AsyncFetcher, 'post_fetch', fake_post_fetch) + search = sherlockeye.SearchSherlockeye('example.com') + await search.process() + + assert search.execution_status == 'failed' + assert search.stop_reason == 'invalid-response' + + +def test_malformed_link_does_not_discard_later_valid_results(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sherlockeye.Core, 'sherlockeye_key', lambda: 'dummy-key') + search = sherlockeye.SearchSherlockeye('example.com') + + search._extract_response( + { + 'success': True, + 'data': { + 'results': [ + {'attributes': {'link': 'https://[invalid'}}, + {'attributes': {'link': 'https://api.example.com/path'}}, + ] + }, + } + ) + + assert search.totalhosts == {'api.example.com'} + assert search.execution_status == 'partial' + assert search.stop_reason == 'invalid-response' + + +@pytest.mark.asyncio +async def test_transport_failure_and_cancellation_are_distinct(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sherlockeye.Core, 'sherlockeye_key', lambda: 'dummy-key') + session_exit_count = 0 + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + nonlocal session_exit_count + try: + yield object() + finally: + session_exit_count += 1 + + monkeypatch.setattr(sherlockeye.AsyncFetcher, 'open_session', fake_open_session) + + async def failed_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse: + raise RuntimeError('provider-secret') + + monkeypatch.setattr(sherlockeye.AsyncFetcher, 'post_fetch', failed_post_fetch) + search = sherlockeye.SearchSherlockeye('example.com') + await search.process() + assert search.execution_status == 'failed' + assert search.stop_reason == 'transport-error' + assert session_exit_count == 1 + + async def cancelled_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse: + raise asyncio.CancelledError + + monkeypatch.setattr(sherlockeye.AsyncFetcher, 'post_fetch', cancelled_post_fetch) + with pytest.raises(asyncio.CancelledError): + await sherlockeye.SearchSherlockeye('example.com').process() + assert session_exit_count == 2 + + +pytestmark = pytest.mark.provider_contract('sherlockeye') diff --git a/tests/discovery/test_shodan_engine.py b/tests/discovery/test_shodan_engine.py index 5178b29b..2ca2816b 100644 --- a/tests/discovery/test_shodan_engine.py +++ b/tests/discovery/test_shodan_engine.py @@ -697,3 +697,9 @@ class TestShodanEngine: assert not await search.get_ips() assert 'Shodan InternetDB request failed' in caplog.text assert 'provider-secret-payload' not in caplog.text + + +pytestmark = [ + pytest.mark.provider_contract('shodan'), + pytest.mark.provider_contract('shodanInternetDB'), +] diff --git a/tests/discovery/test_shodanct.py b/tests/discovery/test_shodanct.py index 3c6868c7..d9a8240d 100644 --- a/tests/discovery/test_shodanct.py +++ b/tests/discovery/test_shodanct.py @@ -99,3 +99,6 @@ async def test_process_reports_transport_and_malformed_responses( assert await search.get_hostnames() == set() assert message in caplog.text + + +pytestmark = pytest.mark.provider_contract('shodanct') diff --git a/tests/discovery/test_sourcegraph.py b/tests/discovery/test_sourcegraph.py index cb6ee6dd..014720cf 100644 --- a/tests/discovery/test_sourcegraph.py +++ b/tests/discovery/test_sourcegraph.py @@ -483,3 +483,6 @@ async def test_sourcegraph_partial_outcome_reaches_completed_result( assert execution.result_count == 1 summary = json.loads(report.with_suffix('.jsonl').read_text().splitlines()[0]) assert summary['source_executions'][0]['stop_reason'] == 'provider-limited' + + +pytestmark = pytest.mark.provider_contract('sourcegraph') diff --git a/tests/discovery/test_subdomaincenter.py b/tests/discovery/test_subdomaincenter.py index a9dc80c4..3cc2458e 100644 --- a/tests/discovery/test_subdomaincenter.py +++ b/tests/discovery/test_subdomaincenter.py @@ -82,3 +82,6 @@ async def test_process_attributes_http_failures( assert await search.get_hostnames() == set() assert 'SubdomainCenter request failed with HTTP 429' in caplog.text + + +pytestmark = pytest.mark.provider_contract('subdomaincenter') diff --git a/tests/discovery/test_subdomainfinderc99.py b/tests/discovery/test_subdomainfinderc99.py index a10ca258..fb903e99 100644 --- a/tests/discovery/test_subdomainfinderc99.py +++ b/tests/discovery/test_subdomainfinderc99.py @@ -1,20 +1,41 @@ +import asyncio +import contextlib +from collections.abc import AsyncIterator +from typing import Any + import pytest from theHarvester.discovery import subdomainfinderc99 +from theHarvester.lib.core import FetcherResponse @pytest.mark.asyncio async def test_successful_response_returns_only_scoped_hostnames(monkeypatch) -> None: - async def fake_fetch_all(*_args, **_kwargs): - return ['
'] + session = object() + session_exited = False + calls: list[tuple[str, object]] = [] - async def fake_post_fetch(*_args, **_kwargs): - return 'api.example.test www.notexample.test' + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + nonlocal session_exited + try: + yield session + finally: + session_exited = True + + async def fake_fetch(*_args, **kwargs): + calls.append(('get', kwargs['session'])) + return FetcherResponse('
', 200, {}) + + async def fake_post_fetch(*_args, **kwargs): + calls.append(('post', kwargs['session'])) + return FetcherResponse('api.example.test www.notexample.test', 200, {}) async def no_sleep(*_args, **_kwargs): return None - monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'fetch_all', fake_fetch_all) + monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'fetch', fake_fetch) monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'post_fetch', fake_post_fetch) monkeypatch.setattr(subdomainfinderc99.asyncio, 'sleep', no_sleep) @@ -22,33 +43,61 @@ async def test_successful_response_returns_only_scoped_hostnames(monkeypatch) -> await search.process() assert set(await search.get_hostnames()) == {'api.example.test'} + assert search.execution_status == 'completed' + assert search.stop_reason is None + assert calls == [('get', session), ('post', session)] + assert session_exited is True @pytest.mark.asyncio -async def test_empty_initial_response_completes_without_evidence(monkeypatch) -> None: - async def fake_fetch_all(*_args, **_kwargs): - return [] +async def test_empty_initial_response_is_transport_failure(monkeypatch) -> None: + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + yield object() - monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'fetch_all', fake_fetch_all) + async def fake_fetch(*_args, **_kwargs): + return None + + monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'fetch', fake_fetch) search = subdomainfinderc99.SearchSubdomainfinderc99('example.test') await search.process() assert not await search.get_hostnames() + assert search.execution_status == 'failed' + assert search.stop_reason == 'transport-error' +@pytest.mark.parametrize( + ('response', 'status', 'reason'), + [ + (FetcherResponse('', 403, {}), 'failed', 'access-denied'), + (FetcherResponse(None, 200, {}), 'failed', 'invalid-response'), + ], +) @pytest.mark.asyncio -async def test_malformed_scan_response_completes_without_evidence(monkeypatch) -> None: - async def fake_fetch_all(*_args, **_kwargs): - return ['
'] +async def test_scan_failures_are_structured( + monkeypatch: pytest.MonkeyPatch, + response: FetcherResponse, + status: str, + reason: str, +) -> None: + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + yield object() + + async def fake_fetch(*_args, **_kwargs): + return FetcherResponse('
', 200, {}) async def fake_post_fetch(*_args, **_kwargs): - return None + return response async def no_sleep(*_args, **_kwargs): return None - monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'fetch_all', fake_fetch_all) + monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'fetch', fake_fetch) monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'post_fetch', fake_post_fetch) monkeypatch.setattr(subdomainfinderc99.asyncio, 'sleep', no_sleep) @@ -56,3 +105,63 @@ async def test_malformed_scan_response_completes_without_evidence(monkeypatch) - await search.process() assert not await search.get_hostnames() + assert search.execution_status == status + assert search.stop_reason == reason + + +@pytest.mark.parametrize( + ('response', 'status', 'reason'), + [ + (FetcherResponse('', 401, {}), 'failed', 'access-denied'), + (FetcherResponse('', 403, {}), 'failed', 'access-denied'), + (FetcherResponse('', 429, {}), 'rate-limited', 'http-429'), + (FetcherResponse('', 503, {}), 'failed', 'http-503'), + (FetcherResponse({}, 200, {}), 'failed', 'invalid-response'), + ], +) +@pytest.mark.asyncio +async def test_initial_failures_are_structured( + monkeypatch: pytest.MonkeyPatch, + response: FetcherResponse, + status: str, + reason: str, +) -> None: + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + yield object() + + async def fake_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse: + return response + + monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'fetch', fake_fetch) + search = subdomainfinderc99.SearchSubdomainfinderc99('example.test') + await search.process() + + assert search.execution_status == status + assert search.stop_reason == reason + + +@pytest.mark.asyncio +async def test_cancellation_propagates(monkeypatch: pytest.MonkeyPatch) -> None: + session_exited = False + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + nonlocal session_exited + try: + yield object() + finally: + session_exited = True + + async def fake_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse: + raise asyncio.CancelledError + + monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(subdomainfinderc99.AsyncFetcher, 'fetch', fake_fetch) + with pytest.raises(asyncio.CancelledError): + await subdomainfinderc99.SearchSubdomainfinderc99('example.test').process() + assert session_exited is True + + +pytestmark = pytest.mark.provider_contract('subdomainfinderc99') diff --git a/tests/discovery/test_thc.py b/tests/discovery/test_thc.py index d9fe225f..05999439 100644 --- a/tests/discovery/test_thc.py +++ b/tests/discovery/test_thc.py @@ -444,3 +444,6 @@ class TestThcIntegration: if __name__ == '__main__': pytest.main() + + +pytestmark = pytest.mark.provider_contract('thc') diff --git a/tests/discovery/test_tombasearch.py b/tests/discovery/test_tombasearch.py index 6bba83e7..99554328 100644 --- a/tests/discovery/test_tombasearch.py +++ b/tests/discovery/test_tombasearch.py @@ -55,6 +55,8 @@ async def test_tomba_http_failures_return_no_results(monkeypatch, caplog, status assert await search.get_emails() == [] assert await search.get_hostnames() == [] + + assert f'Tomba request failed with HTTP {status}' in caplog.text assert 'provider detail' not in caplog.text @@ -294,3 +296,6 @@ async def test_paid_tomba_search_stops_before_exceeding_quota(monkeypatch) -> No ] assert await search.get_emails() == [] assert await search.get_hostnames() == [] + + +pytestmark = pytest.mark.provider_contract('tomba') diff --git a/tests/discovery/test_urlscan.py b/tests/discovery/test_urlscan.py index 19c8c716..74472881 100644 --- a/tests/discovery/test_urlscan.py +++ b/tests/discovery/test_urlscan.py @@ -1,4 +1,6 @@ import asyncio +import contextlib +from collections.abc import AsyncIterator from datetime import UTC, datetime, timedelta from typing import Any @@ -8,9 +10,30 @@ from theHarvester.discovery import urlscan from theHarvester.lib.core import FetcherResponse +class ProviderSession: + def __init__(self) -> None: + self.exited = False + + +@pytest.fixture(autouse=True) +def provider_session(monkeypatch: pytest.MonkeyPatch) -> ProviderSession: + session = ProviderSession() + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + try: + yield session + finally: + session.exited = True + + monkeypatch.setattr(urlscan.AsyncFetcher, 'open_session', fake_open_session) + return session + + @pytest.mark.asyncio async def test_process_collects_sequential_pages_and_preserves_all_routes( monkeypatch: pytest.MonkeyPatch, + provider_session: ProviderSession, ) -> None: responses = [ FetcherResponse( @@ -49,7 +72,6 @@ async def test_process_collects_sequential_pages_and_preserves_all_routes( status=200, headers={}, ), - FetcherResponse(body={'results': []}, status=200, headers={}), ] calls: list[dict[str, Any]] = [] @@ -58,7 +80,7 @@ async def test_process_collects_sequential_pages_and_preserves_all_routes( return responses.pop(0) monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch) - search = urlscan.SearchUrlscan('example.com') + search = urlscan.SearchUrlscan('example.com', 2) await search.process(proxy=True) @@ -84,15 +106,15 @@ async def test_process_collects_sequential_pages_and_preserves_all_routes( ('AS64497', 'Example Transit Two', 'ip', '2001:db8::10'), } assert [call['params'] for call in calls] == [ - {'q': 'domain:example.com'}, - {'q': 'domain:example.com', 'search_after': '200,first'}, - {'q': 'domain:example.com', 'search_after': '100,second'}, + {'q': 'domain:example.com', 'size': 2}, + {'q': 'domain:example.com', 'size': 1, 'search_after': '200,first'}, ] assert all(call['url'] == 'https://urlscan.io/api/v1/search/' for call in calls) + assert all(call['session'] is provider_session for call in calls) assert all(call['json'] is True for call in calls) assert all(call['include_metadata'] is True for call in calls) - assert all(call['proxy'] is True for call in calls) - assert all(call['request_timeout'] == 60 for call in calls) + assert all('request_timeout' not in call for call in calls) + assert provider_session.exited is True assert search.execution_status == 'completed' assert search.stop_reason is None @@ -125,7 +147,7 @@ async def test_repeated_asn_relationship_is_retained_once_per_source_run(monkeyp monkeypatch.setattr(urlscan, 'datetime', TickingDateTime) monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch) - search = urlscan.SearchUrlscan('example.com') + search = urlscan.SearchUrlscan('example.com', 10) await search.process() @@ -138,7 +160,7 @@ async def test_valid_empty_response_is_completed(monkeypatch: pytest.MonkeyPatch return FetcherResponse(body={'results': []}, status=200, headers={}) monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch) - search = urlscan.SearchUrlscan('example.com') + search = urlscan.SearchUrlscan('example.com', 10) await search.process() @@ -165,7 +187,7 @@ async def test_missing_optional_fields_are_skipped(monkeypatch: pytest.MonkeyPat return responses.pop(0) monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch) - search = urlscan.SearchUrlscan('example.com') + search = urlscan.SearchUrlscan('example.com', 10) await search.process() @@ -194,7 +216,7 @@ async def test_malformed_nested_fields_preserve_valid_partial_results(monkeypatc return responses.pop(0) monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch) - search = urlscan.SearchUrlscan('example.com') + search = urlscan.SearchUrlscan('example.com', 10) await search.process() @@ -240,7 +262,7 @@ async def test_results_are_typed_and_scoped_before_insertion(monkeypatch: pytest return responses.pop(0) monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch) - search = urlscan.SearchUrlscan('example.com') + search = urlscan.SearchUrlscan('example.com', 10) await search.process() @@ -275,7 +297,7 @@ async def test_failed_first_page_is_attributed( return response monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch) - search = urlscan.SearchUrlscan('example.com') + search = urlscan.SearchUrlscan('example.com', 10) await search.process() @@ -312,7 +334,7 @@ async def test_later_failure_preserves_partial_results( return responses.pop(0) monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch) - search = urlscan.SearchUrlscan('example.com') + search = urlscan.SearchUrlscan('example.com', 10) await search.process() @@ -327,7 +349,7 @@ async def test_fetch_exception_is_transport_failure(monkeypatch: pytest.MonkeyPa raise OSError('private transport details') monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch) - search = urlscan.SearchUrlscan('example.com') + search = urlscan.SearchUrlscan('example.com', 10) await search.process() @@ -349,7 +371,7 @@ async def test_missing_cursor_stops_after_first_page(monkeypatch: pytest.MonkeyP ) monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch) - search = urlscan.SearchUrlscan('example.com') + search = urlscan.SearchUrlscan('example.com', 10) await search.process() @@ -381,7 +403,7 @@ async def test_repeated_cursor_stops_without_a_third_request(monkeypatch: pytest return responses.pop(0) monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch) - search = urlscan.SearchUrlscan('example.com') + search = urlscan.SearchUrlscan('example.com', 10) await search.process() @@ -392,43 +414,89 @@ async def test_repeated_cursor_stops_without_a_third_request(monkeypatch: pytest @pytest.mark.asyncio -async def test_page_limit_preserves_results(monkeypatch: pytest.MonkeyPatch) -> None: - calls = 0 - - async def fake_fetch(**_kwargs: Any) -> FetcherResponse: - nonlocal calls - calls += 1 - return FetcherResponse( +async def test_pagination_continues_beyond_the_removed_local_page_ceiling(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[dict[str, Any]] = [] + first_page = [ + {'page': {'domain': f'page-{index}.example.com'}, 'sort': [10_001 - index, f'cursor-{index}']} + for index in range(1, 10_001) + ] + responses = [ + FetcherResponse(body={'results': first_page}, status=200, headers={}), + FetcherResponse( body={ 'results': [ { - 'page': {'domain': f'page-{calls}.example.com'}, - 'sort': [calls, f'cursor-{calls}'], + 'page': {'domain': 'page-10001.example.com'}, + 'sort': [0, 'cursor-10001'], } ] }, status=200, headers={}, - ) + ), + ] + + async def fake_fetch(**kwargs: Any) -> FetcherResponse: + calls.append(kwargs['params']) + return responses.pop(0) monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch) - monkeypatch.setattr(urlscan.SearchUrlscan, 'MAX_PAGES', 2) - search = urlscan.SearchUrlscan('example.com') + search = urlscan.SearchUrlscan('example.com', 10_001) await search.process() - assert calls == 2 - assert await search.get_hostnames() == {'page-1.example.com', 'page-2.example.com'} - assert search.execution_status == 'partial' - assert search.stop_reason == 'page-limit' + assert calls == [ + {'q': 'domain:example.com', 'size': 10_000}, + {'q': 'domain:example.com', 'size': 1, 'search_after': '1,cursor-10000'}, + ] + assert await search.get_hostnames() == {f'page-{page}.example.com' for page in range(1, 10_002)} + assert search.execution_status == 'completed' + assert search.stop_reason is None @pytest.mark.asyncio -async def test_cancellation_propagates(monkeypatch: pytest.MonkeyPatch) -> None: +async def test_operator_limit_sets_page_size_and_stops_without_an_extra_request( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[dict[str, Any]] = [] + results = [ + {'page': {'domain': f'result-{index}.example.com'}, 'sort': [10 - index, f'cursor-{index}']} for index in range(10) + ] + + async def fake_fetch(**kwargs: Any) -> FetcherResponse: + calls.append(kwargs['params']) + return FetcherResponse(body={'results': results}, status=200, headers={}) + + monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch) + search = urlscan.SearchUrlscan('example.com', 10) + + await search.process() + + assert calls == [{'q': 'domain:example.com', 'size': 10}] + assert len(await search.get_hostnames()) == 10 + assert search.execution_status == 'completed' + assert search.stop_reason is None + + +@pytest.mark.parametrize('limit', [0, -1, True, 1.5]) +def test_limit_must_be_a_positive_integer(limit: Any) -> None: + with pytest.raises(ValueError, match='positive integer'): + urlscan.SearchUrlscan('example.com', limit) + + +@pytest.mark.asyncio +async def test_cancellation_propagates( + monkeypatch: pytest.MonkeyPatch, + provider_session: ProviderSession, +) -> None: async def fake_fetch(**_kwargs: Any) -> FetcherResponse: raise asyncio.CancelledError monkeypatch.setattr(urlscan.AsyncFetcher, 'fetch', fake_fetch) with pytest.raises(asyncio.CancelledError): - await urlscan.SearchUrlscan('example.com').process() + await urlscan.SearchUrlscan('example.com', 10).process() + assert provider_session.exited is True + + +pytestmark = pytest.mark.provider_contract('urlscan') diff --git a/tests/discovery/test_virustotal.py b/tests/discovery/test_virustotal.py index 584de772..2aa39277 100644 --- a/tests/discovery/test_virustotal.py +++ b/tests/discovery/test_virustotal.py @@ -1,6 +1,89 @@ +from __future__ import annotations + +import asyncio +import contextlib +from typing import TYPE_CHECKING, Any + import pytest from theHarvester.discovery import virustotal +from theHarvester.discovery.constants import MissingKey +from theHarvester.lib.core import FetcherResponse + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + +@pytest.mark.provider_contract('virustotal') +@pytest.mark.asyncio +async def test_process_paginates_without_fixed_sleeps_and_keeps_scoped_evidence( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(virustotal.Core, 'virustotal_key', staticmethod(lambda: 'test-key')) + session = object() + session_exited = False + calls: list[dict[str, Any]] = [] + responses = [ + FetcherResponse( + { + 'data': [ + { + 'id': 'API.Example.COM.', + 'attributes': { + 'last_dns_records': [{'value': 'dns.example.com'}], + 'last_https_certificate': { + 'extensions': {'subject_alternative_name': ['tls.example.com', 'outside.test']} + }, + }, + } + ], + 'meta': {'cursor': 'next-page'}, + }, + 200, + {}, + ), + FetcherResponse( + {'data': [{'id': 'mail.example.com', 'attributes': {'last_dns_records': []}}], 'meta': {}}, + 200, + {}, + ), + ] + + @contextlib.asynccontextmanager + async def fake_open_session(**kwargs: Any) -> AsyncIterator[object]: + nonlocal session_exited + assert kwargs == { + 'headers': { + 'Accept': 'application/json', + 'x-apikey': 'test-key', + }, + 'proxy': True, + } + try: + yield session + finally: + session_exited = True + + async def fake_fetch(**kwargs: Any) -> FetcherResponse: + calls.append(kwargs) + return responses.pop(0) + + monkeypatch.setattr(virustotal.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(virustotal.AsyncFetcher, 'fetch', fake_fetch) + search = virustotal.SearchVirustotal('example.com', limit=10) + await search.process(proxy=True) + + assert await search.get_hostnames() == { + 'api.example.com', + 'dns.example.com', + 'mail.example.com', + 'tls.example.com', + } + assert search.execution_status == 'completed' + assert search.stop_reason is None + assert [call['params'] for call in calls] == [{'limit': 10}, {'limit': 9, 'cursor': 'next-page'}] + assert all(call['session'] is session for call in calls) + assert session_exited is True @pytest.mark.asyncio @@ -10,15 +93,140 @@ async def test_parse_hostnames_preserves_www_evidence() -> None: 'id': 'www.example.com', 'attributes': { 'last_dns_records': [{'value': 'www.api.example.com'}], - 'last_https_certificate': { - 'extensions': {'subject_alternative_name': ['www.mail.example.com']} - }, + 'last_https_certificate': {'extensions': {'subject_alternative_name': ['www.mail.example.com']}}, }, } ] - assert await virustotal.SearchVirustotal.parse_hostnames(data, 'example.com') == [ - 'www.api.example.com', - 'www.example.com', - 'www.mail.example.com', + hostnames, malformed = virustotal.SearchVirustotal.parse_hostnames(data, 'example.com') + assert hostnames == {'www.api.example.com', 'www.example.com', 'www.mail.example.com'} + assert malformed is False + + +@pytest.mark.parametrize('key', [None, '', ' ']) +def test_missing_or_blank_key_fails_closed(monkeypatch: pytest.MonkeyPatch, key: str | None) -> None: + monkeypatch.setattr(virustotal.Core, 'virustotal_key', staticmethod(lambda: key)) + with pytest.raises(MissingKey): + virustotal.SearchVirustotal('example.com', limit=10) + + +@pytest.mark.parametrize('limit', [0, -1, True, 1.5]) +def test_limit_must_be_a_positive_integer(monkeypatch: pytest.MonkeyPatch, limit: Any) -> None: + monkeypatch.setattr(virustotal.Core, 'virustotal_key', staticmethod(lambda: 'test-key')) + with pytest.raises(ValueError, match='positive integer'): + virustotal.SearchVirustotal('example.com', limit=limit) + + +@pytest.mark.parametrize( + ('response', 'status', 'reason'), + [ + (None, 'failed', 'transport-error'), + (FetcherResponse({}, 401, {}), 'failed', 'access-denied'), + (FetcherResponse({}, 429, {}), 'rate-limited', 'http-429'), + (FetcherResponse({}, 503, {}), 'failed', 'http-503'), + (FetcherResponse([], 200, {}), 'failed', 'invalid-response'), + ], +) +@pytest.mark.asyncio +async def test_provider_failures_are_truthful( + monkeypatch: pytest.MonkeyPatch, + response: FetcherResponse | None, + status: str, + reason: str, +) -> None: + monkeypatch.setattr(virustotal.Core, 'virustotal_key', staticmethod(lambda: 'test-key')) + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + yield object() + + async def fake_fetch(**_kwargs: Any) -> FetcherResponse | None: + return response + + monkeypatch.setattr(virustotal.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(virustotal.AsyncFetcher, 'fetch', fake_fetch) + search = virustotal.SearchVirustotal('example.com', limit=10) + await search.process() + + assert search.execution_status == status + assert search.stop_reason == reason + + +@pytest.mark.asyncio +async def test_later_rate_limit_preserves_partial_results(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(virustotal.Core, 'virustotal_key', staticmethod(lambda: 'test-key')) + responses = [ + FetcherResponse( + {'data': [{'id': 'api.example.com', 'attributes': {}}], 'meta': {'cursor': 'next'}}, + 200, + {}, + ), + FetcherResponse({}, 429, {}), ] + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + yield object() + + async def fake_fetch(**_kwargs: Any) -> FetcherResponse: + return responses.pop(0) + + monkeypatch.setattr(virustotal.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(virustotal.AsyncFetcher, 'fetch', fake_fetch) + search = virustotal.SearchVirustotal('example.com', limit=10) + await search.process() + + assert await search.get_hostnames() == {'api.example.com'} + assert search.execution_status == 'partial' + assert search.stop_reason == 'http-429' + + +@pytest.mark.asyncio +async def test_repeated_cursor_stops_without_spending_more_quota(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(virustotal.Core, 'virustotal_key', staticmethod(lambda: 'test-key')) + responses = [ + FetcherResponse({'data': [{'id': 'outside.test', 'attributes': {}}], 'meta': {'cursor': 'same'}}, 200, {}), + FetcherResponse({'data': [{'id': 'api.example.com', 'attributes': {}}], 'meta': {'cursor': 'same'}}, 200, {}), + ] + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + yield object() + + async def fake_fetch(**_kwargs: Any) -> FetcherResponse: + return responses.pop(0) + + monkeypatch.setattr(virustotal.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(virustotal.AsyncFetcher, 'fetch', fake_fetch) + search = virustotal.SearchVirustotal('example.com', limit=10) + + await search.process() + + assert await search.get_hostnames() == {'api.example.com'} + assert search.execution_status == 'partial' + assert search.stop_reason == 'repeated-cursor' + assert responses == [] + + +@pytest.mark.asyncio +async def test_cancellation_closes_provider_session(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(virustotal.Core, 'virustotal_key', staticmethod(lambda: 'test-key')) + session_exited = False + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + nonlocal session_exited + try: + yield object() + finally: + session_exited = True + + async def fake_fetch(**_kwargs: Any) -> FetcherResponse: + raise asyncio.CancelledError('operator-stop') + + monkeypatch.setattr(virustotal.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(virustotal.AsyncFetcher, 'fetch', fake_fetch) + + with pytest.raises(asyncio.CancelledError, match='operator-stop'): + await virustotal.SearchVirustotal('example.com', limit=10).process() + assert session_exited is True diff --git a/tests/discovery/test_waybackarchive.py b/tests/discovery/test_waybackarchive.py index 200dea90..4f8a090d 100644 --- a/tests/discovery/test_waybackarchive.py +++ b/tests/discovery/test_waybackarchive.py @@ -250,3 +250,6 @@ async def test_process_keeps_an_earlier_failure_when_a_later_pattern_reaches_the assert await search.get_hostnames() == {'example.com'} assert search.execution_status == 'partial' assert search.stop_reason == 'invalid-response' + + +pytestmark = pytest.mark.provider_contract('waybackarchive') diff --git a/tests/discovery/test_whoisxml.py b/tests/discovery/test_whoisxml.py index 30891b2a..33d02fc2 100644 --- a/tests/discovery/test_whoisxml.py +++ b/tests/discovery/test_whoisxml.py @@ -1,28 +1,210 @@ -import logging +from __future__ import annotations + +import asyncio +import contextlib +from typing import TYPE_CHECKING, Any import pytest from theHarvester.discovery import whoisxml +from theHarvester.discovery.constants import MissingKey +from theHarvester.lib.core import FetcherResponse + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + +class ProviderSession: + def __init__(self) -> None: + self.exited = False + + +@pytest.fixture(autouse=True) +def provider_session(monkeypatch: pytest.MonkeyPatch) -> ProviderSession: + session = ProviderSession() + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + try: + yield session + finally: + session.exited = True + + monkeypatch.setattr(whoisxml.AsyncFetcher, 'open_session', fake_open_session) + return session + + +@pytest.mark.provider_contract('whoisxml') +@pytest.mark.asyncio +async def test_response_body_is_not_logged_and_scoped_records_are_returned( + monkeypatch: pytest.MonkeyPatch, + provider_session: ProviderSession, +) -> None: + monkeypatch.setattr(whoisxml.Core, 'whoisxml_key', staticmethod(lambda: 'test-key')) + calls: list[dict[str, Any]] = [] + responses = [ + FetcherResponse( + { + 'secret': 'provider-secret-payload', + 'result': { + 'count': 2, + 'nextPageSearchAfter': 'www.example.com', + 'records': [ + {'domain': 'API.Example.COM.'}, + {'domain': 'outside.test'}, + ], + }, + }, + 200, + {}, + ), + FetcherResponse( + { + 'result': { + 'count': 1, + 'nextPageSearchAfter': '', + 'records': [{'domain': 'www.example.com'}], + }, + }, + 200, + {}, + ), + ] + + async def fake_fetch(**kwargs: Any) -> FetcherResponse: + calls.append(kwargs) + return responses.pop(0) + + monkeypatch.setattr(whoisxml.AsyncFetcher, 'fetch', fake_fetch) + search = whoisxml.SearchWhoisXML('example.com', 3) + await search.process(proxy=True) + + assert await search.get_hostnames() == {'api.example.com', 'www.example.com'} + assert search.execution_status == 'completed' + assert search.stop_reason is None + assert [call['url'] for call in calls] == ['https://subdomains.whoisxmlapi.com/api/v2'] * 2 + assert all(call['session'] is provider_session for call in calls) + assert [call['params'] for call in calls] == [ + {'apiKey': 'test-key', 'domainName': 'example.com'}, + {'apiKey': 'test-key', 'domainName': 'example.com', 'searchAfter': 'www.example.com'}, + ] + assert all(call['json'] is True and call['include_metadata'] is True for call in calls) + assert provider_session.exited is True + + +@pytest.mark.parametrize('key', [None, '', ' ']) +def test_missing_or_blank_key_fails_closed(monkeypatch: pytest.MonkeyPatch, key: str | None) -> None: + monkeypatch.setattr(whoisxml.Core, 'whoisxml_key', staticmethod(lambda: key)) + with pytest.raises(MissingKey): + whoisxml.SearchWhoisXML('example.com', 10) + + +@pytest.mark.parametrize('limit', [0, -1, True, 1.5]) +def test_limit_must_be_positive(monkeypatch: pytest.MonkeyPatch, limit: Any) -> None: + monkeypatch.setattr(whoisxml.Core, 'whoisxml_key', staticmethod(lambda: 'test-key')) + with pytest.raises(ValueError, match='positive integer'): + whoisxml.SearchWhoisXML('example.com', limit) + + +@pytest.mark.parametrize( + ('response', 'status', 'reason'), + [ + (None, 'failed', 'transport-error'), + (FetcherResponse({}, 401, {}), 'failed', 'access-denied'), + (FetcherResponse({}, 429, {}), 'rate-limited', 'http-429'), + (FetcherResponse({}, 503, {}), 'failed', 'http-503'), + (FetcherResponse([], 200, {}), 'failed', 'invalid-response'), + (FetcherResponse({'result': {'records': 'many'}}, 200, {}), 'failed', 'invalid-response'), + ], +) +@pytest.mark.asyncio +async def test_provider_failures_are_truthful( + monkeypatch: pytest.MonkeyPatch, + response: FetcherResponse | None, + status: str, + reason: str, +) -> None: + monkeypatch.setattr(whoisxml.Core, 'whoisxml_key', staticmethod(lambda: 'test-key')) + + async def fake_fetch(**_kwargs: Any) -> FetcherResponse | None: + return response + + monkeypatch.setattr(whoisxml.AsyncFetcher, 'fetch', fake_fetch) + search = whoisxml.SearchWhoisXML('example.com', 10) + await search.process() + + assert search.execution_status == status + assert search.stop_reason == reason @pytest.mark.asyncio -async def test_response_body_is_not_logged_and_records_are_returned(monkeypatch, caplog) -> None: - monkeypatch.setattr(whoisxml.Core, 'whoisxml_key', lambda: 'test-key') - monkeypatch.setattr(whoisxml.Core, 'get_user_agent', lambda: 'test-agent') +async def test_malformed_rows_preserve_valid_partial_results(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(whoisxml.Core, 'whoisxml_key', staticmethod(lambda: 'test-key')) - async def fake_fetch_all(*args, **kwargs): - return [ + async def fake_fetch(**_kwargs: Any) -> FetcherResponse: + return FetcherResponse( { - 'secret': 'provider-secret-payload', - 'result': {'records': [{'domain': 'www.example.com'}]}, - } - ] + 'result': { + 'count': 2, + 'nextPageSearchAfter': '', + 'records': [{'domain': 'ok.example.com'}, {'domain': 7}], + } + }, + 200, + {}, + ) - monkeypatch.setattr(whoisxml.AsyncFetcher, 'fetch_all', fake_fetch_all) - caplog.set_level(logging.INFO, logger=whoisxml.__name__) - - search = whoisxml.SearchWhoisXML('example.com') + monkeypatch.setattr(whoisxml.AsyncFetcher, 'fetch', fake_fetch) + search = whoisxml.SearchWhoisXML('example.com', 10) await search.process() - assert await search.get_hostnames() == ['www.example.com'] - assert 'provider-secret-payload' not in caplog.text + assert await search.get_hostnames() == {'ok.example.com'} + assert search.execution_status == 'partial' + assert search.stop_reason == 'invalid-response' + + +@pytest.mark.asyncio +async def test_later_page_failure_preserves_partial_results(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(whoisxml.Core, 'whoisxml_key', staticmethod(lambda: 'test-key')) + responses = [ + FetcherResponse( + { + 'result': { + 'count': 1, + 'nextPageSearchAfter': 'next.example.com', + 'records': [{'domain': 'api.example.com'}], + } + }, + 200, + {}, + ), + FetcherResponse({}, 429, {}), + ] + + async def fake_fetch(**_kwargs: Any) -> FetcherResponse: + return responses.pop(0) + + monkeypatch.setattr(whoisxml.AsyncFetcher, 'fetch', fake_fetch) + search = whoisxml.SearchWhoisXML('example.com', 10) + await search.process() + + assert await search.get_hostnames() == {'api.example.com'} + assert search.execution_status == 'partial' + assert search.stop_reason == 'http-429' + + +@pytest.mark.asyncio +async def test_cancellation_closes_provider_session( + monkeypatch: pytest.MonkeyPatch, + provider_session: ProviderSession, +) -> None: + monkeypatch.setattr(whoisxml.Core, 'whoisxml_key', staticmethod(lambda: 'test-key')) + + async def fake_fetch(**_kwargs: Any) -> FetcherResponse: + raise asyncio.CancelledError('operator-stop') + + monkeypatch.setattr(whoisxml.AsyncFetcher, 'fetch', fake_fetch) + + with pytest.raises(asyncio.CancelledError, match='operator-stop'): + await whoisxml.SearchWhoisXML('example.com', 10).process() + assert provider_session.exited is True diff --git a/tests/discovery/test_windvane.py b/tests/discovery/test_windvane.py index 66a962d8..161da274 100644 --- a/tests/discovery/test_windvane.py +++ b/tests/discovery/test_windvane.py @@ -79,3 +79,6 @@ async def test_keyless_provider_failure_does_not_guess_dns_names(monkeypatch) -> assert requests == 1 assert await search.get_hostnames() == set() assert await search.get_ips() == set() + + +pytestmark = pytest.mark.provider_contract('windvane') diff --git a/tests/discovery/test_yahoosearch.py b/tests/discovery/test_yahoosearch.py index e6438218..dcf48f0d 100644 --- a/tests/discovery/test_yahoosearch.py +++ b/tests/discovery/test_yahoosearch.py @@ -61,3 +61,6 @@ async def test_yahoo_unusable_responses_return_no_evidence( assert await search.get_emails() == [] assert await search.get_hostnames() == [] + + +pytestmark = pytest.mark.provider_contract('yahoo') diff --git a/tests/discovery/test_zoomeyesearch.py b/tests/discovery/test_zoomeyesearch.py index 224f597e..c655298e 100644 --- a/tests/discovery/test_zoomeyesearch.py +++ b/tests/discovery/test_zoomeyesearch.py @@ -1,6 +1,210 @@ +import asyncio +import base64 +import contextlib +from collections.abc import AsyncIterator +from typing import Any + import pytest from theHarvester.discovery import zoomeyesearch +from theHarvester.discovery.constants import MissingKey +from theHarvester.lib.core import FetcherResponse + + +@pytest.mark.provider_contract('zoomeye') +@pytest.mark.asyncio +async def test_process_reuses_session_and_collects_all_capabilities(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(zoomeyesearch.Core, 'zoomeye_key', staticmethod(lambda: 'test-key')) + session = object() + session_exited = False + calls: list[dict[str, Any]] = [] + responses = [ + FetcherResponse( + { + 'code': 60000, + 'total': 1, + 'data': [ + { + 'ip': '192.0.2.1', + 'domain': 'api.example.com', + 'hostname': 'host.example.com', + 'asn': 64500, + 'url': 'https://portal.example.com/v1', + 'banner': 'admin@example.com', + } + ], + }, + 200, + {}, + ) + ] + + @contextlib.asynccontextmanager + async def fake_open_session(**kwargs: Any) -> AsyncIterator[object]: + nonlocal session_exited + assert kwargs == { + 'headers': {'API-KEY': 'test-key', 'Content-Type': 'application/json'}, + 'proxy': True, + } + try: + yield session + finally: + session_exited = True + + async def fake_post_fetch(*args: Any, **kwargs: Any) -> FetcherResponse: + calls.append({'url': args[0], **kwargs}) + return responses.pop(0) + + monkeypatch.setattr(zoomeyesearch.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(zoomeyesearch.AsyncFetcher, 'post_fetch', fake_post_fetch) + search = zoomeyesearch.SearchZoomEye('example.com', 2) + await search.process(proxy=True) + + assert await search.get_hostnames() == {'api.example.com', 'host.example.com', 'portal.example.com'} + assert await search.get_ips() == {'192.0.2.1'} + assert await search.get_asns() == {'AS64500'} + assert await search.get_urls() == {'https://portal.example.com/v1'} + assert await search.get_emails() == {'admin@example.com'} + assert search.execution_status == 'completed' + assert search.stop_reason is None + assert [call['url'] for call in calls] == [search.baseurl] + assert all(call['session'] is session for call in calls) + assert session_exited is True + assert base64.b64decode(calls[0]['json_body']['qbase64']).decode() == 'domain="example.com"' + assert calls[0]['json_body']['page'] == 1 + assert calls[0]['json_body']['pagesize'] == 2 + + +@pytest.mark.parametrize('key', [None, '', ' ']) +def test_missing_or_blank_key_fails_closed(monkeypatch: pytest.MonkeyPatch, key: str | None) -> None: + monkeypatch.setattr(zoomeyesearch.Core, 'zoomeye_key', staticmethod(lambda: key)) + with pytest.raises(MissingKey): + zoomeyesearch.SearchZoomEye('example.com', 1) + + +@pytest.mark.parametrize('limit', [0, -1, True, 1.5]) +def test_limit_must_be_a_positive_integer(monkeypatch: pytest.MonkeyPatch, limit: Any) -> None: + monkeypatch.setattr(zoomeyesearch.Core, 'zoomeye_key', staticmethod(lambda: 'test-key')) + with pytest.raises(ValueError, match='positive integer'): + zoomeyesearch.SearchZoomEye('example.com', limit) + + +@pytest.mark.parametrize( + ('response', 'status', 'reason'), + [ + (None, 'failed', 'transport-error'), + (FetcherResponse({}, 401, {}), 'failed', 'access-denied'), + (FetcherResponse({}, 429, {}), 'rate-limited', 'http-429'), + (FetcherResponse({}, 503, {}), 'failed', 'http-503'), + (FetcherResponse([], 200, {}), 'failed', 'invalid-response'), + (FetcherResponse({'code': 60001}, 200, {}), 'failed', 'provider-error'), + ], +) +@pytest.mark.asyncio +async def test_provider_failures_are_truthful( + monkeypatch: pytest.MonkeyPatch, + response: FetcherResponse | None, + status: str, + reason: str, +) -> None: + monkeypatch.setattr(zoomeyesearch.Core, 'zoomeye_key', staticmethod(lambda: 'test-key')) + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + yield object() + + async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse | None: + return response + + monkeypatch.setattr(zoomeyesearch.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(zoomeyesearch.AsyncFetcher, 'post_fetch', fake_post_fetch) + search = zoomeyesearch.SearchZoomEye('example.com', 1) + await search.process() + + assert search.execution_status == status + assert search.stop_reason == reason + + +@pytest.mark.asyncio +async def test_empty_pages_do_not_hide_later_provider_results(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(zoomeyesearch.Core, 'zoomeye_key', staticmethod(lambda: 'test-key')) + responses = [ + FetcherResponse( + { + 'code': 60000, + 'total': 70_000, + 'data': ([{'hostname': 'late.example.com'}] if page == 6 else []), + }, + 200, + {}, + ) + for page in range(1, 8) + ] + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + yield object() + + async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse: + return responses.pop(0) + + monkeypatch.setattr(zoomeyesearch.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(zoomeyesearch.AsyncFetcher, 'post_fetch', fake_post_fetch) + search = zoomeyesearch.SearchZoomEye('example.com', 70_000) + await search.process() + + assert await search.get_hostnames() == {'late.example.com'} + assert responses == [] + assert search.execution_status == 'completed' + + +@pytest.mark.asyncio +async def test_numbered_pages_keep_a_stable_size_and_slice_the_final_page(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(zoomeyesearch.Core, 'zoomeye_key', staticmethod(lambda: 'test-key')) + calls: list[dict[str, Any]] = [] + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + yield object() + + async def fake_post_fetch(*_args: Any, **kwargs: Any) -> FetcherResponse: + calls.append(kwargs['json_body']) + return FetcherResponse({'code': 60000, 'total': 10_005, 'data': []}, 200, {}) + + monkeypatch.setattr(zoomeyesearch.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(zoomeyesearch.AsyncFetcher, 'post_fetch', fake_post_fetch) + search = zoomeyesearch.SearchZoomEye('example.com', 10_005) + + await search.process() + + assert [(call['page'], call['pagesize']) for call in calls] == [(1, 10_000), (2, 10_000)] + + +@pytest.mark.asyncio +async def test_early_malformed_rows_and_later_valid_evidence_are_partial(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(zoomeyesearch.Core, 'zoomeye_key', staticmethod(lambda: 'test-key')) + monkeypatch.setattr(zoomeyesearch.SearchZoomEye, 'PAGE_SIZE', 1) + responses = [ + FetcherResponse({'code': 60000, 'total': 2, 'data': [7]}, 200, {}), + FetcherResponse({'code': 60000, 'total': 2, 'data': [{'hostname': 'api.example.com'}]}, 200, {}), + ] + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + yield object() + + async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse: + return responses.pop(0) + + monkeypatch.setattr(zoomeyesearch.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(zoomeyesearch.AsyncFetcher, 'post_fetch', fake_post_fetch) + search = zoomeyesearch.SearchZoomEye('example.com', 2) + + await search.process() + + assert await search.get_hostnames() == {'api.example.com'} + assert search.execution_status == 'partial' + assert search.stop_reason == 'invalid-response' @pytest.mark.asyncio @@ -18,6 +222,31 @@ async def test_banner_urls_are_absolute_http_and_scoped(monkeypatch: pytest.Monk ) ) - _hostnames, _emails, _ips, _asns, urls = await search.parse_matches([{'service': {'banner': banner}}]) + _hostnames, _emails, _ips, _asns, urls, malformed = await search.parse_matches([{'banner': banner}]) assert urls == {'https://api.example.com/v1'} + assert malformed is False + + +@pytest.mark.asyncio +async def test_cancellation_closes_provider_session(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(zoomeyesearch.Core, 'zoomeye_key', staticmethod(lambda: 'test-key')) + session_exited = False + + @contextlib.asynccontextmanager + async def fake_open_session(**_kwargs: Any) -> AsyncIterator[object]: + nonlocal session_exited + try: + yield object() + finally: + session_exited = True + + async def fake_post_fetch(*_args: Any, **_kwargs: Any) -> FetcherResponse: + raise asyncio.CancelledError('operator-stop') + + monkeypatch.setattr(zoomeyesearch.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(zoomeyesearch.AsyncFetcher, 'post_fetch', fake_post_fetch) + + with pytest.raises(asyncio.CancelledError, match='operator-stop'): + await zoomeyesearch.SearchZoomEye('example.com', 10).process() + assert session_exited is True diff --git a/tests/lib/test_core.py b/tests/lib/test_core.py index e9e81745..a7a0e6ba 100644 --- a/tests/lib/test_core.py +++ b/tests/lib/test_core.py @@ -1127,6 +1127,26 @@ async def test_fetch_all_propagates_metadata_opt_in(monkeypatch) -> None: assert [result.status for result in results] == [429, 429] +@pytest.mark.asyncio +async def test_fetch_all_reuses_a_caller_owned_session(monkeypatch: pytest.MonkeyPatch) -> None: + session = object() + seen_sessions: list[object] = [] + + async def fake_fetch(*_args: Any, session: object, **_kwargs: Any) -> str: + seen_sessions.append(session) + return 'ok' + + monkeypatch.setattr(AsyncFetcher, 'fetch', fake_fetch) + + results = await AsyncFetcher.fetch_all( + ['https://one.example', 'https://two.example'], + session=session, + ) + + assert results == ['ok', 'ok'] + assert seen_sessions == [session, session] + + @pytest.mark.asyncio async def test_fetch_uses_http_proxy_when_enabled(monkeypatch) -> None: reset_dummy_sessions() diff --git a/tests/lib/test_source_runner.py b/tests/lib/test_source_runner.py index 3bf335e1..3e9c5ae2 100644 --- a/tests/lib/test_source_runner.py +++ b/tests/lib/test_source_runner.py @@ -74,7 +74,7 @@ def test_source_factories_match_the_catalog() -> None: ), ('duckduckgo', 'theHarvester.lib.source_runner.duckduckgosearch.SearchDuckDuckGo', ('example.test', 25), {}), ('dymo', 'theHarvester.lib.source_runner.dymosearch.SearchDymo', ('example.test',), {}), - ('fofa', 'theHarvester.lib.source_runner.fofa.SearchFofa', ('example.test',), {}), + ('fofa', 'theHarvester.lib.source_runner.fofa.SearchFofa', ('example.test', 25), {}), ('fullhunt', 'theHarvester.lib.source_runner.fullhuntsearch.SearchFullHunt', ('example.test',), {}), ('github-code', 'theHarvester.lib.source_runner.githubcode.SearchGithubCode', ('example.test', 25), {}), ('gitlab', 'theHarvester.lib.source_runner.gitlabsearch.SearchGitlab', ('example.test',), {}), @@ -98,13 +98,13 @@ def test_source_factories_match_the_catalog() -> None: ), ('hudsonrock', 'theHarvester.lib.source_runner.hudsonrocksearch.SearchHudsonRock', ('example.test',), {}), ('hunter', 'theHarvester.lib.source_runner.huntersearch.SearchHunter', ('example.test', 25, 5), {}), - ('hunterhow', 'theHarvester.lib.source_runner.searchhunterhow.SearchHunterHow', ('example.test',), {}), + ('hunterhow', 'theHarvester.lib.source_runner.searchhunterhow.SearchHunterHow', ('example.test', 25), {}), ('intelx', 'theHarvester.lib.source_runner.intelxsearch.SearchIntelx', ('example.test',), {}), ('leakix', 'theHarvester.lib.source_runner.leakix.SearchLeakix', ('example.test',), {}), ('leaklookup', 'theHarvester.lib.source_runner.leaklookup.SearchLeakLookup', ('example.test',), {}), ('mojeek', 'theHarvester.lib.source_runner.mojeek.SearchMojeek', ('example.test', 25), {}), ('netlas', 'theHarvester.lib.source_runner.netlas.SearchNetlas', ('example.test', 25), {}), - ('onyphe', 'theHarvester.lib.source_runner.onyphe.SearchOnyphe', ('example.test',), {}), + ('onyphe', 'theHarvester.lib.source_runner.onyphe.SearchOnyphe', ('example.test', 25), {}), ('otx', 'theHarvester.lib.source_runner.otxsearch.SearchOtx', ('example.test',), {}), ( 'pentesttools', @@ -130,7 +130,7 @@ def test_source_factories_match_the_catalog() -> None: ( 'securityscorecard', 'theHarvester.lib.source_runner.securityscorecard.SearchSecurityScorecard', - ('example.test',), + ('example.test', 25), {}, ), ( @@ -162,15 +162,15 @@ def test_source_factories_match_the_catalog() -> None: ), ('thc', 'theHarvester.lib.source_runner.thc.SearchThc', ('example.test',), {}), ('tomba', 'theHarvester.lib.source_runner.tombasearch.SearchTomba', ('example.test', 25, 5), {}), - ('urlscan', 'theHarvester.lib.source_runner.urlscan.SearchUrlscan', ('example.test',), {}), - ('virustotal', 'theHarvester.lib.source_runner.virustotal.SearchVirustotal', ('example.test',), {}), + ('urlscan', 'theHarvester.lib.source_runner.urlscan.SearchUrlscan', ('example.test', 25), {}), + ('virustotal', 'theHarvester.lib.source_runner.virustotal.SearchVirustotal', ('example.test', 25), {}), ( 'waybackarchive', 'theHarvester.lib.source_runner.waybackarchive.SearchWaybackarchive', ('example.test', 25), {}, ), - ('whoisxml', 'theHarvester.lib.source_runner.whoisxml.SearchWhoisXML', ('example.test',), {}), + ('whoisxml', 'theHarvester.lib.source_runner.whoisxml.SearchWhoisXML', ('example.test', 25), {}), ('windvane', 'theHarvester.lib.source_runner.windvane.SearchWindvane', ('example.test',), {}), ('yahoo', 'theHarvester.lib.source_runner.yahoosearch.SearchYahoo', ('example.test', 25), {}), ('zoomeye', 'theHarvester.lib.source_runner.zoomeyesearch.SearchZoomEye', ('example.test', 25), {}), diff --git a/tests/test_hackertarget_apikey.py b/tests/test_hackertarget_apikey.py index 5a0fae50..715a84d9 100644 --- a/tests/test_hackertarget_apikey.py +++ b/tests/test_hackertarget_apikey.py @@ -291,3 +291,6 @@ class TestHackerTargetApiKey: with pytest.raises(asyncio.CancelledError): await search.process(proxy=True) + + +pytestmark = pytest.mark.provider_contract('hackertarget') diff --git a/tests/test_main.py b/tests/test_main.py index 3c495c4f..0dbb52f2 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1231,8 +1231,9 @@ async def test_rest_dns_lookup_runs_before_return_and_retains_action_evidence(mo completed.append(result) class FakeSecurityScorecard: - def __init__(self, domain: str) -> None: + def __init__(self, domain: str, limit: int) -> None: assert domain == 'example.com' + assert limit == 500 async def process(self, _proxy: bool) -> None: return None @@ -1315,8 +1316,8 @@ async def test_dns_lookup_cancellation_persists_partial_evidence( completed.append(result) class FakeSecurityScorecard: - def __init__(self, _domain: str) -> None: - pass + def __init__(self, _domain: str, limit: int) -> None: + assert limit == 500 async def process(self, _proxy: bool) -> None: return None @@ -3830,8 +3831,8 @@ async def test_routeviews_pivots_from_attributed_ips_without_expanding_discovere calls: list[tuple[tuple[object, ...], tuple[str, ...]]] = [] class FakeUrlscan: - def __init__(self, _word: str) -> None: - pass + def __init__(self, _word: str, limit: int) -> None: + assert limit == 500 async def process(self, _proxy: bool) -> None: return None diff --git a/tests/test_mojeek.py b/tests/test_mojeek.py index dddbb5b0..2cc5796b 100644 --- a/tests/test_mojeek.py +++ b/tests/test_mojeek.py @@ -207,3 +207,6 @@ class TestMojeekSearch: } assert search.execution_status == 'completed' assert search.stop_reason is None + + +pytestmark = pytest.mark.provider_contract('mojeek') diff --git a/tests/test_provider_contract_coverage.py b/tests/test_provider_contract_coverage.py new file mode 100644 index 00000000..28e5229d --- /dev/null +++ b/tests/test_provider_contract_coverage.py @@ -0,0 +1,34 @@ +from collections import Counter + +from theHarvester.lib.source_catalog import SOURCE_SPECS + + +def _provider_contract_problems( + sources: tuple[str, ...], + canonical_sources: set[str], +) -> list[str]: + counts = Counter(sources) + marked_sources = set(counts) + problems: list[str] = [] + if unknown := sorted(marked_sources - canonical_sources): + problems.append(f'unknown provider contracts: {", ".join(unknown)}') + if duplicates := sorted(source for source, count in counts.items() if count > 1): + problems.append(f'duplicate provider contracts: {", ".join(duplicates)}') + if missing := sorted(canonical_sources - marked_sources): + problems.append(f'missing provider contracts: {", ".join(missing)}') + return problems + + +def test_every_canonical_source_has_one_offline_provider_contract( + provider_contract_sources: tuple[str, ...], +) -> None: + problems = _provider_contract_problems(provider_contract_sources, set(SOURCE_SPECS)) + assert not problems, '; '.join(problems) + + +def test_provider_contract_failures_name_unknown_duplicate_and_missing_sources() -> None: + assert _provider_contract_problems(('known', 'known', 'unknown'), {'known', 'missing'}) == [ + 'unknown provider contracts: unknown', + 'duplicate provider contracts: known', + 'missing provider contracts: missing', + ] diff --git a/tests/test_workflow_contract.py b/tests/test_workflow_contract.py index dcfd5efa..4285894c 100644 --- a/tests/test_workflow_contract.py +++ b/tests/test_workflow_contract.py @@ -25,6 +25,8 @@ def test_routine_ci_is_read_only_and_offline() -> None: assert 'git push' not in commands assert 'theHarvester -d' not in commands assert '\npytest\n' in f'\n{commands.strip()}\n' + assert 'mypy theHarvester' in commands + assert routine_job['strategy']['matrix']['python-version'] == ['3.12', '3.13', '3.14'] def test_live_provider_smoke_requires_manual_dispatch() -> None: @@ -36,6 +38,9 @@ def test_live_provider_smoke_requires_manual_dispatch() -> None: assert workflow['permissions'] == {'contents': 'read'} assert smoke_job['env']['SMOKE_TEST_DOMAIN'] == 'mozilla.org' assert 'pytest --run-live-network -m live_network' in commands + cli_smokes = [line for line in commands.splitlines() if line.startswith('theHarvester -d')] + assert cli_smokes + assert all('-l 10 -q' in command for command in cli_smokes) def test_harvestview_browser_failures_keep_only_targeted_diagnostics() -> None: diff --git a/theHarvester/discovery/bevigil.py b/theHarvester/discovery/bevigil.py index af52b72b..48bf1e53 100644 --- a/theHarvester/discovery/bevigil.py +++ b/theHarvester/discovery/bevigil.py @@ -1,37 +1,106 @@ +from urllib.parse import urlsplit, urlunsplit + from theHarvester.discovery.constants import MissingKey -from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.discovery.provider_response import provider_http_error +from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse +from theHarvester.lib.hostnames import normalize_scoped_hostname class SearchBeVigil: - def __init__(self, word) -> None: + def __init__(self, word: str) -> None: self.word = word - self.totalhosts: set = set() - self.urls: set = set() + self.totalhosts: set[str] = set() + self.urls: set[str] = set() self.key = Core.bevigil_key() - if self.key is None: - self.key = '' + if not isinstance(self.key, str) or not self.key.strip(): raise MissingKey('bevigil') self.proxy = False + self.execution_status: str | None = None + self.stop_reason: str | None = None + + def _has_results(self) -> bool: + return bool(self.totalhosts or self.urls) + + def _stop(self, status: str, reason: str) -> None: + self.execution_status = 'partial' if self._has_results() else status + self.stop_reason = reason + + def _scoped_url(self, value: object) -> str | None: + if not isinstance(value, str): + return None + try: + parsed = urlsplit(value.strip()) + hostname = normalize_scoped_hostname(parsed.hostname, self.word) + port = parsed.port + except ValueError: + return None + if parsed.scheme.casefold() not in {'http', 'https'} or hostname is None or parsed.username or parsed.password: + return None + netloc = f'{hostname}:{port}' if port is not None else hostname + return urlunsplit((parsed.scheme.casefold(), netloc, parsed.path, parsed.query, '')) async def do_search(self) -> None: + self.execution_status = None + self.stop_reason = None subdomain_endpoint = f'https://osint.bevigil.com/api/{self.word}/subdomains/' url_endpoint = f'https://osint.bevigil.com/api/{self.word}/urls/' headers = {'X-Access-Token': self.key} + requests = ( + (subdomain_endpoint, 'subdomains'), + (url_endpoint, 'urls'), + ) - responses = await AsyncFetcher.fetch_all([subdomain_endpoint], json=True, proxy=self.proxy, headers=headers) - response = responses[0] - for subdomain in response['subdomains']: - self.totalhosts.add(subdomain) + try: + async with AsyncFetcher.open_session( + headers=headers, + proxy=self.proxy, + request_timeout=60, + ) as session: + for endpoint, field in requests: + responses = await AsyncFetcher.fetch_all( + [endpoint], + json=True, + proxy=self.proxy, + headers=headers, + include_metadata=True, + session=session, + ) + response = responses[0] if responses else None + if error := provider_http_error(response): + self._stop(*error) + return + assert isinstance(response, FetcherResponse) + if not isinstance(response.body, dict) or not isinstance(response.body.get(field), list): + self._stop('failed', 'invalid-response') + return - responses = await AsyncFetcher.fetch_all([url_endpoint], json=True, proxy=self.proxy, headers=headers) - response = responses[0] - for url in response['urls']: - self.urls.add(url) + malformed = False + for value in response.body[field]: + if field == 'subdomains': + if not isinstance(value, str): + malformed = True + elif hostname := normalize_scoped_hostname(value, self.word): + self.totalhosts.add(hostname) + elif url := self._scoped_url(value): + self.urls.add(url) + elif not isinstance(value, str): + malformed = True + if malformed: + self._stop('failed', 'invalid-response') + except Exception: + self._stop('failed', 'transport-error') + return - async def get_hostnames(self) -> set: + if self.execution_status is not None and self._has_results(): + self.execution_status = 'partial' + elif self.execution_status is None: + self.execution_status = 'completed' + self.stop_reason = None if self._has_results() else 'no-results' + + async def get_hostnames(self) -> set[str]: return self.totalhosts - async def get_urls(self) -> set: + async def get_urls(self) -> set[str]: return self.urls async def process(self, proxy: bool = False) -> None: diff --git a/theHarvester/discovery/dymosearch.py b/theHarvester/discovery/dymosearch.py index 5bcfaaac..ec49923d 100644 --- a/theHarvester/discovery/dymosearch.py +++ b/theHarvester/discovery/dymosearch.py @@ -1,7 +1,9 @@ from typing import Any from theHarvester.discovery.constants import MissingKey -from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.discovery.provider_response import provider_http_error +from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse +from theHarvester.lib.hostnames import normalize_scoped_hostname class SearchDymo: @@ -24,9 +26,15 @@ class SearchDymo: self.totalhosts: set[str] = set() self.results: dict[str, Any] = {} self.key = Core.dymo_key() - if self.key is None: + if not isinstance(self.key, str) or not self.key.strip(): raise MissingKey('dymo') self.proxy = False + self.execution_status: str | None = None + self.stop_reason: str | None = None + + def _stop(self, status: str, reason: str) -> None: + self.execution_status = 'partial' if self.totalhosts else status + self.stop_reason = reason def _headers(self) -> dict[str, str]: return { @@ -43,26 +51,45 @@ class SearchDymo: response = await AsyncFetcher.post_fetch( self.VERIFY_URL, headers=self._headers(), - data=payload, json=True, + json_body=payload, proxy=self.proxy, + include_metadata=True, ) - if not isinstance(response, dict): + if error := provider_http_error(response): + self._stop(*error) + return + assert isinstance(response, FetcherResponse) + if not isinstance(response.body, dict): + self._stop('failed', 'invalid-response') return - self.results = response + self.results = response.body - domain_block = response.get('domain') if isinstance(response.get('domain'), dict) else {} - url_block = response.get('url') if isinstance(response.get('url'), dict) else {} + raw_domain_block = response.body.get('domain') + raw_url_block = response.body.get('url') + malformed = any(block is not None and not isinstance(block, dict) for block in (raw_domain_block, raw_url_block)) + domain_block = raw_domain_block if isinstance(raw_domain_block, dict) else {} + url_block = raw_url_block if isinstance(raw_url_block, dict) else {} for block in (domain_block, url_block): candidate = block.get('domain') if isinstance(block, dict) else None - if isinstance(candidate, str) and self.word in candidate: - self.totalhosts.add(candidate) + if normalized := normalize_scoped_hostname(candidate, self.word): + self.totalhosts.add(normalized) + elif candidate is not None and not isinstance(candidate, str): + malformed = True suggestion = block.get('didYouMean') if isinstance(block, dict) else None - if isinstance(suggestion, str) and self.word in suggestion: - self.totalhosts.add(suggestion) + if normalized := normalize_scoped_hostname(suggestion, self.word): + self.totalhosts.add(normalized) + elif suggestion is not None and not isinstance(suggestion, str): + malformed = True + + if malformed: + self._stop('failed', 'invalid-response') + else: + self.execution_status = 'completed' + self.stop_reason = None if self.totalhosts else 'no-results' async def get_hostnames(self) -> set: return self.totalhosts @@ -72,4 +99,9 @@ class SearchDymo: async def process(self, proxy: bool = False) -> None: self.proxy = proxy - await self.do_search() + self.execution_status = None + self.stop_reason = None + try: + await self.do_search() + except Exception: + self._stop('failed', 'transport-error') diff --git a/theHarvester/discovery/fofa.py b/theHarvester/discovery/fofa.py index b890ada0..f8f29af7 100644 --- a/theHarvester/discovery/fofa.py +++ b/theHarvester/discovery/fofa.py @@ -1,30 +1,33 @@ import base64 -import logging from ipaddress import ip_address +from typing import Any from urllib.parse import urlparse from theHarvester.discovery.constants import MissingKey +from theHarvester.discovery.provider_response import provider_http_error from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse from theHarvester.lib.hostnames import normalize_scoped_hostname -logger = logging.getLogger(__name__) - class SearchFofa: - """Class uses Fofa API to search for domain and host intelligence - Fofa is a Chinese search engine for network-connected devices - """ + """Collect scoped domain assets through FOFA's cursor search API.""" - def __init__(self, word) -> None: + MAX_PAGE_SIZE = 10_000 + + def __init__(self, word: str, limit: int) -> None: + if isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0: + raise ValueError('FOFA limit must be a positive integer') self.word = word - self.totalhosts: set = set() - self.totalips: set = set() + self.limit = limit + self.totalhosts: set[str] = set() + self.totalips: set[str] = set() self.proxy = False self.hostname = 'https://fofa.info' self.api_key, self.email = self._get_api_credentials() + self.execution_status: str | None = None + self.stop_reason: str | None = None def _get_api_credentials(self) -> tuple[str, str]: - """Get Fofa API credentials""" try: api_key, email = Core.fofa_key() except Exception as error: @@ -33,99 +36,122 @@ class SearchFofa: raise MissingKey('Fofa API (key and email required)') return api_key, email + def _has_results(self) -> bool: + return bool(self.totalhosts or self.totalips) + + def _stop(self, status: str, reason: str) -> None: + self.execution_status = 'partial' if self._has_results() else status + self.stop_reason = reason + + def _store_results(self, results: list[Any]) -> bool: + malformed = False + for result in results: + if not isinstance(result, list) or len(result) < 2: + malformed = True + continue + host, address = result[:2] + if isinstance(host, str): + try: + parsed_hostname = urlparse(host if '://' in host else f'//{host}').hostname + except ValueError: + malformed = True + else: + if clean_host := normalize_scoped_hostname(parsed_hostname, self.word): + self.totalhosts.add(clean_host) + else: + malformed = True + if isinstance(address, str): + try: + self.totalips.add(str(ip_address(address))) + except ValueError: + malformed = True + else: + malformed = True + return malformed + + def _provider_error(self, body: dict[str, Any]) -> None: + message = body.get('errmsg') + normalized = message.casefold() if isinstance(message, str) else '' + if 'invalid' in normalized or '账号无效' in normalized: + self._stop('failed', 'access-denied') + elif any(term in normalized for term in ('quota', 'limit', 'plan')): + self._stop('failed', 'quota-exhausted') + else: + self._stop('failed', 'provider-error') + async def do_search(self) -> None: + query = base64.b64encode(f'domain="{self.word}"'.encode()).decode() + cursor: str | None = None + seen_cursors: set[str] = set() + records_seen = 0 try: - headers = {'User-agent': Core.get_user_agent()} - - # Fofa search query - encode in base64 - query = f'domain="{self.word}"' - query_encoded = base64.b64encode(query.encode()).decode() - - # Fofa API endpoint - url = f'{self.hostname}/api/v1/search/all' - params = { - 'email': self.email, - 'key': self.api_key, - 'qbase64': query_encoded, - 'fields': 'host,ip,port,protocol,title', - 'size': 100, # Limit results - } - - # Build URL with parameters - param_string = '&'.join([f'{k}={v}' for k, v in params.items()]) - full_url = f'{url}?{param_string}' - - response = await AsyncFetcher.fetch_all( - [full_url], - headers=headers, + async with AsyncFetcher.open_session( + headers={'User-Agent': Core.get_user_agent()}, proxy=self.proxy, - json=True, - include_metadata=True, - ) + ) as session: + while records_seen < self.limit: + remaining = self.limit - records_seen + params: dict[str, str | int] = { + 'email': self.email, + 'key': self.api_key, + 'qbase64': query, + 'fields': 'host,ip', + 'size': min(self.MAX_PAGE_SIZE, remaining), + } + if cursor is not None: + params['next'] = cursor + response = await AsyncFetcher.fetch( + session=session, + url=f'{self.hostname}/api/v1/search/next', + params=params, + json=True, + include_metadata=True, + ) + if error := provider_http_error(response): + self._stop(*error) + return + assert isinstance(response, FetcherResponse) + if not isinstance(response.body, dict): + self._stop('failed', 'invalid-response') + return + if response.body.get('error') is True: + self._provider_error(response.body) + return + results = response.body.get('results') + if not isinstance(results, list): + self._stop('failed', 'invalid-response') + return - metadata = response[0] if response and isinstance(response[0], FetcherResponse) else None - if metadata is None: - logger.info(f'No response from Fofa API for: {self.word}') - return - if not 200 <= metadata.status < 300: - logger.info(f'Fofa request failed with HTTP {metadata.status}') - return + page_results = results[:remaining] + records_seen += len(page_results) + if self._store_results(page_results): + self._stop('failed', 'invalid-response') + next_cursor = response.body.get('next') + if not results or not isinstance(next_cursor, str) or not next_cursor: + break + if next_cursor in seen_cursors: + self._stop('failed', 'repeated-cursor') + break + seen_cursors.add(next_cursor) + cursor = next_cursor + except Exception: + self._stop('failed', 'transport-error') + return - try: - data = metadata.body - if not isinstance(data, dict): - logger.info('Fofa returned malformed data') - return + if self.execution_status is not None and self._has_results(): + self.execution_status = 'partial' + elif self.execution_status is None: + self.execution_status = 'completed' + self.stop_reason = None if self._has_results() else 'no-results' - # Check for errors - if data.get('error', False): - error_message = data.get('errmsg') - normalized_error = error_message.casefold() if isinstance(error_message, str) else '' - if 'invalid' in normalized_error or '账号无效' in normalized_error: - logger.info('Fofa API rejected the configured credentials') - elif any(term in normalized_error for term in ('quota', 'limit', 'plan')): - logger.info('Fofa API quota or plan limit was reached') - else: - logger.info('Fofa API returned an error') - return - - # Extract results - results = data.get('results', []) - if not isinstance(results, list): - logger.info('Fofa returned malformed results') - return - for result in results: - if isinstance(result, list) and len(result) >= 2: - host = result[0] # host field - ip = result[1] # ip field - - # Add host if it's related to our domain - if isinstance(host, str): - parsed = urlparse(host if '://' in host else f'//{host}') - if clean_host := normalize_scoped_hostname(parsed.hostname, self.word): - self.totalhosts.add(clean_host) - - # Add IP - if isinstance(ip, str) and ip: - try: - self.totalips.add(str(ip_address(ip))) - except ValueError: - continue - - except Exception as e: - logger.info(f'Failed to parse Fofa response: {e}') - - except MissingKey: - raise - except Exception as e: - logger.info(f'Fofa API error: {e}') - - async def get_hostnames(self) -> set: + async def get_hostnames(self) -> set[str]: return self.totalhosts - async def get_ips(self) -> set: + async def get_ips(self) -> set[str]: return self.totalips async def process(self, proxy: bool = False) -> None: self.proxy = proxy + self.execution_status = None + self.stop_reason = None await self.do_search() diff --git a/theHarvester/discovery/fullhuntsearch.py b/theHarvester/discovery/fullhuntsearch.py index a8d706d9..f9b65a31 100644 --- a/theHarvester/discovery/fullhuntsearch.py +++ b/theHarvester/discovery/fullhuntsearch.py @@ -4,6 +4,7 @@ from typing import Any, ClassVar from urllib.parse import quote from theHarvester.discovery.constants import MissingKey +from theHarvester.discovery.provider_response import provider_http_error from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse from theHarvester.lib.hostnames import normalize_scoped_hostname @@ -140,12 +141,21 @@ class SearchFullHunt: } self.proxy = False self.filters: dict[str, str] = {} # Store filters for advanced searches + self.execution_status: str | None = None + self.stop_reason: str | None = None + + def _has_results(self) -> bool: + return bool(self.total_results['hosts'] or self.total_results['ips']) + + def _stop(self, status: str, reason: str) -> None: + self.execution_status = 'partial' if self._has_results() else status + self.stop_reason = reason def _get_headers(self) -> dict[str, str]: """Returns the headers needed for API requests""" return {'User-Agent': Core.get_user_agent(), 'X-API-KEY': self.key} - async def _fetch_data(self, endpoint: str) -> dict[str, Any]: + async def _fetch_data(self, endpoint: str, session: Any | None = None) -> dict[str, Any]: """Generic method to fetch data from a specific endpoint""" url = f'{self.BASE_URL}/{endpoint}' response = await AsyncFetcher.fetch_all( @@ -154,13 +164,15 @@ class SearchFullHunt: headers=self._get_headers(), proxy=self.proxy, include_metadata=True, + session=session, ) - metadata = response[0] if response and isinstance(response[0], FetcherResponse) else None - if metadata is None: - raise RuntimeError('FullHunt request failed') - if not 200 <= metadata.status < 300: - raise RuntimeError(f'FullHunt request failed with HTTP {metadata.status}') + metadata = response[0] if response else None + if error := provider_http_error(metadata): + self._stop(*error) + raise RuntimeError(f'FullHunt request failed: {error[1]}') + assert isinstance(metadata, FetcherResponse) if not isinstance(metadata.body, dict): + self._stop('failed', 'invalid-response') raise ValueError('FullHunt returned malformed data') return metadata.body @@ -213,7 +225,7 @@ class SearchFullHunt: return ' '.join(query_parts) - async def advanced_search(self) -> dict[str, Any]: + async def advanced_search(self, session: Any | None = None) -> dict[str, Any]: """Perform an advanced search using the configured filters This method uses the search endpoint with the filters configured via add_filter @@ -226,17 +238,17 @@ class SearchFullHunt: query = self._build_query_string() encoded_query = quote(query) endpoint = f'search?query={encoded_query}' - return await self._fetch_data(endpoint) + return await self._fetch_data(endpoint, session) - async def get_domain_details(self) -> dict[str, Any]: + async def get_domain_details(self, session: Any | None = None) -> dict[str, Any]: """Get comprehensive details about a domain""" endpoint = f'domain/{self.word}/details' - return await self._fetch_data(endpoint) + return await self._fetch_data(endpoint, session) - async def get_subdomains(self) -> dict[str, Any]: + async def get_subdomains(self, session: Any | None = None) -> dict[str, Any]: """Get subdomains for a domain""" endpoint = f'domain/{self.word}/subdomains' - return await self._fetch_data(endpoint) + return await self._fetch_data(endpoint, session) async def get_host_details(self, host: str) -> dict[str, Any]: """Get detailed information about a specific host""" @@ -301,6 +313,7 @@ class SearchFullHunt: hosts = details['hosts'] for host_data in hosts: if not isinstance(host_data, dict): + self._stop('failed', 'invalid-response') logger.info('FullHunt ignored a malformed host item') continue hostname = normalize_scoped_hostname(host_data.get('host'), self.word) @@ -332,6 +345,7 @@ class SearchFullHunt: for field in ('dns_records', 'http_response', 'geo', 'cloud', 'certificate') ) ): + self._stop('failed', 'invalid-response') logger.info('FullHunt ignored a malformed host item') continue # Extract subdomains @@ -418,32 +432,48 @@ class SearchFullHunt: async def do_search(self) -> None: """Main search method that calls the various endpoints""" try: - # First get domain details which includes most information - domain_details = await self.get_domain_details() - if not isinstance(domain_details.get('hosts'), list): - raise ValueError('FullHunt returned malformed domain details') - self.total_results['domain_details'] = domain_details - await self.extract_data_from_domain_details(domain_details) + async with AsyncFetcher.open_session( + headers=self._get_headers(), + proxy=self.proxy, + request_timeout=60, + ) as session: + # First get domain details which includes most information + domain_details = await self.get_domain_details(session) + if not isinstance(domain_details.get('hosts'), list): + raise ValueError('FullHunt returned malformed domain details') + self.total_results['domain_details'] = domain_details + await self.extract_data_from_domain_details(domain_details) - # If no hosts found in domain details, try the dedicated subdomains endpoint - if not self.total_results['hosts']: - subdomains_response = await self.get_subdomains() - hosts = subdomains_response.get('hosts') - if not isinstance(hosts, list): - raise ValueError('FullHunt returned malformed subdomains') - for host in hosts: - if normalized_host := normalize_scoped_hostname(host, self.word): - self.total_results['hosts'].append(normalized_host) - else: - logger.info('FullHunt ignored a malformed subdomain item') + # If no hosts found in domain details, try the dedicated subdomains endpoint + if not self.total_results['hosts']: + subdomains_response = await self.get_subdomains(session) + hosts = subdomains_response.get('hosts') + if not isinstance(hosts, list): + raise ValueError('FullHunt returned malformed subdomains') + for host in hosts: + if normalized_host := normalize_scoped_hostname(host, self.word): + self.total_results['hosts'].append(normalized_host) + else: + self._stop('failed', 'invalid-response') + logger.info('FullHunt ignored a malformed subdomain item') - # If filters are set, perform an advanced search - if self.filters: - search_results = await self.advanced_search() - await self.extract_data_from_search_results(search_results) + # If filters are set, perform an advanced search + if self.filters: + search_results = await self.advanced_search(session) + await self.extract_data_from_search_results(search_results) - except Exception as e: - logger.info(f'Error during FullHunt search: {e}') + except Exception as error: + if self.execution_status is None: + reason = 'invalid-response' if isinstance(error, ValueError) else 'transport-error' + self._stop('failed', reason) + logger.info('Error during FullHunt search: %s', type(error).__name__) + return + + if self.execution_status is not None and self._has_results(): + self.execution_status = 'partial' + elif self.execution_status is None: + self.execution_status = 'completed' + self.stop_reason = None if self._has_results() else 'no-results' async def get_hostnames(self) -> list[str]: """Return list of discovered subdomains""" @@ -498,6 +528,8 @@ class SearchFullHunt: """ self.proxy = proxy + self.execution_status = None + self.stop_reason = None # Apply filters if provided if filters: diff --git a/theHarvester/discovery/netlas.py b/theHarvester/discovery/netlas.py index ea5afc05..8c8a5a29 100644 --- a/theHarvester/discovery/netlas.py +++ b/theHarvester/discovery/netlas.py @@ -1,63 +1,94 @@ -import json +from __future__ import annotations + +from typing import Any from theHarvester.discovery.constants import MissingKey -from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.discovery.provider_response import provider_http_error +from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse +from theHarvester.lib.hostnames import normalize_scoped_hostname class SearchNetlas: - def __init__(self, word, limit: int) -> None: + def __init__(self, word: str, limit: int) -> None: + if isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0: + raise ValueError('Netlas limit must be a positive integer') self.word = word - self.totalhosts: list = [] - self.totalips: list = [] - self.key = Core.netlas_key() self.limit = limit - if self.key is None: + self.totalhosts: set[str] = set() + self.key = Core.netlas_key() + if not isinstance(self.key, str) or not self.key.strip(): raise MissingKey('netlas') self.proxy = False + self.execution_status: str | None = None + self.stop_reason: str | None = None - async def do_count(self) -> None: - """Counts the total number of subdomains + def _stop(self, status: str, reason: str) -> None: + self.execution_status = 'partial' if self.totalhosts else status + self.stop_reason = reason - :return: None - """ - api = f'https://app.netlas.io/api/domains_count/?q=*.{self.word}' - headers = {'X-API-Key': self.key} - response = await AsyncFetcher.fetch_all([api], json=True, headers=headers, proxy=self.proxy) - amount_size = response[0]['count'] - self.limit = min(self.limit, amount_size) + def _response_body(self, response: Any) -> Any | None: + if isinstance(response, FetcherResponse) and response.status == 402: + self._stop('failed', 'quota-exhausted') + return None + if error := provider_http_error(response): + self._stop(*error) + return None + assert isinstance(response, FetcherResponse) + if response.body is None: + self._stop('failed', 'invalid-response') + return None + return response.body - async def do_search(self) -> None: - """Download domains for query 'q' size of 'limit' + async def do_search(self, session: Any, size: int) -> None: + response = await AsyncFetcher.post_fetch( + 'https://app.netlas.io/api/domains/download/', + session=session, + json=True, + include_metadata=True, + json_body={ + 'q': f'*.{self.word}', + 'size': size, + 'fields': ['domain'], + 'source_type': 'include', + }, + ) + body = self._response_body(response) + if body is None: + return + if not isinstance(body, list): + self._stop('failed', 'invalid-response') + return - :return: None - """ - user_agent = Core.get_user_agent() - url = 'https://app.netlas.io/api/domains/download/' + malformed = False + for row in body[:size]: + if not isinstance(row, dict) or not isinstance(row.get('data'), dict): + malformed = True + continue + domain = row['data'].get('domain') + if not isinstance(domain, str): + malformed = True + continue + if hostname := normalize_scoped_hostname(domain, self.word): + self.totalhosts.add(hostname) + if malformed: + self._stop('failed', 'invalid-response') - payload = { - 'q': f'*.{self.word}', - 'fields': json.dumps(['domain']), # Convert the list to a JSON string - 'source_type': 'include', - 'size': str(self.limit), # Convert integer to string - 'type': 'json', - 'indice': json.dumps([0]), # Convert the list to a JSON string - } - - headers = { - 'X-API-Key': self.key, - 'User-Agent': user_agent, - } - response = await AsyncFetcher.post_fetch(url, data=payload, headers=headers, proxy=self.proxy) - resp_json = json.loads(response) - - for data in resp_json: - domain = data['data']['domain'] - self.totalhosts.append(domain) - - async def get_hostnames(self) -> list: + async def get_hostnames(self) -> set[str]: return self.totalhosts async def process(self, proxy: bool = False) -> None: self.proxy = proxy - await self.do_count() - await self.do_search() + self.execution_status = None + self.stop_reason = None + try: + async with AsyncFetcher.open_session( + headers={'Authorization': f'Bearer {self.key}'}, + proxy=proxy, + ) as session: + await self.do_search(session, self.limit) + except Exception: + self._stop('failed', 'transport-error') + return + if self.execution_status is None: + self.execution_status = 'completed' + self.stop_reason = None if self.totalhosts else 'no-results' diff --git a/theHarvester/discovery/onyphe.py b/theHarvester/discovery/onyphe.py index d5ed0878..dd47fa6c 100644 --- a/theHarvester/discovery/onyphe.py +++ b/theHarvester/discovery/onyphe.py @@ -4,6 +4,7 @@ from ipaddress import ip_address from urllib.parse import urlparse from theHarvester.discovery.constants import MissingKey +from theHarvester.discovery.provider_response import provider_http_error from theHarvester.lib.asn_attribution import AsnAttributionObservation, SubjectKind from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse from theHarvester.lib.hostnames import normalize_scoped_hostname @@ -22,81 +23,113 @@ class SearchOnyphe: WHOIS data. Both stay separate and are linked to the record's primary IP. """ - def __init__(self, word) -> None: + MAX_RESULTS = 10_000 + + def __init__(self, word: str, limit: int) -> None: + if isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0: + raise ValueError('ONYPHE limit must be a positive integer') self.word = word - self.response = '' + self.limit = limit + self.response: object = {} self.totalhosts: set = set() self.totalips: set = set() self.asns: set = set() self.asn_attributions: set[AsnAttributionObservation] = set() self.key = Core.onyphe_key() - if self.key is None: + if not isinstance(self.key, str) or not self.key.strip(): raise MissingKey('onyphe') self.proxy = False self.execution_status: str | None = None self.stop_reason: str | None = None + def _has_results(self) -> bool: + return bool(self.totalhosts or self.totalips or self.asns) + + def _stop(self, status: str, reason: str) -> None: + self.execution_status = 'partial' if self._has_results() else status + self.stop_reason = reason + async def do_search(self) -> None: - # https://www.onyphe.io/docs/apis/search - # https://www.onyphe.io/search?q=domain%3Acharter.com&captcharesponse=j5cGT - # base_url = f'https://www.onyphe.io/api/v2/search/?q=domain:domain:{self.word}' - base_url = f'https://www.onyphe.io/api/v2/search/?q=domain:{self.word}' + base_url = 'https://www.onyphe.io/api/v2/search/' headers = { 'User-Agent': Core.get_user_agent(), 'Content-Type': 'application/json', 'Authorization': f'bearer {self.key}', } + page = 1 + records_seen = 0 + result_limit = min(self.limit, self.MAX_RESULTS) + page_size = min(result_limit, self.MAX_RESULTS) + last_total = 0 try: - response = await AsyncFetcher.fetch_all( - [base_url], - json=True, - headers=headers, - proxy=self.proxy, - include_metadata=True, - ) + async with AsyncFetcher.open_session(headers=headers, proxy=self.proxy) as session: + while records_seen < result_limit: + remaining = result_limit - records_seen + metadata = await AsyncFetcher.fetch( + session=session, + url=base_url, + params={'q': f'domain:{self.word}', 'page': page, 'size': page_size}, + json=True, + include_metadata=True, + ) + if error := provider_http_error(metadata): + self._stop(*error) + return + assert isinstance(metadata, FetcherResponse) + if not isinstance(metadata.body, dict): + self._stop('failed', 'invalid-response') + return + response_text = metadata.body.get('text') + if response_text != 'Success': + self._stop('failed', 'provider-error' if isinstance(response_text, str) else 'invalid-response') + return + results = metadata.body.get('results') + max_page = metadata.body.get('max_page', 1) + total = metadata.body.get('total', len(results) if isinstance(results, list) else None) + if ( + not isinstance(results, list) + or isinstance(max_page, bool) + or not isinstance(max_page, int) + or max_page < 1 + or isinstance(total, bool) + or not isinstance(total, int) + or total < 0 + ): + self._stop('failed', 'invalid-response') + return + + page_results = results[:remaining] + records_seen += len(page_results) + last_total = total + self.response = {**metadata.body, 'results': page_results} + if await self.parse_onyphe_resp_json(): + self._stop('failed', 'invalid-response') + expected_records = min(total, result_limit) + if (not results or page >= max_page) and records_seen < expected_records: + self._stop('failed', 'invalid-response') + break + if not results or page >= max_page or records_seen >= expected_records: + break + page += 1 except Exception as error: - self.execution_status = 'failed' - self.stop_reason = 'transport-error' + self._stop('failed', 'transport-error') logger.info('Onyphe request failed: %s', type(error).__name__) return - metadata = response[0] if response and isinstance(response[0], FetcherResponse) else None - if metadata is None: - self.execution_status = 'failed' - self.stop_reason = 'transport-error' - return - if metadata.status == 429: - self.execution_status = 'rate-limited' - self.stop_reason = 'http-429' - return - if metadata.status in {401, 403}: - self.execution_status = 'failed' - self.stop_reason = 'access-denied' - return - if not 200 <= metadata.status < 300: - self.execution_status = 'failed' - self.stop_reason = f'http-{metadata.status}' - return + if self.limit > self.MAX_RESULTS and last_total > self.MAX_RESULTS and records_seen >= self.MAX_RESULTS: + self._stop('failed', 'provider-limit') + if self.execution_status is not None and self._has_results(): + self.execution_status = 'partial' + elif self.execution_status is None: + self.execution_status = 'completed' + self.stop_reason = None if self._has_results() else 'no-results' - self.response = metadata.body - await self.parse_onyphe_resp_json() - - async def parse_onyphe_resp_json(self): + async def parse_onyphe_resp_json(self) -> bool: if not isinstance(self.response, dict): - self.execution_status = 'failed' - self.stop_reason = 'invalid-response' - return - response_text = self.response.get('text') - if response_text != 'Success': - self.execution_status = 'failed' - self.stop_reason = 'provider-error' if isinstance(response_text, str) else 'invalid-response' - logger.info('Onyphe API query did not succeed') - return + return True results = self.response.get('results') if not isinstance(results, list): - self.execution_status = 'failed' - self.stop_reason = 'invalid-response' - return + return True malformed = False for result in results: @@ -212,12 +245,7 @@ class SearchOnyphe: except ValueError: malformed = True - if malformed: - self.execution_status = 'partial' if self.totalhosts or self.totalips or self.asns else 'failed' - self.stop_reason = 'invalid-response' - else: - self.execution_status = 'completed' - self.stop_reason = None if self.totalhosts or self.totalips or self.asns else 'no-results' + return malformed async def get_asns(self) -> set: return self.asns @@ -233,4 +261,6 @@ class SearchOnyphe: async def process(self, proxy: bool = False) -> None: self.proxy = proxy + self.execution_status = None + self.stop_reason = None await self.do_search() diff --git a/theHarvester/discovery/provider_response.py b/theHarvester/discovery/provider_response.py new file mode 100644 index 00000000..4569f7de --- /dev/null +++ b/theHarvester/discovery/provider_response.py @@ -0,0 +1,14 @@ +from theHarvester.lib.core import FetcherResponse + + +def provider_http_error(response: object) -> tuple[str, str] | None: + """Classify transport and HTTP failures shared by provider adapters.""" + if not isinstance(response, FetcherResponse): + return 'failed', 'transport-error' + if response.status in {401, 403}: + return 'failed', 'access-denied' + if response.status == 429: + return 'rate-limited', 'http-429' + if not 200 <= response.status < 300: + return 'failed', f'http-{response.status}' + return None diff --git a/theHarvester/discovery/searchhunterhow.py b/theHarvester/discovery/searchhunterhow.py index 99eae2a0..1850057f 100644 --- a/theHarvester/discovery/searchhunterhow.py +++ b/theHarvester/discovery/searchhunterhow.py @@ -1,58 +1,126 @@ +import asyncio import base64 -import logging -from datetime import datetime +from datetime import UTC, datetime +from typing import Any from dateutil.relativedelta import relativedelta from theHarvester.discovery.constants import MissingKey -from theHarvester.lib.core import AsyncFetcher, Core - -logger = logging.getLogger(__name__) +from theHarvester.discovery.provider_response import provider_http_error +from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse +from theHarvester.lib.hostnames import normalize_scoped_hostname class SearchHunterHow: - def __init__(self, word) -> None: + REQUEST_DELAY_SECONDS = 2.0 + + def __init__(self, word: str, limit: int = 500) -> None: + if isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0: + raise ValueError('Hunter.how limit must be a positive integer') self.word = word - self.total_hostnames: set = set() + self.limit = limit + self.total_hostnames: set[str] = set() self.key = Core.hunterhow_key() - if self.key is None: + if not isinstance(self.key, str) or not self.key.strip(): raise MissingKey('hunterhow') self.proxy = False + self.execution_status: str | None = None + self.stop_reason: str | None = None + + def _stop(self, status: str, reason: str) -> None: + self.execution_status = 'partial' if self.total_hostnames else status + self.stop_reason = reason + + @staticmethod + def _page_size(remaining: int) -> int: + for size in (10, 20, 50, 100, 1000): + if remaining <= size: + return size + return 1000 async def do_search(self) -> None: - # https://hunter.how/search-api - query = f'domain.suffix="{self.word}"' - # second_query = f'domain="{self.word}"' - encoded_query = base64.urlsafe_b64encode(query.encode('utf-8')).decode('ascii') + self.execution_status = None + self.stop_reason = None + query = base64.urlsafe_b64encode(f'domain.suffix="{self.word}"'.encode()).decode('ascii') + end = datetime.now(UTC).date() + start = end - relativedelta(days=364) page = 1 - page_size = 100 # can be either: 10,20,50,100) - # The interval between the start time and the end time cannot exceed one year - # Can not exceed one year, but years=1 does not work due to their backend, 364 will suffice - today = datetime.today() - one_year_ago = today - relativedelta(days=364) - start_time = one_year_ago.strftime('%Y-%m-%d') - end_time = today.strftime('%Y-%m-%d') - # two_years_ago = one_year_ago - relativedelta(days=364) - # start_time = two_years_ago.strftime('%Y-%m-%d') - # end_time = one_year_ago.strftime('%Y-%m-%d') - url = f'https://api.hunter.how/search?api-key={self.key}&query={encoded_query}&page={page}&page_size={page_size}&start_time={start_time}&end_time={end_time}' - response = await AsyncFetcher.fetch_all( - [url], - json=True, - headers={'User-Agent': Core.get_user_agent(), 'x-api-key': f'{self.key}'}, - proxy=self.proxy, - ) - dct = response[0] - if 'code' in dct.keys(): - if dct['code'] == 40001: - logger.info('SearchHunterHow API returned code 40001') - return - # total = dct['data']['total'] - # TODO determine if total is ever 100 how to get more subdomains? - for sub in dct['data']['list']: - self.total_hostnames.add(sub['domain']) + returned = 0 + params: dict[str, Any] = { + 'api-key': self.key, + 'query': query, + 'start_time': start.isoformat(), + 'end_time': end.isoformat(), + 'fields': 'domain', + } + try: + async with AsyncFetcher.open_session( + headers={'User-Agent': Core.get_user_agent()}, + proxy=self.proxy, + ) as session: + while returned < self.limit: + request_params = { + **params, + 'page': page, + 'page_size': self._page_size(self.limit - returned), + } + response = await AsyncFetcher.fetch( + session=session, + url='https://api.hunter.how/search', + params=request_params, + include_metadata=True, + ) + if error := provider_http_error(response): + self._stop(*error) + return + assert isinstance(response, FetcherResponse) + if not isinstance(response.body, dict): + self._stop('failed', 'invalid-response') + return + code = response.body.get('code') + if code == 40001: + self._stop('failed', 'access-denied') + return + if code != 200: + self._stop('failed', 'provider-error') + return + data = response.body.get('data') + if not isinstance(data, dict): + self._stop('failed', 'invalid-response') + return + total = data.get('total') + rows = data.get('list') + if isinstance(total, bool) or not isinstance(total, int) or total < 0 or not isinstance(rows, list): + self._stop('failed', 'invalid-response') + return - async def get_hostnames(self) -> set: + remaining = self.limit - returned + malformed = False + for row in rows[:remaining]: + if not isinstance(row, dict) or not isinstance(row.get('domain'), str): + malformed = True + continue + if hostname := normalize_scoped_hostname(row['domain'], self.word): + self.total_hostnames.add(hostname) + if malformed: + self._stop('failed', 'invalid-response') + + returned += len(rows) + if not rows or returned >= min(total, self.limit): + break + page += 1 + await asyncio.sleep(self.REQUEST_DELAY_SECONDS) + except Exception: + self._stop('failed', 'transport-error') + return + + if self.execution_status is not None and self.total_hostnames: + self.execution_status = 'partial' + elif self.execution_status is None: + self.execution_status = 'completed' + self.stop_reason = None if self.total_hostnames else 'no-results' + + async def get_hostnames(self) -> set[str]: return self.total_hostnames async def process(self, proxy: bool = False) -> None: diff --git a/theHarvester/discovery/securityscorecard.py b/theHarvester/discovery/securityscorecard.py index f785ba4d..40deda28 100644 --- a/theHarvester/discovery/securityscorecard.py +++ b/theHarvester/discovery/securityscorecard.py @@ -1,18 +1,24 @@ -import logging +from __future__ import annotations -import aiohttp +from ipaddress import ip_address +from typing import Any from theHarvester.discovery.constants import MissingKey -from theHarvester.lib.core import AsyncFetcher, Core - -logger = logging.getLogger(__name__) +from theHarvester.discovery.provider_response import provider_http_error +from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse +from theHarvester.lib.hostnames import normalize_scoped_hostname class SearchSecurityScorecard: - def __init__(self, word: str): + PAGE_SIZE = 50 + + def __init__(self, word: str, limit: int) -> None: + if isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0: + raise ValueError('SecurityScorecard limit must be a positive integer') self.word = word + self.limit = limit self.api_key = Core.securityscorecard_key() - if self.api_key is None: + if not isinstance(self.api_key, str) or not self.api_key.strip(): raise MissingKey('SecurityScorecard') self.base_url = 'https://api.securityscorecard.io' self.headers = { @@ -22,79 +28,143 @@ class SearchSecurityScorecard: } self.hosts: set[str] = set() self.score: int = 0 - self.grades: dict = {} - self.issues: list[dict] = [] - self.recommendations: list[dict] = [] - self.history: list[dict] = [] - self.ips: list[str] = [] + self.grades: dict[str, Any] = {} + self.issues: list[dict[str, Any]] = [] + self.recommendations: list[dict[str, Any]] = [] + self.history: list[dict[str, Any]] = [] + self.ips: set[str] = set() + self.execution_status: str | None = None + self.stop_reason: str | None = None + + def _stop(self, status: str, reason: str) -> None: + self.execution_status = 'partial' if self.hosts or self.ips else status + self.stop_reason = reason + + def _response_body(self, response: Any) -> dict[str, Any] | None: + if error := provider_http_error(response): + self._stop(*error) + return None + assert isinstance(response, FetcherResponse) + if not isinstance(response.body, dict): + self._stop('failed', 'invalid-response') + return None + return response.body + + def _extract_summary(self, data: dict[str, Any]) -> bool: + malformed = False + score = data.get('score') + if score is not None: + if isinstance(score, bool) or not isinstance(score, int): + malformed = True + else: + self.score = score + + grade = data.get('grade') + if grade is not None: + if isinstance(grade, str) and grade.strip(): + self.grades['overall'] = grade.strip() + else: + malformed = True + factor_grades = data.get('factor_grades') + if factor_grades is not None: + if isinstance(factor_grades, dict): + self.grades.update(factor_grades) + else: + malformed = True + return malformed + + async def _collect_assets(self, session: Any, route: str, field: str) -> bool: + page = 0 + records_seen = 0 + page_size = min(self.PAGE_SIZE, self.limit) + while records_seen < self.limit: + response = await AsyncFetcher.post_fetch( + f'{self.base_url}/parent-domains/{self.word}/{route}', + session=session, + json=True, + include_metadata=True, + json_body={'page': page, 'page_size': page_size}, + ) + body = self._response_body(response) + if body is None: + return False + entries = body.get('entries') + size = body.get('size') + if not isinstance(entries, list) or isinstance(size, bool) or not isinstance(size, int | float) or size < 0: + self._stop('failed', 'invalid-response') + return False + + remaining = self.limit - records_seen + page_entries = entries[:remaining] + records_seen += len(page_entries) + malformed = False + for entry in page_entries: + if not isinstance(entry, dict) or not isinstance(entry.get(field), str): + malformed = True + continue + value = entry[field] + if field == 'domain': + if hostname := normalize_scoped_hostname(value, self.word): + self.hosts.add(hostname) + else: + try: + self.ips.add(str(ip_address(value.strip()))) + except ValueError: + malformed = True + if malformed: + self._stop('failed', 'invalid-response') + if records_seen >= self.limit or len(entries) < page_size: + return True + page += 1 + return True async def process(self, proxy: bool = False) -> None: - """Get security scorecard information for a domain.""" + self.execution_status = None + self.stop_reason = None try: - if proxy: + async with AsyncFetcher.open_session(headers=self.headers, proxy=proxy) as session: response = await AsyncFetcher.fetch( - session=None, url=f'{self.base_url}/companies/{self.word}', headers=self.headers, proxy=proxy + session=session, + url=f'{self.base_url}/companies/{self.word}', + json=True, + include_metadata=True, ) - if response: - self._extract_data(response) - else: - async with aiohttp.ClientSession(headers=self.headers) as session: - async with session.get(f'{self.base_url}/companies/{self.word}') as response: - if response.status == 200: - data = await response.json() - self._extract_data(data) - except Exception as e: - logger.info(f'Error in SecurityScorecard search: {e}') + body = self._response_body(response) + if body is None: + return + if self._extract_summary(body): + self._stop('failed', 'invalid-response') + if not await self._collect_assets(session, 'domains', 'domain'): + return + if not await self._collect_assets(session, 'ips', 'ip'): + return + except Exception: + self._stop('failed', 'transport-error') + return - def _extract_data(self, data: dict) -> None: - """Extract and categorize security scorecard information.""" - if 'grade' in data: - self.score = data.get('grade', 0) - - if 'factor_grades' in data: - self.grades = data['factor_grades'] - - if 'issues' in data: - self.issues = data['issues'] - - if 'recommendations' in data: - self.recommendations = data['recommendations'] - - if 'history' in data: - self.history = data['history'] - - if 'domains' in data: - self.hosts.update(data['domains']) - - # Some responses may include IP addresses under different keys - ips = [] - if isinstance(data.get('ips'), list): - ips = [str(ip) for ip in data.get('ips', []) if isinstance(ip, str | int)] - elif isinstance(data.get('ip_addresses'), list): - ips = [str(ip) for ip in data.get('ip_addresses', []) if isinstance(ip, str | int)] - elif isinstance(data.get('associated_ips'), list): - ips = [str(ip) for ip in data.get('associated_ips', []) if isinstance(ip, str | int)] - if ips: - # Deduplicate while preserving already stored entries - self.ips = list({*self.ips, *ips}) + if self.execution_status is not None and (self.hosts or self.ips): + self.execution_status = 'partial' + elif self.execution_status is None: + self.execution_status = 'completed' + self.stop_reason = None if self.hosts or self.ips else 'no-results' async def get_hostnames(self) -> set[str]: return self.hosts - async def get_ips(self) -> list[str]: + async def get_ips(self) -> set[str]: return self.ips async def get_score(self) -> int: return self.score - async def get_grades(self) -> dict: + async def get_grades(self) -> dict[str, Any]: return self.grades - async def get_issues(self) -> list[dict]: + async def get_issues(self) -> list[dict[str, Any]]: return self.issues - async def get_recommendations(self) -> list[dict]: + async def get_recommendations(self) -> list[dict[str, Any]]: return self.recommendations - async def get_history(self) -> list[dict]: + async def get_history(self) -> list[dict[str, Any]]: return self.history diff --git a/theHarvester/discovery/securitytrailssearch.py b/theHarvester/discovery/securitytrailssearch.py index 40c7db91..39d25089 100644 --- a/theHarvester/discovery/securitytrailssearch.py +++ b/theHarvester/discovery/securitytrailssearch.py @@ -1,95 +1,118 @@ -import asyncio -import logging +from __future__ import annotations + +from ipaddress import ip_address +from typing import Any from theHarvester.discovery.constants import MissingKey -from theHarvester.lib.core import AsyncFetcher, Core -from theHarvester.parsers import securitytrailsparser - -logger = logging.getLogger(__name__) +from theHarvester.discovery.provider_response import provider_http_error +from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse +from theHarvester.lib.hostnames import normalize_scoped_hostname class SearchSecuritytrail: - def __init__(self, word) -> None: + def __init__(self, word: str) -> None: self.word = word self.key = Core.security_trails_key() - if self.key is None: + if not isinstance(self.key, str) or not self.key.strip(): raise MissingKey('Securitytrail') - self.results = '' - self.totalresults = '' self.api = 'https://api.securitytrails.com/v1/' - self.info: tuple[set, set] = (set(), set()) + self.info: tuple[set[str], set[str]] = (set(), set()) self.proxy = False - # Hold structured responses for robust parsing - self.domain_data: dict = {} - self.subdomains_data: dict = {} + self.domain_data: dict[str, Any] = {} + self.subdomains_data: dict[str, Any] = {} + self.execution_status: str | None = None + self.stop_reason: str | None = None - async def authenticate(self) -> None: - # Method to authenticate API key before sending requests. - headers = {'APIKEY': self.key} - url = f'{self.api}ping' - auth_responses = await AsyncFetcher.fetch_all([url], headers=headers, proxy=self.proxy) - auth_responses = auth_responses[0] - if 'False' in auth_responses or 'Invalid authentication' in auth_responses: - logger.info('\tKey could not be authenticated exiting program.') - await asyncio.sleep(5) + def _stop(self, status: str, reason: str) -> None: + self.execution_status = 'partial' if self.info[0] or self.info[1] else status + self.stop_reason = reason - async def do_search(self) -> None: - try: - # https://api.securitytrails.com/v1/domain/domain.com - domain_url = f'{self.api}domain/{self.word}' - headers = {'APIKEY': self.key, 'Accept': 'application/json'} - # Request JSON payloads for robust parsing - domain_response = await AsyncFetcher.fetch_all([domain_url], headers=headers, json=True, proxy=self.proxy) - await asyncio.sleep(5) # 2+ seconds is required due to rate limit. + def _body(self, response: Any) -> dict[str, Any] | None: + if error := provider_http_error(response): + self._stop(*error) + return None + assert isinstance(response, FetcherResponse) + if not isinstance(response.body, dict): + self._stop('failed', 'invalid-response') + return None + return response.body - if domain_response and isinstance(domain_response[0], dict | list): - self.domain_data = domain_response[0] if isinstance(domain_response[0], dict) else {} - else: - logger.info('SecurityTrails: No JSON response received for domain query') - # keep legacy string totalresults for any downstream reliance - if domain_response and domain_response[0]: - self.results = str(domain_response[0]) - self.totalresults += self.results - return + def _parse_domain(self, data: dict[str, Any]) -> bool: + malformed = False + current_dns = data.get('current_dns', {}) + if not isinstance(current_dns, dict): + return True + ips = self.info[0] + for record_type, key in (('a', 'ip'), ('aaaa', 'ipv6')): + records = current_dns.get(record_type, {}) + if not isinstance(records, dict) or not isinstance(records.get('values', []), list): + malformed = True + continue + for record in records.get('values', []): + if not isinstance(record, dict) or not isinstance(record.get(key), str): + malformed = True + continue + try: + ips.add(str(ip_address(record[key].strip()))) + except ValueError: + malformed = True + return malformed - # Get subdomains now. - subdomains_url = f'{domain_url}/subdomains' - subdomain_response = await AsyncFetcher.fetch_all([subdomains_url], headers=headers, json=True, proxy=self.proxy) - await asyncio.sleep(5) - - if subdomain_response and isinstance(subdomain_response[0], dict | list): - self.subdomains_data = subdomain_response[0] if isinstance(subdomain_response[0], dict) else {} - else: - logger.info('SecurityTrails: No JSON response received for subdomain query') - if subdomain_response and subdomain_response[0]: - self.results = str(subdomain_response[0]) - self.totalresults += self.results - except Exception as e: - logger.info(f'SecurityTrails API error: {e}') - return + def _parse_subdomains(self, data: dict[str, Any]) -> bool: + values = data.get('subdomains') + if not isinstance(values, list): + return True + malformed = False + hostnames = self.info[1] + for value in values: + if not isinstance(value, str) or not value.strip(): + malformed = True + continue + if hostname := normalize_scoped_hostname(f'{value}.{self.word}', self.word): + hostnames.add(hostname) + return malformed async def process(self, proxy: bool = False) -> None: self.proxy = proxy - await self.authenticate() - await self.do_search() - # Prefer structured JSON if available; fallback to legacy text - combined_payload = None - if isinstance(self.domain_data, dict) or isinstance(self.subdomains_data, dict): - combined_payload = { - 'domain': self.domain_data if isinstance(self.domain_data, dict) else {}, - 'subdomains': self.subdomains_data if isinstance(self.subdomains_data, dict) else {}, - } - parser_input = ( - combined_payload - if combined_payload and (combined_payload['domain'] or combined_payload['subdomains']) - else self.totalresults - ) - parser = securitytrailsparser.Parser(word=self.word, text=parser_input) - self.info = await parser.parse_text() - # Create parser and set self.info to tuple returned from parsing text. + self.execution_status = None + self.stop_reason = None + headers = {'APIKEY': self.key, 'Accept': 'application/json'} + try: + async with AsyncFetcher.open_session(headers=headers, proxy=proxy) as session: + domain_response = await AsyncFetcher.fetch( + session=session, + url=f'{self.api}domain/{self.word}', + json=True, + include_metadata=True, + ) + domain_body = self._body(domain_response) + if domain_body is None: + return + self.domain_data = domain_body + malformed = self._parse_domain(domain_body) - async def get_ips(self) -> set: + subdomain_response = await AsyncFetcher.fetch( + session=session, + url=f'{self.api}domain/{self.word}/subdomains', + json=True, + include_metadata=True, + ) + subdomain_body = self._body(subdomain_response) + if subdomain_body is None: + return + self.subdomains_data = subdomain_body + malformed = self._parse_subdomains(subdomain_body) or malformed + except Exception: + self._stop('failed', 'transport-error') + return + if malformed: + self._stop('failed', 'invalid-response') + if self.execution_status is None: + self.execution_status = 'completed' + self.stop_reason = None if self.info[0] or self.info[1] else 'no-results' + + async def get_ips(self) -> set[str]: return self.info[0] - async def get_hostnames(self) -> set: + async def get_hostnames(self) -> set[str]: return self.info[1] diff --git a/theHarvester/discovery/sherlockeye.py b/theHarvester/discovery/sherlockeye.py index 6317981d..a264764a 100644 --- a/theHarvester/discovery/sherlockeye.py +++ b/theHarvester/discovery/sherlockeye.py @@ -1,12 +1,12 @@ import logging -import random +from ipaddress import ip_address as normalize_ip_address from typing import Any from urllib.parse import urlparse -import aiohttp - from theHarvester.discovery.constants import MissingKey -from theHarvester.lib.core import Core +from theHarvester.discovery.provider_response import provider_http_error +from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse +from theHarvester.lib.hostnames import normalize_scoped_hostname logger = logging.getLogger(__name__) @@ -26,13 +26,22 @@ class SearchSherlockeye: def __init__(self, word: str) -> None: self.word = word self.key = Core.sherlockeye_key() - if self.key is None: + if not isinstance(self.key, str) or not self.key.strip(): raise MissingKey('sherlockeye') self.totalhosts: set[str] = set() self.totalemails: set[str] = set() self.totalips: set[str] = set() self.results: list[dict[str, Any]] = [] self.proxy: bool | str = False + self.execution_status: str | None = None + self.stop_reason: str | None = None + + def _has_results(self) -> bool: + return bool(self.totalhosts or self.totalemails or self.totalips) + + def _stop(self, status: str, reason: str) -> None: + self.execution_status = 'partial' if self._has_results() else status + self.stop_reason = reason def _headers(self) -> dict[str, str]: return { @@ -41,77 +50,91 @@ class SearchSherlockeye: 'Content-Type': 'application/json', } - def _proxy_url(self) -> str | None: - if isinstance(self.proxy, str) and self.proxy: - return self.proxy - if isinstance(self.proxy, bool) and self.proxy: - try: - proxy_list = Core.proxy_list() - proxy_urls = [*proxy_list.get('http', []), *proxy_list.get('socks5', [])] - if proxy_urls: - return random.choice(proxy_urls) - except Exception: - return None - return None - def _add_hostname(self, hostname: str) -> None: - hostname = hostname.strip().lower() - if hostname.endswith(f'.{self.word}') or hostname == self.word: - self.totalhosts.add(hostname) + if normalized := normalize_scoped_hostname(hostname, self.word): + self.totalhosts.add(normalized) def _add_email(self, email: str) -> None: - email = email.strip().lower() - if '@' in email and self.word in email: - self.totalemails.add(email) + normalized_email = email.strip().lower() + local_part, separator, domain = normalized_email.rpartition('@') + if local_part and separator and (normalized_domain := normalize_scoped_hostname(domain, self.word)): + self.totalemails.add(f'{local_part}@{normalized_domain}') def _add_ip(self, ip_address: str) -> None: - ip_address = ip_address.strip() - if ip_address: - self.totalips.add(ip_address) + try: + self.totalips.add(str(normalize_ip_address(ip_address.strip()))) + except ValueError: + return - def _extract_from_link(self, link: str) -> None: - parsed = urlparse(link.strip()) + def _extract_from_link(self, link: str) -> bool: + try: + parsed = urlparse(link.strip()) + except ValueError: + return True if parsed.hostname: self._add_hostname(parsed.hostname) + return False - def _extract_result(self, result: dict[str, Any]) -> None: + def _extract_result(self, result: dict[str, Any]) -> bool: attributes = result.get('attributes') if not isinstance(attributes, dict): - return + return True + + malformed = False domain = attributes.get('domain') if isinstance(domain, str): self._add_hostname(domain) + elif domain is not None: + malformed = True email = attributes.get('email') if isinstance(email, str): self._add_email(email) + elif email is not None: + malformed = True ip_address = attributes.get('ip') if isinstance(ip_address, str): self._add_ip(ip_address) + elif ip_address is not None: + malformed = True link = attributes.get('link') if isinstance(link, str): - self._extract_from_link(link) + malformed |= self._extract_from_link(link) + elif link is not None: + malformed = True + return malformed def _extract_response(self, response: dict[str, Any]) -> None: if response.get('success') is False: logger.info('Sherlockeye API error') + self._stop('failed', 'provider-error') return data = response.get('data') if not isinstance(data, dict): + self._stop('failed', 'invalid-response') return search_results = data.get('results') if not isinstance(search_results, list): + self._stop('failed', 'invalid-response') return self.results = search_results + malformed = False for result in search_results: if isinstance(result, dict): - self._extract_result(result) + malformed |= self._extract_result(result) + else: + malformed = True + if malformed: + self._stop('failed', 'invalid-response') + elif self.execution_status is None: + self.execution_status = 'completed' + self.stop_reason = None if self._has_results() else 'no-results' async def do_search(self) -> None: payload = { @@ -119,24 +142,36 @@ class SearchSherlockeye: 'value': self.word, 'timeoutSeconds': self.DEFAULT_TIMEOUT_SECONDS, } - timeout = aiohttp.ClientTimeout(total=self.DEFAULT_TIMEOUT_SECONDS + 30) - try: - async with aiohttp.ClientSession(headers=self._headers(), timeout=timeout) as session: - async with session.post( + async with AsyncFetcher.open_session( + headers=self._headers(), + proxy=self.proxy, + request_timeout=self.DEFAULT_TIMEOUT_SECONDS + 30, + ) as session: + response = await AsyncFetcher.post_fetch( self.SYNC_SEARCH_URL, - json=payload, - proxy=self._proxy_url(), - ) as response: - if response.status != 200: - logger.info(f'Sherlockeye API request failed with status {response.status}') - return - - response_data = await response.json() - if isinstance(response_data, dict): - self._extract_response(response_data) + session=session, + json=True, + include_metadata=True, + json_body=payload, + ) + if error := provider_http_error(response): + self._stop(*error) + status = response.status if isinstance(response, FetcherResponse) else 'transport' + logger.info('Sherlockeye API request failed with status %s: %s', status, error[1]) + return + assert isinstance(response, FetcherResponse) + if response.status != 200: + self._stop('failed', f'http-{response.status}') + logger.info('Sherlockeye API request failed with status %s', response.status) + return + if isinstance(response.body, dict): + self._extract_response(response.body) + else: + self._stop('failed', 'invalid-response') except Exception as error: - logger.info(f'Sherlockeye API error: {error}') + self._stop('failed', 'transport-error') + logger.info('Sherlockeye API error: %s', type(error).__name__) async def get_hostnames(self) -> set[str]: return self.totalhosts @@ -152,4 +187,6 @@ class SearchSherlockeye: async def process(self, proxy: bool = False) -> None: self.proxy = proxy + self.execution_status = None + self.stop_reason = None await self.do_search() diff --git a/theHarvester/discovery/subdomainfinderc99.py b/theHarvester/discovery/subdomainfinderc99.py index 2adaf35a..52fa9585 100644 --- a/theHarvester/discovery/subdomainfinderc99.py +++ b/theHarvester/discovery/subdomainfinderc99.py @@ -5,7 +5,8 @@ from bs4 import BeautifulSoup from bs4.element import Tag from theHarvester.discovery.constants import get_delay -from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.discovery.provider_response import provider_http_error +from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse from theHarvester.parsers import myparser @@ -17,22 +18,54 @@ class SearchSubdomainfinderc99: # TODO add api support self.server = 'https://subdomainfinder.c99.nl/' self.totalresults = '' + self.execution_status: str | None = None + self.stop_reason: str | None = None + + def _stop(self, status: str, reason: str) -> None: + self.execution_status = status + self.stop_reason = reason async def do_search(self) -> None: # Based on https://gist.github.com/th3gundy/bc83580cbe04031e9164362b33600962 headers = {'User-Agent': Core.get_browser_user_agent()} - resp = await AsyncFetcher.fetch_all([self.server], headers=headers, proxy=self.proxy) - if not resp or not isinstance(resp[0], str): - return - data = await self.get_csrf_params(resp[0]) + async with AsyncFetcher.open_session(headers=headers, proxy=self.proxy) as session: + metadata = await AsyncFetcher.fetch( + session=session, + url=self.server, + include_metadata=True, + ) + if error := provider_http_error(metadata): + self._stop(*error) + return + assert isinstance(metadata, FetcherResponse) + if not isinstance(metadata.body, str): + self._stop('failed', 'invalid-response') + return + data = await self.get_csrf_params(metadata.body) + if not data: + self._stop('failed', 'invalid-response') + return - data['scan_subdomains'] = '' - data['domain'] = self.word - data['privatequery'] = 'on' - await asyncio.sleep(get_delay()) - second_resp = await AsyncFetcher.post_fetch(self.server, headers=headers, proxy=self.proxy, data=ujson.dumps(data)) - if isinstance(second_resp, str): - self.totalresults += second_resp + data['scan_subdomains'] = '' + data['domain'] = self.word + data['privatequery'] = 'on' + await asyncio.sleep(get_delay()) + second_resp = await AsyncFetcher.post_fetch( + self.server, + session=session, + data=ujson.dumps(data), + include_metadata=True, + ) + if error := provider_http_error(second_resp): + self._stop(*error) + return + assert isinstance(second_resp, FetcherResponse) + if not isinstance(second_resp.body, str): + self._stop('failed', 'invalid-response') + return + self.totalresults += second_resp.body + self.execution_status = 'completed' + self.stop_reason = None if await self.get_hostnames() else 'no-results' async def get_hostnames(self): rawres = myparser.Parser(self.totalresults, self.word) @@ -40,7 +73,12 @@ class SearchSubdomainfinderc99: async def process(self, proxy: bool = False) -> None: self.proxy = proxy - await self.do_search() + self.execution_status = None + self.stop_reason = None + try: + await self.do_search() + except Exception: + self._stop('failed', 'transport-error') @staticmethod async def get_csrf_params(data): diff --git a/theHarvester/discovery/urlscan.py b/theHarvester/discovery/urlscan.py index 8f095397..29adb7ca 100644 --- a/theHarvester/discovery/urlscan.py +++ b/theHarvester/discovery/urlscan.py @@ -3,6 +3,7 @@ from datetime import UTC, datetime from ipaddress import ip_address from urllib.parse import urlsplit +from theHarvester.discovery.provider_response import provider_http_error from theHarvester.lib.asn_attribution import AsnAttributionObservation, SubjectKind from theHarvester.lib.core import AsyncFetcher, FetcherResponse from theHarvester.lib.hostnames import normalize_scoped_hostname @@ -12,11 +13,13 @@ logger = logging.getLogger(__name__) class SearchUrlscan: - # ponytail: hard cap protects against endless unique cursors; raise only if real targets exceed 1,000 pages. - MAX_PAGES = 1000 + MAX_PAGE_SIZE = 10_000 - def __init__(self, word) -> None: + def __init__(self, word: str, limit: int) -> None: + if isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0: + raise ValueError('URLScan limit must be a positive integer') self.word = word + self.limit = limit self.totalhosts: set = set() self.totalips: set = set() self.urls: set = set() @@ -144,63 +147,67 @@ class SearchUrlscan: collected_at = datetime.now(UTC) cursor = None seen_cursors: set[str] = set() + records_seen = 0 malformed = False - for _ in range(self.MAX_PAGES): - params = {'q': f'domain:{self.word}'} - if cursor is not None: - params['search_after'] = cursor - try: - response = await AsyncFetcher.fetch( - url=url, - params=params, - json=True, - proxy=self.proxy, - request_timeout=60, - include_metadata=True, - ) - except Exception as error: - self._stop('failed', 'transport-error') - logger.info('URLScan request failed: %s', type(error).__name__) - return + try: + async with AsyncFetcher.open_session(proxy=self.proxy) as session: + while records_seen < self.limit: + remaining = self.limit - records_seen + params: dict[str, str | int] = { + 'q': f'domain:{self.word}', + 'size': min(self.MAX_PAGE_SIZE, remaining), + } + if cursor is not None: + params['search_after'] = cursor + response = await AsyncFetcher.fetch( + session=session, + url=url, + params=params, + json=True, + include_metadata=True, + ) - if not isinstance(response, FetcherResponse): - self._stop('failed', 'transport-error') - return - if response.status == 429: - self._stop('rate-limited', 'http-429') - return - if response.status in {401, 403}: - self._stop('failed', 'access-denied') - return - if not 200 <= response.status < 300: - self._stop('failed', f'http-{response.status}') - return - if not isinstance(response.body, dict) or not isinstance(response.body.get('results'), list): - self._stop('failed', 'invalid-response') - return + if error := provider_http_error(response): + self._stop(*error) + return + assert isinstance(response, FetcherResponse) + if not isinstance(response.body, dict) or not isinstance(response.body.get('results'), list): + self._stop('failed', 'invalid-response') + return - results = response.body['results'] - if not results: - if malformed: - self._stop('failed', 'invalid-response') - else: - self.execution_status = 'completed' - self.stop_reason = None if self._has_results() else 'no-results' - return + results = response.body['results'] + if not results: + if malformed: + self._stop('failed', 'invalid-response') + else: + self.execution_status = 'completed' + self.stop_reason = None if self._has_results() else 'no-results' + return - malformed = self._parse_results(results, collected_at) or malformed - next_cursor = self._cursor(results[-1]) - if next_cursor is None: - self._stop('failed', 'invalid-cursor') - return - if next_cursor in seen_cursors: - self._stop('failed', 'repeated-cursor') - return - seen_cursors.add(next_cursor) - cursor = next_cursor + page_results = results[:remaining] + records_seen += len(page_results) + malformed = self._parse_results(page_results, collected_at) or malformed + if records_seen >= self.limit: + break + next_cursor = self._cursor(page_results[-1]) + if next_cursor is None: + self._stop('failed', 'invalid-cursor') + return + if next_cursor in seen_cursors: + self._stop('failed', 'repeated-cursor') + return + seen_cursors.add(next_cursor) + cursor = next_cursor + except Exception as error: + self._stop('failed', 'transport-error') + logger.info('URLScan request failed: %s', type(error).__name__) + return - self.execution_status = 'partial' - self.stop_reason = 'page-limit' + if self.execution_status is not None and self._has_results(): + self.execution_status = 'partial' + elif self.execution_status is None: + self.execution_status = 'completed' + self.stop_reason = None if self._has_results() else 'no-results' async def get_hostnames(self) -> set: return self.totalhosts diff --git a/theHarvester/discovery/virustotal.py b/theHarvester/discovery/virustotal.py index f703859c..e040f49b 100644 --- a/theHarvester/discovery/virustotal.py +++ b/theHarvester/discovery/virustotal.py @@ -1,98 +1,139 @@ -import asyncio +from __future__ import annotations + +from typing import Any from theHarvester.discovery.constants import MissingKey -from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.discovery.provider_response import provider_http_error +from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse +from theHarvester.lib.hostnames import normalize_scoped_hostname class SearchVirustotal: - def __init__(self, word) -> None: + def __init__(self, word: str, limit: int) -> None: + if isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0: + raise ValueError('VirusTotal limit must be a positive integer') self.key = Core.virustotal_key() - if self.key is None: + if not isinstance(self.key, str) or not self.key.strip(): raise MissingKey('virustotal') self.word = word + self.limit = limit self.proxy = False - self.hostnames: list = [] + self.hostnames: set[str] = set() + self.execution_status: str | None = None + self.stop_reason: str | None = None + + def _stop(self, status: str, reason: str) -> None: + self.execution_status = 'partial' if self.hostnames else status + self.stop_reason = reason async def do_search(self) -> None: - # TODO determine if more endpoints can yield useful info given a domain - # based on: https://developers.virustotal.com/reference/domains-relationships - # base_url = "https://www.virustotal.com/api/v3/domains/domain/subdomains?limit=40" - headers = { - 'User-Agent': Core.get_user_agent(), - 'Accept': 'application/json', - 'x-apikey': self.key, - } - base_url = f'https://www.virustotal.com/api/v3/domains/{self.word}/subdomains?limit=40' - cursor = '' - count = 0 - fail_counter = 0 - counter = 0 - breakcon = False - while True: - if breakcon: - break - # rate limit is 4 per minute - # TODO add timer logic if proven to be needed - # in the meantime sleeping 16 seconds should eliminate hitting the rate limit - # in case rate limit is hit, fail counter exists and sleep for 65 seconds - send_url = base_url + '&cursor=' + cursor if cursor != '' and len(cursor) > 2 else base_url - responses = await AsyncFetcher.fetch_all([send_url], headers=headers, proxy=self.proxy, json=True) - jdata = responses[0] - if 'data' not in jdata: - await asyncio.sleep(60 + 5) - fail_counter += 1 - if 'meta' in jdata: - cursor = jdata['meta']['cursor'] if 'cursor' in jdata['meta'] else '' - if len(cursor) == 0 and 'data' in jdata: - # if cursor no longer is within the meta field have hit last entry - breakcon = True - count += jdata['meta']['count'] - if count == 0 or fail_counter >= 2: - break - if 'data' in jdata: - data = jdata['data'] - self.hostnames.extend(await self.parse_hostnames(data, self.word)) - counter += 1 - await asyncio.sleep(16) - self.hostnames = list(sorted(set(self.hostnames))) - # verify domains such as x.x.com.multicdn.x.com are parsed properly - self.hostnames = [ - host for host in self.hostnames if ((len(host.split('.')) >= 3) and host.split('.')[-2] == self.word.split('.')[-2]) - ] + headers = {'Accept': 'application/json', 'x-apikey': self.key} + cursor: str | None = None + seen_cursors: set[str] = set() + records_seen = 0 + try: + async with AsyncFetcher.open_session(headers=headers, proxy=self.proxy) as session: + while records_seen < self.limit: + remaining = self.limit - records_seen + params: dict[str, int | str] = {'limit': min(40, remaining)} + if cursor: + params['cursor'] = cursor + response = await AsyncFetcher.fetch( + session=session, + url=f'https://www.virustotal.com/api/v3/domains/{self.word}/subdomains', + params=params, + json=True, + include_metadata=True, + ) + if error := provider_http_error(response): + self._stop(*error) + return + assert isinstance(response, FetcherResponse) + if not isinstance(response.body, dict): + self._stop('failed', 'invalid-response') + return + data = response.body.get('data') + meta = response.body.get('meta', {}) + if not isinstance(data, list) or not isinstance(meta, dict): + self._stop('failed', 'invalid-response') + return + page_data = data[:remaining] + records_seen += len(page_data) + hostnames, malformed = self.parse_hostnames(page_data, self.word) + for hostname in sorted(hostnames): + if len(self.hostnames) >= self.limit: + break + self.hostnames.add(hostname) + if malformed: + self._stop('failed', 'invalid-response') + next_cursor = meta.get('cursor') + if not data or not isinstance(next_cursor, str) or not next_cursor: + break + if next_cursor in seen_cursors: + self._stop('failed', 'repeated-cursor') + break + seen_cursors.add(next_cursor) + cursor = next_cursor + except Exception: + self._stop('failed', 'transport-error') + return - async def get_hostnames(self) -> list: + if self.execution_status is not None and self.hostnames: + self.execution_status = 'partial' + elif self.execution_status is None: + self.execution_status = 'completed' + self.stop_reason = None if self.hostnames else 'no-results' + + async def get_hostnames(self) -> set[str]: return self.hostnames @staticmethod - async def parse_hostnames(data, word): - total_subdomains: set[str] = set() - for attribute in data: - total_subdomains.add(attribute['id'].replace('"', '')) - attributes = attribute['attributes'] - total_subdomains.update( - {value['value'].replace('"', '') for value in attributes['last_dns_records'] if word in value['value']} - ) - if 'last_https_certificate' in attributes: - total_subdomains.update( - { - value.replace('"', '') - for value in attributes['last_https_certificate']['extensions']['subject_alternative_name'] - if word in value - } - ) - # Convert to list for further processing without changing variable type mid-function - subdomains_list: list[str] = list(sorted(total_subdomains)) - # Other false positives may occur over time and yes there are other ways to parse this, feel free to implement - # them and submit a PR or raise an issue if you run into this filtering not being enough - # TODO determine if parsing 'v=spf1 include:_spf-x.acme.com include:_spf-x.acme.com' is worth parsing - subdomains_list = [ - x - for x in subdomains_list - if 'edgekey.net' not in str(x) and 'akadns.net' not in str(x) and 'include:_spf' not in str(x) - ] - subdomains_list.sort() - return subdomains_list + def parse_hostnames(data: list[Any], word: str) -> tuple[set[str], bool]: + hostnames: set[str] = set() + malformed = False + + def add(value: Any) -> None: + nonlocal malformed + if not isinstance(value, str): + malformed = True + return + if hostname := normalize_scoped_hostname(value.replace('"', ''), word): + hostnames.add(hostname) + + for item in data: + if not isinstance(item, dict): + malformed = True + continue + add(item.get('id')) + attributes = item.get('attributes', {}) + if not isinstance(attributes, dict): + malformed = True + continue + records = attributes.get('last_dns_records', []) + if not isinstance(records, list): + malformed = True + else: + for record in records: + if not isinstance(record, dict): + malformed = True + else: + add(record.get('value')) + certificate = attributes.get('last_https_certificate') + if certificate is None: + continue + if not isinstance(certificate, dict) or not isinstance(certificate.get('extensions'), dict): + malformed = True + continue + names = certificate['extensions'].get('subject_alternative_name', []) + if not isinstance(names, list): + malformed = True + continue + for name in names: + add(name) + return hostnames, malformed async def process(self, proxy: bool = False) -> None: self.proxy = proxy + self.execution_status = None + self.stop_reason = None await self.do_search() diff --git a/theHarvester/discovery/whoisxml.py b/theHarvester/discovery/whoisxml.py index 1da2a0be..0c244dda 100644 --- a/theHarvester/discovery/whoisxml.py +++ b/theHarvester/discovery/whoisxml.py @@ -1,38 +1,92 @@ +from __future__ import annotations + from theHarvester.discovery.constants import MissingKey -from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.discovery.provider_response import provider_http_error +from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse +from theHarvester.lib.hostnames import normalize_scoped_hostname class SearchWhoisXML: - def __init__(self, word) -> None: + def __init__(self, word: str, limit: int) -> None: + if isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0: + raise ValueError('WhoisXML limit must be a positive integer') self.word = word + self.limit = limit self.key = Core.whoisxml_key() - if self.key is None: + if not isinstance(self.key, str) or not self.key.strip(): raise MissingKey('whoisxml') - self.total_results: list[str] = [] - self.proxy: bool = False + self.total_results: set[str] = set() + self.proxy = False + self.execution_status: str | None = None + self.stop_reason: str | None = None - async def do_search(self): - # https://subdomains.whoisxmlapi.com/api/documentation/making-requests - url = 'https://subdomains.whoisxmlapi.com/api/v1' - params = {'apiKey': self.key, 'domainName': self.word} - response = await AsyncFetcher.fetch_all( - [url], - json=True, - params=params, - headers={'User-Agent': Core.get_user_agent()}, - proxy=self.proxy, - ) - # Parse the response according to the example JSON structure: - # {"search":"example.com.com","result":{"count":10000,"records":[{"domain":"test.example.com","firstSeen":1678169400,"lastSeen":1678169400}]}} - self.total_results = [] - if response and response[0]: - # Extract domains from the records array - if 'result' in response[0] and 'records' in response[0]['result']: - self.total_results = [record['domain'] for record in response[0]['result']['records']] + def _stop(self, status: str, reason: str) -> None: + self.execution_status = 'partial' if self.total_results else status + self.stop_reason = reason - async def get_hostnames(self): + async def do_search(self) -> None: + cursor: str | None = None + seen_cursors: set[str] = set() + records_seen = 0 + async with AsyncFetcher.open_session(proxy=self.proxy) as session: + while records_seen < self.limit: + params = {'apiKey': self.key, 'domainName': self.word} + if cursor is not None: + params['searchAfter'] = cursor + response = await AsyncFetcher.fetch( + session=session, + url='https://subdomains.whoisxmlapi.com/api/v2', + params=params, + json=True, + include_metadata=True, + ) + if error := provider_http_error(response): + self._stop(*error) + return + assert isinstance(response, FetcherResponse) + if not isinstance(response.body, dict): + self._stop('failed', 'invalid-response') + return + result = response.body.get('result') + if not isinstance(result, dict) or not isinstance(result.get('records'), list): + self._stop('failed', 'invalid-response') + return + next_cursor = result.get('nextPageSearchAfter') + if not isinstance(next_cursor, str): + self._stop('failed', 'invalid-response') + return + + remaining = self.limit - records_seen + records = result['records'][:remaining] + records_seen += len(records) + malformed = False + for record in records: + if not isinstance(record, dict) or not isinstance(record.get('domain'), str): + malformed = True + continue + if hostname := normalize_scoped_hostname(record['domain'], self.word): + self.total_results.add(hostname) + if malformed: + self._stop('failed', 'invalid-response') + if records_seen >= self.limit or not next_cursor: + break + if next_cursor in seen_cursors: + self._stop('failed', 'repeated-cursor') + break + seen_cursors.add(next_cursor) + cursor = next_cursor + if self.execution_status is None: + self.execution_status = 'completed' + self.stop_reason = None if self.total_results else 'no-results' + + async def get_hostnames(self) -> set[str]: return self.total_results async def process(self, proxy: bool = False) -> None: self.proxy = proxy - await self.do_search() + self.execution_status = None + self.stop_reason = None + try: + await self.do_search() + except Exception: + self._stop('failed', 'transport-error') diff --git a/theHarvester/discovery/zoomeyesearch.py b/theHarvester/discovery/zoomeyesearch.py index fc07afa2..6ea42feb 100644 --- a/theHarvester/discovery/zoomeyesearch.py +++ b/theHarvester/discovery/zoomeyesearch.py @@ -1,390 +1,236 @@ -import asyncio -import logging +from __future__ import annotations + +import base64 import math import re -from collections.abc import Iterable +from ipaddress import ip_address from typing import Any -from urllib.parse import urlparse +from urllib.parse import urlsplit, urlunsplit -from theHarvester.discovery.constants import MissingKey, get_delay -from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.discovery.constants import MissingKey +from theHarvester.discovery.provider_response import provider_http_error +from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse from theHarvester.lib.hostnames import normalize_scoped_hostname from theHarvester.parsers import myparser -logger = logging.getLogger(__name__) - class SearchZoomEye: - def __init__(self, word, limit) -> None: - self.word = word - self.limit = limit - self.key = Core.zoomeye_key() - # NOTE for ZoomEye you get a system recharge on the 1st of every month - # Which resets your balance to 10000 requests - # If you wish to extract as many subdomains as possible visit the fetch_subdomains - # To see how - if self.key is None: + PAGE_SIZE = 10_000 + RESPONSE_FIELDS = ','.join( + ( + 'ip', + 'domain', + 'hostname', + 'rdns', + 'asn', + 'url', + 'banner', + 'header', + 'body', + 'ssl', + ) + ) + URL_PATTERN = re.compile(r'https?://[^\s"\'<>]+') + + def __init__(self, word: str, limit: int) -> None: + if isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0: + raise ValueError('ZoomEye limit must be a positive integer') + key = Core.zoomeye_key() + if not isinstance(key, str) or not key.strip(): raise MissingKey('zoomeye') - # API v2 base - self.baseurl = 'https://api.zoomeye.ai/host/search' - self.domain_url = 'https://api.zoomeye.ai/domain/search' + self.word = word + self.target = word.strip().lower().removeprefix('www.').rstrip('.') + self.limit = limit + self.key = key + self.baseurl = 'https://api.zoomeye.ai/v2/search' self.proxy = False - self.totalasns: list = list() - self.totalhosts: list = list() - self.urls: list = list() - self.totalips: list = list() - self.totalemails: list = list() - # Regex used is directly from: https://github.com/GerbenJavado/LinkFinder/blob/master/linkfinder.py#L29 - # Maybe one day it will be a pip package - # Regardless LinkFinder is an amazing tool! - regex_str = r""" - (?:"|') # Start newline delimiter - ( - ((?:[a-zA-Z]{1,10}://|//) # Match a scheme [a-Z]*1-10 or // - [^"'/]{1,}\. # Match a domainname (any character + dot) - [a-zA-Z]{2,}[^"']{0,}) # The domainextension and/or path - | - ((?:/|\.\./|\./) # Start with /,../,./ - [^"'><,;| *()(%%$^/\\\[\]] # Next character can't be... - [^"'><,;|()]{1,}) # Rest of the characters can't be - | - ([a-zA-Z0-9_\-/]{1,}/ # Relative endpoint with / - [a-zA-Z0-9_\-/]{1,} # Resource name - \.(?:[a-zA-Z]{1,4}|action) # Rest + extension (length 1-4 or action) - (?:[\?|#][^"|']{0,}|)) # ? or # mark with parameters - | - ([a-zA-Z0-9_\-/]{1,}/ # REST API (no extension) with / - [a-zA-Z0-9_\-/]{3,} # Proper REST endpoints usually have 3+ chars - (?:[\?|#][^"|']{0,}|)) # ? or # mark with parameters - | - ([a-zA-Z0-9_\-]{1,} # filename - \.(?:php|asp|aspx|jsp|json| - action|html|js|txt|xml) # . + extension - (?:[\?|#][^"|']{0,}|)) # ? or # mark with parameters - ) - (?:"|') # End newline delimiter - """ - self.url_regex = re.compile(regex_str, re.VERBOSE) + self.totalasns: set[str] = set() + self.totalhosts: set[str] = set() + self.urls: set[str] = set() + self.totalips: set[str] = set() + self.totalemails: set[str] = set() + self.execution_status: str | None = None + self.stop_reason: str | None = None - def _build_headers(self) -> dict[str, str]: - # API v2 uses API-KEY header - return {'API-KEY': self.key, 'User-Agent': Core.get_user_agent()} + def _stop(self, status: str, reason: str) -> None: + has_results = any((self.totalhosts, self.totalemails, self.totalips, self.totalasns, self.urls)) + self.execution_status = 'partial' if has_results else status + self.stop_reason = reason - @staticmethod - def _is_success(resp: dict[str, Any]) -> bool: - # Accept multiple success indicators across versions + def _normalize_url(self, value: Any) -> str | None: + if not isinstance(value, str): + return None try: - if 'code' in resp: - # v2 style often uses code==0 for success - return resp.get('code') in (0, 200) - if 'status' in resp and isinstance(resp.get('status'), int): - # some responses put HTTP-like code here - return resp.get('status') in (0, 200) - except Exception as e: - logger.info(f'ZoomEye response status parsing failed with {type(e).__name__}') - return False + parsed = urlsplit(value.rstrip('),.;')) + except ValueError: + return None + hostname = normalize_scoped_hostname(parsed.hostname, self.target) + if parsed.scheme not in {'http', 'https'} or not parsed.netloc or hostname is None: + return None + if parsed.username is not None or parsed.password is not None: + return None + try: + port = f':{parsed.port}' if parsed.port is not None else '' + except ValueError: + return None + return urlunsplit((parsed.scheme, f'{hostname}{port}', parsed.path, parsed.query, '')) - # If no explicit status, assume success and let parsing validate - return True - - @staticmethod - def _unwrap_data(resp: dict[str, Any]) -> dict[str, Any]: - # Many v2 endpoints return {'code':0,'data':{...}} - data = resp.get('data') - return data if isinstance(data, dict) else resp - - @staticmethod - def _page_total_from_payload(payload: dict[str, Any], page_size: int) -> int: - # Prefer explicit page total if provided - if 'available' in payload: - try: - return int(payload['available']) - except ValueError: - logger.info('Payload availablity is not a integer') - except Exception as e: - logger.info(f'An error occurred in page_total_from_payload : {e}') - total_results = payload.get('total') or payload.get('count') or payload.get('total_count') - if isinstance(total_results, int) and total_results >= 0: - size = payload.get('size') or page_size - try: - size_int = int(size) - size_int = size_int if size_int > 0 else page_size - except Exception: - size_int = page_size - return max(1, math.ceil(total_results / size_int)) - # Fallback: if a list/matches is present, consider at least one page - if any(k in payload for k in ('matches', 'list', 'results', 'items')): - return 1 - return 1 - - @staticmethod - def _safe_add_hostname(container: set, value: str | None) -> None: - if not value or not isinstance(value, str): - return - v = value.strip() - if not v: - return - v = v.removesuffix('.') - container.add(v) - - async def fetch_subdomains(self) -> None: - headers = self._build_headers() - # type=0 for subdomain search per docs - size = 30 - params = (('q', self.word), ('type', '0'), ('page', '1'), ('size', str(size))) - response = await AsyncFetcher.fetch_all( - [self.domain_url], + async def _fetch_page(self, session: Any, page: int, page_size: int) -> dict[str, Any] | None: + query = base64.b64encode(f'domain="{self.target}"'.encode()).decode() + response = await AsyncFetcher.post_fetch( + self.baseurl, + session=session, json=True, - proxy=self.proxy, - headers=headers, - params=params, + include_metadata=True, + json_body={ + 'qbase64': query, + 'sub_type': 'all', + 'page': page, + 'pagesize': page_size, + 'fields': self.RESPONSE_FIELDS, + }, ) - if not response: + if error := provider_http_error(response): + self._stop(*error) + return None + assert isinstance(response, FetcherResponse) + if not isinstance(response.body, dict): + self._stop('failed', 'invalid-response') + return None + if response.body.get('code') != 60000: + self._stop('failed', 'provider-error') + return None + data = response.body.get('data') + total = response.body.get('total') + if not isinstance(data, list) or isinstance(total, bool) or not isinstance(total, int) or total < 0: + self._stop('failed', 'invalid-response') + return None + return response.body + + async def do_search(self, session: Any) -> None: + page_size = min(self.PAGE_SIZE, self.limit) + first = await self._fetch_page(session, 1, page_size) + if first is None: return - raw = response[0] or {} - if not self._is_success(raw): - return - payload = self._unwrap_data(raw) - total_pages = self._page_total_from_payload(payload, size) - # If user requested more pages than available, clamp to available - self.limit = min(self.limit, total_pages) if total_pages >= 1 else self.limit + await self._store_matches(first['data'][:page_size]) + page_limit = math.ceil(min(first['total'], self.limit) / page_size) if first['total'] else 1 + for page in range(2, page_limit + 1): + remaining = self.limit - ((page - 1) * page_size) + response = await self._fetch_page(session, page, page_size) + if response is None: + return + await self._store_matches(response['data'][:remaining]) - # Parse first page - first_list = payload.get('list') or payload.get('results') or [] - self.totalhosts.extend( - [item.get('name') or item.get('domain') or item.get('host') for item in first_list if isinstance(item, dict)] - ) + async def _store_matches(self, matches: list[Any]) -> None: + hostnames, emails, ips, asns, urls, malformed = await self.parse_matches(matches) + self.totalhosts.update(hostnames) + self.totalemails.update(emails) + self.totalips.update(ips) + self.totalasns.update(asns) + self.urls.update(urls) + if malformed: + self._stop('failed', 'invalid-response') - # Iterate remaining pages - for i in range(2, self.limit + 1): - params = (('q', self.word), ('type', '0'), ('page', str(i)), ('size', str(size))) - response = await AsyncFetcher.fetch_all( - [self.domain_url], - json=True, - proxy=self.proxy, - headers=headers, - params=params, - ) - if not response: - break - raw = response[0] or {} - if not self._is_success(raw): - break - payload = self._unwrap_data(raw) - page_list = payload.get('list') or payload.get('results') or [] - found_subdomains = [ - item.get('name') or item.get('domain') or item.get('host') for item in page_list if isinstance(item, dict) - ] - found_subdomains = [x for x in found_subdomains if x] - if not found_subdomains: - break - self.totalhosts.extend(found_subdomains) - if i % 10 == 0: - await asyncio.sleep(get_delay() + 1) - - async def do_search(self) -> None: - headers = self._build_headers() - # Fetch subdomains first - await self.fetch_subdomains() - - size = 20 - params = (('query', f'site:{self.word}'), ('page', '1'), ('size', str(size))) - response = await AsyncFetcher.fetch_all([self.baseurl], json=True, proxy=self.proxy, headers=headers, params=params) - if not response: - return - raw = response[0] or {} - payload = self._unwrap_data(raw) - total_pages = self._page_total_from_payload(payload, size) - self.limit = min(self.limit, total_pages) if total_pages >= 1 else self.limit - cur_page = 2 if self.limit >= 2 else -1 - - nomatches_counter = 0 - - def extract_matches(p: dict[str, Any]) -> Iterable[dict]: - return p.get('matches') or p.get('list') or p.get('results') or [] - - if cur_page == -1: - if isinstance(payload, dict): - matches = extract_matches(payload) - if matches: - hostnames, emails, ips, asns, urls = await self.parse_matches(matches) - self.totalhosts.extend(hostnames) - self.totalemails.extend(emails) - self.totalips.extend(ips) - self.totalasns.extend(asns) - self.urls.extend(urls) - return - - # Parse first page then loop - if isinstance(payload, dict): - matches = extract_matches(payload) - if matches: - hostnames, emails, ips, asns, urls = await self.parse_matches(matches) - self.totalhosts.extend(hostnames) - self.totalemails.extend(emails) - self.totalips.extend(ips) - self.totalasns.extend(asns) - self.urls.extend(urls) - - for num in range(2, self.limit + 1): - params = (('query', f'site:{self.word}'), ('page', str(num)), ('size', str(size))) - response = await AsyncFetcher.fetch_all( - [self.baseurl], - json=True, - proxy=self.proxy, - headers=headers, - params=params, - ) - if not response: - break - raw = response[0] or {} - payload = self._unwrap_data(raw) - matches = extract_matches(payload) - if not matches: - nomatches_counter += 1 - if nomatches_counter >= 5: - break - continue - - hostnames, emails, ips, asns, urls = await self.parse_matches(matches) - - if len(hostnames) == 0 and len(emails) == 0 and len(ips) == 0 and len(asns) == 0 and len(urls) == 0: - nomatches_counter += 1 - if nomatches_counter >= 5: - break - - self.totalhosts.extend(hostnames) - self.totalemails.extend(emails) - self.totalips.extend(ips) - self.totalasns.extend(asns) - self.urls.extend(urls) - - if num % 10 == 0: - await asyncio.sleep(get_delay() + 1) - - async def parse_matches(self, matches): - # Helper function to parse items from match json + async def parse_matches( + self, + matches: list[Any], + ) -> tuple[set[str], set[str], set[str], set[str], set[str], bool]: ips: set[str] = set() urls: set[str] = set() hostnames: set[str] = set() asns: set[str] = set() emails: set[str] = set() + malformed = False for match in matches: if not isinstance(match, dict): + malformed = True continue - try: - # IPs - ip = match.get('ip') or match.get('ip_str') or match.get('ip_str_v4') or match.get('address') - if isinstance(ip, str): - ips.add(ip) + raw_ip = match.get('ip') + if raw_ip is not None: + try: + ips.add(str(ip_address(str(raw_ip).strip()))) + except ValueError: + malformed = True - # ASNs - asn_val = None - if isinstance(match.get('geoinfo'), dict): - asn_val = match['geoinfo'].get('asn') - asn_val = asn_val or match.get('asn') - if asn_val: - try: - asns.add(f'AS{int(asn_val)}') - except Exception: - # if already a string like 'AS12345' - asns.add(str(asn_val) if str(asn_val).startswith('AS') else f'AS{asn_val!s}') + raw_asn = match.get('asn') + if raw_asn is not None: + try: + asns.add(f'AS{int(str(raw_asn).removeprefix("AS"))}') + except ValueError: + malformed = True - # Reverse DNS and hostnames - rdns_new = match.get('rdns_new') - if isinstance(rdns_new, str) and rdns_new: - if ',' in rdns_new: - parts = str(rdns_new).split(',') - primary = parts[0] - secondary = parts[1] if len(parts) == 2 else None - if primary: - self._safe_add_hostname(hostnames, primary) - if secondary: - self._safe_add_hostname(hostnames, secondary) - else: - self._safe_add_hostname(hostnames, rdns_new) + for field in ('domain', 'hostname', 'rdns'): + value = match.get(field) + if value is None: + continue + if not isinstance(value, str): + malformed = True + elif (hostname := normalize_scoped_hostname(value, self.target)) and hostname != self.target: + hostnames.add(hostname) - rdns = match.get('rdns') - if isinstance(rdns, str) and rdns: - self._safe_add_hostname(hostnames, rdns) + if raw_url := match.get('url'): + if normalized_url := self._normalize_url(raw_url): + urls.add(normalized_url) + if url_hostname := normalize_scoped_hostname(urlsplit(normalized_url).hostname, self.target): + if url_hostname != self.target: + hostnames.add(url_hostname) + elif isinstance(raw_url, str): + malformed = True - # Additional hostname-like fields - for f in ('hostname', 'host', 'domain', 'site', 'fqdn'): - self._safe_add_hostname(hostnames, match.get(f)) - for f in ('hostnames', 'domains', 'names'): - vals = match.get(f) - if isinstance(vals, list): - for v in vals: - if isinstance(v, str): - self._safe_add_hostname(hostnames, v) + text_values: list[str] = [] + for field in ('banner', 'header', 'body', 'ssl'): + value = match.get(field) + if value is None: + continue + if isinstance(value, str): + text_values.append(value) + else: + malformed = True + if not text_values: + continue + content = '\n'.join(text_values) + parser = myparser.Parser(content, self.word) + emails.update(await parser.emails()) + parser = myparser.Parser(content, self.word) + hostnames.update(hostname for hostname in await parser.hostnames() if hostname != self.target) + for candidate in self.URL_PATTERN.findall(content): + if normalized_url := self._normalize_url(candidate): + urls.add(normalized_url) - # Banner/content extraction for emails, hostnames, and URLs - banners = [] - - portinfo = match.get('portinfo') - if isinstance(portinfo, dict): - b = portinfo.get('banner') - if isinstance(b, str) and b: - banners.append(b) - - service = match.get('service') - if isinstance(service, dict): - for key in ('banner', 'data', 'raw'): - v = service.get(key) - if isinstance(v, str) and v: - banners.append(v) - http = service.get('http') - if isinstance(http, dict): - for key in ('title', 'html', 'body', 'server', 'raw'): - v = http.get(key) - if isinstance(v, str) and v: - banners.append(v) - - content_blob = '\n'.join(banners) - if content_blob: - temp_emails = set(await self.parse_emails(content_blob)) - emails.update(temp_emails) - hostnames.update(set(await self.parse_hostnames(content_blob))) - for url_match in re.finditer(self.url_regex, content_blob): - candidate = str(url_match.group(1)).replace('"', '') - try: - parsed = urlparse(candidate) - hostname = normalize_scoped_hostname(parsed.hostname, self.word) - except ValueError: - continue - if parsed.scheme in {'http', 'https'} and parsed.netloc and hostname: - urls.add(candidate) - - except Exception as e: - # Continue processing other matches instead of failing completely - logger.info(f'ZoomEye parsing error: {e}') - - return hostnames, emails, ips, asns, urls + return hostnames, emails, ips, asns, urls, malformed async def process(self, proxy: bool = False) -> None: self.proxy = proxy - await self.do_search() # Only need to do it once. + self.execution_status = None + self.stop_reason = None + try: + async with AsyncFetcher.open_session( + headers={'API-KEY': self.key, 'Content-Type': 'application/json'}, + proxy=proxy, + ) as session: + await self.do_search(session) + except Exception: + self._stop('failed', 'transport-error') + return + has_results = any((self.totalhosts, self.totalemails, self.totalips, self.totalasns, self.urls)) + if self.execution_status is not None and has_results: + self.execution_status = 'partial' + elif self.execution_status is None: + self.execution_status = 'completed' + self.stop_reason = None if has_results else 'no-results' - async def parse_emails(self, content): - rawres = myparser.Parser(content, self.word) - return await rawres.emails() + async def get_hostnames(self) -> set[str]: + return self.totalhosts - async def parse_hostnames(self, content): - rawres = myparser.Parser(content, self.word) - return await rawres.hostnames() + async def get_emails(self) -> set[str]: + return self.totalemails - async def get_hostnames(self): - return set(self.totalhosts) + async def get_ips(self) -> set[str]: + return self.totalips - async def get_emails(self): - return set(self.totalemails) + async def get_asns(self) -> set[str]: + return self.totalasns - async def get_ips(self): - return set(self.totalips) - - async def get_asns(self): - return set(self.totalasns) - - async def get_urls(self): - return set(self.urls) + async def get_urls(self) -> set[str]: + return self.urls diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index 28fec96d..1f14bd34 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -945,13 +945,30 @@ class AsyncFetcher: @classmethod async def fetch_all( cls, - urls, - headers=None, + urls: list[str], + headers: dict[str, str] | None = None, params: Sized = '', json: bool = False, - proxy: bool = False, + proxy: str | bool | None = False, include_metadata: bool = False, - ) -> list: + *, + session: aiohttp.ClientSession | None = None, + ) -> list[Any]: + if session is not None: + return list( + await asyncio.gather( + *[ + AsyncFetcher.fetch( + session=session, + url=url, + params=params, + json=json, + include_metadata=include_metadata, + ) + for url in urls + ] + ) + ) # By default, timeout is 5 minutes; 60 seconds should suffice headers = cls._default_headers(headers) timeout = cls._request_timeout(60) diff --git a/theHarvester/lib/source_runner.py b/theHarvester/lib/source_runner.py index 7f5f37c2..07adb3e3 100644 --- a/theHarvester/lib/source_runner.py +++ b/theHarvester/lib/source_runner.py @@ -143,7 +143,7 @@ SOURCE_FACTORIES: dict[str, SourceFactory] = { 'dnsdumpster': lambda request: search_dnsdumpster.SearchDNSDumpster(request.target), 'duckduckgo': lambda request: duckduckgosearch.SearchDuckDuckGo(request.target, request.limit), 'dymo': lambda request: dymosearch.SearchDymo(request.target), - 'fofa': lambda request: fofa.SearchFofa(request.target), + 'fofa': lambda request: fofa.SearchFofa(request.target, request.limit), 'fullhunt': lambda request: fullhuntsearch.SearchFullHunt(request.target), 'github-code': lambda request: githubcode.SearchGithubCode(request.target, request.limit), 'gitlab': lambda request: gitlabsearch.SearchGitlab(request.target), @@ -152,13 +152,13 @@ SOURCE_FACTORIES: dict[str, SourceFactory] = { 'hibpverified': lambda request: hibpverified.SearchHibpVerified(request.target), 'hudsonrock': lambda request: hudsonrocksearch.SearchHudsonRock(request.target), 'hunter': lambda request: huntersearch.SearchHunter(request.target, request.limit, request.start), - 'hunterhow': lambda request: searchhunterhow.SearchHunterHow(request.target), + 'hunterhow': lambda request: searchhunterhow.SearchHunterHow(request.target, request.limit), 'intelx': lambda request: intelxsearch.SearchIntelx(request.target), 'leakix': lambda request: leakix.SearchLeakix(request.target), 'leaklookup': lambda request: leaklookup.SearchLeakLookup(request.target), 'mojeek': lambda request: mojeek.SearchMojeek(request.target, request.limit), 'netlas': lambda request: netlas.SearchNetlas(request.target, request.limit), - 'onyphe': lambda request: onyphe.SearchOnyphe(request.target), + 'onyphe': lambda request: onyphe.SearchOnyphe(request.target, request.limit), 'otx': lambda request: otxsearch.SearchOtx(request.target), 'pentesttools': lambda request: pentesttools.SearchPentestTools(request.target), 'projectdiscovery': lambda request: projectdiscovery.SearchDiscovery(request.target), @@ -166,7 +166,7 @@ SOURCE_FACTORIES: dict[str, SourceFactory] = { 'robtex': lambda request: robtex.SearchRobtex(request.target), 'rocketreach': lambda request: rocketreach.SearchRocketReach(request.target, request.limit), 'securityTrails': lambda request: securitytrailssearch.SearchSecuritytrail(request.target), - 'securityscorecard': lambda request: securityscorecard.SearchSecurityScorecard(request.target), + 'securityscorecard': lambda request: securityscorecard.SearchSecurityScorecard(request.target, request.limit), 'sherlockeye': lambda request: sherlockeye.SearchSherlockeye(request.target), 'shodan': lambda request: shodansearch.SearchShodan(request.target), 'shodanInternetDB': lambda request: shodan_internetdb.SearchShodanInternetDB(request.target), @@ -176,10 +176,10 @@ SOURCE_FACTORIES: dict[str, SourceFactory] = { 'subdomainfinderc99': lambda request: subdomainfinderc99.SearchSubdomainfinderc99(request.target), 'thc': lambda request: thc.SearchThc(request.target), 'tomba': lambda request: tombasearch.SearchTomba(request.target, request.limit, request.start), - 'urlscan': lambda request: urlscan.SearchUrlscan(request.target), - 'virustotal': lambda request: virustotal.SearchVirustotal(request.target), + 'urlscan': lambda request: urlscan.SearchUrlscan(request.target, request.limit), + 'virustotal': lambda request: virustotal.SearchVirustotal(request.target, request.limit), 'waybackarchive': lambda request: waybackarchive.SearchWaybackarchive(request.target, request.limit), - 'whoisxml': lambda request: whoisxml.SearchWhoisXML(request.target), + 'whoisxml': lambda request: whoisxml.SearchWhoisXML(request.target, request.limit), 'windvane': lambda request: windvane.SearchWindvane(request.target), 'yahoo': lambda request: yahoosearch.SearchYahoo(request.target, request.limit), 'zoomeye': lambda request: zoomeyesearch.SearchZoomEye(request.target, request.limit),