From f44807a0468fd75bfa42b00a02c303973108b4e2 Mon Sep 17 00:00:00 2001 From: Matt <36310667+NotoriousRebel@users.noreply.github.com> Date: Sat, 15 Aug 2026 01:48:33 -0400 Subject: [PATCH] refactor: run discovery sources with TaskGroup (#2544) * refactor: run discovery sources with TaskGroup * fix: preserve source runner compatibility diagnostics * feat: add configurable source workers --- CHANGELOG.md | 4 + CONTRIBUTING.md | 2 + README.md | 8 +- docs/wiki/How-to-add-a-new-module.md | 10 +- docs/wiki/Rest-API.md | 4 + tests/discovery/test_builtwith.py | 7 +- tests/discovery/test_crtname.py | 4 +- tests/discovery/test_crtsh.py | 6 +- tests/discovery/test_dymosearch.py | 6 +- tests/discovery/test_gitlabsearch.py | 4 +- tests/discovery/test_haveibeenpwned.py | 2 +- tests/discovery/test_hibpverified.py | 2 +- tests/discovery/test_hudsonrock.py | 2 +- tests/discovery/test_leaklookup.py | 4 +- tests/discovery/test_rapiddns.py | 5 +- tests/e2e/test_harvestview.py | 20 +- tests/lib/test_core.py | 26 +- tests/lib/test_enumeration.py | 4 + tests/lib/test_harvestview_ui.py | 2 + tests/lib/test_run_backend.py | 40 +- tests/lib/test_source_catalog.py | 54 +- tests/lib/test_source_runner.py | 835 ++++++++++++ tests/test_all_source_orchestration.py | 70 +- tests/test_main.py | 735 +++++++++- tests/test_no_hosts.py | 7 +- theHarvester/__main__.py | 1204 ++--------------- theHarvester/lib/api/harvestview.py | 2 + theHarvester/lib/api/run_models.py | 7 + theHarvester/lib/api/run_worker.py | 2 + .../lib/api/static/harvestview/app.js | 2 + .../lib/api/static/harvestview/index.html | 4 + theHarvester/lib/core.py | 70 +- theHarvester/lib/enumeration.py | 5 + theHarvester/lib/source_catalog.py | 25 +- theHarvester/lib/source_runner.py | 504 +++++++ 35 files changed, 2368 insertions(+), 1320 deletions(-) create mode 100644 tests/lib/test_source_runner.py create mode 100644 theHarvester/lib/source_runner.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ce746692..62a59ef3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added root contributor and security policies, structured issue forms, repository agent guidance, discovery terminology, and an operator-focused documentation wiki ([d090a29a](https://github.com/laramies/theHarvester/commit/d090a29a), [7c491ef5](https://github.com/laramies/theHarvester/commit/7c491ef5), [8b9d420b](https://github.com/laramies/theHarvester/commit/8b9d420b)). ### Changed +- Routed discovery sources through immutable source jobs with bounded `TaskGroup` ownership, typed outcomes, and native cancellation propagation instead of queuing live coroutine objects. +- Discovery now uses a fixed pool of six source workers by default. CLI `-j` or `--source-workers`, REST + `source_workers`, and HarvestView can set another positive count without skipping sources or limiting their results. - Replaced Shodan's synchronous Python SDK with cancellable async Host API requests that honor configured proxies, query every unique resolved IPv4, paginate target-bound hostname and TLS-certificate searches without an adapter-specific result cap, retain successful partial results, and add no source-local deadline. Shodan now stores one canonical `shodan-host` result per IP with every normalized TCP or UDP service and scoped certificate CN/SAN metadata in native JSONL, SQLite, API, and HarvestView details instead of an escaped JSON value. - Reworked screenshot scans to use one bounded aiohttp session and one shared browser, with isolated per-target contexts, status-based reachability, and deterministic async cleanup. - Migrated BuiltWith to the current v23 Domain API with privacy-preserving request controls, nested result parsing, and truthful partial or failed outcomes. @@ -65,6 +68,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Expanded offline regression coverage for discovery providers, configuration contracts, logging, output, documentation, workflow policy, and scope boundaries. ### Removed +- Removed the inert legacy source identifiers `linkedin`, `netcraft`, `omnisint`, `sublist3r`, and `zoomeyeapi`; use the source catalog and shared factory registry for supported providers. - Removed the obsolete bundled IP-range and resolver snapshots. - Removed the REST API's built-in SlowAPI request limiter and its launcher option without adding a replacement. - Removed the duplicate `chaos` source name and module; ProjectDiscovery remains available through `projectdiscovery` with the same dataset and credential. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8de9d923..b123b857 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,6 +38,8 @@ Use a short branch name that describes the change, such as `fix/certspotter-pagi ### Discovery providers +Register an ordinary provider with one `SourceSpec` catalog entry and one `SOURCE_FACTORIES` entry. Do not edit CLI orchestration, persistence, or output code: the catalog drives selection/help/activity metadata and the shared runner owns construction, collection, and completed-result output. + If you can add coverage for a new or changed discovery provider, keep it focused and mock HTTP, DNS, and provider responses. Tests must not require API keys or external network access. [The Baidu discovery tests](tests/discovery/test_baidusearch.py) are a small example you can copy and adapt. Useful cases include: diff --git a/README.md b/README.md index ec13a785..874a3509 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,10 @@ Query several passive sources: uv run theHarvester -d example.com -b crtsh,certspotter,commoncrawl ``` +Six discovery sources run at once by default. Use `-j` or `--source-workers` with a positive number to change that +concurrency. The worker count never skips a selected source or limits its results, and it is automatically reduced when +fewer sources are selected. REST `source_workers` and HarvestView use the same setting. + Run every source that can contribute subdomains: ```bash @@ -157,6 +161,8 @@ The table shows which result types each source can add to consolidated CLI resul JSON and XML group findings by result type without source attribution. JSONL and SQLite retain source attribution when the collection adapter provides it. Empty optional fields may be omitted. BuiltWith's normalized frameworks, languages, servers, CMS products, and analytics products are retained in JSONL and completed-result SQLite rows. +Contributors add an ordinary discovery provider with one catalog entry and one factory entry; the shared runner handles CLI execution, persistence, and output. See [the contributor module guide](docs/wiki/How-to-add-a-new-module.md). + A checkmark means the source can add that result type. The **Additional action output** column lists optional actions that return other data. Read the **API key** column as follows: @@ -241,7 +247,7 @@ Provider pricing is intentionally omitted because plans and quotas change freque `haveibeenpwned` remains the keyless public breach catalogue. `hibpverified` is a separate authenticated source for HIBP's `breachedDomain` endpoint. It participates in `all` and matching capability selectors just like every other P0 source, and skips normally when its provider key is absent. API run requests can select it through the shared source contract and return normalized emails plus stable breach names. A live run requires a user-owned paid HIBP API key and a user-owned domain verified in that account; routine tests use offline responses. -The runtime registry also reports the legacy identifiers `linkedin`, `netcraft`, `omnisint`, `sublist3r`, and `zoomeyeapi`. These identifiers have no active CLI handlers. The table does not present them as usable sources. +The inert legacy identifiers `linkedin`, `netcraft`, `omnisint`, `sublist3r`, and `zoomeyeapi` are no longer registered. Use the table above, `SOURCE_SPECS`, and `SOURCE_FACTORIES` as the supported provider inventory. ## Configuration diff --git a/docs/wiki/How-to-add-a-new-module.md b/docs/wiki/How-to-add-a-new-module.md index b555ec4e..069742e6 100644 --- a/docs/wiki/How-to-add-a-new-module.md +++ b/docs/wiki/How-to-add-a-new-module.md @@ -27,13 +27,7 @@ Do not return fields the provider did not supply. Normalize and deduplicate befo ## 3. Register the source -Update the current symbols rather than following fixed line numbers: - -1. Import the adapter in [`theHarvester/__main__.py`](https://github.com/laramies/theHarvester/blob/dev/theHarvester/__main__.py). -2. Add its source handler to the existing alphabetical source-selection chain. -3. Call the central `store()` helper once with only the result flags the adapter supports. -4. Add the source identifier to `Core.get_supportedengines()` in [`theHarvester/lib/core.py`](https://github.com/laramies/theHarvester/blob/dev/theHarvester/lib/core.py). -5. Add the identifier to the CLI `--source` help list. +Add one catalog entry in [`theHarvester/lib/source_catalog.py`](https://github.com/laramies/theHarvester/blob/dev/theHarvester/lib/source_catalog.py) and one factory entry in [`theHarvester/lib/source_runner.py`](https://github.com/laramies/theHarvester/blob/dev/theHarvester/lib/source_runner.py). The catalog supplies CLI help, source selection, and activity classification; the factory constructs the adapter; the runner collects declared result routes and persists them through the existing completed-result flow. Keep the public source identifier stable and use the same spelling everywhere. @@ -64,6 +58,6 @@ Tests must not require external network access or real provider credentials. ## 6. Update operator documentation -Add the source to the README source/result matrix with its actual output columns and key requirement. The matrix contract test checks that documented result types match the flags passed to `store()`. +Add the source to the README source/result matrix with its actual output columns and key requirement. The matrix contract test checks that documented result types match the catalog entry. In the pull request, link the provider API documentation and explain any intentional exception to shared transport behavior. diff --git a/docs/wiki/Rest-API.md b/docs/wiki/Rest-API.md index 72c4dd86..4798743e 100644 --- a/docs/wiki/Rest-API.md +++ b/docs/wiki/Rest-API.md @@ -66,6 +66,7 @@ run_id="$(curl -s http://127.0.0.1:5000/api/v1/runs \ "target": "example.com", "sources": ["emails", "crtsh"], "limit": 500, + "source_workers": 6, "deadline_seconds": 1800 }' \ | jq -r '.run_id')" @@ -77,6 +78,9 @@ curl -s "http://127.0.0.1:5000/api/v1/runs/$run_id" \ Run submission is asynchronous. Lifecycle status is `queued`, `running`, `cancelling`, `cancelled`, `completed`, or `failed`. Terminal evidence status is reported separately as `complete`, `partial`, or `failed` when evidence exists. +`source_workers` is the same positive concurrency used by CLI `-j` or `--source-workers` and HarvestView. It defaults +to six, is reduced when fewer sources are selected, and never skips sources or limits their results. + P1 DNS and P2 direct options are fields on the same run request. The OpenAPI schema shows their current defaults, limits, and descriptions. The server uses the operator-selected target and does not impose a public-only egress policy. RouteViews is the explicit P0 `routeviews` action. When selected for a domain run, it automatically enriches harvested IPs that have sourced IP-to-ASN attribution. It also accepts an AS-prefixed ASN or IP address supplied as the run target. Harvested IPs without that attribution are not sent, and bare ASN findings are not expanded into complete prefix inventories. For IP pivots, only the most-specific matching prefix is retained, including every origin when that prefix is multi-origin. An explicit ASN target still requests its complete prefix inventory. The fixed internal budget is 300 sequential requests and 300 seconds; `limit` does not change it. A server-side `routeviews.key` is used automatically for PeeringDB-verified authenticated access at the documented 10-request-per-second allowance; otherwise the action uses guest access at one request per second. Provider credentials cannot be supplied in a run request. Returned prefixes remain external relationships and are never scheduled as DNS or P2 targets. The CLI also accepts a literal CIDR target. diff --git a/tests/discovery/test_builtwith.py b/tests/discovery/test_builtwith.py index e7eff39c..95072e13 100644 --- a/tests/discovery/test_builtwith.py +++ b/tests/discovery/test_builtwith.py @@ -21,6 +21,7 @@ if 'aiohttp_socks' not in sys.modules: from theHarvester import __main__ as theharvester_main from theHarvester.discovery import builtwith from theHarvester.discovery.constants import MissingKey +from theHarvester.lib import source_runner from theHarvester.lib.completed_result import CompletedResult from theHarvester.lib.core import FetcherResponse, ResponseStreamError @@ -326,7 +327,7 @@ async def test_normalized_builtwith_results_reach_completed_jsonl( report = tmp_path / 'builtwith-report' monkeypatch.setattr(theharvester_main, 'ResultStore', FakeResultStore) - monkeypatch.setattr(theharvester_main.builtwith, 'SearchBuiltWith', FakeBuiltWith) + monkeypatch.setattr(source_runner.builtwith, 'SearchBuiltWith', FakeBuiltWith) monkeypatch.setattr(sys, 'argv', ['theHarvester', '-d', 'example.com', '-b', 'builtwith', '-f', str(report)]) with pytest.raises(SystemExit) as exit_info: @@ -341,6 +342,10 @@ async def test_normalized_builtwith_results_reach_completed_jsonl( ('server', 'nginx'), ('url', 'https://example.com/login'), ) + assert completed_results[0].source_executions[0].source == 'builtwith' + assert completed_results[0].source_executions[0].status == 'completed' + assert completed_results[0].source_executions[0].result_count == 6 + assert {observation.source for observation in completed_results[0].observations} == {'builtwith'} records = [json.loads(line) for line in report.with_suffix('.jsonl').read_text().splitlines()] assert {'type': 'url', 'value': 'https://example.com/login', 'sources': ['builtwith']} in records assert {'type': 'framework', 'value': 'Django', 'sources': ['builtwith']} in records diff --git a/tests/discovery/test_crtname.py b/tests/discovery/test_crtname.py index b2310842..f7e15fbb 100644 --- a/tests/discovery/test_crtname.py +++ b/tests/discovery/test_crtname.py @@ -9,7 +9,7 @@ from typing import Any import pytest from theHarvester import __main__ as theharvester_main -from theHarvester.discovery import crtname +from theHarvester.discovery import crtname, crtsh from theHarvester.lib.completed_result import CompletedResult from theHarvester.lib.core import ResponseStreamError @@ -302,7 +302,7 @@ async def test_crt_name_and_crtsh_share_one_result_with_both_sources( report = tmp_path / 'crt-name-overlap' monkeypatch.setattr(crtname.AsyncFetcher, 'stream_records', stream_records) - monkeypatch.setattr(theharvester_main.crtsh, 'SearchCrtsh', FakeCrtsh) + monkeypatch.setattr(crtsh, 'SearchCrtsh', FakeCrtsh) monkeypatch.setattr(theharvester_main, 'ResultStore', FakeResultStore) monkeypatch.setattr( sys, diff --git a/tests/discovery/test_crtsh.py b/tests/discovery/test_crtsh.py index 7125bd20..8ef5cea7 100644 --- a/tests/discovery/test_crtsh.py +++ b/tests/discovery/test_crtsh.py @@ -165,7 +165,7 @@ class TestCrtshSearch: class TestCrtshIntegration: - def test_supportedengines_lists_crtsh(self): - from theHarvester.lib.core import Core + def test_source_catalog_lists_crtsh(self): + from theHarvester.lib.source_catalog import SOURCE_SPECS - assert 'crtsh' in Core.get_supportedengines() + assert 'crtsh' in SOURCE_SPECS diff --git a/tests/discovery/test_dymosearch.py b/tests/discovery/test_dymosearch.py index 640d9f58..5a86f3fc 100644 --- a/tests/discovery/test_dymosearch.py +++ b/tests/discovery/test_dymosearch.py @@ -139,7 +139,7 @@ class TestDymoIntegration: assert hasattr(mod, 'SearchDymo') - def test_supportedengines_lists_dymo(self): - from theHarvester.lib.core import Core + def test_source_catalog_lists_dymo(self): + from theHarvester.lib.source_catalog import SOURCE_SPECS - assert 'dymo' in Core.get_supportedengines() + assert 'dymo' in SOURCE_SPECS diff --git a/tests/discovery/test_gitlabsearch.py b/tests/discovery/test_gitlabsearch.py index d172572f..4d75c3cb 100644 --- a/tests/discovery/test_gitlabsearch.py +++ b/tests/discovery/test_gitlabsearch.py @@ -5,8 +5,8 @@ from typing import Any import pytest -from theHarvester.discovery import gitlabsearch from theHarvester import __main__ as theharvester_main +from theHarvester.discovery import gitlabsearch from theHarvester.lib.completed_result import CompletedResult @@ -164,7 +164,7 @@ async def test_gitlab_urls_reach_completed_jsonl( report = tmp_path / 'gitlab-report' monkeypatch.setattr(theharvester_main, 'ResultStore', FakeResultStore) - monkeypatch.setattr(theharvester_main.gitlabsearch, 'SearchGitlab', FakeGitlab) + monkeypatch.setattr(gitlabsearch, 'SearchGitlab', FakeGitlab) monkeypatch.setattr(sys, 'argv', ['theHarvester', '-d', 'example.test', '-b', 'gitlab', '-f', str(report)]) with pytest.raises(SystemExit) as exit_info: diff --git a/tests/discovery/test_haveibeenpwned.py b/tests/discovery/test_haveibeenpwned.py index 75cb9d3f..0ba4b55a 100644 --- a/tests/discovery/test_haveibeenpwned.py +++ b/tests/discovery/test_haveibeenpwned.py @@ -193,7 +193,7 @@ async def test_public_breach_names_reach_completed_result_and_jsonl( report = tmp_path / 'hibp-report' monkeypatch.setattr(theharvester_main, 'ResultStore', FakeResultStore) - monkeypatch.setattr(theharvester_main.haveibeenpwned, 'SearchHaveIBeenPwned', FakeHaveIBeenPwned) + monkeypatch.setattr(haveibeenpwned, 'SearchHaveIBeenPwned', FakeHaveIBeenPwned) monkeypatch.setattr( sys, 'argv', diff --git a/tests/discovery/test_hibpverified.py b/tests/discovery/test_hibpverified.py index caaa49b2..da209efa 100644 --- a/tests/discovery/test_hibpverified.py +++ b/tests/discovery/test_hibpverified.py @@ -172,7 +172,7 @@ async def test_verified_domain_results_reach_completed_jsonl_and_sqlite_handoff( report = tmp_path / 'hibp-verified-report' monkeypatch.setattr(theharvester_main, 'ResultStore', FakeResultStore) - monkeypatch.setattr(theharvester_main.hibpverified, 'SearchHibpVerified', FakeHibpVerified) + monkeypatch.setattr(hibpverified, 'SearchHibpVerified', FakeHibpVerified) monkeypatch.setattr( sys, 'argv', diff --git a/tests/discovery/test_hudsonrock.py b/tests/discovery/test_hudsonrock.py index 9d2cba5f..e69e7fdc 100644 --- a/tests/discovery/test_hudsonrock.py +++ b/tests/discovery/test_hudsonrock.py @@ -241,7 +241,7 @@ async def test_infostealer_data_reaches_completed_result_and_jsonl( report = tmp_path / 'hudsonrock-report' monkeypatch.setattr(theharvester_main, 'ResultStore', FakeResultStore) - monkeypatch.setattr(theharvester_main.hudsonrocksearch, 'SearchHudsonRock', FakeHudsonRock) + monkeypatch.setattr(hudsonrocksearch, 'SearchHudsonRock', FakeHudsonRock) monkeypatch.setattr( sys, 'argv', diff --git a/tests/discovery/test_leaklookup.py b/tests/discovery/test_leaklookup.py index ca352265..a8913081 100644 --- a/tests/discovery/test_leaklookup.py +++ b/tests/discovery/test_leaklookup.py @@ -6,11 +6,11 @@ from typing import Any import pytest +from theHarvester import __main__ as theharvester_main from theHarvester.discovery import leaklookup from theHarvester.discovery.constants import MissingKey from theHarvester.lib.completed_result import CompletedResult from theHarvester.lib.core import FetcherResponse -from theHarvester import __main__ as theharvester_main @pytest.mark.parametrize('key', [None, '', ' ']) @@ -178,7 +178,7 @@ async def test_leaklookup_emails_and_breaches_reach_completed_result_and_jsonl(m report = tmp_path / 'leaklookup-report' monkeypatch.setattr(theharvester_main, 'ResultStore', FakeResultStore) - monkeypatch.setattr(theharvester_main.leaklookup, 'SearchLeakLookup', FakeLeakLookup) + monkeypatch.setattr(leaklookup, 'SearchLeakLookup', FakeLeakLookup) monkeypatch.setattr( sys, 'argv', diff --git a/tests/discovery/test_rapiddns.py b/tests/discovery/test_rapiddns.py index 1b34c0e2..c2ff47db 100644 --- a/tests/discovery/test_rapiddns.py +++ b/tests/discovery/test_rapiddns.py @@ -12,6 +12,7 @@ import pytest import theHarvester.__main__ as theharvester_main from theHarvester.discovery import rapiddns +from theHarvester.lib import source_runner from theHarvester.lib.completed_result import CompletedResult from theHarvester.lib.core import FetcherResponse from theHarvester.lib.output import configure_logging @@ -214,9 +215,9 @@ async def test_rapiddns_evidence_reaches_existing_outputs( monkeypatch.setattr(rapiddns.AsyncFetcher, 'fetch_all', fake_fetch_all) monkeypatch.setattr(theharvester_main, 'ResultStore', FakeResultStore) monkeypatch.setattr(theharvester_main.hostchecker, 'Checker', UnexpectedChecker) - monkeypatch.setattr(theharvester_main.search_dehashed, 'SearchDehashed', FakeDehashed) + monkeypatch.setattr(source_runner.search_dehashed, 'SearchDehashed', FakeDehashed) monkeypatch.setattr(theharvester_main.api_endpoints, 'SearchApiEndpoints', FakeApiEndpoints) - monkeypatch.setattr(theharvester_main.securityscorecard, 'SearchSecurityScorecard', FakeSecurityScorecard) + monkeypatch.setattr(source_runner.securityscorecard, 'SearchSecurityScorecard', FakeSecurityScorecard) monkeypatch.setattr(theharvester_main.dnssearch, 'reverse_ip_ranges', fake_reverse_ip_ranges) monkeypatch.setattr( sys, diff --git a/tests/e2e/test_harvestview.py b/tests/e2e/test_harvestview.py index efa2b5eb..39638f0f 100644 --- a/tests/e2e/test_harvestview.py +++ b/tests/e2e/test_harvestview.py @@ -172,6 +172,7 @@ def test_harvestview_can_submit_overridable_execution_controls( page.locator('#run-target').fill('example.com') page.get_by_text('Advanced execution controls', exact=True).click() page.locator('#run-start').fill('25') + page.locator('#source-workers').fill('7') page.locator('#run-deadline').fill('86400') page.locator('[name="proxies"]').check() page.locator('[name="shodan"]').check() @@ -198,6 +199,7 @@ def test_harvestview_can_submit_overridable_execution_controls( 'sources': ['crtsh'], 'limit': 500, 'start': 25, + 'source_workers': 7, 'deadline_seconds': 86_400, 'proxies': True, 'no_hosts': False, @@ -1181,17 +1183,13 @@ def test_harvestview_can_import_and_analyze_fixture_evidence_through_the_real_ui for index, source in enumerate(ordered_sources) ], 'results': [ - { - 'type': result_type, - 'value': ( - '192.0.2.10' - if result_type == 'ip' - else 'AS64500' - if result_type == 'asn' - else f'{result_type}.example.com' - ), - 'sources': [ordered_sources[index]['name']] if index < 3 else [], - } + { + 'type': result_type, + 'value': ( + '192.0.2.10' if result_type == 'ip' else 'AS64500' if result_type == 'asn' else f'{result_type}.example.com' + ), + 'sources': [ordered_sources[index]['name']] if index < 3 else [], + } for index, result_type in enumerate(('hostname', 'ip', 'asn', 'email', 'url', 'framework', 'person', 'language')) ] + [ diff --git a/tests/lib/test_core.py b/tests/lib/test_core.py index 9cffe397..9da42fa3 100644 --- a/tests/lib/test_core.py +++ b/tests/lib/test_core.py @@ -19,7 +19,7 @@ from theHarvester.lib.core import ( ResponseStreamError, ) from theHarvester.lib.output import configure_logging -from theHarvester.lib.source_catalog import SOURCE_SPECS, ActivityClass +from theHarvester.lib.source_catalog import SOURCE_SPECS, ActivityClass, resolve_sources @pytest.fixture(autouse=True) @@ -28,7 +28,7 @@ def mock_environ(monkeypatch, tmp_path: Path): def test_email_capability_expands_to_email_sources() -> None: - assert Core.expand_source_selection("emails") == [ + assert resolve_sources("emails") == [ "apis-guru", "baidu", "brave", @@ -53,7 +53,7 @@ def test_email_capability_expands_to_email_sources() -> None: def test_capabilities_and_explicit_sources_form_a_union() -> None: - assert Core.expand_source_selection("certspotter, urls") == [ + assert resolve_sources("certspotter, urls") == [ "apis-guru", "bevigil", "builtwith", @@ -67,7 +67,7 @@ def test_capabilities_and_explicit_sources_form_a_union() -> None: def test_multiple_capabilities_form_a_union() -> None: - assert Core.expand_source_selection("asns,people") == [ + assert resolve_sources("asns,people") == [ "criminalip", "onyphe", "urlscan", @@ -76,15 +76,23 @@ def test_multiple_capabilities_form_a_union() -> None: def test_breach_capability_includes_every_matching_source() -> None: - assert Core.expand_source_selection('breaches') == ['haveibeenpwned', 'hibpverified', 'leaklookup'] + assert resolve_sources('breaches') == ['haveibeenpwned', 'hibpverified', 'leaklookup'] def test_named_source_can_be_combined_with_a_capability() -> None: - assert Core.expand_source_selection('breaches,hibpverified') == ['haveibeenpwned', 'hibpverified', 'leaklookup'] + assert resolve_sources('breaches,hibpverified') == ['haveibeenpwned', 'hibpverified', 'leaklookup'] + + +def test_core_supported_engines_compatibility_uses_the_catalog() -> None: + assert Core.get_supportedengines() == sorted(SOURCE_SPECS) + + +def test_core_source_selection_compatibility_uses_the_catalog() -> None: + assert Core.expand_source_selection('breaches') == resolve_sources('breaches') def test_all_selects_only_passive_catalog_sources() -> None: - assert Core.expand_source_selection("ALL") == sorted( + assert resolve_sources("ALL") == sorted( spec.name for spec in SOURCE_SPECS.values() if spec.activity is ActivityClass.PASSIVE @@ -105,8 +113,8 @@ def test_all_selects_only_passive_catalog_sources() -> None: ['criminalip', 'pentesttools', 'shodan', 'shodanInternetDB', 'subdomainfinderc99'], ) def test_non_passive_sources_run_only_when_explicitly_selected(source: str) -> None: - assert source not in Core.expand_source_selection('all') - assert Core.expand_source_selection(source) == [source] + assert source not in resolve_sources('all') + assert resolve_sources(source) == [source] def mock_read_text(mocked: dict[Path, str | Exception]): diff --git a/tests/lib/test_enumeration.py b/tests/lib/test_enumeration.py index 1ff008c8..96c3448b 100644 --- a/tests/lib/test_enumeration.py +++ b/tests/lib/test_enumeration.py @@ -6,6 +6,7 @@ from theHarvester.lib.enumeration import ( DEFAULT_DNS_RECURSIVE_QUERY_LIMIT, DEFAULT_DNS_RECURSIVE_RUNTIME_SECONDS, DEFAULT_RESULT_LIMIT, + DEFAULT_SOURCE_WORKERS, EnumerationOptions, ) from theHarvester.lib.source_catalog import selected_action_names @@ -19,6 +20,7 @@ def test_enumeration_options_fill_the_shared_execution_defaults() -> None: assert options.start == 0 assert options.dns_recursive_query_limit == DEFAULT_DNS_RECURSIVE_QUERY_LIMIT is None assert options.dns_recursive_runtime_seconds == DEFAULT_DNS_RECURSIVE_RUNTIME_SECONDS is None + assert options.source_workers == DEFAULT_SOURCE_WORKERS == 6 def test_enumeration_options_preserve_explicit_transport_values() -> None: @@ -31,6 +33,7 @@ def test_enumeration_options_preserve_explicit_transport_values() -> None: proxies=True, quiet=True, screenshot='/tmp/managed-screenshots', + source_workers=7, ) ) @@ -39,6 +42,7 @@ def test_enumeration_options_preserve_explicit_transport_values() -> None: assert options.proxies is True assert options.quiet is True assert options.screenshot == '/tmp/managed-screenshots' + assert options.source_workers == 7 def test_routeviews_is_an_explicit_passive_action_independent_of_source_limits() -> None: diff --git a/tests/lib/test_harvestview_ui.py b/tests/lib/test_harvestview_ui.py index 741515cc..4ef18803 100644 --- a/tests/lib/test_harvestview_ui.py +++ b/tests/lib/test_harvestview_ui.py @@ -23,6 +23,7 @@ def test_harvestview_owns_root_and_issues_an_http_only_session(tmp_path, monkeyp assert 'Advanced safety controls' in root.text assert f'value="{",".join(DEFAULT_DNS_RESOLVERS)}"' in root.text assert 'Resolve with the configured resolver addresses.' in root.text + assert 'id="source-workers" name="source_workers" type="number" min="1"' in root.text assert legacy.status_code == 404 cookie = root.headers['set-cookie'] assert 'theharvester-api-key=' in cookie @@ -53,6 +54,7 @@ def test_harvestview_assets_load_outside_the_repository_directory(tmp_path, monk assert "request.dns_recursive_runtime_seconds === undefined ? 'Not recorded'" in response.text assert "request.deadline_seconds === null ? 'Unlimited'" in response.text assert "deadline_seconds: form.get('deadline_seconds') ? Number(form.get('deadline_seconds')) : null" in response.text + assert "source_workers: Number(form.get('source_workers'))" in response.text def test_harvestview_has_an_operator_readable_shodan_host_route(tmp_path, monkeypatch) -> None: diff --git a/tests/lib/test_run_backend.py b/tests/lib/test_run_backend.py index 78d3e9ed..acc5fb56 100644 --- a/tests/lib/test_run_backend.py +++ b/tests/lib/test_run_backend.py @@ -367,6 +367,7 @@ def test_child_screenshot_run_persists_downloadable_artifact_metadata(tmp_path, def test_child_screenshot_cancellation_reuses_the_checkpointed_evidence(tmp_path, monkeypatch) -> None: from theHarvester import __main__ as main_module + from theHarvester.lib import source_runner from theHarvester.lib.api.run_artifacts import read_child_evidence from theHarvester.lib.api.run_models import RunRequest from theHarvester.lib.api.run_store import RunStore @@ -410,7 +411,7 @@ def test_child_screenshot_cancellation_reuses_the_checkpointed_evidence(tmp_path database = tmp_path / 'runs.sqlite' monkeypatch.setenv('THEHARVESTER_RUN_ARTIFACTS', str(tmp_path / 'artifacts')) - monkeypatch.setattr(main_module.crtsh, 'SearchCrtsh', TwoHostSource) + monkeypatch.setattr(source_runner.crtsh, 'SearchCrtsh', TwoHostSource) monkeypatch.setattr(main_module, 'ScreenShotter', FakeScreenShotter) async def scenario(): @@ -1205,3 +1206,40 @@ def test_worker_fails_run_without_attaching_child_evidence_for_another_target(tm assert detail['status'] == 'failed' assert detail['results'] == [] assert 'does not match run target' in detail['error'] + + +@pytest.mark.parametrize('source_workers', [0, -1, True, 1.5]) +def test_run_request_requires_positive_source_workers(source_workers: object) -> None: + from pydantic import ValidationError + + from theHarvester.lib.api.run_models import RunRequest + + with pytest.raises(ValidationError): + RunRequest(target='example.test', sources=['crtsh'], source_workers=source_workers) + + +def test_source_workers_are_preserved_from_rest_request_to_child(tmp_path, monkeypatch) -> None: + from theHarvester import __main__ as main_module + from theHarvester.lib.api import run_worker + from theHarvester.lib.api.run_models import RunRequest + from theHarvester.lib.api.run_store import RunStore + from theHarvester.lib.completed_result import CompletedResult + + received_options = [] + + async def fake_start(options, **_kwargs): + received_options.append(options) + now = datetime.now(UTC) + return (CompletedResult.finish(target=options.domain, started_at=now, completed_at=now, groups={}),) + + monkeypatch.setattr(main_module, 'start', fake_start) + + async def scenario() -> None: + store = RunStore(tmp_path / 'runs.sqlite') + created = await store.create(RunRequest(target='example.test', sources=['crtsh'], source_workers=7)) + assert await store.claim_next() is not None + await run_worker._child_execute(created['run_id'], store.database) + + asyncio.run(scenario()) + + assert received_options[0].source_workers == 7 diff --git a/tests/lib/test_source_catalog.py b/tests/lib/test_source_catalog.py index 73cbdef6..d2366b76 100644 --- a/tests/lib/test_source_catalog.py +++ b/tests/lib/test_source_catalog.py @@ -1,52 +1,26 @@ -import ast -from pathlib import Path - from theHarvester.discovery import apisguru, bevigil, builtwith, gitlabsearch, intelxsearch, rocketreach, urlscan, zoomeyesearch -from theHarvester.lib.core import Core from theHarvester.lib.source_catalog import SOURCE_SPECS, ActivityClass, ResultRoute, SourceSpec, get_source_spec -def _scheduled_source_names() -> list[str]: - tree = ast.parse(Path('theHarvester/__main__.py').read_text()) - names: list[str] = [] - for node in ast.walk(tree): - if not isinstance(node, ast.Compare) or len(node.ops) != 1 or not isinstance(node.ops[0], ast.Eq): - continue - if not isinstance(node.left, ast.Name) or node.left.id != 'engineitem' or len(node.comparators) != 1: - continue - comparator = node.comparators[0] - if isinstance(comparator, ast.Constant) and isinstance(comparator.value, str): - names.append(comparator.value) - return names - - -def test_source_specs_cover_supported_sources() -> None: - scheduled = _scheduled_source_names() - - assert len(scheduled) == len(set(scheduled)) - assert set(SOURCE_SPECS) == set(scheduled) - assert set(SOURCE_SPECS) <= set(Core.get_supportedengines()) - - def test_dead_threatcrowd_source_is_not_selectable() -> None: - assert 'threatcrowd' not in Core.get_supportedengines() assert 'threatcrowd' not in SOURCE_SPECS - assert 'threatcrowd' not in _scheduled_source_names() def test_invalid_bitbucket_domain_source_is_not_selectable() -> None: - assert 'bitbucket' not in Core.get_supportedengines() assert 'bitbucket' not in SOURCE_SPECS - assert 'bitbucket' not in _scheduled_source_names() def test_projectdiscovery_is_the_only_selector_for_the_chaos_corpus() -> None: - selected = Core.expand_source_selection('all') + selected = [spec.name for spec in SOURCE_SPECS.values() if spec.activity is ActivityClass.PASSIVE] assert 'projectdiscovery' in selected assert 'chaos' not in selected - assert 'chaos' not in Core.get_supportedengines() - assert 'chaos' not in _scheduled_source_names() + + +def test_removed_inert_sources_are_not_supported() -> None: + removed = {'linkedin', 'netcraft', 'omnisint', 'sublist3r', 'zoomeyeapi'} + + assert not removed & set(SOURCE_SPECS) def test_subdomain_route_drives_subdomain_capability() -> None: @@ -59,9 +33,7 @@ def test_subdomain_route_drives_subdomain_capability() -> None: def test_source_specs_describe_consolidated_routes_not_getter_presence() -> None: - assert SOURCE_SPECS['apis-guru'].routes == frozenset( - {ResultRoute.SUBDOMAINS, ResultRoute.EMAILS, ResultRoute.URLS} - ) + assert SOURCE_SPECS['apis-guru'].routes == frozenset({ResultRoute.SUBDOMAINS, ResultRoute.EMAILS, ResultRoute.URLS}) assert SOURCE_SPECS['gitlab'].routes == frozenset({ResultRoute.SUBDOMAINS, ResultRoute.EMAILS, ResultRoute.URLS}) assert SOURCE_SPECS['sourcegraph'].routes == frozenset({ResultRoute.SUBDOMAINS}) assert SOURCE_SPECS['haveibeenpwned'].routes == frozenset({ResultRoute.BREACHES}) @@ -114,6 +86,14 @@ def test_rapiddns_declares_separate_subdomain_and_ip_routes() -> None: assert SOURCE_SPECS['rapiddns'].routes == frozenset({ResultRoute.SUBDOMAINS, ResultRoute.IPS}) +def test_catalog_declares_sources_that_retain_provider_hostname_evidence_without_dns_confirmation() -> None: + assert {spec.name for spec in SOURCE_SPECS.values() if spec.retains_unresolved_hostnames} == { + 'hackertarget', + 'pentesttools', + 'rapiddns', + } + + def test_pentesttools_declares_its_normalized_subdomain_and_ip_routes() -> None: assert SOURCE_SPECS['pentesttools'].routes == frozenset({ResultRoute.SUBDOMAINS, ResultRoute.IPS}) @@ -127,9 +107,7 @@ def test_leakix_declares_only_its_documented_subdomain_route() -> None: def test_unavailable_venacus_source_is_not_selectable() -> None: - assert 'venacus' not in Core.get_supportedengines() assert 'venacus' not in SOURCE_SPECS - assert 'venacus' not in _scheduled_source_names() def test_source_lookup_preserves_case_insensitive_legacy_labels() -> None: diff --git a/tests/lib/test_source_runner.py b/tests/lib/test_source_runner.py new file mode 100644 index 00000000..3bf335e1 --- /dev/null +++ b/tests/lib/test_source_runner.py @@ -0,0 +1,835 @@ +from __future__ import annotations + +import asyncio +import logging +from dataclasses import FrozenInstanceError +from datetime import UTC, datetime +from typing import Any + +import pytest + +from theHarvester.discovery.constants import MissingKeyError +from theHarvester.lib.asn_attribution import AsnAttributionObservation +from theHarvester.lib.completed_result import ResultObservation, SourceExecution +from theHarvester.lib.source_catalog import SOURCE_SPECS +from theHarvester.lib.source_runner import ( + SOURCE_FACTORIES, + SourceJob, + SourceOutcome, + SourceRequest, + create_source, + run_source, + run_source_jobs, +) + + +@pytest.mark.parametrize('workers', [0, -1, True, 1.5]) +@pytest.mark.asyncio +async def test_source_jobs_require_a_positive_worker_count(workers: object) -> None: + with pytest.raises(ValueError, match='source workers must be a positive integer'): + await run_source_jobs((), workers=workers) # type: ignore[arg-type] + + +def test_source_contracts_are_immutable() -> None: + request = SourceRequest('APIS-GURU', 'example.test', 25, 5, True, True) + job = SourceJob(request) + outcome = SourceOutcome(SourceExecution('apis-guru', 'completed', 0, 0)) + + with pytest.raises(FrozenInstanceError): + request.target = 'changed.test' # type: ignore[misc] + with pytest.raises(FrozenInstanceError): + job.request = request # type: ignore[misc] + with pytest.raises(FrozenInstanceError): + outcome.observations = () # type: ignore[misc] + assert request.source == 'apis-guru' + + +def test_source_factories_match_the_catalog() -> None: + assert set(SOURCE_FACTORIES) == set(SOURCE_SPECS) + + +@pytest.mark.parametrize( + ('source', 'patch_target', 'expected_args', 'expected_kwargs'), + [ + ('apis-guru', 'theHarvester.lib.source_runner.apisguru.SearchApisGuru', ('example.test', 25), {}), + ('arquivo', 'theHarvester.lib.source_runner.arquivo.SearchArquivo', ('example.test', 25), {}), + ('baidu', 'theHarvester.lib.source_runner.baidusearch.SearchBaidu', ('example.test', 25), {}), + ('bevigil', 'theHarvester.lib.source_runner.bevigil.SearchBeVigil', ('example.test',), {}), + ('brave', 'theHarvester.lib.source_runner.bravesearch.SearchBrave', ('example.test', 25), {}), + ('bufferoverun', 'theHarvester.lib.source_runner.bufferoverun.SearchBufferover', ('example.test',), {}), + ('builtwith', 'theHarvester.lib.source_runner.builtwith.SearchBuiltWith', ('example.test',), {}), + ('censys', 'theHarvester.lib.source_runner.censysearch.SearchCensys', ('example.test', 25), {}), + ('certspotter', 'theHarvester.lib.source_runner.certspottersearch.SearchCertspoter', ('example.test',), {}), + ('commoncrawl', 'theHarvester.lib.source_runner.commoncrawl.SearchCommoncrawl', ('example.test', 25), {}), + ('criminalip', 'theHarvester.lib.source_runner.criminalip.SearchCriminalIP', ('example.test',), {}), + ('crt-name', 'theHarvester.lib.source_runner.crtname.SearchCrtName', ('example.test',), {}), + ('crtsh', 'theHarvester.lib.source_runner.crtsh.SearchCrtsh', ('example.test',), {}), + ('dehashed', 'theHarvester.lib.source_runner.search_dehashed.SearchDehashed', ('example.test',), {'limit': 25}), + ('dnsdb', 'theHarvester.lib.source_runner.dnsdb.SearchDNSDB', ('example.test',), {}), + ( + 'dnsdumpster', + 'theHarvester.lib.source_runner.search_dnsdumpster.SearchDNSDumpster', + ('example.test',), + {}, + ), + ('duckduckgo', 'theHarvester.lib.source_runner.duckduckgosearch.SearchDuckDuckGo', ('example.test', 25), {}), + ('dymo', 'theHarvester.lib.source_runner.dymosearch.SearchDymo', ('example.test',), {}), + ('fofa', 'theHarvester.lib.source_runner.fofa.SearchFofa', ('example.test',), {}), + ('fullhunt', 'theHarvester.lib.source_runner.fullhuntsearch.SearchFullHunt', ('example.test',), {}), + ('github-code', 'theHarvester.lib.source_runner.githubcode.SearchGithubCode', ('example.test', 25), {}), + ('gitlab', 'theHarvester.lib.source_runner.gitlabsearch.SearchGitlab', ('example.test',), {}), + ( + 'hackertarget', + 'theHarvester.lib.source_runner.hackertarget.SearchHackerTarget', + ('example.test',), + {}, + ), + ( + 'haveibeenpwned', + 'theHarvester.lib.source_runner.haveibeenpwned.SearchHaveIBeenPwned', + ('example.test',), + {}, + ), + ( + 'hibpverified', + 'theHarvester.lib.source_runner.hibpverified.SearchHibpVerified', + ('example.test',), + {}, + ), + ('hudsonrock', 'theHarvester.lib.source_runner.hudsonrocksearch.SearchHudsonRock', ('example.test',), {}), + ('hunter', 'theHarvester.lib.source_runner.huntersearch.SearchHunter', ('example.test', 25, 5), {}), + ('hunterhow', 'theHarvester.lib.source_runner.searchhunterhow.SearchHunterHow', ('example.test',), {}), + ('intelx', 'theHarvester.lib.source_runner.intelxsearch.SearchIntelx', ('example.test',), {}), + ('leakix', 'theHarvester.lib.source_runner.leakix.SearchLeakix', ('example.test',), {}), + ('leaklookup', 'theHarvester.lib.source_runner.leaklookup.SearchLeakLookup', ('example.test',), {}), + ('mojeek', 'theHarvester.lib.source_runner.mojeek.SearchMojeek', ('example.test', 25), {}), + ('netlas', 'theHarvester.lib.source_runner.netlas.SearchNetlas', ('example.test', 25), {}), + ('onyphe', 'theHarvester.lib.source_runner.onyphe.SearchOnyphe', ('example.test',), {}), + ('otx', 'theHarvester.lib.source_runner.otxsearch.SearchOtx', ('example.test',), {}), + ( + 'pentesttools', + 'theHarvester.lib.source_runner.pentesttools.SearchPentestTools', + ('example.test',), + {}, + ), + ( + 'projectdiscovery', + 'theHarvester.lib.source_runner.projectdiscovery.SearchDiscovery', + ('example.test',), + {}, + ), + ('rapiddns', 'theHarvester.lib.source_runner.rapiddns.SearchRapidDns', ('example.test',), {}), + ('robtex', 'theHarvester.lib.source_runner.robtex.SearchRobtex', ('example.test',), {}), + ('rocketreach', 'theHarvester.lib.source_runner.rocketreach.SearchRocketReach', ('example.test', 25), {}), + ( + 'securityTrails', + 'theHarvester.lib.source_runner.securitytrailssearch.SearchSecuritytrail', + ('example.test',), + {}, + ), + ( + 'securityscorecard', + 'theHarvester.lib.source_runner.securityscorecard.SearchSecurityScorecard', + ('example.test',), + {}, + ), + ( + 'sherlockeye', + 'theHarvester.lib.source_runner.sherlockeye.SearchSherlockeye', + ('example.test',), + {}, + ), + ('shodan', 'theHarvester.lib.source_runner.shodansearch.SearchShodan', ('example.test',), {}), + ( + 'shodanInternetDB', + 'theHarvester.lib.source_runner.shodan_internetdb.SearchShodanInternetDB', + ('example.test',), + {}, + ), + ('shodanct', 'theHarvester.lib.source_runner.shodanct.SearchShodanCt', ('example.test',), {}), + ('sourcegraph', 'theHarvester.lib.source_runner.sourcegraph.SearchSourcegraph', ('example.test', 25), {}), + ( + 'subdomaincenter', + 'theHarvester.lib.source_runner.subdomaincenter.SubdomainCenter', + ('example.test',), + {}, + ), + ( + 'subdomainfinderc99', + 'theHarvester.lib.source_runner.subdomainfinderc99.SearchSubdomainfinderc99', + ('example.test',), + {}, + ), + ('thc', 'theHarvester.lib.source_runner.thc.SearchThc', ('example.test',), {}), + ('tomba', 'theHarvester.lib.source_runner.tombasearch.SearchTomba', ('example.test', 25, 5), {}), + ('urlscan', 'theHarvester.lib.source_runner.urlscan.SearchUrlscan', ('example.test',), {}), + ('virustotal', 'theHarvester.lib.source_runner.virustotal.SearchVirustotal', ('example.test',), {}), + ( + 'waybackarchive', + 'theHarvester.lib.source_runner.waybackarchive.SearchWaybackarchive', + ('example.test', 25), + {}, + ), + ('whoisxml', 'theHarvester.lib.source_runner.whoisxml.SearchWhoisXML', ('example.test',), {}), + ('windvane', 'theHarvester.lib.source_runner.windvane.SearchWindvane', ('example.test',), {}), + ('yahoo', 'theHarvester.lib.source_runner.yahoosearch.SearchYahoo', ('example.test', 25), {}), + ('zoomeye', 'theHarvester.lib.source_runner.zoomeyesearch.SearchZoomEye', ('example.test', 25), {}), + ], +) +def test_factory_constructor_shapes( + monkeypatch: pytest.MonkeyPatch, + source: str, + patch_target: str, + expected_args: tuple[object, ...], + expected_kwargs: dict[str, object], +) -> None: + calls: list[tuple[tuple[object, ...], dict[str, object]]] = [] + + def constructor(*args: object, **kwargs: object) -> object: + calls.append((args, kwargs)) + return object() + + monkeypatch.setattr(patch_target, constructor) + + create_source(SourceRequest(source, 'example.test', 25, 5, True, False)) + + assert calls == [(expected_args, expected_kwargs)] + + +@pytest.mark.asyncio +async def test_runner_normalizes_only_declared_apis_guru_routes(monkeypatch: pytest.MonkeyPatch) -> None: + class FakeApisGuru: + execution_status = 'completed' + stop_reason = None + + def __init__(self, target: str, limit: int) -> None: + assert (target, limit) == ('example.test', 25) + + async def process(self, proxy: bool) -> None: + assert proxy is True + + async def get_hostnames(self) -> list[str]: + return ['API.Example.TEST.', 'api.example.test', 'outside.test', 'example.test'] + + async def get_emails(self) -> list[str]: + return ['User@Example.TEST', 'user@example.test'] + + async def get_urls(self) -> list[str]: + return ['https://api.example.test/v1', 'https://api.example.test/v1'] + + async def get_ips(self) -> set[str]: + raise AssertionError('undeclared getter must not be read') + + monkeypatch.setitem(SOURCE_FACTORIES, 'apis-guru', lambda request: FakeApisGuru(request.target, request.limit)) + + outcome = await run_source(SourceRequest('apis-guru', 'example.test', 25, 5, True, True)) + + assert outcome.execution.source == 'apis-guru' + assert outcome.execution.status == 'completed' + assert outcome.execution.result_count == 3 + assert outcome.execution.stop_reason is None + assert outcome.observations == ( + ResultObservation('apis-guru', 'email', 'user@example.test'), + ResultObservation('apis-guru', 'hostname', 'api.example.test'), + ResultObservation('apis-guru', 'url', 'https://api.example.test/v1'), + ) + assert outcome.asn_attributions == () + + +@pytest.mark.asyncio +async def test_runner_times_construction_and_records_missing_credentials(monkeypatch: pytest.MonkeyPatch) -> None: + ticks = iter((10.0, 10.125)) + events: list[str] = [] + + def clock() -> float: + events.append('clock') + return next(ticks) + + def missing_factory(_request: SourceRequest) -> Any: + assert events == ['clock'] + raise MissingKeyError('apis-guru') + + monkeypatch.setattr('theHarvester.lib.source_runner.time.perf_counter', clock) + monkeypatch.setitem(SOURCE_FACTORIES, 'apis-guru', missing_factory) + + outcome = await run_source(SourceRequest('apis-guru', 'example.test', 25, 0, False, True)) + + assert outcome.execution == SourceExecution( + 'apis-guru', + 'skipped', + 125, + 0, + 'MissingKeyError', + 'missing-credentials', + ) + + +@pytest.mark.asyncio +async def test_start_reporter_failure_is_sanitized_and_does_not_change_provider_outcome( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + process_called = False + + class SuccessfulAdapter: + async def process(self, _proxy: bool) -> None: + nonlocal process_called + process_called = True + + async def get_hostnames(self) -> set[str]: + return {'fresh.example.test'} + + async def get_emails(self) -> set[str]: + return set() + + async def get_ips(self) -> set[str]: + return set() + + async def get_urls(self) -> set[str]: + return set() + + def broken_reporter(_request: SourceRequest) -> None: + raise RuntimeError('sensitive callback payload') + + monkeypatch.setitem(SOURCE_FACTORIES, 'apis-guru', lambda _request: SuccessfulAdapter()) + caplog.set_level(logging.WARNING, logger='theHarvester.lib.source_runner') + + outcome = await run_source( + SourceRequest('apis-guru', 'example.test', 25, 0, False, True), + on_started=broken_reporter, + ) + + assert process_called is True + assert outcome.execution.status == 'completed' + assert outcome.execution.error_type is None + assert outcome.observations == (ResultObservation('apis-guru', 'hostname', 'fresh.example.test'),) + assert 'Source start reporter failed for apis-guru: RuntimeError' in caplog.text + assert 'sensitive callback payload' not in caplog.text + + +@pytest.mark.asyncio +async def test_start_reporter_cancellation_propagates_without_collecting_pre_process_results( + monkeypatch: pytest.MonkeyPatch, +) -> None: + cancellation = asyncio.CancelledError('reporter cancelled') + committed: list[SourceOutcome] = [] + process_called = False + getter_called = False + + class UnstartedAdapter: + async def process(self, _proxy: bool) -> None: + nonlocal process_called + process_called = True + + async def get_hostnames(self) -> set[str]: + nonlocal getter_called + getter_called = True + return {'stale.example.test'} + + def cancelled_reporter(_request: SourceRequest) -> None: + raise cancellation + + monkeypatch.setitem(SOURCE_FACTORIES, 'apis-guru', lambda _request: UnstartedAdapter()) + + with pytest.raises(asyncio.CancelledError) as raised: + await run_source( + SourceRequest('apis-guru', 'example.test', 25, 0, False, True), + commit_cancelled=committed.append, + on_started=cancelled_reporter, + ) + + assert raised.value is cancellation + assert process_called is False + assert getter_called is False + assert committed[0].execution.status == 'failed' + assert committed[0].execution.stop_reason == 'cancelled' + assert committed[0].observations == () + + +@pytest.mark.asyncio +async def test_runner_retains_earlier_observations_when_a_later_getter_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class PartiallyFailingApisGuru: + async def process(self, _proxy: bool) -> None: + return None + + async def get_hostnames(self) -> set[str]: + return {'API.Example.TEST.'} + + async def get_emails(self) -> set[str]: + raise RuntimeError('email projection failed') + + monkeypatch.setitem(SOURCE_FACTORIES, 'apis-guru', lambda _request: PartiallyFailingApisGuru()) + + outcome = await run_source(SourceRequest('apis-guru', 'example.test', 25, 0, False, True)) + + assert outcome.execution.status == 'partial' + assert outcome.execution.error_type == 'RuntimeError' + assert outcome.execution.result_count == 1 + assert outcome.observations == (ResultObservation('apis-guru', 'hostname', 'api.example.test'),) + + +@pytest.mark.asyncio +async def test_runner_reads_retained_adapter_evidence_after_process_failure(monkeypatch: pytest.MonkeyPatch) -> None: + class PartiallyFailingSourcegraph: + async def process(self, _proxy: bool) -> None: + raise RuntimeError('stream failed') + + async def get_hostnames(self) -> set[str]: + return {'partial.example.test'} + + monkeypatch.setitem(SOURCE_FACTORIES, 'sourcegraph', lambda _request: PartiallyFailingSourcegraph()) + + outcome = await run_source(SourceRequest('sourcegraph', 'example.test', 25, 0, False, True)) + + assert outcome.execution.status == 'partial' + assert outcome.execution.error_type == 'RuntimeError' + assert outcome.observations == (ResultObservation('sourcegraph', 'hostname', 'partial.example.test'),) + + +@pytest.mark.asyncio +async def test_runner_reports_normal_zero_yield_as_completed_no_results(monkeypatch: pytest.MonkeyPatch) -> None: + class EmptySourcegraph: + async def process(self, _proxy: bool) -> None: + return None + + async def get_hostnames(self) -> tuple[()]: + return () + + monkeypatch.setitem(SOURCE_FACTORIES, 'sourcegraph', lambda _request: EmptySourcegraph()) + + outcome = await run_source(SourceRequest('sourcegraph', 'example.test', 25, 0, False, True)) + + assert outcome.execution.status == 'completed' + assert outcome.execution.stop_reason == 'no-results' + assert outcome.execution.result_count == 0 + + +@pytest.mark.parametrize('source', ['builtwith', 'hudsonrock', 'shodan']) +@pytest.mark.parametrize( + ('reported_status', 'reported_reason', 'has_results', 'expected_count'), + [ + ('completed', None, False, 0), + ('partial', 'provider-partial', True, 1), + ('failed', 'provider-failure', False, 0), + ('rate-limited', 'http-429', False, 0), + ], +) +@pytest.mark.asyncio +async def test_special_sources_share_runner_outcome_semantics( + monkeypatch: pytest.MonkeyPatch, + source: str, + reported_status: str, + reported_reason: str | None, + has_results: bool, + expected_count: int, +) -> None: + class FakeSpecialSource: + execution_status = reported_status + stop_reason = reported_reason + + async def process(self, _proxy: bool) -> None: + return None + + async def get_hostnames(self) -> set[str]: + return {'partial.example.test'} if has_results else set() + + async def get_emails(self) -> set[str]: + return set() + + async def get_ips(self) -> set[str]: + return set() + + async def get_urls(self) -> set[str]: + return set() + + async def get_frameworks(self) -> set[str]: + return set() + + async def get_languages(self) -> set[str]: + return set() + + async def get_servers(self) -> set[str]: + return set() + + async def get_cms(self) -> set[str]: + return set() + + async def get_analytics(self) -> set[str]: + return set() + + async def get_infostealers(self) -> list[dict[str, object]]: + return [] + + async def get_shodan_hosts(self) -> tuple[()]: + return () + + monkeypatch.setitem(SOURCE_FACTORIES, source, lambda _request: FakeSpecialSource()) + + outcome = await run_source(SourceRequest(source, 'example.test', 25, 0, False, True)) + + assert outcome.execution.status == reported_status + assert outcome.execution.stop_reason == (reported_reason or 'no-results') + assert outcome.execution.result_count == expected_count + assert {observation.source for observation in outcome.observations} == ({source} if has_results else set()) + + +@pytest.mark.asyncio +async def test_runner_collects_builtwith_compatibility_observations(monkeypatch: pytest.MonkeyPatch) -> None: + class FakeBuiltWith: + async def process(self, _proxy: bool) -> None: + return None + + async def get_hostnames(self) -> set[str]: + return {'app.example.test'} + + async def get_urls(self) -> set[str]: + return {'https://app.example.test'} + + async def get_frameworks(self) -> set[str]: + return {'Django'} + + async def get_languages(self) -> set[str]: + return {'Python'} + + async def get_servers(self) -> set[str]: + return {'nginx'} + + async def get_cms(self) -> set[str]: + return {'Wagtail'} + + async def get_analytics(self) -> set[str]: + return {'Plausible'} + + monkeypatch.setitem(SOURCE_FACTORIES, 'builtwith', lambda _request: FakeBuiltWith()) + + outcome = await run_source(SourceRequest('builtwith', 'example.test', 25, 0, False, True)) + + assert {(item.kind, item.value) for item in outcome.observations} == { + ('analytics', 'Plausible'), + ('cms', 'Wagtail'), + ('framework', 'Django'), + ('hostname', 'app.example.test'), + ('language', 'Python'), + ('server', 'nginx'), + ('url', 'https://app.example.test'), + } + assert outcome.execution.result_count == 7 + + +@pytest.mark.asyncio +async def test_runner_collects_hudson_rock_infostealers(monkeypatch: pytest.MonkeyPatch) -> None: + infostealer = {'type': 'employee', 'url': 'https://legacy.example.test'} + + class FakeHudsonRock: + async def process(self, _proxy: bool) -> None: + return None + + async def get_hostnames(self) -> set[str]: + return {'portal.example.test'} + + async def get_emails(self) -> set[str]: + return {'user@example.test'} + + async def get_ips(self) -> set[str]: + return set() + + async def get_urls(self) -> set[str]: + return {'https://portal.example.test'} + + async def get_infostealers(self) -> list[dict[str, object]]: + return [infostealer] + + monkeypatch.setitem(SOURCE_FACTORIES, 'hudsonrock', lambda _request: FakeHudsonRock()) + + outcome = await run_source(SourceRequest('hudsonrock', 'example.test', 25, 0, False, True)) + + assert {(item.kind, item.value) for item in outcome.observations} == { + ('email', 'user@example.test'), + ('hostname', 'portal.example.test'), + ('infostealer', '{"type":"employee","url":"https://legacy.example.test"}'), + } + assert outcome.execution.result_count == 3 + + +@pytest.mark.asyncio +async def test_runner_keeps_only_asn_attributions_backed_by_accepted_observations( + monkeypatch: pytest.MonkeyPatch, +) -> None: + collected_at = datetime.now(UTC) + + class FakeOnyphe: + async def process(self, _proxy: bool) -> None: + return None + + async def get_hostnames(self) -> set[str]: + raise AssertionError('no-hosts must not read hostname results') + + async def get_ips(self) -> set[str]: + return {'192.0.2.1'} + + async def get_asns(self) -> set[str]: + return {'AS64500'} + + async def get_asn_attributions(self) -> set[AsnAttributionObservation]: + return { + AsnAttributionObservation('source', 'onyphe', 'AS64500', 'Accepted Org', 'ip', '192.0.2.1', collected_at), + AsnAttributionObservation('source', 'onyphe', 'AS64501', 'Wrong ASN', 'ip', '192.0.2.1', collected_at), + AsnAttributionObservation( + 'source', 'onyphe', 'AS64500', 'Excluded Host', 'hostname', 'host.example.test', collected_at + ), + } + + monkeypatch.setitem(SOURCE_FACTORIES, 'onyphe', lambda _request: FakeOnyphe()) + + outcome = await run_source(SourceRequest('onyphe', 'example.test', 25, 0, False, False)) + + assert len(outcome.asn_attributions) == 1 + assert outcome.asn_attributions[0].organization_label == 'Accepted Org' + + +@pytest.mark.asyncio +async def test_runner_reports_invalid_asn_as_partial_and_keeps_prior_ip(monkeypatch: pytest.MonkeyPatch) -> None: + class FakeOnyphe: + async def process(self, _proxy: bool) -> None: + return None + + async def get_ips(self) -> set[str]: + return {'192.0.2.1'} + + async def get_asns(self) -> set[str]: + return {'not-an-asn'} + + monkeypatch.setitem(SOURCE_FACTORIES, 'onyphe', lambda _request: FakeOnyphe()) + + outcome = await run_source(SourceRequest('onyphe', 'example.test', 25, 0, False, False)) + + assert outcome.execution.status == 'partial' + assert outcome.execution.error_type == 'ValueError' + assert outcome.observations == (ResultObservation('onyphe', 'ip', '192.0.2.1'),) + + +@pytest.mark.asyncio +async def test_runner_collects_url_before_later_asn_getter_failure(monkeypatch: pytest.MonkeyPatch) -> None: + class FakeZoomEye: + async def process(self, _proxy: bool) -> None: + return None + + async def get_emails(self) -> set[str]: + return set() + + async def get_ips(self) -> set[str]: + return set() + + async def get_people(self) -> set[str]: + return set() + + async def get_urls(self) -> set[str]: + return {'https://portal.example.test'} + + async def get_asns(self) -> set[str]: + raise RuntimeError('asn getter failed') + + monkeypatch.setitem(SOURCE_FACTORIES, 'zoomeye', lambda _request: FakeZoomEye()) + + outcome = await run_source(SourceRequest('zoomeye', 'example.test', 25, 0, False, False)) + + assert outcome.execution.status == 'partial' + assert outcome.execution.error_type == 'RuntimeError' + assert outcome.observations == (ResultObservation('zoomeye', 'url', 'https://portal.example.test'),) + + +@pytest.mark.asyncio +async def test_source_jobs_use_a_clamped_worker_pool_and_isolate_failures( + monkeypatch: pytest.MonkeyPatch, +) -> None: + active = 0 + peak = 0 + three_active = asyncio.Event() + task_names: set[str] = set() + + class GatedAdapter: + def __init__(self, source: str) -> None: + self.source = source + + async def process(self, _proxy: bool) -> None: + nonlocal active, peak + task = asyncio.current_task() + assert task is not None + task_names.add(task.get_name()) + active += 1 + peak = max(peak, active) + if active == 3: + three_active.set() + await three_active.wait() + await asyncio.sleep(0) + active -= 1 + if self.source == 'apis-guru': + raise RuntimeError('provider failed') + + async def get_hostnames(self) -> set[str]: + return {f'{self.source}.example.test'} + + source_names = ('apis-guru', 'sourcegraph', 'crtsh', 'crt-name') + for source in source_names: + monkeypatch.setitem(SOURCE_FACTORIES, source, lambda _request, source=source: GatedAdapter(source)) + jobs = tuple(SourceJob(SourceRequest(source, 'example.test', 25, 0, False, True)) for source in source_names) + + outcomes = await run_source_jobs(jobs, workers=3) + + assert peak == 3 + assert task_names == {'source-worker:0', 'source-worker:1', 'source-worker:2'} + assert [outcome.execution.source for outcome in outcomes] == list(source_names) + assert outcomes[0].execution.status == 'partial' + assert outcomes[0].execution.error_type == 'RuntimeError' + assert all(outcome.execution.status == 'completed' for outcome in outcomes[1:]) + assert not [task for task in asyncio.all_tasks() if task.get_name().startswith('source-worker:') and not task.done()] + + +@pytest.mark.parametrize('workers', [1, 3, 8]) +@pytest.mark.asyncio +async def test_source_worker_count_does_not_change_completed_sources_or_results( + monkeypatch: pytest.MonkeyPatch, + workers: int, +) -> None: + starts: list[str] = [] + task_names: set[str] = set() + + class CompleteAdapter: + def __init__(self, source: str) -> None: + self.source = source + + async def process(self, _proxy: bool) -> None: + task = asyncio.current_task() + assert task is not None + task_names.add(task.get_name()) + starts.append(self.source) + await asyncio.sleep(0) + + async def get_hostnames(self) -> set[str]: + return {f'{self.source}.example.test'} + + source_names = ('apis-guru', 'sourcegraph', 'crtsh', 'crt-name') + for source in source_names: + monkeypatch.setitem(SOURCE_FACTORIES, source, lambda _request, source=source: CompleteAdapter(source)) + jobs = tuple(SourceJob(SourceRequest(source, 'example.test', 25, 0, False, True)) for source in source_names) + + outcomes = await run_source_jobs(jobs, workers=workers) + + assert sorted(starts) == sorted(source_names) + assert [outcome.execution.source for outcome in outcomes] == list(source_names) + assert [outcome.execution.result_count for outcome in outcomes] == [1, 1, 1, 1] + assert len(task_names) == min(workers, len(jobs)) + + +@pytest.mark.asyncio +async def test_cancelled_source_commits_immutable_partial_outcome_then_propagates( + monkeypatch: pytest.MonkeyPatch, +) -> None: + committed: list[SourceOutcome] = [] + cancellation = asyncio.CancelledError() + + class CancelledSourcegraph: + async def process(self, _proxy: bool) -> None: + raise cancellation + + async def get_hostnames(self) -> set[str]: + return {'partial.example.test'} + + monkeypatch.setitem(SOURCE_FACTORIES, 'sourcegraph', lambda _request: CancelledSourcegraph()) + + with pytest.raises(asyncio.CancelledError) as raised: + await run_source( + SourceRequest('sourcegraph', 'example.test', 25, 0, False, True), + commit_cancelled=committed.append, + ) + + assert raised.value is cancellation + assert len(committed) == 1 + outcome = committed[0] + assert outcome.execution.status == 'partial' + assert outcome.execution.error_type == 'CancelledError' + assert outcome.execution.stop_reason == 'cancelled' + assert outcome.observations == (ResultObservation('sourcegraph', 'hostname', 'partial.example.test'),) + with pytest.raises(FrozenInstanceError): + outcome.observations = () # type: ignore[misc] + + +@pytest.mark.asyncio +async def test_child_cancellation_promptly_cleans_blocking_sibling_and_preserves_original( + monkeypatch: pytest.MonkeyPatch, +) -> None: + sibling_started = asyncio.Event() + sibling_cancelled = asyncio.Event() + cancellation = asyncio.CancelledError('source cancelled') + + class CancellingAdapter: + async def process(self, _proxy: bool) -> None: + await sibling_started.wait() + raise cancellation + + async def get_hostnames(self) -> set[str]: + return set() + + class BlockingAdapter: + async def process(self, _proxy: bool) -> None: + sibling_started.set() + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + sibling_cancelled.set() + raise + + async def get_hostnames(self) -> set[str]: + return set() + + monkeypatch.setitem(SOURCE_FACTORIES, 'sourcegraph', lambda _request: CancellingAdapter()) + monkeypatch.setitem(SOURCE_FACTORIES, 'crtsh', lambda _request: BlockingAdapter()) + jobs = tuple(SourceJob(SourceRequest(source, 'example.test', 25, 0, False, True)) for source in ('crtsh', 'sourcegraph')) + + with pytest.raises(asyncio.CancelledError) as raised: + async with asyncio.timeout(0.5): + await run_source_jobs(jobs) + + assert raised.value is cancellation + assert sibling_cancelled.is_set() + + +@pytest.mark.asyncio +async def test_parent_cancellation_commits_active_jobs_and_cleans_structured_tasks( + monkeypatch: pytest.MonkeyPatch, +) -> None: + started = asyncio.Event() + started_count = 0 + committed: list[SourceOutcome] = [] + + class BlockingAdapter: + def __init__(self, source: str) -> None: + self.source = source + + async def process(self, _proxy: bool) -> None: + nonlocal started_count + started_count += 1 + if started_count == 3: + started.set() + await asyncio.Event().wait() + + async def get_hostnames(self) -> set[str]: + return {f'{self.source}.example.test'} + + source_names = ('sourcegraph', 'crtsh', 'crt-name', 'apis-guru') + for source in source_names: + monkeypatch.setitem(SOURCE_FACTORIES, source, lambda _request, source=source: BlockingAdapter(source)) + jobs = tuple(SourceJob(SourceRequest(source, 'example.test', 25, 0, False, True)) for source in source_names) + task = asyncio.create_task(run_source_jobs(jobs, commit=committed.append)) + await started.wait() + task.cancel('parent-marker') + + with pytest.raises(asyncio.CancelledError) as raised: + await task + + assert raised.value.args == ('parent-marker',) + assert {outcome.execution.source for outcome in committed} == set(source_names) + assert all(outcome.execution.stop_reason == 'cancelled' for outcome in committed) + assert not [task for task in asyncio.all_tasks() if task.get_name().startswith('source-worker:') and not task.done()] diff --git a/tests/test_all_source_orchestration.py b/tests/test_all_source_orchestration.py index 5036d373..36ba2915 100644 --- a/tests/test_all_source_orchestration.py +++ b/tests/test_all_source_orchestration.py @@ -10,7 +10,8 @@ from uuid import UUID import pytest from theHarvester import __main__ as theharvester_main -from theHarvester.lib.source_catalog import SOURCE_SPECS, ActivityClass +from theHarvester.lib import source_catalog, source_runner +from theHarvester.lib.source_catalog import SOURCE_SPECS, ActivityClass, ResultRoute, SourceSpec NON_PASSIVE_SOURCES = ( 'criminalip', @@ -38,6 +39,46 @@ async def test_source_help_uses_the_runtime_catalog( assert 'linkedin_links' not in help_output +@pytest.mark.asyncio +async def test_catalog_and_factory_are_the_only_source_registration_points( + monkeypatch: pytest.MonkeyPatch, +) -> None: + runs = 0 + + class FakeResultStore: + async def initialize(self) -> None: + return None + + async def record_observations(self, *_args: object) -> None: + return None + + async def save_run(self, _result: object) -> None: + return None + + class CatalogOnlyAdapter: + async def process(self, _proxy: bool) -> None: + nonlocal runs + runs += 1 + return None + + async def get_hostnames(self) -> set[str]: + return {'catalog.example.test'} + + source = 'catalog-only-source' + spec = SourceSpec(source, frozenset({ResultRoute.SUBDOMAINS})) + monkeypatch.setitem(SOURCE_SPECS, source, spec) + monkeypatch.setitem(source_catalog._CASEFOLDED_SOURCE_SPECS, source, spec) + monkeypatch.setitem(source_runner.SOURCE_FACTORIES, source, lambda _request: CatalogOnlyAdapter()) + monkeypatch.setattr(theharvester_main, 'ResultStore', FakeResultStore) + monkeypatch.setattr(sys, 'argv', ['theHarvester', '-d', 'example.test', '-b', source]) + + with pytest.raises(SystemExit) as exit_info: + await theharvester_main.start() + + assert exit_info.value.code == 0 + assert runs == 1 + + @pytest.mark.asyncio async def test_activity_summary_includes_source_and_option_classes( monkeypatch: pytest.MonkeyPatch, @@ -69,7 +110,10 @@ async def test_activity_summary_includes_source_and_option_classes( async def get_asns(self) -> set[str]: return set() - monkeypatch.setattr(theharvester_main.criminalip, 'SearchCriminalIP', FakeCriminalIP) + async def get_shodan_hosts(self) -> tuple[()]: + return () + + monkeypatch.setattr(source_runner.criminalip, 'SearchCriminalIP', FakeCriminalIP) monkeypatch.setattr(theharvester_main, 'ResultStore', FakeResultStore) monkeypatch.setattr(sys, 'argv', ['theHarvester', '-d', 'example.test', '-b', 'criminalip', '-n', '-s']) @@ -110,8 +154,11 @@ async def test_activity_summary_covers_api_scan_without_sources( @pytest.mark.asyncio -async def test_legacy_handlerless_source_does_not_break_activity_summary( +@pytest.mark.parametrize('source', ['linkedin', 'netcraft', 'omnisint', 'sublist3r', 'zoomeyeapi']) +async def test_removed_source_is_rejected_before_source_execution( monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + source: str, ) -> None: class FakeResultStore: async def initialize(self) -> None: @@ -121,12 +168,13 @@ async def test_legacy_handlerless_source_does_not_break_activity_summary( return None monkeypatch.setattr(theharvester_main, 'ResultStore', FakeResultStore) - monkeypatch.setattr(sys, 'argv', ['theHarvester', '-d', 'example.test', '-b', 'linkedin']) + monkeypatch.setattr(sys, 'argv', ['theHarvester', '-d', 'example.test', '-b', source]) with pytest.raises(SystemExit) as exit_info: await theharvester_main.start() - assert exit_info.value.code == 0 + assert exit_info.value.code == 1 + assert 'SourceDidNotStart' not in capsys.readouterr().out @pytest.mark.asyncio @@ -168,10 +216,11 @@ async def test_explicit_non_passive_source_is_scheduled_once( monkeypatch.setattr(theharvester_main.shodansearch, 'SearchShodan', lambda *_args: FakeAdapter()) else: module, constructor_name = { - 'criminalip': (theharvester_main.criminalip, 'SearchCriminalIP'), - 'pentesttools': (theharvester_main.pentesttools, 'SearchPentestTools'), - 'shodanInternetDB': (theharvester_main.shodan_internetdb, 'SearchShodanInternetDB'), - 'subdomainfinderc99': (theharvester_main.subdomainfinderc99, 'SearchSubdomainfinderc99'), + 'criminalip': (source_runner.criminalip, 'SearchCriminalIP'), + 'pentesttools': (source_runner.pentesttools, 'SearchPentestTools'), + 'shodanInternetDB': (source_runner.shodan_internetdb, 'SearchShodanInternetDB'), + 'subdomainfinderc99': (source_runner.subdomainfinderc99, 'SearchSubdomainfinderc99'), + 'windvane': (source_runner.windvane, 'SearchWindvane'), }[source] monkeypatch.setattr(module, constructor_name, lambda *_args, **_kwargs: FakeAdapter()) @@ -243,7 +292,8 @@ async def test_all_schedules_each_passive_catalog_source_once_and_reports_result discovery_modules = sorted( { value - for value in vars(theharvester_main).values() + for namespace in (vars(theharvester_main), vars(source_runner)) + for value in namespace.values() if isinstance(value, ModuleType) and value.__name__.startswith('theHarvester.discovery.') }, key=lambda module: module.__name__, diff --git a/tests/test_main.py b/tests/test_main.py index 86e0726f..89c4226e 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -13,8 +13,9 @@ import pytest import theHarvester.screenshot.screenshot as screenshot_module from theHarvester import __main__ as theharvester_main from theHarvester.discovery.constants import MissingKey +from theHarvester.lib import source_runner from theHarvester.lib.asn_attribution import AsnAttributionObservation -from theHarvester.lib.completed_result import CompletedResult, ResultObservation +from theHarvester.lib.completed_result import CompletedResult, ResultObservation, SourceExecution from theHarvester.lib.core import FetcherResponse from theHarvester.lib.dns_consensus import Addressability from theHarvester.lib.enumeration import EnumerationOptions @@ -59,6 +60,8 @@ async def test_cli_help_explains_proxy_and_direct_action_scope( assert 'For normal use, pass only --vhost; bounded safety defaults apply automatically.' in help_text assert 'virtual host advanced controls' in help_text assert 'Candidate names are never resolved through DNS.' in help_text + assert '-j SOURCE_WORKERS' in help_text + assert '--source-workers SOURCE_WORKERS' in help_text def _confirmed_vhost(endpoint: str = 'http://192.0.2.10:80/') -> VirtualHostObservation: @@ -181,7 +184,7 @@ async def test_virtual_host_action_uses_harvested_hostnames_and_ips(monkeypatch: return HarvestedVirtualHostResult((), 5, 1, 1, 1, 1, 'completed') monkeypatch.setattr(theharvester_main, 'ResultStore', _NoopResultStore) - monkeypatch.setattr(theharvester_main.rapiddns, 'SearchRapidDns', FakeRapidDNS) + monkeypatch.setattr(source_runner.rapiddns, 'SearchRapidDns', FakeRapidDNS) monkeypatch.setattr(theharvester_main, 'discover_harvested_virtual_hosts', fake_discover) response = await theharvester_main.start( @@ -593,8 +596,8 @@ async def test_rapiddns_hostnames_honor_explicit_dns_resolution(monkeypatch: pyt ) monkeypatch.setattr(theharvester_main, 'ResultStore', FakeResultStore) - monkeypatch.setattr(theharvester_main.rapiddns, 'SearchRapidDns', FakeRapidDNS) - monkeypatch.setattr(theharvester_main.crtsh, 'SearchCrtsh', FakeCrtsh) + monkeypatch.setattr(source_runner.rapiddns, 'SearchRapidDns', FakeRapidDNS) + monkeypatch.setattr(source_runner.crtsh, 'SearchCrtsh', FakeCrtsh) monkeypatch.setattr(theharvester_main.hostchecker, 'Checker', FakeChecker) monkeypatch.setattr( @@ -696,7 +699,7 @@ async def test_provider_resolved_hostname_survives_no_answer_dns_validation( return [], [], [] monkeypatch.setattr(theharvester_main, 'ResultStore', FakeResultStore) - monkeypatch.setattr(theharvester_main.hackertarget, 'SearchHackerTarget', FakeHackerTarget) + monkeypatch.setattr(source_runner.hackertarget, 'SearchHackerTarget', FakeHackerTarget) monkeypatch.setattr(theharvester_main.hostchecker, 'Checker', NoAnswerChecker) output_path = tmp_path / 'provider-no-answer' @@ -938,10 +941,10 @@ async def test_dns_resolve_cancellation_closes_workers_and_persists_before_propa raise asyncio.CancelledError monkeypatch.setattr(theharvester_main, 'ResultStore', FakeResultStore) - monkeypatch.setattr(theharvester_main.certspottersearch, 'SearchCertspoter', FakeSource) - monkeypatch.setattr(theharvester_main.crtsh, 'SearchCrtsh', FakeSource) - monkeypatch.setattr(theharvester_main.shodanct, 'SearchShodanCt', FakeSource) - monkeypatch.setattr(theharvester_main.subdomaincenter, 'SubdomainCenter', FakeSource) + monkeypatch.setattr(source_runner.certspottersearch, 'SearchCertspoter', FakeSource) + monkeypatch.setattr(source_runner.crtsh, 'SearchCrtsh', FakeSource) + monkeypatch.setattr(source_runner.shodanct, 'SearchShodanCt', FakeSource) + monkeypatch.setattr(source_runner.subdomaincenter, 'SubdomainCenter', FakeSource) monkeypatch.setattr(theharvester_main.hostchecker, 'Checker', CancelledChecker) with pytest.raises(asyncio.CancelledError): @@ -986,7 +989,7 @@ async def test_source_cancellation_before_dns_resolution_does_not_claim_dns_fail raise asyncio.CancelledError monkeypatch.setattr(theharvester_main, 'ResultStore', FakeResultStore) - monkeypatch.setattr(theharvester_main.certspottersearch, 'SearchCertspoter', CancelledSource) + monkeypatch.setattr(source_runner.certspottersearch, 'SearchCertspoter', CancelledSource) with pytest.raises(asyncio.CancelledError): await theharvester_main.start( @@ -1039,7 +1042,7 @@ async def test_dns_resolve_query_errors_are_partial_even_without_findings(monkey return [], [], [] monkeypatch.setattr(theharvester_main, 'ResultStore', FakeResultStore) - monkeypatch.setattr(theharvester_main.certspottersearch, 'SearchCertspoter', FakeSource) + monkeypatch.setattr(source_runner.certspottersearch, 'SearchCertspoter', FakeSource) monkeypatch.setattr(theharvester_main.hostchecker, 'Checker', EmptyChecker) response = await theharvester_main.start( @@ -1094,7 +1097,7 @@ async def test_dns_resolve_budget_stop_retains_partial_evidence(monkeypatch: pyt return ['api.example.com:192.0.2.10'], ['api.example.com'], ['192.0.2.10'] monkeypatch.setattr(theharvester_main, 'ResultStore', FakeResultStore) - monkeypatch.setattr(theharvester_main.certspottersearch, 'SearchCertspoter', FakeSource) + monkeypatch.setattr(source_runner.certspottersearch, 'SearchCertspoter', FakeSource) monkeypatch.setattr(theharvester_main.hostchecker, 'Checker', LimitedChecker) response = await theharvester_main.start( @@ -1134,7 +1137,7 @@ async def test_explicit_dns_resolvers_normalizing_empty_fail_closed(monkeypatch: raise AssertionError('explicit invalid resolvers must not fall back to system DNS') monkeypatch.setattr(theharvester_main, 'ResultStore', FakeResultStore) - monkeypatch.setattr(theharvester_main.certspottersearch, 'SearchCertspoter', FakeSource) + monkeypatch.setattr(source_runner.certspottersearch, 'SearchCertspoter', FakeSource) monkeypatch.setattr(theharvester_main, 'normalize_resolver_addresses', lambda _values: []) monkeypatch.setattr(theharvester_main.hostchecker, 'Checker', UnexpectedChecker) @@ -1212,7 +1215,7 @@ async def test_rest_dns_lookup_runs_before_return_and_retains_action_evidence(mo return theharvester_main.dnssearch.ReverseDNSResult(254, 254) monkeypatch.setattr(theharvester_main, 'ResultStore', FakeResultStore) - monkeypatch.setattr(theharvester_main.securityscorecard, 'SearchSecurityScorecard', FakeSecurityScorecard) + monkeypatch.setattr(source_runner.securityscorecard, 'SearchSecurityScorecard', FakeSecurityScorecard) monkeypatch.setattr(theharvester_main.dnssearch, 'reverse_ip_ranges', fake_reverse) response = await theharvester_main.start( @@ -1296,7 +1299,7 @@ async def test_dns_lookup_cancellation_persists_partial_evidence( raise asyncio.CancelledError monkeypatch.setattr(theharvester_main, 'ResultStore', FakeResultStore) - monkeypatch.setattr(theharvester_main.securityscorecard, 'SearchSecurityScorecard', FakeSecurityScorecard) + monkeypatch.setattr(source_runner.securityscorecard, 'SearchSecurityScorecard', FakeSecurityScorecard) monkeypatch.setattr(theharvester_main.dnssearch, 'reverse_ip_ranges', fake_reverse) with pytest.raises(asyncio.CancelledError): @@ -1319,8 +1322,11 @@ async def test_dns_lookup_cancellation_persists_partial_evidence( @pytest.mark.parametrize( ('source', 'module', 'constructor_name'), [ - ('projectdiscovery', theharvester_main.projectdiscovery, 'SearchDiscovery'), - ('bevigil', theharvester_main.bevigil, 'SearchBeVigil'), + ('projectdiscovery', source_runner.projectdiscovery, 'SearchDiscovery'), + ('bevigil', source_runner.bevigil, 'SearchBeVigil'), + ('builtwith', source_runner.builtwith, 'SearchBuiltWith'), + ('hudsonrock', source_runner.hudsonrocksearch, 'SearchHudsonRock'), + ('shodan', source_runner.shodansearch, 'SearchShodan'), ], ) @pytest.mark.asyncio @@ -1364,6 +1370,440 @@ async def test_constructor_missing_credentials_are_persisted_as_skipped_source( assert execution.stop_reason == 'missing-credentials' +@pytest.mark.asyncio +async def test_target_only_source_reports_search_progress_at_the_cli_seam( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + class EmptyBeVigil: + def __init__(self, _word: str) -> None: + pass + + async def process(self, _proxy: bool) -> None: + return None + + async def get_hostnames(self) -> set[str]: + return set() + + async def get_urls(self) -> set[str]: + return set() + + monkeypatch.setattr(theharvester_main, 'ResultStore', _NoopResultStore) + monkeypatch.setattr(source_runner.bevigil, 'SearchBeVigil', EmptyBeVigil) + monkeypatch.setattr(sys, 'argv', ['theHarvester', '-d', 'example.test', '-b', 'bevigil']) + + with pytest.raises(SystemExit) as exit_info: + await theharvester_main.start() + + assert exit_info.value.code == 0 + assert '[*] Searching Bevigil.' in capsys.readouterr().out + + +@pytest.mark.asyncio +async def test_source_progress_waits_for_runner_admission( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + sources = ('certspotter', 'crt-name', 'crtsh', 'dymo') + requested_workers = 3 + admitted_sources = sources[:requested_workers] + constructed: list[str] = [] + processing: set[str] = set() + admitted = asyncio.Event() + release = asyncio.Event() + + class GatedSource: + def __init__(self, source: str) -> None: + self.source = source + + async def process(self, _proxy: bool) -> None: + processing.add(self.source) + if len(processing) == requested_workers: + admitted.set() + await release.wait() + + async def get_hostnames(self) -> set[str]: + return set() + + def factory(source: str) -> source_runner.SourceFactory: + def create(_request: source_runner.SourceRequest) -> GatedSource: + constructed.append(source) + return GatedSource(source) + + return create + + for source in sources: + monkeypatch.setitem(source_runner.SOURCE_FACTORIES, source, factory(source)) + monkeypatch.setattr(theharvester_main, 'ResultStore', _NoopResultStore) + theharvester_main.configure_logging(verbose=False) + + task = asyncio.create_task( + theharvester_main.start( + EnumerationOptions(domain='example.test', source=','.join(sources), source_workers=requested_workers, quiet=False), + return_completed_result=True, + ) + ) + await asyncio.wait_for(admitted.wait(), timeout=1) + interim_output = capsys.readouterr().out + release.set() + await task + + assert constructed[:requested_workers] == list(admitted_sources) + assert all(f'[*] Searching {source[0].upper() + source[1:]}.' in interim_output for source in admitted_sources) + assert '[*] Searching Dymo.' not in interim_output + + +@pytest.mark.asyncio +async def test_source_completion_reports_verbose_terminal_summary( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + class CompletedCertSpotter: + async def process(self, _proxy: bool) -> None: + return None + + async def get_hostnames(self) -> set[str]: + return {'api.example.test'} + + monkeypatch.setattr(theharvester_main, 'ResultStore', _NoopResultStore) + monkeypatch.setattr(source_runner.certspottersearch, 'SearchCertspoter', lambda _word: CompletedCertSpotter()) + caplog.set_level(logging.INFO, logger='theHarvester.__main__') + + await theharvester_main.start( + EnumerationOptions(domain='example.test', source='certspotter', quiet=True), + return_completed_result=True, + ) + + assert any( + message.startswith('Source certspotter finished in ') and message.endswith('s: status=completed; results=1') + for message in caplog.messages + ) + + +@pytest.mark.parametrize( + ('error', 'expected_status', 'expected_error_type', 'expected_stop_reason', 'expected_diagnostic'), + [ + ( + MissingKey('sensitive-provider-detail'), + 'skipped', + 'MissingKeyError', + 'missing-credentials', + '[!] Source bevigil skipped: missing credentials.', + ), + ( + RuntimeError('sensitive-provider-detail'), + 'failed', + 'RuntimeError', + None, + '[!] Source bevigil failed: RuntimeError.', + ), + ], +) +@pytest.mark.asyncio +async def test_target_only_constructor_failure_reports_sanitized_cli_diagnostic( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + error: Exception, + expected_status: str, + expected_error_type: str, + expected_stop_reason: str | None, + expected_diagnostic: str, +) -> None: + saved_results: list[CompletedResult] = [] + + class RecordingResultStore(_NoopResultStore): + async def save_run(self, result: CompletedResult) -> None: + saved_results.append(result) + + class BrokenBeVigil: + def __init__(self, _word: str) -> None: + raise error + + monkeypatch.setattr(theharvester_main, 'ResultStore', RecordingResultStore) + monkeypatch.setattr(source_runner.bevigil, 'SearchBeVigil', BrokenBeVigil) + monkeypatch.setattr(sys, 'argv', ['theHarvester', '-d', 'example.test', '-b', 'bevigil']) + + with pytest.raises(SystemExit) as exit_info: + await theharvester_main.start() + + output = capsys.readouterr().out + assert exit_info.value.code == 0 + assert expected_diagnostic in output + assert '[*] Searching Bevigil.' not in output + assert 'sensitive-provider-detail' not in output + execution = saved_results[-1].source_executions[0] + assert execution.status == expected_status + assert execution.error_type == expected_error_type + assert execution.stop_reason == expected_stop_reason + + +@pytest.mark.asyncio +async def test_partial_source_reports_sanitized_cli_diagnostic( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + class PartialBeVigil: + async def process(self, _proxy: bool) -> None: + raise RuntimeError('sensitive-provider-detail') + + async def get_hostnames(self) -> set[str]: + return {'partial.example.test'} + + async def get_urls(self) -> set[str]: + return set() + + monkeypatch.setattr(theharvester_main, 'ResultStore', _NoopResultStore) + monkeypatch.setattr(source_runner.bevigil, 'SearchBeVigil', lambda _word: PartialBeVigil()) + theharvester_main.configure_logging(verbose=False) + + response = await theharvester_main.start( + EnumerationOptions(domain='example.test', source='bevigil', quiet=False), + return_completed_result=True, + ) + + output = capsys.readouterr().out + assert '[!] Source bevigil partial: RuntimeError.' in output + assert 'sensitive-provider-detail' not in output + assert response[-1].source_executions[0].status == 'partial' + + +@pytest.mark.asyncio +async def test_source_checkpoint_failure_does_not_log_sensitive_exception_text( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + class EmptyBeVigil: + async def process(self, _proxy: bool) -> None: + return None + + async def get_hostnames(self) -> set[str]: + return set() + + async def get_urls(self) -> set[str]: + return set() + + checkpoint_calls = 0 + + async def broken_checkpoint(_result: CompletedResult) -> None: + nonlocal checkpoint_calls + checkpoint_calls += 1 + if checkpoint_calls == 1: + raise RuntimeError('sensitive-checkpoint-detail') + + monkeypatch.setattr(theharvester_main, 'ResultStore', _NoopResultStore) + monkeypatch.setattr(source_runner.bevigil, 'SearchBeVigil', lambda _word: EmptyBeVigil()) + theharvester_main.configure_logging(verbose=False) + + await theharvester_main.start( + EnumerationOptions(domain='example.test', source='bevigil', quiet=True), + completed_result_checkpoint=broken_checkpoint, + return_completed_result=True, + ) + + output = capsys.readouterr().out + assert 'An error occurred while committing bevigil: RuntimeError.' in output + assert 'sensitive-checkpoint-detail' not in output + + +@pytest.mark.asyncio +async def test_source_cancellation_does_not_announce_never_started_queued_source( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + sources = ('certspotter', 'crt-name', 'crtsh', 'dymo') + requested_workers = 3 + cancellation = asyncio.CancelledError('source cancelled') + constructed: list[str] = [] + processing: set[str] = set() + admitted = asyncio.Event() + + class CancellingSource: + def __init__(self, source: str) -> None: + self.source = source + + async def process(self, _proxy: bool) -> None: + processing.add(self.source) + if len(processing) == requested_workers: + admitted.set() + await admitted.wait() + if self.source == sources[0]: + raise cancellation + await asyncio.Event().wait() + + async def get_hostnames(self) -> set[str]: + return set() + + def factory(source: str) -> source_runner.SourceFactory: + def create(_request: source_runner.SourceRequest) -> CancellingSource: + constructed.append(source) + return CancellingSource(source) + + return create + + for source in sources: + monkeypatch.setitem(source_runner.SOURCE_FACTORIES, source, factory(source)) + monkeypatch.setattr(theharvester_main, 'ResultStore', _NoopResultStore) + theharvester_main.configure_logging(verbose=False) + + with pytest.raises(asyncio.CancelledError) as raised: + async with asyncio.timeout(1): + await theharvester_main.start( + EnumerationOptions( + domain='example.test', source=','.join(sources), source_workers=requested_workers, quiet=False + ), + return_completed_result=True, + ) + + output = capsys.readouterr().out + assert raised.value is cancellation + assert constructed == list(sources[:requested_workers]) + assert '[*] Searching Dymo.' not in output + + +@pytest.mark.parametrize( + ('source', 'module', 'constructor_name'), + [ + ('builtwith', source_runner.builtwith, 'SearchBuiltWith'), + ('hudsonrock', source_runner.hudsonrocksearch, 'SearchHudsonRock'), + ('shodan', source_runner.shodansearch, 'SearchShodan'), + ], +) +@pytest.mark.asyncio +async def test_special_source_constructor_failure_uses_runner_outcome( + monkeypatch: pytest.MonkeyPatch, + source: str, + module: ModuleType, + constructor_name: str, +) -> None: + class BrokenSource: + def __init__(self, _word: str) -> None: + raise RuntimeError('construction failed') + + monkeypatch.setattr(theharvester_main, 'ResultStore', _NoopResultStore) + monkeypatch.setattr(module, constructor_name, BrokenSource) + + response = await theharvester_main.start( + EnumerationOptions(domain='example.test', source=source, quiet=True), + return_completed_result=True, + ) + + completed = response[-1] + assert isinstance(completed, CompletedResult) + assert completed.source_executions == ( + SourceExecution(source, 'failed', completed.source_executions[0].duration_ms, 0, 'RuntimeError'), + ) + + +@pytest.mark.parametrize( + ('source', 'module', 'constructor_name', 'expected_results'), + [ + ( + 'builtwith', + source_runner.builtwith, + 'SearchBuiltWith', + { + ('analytics', 'Plausible'), + ('cms', 'Wagtail'), + ('framework', 'Django'), + ('hostname', 'partial.example.test'), + ('language', 'Python'), + ('server', 'nginx'), + ('url', 'https://partial.example.test'), + }, + ), + ( + 'hudsonrock', + source_runner.hudsonrocksearch, + 'SearchHudsonRock', + { + ('email', 'user@example.test'), + ('hostname', 'partial.example.test'), + ('infostealer', '{"email":"user@example.test"}'), + }, + ), + ('shodan', source_runner.shodansearch, 'SearchShodan', {('hostname', 'partial.example.test')}), + ], +) +@pytest.mark.asyncio +async def test_special_source_cancellation_retains_partial_evidence_and_original_error( + monkeypatch: pytest.MonkeyPatch, + source: str, + module: ModuleType, + constructor_name: str, + expected_results: set[tuple[str, str]], +) -> None: + cancellation = asyncio.CancelledError(f'cancel {source}') + saved_results: list[CompletedResult] = [] + + class RecordingResultStore(_NoopResultStore): + async def save_run(self, result: CompletedResult) -> None: + saved_results.append(result) + + class CancellingSource: + def __init__(self, _word: str) -> None: + pass + + async def process(self, _proxy: bool) -> None: + raise cancellation + + async def get_hostnames(self) -> set[str]: + return {'partial.example.test'} + + async def get_emails(self) -> set[str]: + return {'user@example.test'} + + async def get_ips(self) -> set[str]: + return set() + + async def get_urls(self) -> set[str]: + return {'https://partial.example.test'} + + async def get_frameworks(self) -> set[str]: + return {'Django'} + + async def get_languages(self) -> set[str]: + return {'Python'} + + async def get_servers(self) -> set[str]: + return {'nginx'} + + async def get_cms(self) -> set[str]: + return {'Wagtail'} + + async def get_analytics(self) -> set[str]: + return {'Plausible'} + + async def get_infostealers(self) -> list[dict[str, str]]: + return [{'email': 'user@example.test'}] + + async def get_shodan_hosts(self) -> tuple[()]: + return () + + monkeypatch.setattr(theharvester_main, 'ResultStore', RecordingResultStore) + monkeypatch.setattr(module, constructor_name, CancellingSource) + + with pytest.raises(asyncio.CancelledError) as raised: + await theharvester_main.start( + EnumerationOptions(domain='example.test', source=source, quiet=True), + persist_completed_result=True, + ) + + assert raised.value is cancellation + assert len(saved_results) == 1 + assert set(saved_results[0].results) == expected_results + assert saved_results[0].source_executions == ( + SourceExecution( + source, + 'partial', + saved_results[0].source_executions[0].duration_ms, + len(expected_results), + 'CancelledError', + 'cancelled', + ), + ) + assert {observation.source for observation in saved_results[0].observations} == {source} + + @pytest.mark.asyncio async def test_source_failure_retains_normalized_partial_results(monkeypatch: pytest.MonkeyPatch) -> None: completed: list[CompletedResult] = [] @@ -1389,7 +1829,7 @@ async def test_source_failure_retains_normalized_partial_results(monkeypatch: py raise RuntimeError('provider page failed') monkeypatch.setattr(theharvester_main, 'ResultStore', FakeResultStore) - monkeypatch.setattr(theharvester_main.builtwith, 'SearchBuiltWith', PartiallyFailingBuiltWith) + monkeypatch.setattr(source_runner.builtwith, 'SearchBuiltWith', PartiallyFailingBuiltWith) monkeypatch.setattr(sys, 'argv', ['theHarvester', '-d', 'example.com', '-b', 'builtwith']) with pytest.raises(SystemExit) as exit_info: @@ -1464,8 +1904,8 @@ async def test_source_checkpoint_excludes_other_source_work_in_progress(monkeypa release_builtwith.set() monkeypatch.setattr(theharvester_main, 'ResultStore', FakeResultStore) - monkeypatch.setattr(theharvester_main.builtwith, 'SearchBuiltWith', PausedBuiltWith) - monkeypatch.setattr(theharvester_main.crtsh, 'SearchCrtsh', FastCrtsh) + monkeypatch.setattr(source_runner.builtwith, 'SearchBuiltWith', PausedBuiltWith) + monkeypatch.setattr(source_runner.crtsh, 'SearchCrtsh', FastCrtsh) monkeypatch.setattr(sys, 'argv', ['theHarvester', '-d', 'example.com', '-b', 'builtwith,crtsh']) with pytest.raises(SystemExit) as exit_info: @@ -1483,6 +1923,204 @@ async def test_source_checkpoint_excludes_other_source_work_in_progress(monkeypa } +@pytest.mark.parametrize( + ('source', 'main_module_name', 'constructor_name'), + [ + ('arquivo', 'arquivo', 'SearchArquivo'), + ('baidu', 'baidusearch', 'SearchBaidu'), + ('brave', 'bravesearch', 'SearchBrave'), + ('censys', 'censysearch', 'SearchCensys'), + ('commoncrawl', 'commoncrawl', 'SearchCommoncrawl'), + ('dehashed', 'search_dehashed', 'SearchDehashed'), + ('github-code', 'githubcode', 'SearchGithubCode'), + ('hunter', 'huntersearch', 'SearchHunter'), + ('mojeek', 'mojeek', 'SearchMojeek'), + ('netlas', 'netlas', 'SearchNetlas'), + ('rocketreach', 'rocketreach', 'SearchRocketReach'), + ('tomba', 'tombasearch', 'SearchTomba'), + ('waybackarchive', 'waybackarchive', 'SearchWaybackarchive'), + ('yahoo', 'yahoosearch', 'SearchYahoo'), + ('zoomeye', 'zoomeyesearch', 'SearchZoomEye'), + ], +) +@pytest.mark.asyncio +async def test_limited_source_orchestration_uses_immutable_runner_request( + monkeypatch: pytest.MonkeyPatch, + source: str, + main_module_name: str, + constructor_name: str, +) -> None: + requests: list[source_runner.SourceRequest] = [] + processed_with_proxy: list[bool] = [] + + class FakeAdapter: + execution_status = 'partial' + stop_reason = 'provider-boundary' + + async def process(self, proxy: bool) -> None: + processed_with_proxy.append(proxy) + + async def get_hostnames(self) -> set[str]: + return {'sub.example.test'} + + async def get_emails(self) -> set[str]: + return {'user@example.test'} + + async def get_ips(self) -> set[str]: + return {'192.0.2.1'} + + async def get_asns(self) -> set[str]: + return {'AS64500'} + + async def get_urls(self) -> set[str]: + return {'https://sub.example.test/evidence'} + + def runner_factory(request: source_runner.SourceRequest) -> FakeAdapter: + requests.append(request) + return FakeAdapter() + + def legacy_constructor(*_args: object, **_kwargs: object) -> object: + raise AssertionError('legacy constructor branch was used') + + legacy_module = ModuleType(f'test_legacy_{main_module_name}') + setattr(legacy_module, constructor_name, legacy_constructor) + monkeypatch.setattr(theharvester_main, main_module_name, legacy_module, raising=False) + monkeypatch.setitem(source_runner.SOURCE_FACTORIES, source, runner_factory) + monkeypatch.setattr(theharvester_main, 'ResultStore', _NoopResultStore) + + response = await theharvester_main.start( + EnumerationOptions( + domain='example.test', + source=source, + limit=37, + start=11, + proxies=True, + quiet=True, + ), + return_completed_result=True, + ) + + assert requests == [source_runner.SourceRequest(source, 'example.test', 37, 11, True, True)] + assert processed_with_proxy == [True] + execution = response[-1].source_executions[0] + assert execution.source == source + assert execution.status == 'partial' + assert execution.stop_reason == 'provider-boundary' + + +@pytest.mark.parametrize( + ('source', 'main_module_name', 'constructor_name'), + [ + ('bevigil', 'bevigil', 'SearchBeVigil'), + ('bufferoverun', 'bufferoverun', 'SearchBufferover'), + ('certspotter', 'certspottersearch', 'SearchCertspoter'), + ('criminalip', 'criminalip', 'SearchCriminalIP'), + ('crt-name', 'crtname', 'SearchCrtName'), + ('crtsh', 'crtsh', 'SearchCrtsh'), + ('dnsdb', 'dnsdb', 'SearchDNSDB'), + ('dnsdumpster', 'search_dnsdumpster', 'SearchDNSDumpster'), + ('dymo', 'dymosearch', 'SearchDymo'), + ('fofa', 'fofa', 'SearchFofa'), + ('fullhunt', 'fullhuntsearch', 'SearchFullHunt'), + ('gitlab', 'gitlabsearch', 'SearchGitlab'), + ('hackertarget', 'hackertarget', 'SearchHackerTarget'), + ('haveibeenpwned', 'haveibeenpwned', 'SearchHaveIBeenPwned'), + ('hibpverified', 'hibpverified', 'SearchHibpVerified'), + ('hunterhow', 'searchhunterhow', 'SearchHunterHow'), + ('intelx', 'intelxsearch', 'SearchIntelx'), + ('leakix', 'leakix', 'SearchLeakix'), + ('leaklookup', 'leaklookup', 'SearchLeakLookup'), + ('onyphe', 'onyphe', 'SearchOnyphe'), + ('otx', 'otxsearch', 'SearchOtx'), + ('pentesttools', 'pentesttools', 'SearchPentestTools'), + ('projectdiscovery', 'projectdiscovery', 'SearchDiscovery'), + ('rapiddns', 'rapiddns', 'SearchRapidDns'), + ('robtex', 'robtex', 'SearchRobtex'), + ('securityscorecard', 'securityscorecard', 'SearchSecurityScorecard'), + ('securityTrails', 'securitytrailssearch', 'SearchSecuritytrail'), + ('sherlockeye', 'sherlockeye', 'SearchSherlockeye'), + ('shodanInternetDB', 'shodan_internetdb', 'SearchShodanInternetDB'), + ('shodanct', 'shodanct', 'SearchShodanCt'), + ('subdomaincenter', 'subdomaincenter', 'SubdomainCenter'), + ('subdomainfinderc99', 'subdomainfinderc99', 'SearchSubdomainfinderc99'), + ('thc', 'thc', 'SearchThc'), + ('urlscan', 'urlscan', 'SearchUrlscan'), + ('virustotal', 'virustotal', 'SearchVirustotal'), + ('whoisxml', 'whoisxml', 'SearchWhoisXML'), + ('windvane', 'windvane', 'SearchWindvane'), + ], +) +@pytest.mark.asyncio +async def test_target_only_source_orchestration_uses_immutable_runner_request( + monkeypatch: pytest.MonkeyPatch, + source: str, + main_module_name: str, + constructor_name: str, +) -> None: + requests: list[source_runner.SourceRequest] = [] + processed_with_proxy: list[bool] = [] + + class FakeAdapter: + execution_status = 'partial' + stop_reason = 'provider-boundary' + + async def process(self, proxy: bool) -> None: + processed_with_proxy.append(proxy) + + async def get_hostnames(self) -> set[str]: + return {'sub.example.test'} + + async def get_emails(self) -> set[str]: + return {'user@example.test'} + + async def get_ips(self) -> set[str]: + return {'192.0.2.1'} + + async def get_asns(self) -> set[str]: + return {'AS64500'} + + async def get_urls(self) -> set[str]: + return {'https://sub.example.test/evidence'} + + async def get_breach_names(self) -> set[str]: + return {'Example Breach'} + + async def get_host_ip_pairs(self) -> set[tuple[str, str]]: + return set() + + def runner_factory(request: source_runner.SourceRequest) -> FakeAdapter: + requests.append(request) + return FakeAdapter() + + def legacy_constructor(*_args: object, **_kwargs: object) -> object: + raise AssertionError('legacy constructor branch was used') + + legacy_module = ModuleType(f'test_legacy_{main_module_name}') + setattr(legacy_module, constructor_name, legacy_constructor) + monkeypatch.setattr(theharvester_main, main_module_name, legacy_module, raising=False) + monkeypatch.setitem(source_runner.SOURCE_FACTORIES, source, runner_factory) + monkeypatch.setattr(theharvester_main, 'ResultStore', _NoopResultStore) + + response = await theharvester_main.start( + EnumerationOptions( + domain='example.test', + source=source, + limit=37, + start=11, + proxies=True, + quiet=True, + ), + return_completed_result=True, + ) + + assert requests == [source_runner.SourceRequest(source, 'example.test', 37, 11, True, True)] + assert processed_with_proxy == [True] + execution = response[-1].source_executions[0] + assert execution.source == source + assert execution.status == 'partial' + assert execution.stop_reason == 'provider-boundary' + + @pytest.mark.asyncio async def test_invalid_source_outcome_is_not_recorded_as_completed(monkeypatch: pytest.MonkeyPatch) -> None: completed: list[CompletedResult] = [] @@ -1507,7 +2145,7 @@ async def test_invalid_source_outcome_is_not_recorded_as_completed(monkeypatch: return set() monkeypatch.setattr(theharvester_main, 'ResultStore', FakeResultStore) - monkeypatch.setattr(theharvester_main.crtsh, 'SearchCrtsh', InvalidOutcomeCrtsh) + monkeypatch.setattr(source_runner.crtsh, 'SearchCrtsh', InvalidOutcomeCrtsh) monkeypatch.setattr(sys, 'argv', ['theHarvester', '-d', 'example.com', '-b', 'crtsh']) with pytest.raises(SystemExit) as exit_info: @@ -1708,7 +2346,7 @@ async def test_dns_proven_cname_hosts_reach_screenshot_filter( return Path(self.output) / f'{host.removeprefix("https://")}.png' monkeypatch.setattr(theharvester_main, 'ResultStore', FakeResultStore) - monkeypatch.setattr(theharvester_main.crtsh, 'SearchCrtsh', FakeCrtsh) + monkeypatch.setattr(source_runner.crtsh, 'SearchCrtsh', FakeCrtsh) monkeypatch.setattr(theharvester_main.hostchecker, 'Checker', FakeChecker) monkeypatch.setattr(theharvester_main, 'ScreenShotter', FakeScreenShotter) monkeypatch.setattr( @@ -1745,6 +2383,29 @@ class _NoopResultStore: return None +@pytest.mark.asyncio +async def test_source_worker_count_reaches_the_runner(monkeypatch: pytest.MonkeyPatch) -> None: + captured_workers: list[int] = [] + + async def fake_run_source_jobs(_jobs, *, workers: int, **_kwargs): + captured_workers.append(workers) + return () + + monkeypatch.setattr(theharvester_main, 'ResultStore', _NoopResultStore) + monkeypatch.setattr(theharvester_main, 'run_source_jobs', fake_run_source_jobs) + monkeypatch.setattr( + sys, + 'argv', + ['theHarvester', '-d', 'example.test', '-b', 'crtsh', '-j', '7', '--quiet'], + ) + + with pytest.raises(SystemExit) as exit_info: + await theharvester_main.start() + + assert exit_info.value.code == 0 + assert captured_workers == [7] + + class _ApiHostSource: def __init__(self, _word: str) -> None: pass @@ -2166,7 +2827,7 @@ async def test_screenshot_redirects_to_one_login_keep_distinct_subject_artifacts return {'first.example.test', 'second.example.test'} monkeypatch.setattr(theharvester_main, 'ResultStore', _NoopResultStore) - monkeypatch.setattr(theharvester_main.crtsh, 'SearchCrtsh', TwoHostSource) + monkeypatch.setattr(source_runner.crtsh, 'SearchCrtsh', TwoHostSource) monkeypatch.setattr(theharvester_main, 'ScreenShotter', FakeScreenShotter) result = await theharvester_main.start( @@ -2232,7 +2893,7 @@ async def test_screenshot_cancellation_persists_failed_execution_and_propagates( return {'first.example.test', 'second.example.test'} monkeypatch.setattr(theharvester_main, 'ResultStore', RecordingResultStore) - monkeypatch.setattr(theharvester_main.crtsh, 'SearchCrtsh', TwoHostSource) + monkeypatch.setattr(source_runner.crtsh, 'SearchCrtsh', TwoHostSource) monkeypatch.setattr(theharvester_main, 'ScreenShotter', FakeScreenShotter) task = asyncio.create_task( @@ -2299,7 +2960,7 @@ async def test_screenshot_capture_failure_cancels_sibling_tasks( return {'first.example.test', 'second.example.test'} monkeypatch.setattr(theharvester_main, 'ResultStore', _NoopResultStore) - monkeypatch.setattr(theharvester_main.crtsh, 'SearchCrtsh', TwoHostSource) + monkeypatch.setattr(source_runner.crtsh, 'SearchCrtsh', TwoHostSource) monkeypatch.setattr(theharvester_main, 'ScreenShotter', FakeScreenShotter) result = await theharvester_main.start( @@ -2459,7 +3120,7 @@ async def test_direct_action_evidence_reaches_completed_result(monkeypatch: pyte wordlist = tmp_path / 'api.txt' wordlist.write_text('/api/v1\n', encoding='utf-8') monkeypatch.setattr(theharvester_main, 'ResultStore', FakeResultStore) - monkeypatch.setattr(theharvester_main.crtsh, 'SearchCrtsh', FakeCrtsh) + monkeypatch.setattr(source_runner.crtsh, 'SearchCrtsh', FakeCrtsh) monkeypatch.setattr(theharvester_main.hostchecker, 'Checker', FakeChecker) monkeypatch.setattr(theharvester_main.takeover, 'TakeOver', FakeTakeOver) monkeypatch.setattr(theharvester_main, 'ScreenShotter', FakeScreenShotter) @@ -2643,7 +3304,7 @@ async def test_takeover_action_records_suppressed_outcome( return {} monkeypatch.setattr(theharvester_main, 'ResultStore', _NoopResultStore) - monkeypatch.setattr(theharvester_main.crtsh, 'SearchCrtsh', _ApiHostSource) + monkeypatch.setattr(source_runner.crtsh, 'SearchCrtsh', _ApiHostSource) monkeypatch.setattr(theharvester_main.takeover, 'TakeOver', FakeTakeOver) result = await theharvester_main.start( @@ -2672,7 +3333,7 @@ async def test_shodan_action_records_all_target_errors_as_failed( raise RuntimeError(f'provider-secret-payload for {ip}') monkeypatch.setattr(theharvester_main, 'ResultStore', _NoopResultStore) - monkeypatch.setattr(theharvester_main.crtsh, 'SearchCrtsh', _ApiHostSource) + monkeypatch.setattr(source_runner.crtsh, 'SearchCrtsh', _ApiHostSource) monkeypatch.setattr(theharvester_main.hostchecker, 'Checker', _ApiHostChecker) monkeypatch.setattr(theharvester_main.shodansearch, 'SearchShodan', FailedShodan) @@ -2706,7 +3367,7 @@ async def test_shodan_no_data_is_a_completed_zero_yield_action(monkeypatch: pyte return {} monkeypatch.setattr(theharvester_main, 'ResultStore', _NoopResultStore) - monkeypatch.setattr(theharvester_main.crtsh, 'SearchCrtsh', _ApiHostSource) + monkeypatch.setattr(source_runner.crtsh, 'SearchCrtsh', _ApiHostSource) monkeypatch.setattr(theharvester_main.hostchecker, 'Checker', _ApiHostChecker) monkeypatch.setattr(theharvester_main.shodansearch, 'SearchShodan', EmptyShodan) @@ -2877,9 +3538,7 @@ async def test_api_scan_logs_result_collections_deterministically( ) rendered_endpoints = [ - message.removeprefix(' - ') - for message in caplog.messages - if message.startswith(' - https://example.com/api') + message.removeprefix(' - ') for message in caplog.messages if message.startswith(' - https://example.com/api') ] assert rendered_endpoints == sorted(endpoint_values) assert f'[*] HTTP methods used: {", ".join(sorted(method_values))}' in caplog.text @@ -3127,7 +3786,7 @@ async def test_routeviews_pivots_from_attributed_ips_without_expanding_discovere return RouteViewsResult((), (), (), 2, 0, 'completed', stop_reason='no-results') monkeypatch.setattr(theharvester_main, 'ResultStore', _NoopResultStore) - monkeypatch.setattr(theharvester_main.urlscan, 'SearchUrlscan', FakeUrlscan) + monkeypatch.setattr(source_runner.urlscan, 'SearchUrlscan', FakeUrlscan) monkeypatch.setattr(theharvester_main, 'enrich_routeviews', fake_routeviews) monkeypatch.setattr(theharvester_main.Core, 'routeviews_key', staticmethod(lambda: None)) @@ -3398,7 +4057,7 @@ async def test_takeover_failure_persists_and_propagates( return {} monkeypatch.setattr(theharvester_main, 'ResultStore', _recording_result_store(saved)) - monkeypatch.setattr(theharvester_main.crtsh, 'SearchCrtsh', _ApiHostSource) + monkeypatch.setattr(source_runner.crtsh, 'SearchCrtsh', _ApiHostSource) monkeypatch.setattr(theharvester_main.takeover, 'TakeOver', CancelledTakeOver) with pytest.raises(type(raised_error)): @@ -3453,7 +4112,7 @@ async def test_shodan_cancellation_persists_failure_and_propagates(monkeypatch: return tuple(self.hosts.values()) monkeypatch.setattr(theharvester_main, 'ResultStore', _recording_result_store(saved)) - monkeypatch.setattr(theharvester_main.crtsh, 'SearchCrtsh', _ApiHostSource) + monkeypatch.setattr(source_runner.crtsh, 'SearchCrtsh', _ApiHostSource) monkeypatch.setattr(theharvester_main.hostchecker, 'Checker', FakeChecker) monkeypatch.setattr(theharvester_main.shodansearch, 'SearchShodan', CancelledShodan) @@ -3525,7 +4184,7 @@ async def test_direct_action_checkpoint_cancellation_persists_and_propagates( raise asyncio.CancelledError monkeypatch.setattr(theharvester_main, 'ResultStore', _recording_result_store(saved)) - monkeypatch.setattr(theharvester_main.crtsh, 'SearchCrtsh', _ApiHostSource) + monkeypatch.setattr(source_runner.crtsh, 'SearchCrtsh', _ApiHostSource) monkeypatch.setattr(theharvester_main.takeover, 'TakeOver', FakeTakeOver) monkeypatch.setattr(theharvester_main.hostchecker, 'Checker', _ApiHostChecker) monkeypatch.setattr(theharvester_main.shodansearch, 'SearchShodan', FakeShodan) @@ -3622,7 +4281,7 @@ async def test_recursive_dns_results_reach_completed_output_without_changing_leg ) monkeypatch.setattr(theharvester_main, 'ResultStore', FakeResultStore) - monkeypatch.setattr(theharvester_main.crtsh, 'SearchCrtsh', FakeCrtsh) + monkeypatch.setattr(source_runner.crtsh, 'SearchCrtsh', FakeCrtsh) monkeypatch.setattr(theharvester_main.hostchecker, 'Checker', FakeChecker) monkeypatch.setattr(theharvester_main, 'AioDNSResolverVantage', FakeResolver) monkeypatch.setattr(theharvester_main, 'discover_recursive_dns', fake_recursive) @@ -3796,7 +4455,7 @@ async def test_recursive_dns_closes_resolvers_on_failure_and_preserves_cancellat raise error_type() monkeypatch.setattr(theharvester_main, 'ResultStore', FakeResultStore) - monkeypatch.setattr(theharvester_main.crtsh, 'SearchCrtsh', FakeCrtsh) + monkeypatch.setattr(source_runner.crtsh, 'SearchCrtsh', FakeCrtsh) monkeypatch.setattr(theharvester_main.hostchecker, 'Checker', FakeChecker) monkeypatch.setattr(theharvester_main, 'AioDNSResolverVantage', FakeResolver) monkeypatch.setattr(theharvester_main, 'discover_recursive_dns', fail_recursive) diff --git a/tests/test_no_hosts.py b/tests/test_no_hosts.py index c843677e..e8c7ad88 100644 --- a/tests/test_no_hosts.py +++ b/tests/test_no_hosts.py @@ -8,6 +8,7 @@ from typing import TYPE_CHECKING import pytest from theHarvester import __main__ as theharvester_main +from theHarvester.lib import source_runner from theHarvester.lib.completed_result import CompletedResult from theHarvester.lib.enumeration import EnumerationOptions @@ -69,7 +70,7 @@ async def test_no_hosts_keeps_mixed_source_ip_without_retrieving_hostnames( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(theharvester_main, 'ResultStore', NoopResultStore) - monkeypatch.setattr(theharvester_main.bufferoverun, 'SearchBufferover', lambda _word: MixedResultSource()) + monkeypatch.setattr(source_runner.bufferoverun, 'SearchBufferover', lambda _word: MixedResultSource()) response = await theharvester_main.start( EnumerationOptions(domain='example.com', source='bufferoverun', no_hosts=True, quiet=True), @@ -92,7 +93,7 @@ async def test_no_hosts_skips_host_only_source_before_construction(monkeypatch: raise AssertionError('host-only source must not be constructed') monkeypatch.setattr(theharvester_main, 'ResultStore', NoopResultStore) - monkeypatch.setattr(theharvester_main.crtsh, 'SearchCrtsh', HostOnlySource) + monkeypatch.setattr(source_runner.crtsh, 'SearchCrtsh', HostOnlySource) response = await theharvester_main.start( EnumerationOptions(domain='example.com', source='crtsh', no_hosts=True, quiet=True), @@ -114,7 +115,7 @@ async def test_no_hosts_omits_hostname_records_from_every_file_report( ) -> None: output = tmp_path / 'non-host-results' monkeypatch.setattr(theharvester_main, 'ResultStore', NoopResultStore) - monkeypatch.setattr(theharvester_main.bufferoverun, 'SearchBufferover', lambda _word: MixedResultSource()) + monkeypatch.setattr(source_runner.bufferoverun, 'SearchBufferover', lambda _word: MixedResultSource()) monkeypatch.setattr( sys, 'argv', diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index bf99637a..10828e24 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -1,7 +1,6 @@ import argparse import asyncio import hashlib -import inspect import json import logging import os @@ -15,7 +14,7 @@ from contextlib import AsyncExitStack from datetime import UTC, datetime from ipaddress import ip_address, ip_network from pathlib import Path -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING from urllib.parse import urlsplit from uuid import UUID, uuid4 @@ -25,86 +24,29 @@ import ujson from theHarvester.discovery import ( api_endpoints, - apisguru, - arquivo, - baidusearch, - bevigil, - bravesearch, - bufferoverun, - builtwith, - censysearch, - certspottersearch, - commoncrawl, - criminalip, - crtname, - crtsh, - dnsdb, dnssearch, - duckduckgosearch, - dymosearch, - fofa, - fullhuntsearch, - githubcode, - gitlabsearch, - hackertarget, - haveibeenpwned, - hibpverified, - hudsonrocksearch, - huntersearch, - intelxsearch, - leakix, - leaklookup, - mojeek, - netlas, - onyphe, - otxsearch, - pentesttools, - projectdiscovery, - rapiddns, - robtex, - rocketreach, - search_dehashed, - search_dnsdumpster, - searchhunterhow, - securityscorecard, - securitytrailssearch, - sherlockeye, - shodan_internetdb, - shodanct, shodansearch, - sourcegraph, - subdomaincenter, - subdomainfinderc99, takeover, - thc, - tombasearch, - urlscan, - virustotal, - waybackarchive, - whoisxml, - windvane, - yahoosearch, - zoomeyesearch, ) from theHarvester.discovery.constants import MissingKey from theHarvester.lib import hostchecker from theHarvester.lib.active_evidence import ActionExecution, ActiveEvidence, ArtifactReference from theHarvester.lib.asn_attribution import AsnAttributionObservation from theHarvester.lib.completed_result import ( - EXECUTION_STATUSES, CompletedResult, ExecutionStatus, ResultKind, ResultObservation, SourceExecution, ) -from theHarvester.lib.core import DATA_DIR, Core, show_default_error_message +from theHarvester.lib.core import DATA_DIR, Core from theHarvester.lib.database import ResultStore from theHarvester.lib.dns_consensus import AioDNSResolverVantage from theHarvester.lib.enumeration import ( DEFAULT_DNS_RECURSIVE_RUNTIME_SECONDS, DEFAULT_RESULT_LIMIT, DEFAULT_RESULT_START, + DEFAULT_SOURCE_WORKERS, EnumerationOptions, ) from theHarvester.lib.hostnames import normalize_hostname, normalize_scoped_hostname @@ -122,10 +64,11 @@ from theHarvester.lib.source_catalog import ( SOURCE_SPECS, ActivityClass, ResultRoute, - SourceSpec, get_source_spec, hostname_collection_conflicts, + resolve_sources, ) +from theHarvester.lib.source_runner import SourceJob, SourceOutcome, SourceRequest, run_source_jobs from theHarvester.lib.virtual_host import ( DEFAULT_VHOST_CONCURRENCY, DEFAULT_VHOST_REQUEST_LIMIT, @@ -202,7 +145,7 @@ async def start( result_database: str | Path | None = None, completed_run_id: UUID | None = None, ): - """Main program function""" + """Run one CLI or transport-neutral enumeration request.""" parser = argparse.ArgumentParser( description='theHarvester is used to gather open source intelligence (OSINT) on a company or domain.' ) @@ -226,6 +169,13 @@ async def start( default=DEFAULT_RESULT_START, type=int, ) + parser.add_argument( + '-j', + '--source-workers', + help='Maximum discovery sources to run at once (default: %(default)s).', + default=DEFAULT_SOURCE_WORKERS, + type=int, + ) parser.add_argument( '-p', '--proxies', @@ -431,7 +381,7 @@ async def start( # indicates this from the rest API if rest_args: if rest_args.source and rest_args.source == 'getsources': - return list(sorted(Core.get_supportedengines())) + return list(sorted(SOURCE_SPECS)) args = EnumerationOptions.from_namespace(rest_args) filename = args.filename if args.dns_brute: @@ -448,6 +398,8 @@ async def start( configure_logging(verbose=args.verbose) if args.verbose: logger.info('Verbose logging enabled') + if isinstance(args.source_workers, bool) or not isinstance(args.source_workers, int) or args.source_workers <= 0: + raise ValueError('--source-workers must be a positive integer') collect_hosts = not args.no_hosts action_request = { 'no_hosts': args.no_hosts, @@ -602,9 +554,6 @@ async def start( dns_resolution_cancelled = False dns_resolution_stop_reason: str | None = None - def record_missing_credentials(source: str) -> None: - source_executions.append(SourceExecution(source, 'skipped', 0, 0, 'MissingKeyError', 'missing-credentials')) - def confirmed_virtual_hostnames() -> list[str]: return sorted({observation.hostname for observation in vhost_observations}) @@ -797,166 +746,61 @@ async def start( ) ) - async def collect_and_store( - search_engine: Any, - source_spec: SourceSpec, - source_observations: set[ResultObservation], - ) -> None: - """Process a source and persist its declared consolidated result routes. - - :param search_engine: search engine to fetch details from - :param source_spec: canonical source identity and declared result routes - """ - source = source_spec.name - routes = source_spec.routes - if source: - output_logger.info(f'[*] Searching {source[0].upper() + source[1:]}. ') - await search_engine.process(use_proxy) - - def record_source_observations(source_name: str, kind: ResultKind, values: Iterable[object]) -> None: - source_observations.update( - ResultObservation(source_name, kind, value) for item in values if (value := str(item).strip()) - ) - - if collect_hosts and ResultRoute.SUBDOMAINS in routes: - discovered_hosts = await search_engine.get_hostnames() - host_names = list(_normalize_hosts_for_storage(discovered_hosts, word)) - if source == 'rapiddns': - for host, address in await search_engine.get_host_ip_pairs(): - normalized = normalize_scoped_hostname(host, word) - if normalized and normalized in host_names: - reported_host_ip_pairs.add((normalized, address)) - all_hosts.extend(host_names) - record_source_observations(source, 'hostname', host_names) - - if ResultRoute.EMAILS in routes: - email_list = await search_engine.get_emails() - all_emails.extend(email_list) - record_source_observations(source, 'email', email_list) - - if ResultRoute.IPS in routes: - ips_list = await search_engine.get_ips() - all_ip.extend(ips_list) - record_source_observations(source, 'ip', _normalize_ip_addresses(ips_list)) - - if ResultRoute.PEOPLE in routes: - people_list = await search_engine.get_people() - all_people.extend(people_list) - people_evidence = ( - json.dumps(person, ensure_ascii=False, separators=(',', ':'), sort_keys=True) for person in people_list - ) - record_source_observations(source, 'person', people_evidence) - - if ResultRoute.URLS in routes: - urls = await search_engine.get_urls() - all_urls.extend(urls) - record_source_observations(source, 'url', urls) - - if ResultRoute.ASNS in routes: - fasns = await search_engine.get_asns() - total_asns.extend(fasns) - record_source_observations(source, 'asn', fasns) - get_asn_attributions = getattr(search_engine, 'get_asn_attributions', None) - if get_asn_attributions is not None: - asn_attributions.extend( - attribution - for attribution in await get_asn_attributions() - if ResultObservation(source, 'asn', attribution.asn) in source_observations - and ResultObservation(source, attribution.subject_kind, attribution.subject_value) in source_observations + def commit_source_outcome(outcome: SourceOutcome) -> None: + source_executions.append(outcome.execution) + observations.update(outcome.observations) + asn_attributions.extend(outcome.asn_attributions) + record_shodan_host_observations(outcome.shodan_hosts) + reported_host_ip_pairs.update(outcome.reported_host_ip_pairs) + if not args.quiet: + if outcome.execution.stop_reason == 'missing-credentials': + output_logger.info(f'[!] Source {outcome.execution.source} skipped: missing credentials.') + elif outcome.execution.status in {'failed', 'partial'} and outcome.execution.error_type is not None: + output_logger.info( + f'[!] Source {outcome.execution.source} {outcome.execution.status}: {outcome.execution.error_type}.' ) + for observation in outcome.observations: + if observation.kind == 'hostname': + all_hosts.append(observation.value) + elif observation.kind == 'email': + all_emails.append(observation.value) + elif observation.kind == 'ip': + all_ip.append(observation.value) + elif observation.kind == 'asn': + total_asns.append(observation.value) + elif observation.kind == 'breach': + all_breaches.append(observation.value) + elif observation.kind == 'person': + all_people.append(json.loads(observation.value)) + elif observation.kind == 'url': + all_urls.append(observation.value) + elif observation.kind == 'framework': + all_frameworks.append(observation.value) + elif observation.kind == 'language': + all_languages.append(observation.value) + elif observation.kind == 'server': + all_servers.append(observation.value) + elif observation.kind == 'cms': + all_cms.append(observation.value) + elif observation.kind == 'analytics': + all_analytics.append(observation.value) + elif observation.kind == 'infostealer': + all_infostealers.append(json.loads(observation.value)) - if ResultRoute.BREACHES in routes: - breach_names = await search_engine.get_breach_names() - all_breaches.extend(breach_names) - record_source_observations(source, 'breach', breach_names) - if source == 'builtwith': - technology_results: tuple[tuple[str, list[Any], ResultKind], ...] = ( - ('get_frameworks', all_frameworks, 'framework'), - ('get_languages', all_languages, 'language'), - ('get_servers', all_servers, 'server'), - ('get_cms', all_cms, 'cms'), - ('get_analytics', all_analytics, 'analytics'), - ) - for getter_name, results, result_type in technology_results: - values = await getattr(search_engine, getter_name)() - results.extend(values) - record_source_observations(source, result_type, values) - if source == 'hudsonrock': - infostealers = await search_engine.get_infostealers() - all_infostealers.extend(infostealers) - record_source_observations( - source, - 'infostealer', - (json.dumps(stealer, ensure_ascii=False, separators=(',', ':'), sort_keys=True) for stealer in infostealers), - ) - if source == 'shodan': - sourced_shodan_hosts = record_shodan_host_observations(await search_engine.get_shodan_hosts()) - record_source_observations(source, 'shodan-host', (host.ip for host in sourced_shodan_hosts)) - - async def store(search_engine: Any, source: str) -> None: - source_spec = get_source_spec(source) - source_name = source_spec.name - source_observations: set[ResultObservation] = set() - logger.info(f'Source {source_name} started') - started = time.perf_counter() + async def finish_source_outcome(outcome: SourceOutcome) -> None: try: - await collect_and_store(search_engine, source_spec, source_observations) - reported_status = getattr(search_engine, 'execution_status', None) - if reported_status is None: - execution_status: ExecutionStatus = 'completed' - elif isinstance(reported_status, str) and reported_status in EXECUTION_STATUSES: - execution_status = cast('ExecutionStatus', reported_status) - else: - raise ValueError(f'Source {source_name} reported invalid execution status: {reported_status!r}') + await checkpoint_completed_result(committed_sources_only=True) except asyncio.CancelledError: - result_count = len(source_observations) - source_executions.append( - SourceExecution( - source_name, - 'partial' if result_count else 'failed', - (time.perf_counter() - started) * 1000, - result_count, - 'CancelledError', - 'cancelled', - ) - ) - observations.update(source_observations) raise except Exception as error: - result_count = len(source_observations) - duration_ms = (time.perf_counter() - started) * 1000 - logger.exception(f'Source {source_name} failed after {duration_ms / 1000:.2f}s with {result_count} result(s)') - source_executions.append( - SourceExecution( - source_name, - 'partial' if result_count else 'failed', - duration_ms, - result_count, - type(error).__name__, - ) + output_logger.info(f'\n An error occurred while committing {outcome.execution.source}: {type(error).__name__}.\n') + else: + execution = outcome.execution + stop_summary = f'; stop={execution.stop_reason}' if execution.stop_reason is not None else '' + logger.info( + f'Source {execution.source} finished in {execution.duration_ms / 1000:.2f}s: ' + f'status={execution.status}; results={execution.result_count}{stop_summary}' ) - observations.update(source_observations) - await checkpoint_completed_result(committed_sources_only=True) - raise - result_count = len(source_observations) - duration_ms = (time.perf_counter() - started) * 1000 - stop_reason = getattr(search_engine, 'stop_reason', None) - source_executions.append( - SourceExecution( - source_name, - execution_status, - duration_ms, - result_count, - stop_reason=stop_reason if isinstance(stop_reason, str) else None, - ) - ) - observations.update(source_observations) - await checkpoint_completed_result(committed_sources_only=True) - stop_summary = f'; stop={stop_reason}' if isinstance(stop_reason, str) else '' - logger.info( - f'Source {source_name} finished in {duration_ms / 1000:.2f}s: ' - f'status={execution_status}; results={result_count}{stop_summary}' - ) async def resolve_source_hostnames() -> None: nonlocal dns_resolution_cancelled, dns_resolution_completed_count @@ -967,13 +811,10 @@ async def start( full.extend(sorted({item.value for item in hostname_observations})) return - provider_resolved_sources = {'hackertarget', 'pentesttools'} - rapiddns_paired_hosts = {host for host, _address in reported_host_ip_pairs} - full.extend( - sorted( - {item.value for item in hostname_observations if item.source in provider_resolved_sources} | rapiddns_paired_hosts - ) - ) + retained_unresolved_hosts = { + item.value for item in hostname_observations if get_source_spec(item.source).retains_unresolved_hostnames + } + full.extend(sorted(retained_unresolved_hosts)) if dnsresolve is not None and not final_dns_resolver_list: return @@ -981,9 +822,6 @@ async def start( if not host_names: return - rapiddns_candidates = { - item.value for item in hostname_observations if item.source == 'rapiddns' and item.value not in rapiddns_paired_hosts - } dns_resolution_started = time.perf_counter() full_hosts_checker = hostchecker.Checker(host_names, final_dns_resolver_list) @@ -992,7 +830,6 @@ async def start( dns_resolution_ips.update(_normalize_ip_addresses(temp_ips)) all_ip.extend(temp_ips) full.extend(resolved_pair) - full.extend(sorted(rapiddns_candidates - set(resolved_hosts))) resolved_screenshot_hosts.update(resolved_hosts) try: @@ -1022,9 +859,9 @@ async def start( dns_resolution_error_types.update(getattr(full_hosts_checker, 'query_error_types', set())) dns_resolution_stop_reason = getattr(full_hosts_checker, 'stop_reason', None) - stor_lst = [] + source_jobs: list[SourceJob] = [] if args.source is not None: - engines = Core.expand_source_selection(args.source) + engines = resolve_sources(args.source) if not collect_hosts: hostname_only_engines = [ engine @@ -1073,886 +910,33 @@ async def start( output_logger.info(f'[*] Activity: {", ".join(activity_labels[item] for item in ActivityClass if item in activities)}') if args.source is not None: - # Iterate through search engines in order - if set(engines).issubset(Core.get_supportedengines()): - output_logger.info(f'\n[*] Target: {word} \n') - - for engineitem in engines: - if engineitem == 'apis-guru': - try: - apis_guru_search = apisguru.SearchApisGuru(word, limit) - stor_lst.append(store(apis_guru_search, engineitem)) - except Exception as e: - show_default_error_message(engineitem, word, e) - - elif engineitem == 'arquivo': - try: - arquivo_search = arquivo.SearchArquivo(word, limit) - stor_lst.append(store(arquivo_search, engineitem)) - except Exception as e: - show_default_error_message(engineitem, word, e) - - elif engineitem == 'baidu': - try: - baidu_search = baidusearch.SearchBaidu(word, limit) - stor_lst.append( - store( - baidu_search, - engineitem, - ) - ) - except Exception as e: - show_default_error_message(engineitem, word, e) - - elif engineitem == 'bevigil': - try: - bevigil_search = bevigil.SearchBeVigil(word) - stor_lst.append( - store( - bevigil_search, - engineitem, - ) - ) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - show_default_error_message(engineitem, word, error=e) - - elif engineitem == 'brave': - try: - brave_search = bravesearch.SearchBrave(word, limit) - stor_lst.append( - store( - brave_search, - engineitem, - ) - ) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - show_default_error_message(engineitem, word, error=e) - - elif engineitem == 'bufferoverun': - try: - bufferoverun_search = bufferoverun.SearchBufferover(word) - stor_lst.append( - store( - bufferoverun_search, - engineitem, - ) - ) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - show_default_error_message(engineitem, word, e) - - elif engineitem == 'builtwith': - try: - builtwith_search = builtwith.SearchBuiltWith(word) - stor_lst.append(store(builtwith_search, engineitem)) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - output_logger.info(f"Failed to perform BuiltWith search for word: '{word}'") - output_logger.info(f'A Missing Key Error occurred in builtwith: {e}') - else: - show_default_error_message(engineitem, word, e) - - elif engineitem == 'censys': - try: - censys_search = censysearch.SearchCensys(word, limit) - stor_lst.append( - store( - censys_search, - engineitem, - ) - ) - except MissingKey as mk: - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(f'Censys Platform credentials are missing or invalid: {mk}') - except ConnectionError as ce: - if not args.quiet: - output_logger.info(f'Network error while querying Censys: {ce}') - except TimeoutError as te: - if not args.quiet: - output_logger.info(f'Timeout occurred while contacting Censys: {te}') - except ValueError as ve: - if not args.quiet: - output_logger.info(f'Censys returned unexpected data: {ve}') - except Exception as e: - if not args.quiet: - output_logger.info(f'Unexpected error occurred in Censys module: {e}') - - elif engineitem == 'certspotter': - try: - certspotter_search = certspottersearch.SearchCertspoter(word) - stor_lst.append(store(certspotter_search, engineitem)) - except ConnectionError as ce: - if not args.quiet: - output_logger.info(f'Network connection error while accessing Certspotter: {ce}') - except TimeoutError as te: - if not args.quiet: - output_logger.info(f'Request to Certspotter timed out: {te}') - except ValueError as ve: - if not args.quiet: - output_logger.info(f'Certspotter returned invalid data: {ve}') - except MissingKey as mk: - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(f'Unexpected response structure from Certspotter (missing key): {mk}') - except Exception as e: - if not args.quiet: - output_logger.info(f'Unexpected error occurred in Certspotter module: {e}') - - elif engineitem == 'commoncrawl': - try: - commoncrawl_search = commoncrawl.SearchCommoncrawl(word, limit) - stor_lst.append( - store( - commoncrawl_search, - engineitem, - ) - ) - except Exception as e: - show_default_error_message(engineitem, word, e) - - elif engineitem == 'criminalip': - try: - criminalip_search = criminalip.SearchCriminalIP(word) - stor_lst.append( - store( - criminalip_search, - engineitem, - ) - ) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(f'A Missing key error occurred in criminalip: {e}') - else: - show_default_error_message(engineitem, word, e) - - elif engineitem == 'crt-name': - try: - crt_name_search = crtname.SearchCrtName(word) - stor_lst.append(store(crt_name_search, engineitem)) - except Exception as e: - show_default_error_message(engineitem, word, e) - - elif engineitem == 'crtsh': - try: - crtsh_search = crtsh.SearchCrtsh(word) - stor_lst.append(store(crtsh_search, 'CRTsh')) - except Exception as e: - output_logger.info(f'[!] A timeout occurred with crtsh, cannot find {args.domain}\n {e}') - - elif engineitem == 'dehashed': - try: - dehashed_search = search_dehashed.SearchDehashed(word, limit=limit) - stor_lst.append( - store( - dehashed_search, - engineitem, - ) - ) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(f'A Missing Key error occurred in dehashed: {e}') - else: - show_default_error_message(engineitem, word, e) - - elif engineitem == 'dnsdb': - try: - dnsdb_search = dnsdb.SearchDNSDB(word) - stor_lst.append(store(dnsdb_search, engineitem)) - except MissingKey as e: - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(e) - except Exception as e: - show_default_error_message(engineitem, word, e) - - elif engineitem == 'dnsdumpster': - try: - dnsdumpster_search = search_dnsdumpster.SearchDNSDumpster(word) - stor_lst.append( - store( - dnsdumpster_search, - engineitem, - ) - ) - except MissingKey as e: - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(e) - except Exception as e: - show_default_error_message(engineitem, word, e) - - elif engineitem == 'duckduckgo': - duckduckgo_search = duckduckgosearch.SearchDuckDuckGo(word, limit) - stor_lst.append( - store( - duckduckgo_search, - engineitem, - ) - ) - - elif engineitem == 'dymo': - try: - dymo_search = dymosearch.SearchDymo(word) - stor_lst.append(store(dymo_search, engineitem)) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(f'A Missing Key error occurred in dymo: {e}') - else: - show_default_error_message(engineitem, word, e) - - elif engineitem == 'fofa': - try: - fofa_search = fofa.SearchFofa(word) - stor_lst.append( - store( - fofa_search, - engineitem, - ) - ) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(f'A Missing Key error occurred in Fofa: {e}') - else: - show_default_error_message(engineitem, word, e) - - elif engineitem == 'fullhunt': - try: - fullhunt_search = fullhuntsearch.SearchFullHunt(word) - stor_lst.append(store(fullhunt_search, engineitem)) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(f'A Missing Key error occurred in fullhunt: {e}') - - elif engineitem == 'github-code': - try: - github_search = githubcode.SearchGithubCode(word, limit) - stor_lst.append( - store( - github_search, - engineitem, - ) - ) - except MissingKey as ex: - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(f'A Missing Key error occurred in github-code: {ex}') - - elif engineitem == 'gitlab': - try: - gitlab_search = gitlabsearch.SearchGitlab(word) - stor_lst.append( - store( - gitlab_search, - engineitem, - ) - ) - except Exception as e: - show_default_error_message(engineitem, word, e) - - elif engineitem == 'hackertarget': - try: - hackertarget_search = hackertarget.SearchHackerTarget(word) - stor_lst.append(store(hackertarget_search, engineitem)) - except Exception as e: - show_default_error_message(engineitem, word, e) - - elif engineitem == 'haveibeenpwned': - try: - haveibeenpwned_search = haveibeenpwned.SearchHaveIBeenPwned(word) - stor_lst.append( - store( - haveibeenpwned_search, - engineitem, - ) - ) - except Exception as e: - show_default_error_message(engineitem, word, e) - - elif engineitem == 'hibpverified': - try: - hibp_search = hibpverified.SearchHibpVerified(word) - stor_lst.append(store(hibp_search, engineitem)) - except MissingKey as error: - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(f'A Missing Key error occurred in hibpverified: {error}') - except Exception as error: - show_default_error_message(engineitem, word, error) - - elif engineitem == 'hudsonrock': - try: - hudsonrock_search = hudsonrocksearch.SearchHudsonRock(word) - stor_lst.append( - store( - hudsonrock_search, - engineitem, - ) - ) - except Exception as e: - output_logger.info(f'An exception has occurred in Hudson Rock search: {e}') - - elif engineitem == 'hunter': - try: - hunter_search = huntersearch.SearchHunter(word, limit, start) - stor_lst.append( - store( - hunter_search, - engineitem, - ) - ) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(f'A Missing Key error occurred in Hunter: {e}') - - elif engineitem == 'hunterhow': - try: - hunterhow_search = searchhunterhow.SearchHunterHow(word) - stor_lst.append(store(hunterhow_search, engineitem)) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(f'A Missing Key error occurred in Hunter How: {e}') - else: - output_logger.info(f'An exception has occurred in hunterhow search: {e}') - - elif engineitem == 'intelx': - try: - intelx_search = intelxsearch.SearchIntelx(word) - stor_lst.append( - store( - intelx_search, - engineitem, - ) - ) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(f'A Missing Key error occurred in intelx: {e}') - else: - output_logger.info(f'An exception has occurred in Intelx search: {e}') - - elif engineitem == 'leakix': - try: - leakix_search = leakix.SearchLeakix(word) - stor_lst.append( - store( - leakix_search, - engineitem, - ) - ) - except MissingKey as e: - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(e) - except Exception as e: - show_default_error_message(engineitem, word, e) - - elif engineitem == 'leaklookup': - try: - leaklookup_search = leaklookup.SearchLeakLookup(word) - stor_lst.append( - store( - leaklookup_search, - engineitem, - ) - ) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - output_logger.info(f'A Missing Key error occurred in LeakLookup: {e}') - else: - output_logger.info(f'An exception has occurred in LeakLookup search: {e}') - - elif engineitem == 'mojeek': - try: - mojeek_search = mojeek.SearchMojeek(word, limit) - stor_lst.append( - store( - mojeek_search, - engineitem, - ) - ) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - output_logger.info(f'A Missing Key error occurred in Mojeek: {e}') - else: - output_logger.info(f'An exception has occurred in Mojeek search: {e}') - - elif engineitem == 'netlas': - try: - netlas_search = netlas.SearchNetlas(word, limit) - stor_lst.append( - store( - netlas_search, - engineitem, - ) - ) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(f'A Missing Key error occurred in Netlas: {e}') - - elif engineitem == 'onyphe': - try: - onyphe_search = onyphe.SearchOnyphe(word) - stor_lst.append( - store( - onyphe_search, - engineitem, - ) - ) - except ConnectionError as ce: - if not args.quiet: - output_logger.info(f'Network connection error while accessing Onyphe: {ce}') - except TimeoutError as te: - if not args.quiet: - output_logger.info(f'Request to Onyphe timed out: {te}') - except ValueError as ve: - if not args.quiet: - output_logger.info(f'Onyphe returned invalid or unexpected data: {ve}') - except KeyError as ke: - if not args.quiet: - output_logger.info(f'Unexpected response structure from Onyphe (missing key): {ke}') - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(f'Unexpected error occurred in Onyphe module: {e}') - - elif engineitem == 'otx': - try: - otxsearch_search = otxsearch.SearchOtx(word) - stor_lst.append( - store( - otxsearch_search, - engineitem, - ) - ) - except ConnectionError as ce: - if not args.quiet: - output_logger.info(f'Network connection error while accessing OTX: {ce}') - except TimeoutError as te: - if not args.quiet: - output_logger.info(f'Request to OTX timed out: {te}') - except ValueError as ve: - if not args.quiet: - output_logger.info(f'OTX returned invalid or unexpected data: {ve}') - except KeyError as ke: - if not args.quiet: - output_logger.info(f'Unexpected response structure from OTX (missing key): {ke}') - except Exception as e: - if not args.quiet: - output_logger.info(f'Unexpected error occurred in OTX module: {e}') - - elif engineitem == 'pentesttools': - try: - pentesttools_search = pentesttools.SearchPentestTools(word) - stor_lst.append(store(pentesttools_search, engineitem)) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(f'A Missing Key error occurred in PentestTools search: {e}') - else: - output_logger.info(f'An exception has occurred in PentestTools search: {e}') - - elif engineitem == 'projectdiscovery': - try: - projectdiscovery_search = projectdiscovery.SearchDiscovery(word) - stor_lst.append(store(projectdiscovery_search, engineitem)) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(f'A Missing Key error occurred in ProjectDiscovery: {e}') - else: - output_logger.info('An exception has occurred in ProjectDiscovery') - - elif engineitem == 'rapiddns': - try: - rapiddns_search = rapiddns.SearchRapidDns(word) - stor_lst.append(store(rapiddns_search, engineitem)) - except ConnectionError as ce: - if not args.quiet: - output_logger.info(f'Network connection error while accessing RapidDNS: {ce}') - except TimeoutError as te: - if not args.quiet: - output_logger.info(f'Request to RapidDNS timed out: {te}') - except ValueError as ve: - if not args.quiet: - output_logger.info(f'RapidDNS returned invalid or unexpected data: {ve}') - except KeyError as ke: - if not args.quiet: - output_logger.info(f'Unexpected response structure from RapidDNS (missing key): {ke}') - except Exception as e: - if not args.quiet: - output_logger.info(f'Unexpected error occurred in RapidDNS module: {e}') - - elif engineitem == 'robtex': - try: - robtex_search = robtex.SearchRobtex(word) - stor_lst.append( - store( - robtex_search, - engineitem, - ) - ) - except Exception as e: - show_default_error_message(engineitem, word, e) - - elif engineitem == 'rocketreach': - try: - rocketreach_search = rocketreach.SearchRocketReach(word, limit) - stor_lst.append(store(rocketreach_search, engineitem)) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(f'A Missing Key error occurred in RocketReach: {e}') - else: - output_logger.info(f'An exception has occurred in RocketReach: {e}') - - elif engineitem == 'securityscorecard': - try: - securityscorecard_search = securityscorecard.SearchSecurityScorecard(word) - stor_lst.append( - store( - securityscorecard_search, - engineitem, - ) - ) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - output_logger.info(MissingKey('SecurityScorecard')) - else: - output_logger.info(f'An exception has occurred in SecurityScorecard search: {e}') - - elif engineitem == 'securityTrails': - try: - securitytrails_search = securitytrailssearch.SearchSecuritytrail(word) - stor_lst.append( - store( - securitytrails_search, - engineitem, - ) - ) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(f'A Missing Key error occurred Security Trails: {e}') - - elif engineitem == 'sherlockeye': - try: - sherlockeye_search = sherlockeye.SearchSherlockeye(word) - stor_lst.append( - store( - sherlockeye_search, - engineitem, - ) - ) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(f'A Missing Key error occurred in sherlockeye: {e}') - else: - show_default_error_message(engineitem, word, e) - - elif engineitem == 'shodan': - try: - stor_lst.append(store(shodansearch.SearchShodan(word), engineitem)) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(f'A Missing Key error occurred in Shodan search: {e}') - else: - output_logger.info(f'An exception has occurred in Shodan search: {e}') - - elif engineitem == 'shodanInternetDB': - try: - shodanidb_search = shodan_internetdb.SearchShodanInternetDB(word) - stor_lst.append( - store( - shodanidb_search, - engineitem, - ) - ) - except ConnectionError as ce: - if not args.quiet: - output_logger.info(f'Network connection error while accessing Shodan InternetDB: {ce}') - except TimeoutError as te: - if not args.quiet: - output_logger.info(f'Request to Shodan InternetDB timed out: {te}') - except Exception as e: - if not args.quiet: - output_logger.info(f'Unexpected error occurred in Shodan InternetDB module: {e}') - - elif engineitem == 'shodanct': - try: - shodanct_search = shodanct.SearchShodanCt(word) - stor_lst.append(store(shodanct_search, engineitem)) - except Exception as e: - show_default_error_message(engineitem, word, e) - - elif engineitem == 'sourcegraph': - sourcegraph_search = sourcegraph.SearchSourcegraph(word, limit) - stor_lst.append(store(sourcegraph_search, engineitem)) - - elif engineitem == 'subdomaincenter': - try: - subdomaincenter_search = subdomaincenter.SubdomainCenter(word) - stor_lst.append(store(subdomaincenter_search, engineitem)) - except ConnectionError as ce: - if not args.quiet: - output_logger.info(f'Network connection error while accessing SubdomainCenter: {ce}') - except TimeoutError as te: - if not args.quiet: - output_logger.info(f'Request to SubdomainCenter timed out: {te}') - except ValueError as ve: - if not args.quiet: - output_logger.info(f'SubdomainCenter returned invalid or unexpected data: {ve}') - except KeyError as ke: - if not args.quiet: - output_logger.info(f'Unexpected response structure from SubdomainCenter (missing key): {ke}') - except Exception as e: - if not args.quiet: - output_logger.info(f'Unexpected error occurred in SubdomainCenter module: {e}') - - elif engineitem == 'subdomainfinderc99': - try: - subdomainfinderc99_search = subdomainfinderc99.SearchSubdomainfinderc99(word) - stor_lst.append(store(subdomainfinderc99_search, engineitem)) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(f'A Missing Key error occurred in Subdomainfinderc99 search: {e}') - else: - output_logger.info(f'An exception has occurred in Subdomainfinderc99 search: {e}') - - elif engineitem == 'thc': - try: - thc_search = thc.SearchThc(word) - stor_lst.append(store(thc_search, engineitem)) - except Exception as e: - show_default_error_message(engineitem, word, e) - - elif engineitem == 'tomba': - try: - tomba_search = tombasearch.SearchTomba(word, limit, start) - stor_lst.append( - store( - tomba_search, - engineitem, - ) - ) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(f'A Missing Key error occurred in Tomba: {e}') - - elif engineitem == 'urlscan': - try: - urlscan_search = urlscan.SearchUrlscan(word) - stor_lst.append( - store( - urlscan_search, - engineitem, - ) - ) - except ConnectionError as ce: - if not args.quiet: - output_logger.info(f'Network connection error while accessing Urlscan: {ce}') - except TimeoutError as te: - if not args.quiet: - output_logger.info(f'Request to Urlscan timed out: {te}') - except ValueError as ve: - if not args.quiet: - output_logger.info(f'Urlscan returned invalid or unexpected data: {ve}') - except KeyError as ke: - if not args.quiet: - output_logger.info(f'Unexpected response structure from Urlscan (missing key): {ke}') - except Exception as e: - if not args.quiet: - output_logger.info(f'Unexpected error occurred in Urlscan module: {e}') - - elif engineitem == 'virustotal': - try: - virustotal_search = virustotal.SearchVirustotal(word) - stor_lst.append(store(virustotal_search, engineitem)) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(f'A Missing Key error occurred in virustotal search: {e}') - - elif engineitem == 'waybackarchive': - try: - waybackarchive_search = waybackarchive.SearchWaybackarchive(word, limit) - stor_lst.append( - store( - waybackarchive_search, - engineitem, - ) - ) - except Exception as e: - show_default_error_message(engineitem, word, e) - - elif engineitem == 'whoisxml': - try: - whoisxml_search = whoisxml.SearchWhoisXML(word) - stor_lst.append(store(whoisxml_search, engineitem)) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(f'A Missing Key error occurred in whoisxml search: {e}') - else: - output_logger.info(f'An exception has occurred in WhoisXML search: {e}') - - elif engineitem == 'windvane': - try: - windvane_search = windvane.SearchWindvane(word) - stor_lst.append( - store( - windvane_search, - engineitem, - ) - ) - except Exception as e: - show_default_error_message(engineitem, word, e) - - elif engineitem == 'yahoo': - try: - yahoo_search = yahoosearch.SearchYahoo(word, limit) - stor_lst.append( - store( - yahoo_search, - engineitem, - ) - ) - except ConnectionError as ce: - if not args.quiet: - output_logger.info(f'Network connection error while accessing Yahoo: {ce}') - except TimeoutError as te: - if not args.quiet: - output_logger.info(f'Request to Yahoo timed out: {te}') - except ValueError as ve: - if not args.quiet: - output_logger.info(f'Yahoo returned invalid or unexpected data: {ve}') - except KeyError as ke: - if not args.quiet: - output_logger.info(f'Unexpected response structure from Yahoo (missing key): {ke}') - except Exception as e: - if not args.quiet: - output_logger.info(f'Unexpected error occurred in Yahoo module: {e}') - - elif engineitem == 'zoomeye': - try: - zoomeye_search = zoomeyesearch.SearchZoomEye(word, limit) - stor_lst.append( - store( - zoomeye_search, - engineitem, - ) - ) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(f'A Missing Key error occurred in zoomeye: {e}') - - elif rest_args is not None: - try: - rest_args.dns_brute - except AttributeError: - output_logger.info('\n[!] Invalid source.\n') - sys.exit(1) - else: - # Print which engines aren't supported - unsupported_engines = set(engines) - set(Core.get_supportedengines()) - if unsupported_engines: - output_logger.info(f'The following engines are not supported: {unsupported_engines}') + unsupported_engines = set(engines) - set(SOURCE_SPECS) + if unsupported_engines: + output_logger.info(f'The following engines are not supported: {unsupported_engines}') output_logger.info('\n[!] Invalid source.\n') sys.exit(1) + output_logger.info(f'\n[*] Target: {word} \n') + source_jobs.extend(SourceJob(SourceRequest(engine, word, limit, start, use_proxy, collect_hosts)) for engine in engines) - async def worker(queue): - while True: - # Get a "work item" out of the queue. - stor = await queue.get() - try: - await stor - except Exception as work_item_error: - output_logger.info( - f'\n An error occurred while processing a "work item": {type(work_item_error).__name__}: {work_item_error}\n' - ) - finally: - # Notify the queue that the "work item" has been processed. - queue.task_done() + async def handler(jobs: list[SourceJob]) -> tuple[SourceOutcome, ...]: + def report_source_started(request: SourceRequest) -> None: + source = request.source + output_logger.info(f'[*] Searching {source[0].upper() + source[1:]}. ') - async def handler(lst): - queue: asyncio.Queue[Awaitable[Any]] = asyncio.Queue() - for stor_method in lst: - # enqueue the coroutines - queue.put_nowait(stor_method) - # Create three worker tasks to process the queue concurrently. - tasks = [] - for _i in range(3): - task = asyncio.create_task(worker(queue)) - tasks.append(task) - - join_task = asyncio.create_task(queue.join()) - try: - done, _pending = await asyncio.wait((join_task, *tasks), return_when=asyncio.FIRST_COMPLETED) - finished_workers = [task for task in tasks if task in done] - if any(task.cancelled() for task in finished_workers): - raise asyncio.CancelledError - for task in finished_workers: - if error := task.exception(): - raise error - if finished_workers: - raise RuntimeError('A source worker stopped before the queue was drained') - await join_task - finally: - join_task.cancel() - for task in tasks: - task.cancel() - await asyncio.gather(join_task, *tasks, return_exceptions=True) - while not queue.empty(): - pending_work = queue.get_nowait() - if inspect.iscoroutine(pending_work): - pending_work.close() - queue.task_done() + if jobs: + output_logger.info( + f'[*] Source workers: requested={args.source_workers}; effective={min(args.source_workers, len(jobs))}.' + ) + return await run_source_jobs( + tuple(jobs), + workers=args.source_workers, + commit=commit_source_outcome, + after_commit=finish_source_outcome, + on_started=report_source_started, + ) try: - await handler(lst=stor_lst) + await handler(source_jobs) await resolve_source_hostnames() except asyncio.CancelledError: record_dns_resolution_execution(handler_cancelled=True) @@ -1960,12 +944,6 @@ async def start( await persist_result(finish_completed_result(committed_sources_only=True)) raise - recorded_sources = {result.source.casefold() for result in source_executions} - source_executions.extend( - SourceExecution(engine, 'skipped', 0, 0, 'SourceDidNotStart') - for engine in engines - if engine.casefold() not in recorded_sources - ) record_dns_resolution_execution() await checkpoint_completed_result() diff --git a/theHarvester/lib/api/harvestview.py b/theHarvester/lib/api/harvestview.py index da119ede..3b9ad415 100644 --- a/theHarvester/lib/api/harvestview.py +++ b/theHarvester/lib/api/harvestview.py @@ -9,6 +9,7 @@ from fastapi.responses import HTMLResponse from theHarvester import __version__ from theHarvester.lib.api.auth import API_KEY_COOKIE_NAME, _configured_api_key, browser_session_token +from theHarvester.lib.enumeration import DEFAULT_SOURCE_WORKERS from theHarvester.lib.resolver_selection import DEFAULT_DNS_RESOLVERS router = APIRouter() @@ -50,6 +51,7 @@ async def harvestview_app(request: Request) -> HTMLResponse: template.replace('{{VERSION}}', __version__) .replace('{{ASSET_VERSION}}', str(asset_version)) .replace('{{DNS_RESOLVERS}}', ','.join(DEFAULT_DNS_RESOLVERS)) + .replace('{{SOURCE_WORKERS}}', str(DEFAULT_SOURCE_WORKERS)) ) if configured_api_key := _configured_api_key(): response.set_cookie( diff --git a/theHarvester/lib/api/run_models.py b/theHarvester/lib/api/run_models.py index a9105e03..9b8b135c 100644 --- a/theHarvester/lib/api/run_models.py +++ b/theHarvester/lib/api/run_models.py @@ -11,6 +11,7 @@ from theHarvester.lib.enumeration import ( DEFAULT_DNS_RECURSIVE_QUERY_LIMIT, DEFAULT_DNS_RECURSIVE_RUNTIME_SECONDS, DEFAULT_RESULT_START, + DEFAULT_SOURCE_WORKERS, ) from theHarvester.lib.evidence_types import EvidenceStatus # noqa: TC001 - Pydantic resolves this annotation at runtime from theHarvester.lib.resolver_selection import DEFAULT_DNS_RESOLVERS, normalize_resolver_addresses @@ -88,6 +89,12 @@ class RunRequest(BaseModel): ge=0, description='Starting result offset for providers that support pagination.', ) + source_workers: int = Field( + default=DEFAULT_SOURCE_WORKERS, + strict=True, + ge=1, + description='Maximum discovery sources run concurrently. Every selected source still runs.', + ) deadline_seconds: int | None = Field( default=None, ge=30, diff --git a/theHarvester/lib/api/run_worker.py b/theHarvester/lib/api/run_worker.py index 0dbcf965..42361e59 100644 --- a/theHarvester/lib/api/run_worker.py +++ b/theHarvester/lib/api/run_worker.py @@ -15,6 +15,7 @@ import anyio from theHarvester.lib.enumeration import ( DEFAULT_RESULT_START, + DEFAULT_SOURCE_WORKERS, EnumerationOptions, ) from theHarvester.lib.resolver_selection import DEFAULT_DNS_RESOLVERS @@ -320,6 +321,7 @@ async def _child_execute(run_id: str, database: Path) -> None: screenshot=str(screenshot_dir) if request.get('screenshot') else '', shodan=request.get('shodan', False), source=','.join(request['sources']), + source_workers=request.get('source_workers', DEFAULT_SOURCE_WORKERS), start=request.get('start', DEFAULT_RESULT_START), take_over=request.get('takeover', False), vhost=request.get('vhost', False), diff --git a/theHarvester/lib/api/static/harvestview/app.js b/theHarvester/lib/api/static/harvestview/app.js index 0b3ca721..0b3dc135 100644 --- a/theHarvester/lib/api/static/harvestview/app.js +++ b/theHarvester/lib/api/static/harvestview/app.js @@ -290,6 +290,7 @@ const options = [ ['Sources', sources], ['Result limit', request.limit ?? 'Imported evidence'], ['Result start offset', request.start ?? 'Not recorded'], + ['Discovery source workers', request.source_workers ?? 'Not recorded'], ['Whole-run deadline', request.deadline_seconds === null ? 'Unlimited' : request.deadline_seconds === undefined ? 'Not recorded' : `${request.deadline_seconds} seconds`], ['Proxy transport', request.proxies ? 'Selected' : 'Off'], ['Hostname results', request.no_hosts ? 'Excluded' : 'Included'], @@ -930,6 +931,7 @@ const payload = { target: form.get('target'), sources: [...state.selectedSources], limit: Number(form.get('limit')), start: Number(form.get('start')), deadline_seconds: form.get('deadline_seconds') ? Number(form.get('deadline_seconds')) : null, + source_workers: Number(form.get('source_workers')), proxies: form.has('proxies'), no_hosts: form.has('no_hosts'), dns_lookup: form.has('dns_lookup'), dns_resolve: form.has('dns_resolve'), dns_resolvers: String(form.get('dns_resolvers')).split(',').map(value => value.trim()), diff --git a/theHarvester/lib/api/static/harvestview/index.html b/theHarvester/lib/api/static/harvestview/index.html index 9caa54b1..ed816719 100644 --- a/theHarvester/lib/api/static/harvestview/index.html +++ b/theHarvester/lib/api/static/harvestview/index.html @@ -209,6 +209,10 @@ Skip this many provider results before collection. +