From 8ead90f7a98c080e2157bf6a8dcce71a95e8a757 Mon Sep 17 00:00:00 2001 From: NotoriousRebel <36310667+NotoriousRebel@users.noreply.github.com> Date: Wed, 26 Aug 2026 22:53:21 -0400 Subject: [PATCH] fix: scope proxy mode to HTTP transport --- CHANGELOG.md | 4 +- CONTEXT.md | 6 +- ...when-explicit-proxy-mode-is-unavailable.md | 2 +- ...0009-scope-proxy-mode-to-http-transport.md | 5 ++ docs/wiki/Configuration-and-API-Keys.md | 9 +-- tests/discovery/test_intelxsearch.py | 1 - tests/discovery/test_rapiddns.py | 2 - tests/e2e/test_harvestview.py | 4 +- tests/lib/test_api_v1.py | 15 +++-- tests/lib/test_enumeration.py | 5 ++ tests/lib/test_source_runner.py | 30 ++++----- tests/test_main.py | 61 ++++++++++--------- theHarvester/__main__.py | 22 ++----- theHarvester/lib/api/run_models.py | 15 +---- theHarvester/lib/api/run_worker.py | 1 - .../lib/api/static/harvestview/app.js | 7 ++- .../lib/api/static/harvestview/index.html | 2 +- theHarvester/lib/enumeration.py | 2 - theHarvester/lib/source_catalog.py | 9 --- theHarvester/lib/source_runner.py | 12 +--- 20 files changed, 94 insertions(+), 120 deletions(-) create mode 100644 docs/adr/0009-scope-proxy-mode-to-http-transport.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 92e9bd3a..7102c9e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added root contributor and security policies, structured issue forms, repository agent guidance, discovery terminology, and an operator-focused documentation wiki ([d090a29a](https://github.com/laramies/theHarvester/commit/d090a29a), [7c491ef5](https://github.com/laramies/theHarvester/commit/7c491ef5), [8b9d420b](https://github.com/laramies/theHarvester/commit/8b9d420b)). ### Changed +- Scoped `--proxies` to supported HTTP(S) requests while allowing DNS queries through operator-selected recursive resolvers in the same run. - Raised the minimum supported Python version to 3.14, selected it for source checkouts, and limited release validation to that runtime. - Replaced the maintenance-only UltraJSON dependency with Python's standard `json` module for provider payloads and legacy reports, removing the native runtime extension without changing JSONL or API serialization. - Centralized passive-source completion reporting in an immutable `SourceExecutionReport` returned by adapters, with the source runner alone deriving partial and no-result outcomes from retained evidence; removed mutable per-adapter `execution_status` and `stop_reason` state and made the runner reject adapters that still expose either field. @@ -77,6 +78,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Expanded offline regression coverage for discovery providers, configuration contracts, logging, output, documentation, workflow policy, and scope boundaries. ### Removed +- Removed the unused legacy `-e`/`--dns-server` CLI option and internal field; use `--dns-resolvers` to select resolver addresses. - Removed the unused legacy SecurityTrails parser that stripped leading `www.` labels outside the shared hostname-scope boundary. - Removed the mutable runtime takeover fingerprint download and silent handwritten fallback rules. - Removed the inert legacy source identifiers `linkedin`, `netcraft`, `omnisint`, `sublist3r`, and `zoomeyeapi`; use the source catalog and shared factory registry for supported providers. @@ -87,7 +89,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Removed the nonfunctional ThreatCrowd source because its service hostnames terminate at deleted AWS load balancers and return NXDOMAIN; OTX remains available through its separate adapter. ### Fixed -- Made explicit proxy mode fail closed with a sanitized `proxy-unavailable` source outcome, and kept one proxy identity and session across the CriminalIP scan, poll, and report conversation. +- Made explicit HTTP proxy mode fail closed with a sanitized `proxy-unavailable` source outcome, and kept one proxy identity and session across the CriminalIP scan, poll, and report conversation. - Reused one connection pool, proxy identity, and cookie jar across Censys and GitHub Code pagination while keeping provider sessions isolated and cancellation-safe. - Sent a stable, versioned theHarvester identity with provider and API requests while preserving explicit browser identities for sources that require them. - Kept API endpoint scan URLs canonical instead of prefixing targets onto already complete URLs. diff --git a/CONTEXT.md b/CONTEXT.md index d172e0f3..9f682849 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -34,7 +34,7 @@ When a change alters one of these boundaries, update this document and the neare - A source adapter returns `None` for ordinary completion or an immutable `SourceExecutionReport` when it must preserve an explicit outcome or stop reason. The central source runner owns observation collection, result counting, no-result classification, exception handling, and final source status. - A result limit of zero removes the shared local cap on results and pages. Adapters continue until the provider is exhausted, but source-owned quotas, protocol maxima, response limits, and runtime safety bounds still apply. A source that stops at one of those boundaries must report an explicit partial outcome and stop reason. - Source execution statuses are `completed`, `partial`, `failed`, `rate-limited`, and `skipped`. Mutable adapter fields such as `execution_status` and `stop_reason` are outside this release contract and are rejected, including when an adapter raises or is cancelled. -- Explicit proxy mode is fail-closed for every supported discovery source and action. If no configured proxy is available, execution makes no direct request and terminates with the sanitized `proxy-unavailable` reason. Sources and actions that require direct DNS are rejected before result persistence; a configured proxy endpoint failure is recorded as `transport-error`. +- HTTP proxy mode is fail-closed for supported HTTP(S) requests. If no configured proxy is available, execution makes no HTTP(S) request and terminates with the sanitized `proxy-unavailable` reason. DNS queries use the operator-selected recursive resolver vantages independently and may coexist with proxied HTTP(S); a configured proxy endpoint failure is recorded as `transport-error`. - Run lifecycle statuses are `queued`, `running`, `cancelling`, `cancelled`, `completed`, and `failed`. Terminal evidence status is independently `complete`, `partial`, or `failed`; retained evidence survives a later cancellation or process failure. - Run schedules support one-time, hourly, daily, weekly, and monthly recurrence. Daily, weekly, and monthly occurrences preserve the selected local wall-clock time; a monthly day that does not exist falls on that month’s final day. - Imported runs enter as completed run records and execute no source or action. Action-only runs create independent run records instead of mutating the evidence of the run that supplied their candidate. @@ -239,6 +239,10 @@ _Avoid_: Raw result, cleaned response The original unprocessed response returned by a discovery provider, which may contain unused, sensitive, or redistribution-restricted fields. _Avoid_: Evidence record, JSONL result +**HTTP proxy mode**: +An enumeration policy that requires supported HTTP(S) provider and target requests to use a configured proxy while DNS queries independently use the operator-selected recursive resolver vantages. +_Avoid_: Fully proxied run, anonymous mode + **P0 passive collection**: An activity that queries an existing provider or dataset without directing traffic toward the target. _Avoid_: Passive scan diff --git a/docs/adr/0008-fail-closed-when-explicit-proxy-mode-is-unavailable.md b/docs/adr/0008-fail-closed-when-explicit-proxy-mode-is-unavailable.md index bf329771..dfb52956 100644 --- a/docs/adr/0008-fail-closed-when-explicit-proxy-mode-is-unavailable.md +++ b/docs/adr/0008-fail-closed-when-explicit-proxy-mode-is-unavailable.md @@ -1,6 +1,6 @@ # Fail closed when explicit proxy mode is unavailable -Status: accepted +Status: superseded by ADR-0009 ## Decision diff --git a/docs/adr/0009-scope-proxy-mode-to-http-transport.md b/docs/adr/0009-scope-proxy-mode-to-http-transport.md new file mode 100644 index 00000000..2c39c762 --- /dev/null +++ b/docs/adr/0009-scope-proxy-mode-to-http-transport.md @@ -0,0 +1,5 @@ +# Scope proxy mode to HTTP transport + +Status: accepted + +`--proxies` requires supported HTTP(S) provider and target requests to use one configured proxy identity and fail closed rather than fall back to direct HTTP(S). DNS queries remain independent resolver traffic: they use the operator-selected recursive resolver addresses and may coexist with proxied HTTP(S) in the same run. This accepts that the resolver and local network can observe DNS traffic, avoids implying that proxy mode provides anonymity, and keeps HTTP-only actions that cannot honor the proxy requirement unavailable in proxy mode. diff --git a/docs/wiki/Configuration-and-API-Keys.md b/docs/wiki/Configuration-and-API-Keys.md index 52a6ac60..8ecf7e6f 100644 --- a/docs/wiki/Configuration-and-API-Keys.md +++ b/docs/wiki/Configuration-and-API-Keys.md @@ -70,10 +70,11 @@ Network activity: provider-facing passive lookup through a configured proxy. uv run theHarvester -d example.com -b crtsh -p ``` -A proxy does not make an assessment anonymous and does not change the authorization boundary. When proxy mode is -enabled, every supported discovery source and action fails closed with `proxy-unavailable` instead of making a direct -request if no configured proxy is available. Sources and actions that require direct DNS are rejected before result -persistence; disable proxy mode to run them. A configured proxy transport failure is recorded as `transport-error`. +A proxy does not make an assessment anonymous and does not change the authorization boundary. `--proxies` applies to +supported HTTP(S) provider and target requests, which fail closed with `proxy-unavailable` instead of falling back to +direct HTTP(S) when no configured proxy is available. DNS queries independently use the operator-selected recursive +resolver addresses and may run alongside proxied HTTP(S); the resolver and local network can therefore observe that +DNS traffic. A configured proxy transport failure is recorded as `transport-error`. ## API protection diff --git a/tests/discovery/test_intelxsearch.py b/tests/discovery/test_intelxsearch.py index 2d70fc7c..e78933be 100644 --- a/tests/discovery/test_intelxsearch.py +++ b/tests/discovery/test_intelxsearch.py @@ -227,7 +227,6 @@ async def test_orchestrator_stores_intelx_subdomains_without_dns(monkeypatch: py filename='', quiet=True, dns_lookup=False, - dns_server=None, dns_resolve='', limit=500, shodan=False, diff --git a/tests/discovery/test_rapiddns.py b/tests/discovery/test_rapiddns.py index c7239663..d14f3c01 100644 --- a/tests/discovery/test_rapiddns.py +++ b/tests/discovery/test_rapiddns.py @@ -316,7 +316,6 @@ async def test_rapiddns_evidence_reaches_existing_outputs( filename='', quiet=True, dns_lookup=False, - dns_server=None, dns_resolve='', limit=500, shodan=False, @@ -334,7 +333,6 @@ async def test_rapiddns_evidence_reaches_existing_outputs( filename='', quiet=True, dns_lookup=False, - dns_server=None, dns_resolve='', limit=500, shodan=False, diff --git a/tests/e2e/test_harvestview.py b/tests/e2e/test_harvestview.py index 2a2d0b71..5899060c 100644 --- a/tests/e2e/test_harvestview.py +++ b/tests/e2e/test_harvestview.py @@ -1399,9 +1399,9 @@ def test_unchanged_poll_keeps_result_table_filters(harvestview_server_url: str, page.route(f'{harvestview_server_url}/api/v1/runs', lambda route: route.fulfill(json=[run])) page.route(f'{harvestview_server_url}/api/v1/runs/stable-run', lambda route: route.fulfill(json=run)) page.goto(f'{harvestview_server_url}/') - expect(page.locator('#request-options')).to_contain_text('Proxy transportSelected') + expect(page.locator('#request-options')).to_contain_text('HTTP(S) proxy transportSelected') expect(page.locator('#request-options')).to_contain_text('DNS lookup (/24 reverse expansion)Selected') - expect(page.locator('#request-options')).to_contain_text('Takeover transportConfigured proxy') + expect(page.locator('#request-options')).to_contain_text('Takeover transportDNS resolvers + HTTP proxy') value_filter = page.locator('.tabulator-col[tabulator-field="value"] .tabulator-header-filter input') value_filter.fill('api') diff --git a/tests/lib/test_api_v1.py b/tests/lib/test_api_v1.py index c46a5cef..1c0addfb 100644 --- a/tests/lib/test_api_v1.py +++ b/tests/lib/test_api_v1.py @@ -442,10 +442,10 @@ def test_openapi_explains_scope_and_execution_controls(tmp_path, monkeypatch) -> ) assert 'prefix' not in properties['routeviews']['description'] assert 'three resolver' in properties['dns_recursive_query_limit']['description'] - assert 'discovery sources' in properties['proxies']['description'] - assert 'Direct DNS' in properties['proxies']['description'] + assert 'HTTP(S) requests' in properties['proxies']['description'] + assert 'selected resolvers outside this proxy' in properties['proxies']['description'] assert 'Exclude hostname results' in properties['no_hosts']['description'] - assert 'direct transport' in properties['takeover']['description'] + assert 'HTTP confirmation requests use the configured proxy when enabled' in properties['takeover']['description'] assert 'take_over' not in properties assert 'endpoint paths' in properties['api_scan_paths']['description'] import_content = schema['paths']['/api/v1/runs/import']['post']['requestBody']['content'] @@ -597,13 +597,12 @@ def test_fresh_api_uses_catalog_takeover_name_and_rejects_unknown_fields() -> No {'sources': [], 'dns_lookup': True}, ], ) -def test_api_rejects_direct_dns_work_in_proxy_mode(request_fields: dict[str, object]) -> None: - from pydantic import ValidationError - +def test_api_allows_dns_work_in_http_proxy_mode(request_fields: dict[str, object]) -> None: from theHarvester.lib.api.run_models import RunRequest - with pytest.raises(ValidationError, match='Direct DNS'): - RunRequest(target='example.test', proxies=True, **request_fields) + request = RunRequest(target='example.test', proxies=True, **request_fields) + + assert request.proxies is True def test_api_rejects_screenshot_capture_in_proxy_mode() -> None: diff --git a/tests/lib/test_enumeration.py b/tests/lib/test_enumeration.py index c1957c96..1bc22dfd 100644 --- a/tests/lib/test_enumeration.py +++ b/tests/lib/test_enumeration.py @@ -1,4 +1,5 @@ from argparse import Namespace +from dataclasses import fields import pytest @@ -23,6 +24,10 @@ def test_enumeration_options_fill_the_shared_execution_defaults() -> None: assert options.source_workers == DEFAULT_SOURCE_WORKERS == 3 +def test_enumeration_options_omit_removed_dns_server_field() -> None: + assert 'dns_server' not in {field.name for field in fields(EnumerationOptions)} + + def test_enumeration_options_preserve_explicit_transport_values() -> None: options = EnumerationOptions.from_namespace( Namespace( diff --git a/tests/lib/test_source_runner.py b/tests/lib/test_source_runner.py index f9b1b915..e2602412 100644 --- a/tests/lib/test_source_runner.py +++ b/tests/lib/test_source_runner.py @@ -412,26 +412,26 @@ async def test_runner_reports_unavailable_required_proxy_without_starting_source @pytest.mark.asyncio -@pytest.mark.parametrize('source', ['shodan', 'shodanInternetDB']) -async def test_runner_rejects_direct_dns_source_before_starting_adapter( - monkeypatch: pytest.MonkeyPatch, - source: str, -) -> None: - adapter_created = False +async def test_runner_allows_dns_source_http_requests_to_use_proxy(monkeypatch: pytest.MonkeyPatch) -> None: + received: list[str | bool] = [] - def create_adapter(_request: SourceRequest) -> object: - nonlocal adapter_created - adapter_created = True - return object() + class ProxiedDnsSource: + async def process(self, proxy: str | bool) -> None: + received.append(proxy) - monkeypatch.setitem(SOURCE_FACTORIES, source, create_adapter) + async def get_hostnames(self) -> tuple[()]: + return () + + async def get_ips(self) -> tuple[()]: + return () + + monkeypatch.setitem(SOURCE_FACTORIES, 'shodanInternetDB', lambda _request: ProxiedDnsSource()) monkeypatch.setattr(AsyncFetcher, '_proxy_list', {'http': ['http://proxy.example:8080'], 'socks5': []}) - outcome = await run_source(SourceRequest(source, 'example.test', 25, 0, True, True)) + outcome = await run_source(SourceRequest('shodanInternetDB', 'example.test', 25, 0, True, True)) - assert adapter_created is False - assert outcome.execution.status == 'failed' - assert outcome.execution.stop_reason == 'direct-transport-only' + assert received == [True] + assert outcome.execution.status == 'completed' assert outcome.observations == () diff --git a/tests/test_main.py b/tests/test_main.py index 20687614..e1561aa3 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -44,16 +44,16 @@ async def test_cli_help_explains_proxy_and_direct_action_scope( help_text = ' '.join(capsys.readouterr().out.split()) assert exit_info.value.code == 0 - assert 'Use proxies.yaml for supported discovery sources and actions.' in help_text + assert 'Use proxies.yaml for supported HTTP(S) requests.' in help_text assert 'unavailable if no proxy is configured.' in help_text - assert 'Direct DNS sources and actions are rejected.' in help_text + assert 'DNS queries use the selected resolvers outside this proxy.' in help_text assert 'Query the Shodan Host API for discovered IPs, using configured proxies when enabled.' in help_text assert ( 'Enrich discovered IPs with sourced ASN attribution, or an explicitly targeted ASN, IP, or prefix, through ' 'RouteViews.' in help_text ) assert 'Exclude hostname results while retaining other result types returned by selected sources.' in help_text - assert 'Accepted for compatibility but currently unused; use --dns-resolvers to select resolvers.' in help_text + assert '--dns-server' not in help_text assert 'Select resolver IPs for DNS actions without enabling hostname resolution.' in help_text assert 'text file with one IP per line' in help_text assert 'Perform PTR lookups across the /24 network containing each discovered IPv4 address.' in help_text @@ -68,10 +68,24 @@ async def test_cli_help_explains_proxy_and_direct_action_scope( assert '-j SOURCE_WORKERS' in help_text assert '--source-workers SOURCE_WORKERS' in help_text assert '0 continues to provider exhaustion with no local result or page-count cap' in help_text - assert 'DNS requires direct transport, so proxy mode rejects this action before execution.' in help_text + assert 'HTTP confirmation requests use the configured proxy when enabled.' in help_text assert 'Indicators are not confirmed takeovers.' in help_text +@pytest.mark.asyncio +@pytest.mark.parametrize('option', ['-e', '--dns-server']) +async def test_cli_rejects_removed_dns_server_flag( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], option: str +) -> None: + monkeypatch.setattr(sys, 'argv', ['theHarvester', '-d', 'example.com', option, '192.0.2.53']) + + with pytest.raises(SystemExit) as exit_info: + await theharvester_main.start() + + assert exit_info.value.code == 2 + assert f'unrecognized arguments: {option} 192.0.2.53' in capsys.readouterr().err + + def _confirmed_vhost(endpoint: str = 'http://192.0.2.10:80/') -> VirtualHostObservation: return VirtualHostObservation( endpoint=endpoint, @@ -232,42 +246,31 @@ async def test_proxy_mode_rejects_direct_screenshot_action_before_result_store_i ) -@pytest.mark.asyncio -@pytest.mark.parametrize('source', ['shodan', 'shodanInternetDB']) -async def test_proxy_mode_rejects_direct_dns_source_before_result_store_initialization( - monkeypatch: pytest.MonkeyPatch, - source: str, -) -> None: - class UnexpectedResultStore: - def __init__(self, *_args: object) -> None: - raise AssertionError('result store initialization must follow proxy validation') - - monkeypatch.setattr(theharvester_main, 'ResultStore', UnexpectedResultStore) - - with pytest.raises(ValueError, match=f'Direct DNS sources cannot use --proxies: {source}'): - await theharvester_main.start(EnumerationOptions(domain='example.com', proxies=True, quiet=True, source=source)) - - @pytest.mark.asyncio @pytest.mark.parametrize( - ('options', 'action'), + 'options', [ - (EnumerationOptions(domain='example.com', proxies=True, quiet=True, take_over=True), 'takeover'), - (EnumerationOptions(domain='example.com', proxies=True, quiet=True, dns_lookup=True), 'dns-lookup'), + EnumerationOptions(domain='example.com', proxies=True, quiet=True, source='shodan'), + EnumerationOptions(domain='example.com', proxies=True, quiet=True, source='shodanInternetDB'), + EnumerationOptions(domain='example.com', proxies=True, quiet=True, take_over=True), + EnumerationOptions(domain='example.com', proxies=True, quiet=True, dns_lookup=True), ], ) -async def test_proxy_mode_rejects_direct_dns_action_before_result_store_initialization( +async def test_proxy_mode_allows_dns_work_to_reach_result_store_initialization( monkeypatch: pytest.MonkeyPatch, options: EnumerationOptions, - action: str, ) -> None: - class UnexpectedResultStore: + class ResultStoreReached(Exception): + pass + + class ExpectedResultStore: def __init__(self, *_args: object) -> None: - raise AssertionError('result store initialization must follow proxy validation') + raise ResultStoreReached - monkeypatch.setattr(theharvester_main, 'ResultStore', UnexpectedResultStore) + monkeypatch.setattr(theharvester_main, 'ResultStore', ExpectedResultStore) + monkeypatch.setattr(AsyncFetcher, '_proxy_list', {'http': ['http://proxy.example:8080'], 'socks5': []}) - with pytest.raises(ValueError, match=f'Direct DNS actions cannot use --proxies: {action}'): + with pytest.raises(ResultStoreReached): await theharvester_main.start(options) diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 51f9240e..c9bf04f6 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -59,15 +59,12 @@ from theHarvester.lib.result_values import normalize_asn, normalize_ip from theHarvester.lib.routeviews import RouteViewsCancelled, RouteViewsResult, enrich_routeviews from theHarvester.lib.shodan_evidence import ShodanHostObservation, canonical_shodan_hosts from theHarvester.lib.source_catalog import ( - DIRECT_DNS_ACTIONS, SOURCE_SPECS, ActivityClass, ResultRoute, get_source_spec, hostname_collection_conflicts, resolve_sources, - selected_action_names, - source_requires_direct_dns, ) from theHarvester.lib.source_runner import SourceOutcome, SourceRequest, run_source_jobs from theHarvester.lib.virtual_host import ( @@ -187,8 +184,8 @@ async def start( '-p', '--proxies', help=( - 'Use proxies.yaml for supported discovery sources and actions. The run fails immediately with ' - 'proxy-unavailable if no proxy is configured. Direct DNS sources and actions are rejected.' + 'Use proxies.yaml for supported HTTP(S) requests. The run fails immediately with proxy-unavailable if no ' + 'proxy is configured. DNS queries use the selected resolvers outside this proxy.' ), default=False, action='store_true', @@ -224,18 +221,13 @@ async def start( type=str, ) - parser.add_argument( - '-e', - '--dns-server', - help='Accepted for compatibility but currently unused; use --dns-resolvers to select resolvers.', - ) parser.add_argument( '-t', '--take-over', help=( 'Check discovered hosts for provider-gated takeover indicators. Uses configured DNS resolvers and ' - 'wildcard controls and does not follow redirects. DNS requires direct transport, so proxy mode rejects ' - 'this action before execution. Indicators are not confirmed takeovers.' + 'wildcard controls and does not follow redirects. HTTP confirmation requests use the configured proxy ' + 'when enabled. Indicators are not confirmed takeovers.' ), default=False, action='store_true', @@ -431,12 +423,6 @@ async def start( if conflicts := hostname_collection_conflicts(action_request): raise ValueError(f'--no-hosts cannot be combined with: {", ".join(conflicts)}') resolved_source_selection = resolve_sources(args.source) if args.source is not None else [] - direct_dns_sources = [source for source in resolved_source_selection if source_requires_direct_dns(source)] - if args.proxies and direct_dns_sources: - raise ValueError(f'Direct DNS sources cannot use --proxies: {", ".join(direct_dns_sources)}') - direct_dns_actions = [action for action in selected_action_names(action_request) if action in DIRECT_DNS_ACTIONS] - if args.proxies and direct_dns_actions: - raise ValueError(f'Direct DNS actions cannot use --proxies: {", ".join(direct_dns_actions)}') vhost_enabled = args.vhost or bool(args.vhost_endpoint) or bool(args.vhost_candidates) vhost_scope = '' vhost_endpoint = '' diff --git a/theHarvester/lib/api/run_models.py b/theHarvester/lib/api/run_models.py index 073dafdd..c424c716 100644 --- a/theHarvester/lib/api/run_models.py +++ b/theHarvester/lib/api/run_models.py @@ -17,13 +17,10 @@ from theHarvester.lib.evidence_types import EvidenceStatus # noqa: TC001 - Pyda from theHarvester.lib.resolver_selection import DEFAULT_DNS_RESOLVERS, normalize_resolver_addresses from theHarvester.lib.result_values import normalize_asn from theHarvester.lib.source_catalog import ( - DIRECT_DNS_ACTIONS, SOURCE_SPECS, ActivityClass, hostname_collection_conflicts, - resolve_sources, selected_action_names, - source_requires_direct_dns, ) from theHarvester.lib.virtual_host import ( DEFAULT_VHOST_CONCURRENCY, @@ -112,8 +109,8 @@ class RunRequest(BaseModel): proxies: bool = Field( default=False, description=( - 'Use configured proxies for supported discovery sources and actions. Supported requests fail ' - 'closed with proxy-unavailable if no proxy is configured. Direct DNS sources and actions are rejected.' + 'Use configured proxies for supported HTTP(S) requests. HTTP(S) fails closed with proxy-unavailable if no ' + 'proxy is configured. DNS queries use the selected resolvers outside this proxy.' ), ) no_hosts: bool = Field( @@ -172,7 +169,7 @@ class RunRequest(BaseModel): default=False, description=( 'Check discovered hosts for DNS provider-gated takeover indicators, with wildcard controls and no redirects. ' - 'DNS requires direct transport, so proxy mode rejects this action before execution. ' + 'HTTP confirmation requests use the configured proxy when enabled. ' 'Indicators are not confirmed takeovers.' ), ) @@ -275,12 +272,6 @@ class RunRequest(BaseModel): return self if self.screenshot: raise ValueError('Screenshot capture supports direct transport only; proxies must be disabled') - direct_sources = [ - source for source in resolve_sources(self.sources) if source in SOURCE_SPECS and source_requires_direct_dns(source) - ] - direct_actions = [action for action in selected_action_names(self.model_dump()) if action in DIRECT_DNS_ACTIONS] - if direct_work := (*direct_sources, *direct_actions): - raise ValueError(f'Direct DNS work cannot use proxies: {", ".join(direct_work)}') return self @model_validator(mode='after') diff --git a/theHarvester/lib/api/run_worker.py b/theHarvester/lib/api/run_worker.py index bc96d416..7dc1b559 100644 --- a/theHarvester/lib/api/run_worker.py +++ b/theHarvester/lib/api/run_worker.py @@ -310,7 +310,6 @@ async def _child_execute(run_id: str, database: Path) -> None: dns_recursive_runtime_seconds=request.get('dns_recursive_runtime_seconds'), dns_resolve=','.join(resolver_list) if request.get('dns_resolve') else '', dns_resolvers=tuple(resolver_list), - dns_server=None, domain=run['target'], filename='', limit=request['limit'], diff --git a/theHarvester/lib/api/static/harvestview/app.js b/theHarvester/lib/api/static/harvestview/app.js index d166741d..9ecc9d25 100644 --- a/theHarvester/lib/api/static/harvestview/app.js +++ b/theHarvester/lib/api/static/harvestview/app.js @@ -318,7 +318,7 @@ ['Result start offset', request.start ?? 'Not recorded'], ['Discovery source workers', request.source_workers ?? 'Not recorded'], ['Whole-run deadline', request.deadline_seconds === null ? 'Unlimited' : request.deadline_seconds === undefined ? 'Not recorded' : `${request.deadline_seconds} seconds`], - ['Proxy transport', request.proxies ? 'Selected' : 'Off'], + ['HTTP(S) proxy transport', request.proxies ? 'Selected' : 'Off'], ['Hostname results', request.no_hosts ? 'Excluded' : 'Included'], ['DNS lookup (/24 reverse expansion)', request.dns_lookup ? 'Selected' : 'Off'], ['DNS resolution', request.dns_resolve ? 'Selected' : 'Off'], ['DNS brute force', request.dns_brute ? 'Selected' : 'Off'], @@ -328,7 +328,10 @@ ['Recursive DNS runtime', request.dns_recursive_runtime_seconds === null ? 'Unlimited' : request.dns_recursive_runtime_seconds === undefined ? 'Not recorded' : `${request.dns_recursive_runtime_seconds} seconds`], ['RouteViews enrichment', request.routeviews ? 'Selected' : 'Off'], ['Screenshots', request.screenshot ? 'Selected' : 'Off'], - ['Takeover transport', request.takeover ? (request.proxies ? 'Configured proxy' : 'Direct') : 'Off'], + [ + 'Takeover transport', + request.takeover ? (request.proxies ? 'DNS resolvers + HTTP proxy' : 'DNS resolvers + direct HTTP') : 'Off' + ], ['API endpoint interaction', request.api_scan ? 'Selected' : 'Off'], ['Virtual-host discovery', request.vhost ? 'Selected' : 'Off'], ['Virtual-host endpoint override', request.vhost ? request.vhost_endpoint || 'Harvested IPs' : 'Not applicable'], diff --git a/theHarvester/lib/api/static/harvestview/index.html b/theHarvester/lib/api/static/harvestview/index.html index bafcab7e..8db2c2da 100644 --- a/theHarvester/lib/api/static/harvestview/index.html +++ b/theHarvester/lib/api/static/harvestview/index.html @@ -261,7 +261,7 @@ Optional list for API endpoint interaction, one URL path per line. Leave empty to use the bundled list. - + diff --git a/theHarvester/lib/enumeration.py b/theHarvester/lib/enumeration.py index f3224c5d..51695d37 100644 --- a/theHarvester/lib/enumeration.py +++ b/theHarvester/lib/enumeration.py @@ -32,7 +32,6 @@ class EnumerationOptions: no_hosts: bool = False shodan: bool = False screenshot: str = '' - dns_server: str | None = None take_over: bool = False dns_resolve: str | None = '' dns_resolvers: tuple[str, ...] = () @@ -77,7 +76,6 @@ class EnumerationOptions: no_hosts=getattr(value, 'no_hosts', False), shodan=getattr(value, 'shodan', False), screenshot=getattr(value, 'screenshot', ''), - dns_server=getattr(value, 'dns_server', None), take_over=getattr(value, 'take_over', False), dns_resolve=getattr(value, 'dns_resolve', ''), dns_resolvers=tuple(getattr(value, 'dns_resolvers', ())), diff --git a/theHarvester/lib/source_catalog.py b/theHarvester/lib/source_catalog.py index 20f85047..ef91612b 100644 --- a/theHarvester/lib/source_catalog.py +++ b/theHarvester/lib/source_catalog.py @@ -24,10 +24,6 @@ ACTION_ACTIVITIES: Final = { 'takeover': ActivityClass.DIRECT, 'vhost': ActivityClass.DIRECT, } -DIRECT_DNS_ACTIONS: Final = frozenset( - (*[name for name, activity in ACTION_ACTIVITIES.items() if activity is ActivityClass.DNS], 'takeover') -) -DIRECT_DNS_SOURCES: Final = frozenset({'shodan', 'shodanInternetDB'}) ACTION_REQUEST_FIELDS: Final = { **{name: name.replace('-', '_') for name in ACTION_ACTIVITIES}, 'dns-recursive': 'dns_recursive_depth', @@ -225,11 +221,6 @@ def get_source_spec(name: str) -> SourceSpec: return _CASEFOLDED_SOURCE_SPECS[name.casefold()] -def source_requires_direct_dns(name: str) -> bool: - spec = _CASEFOLDED_SOURCE_SPECS.get(name.casefold()) - return spec is not None and spec.name in DIRECT_DNS_SOURCES - - def activity_classes_for_selection( source_names: Iterable[str], action_names: Iterable[str] = (), diff --git a/theHarvester/lib/source_runner.py b/theHarvester/lib/source_runner.py index 586654d3..e0e70ae0 100644 --- a/theHarvester/lib/source_runner.py +++ b/theHarvester/lib/source_runner.py @@ -77,7 +77,7 @@ from theHarvester.lib.enumeration import DEFAULT_SOURCE_WORKERS from theHarvester.lib.hostnames import normalize_scoped_hostname from theHarvester.lib.result_values import normalize_ip from theHarvester.lib.shodan_evidence import ShodanHostObservation, canonical_shodan_hosts -from theHarvester.lib.source_catalog import ResultRoute, get_source_spec, source_requires_direct_dns +from theHarvester.lib.source_catalog import ResultRoute, get_source_spec from theHarvester.lib.source_execution import SourceExecutionReport if TYPE_CHECKING: @@ -315,16 +315,6 @@ async def run_source( process_completed = False proxy_transport_failed = False source_spec = get_source_spec(request.source) - if request.proxy and source_requires_direct_dns(source_spec.name): - return SourceOutcome( - SourceExecution( - source_spec.name, - 'failed', - (time.perf_counter() - started) * 1000, - 0, - stop_reason='direct-transport-only', - ) - ) try: with AsyncFetcher.proxy_scope(request.proxy) as selected_proxy: created_adapter = create_source(request)