From 259455f1c1a29c68a52b52ecd7233ee60f22659d Mon Sep 17 00:00:00 2001 From: Matt <36310667+NotoriousRebel@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:14:53 -0400 Subject: [PATCH] fix: remove Windvane DNS guessing fallback (#2541) --- CHANGELOG.md | 2 +- tests/discovery/test_windvane.py | 23 ++++----- tests/lib/test_core.py | 3 +- tests/test_all_source_orchestration.py | 2 - theHarvester/discovery/windvane.py | 70 ++------------------------ theHarvester/lib/source_catalog.py | 1 - 6 files changed, 15 insertions(+), 86 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20fd34f0..30ac0195 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,7 +70,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Sent a stable, versioned theHarvester identity with provider and API requests while preserving explicit browser identities for sources that require them. - Kept API endpoint scan URLs canonical instead of prefixing targets onto already complete URLs. - Made DeHashed pagination honor the CLI limit, retain only normalized email and IP evidence, and discard raw breach rows; aligned LeakIX with its authenticated subdomain endpoint and documented rate-limit retry. -- Added offline contracts for explicitly selected DNS and direct sources, retained normalized Pentest-Tools host and IP results, and hardened Shodan InternetDB, SubdomainFinder C99, and Windvane evidence boundaries. +- Added offline contracts for explicitly selected DNS and direct sources, retained normalized Pentest-Tools host and IP results, hardened Shodan InternetDB and SubdomainFinder C99 evidence boundaries, and removed Windvane's implicit DNS-guessing fallback while retaining its provider-backed source. - Retained relevant GitLab project, profile, and website URLs in consolidated JSONL and SQLite results while excluding unrelated user URLs. - Standardized BuiltWith and every other URL-producing adapter on `get_urls()`. - Made no-filename REST `/query` executions reach completed-result construction and SQLite persistence without changing the legacy response fields. diff --git a/tests/discovery/test_windvane.py b/tests/discovery/test_windvane.py index 7f7c9ed2..66a962d8 100644 --- a/tests/discovery/test_windvane.py +++ b/tests/discovery/test_windvane.py @@ -1,4 +1,3 @@ -import asyncio import json import socket @@ -59,26 +58,24 @@ async def test_authenticated_results_are_normalized_and_scoped(monkeypatch) -> N @pytest.mark.asyncio -async def test_keyless_fallback_uses_only_the_mocked_dns_boundary(monkeypatch) -> None: +async def test_keyless_provider_failure_does_not_guess_dns_names(monkeypatch) -> None: monkeypatch.setattr(windvane.Core, 'windvane_key', lambda: None) + requests = 0 async def fake_post_fetch(*_args, **_kwargs): + nonlocal requests + requests += 1 return '{"code":1}' - async def no_sleep(*_args, **_kwargs): - return None - - def fake_gethostbyname(hostname: str) -> str: - if hostname == 'api.example.test': - return '203.0.113.10' - raise socket.gaierror + def unexpected_gethostbyname(_hostname: str) -> str: + raise AssertionError('Windvane must not guess common DNS names') monkeypatch.setattr(windvane.AsyncFetcher, 'post_fetch', fake_post_fetch) - monkeypatch.setattr(asyncio, 'sleep', no_sleep) - monkeypatch.setattr(socket, 'gethostbyname', fake_gethostbyname) + monkeypatch.setattr(socket, 'gethostbyname', unexpected_gethostbyname) search = windvane.SearchWindvane('example.test') await search.process() - assert await search.get_hostnames() == {'api.example.test'} - assert await search.get_ips() == {'203.0.113.10'} + assert requests == 1 + assert await search.get_hostnames() == set() + assert await search.get_ips() == set() diff --git a/tests/lib/test_core.py b/tests/lib/test_core.py index d2672261..2d115a4e 100644 --- a/tests/lib/test_core.py +++ b/tests/lib/test_core.py @@ -90,13 +90,12 @@ def test_all_selects_only_passive_catalog_sources() -> None: "shodan": ActivityClass.DNS, "shodanInternetDB": ActivityClass.DNS, "subdomainfinderc99": ActivityClass.DNS, - "windvane": ActivityClass.DNS, } @pytest.mark.parametrize( 'source', - ['criminalip', 'pentesttools', 'shodan', 'shodanInternetDB', 'subdomainfinderc99', 'windvane'], + ['criminalip', 'pentesttools', 'shodan', 'shodanInternetDB', 'subdomainfinderc99'], ) def test_non_passive_sources_run_only_when_explicitly_selected(source: str) -> None: assert source not in Core.expand_source_selection('all') diff --git a/tests/test_all_source_orchestration.py b/tests/test_all_source_orchestration.py index 47c0a260..a53cbfa6 100644 --- a/tests/test_all_source_orchestration.py +++ b/tests/test_all_source_orchestration.py @@ -19,7 +19,6 @@ NON_PASSIVE_SOURCES = ( 'shodan', 'shodanInternetDB', 'subdomainfinderc99', - 'windvane', ) @@ -182,7 +181,6 @@ async def test_explicit_non_passive_source_is_scheduled_once( 'pentesttools': (theharvester_main.pentesttools, 'SearchPentestTools'), 'shodanInternetDB': (theharvester_main.shodan_internetdb, 'SearchShodanInternetDB'), 'subdomainfinderc99': (theharvester_main.subdomainfinderc99, 'SearchSubdomainfinderc99'), - 'windvane': (theharvester_main.windvane, 'SearchWindvane'), }[source] monkeypatch.setattr(module, constructor_name, lambda *_args, **_kwargs: FakeAdapter()) diff --git a/theHarvester/discovery/windvane.py b/theHarvester/discovery/windvane.py index e22d0e88..c2e50b15 100644 --- a/theHarvester/discovery/windvane.py +++ b/theHarvester/discovery/windvane.py @@ -30,7 +30,7 @@ class SearchWindvane: Note: This API requires authentication for full access. - With API key: Full access to all endpoints with pagination - - Without API key: Limited to 5 unauthenticated requests + DNS fallback + - Without API key: Limited unauthenticated API access Set API key via: - Environment variable: export WINDVANE_API_KEY="your-key" @@ -92,7 +92,7 @@ class SearchWindvane: await self._search_dns_history(headers) await self._search_emails(headers) else: - # Without API key, try alternative/limited approaches + # Without API key, use the provider's limited endpoint only. logger.info('[*] Windvane API key not found. Using limited unauthenticated access.') await self._search_subdomains_limited(headers) @@ -235,78 +235,14 @@ class SearchWindvane: logger.info(f'[*] Found {len(subdomains)} subdomains with limited access') else: - # If API call fails, try fallback approaches - await self._fallback_search() + logger.info(f'Windvane limited API returned code {response_data.get("code")}') except Exception as e: logger.info(f'Windvane limited API failed: {e}') - await self._fallback_search() except Exception as e: logger.info(f'Windvane limited search error: {e}') - async def _fallback_search(self) -> None: - """Fallback search using common subdomain patterns when API is unavailable""" - try: - logger.info('[*] API unavailable, using fallback subdomain pattern search...') - - # Common subdomain prefixes to try - common_subdomains = [ - 'www', - 'mail', - 'ftp', - 'admin', - 'test', - 'dev', - 'staging', - 'api', - 'cdn', - 'blog', - 'shop', - 'portal', - 'app', - 'mobile', - 'secure', - 'login', - 'support', - 'help', - 'docs', - 'status', - ] - - # Try to resolve common subdomains (basic DNS lookup approach) - import asyncio - import socket - - found_count = 0 - for sub in common_subdomains: - subdomain = f'{sub}.{self.word}' - try: - # Simple DNS resolution check - await asyncio.sleep(0.1) # Rate limiting - - # Use a simple DNS lookup (non-blocking) - loop = asyncio.get_event_loop() - try: - result = await loop.run_in_executor(None, socket.gethostbyname, subdomain) - if result: - self.totalhosts.add(subdomain.lower()) - self.totalips.add(result) - found_count += 1 - except socket.gaierror: - pass # Subdomain doesn't exist - - except Exception: - continue - - if found_count > 0: - logger.info(f'[*] Found {found_count} subdomains using DNS fallback') - else: - logger.info('[*] No additional subdomains found via fallback methods') - - except Exception as e: - logger.info(f'Fallback search error: {e}') - def set_api_key(self, api_key: str) -> None: """Set the API key for authenticated requests diff --git a/theHarvester/lib/source_catalog.py b/theHarvester/lib/source_catalog.py index 2dfea2bc..0213fa72 100644 --- a/theHarvester/lib/source_catalog.py +++ b/theHarvester/lib/source_catalog.py @@ -180,7 +180,6 @@ _SPECS = ( ResultRoute.SUBDOMAINS, ResultRoute.EMAILS, ResultRoute.IPS, - activity=ActivityClass.DNS, ), _spec('yahoo', ResultRoute.SUBDOMAINS, ResultRoute.EMAILS), _spec(