diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ef9262c..108a1b7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Removed the transport-wide delay before reading ready HTTP responses, bounded Wayback Archive to 30 seconds and Common Crawl to 120 seconds, kept both sources within the requested result limit, and made long-source progress visible in verbose mode. Common Crawl now requests one 50-record page at a time instead of bursting page batches. - Made Baidu, crt.sh, HackerTarget, Have I Been Pwned, Mojeek, OTX, and Robtex report blocked, malformed, or transport failures truthfully. Also fixed HackerTarget CSV parsing and Robtex AAAA results. -- Hardened BufferOver, Chaos, DNSDumpster, ONYPHE, and URLScan parsing and result attribution, including scoped typed results and bounded URLScan pagination. +- Hardened BufferOver, ProjectDiscovery, DNSDumpster, ONYPHE, and URLScan parsing and result attribution, including scoped typed results and bounded URLScan pagination. - Made `harvestview` the sole launcher for the local web application and REST API. - Standardized SQLite, JSONL, API, and HarvestView result names on `hostname` and `ip` without a presentation alias. - Standardized URL-producing adapters, JSON, JSONL, SQLite, and API evidence on one `url` result kind while preserving producer provenance. @@ -59,6 +59,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed - Removed the obsolete bundled IP-range and resolver snapshots. - Removed the REST API's built-in SlowAPI request limiter and its launcher option without adding a replacement. +- Removed the duplicate `chaos` source name and module; ProjectDiscovery remains available through `projectdiscovery` with the same dataset and credential. - Removed Bitbucket domain discovery because its current REST APIs require workspace, repository, or user scope that the domain-only CLI contract cannot provide. - Removed the nonfunctional ThreatCrowd source because its service hostnames terminate at deleted AWS load balancers and return NXDOMAIN; OTX remains available through its separate adapter. @@ -70,7 +71,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Retained relevant GitLab project, profile, and website URLs in consolidated JSONL and SQLite results while excluding unrelated user URLs. - Standardized BuiltWith and every other URL-producing adapter on `get_urls()`. - Made no-filename REST `/query` executions reach completed-result construction and SQLite persistence without changing the legacy response fields. -- Made Chaos reject empty credentials, report HTTP and malformed-response failures, and preserve supported subdomain response shapes. +- Made ProjectDiscovery reject empty credentials, report HTTP and malformed-response failures, and preserve supported subdomain response shapes. - Made Fofa reject incomplete credentials, report HTTP and malformed-response failures, normalize scoped hosts, and discard invalid IP values. - Made FullHunt reject empty credentials, report HTTP and malformed-response failures, and isolate malformed host records. - Made Hudson Rock HTTP failures status-aware, bounded rate-limit retries, removed trailing request delays, isolated malformed provider items, and retained infostealer details in completed JSONL and SQLite results. diff --git a/README.md b/README.md index 93ff46b3..b44deeca 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,6 @@ Read the **API key** column as follows: | `brave` | ✓ | ✓ | No | No | No | No | No | No | ✓ | | `censys` | ✓ | ✓ | No | No | No | No | No | No | ✓ | | `certspotter` | ✓ | No | No | No | No | No | No | No | No | -| `chaos` | ✓ | No | No | No | No | No | No | No | ✓ | | `commoncrawl` | ✓ | No | No | No | No | No | No | No | No | | `criminalip` | ✓ | No | ✓ | ✓ | No | No | No | No | ✓ | | `crt-name` | ✓ | No | No | No | No | No | No | No | No | diff --git a/tests/discovery/test_chaos.py b/tests/discovery/test_projectdiscovery.py similarity index 63% rename from tests/discovery/test_chaos.py rename to tests/discovery/test_projectdiscovery.py index e3cb6e32..380622fb 100644 --- a/tests/discovery/test_chaos.py +++ b/tests/discovery/test_projectdiscovery.py @@ -3,7 +3,7 @@ from typing import Any import pytest -from theHarvester.discovery import chaos +from theHarvester.discovery import projectdiscovery from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import FetcherResponse @@ -13,12 +13,12 @@ async def test_http_failure_is_reported_without_results( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, ) -> None: - monkeypatch.setattr(chaos.Core, 'projectdiscovery_key', lambda: 'test-key') + monkeypatch.setattr(projectdiscovery.Core, 'projectdiscovery_key', lambda: 'test-key') async def fake_fetch_all(urls: list[str], **kwargs: Any) -> list[FetcherResponse]: assert urls == ['https://dns.projectdiscovery.io/dns/example.com/subdomains'] - assert kwargs['headers']['Authorization'] == 'Bearer test-key' - assert isinstance(kwargs['headers']['User-agent'], str) + assert kwargs['headers']['Authorization'] == 'test-key' + assert isinstance(kwargs['headers']['User-Agent'], str) assert {key: value for key, value in kwargs.items() if key != 'headers'} == { 'proxy': False, 'json': True, @@ -26,27 +26,27 @@ async def test_http_failure_is_reported_without_results( } return [FetcherResponse(body={'error': 'forbidden'}, status=403, headers={})] - monkeypatch.setattr(chaos.AsyncFetcher, 'fetch_all', fake_fetch_all) - search = chaos.SearchChaos('example.com') + monkeypatch.setattr(projectdiscovery.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = projectdiscovery.SearchDiscovery('example.com') - with caplog.at_level(logging.INFO, logger=chaos.__name__): + with caplog.at_level(logging.INFO, logger=projectdiscovery.__name__): await search.process() assert await search.get_hostnames() == set() assert search.execution_status == 'failed' assert search.stop_reason == 'access-denied' - assert 'Chaos request failed with HTTP 403' in caplog.text + assert 'ProjectDiscovery request failed with HTTP 403' in caplog.text @pytest.mark.asyncio async def test_http_429_is_rate_limited(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(chaos.Core, 'projectdiscovery_key', lambda: 'test-key') + monkeypatch.setattr(projectdiscovery.Core, 'projectdiscovery_key', lambda: 'test-key') async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: return [FetcherResponse(body={'error': 'too many requests'}, status=429, headers={})] - monkeypatch.setattr(chaos.AsyncFetcher, 'fetch_all', fake_fetch_all) - search = chaos.SearchChaos('example.com') + monkeypatch.setattr(projectdiscovery.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = projectdiscovery.SearchDiscovery('example.com') await search.process() @@ -57,21 +57,21 @@ async def test_http_429_is_rate_limited(monkeypatch: pytest.MonkeyPatch) -> None @pytest.mark.parametrize('key', ['', ' ']) def test_empty_api_key_is_rejected(monkeypatch: pytest.MonkeyPatch, key: str) -> None: - monkeypatch.setattr(chaos.Core, 'projectdiscovery_key', lambda: key) + monkeypatch.setattr(projectdiscovery.Core, 'projectdiscovery_key', lambda: key) with pytest.raises(MissingKey): - chaos.SearchChaos('example.com') + projectdiscovery.SearchDiscovery('example.com') @pytest.mark.asyncio async def test_provider_unauthorized_payload_is_access_denied(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(chaos.Core, 'projectdiscovery_key', lambda: 'test-key') + monkeypatch.setattr(projectdiscovery.Core, 'projectdiscovery_key', lambda: 'test-key') async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: return [FetcherResponse(body={'error': 'unauthorized'}, status=200, headers={})] - monkeypatch.setattr(chaos.AsyncFetcher, 'fetch_all', fake_fetch_all) - search = chaos.SearchChaos('example.com') + monkeypatch.setattr(projectdiscovery.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = projectdiscovery.SearchDiscovery('example.com') await search.process() @@ -85,21 +85,21 @@ async def test_malformed_response_is_reported( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, ) -> None: - monkeypatch.setattr(chaos.Core, 'projectdiscovery_key', lambda: 'test-key') + monkeypatch.setattr(projectdiscovery.Core, 'projectdiscovery_key', lambda: 'test-key') async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: return [FetcherResponse(body=7, status=200, headers={})] - monkeypatch.setattr(chaos.AsyncFetcher, 'fetch_all', fake_fetch_all) - search = chaos.SearchChaos('example.com') + monkeypatch.setattr(projectdiscovery.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = projectdiscovery.SearchDiscovery('example.com') - with caplog.at_level(logging.INFO, logger=chaos.__name__): + with caplog.at_level(logging.INFO, logger=projectdiscovery.__name__): await search.process() assert await search.get_hostnames() == set() assert search.execution_status == 'failed' assert search.stop_reason == 'invalid-response' - assert 'Chaos returned malformed data' in caplog.text + assert 'ProjectDiscovery returned malformed data' in caplog.text @pytest.mark.asyncio @@ -107,7 +107,7 @@ async def test_success_preserves_supported_subdomain_shapes( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, ) -> None: - monkeypatch.setattr(chaos.Core, 'projectdiscovery_key', lambda: 'test-key') + monkeypatch.setattr(projectdiscovery.Core, 'projectdiscovery_key', lambda: 'test-key') async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: return [ @@ -118,16 +118,16 @@ async def test_success_preserves_supported_subdomain_shapes( ) ] - monkeypatch.setattr(chaos.AsyncFetcher, 'fetch_all', fake_fetch_all) - search = chaos.SearchChaos('example.com') + monkeypatch.setattr(projectdiscovery.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = projectdiscovery.SearchDiscovery('example.com') - with caplog.at_level(logging.INFO, logger=chaos.__name__): + with caplog.at_level(logging.INFO, logger=projectdiscovery.__name__): await search.process() assert await search.get_hostnames() == {'api.example.com', 'www.example.com'} assert search.execution_status == 'partial' assert search.stop_reason == 'invalid-response' - assert caplog.text.count('Chaos ignored a malformed subdomain item') == 2 + assert caplog.text.count('ProjectDiscovery ignored a malformed subdomain item') == 2 @pytest.mark.asyncio @@ -135,21 +135,21 @@ async def test_malformed_subdomain_collection_is_reported( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, ) -> None: - monkeypatch.setattr(chaos.Core, 'projectdiscovery_key', lambda: 'test-key') + monkeypatch.setattr(projectdiscovery.Core, 'projectdiscovery_key', lambda: 'test-key') async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: return [FetcherResponse(body={'subdomains': 7}, status=200, headers={})] - monkeypatch.setattr(chaos.AsyncFetcher, 'fetch_all', fake_fetch_all) - search = chaos.SearchChaos('example.com') + monkeypatch.setattr(projectdiscovery.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = projectdiscovery.SearchDiscovery('example.com') - with caplog.at_level(logging.INFO, logger=chaos.__name__): + with caplog.at_level(logging.INFO, logger=projectdiscovery.__name__): await search.process() assert await search.get_hostnames() == set() assert search.execution_status == 'failed' assert search.stop_reason == 'invalid-response' - assert 'Chaos returned malformed subdomain data' in caplog.text + assert 'ProjectDiscovery returned malformed subdomain data' in caplog.text @pytest.mark.asyncio @@ -157,15 +157,15 @@ async def test_fetch_exception_is_transport_failure( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, ) -> None: - monkeypatch.setattr(chaos.Core, 'projectdiscovery_key', lambda: 'test-key') + monkeypatch.setattr(projectdiscovery.Core, 'projectdiscovery_key', lambda: 'test-key') async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: raise OSError('private transport details') - monkeypatch.setattr(chaos.AsyncFetcher, 'fetch_all', fake_fetch_all) - search = chaos.SearchChaos('example.com') + monkeypatch.setattr(projectdiscovery.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = projectdiscovery.SearchDiscovery('example.com') - with caplog.at_level(logging.INFO, logger=chaos.__name__): + with caplog.at_level(logging.INFO, logger=projectdiscovery.__name__): await search.process() assert await search.get_hostnames() == set() @@ -179,7 +179,7 @@ async def test_parser_exception_preserves_valid_partial_results( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, ) -> None: - monkeypatch.setattr(chaos.Core, 'projectdiscovery_key', lambda: 'test-key') + monkeypatch.setattr(projectdiscovery.Core, 'projectdiscovery_key', lambda: 'test-key') class ExplodingSubdomains(list[str]): def __iter__(self): @@ -195,10 +195,10 @@ async def test_parser_exception_preserves_valid_partial_results( ) ] - monkeypatch.setattr(chaos.AsyncFetcher, 'fetch_all', fake_fetch_all) - search = chaos.SearchChaos('example.com') + monkeypatch.setattr(projectdiscovery.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = projectdiscovery.SearchDiscovery('example.com') - with caplog.at_level(logging.INFO, logger=chaos.__name__): + with caplog.at_level(logging.INFO, logger=projectdiscovery.__name__): await search.process() assert await search.get_hostnames() == {'api.example.com'} diff --git a/tests/lib/test_source_catalog.py b/tests/lib/test_source_catalog.py index ca6e7296..73cbdef6 100644 --- a/tests/lib/test_source_catalog.py +++ b/tests/lib/test_source_catalog.py @@ -40,6 +40,15 @@ def test_invalid_bitbucket_domain_source_is_not_selectable() -> None: assert 'bitbucket' not in _scheduled_source_names() +def test_projectdiscovery_is_the_only_selector_for_the_chaos_corpus() -> None: + selected = Core.expand_source_selection('all') + + assert 'projectdiscovery' in selected + assert 'chaos' not in selected + assert 'chaos' not in Core.get_supportedengines() + assert 'chaos' not in _scheduled_source_names() + + def test_subdomain_route_drives_subdomain_capability() -> None: spec = SourceSpec( name='example', diff --git a/tests/test_main.py b/tests/test_main.py index 3339c889..df4b2e9d 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1154,7 +1154,7 @@ async def test_dns_lookup_cancels_sibling_ranges_and_persists_partial_evidence( @pytest.mark.parametrize( ('source', 'module', 'constructor_name'), [ - ('chaos', theharvester_main.chaos, 'SearchChaos'), + ('projectdiscovery', theharvester_main.projectdiscovery, 'SearchDiscovery'), ('bevigil', theharvester_main.bevigil, 'SearchBeVigil'), ], ) diff --git a/tests/test_readme.py b/tests/test_readme.py index 0e1d631b..4b110764 100644 --- a/tests/test_readme.py +++ b/tests/test_readme.py @@ -22,7 +22,7 @@ OPTIONAL_API_KEY_SOURCES = {'hackertarget', 'mojeek', 'windvane'} API_KEY_SOURCE_ALIASES = { 'github': {'github-code'}, 'pentestTools': {'pentesttools'}, - 'projectDiscovery': {'chaos', 'projectdiscovery'}, + 'projectDiscovery': {'projectdiscovery'}, } WIKI_PAGES = { 'Configuration-and-API-Keys.md', @@ -81,8 +81,8 @@ def test_readme_matches_declared_source_contracts() -> None: declared = _declared_source_contracts() assert '| Source | Subdomains | Emails | IPs | ASNs | URLs | People | Breaches |' in readme - assert len(declared) == 59 - assert len(documented) == 59 + assert len(declared) == 58 + assert len(documented) == 58 assert documented == declared assert {'securitytrails', 'shodaninternetdb'}.isdisjoint(documented) diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 4e7d6af7..1814b9c1 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -35,7 +35,6 @@ from theHarvester.discovery import ( builtwith, censysearch, certspottersearch, - chaos, commoncrawl, criminalip, crtname, @@ -1037,23 +1036,6 @@ async def start( if not args.quiet: output_logger.info(f'Unexpected error occurred in Certspotter module: {e}') - elif engineitem == 'chaos': - try: - chaos_search = chaos.SearchChaos(word) - stor_lst.append( - store( - chaos_search, - engineitem, - ) - ) - except Exception as e: - if isinstance(e, MissingKey): - record_missing_credentials(engineitem) - if not args.quiet: - output_logger.info(f'A Missing Key error occurred in Chaos: {e}') - else: - show_default_error_message(engineitem, word, e) - elif engineitem == 'commoncrawl': try: commoncrawl_search = commoncrawl.SearchCommoncrawl(word, limit) diff --git a/theHarvester/discovery/chaos.py b/theHarvester/discovery/chaos.py deleted file mode 100644 index 5682a23c..00000000 --- a/theHarvester/discovery/chaos.py +++ /dev/null @@ -1,130 +0,0 @@ -import logging - -from theHarvester.discovery.constants import MissingKey -from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse - -logger = logging.getLogger(__name__) - - -class SearchChaos: - """Class uses ProjectDiscovery Chaos subdomain enumeration API""" - - def __init__(self, word) -> None: - self.word = word - self.totalhosts: set = set() - self.proxy = False - self.hostname = 'https://dns.projectdiscovery.io' - self.key = self._get_api_key() - self.execution_status: str | None = None - self.stop_reason: str | None = None - - def _get_api_key(self) -> str: - """Get Chaos API key""" - try: - key = Core.projectdiscovery_key() - except Exception as error: - raise MissingKey('Chaos (ProjectDiscovery)') from error - if not isinstance(key, str) or not key.strip(): - raise MissingKey('Chaos (ProjectDiscovery)') - return key - - async def do_search(self) -> None: - try: - headers = {'User-agent': Core.get_user_agent(), 'Authorization': f'Bearer {self.key}'} - - # Chaos API endpoint for subdomain enumeration - url = f'{self.hostname}/dns/{self.word}/subdomains' - - response = await AsyncFetcher.fetch_all( - [url], - headers=headers, - proxy=self.proxy, - json=True, - include_metadata=True, - ) - - metadata = response[0] if response and isinstance(response[0], FetcherResponse) else None - if metadata is None: - self.execution_status = 'failed' - self.stop_reason = 'transport-error' - logger.info(f'No response from Chaos API for: {url}') - return - if not 200 <= metadata.status < 300: - self.execution_status = 'rate-limited' if metadata.status == 429 else 'failed' - self.stop_reason = 'access-denied' if metadata.status in {401, 403} else f'http-{metadata.status}' - logger.info(f'Chaos request failed with HTTP {metadata.status}') - return - - try: - data = metadata.body - if not isinstance(data, (dict, list)): - self.execution_status = 'failed' - self.stop_reason = 'invalid-response' - logger.info('Chaos returned malformed data') - return - - if isinstance(data, dict): - # Check for error messages - if 'error' in data: - error_msg = data.get('message', data.get('error', 'Unknown error')) - logger.info('Chaos API returned an error') - self.execution_status = 'failed' - self.stop_reason = 'access-denied' if 'unauthorized' in str(error_msg).casefold() else 'provider-error' - return - - # Extract subdomains from response - subdomains = data.get('subdomains', []) - if not subdomains: - subdomains = data.get('data', []) - if not subdomains: - subdomains = data.get('results', []) - - else: - subdomains = data - - if not isinstance(subdomains, list): - self.execution_status = 'failed' - self.stop_reason = 'invalid-response' - logger.info('Chaos returned malformed subdomain data') - return - malformed_items = False - for subdomain in subdomains: - if isinstance(subdomain, str): - label = subdomain - elif isinstance(subdomain, dict): - label = subdomain.get('subdomain', '') or subdomain.get('name', '') - if not isinstance(label, str) or not label: - malformed_items = True - logger.info('Chaos ignored a malformed subdomain item') - continue - else: - malformed_items = True - logger.info('Chaos ignored a malformed subdomain item') - continue - full_domain = f'{label}.{self.word}' if label else self.word - self.totalhosts.add(full_domain.lower()) - if malformed_items: - self.execution_status = 'partial' if self.totalhosts else 'failed' - self.stop_reason = 'invalid-response' - else: - self.execution_status = 'completed' - self.stop_reason = None if subdomains else 'no-results' - - except Exception as error: - self.execution_status = 'partial' if self.totalhosts else 'failed' - self.stop_reason = 'invalid-response' - logger.info('Failed to parse Chaos response: %s', type(error).__name__) - - except MissingKey: - raise - except Exception as error: - self.execution_status = 'failed' - self.stop_reason = 'transport-error' - logger.info('Chaos API error: %s', type(error).__name__) - - async def get_hostnames(self) -> set: - return self.totalhosts - - async def process(self, proxy: bool = False) -> None: - self.proxy = proxy - await self.do_search() diff --git a/theHarvester/discovery/projectdiscovery.py b/theHarvester/discovery/projectdiscovery.py index b812ca45..117bfc34 100644 --- a/theHarvester/discovery/projectdiscovery.py +++ b/theHarvester/discovery/projectdiscovery.py @@ -1,28 +1,117 @@ +import logging + from theHarvester.discovery.constants import MissingKey -from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse + +logger = logging.getLogger(__name__) class SearchDiscovery: - def __init__(self, word) -> None: + """Collect subdomains from ProjectDiscovery's passive DNS dataset.""" + + def __init__(self, word: str) -> None: self.word = word - self.key = Core.projectdiscovery_key() - if self.key is None: + self.totalhosts: set[str] = set() + self.proxy = False + self.hostname = 'https://dns.projectdiscovery.io' + self.key = self._get_api_key() + self.execution_status: str | None = None + self.stop_reason: str | None = None + + def _get_api_key(self) -> str: + try: + key = Core.projectdiscovery_key() + except Exception as error: + raise MissingKey('ProjectDiscovery') from error + if not isinstance(key, str) or not key.strip(): raise MissingKey('ProjectDiscovery') - self.total_results: list[str] = [] - self.proxy: bool = False + return key - async def do_search(self): - url = f'https://dns.projectdiscovery.io/dns/{self.word}/subdomains' - response = await AsyncFetcher.fetch_all( - [url], - json=True, - headers={'User-Agent': Core.get_user_agent(), 'Authorization': self.key}, - proxy=self.proxy, - ) - self.total_results = [f'{domains}.{self.word}' for domains in response[0]['subdomains']] + async def do_search(self) -> None: + try: + url = f'{self.hostname}/dns/{self.word}/subdomains' + response = await AsyncFetcher.fetch_all( + [url], + headers={'User-Agent': Core.get_user_agent(), 'Authorization': self.key}, + proxy=self.proxy, + json=True, + include_metadata=True, + ) - async def get_hostnames(self): - return self.total_results + metadata = response[0] if response and isinstance(response[0], FetcherResponse) else None + if metadata is None: + self.execution_status = 'failed' + self.stop_reason = 'transport-error' + logger.info('No response from ProjectDiscovery for: %s', url) + return + if not 200 <= metadata.status < 300: + self.execution_status = 'rate-limited' if metadata.status == 429 else 'failed' + self.stop_reason = 'access-denied' if metadata.status in {401, 403} else f'http-{metadata.status}' + logger.info('ProjectDiscovery request failed with HTTP %s', metadata.status) + return + + try: + data = metadata.body + if not isinstance(data, (dict, list)): + self.execution_status = 'failed' + self.stop_reason = 'invalid-response' + logger.info('ProjectDiscovery returned malformed data') + return + + if isinstance(data, dict): + if 'error' in data: + error_message = data.get('message', data.get('error', 'Unknown error')) + self.execution_status = 'failed' + self.stop_reason = ( + 'access-denied' if 'unauthorized' in str(error_message).casefold() else 'provider-error' + ) + logger.info('ProjectDiscovery returned an error') + return + subdomains = data.get('subdomains', []) or data.get('data', []) or data.get('results', []) + else: + subdomains = data + + if not isinstance(subdomains, list): + self.execution_status = 'failed' + self.stop_reason = 'invalid-response' + logger.info('ProjectDiscovery returned malformed subdomain data') + return + + malformed_items = False + for subdomain in subdomains: + if isinstance(subdomain, str): + label = subdomain + elif isinstance(subdomain, dict): + label = subdomain.get('subdomain', '') or subdomain.get('name', '') + if not isinstance(label, str) or not label: + malformed_items = True + logger.info('ProjectDiscovery ignored a malformed subdomain item') + continue + else: + malformed_items = True + logger.info('ProjectDiscovery ignored a malformed subdomain item') + continue + self.totalhosts.add(f'{label}.{self.word}'.lower() if label else self.word.lower()) + + if malformed_items: + self.execution_status = 'partial' if self.totalhosts else 'failed' + self.stop_reason = 'invalid-response' + else: + self.execution_status = 'completed' + self.stop_reason = None if subdomains else 'no-results' + except Exception as error: + self.execution_status = 'partial' if self.totalhosts else 'failed' + self.stop_reason = 'invalid-response' + logger.info('Failed to parse ProjectDiscovery response: %s', type(error).__name__) + except MissingKey: + raise + except Exception as error: + self.execution_status = 'failed' + self.stop_reason = 'transport-error' + logger.info('ProjectDiscovery API error: %s', type(error).__name__) + + async def get_hostnames(self) -> set[str]: + return self.totalhosts async def process(self, proxy: bool = False) -> None: self.proxy = proxy diff --git a/theHarvester/lib/api/runs.py b/theHarvester/lib/api/runs.py index 46139e67..09e11391 100644 --- a/theHarvester/lib/api/runs.py +++ b/theHarvester/lib/api/runs.py @@ -75,7 +75,7 @@ async def list_runs( async def list_sources(_api_key: Annotated[str, Depends(get_api_key)]) -> SourceCatalogResponse: from theHarvester.lib.core import Core - provider_aliases = {'chaos': 'projectDiscovery', 'github-code': 'github', 'pentesttools': 'pentestTools'} + provider_aliases = {'github-code': 'github', 'pentesttools': 'pentestTools'} api_key_fields = Core.api_key_fields() provider_names = {provider.casefold(): provider for provider in api_key_fields} diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index e5f9a5e9..5283b9c0 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -409,7 +409,6 @@ class Core: 'brave', 'censys', 'certspotter', - 'chaos', 'commoncrawl', 'criminalip', 'crt-name', diff --git a/theHarvester/lib/source_catalog.py b/theHarvester/lib/source_catalog.py index c76c826f..73c0fe86 100644 --- a/theHarvester/lib/source_catalog.py +++ b/theHarvester/lib/source_catalog.py @@ -99,7 +99,6 @@ _SPECS = ( _spec('builtwith', ResultRoute.SUBDOMAINS, ResultRoute.URLS), _spec('censys', ResultRoute.SUBDOMAINS, ResultRoute.EMAILS), _spec('certspotter', ResultRoute.SUBDOMAINS), - _spec('chaos', ResultRoute.SUBDOMAINS), _spec('commoncrawl', ResultRoute.SUBDOMAINS), _spec( 'criminalip',