Remove duplicate Chaos discovery source (#2533)

CHANGELOG.md

@@ -34,7 +34,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 - Removed the transport-wide delay before reading ready HTTP responses, bounded Wayback Archive to 30 seconds and Common Crawl to 120 seconds, kept both sources within the requested result limit, and made long-source progress visible in verbose mode. Common Crawl now requests one 50-record page at a time instead of bursting page batches.
 - Made Baidu, crt.sh, HackerTarget, Have I Been Pwned, Mojeek, OTX, and Robtex report blocked, malformed, or transport failures truthfully. Also fixed HackerTarget CSV parsing and Robtex AAAA results.
-- Hardened BufferOver, Chaos, DNSDumpster, ONYPHE, and URLScan parsing and result attribution, including scoped typed results and bounded URLScan pagination.
+- Hardened BufferOver, ProjectDiscovery, DNSDumpster, ONYPHE, and URLScan parsing and result attribution, including scoped typed results and bounded URLScan pagination.
 - Made `harvestview` the sole launcher for the local web application and REST API.
 - Standardized SQLite, JSONL, API, and HarvestView result names on `hostname` and `ip` without a presentation alias.
 - Standardized URL-producing adapters, JSON, JSONL, SQLite, and API evidence on one `url` result kind while preserving producer provenance.
@@ -59,6 +59,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Removed
 - Removed the obsolete bundled IP-range and resolver snapshots.
 - Removed the REST API's built-in SlowAPI request limiter and its launcher option without adding a replacement.
+- Removed the duplicate `chaos` source name and module; ProjectDiscovery remains available through `projectdiscovery` with the same dataset and credential.
 - Removed Bitbucket domain discovery because its current REST APIs require workspace, repository, or user scope that the domain-only CLI contract cannot provide.
 - Removed the nonfunctional ThreatCrowd source because its service hostnames terminate at deleted AWS load balancers and return NXDOMAIN; OTX remains available through its separate adapter.
 
@@ -70,7 +71,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Retained relevant GitLab project, profile, and website URLs in consolidated JSONL and SQLite results while excluding unrelated user URLs.
 - Standardized BuiltWith and every other URL-producing adapter on `get_urls()`.
 - Made no-filename REST `/query` executions reach completed-result construction and SQLite persistence without changing the legacy response fields.
-- Made Chaos reject empty credentials, report HTTP and malformed-response failures, and preserve supported subdomain response shapes.
+- Made ProjectDiscovery reject empty credentials, report HTTP and malformed-response failures, and preserve supported subdomain response shapes.
 - Made Fofa reject incomplete credentials, report HTTP and malformed-response failures, normalize scoped hosts, and discard invalid IP values.
 - Made FullHunt reject empty credentials, report HTTP and malformed-response failures, and isolate malformed host records.
 - Made Hudson Rock HTTP failures status-aware, bounded rate-limit retries, removed trailing request delays, isolated malformed provider items, and retained infostealer details in completed JSONL and SQLite results.

README.md

@@ -169,7 +169,6 @@ Read the **API key** column as follows:
 | `brave` | ✓ | ✓ | No | No | No | No | No | No | ✓ |
 | `censys` | ✓ | ✓ | No | No | No | No | No | No | ✓ |
 | `certspotter` | ✓ | No | No | No | No | No | No | No | No |
-| `chaos` | ✓ | No | No | No | No | No | No | No | ✓ |
 | `commoncrawl` | ✓ | No | No | No | No | No | No | No | No |
 | `criminalip` | ✓ | No | ✓ | ✓ | No | No | No | No | ✓ |
 | `crt-name` | ✓ | No | No | No | No | No | No | No | No |

tests/discovery/test_projectdiscovery.py

@@ -3,7 +3,7 @@ from typing import Any
 
 import pytest
 
-from theHarvester.discovery import chaos
+from theHarvester.discovery import projectdiscovery
 from theHarvester.discovery.constants import MissingKey
 from theHarvester.lib.core import FetcherResponse
 
@@ -13,12 +13,12 @@ async def test_http_failure_is_reported_without_results(
     monkeypatch: pytest.MonkeyPatch,
     caplog: pytest.LogCaptureFixture,
 ) -> None:
-    monkeypatch.setattr(chaos.Core, 'projectdiscovery_key', lambda: 'test-key')
+    monkeypatch.setattr(projectdiscovery.Core, 'projectdiscovery_key', lambda: 'test-key')
 
     async def fake_fetch_all(urls: list[str], **kwargs: Any) -> list[FetcherResponse]:
         assert urls == ['https://dns.projectdiscovery.io/dns/example.com/subdomains']
-        assert kwargs['headers']['Authorization'] == 'Bearer test-key'
-        assert isinstance(kwargs['headers']['User-agent'], str)
+        assert kwargs['headers']['Authorization'] == 'test-key'
+        assert isinstance(kwargs['headers']['User-Agent'], str)
         assert {key: value for key, value in kwargs.items() if key != 'headers'} == {
             'proxy': False,
             'json': True,
@@ -26,27 +26,27 @@ async def test_http_failure_is_reported_without_results(
         }
         return [FetcherResponse(body={'error': 'forbidden'}, status=403, headers={})]
 
-    monkeypatch.setattr(chaos.AsyncFetcher, 'fetch_all', fake_fetch_all)
-    search = chaos.SearchChaos('example.com')
+    monkeypatch.setattr(projectdiscovery.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    search = projectdiscovery.SearchDiscovery('example.com')
 
-    with caplog.at_level(logging.INFO, logger=chaos.__name__):
+    with caplog.at_level(logging.INFO, logger=projectdiscovery.__name__):
         await search.process()
 
     assert await search.get_hostnames() == set()
     assert search.execution_status == 'failed'
     assert search.stop_reason == 'access-denied'
-    assert 'Chaos request failed with HTTP 403' in caplog.text
+    assert 'ProjectDiscovery request failed with HTTP 403' in caplog.text
 
 
 @pytest.mark.asyncio
 async def test_http_429_is_rate_limited(monkeypatch: pytest.MonkeyPatch) -> None:
-    monkeypatch.setattr(chaos.Core, 'projectdiscovery_key', lambda: 'test-key')
+    monkeypatch.setattr(projectdiscovery.Core, 'projectdiscovery_key', lambda: 'test-key')
 
     async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]:
         return [FetcherResponse(body={'error': 'too many requests'}, status=429, headers={})]
 
-    monkeypatch.setattr(chaos.AsyncFetcher, 'fetch_all', fake_fetch_all)
-    search = chaos.SearchChaos('example.com')
+    monkeypatch.setattr(projectdiscovery.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    search = projectdiscovery.SearchDiscovery('example.com')
 
     await search.process()
 
@@ -57,21 +57,21 @@ async def test_http_429_is_rate_limited(monkeypatch: pytest.MonkeyPatch) -> None
 
 @pytest.mark.parametrize('key', ['', '   '])
 def test_empty_api_key_is_rejected(monkeypatch: pytest.MonkeyPatch, key: str) -> None:
-    monkeypatch.setattr(chaos.Core, 'projectdiscovery_key', lambda: key)
+    monkeypatch.setattr(projectdiscovery.Core, 'projectdiscovery_key', lambda: key)
 
     with pytest.raises(MissingKey):
-        chaos.SearchChaos('example.com')
+        projectdiscovery.SearchDiscovery('example.com')
 
 
 @pytest.mark.asyncio
 async def test_provider_unauthorized_payload_is_access_denied(monkeypatch: pytest.MonkeyPatch) -> None:
-    monkeypatch.setattr(chaos.Core, 'projectdiscovery_key', lambda: 'test-key')
+    monkeypatch.setattr(projectdiscovery.Core, 'projectdiscovery_key', lambda: 'test-key')
 
     async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]:
         return [FetcherResponse(body={'error': 'unauthorized'}, status=200, headers={})]
 
-    monkeypatch.setattr(chaos.AsyncFetcher, 'fetch_all', fake_fetch_all)
-    search = chaos.SearchChaos('example.com')
+    monkeypatch.setattr(projectdiscovery.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    search = projectdiscovery.SearchDiscovery('example.com')
 
     await search.process()
 
@@ -85,21 +85,21 @@ async def test_malformed_response_is_reported(
     monkeypatch: pytest.MonkeyPatch,
     caplog: pytest.LogCaptureFixture,
 ) -> None:
-    monkeypatch.setattr(chaos.Core, 'projectdiscovery_key', lambda: 'test-key')
+    monkeypatch.setattr(projectdiscovery.Core, 'projectdiscovery_key', lambda: 'test-key')
 
     async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]:
         return [FetcherResponse(body=7, status=200, headers={})]
 
-    monkeypatch.setattr(chaos.AsyncFetcher, 'fetch_all', fake_fetch_all)
-    search = chaos.SearchChaos('example.com')
+    monkeypatch.setattr(projectdiscovery.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    search = projectdiscovery.SearchDiscovery('example.com')
 
-    with caplog.at_level(logging.INFO, logger=chaos.__name__):
+    with caplog.at_level(logging.INFO, logger=projectdiscovery.__name__):
         await search.process()
 
     assert await search.get_hostnames() == set()
     assert search.execution_status == 'failed'
     assert search.stop_reason == 'invalid-response'
-    assert 'Chaos returned malformed data' in caplog.text
+    assert 'ProjectDiscovery returned malformed data' in caplog.text
 
 
 @pytest.mark.asyncio
@@ -107,7 +107,7 @@ async def test_success_preserves_supported_subdomain_shapes(
     monkeypatch: pytest.MonkeyPatch,
     caplog: pytest.LogCaptureFixture,
 ) -> None:
-    monkeypatch.setattr(chaos.Core, 'projectdiscovery_key', lambda: 'test-key')
+    monkeypatch.setattr(projectdiscovery.Core, 'projectdiscovery_key', lambda: 'test-key')
 
     async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]:
         return [
@@ -118,16 +118,16 @@ async def test_success_preserves_supported_subdomain_shapes(
             )
         ]
 
-    monkeypatch.setattr(chaos.AsyncFetcher, 'fetch_all', fake_fetch_all)
-    search = chaos.SearchChaos('example.com')
+    monkeypatch.setattr(projectdiscovery.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    search = projectdiscovery.SearchDiscovery('example.com')
 
-    with caplog.at_level(logging.INFO, logger=chaos.__name__):
+    with caplog.at_level(logging.INFO, logger=projectdiscovery.__name__):
         await search.process()
 
     assert await search.get_hostnames() == {'api.example.com', 'www.example.com'}
     assert search.execution_status == 'partial'
     assert search.stop_reason == 'invalid-response'
-    assert caplog.text.count('Chaos ignored a malformed subdomain item') == 2
+    assert caplog.text.count('ProjectDiscovery ignored a malformed subdomain item') == 2
 
 
 @pytest.mark.asyncio
@@ -135,21 +135,21 @@ async def test_malformed_subdomain_collection_is_reported(
     monkeypatch: pytest.MonkeyPatch,
     caplog: pytest.LogCaptureFixture,
 ) -> None:
-    monkeypatch.setattr(chaos.Core, 'projectdiscovery_key', lambda: 'test-key')
+    monkeypatch.setattr(projectdiscovery.Core, 'projectdiscovery_key', lambda: 'test-key')
 
     async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]:
         return [FetcherResponse(body={'subdomains': 7}, status=200, headers={})]
 
-    monkeypatch.setattr(chaos.AsyncFetcher, 'fetch_all', fake_fetch_all)
-    search = chaos.SearchChaos('example.com')
+    monkeypatch.setattr(projectdiscovery.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    search = projectdiscovery.SearchDiscovery('example.com')
 
-    with caplog.at_level(logging.INFO, logger=chaos.__name__):
+    with caplog.at_level(logging.INFO, logger=projectdiscovery.__name__):
         await search.process()
 
     assert await search.get_hostnames() == set()
     assert search.execution_status == 'failed'
     assert search.stop_reason == 'invalid-response'
-    assert 'Chaos returned malformed subdomain data' in caplog.text
+    assert 'ProjectDiscovery returned malformed subdomain data' in caplog.text
 
 
 @pytest.mark.asyncio
@@ -157,15 +157,15 @@ async def test_fetch_exception_is_transport_failure(
     monkeypatch: pytest.MonkeyPatch,
     caplog: pytest.LogCaptureFixture,
 ) -> None:
-    monkeypatch.setattr(chaos.Core, 'projectdiscovery_key', lambda: 'test-key')
+    monkeypatch.setattr(projectdiscovery.Core, 'projectdiscovery_key', lambda: 'test-key')
 
     async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]:
         raise OSError('private transport details')
 
-    monkeypatch.setattr(chaos.AsyncFetcher, 'fetch_all', fake_fetch_all)
-    search = chaos.SearchChaos('example.com')
+    monkeypatch.setattr(projectdiscovery.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    search = projectdiscovery.SearchDiscovery('example.com')
 
-    with caplog.at_level(logging.INFO, logger=chaos.__name__):
+    with caplog.at_level(logging.INFO, logger=projectdiscovery.__name__):
         await search.process()
 
     assert await search.get_hostnames() == set()
@@ -179,7 +179,7 @@ async def test_parser_exception_preserves_valid_partial_results(
     monkeypatch: pytest.MonkeyPatch,
     caplog: pytest.LogCaptureFixture,
 ) -> None:
-    monkeypatch.setattr(chaos.Core, 'projectdiscovery_key', lambda: 'test-key')
+    monkeypatch.setattr(projectdiscovery.Core, 'projectdiscovery_key', lambda: 'test-key')
 
     class ExplodingSubdomains(list[str]):
         def __iter__(self):
@@ -195,10 +195,10 @@ async def test_parser_exception_preserves_valid_partial_results(
             )
         ]
 
-    monkeypatch.setattr(chaos.AsyncFetcher, 'fetch_all', fake_fetch_all)
-    search = chaos.SearchChaos('example.com')
+    monkeypatch.setattr(projectdiscovery.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    search = projectdiscovery.SearchDiscovery('example.com')
 
-    with caplog.at_level(logging.INFO, logger=chaos.__name__):
+    with caplog.at_level(logging.INFO, logger=projectdiscovery.__name__):
         await search.process()
 
     assert await search.get_hostnames() == {'api.example.com'}

tests/lib/test_source_catalog.py

@@ -40,6 +40,15 @@ def test_invalid_bitbucket_domain_source_is_not_selectable() -> None:
     assert 'bitbucket' not in _scheduled_source_names()
 
 
+def test_projectdiscovery_is_the_only_selector_for_the_chaos_corpus() -> None:
+    selected = Core.expand_source_selection('all')
+
+    assert 'projectdiscovery' in selected
+    assert 'chaos' not in selected
+    assert 'chaos' not in Core.get_supportedengines()
+    assert 'chaos' not in _scheduled_source_names()
+
+
 def test_subdomain_route_drives_subdomain_capability() -> None:
     spec = SourceSpec(
         name='example',

tests/test_main.py

@@ -1154,7 +1154,7 @@ async def test_dns_lookup_cancels_sibling_ranges_and_persists_partial_evidence(
 @pytest.mark.parametrize(
     ('source', 'module', 'constructor_name'),
     [
-        ('chaos', theharvester_main.chaos, 'SearchChaos'),
+        ('projectdiscovery', theharvester_main.projectdiscovery, 'SearchDiscovery'),
         ('bevigil', theharvester_main.bevigil, 'SearchBeVigil'),
     ],
 )

tests/test_readme.py

@@ -22,7 +22,7 @@ OPTIONAL_API_KEY_SOURCES = {'hackertarget', 'mojeek', 'windvane'}
 API_KEY_SOURCE_ALIASES = {
     'github': {'github-code'},
     'pentestTools': {'pentesttools'},
-    'projectDiscovery': {'chaos', 'projectdiscovery'},
+    'projectDiscovery': {'projectdiscovery'},
 }
 WIKI_PAGES = {
     'Configuration-and-API-Keys.md',
@@ -81,8 +81,8 @@ def test_readme_matches_declared_source_contracts() -> None:
     declared = _declared_source_contracts()
 
     assert '| Source | Subdomains | Emails | IPs | ASNs | URLs | People | Breaches |' in readme
-    assert len(declared) == 59
-    assert len(documented) == 59
+    assert len(declared) == 58
+    assert len(documented) == 58
     assert documented == declared
     assert {'securitytrails', 'shodaninternetdb'}.isdisjoint(documented)
 

theHarvester/__main__.py

@@ -35,7 +35,6 @@ from theHarvester.discovery import (
     builtwith,
     censysearch,
     certspottersearch,
-    chaos,
     commoncrawl,
     criminalip,
     crtname,
@@ -1037,23 +1036,6 @@ async def start(
                         if not args.quiet:
                             output_logger.info(f'Unexpected error occurred in Certspotter module: {e}')
 
-                elif engineitem == 'chaos':
-                    try:
-                        chaos_search = chaos.SearchChaos(word)
-                        stor_lst.append(
-                            store(
-                                chaos_search,
-                                engineitem,
-                            )
-                        )
-                    except Exception as e:
-                        if isinstance(e, MissingKey):
-                            record_missing_credentials(engineitem)
-                            if not args.quiet:
-                                output_logger.info(f'A Missing Key error occurred in Chaos: {e}')
-                        else:
-                            show_default_error_message(engineitem, word, e)
-
                 elif engineitem == 'commoncrawl':
                     try:
                         commoncrawl_search = commoncrawl.SearchCommoncrawl(word, limit)

theHarvester/discovery/chaos.py

@@ -1,130 +0,0 @@
-import logging
-
-from theHarvester.discovery.constants import MissingKey
-from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse
-
-logger = logging.getLogger(__name__)
-
-
-class SearchChaos:
-    """Class uses ProjectDiscovery Chaos subdomain enumeration API"""
-
-    def __init__(self, word) -> None:
-        self.word = word
-        self.totalhosts: set = set()
-        self.proxy = False
-        self.hostname = 'https://dns.projectdiscovery.io'
-        self.key = self._get_api_key()
-        self.execution_status: str | None = None
-        self.stop_reason: str | None = None
-
-    def _get_api_key(self) -> str:
-        """Get Chaos API key"""
-        try:
-            key = Core.projectdiscovery_key()
-        except Exception as error:
-            raise MissingKey('Chaos (ProjectDiscovery)') from error
-        if not isinstance(key, str) or not key.strip():
-            raise MissingKey('Chaos (ProjectDiscovery)')
-        return key
-
-    async def do_search(self) -> None:
-        try:
-            headers = {'User-agent': Core.get_user_agent(), 'Authorization': f'Bearer {self.key}'}
-
-            # Chaos API endpoint for subdomain enumeration
-            url = f'{self.hostname}/dns/{self.word}/subdomains'
-
-            response = await AsyncFetcher.fetch_all(
-                [url],
-                headers=headers,
-                proxy=self.proxy,
-                json=True,
-                include_metadata=True,
-            )
-
-            metadata = response[0] if response and isinstance(response[0], FetcherResponse) else None
-            if metadata is None:
-                self.execution_status = 'failed'
-                self.stop_reason = 'transport-error'
-                logger.info(f'No response from Chaos API for: {url}')
-                return
-            if not 200 <= metadata.status < 300:
-                self.execution_status = 'rate-limited' if metadata.status == 429 else 'failed'
-                self.stop_reason = 'access-denied' if metadata.status in {401, 403} else f'http-{metadata.status}'
-                logger.info(f'Chaos request failed with HTTP {metadata.status}')
-                return
-
-            try:
-                data = metadata.body
-                if not isinstance(data, (dict, list)):
-                    self.execution_status = 'failed'
-                    self.stop_reason = 'invalid-response'
-                    logger.info('Chaos returned malformed data')
-                    return
-
-                if isinstance(data, dict):
-                    # Check for error messages
-                    if 'error' in data:
-                        error_msg = data.get('message', data.get('error', 'Unknown error'))
-                        logger.info('Chaos API returned an error')
-                        self.execution_status = 'failed'
-                        self.stop_reason = 'access-denied' if 'unauthorized' in str(error_msg).casefold() else 'provider-error'
-                        return
-
-                    # Extract subdomains from response
-                    subdomains = data.get('subdomains', [])
-                    if not subdomains:
-                        subdomains = data.get('data', [])
-                    if not subdomains:
-                        subdomains = data.get('results', [])
-
-                else:
-                    subdomains = data
-
-                if not isinstance(subdomains, list):
-                    self.execution_status = 'failed'
-                    self.stop_reason = 'invalid-response'
-                    logger.info('Chaos returned malformed subdomain data')
-                    return
-                malformed_items = False
-                for subdomain in subdomains:
-                    if isinstance(subdomain, str):
-                        label = subdomain
-                    elif isinstance(subdomain, dict):
-                        label = subdomain.get('subdomain', '') or subdomain.get('name', '')
-                        if not isinstance(label, str) or not label:
-                            malformed_items = True
-                            logger.info('Chaos ignored a malformed subdomain item')
-                            continue
-                    else:
-                        malformed_items = True
-                        logger.info('Chaos ignored a malformed subdomain item')
-                        continue
-                    full_domain = f'{label}.{self.word}' if label else self.word
-                    self.totalhosts.add(full_domain.lower())
-                if malformed_items:
-                    self.execution_status = 'partial' if self.totalhosts else 'failed'
-                    self.stop_reason = 'invalid-response'
-                else:
-                    self.execution_status = 'completed'
-                    self.stop_reason = None if subdomains else 'no-results'
-
-            except Exception as error:
-                self.execution_status = 'partial' if self.totalhosts else 'failed'
-                self.stop_reason = 'invalid-response'
-                logger.info('Failed to parse Chaos response: %s', type(error).__name__)
-
-        except MissingKey:
-            raise
-        except Exception as error:
-            self.execution_status = 'failed'
-            self.stop_reason = 'transport-error'
-            logger.info('Chaos API error: %s', type(error).__name__)
-
-    async def get_hostnames(self) -> set:
-        return self.totalhosts
-
-    async def process(self, proxy: bool = False) -> None:
-        self.proxy = proxy
-        await self.do_search()

theHarvester/discovery/projectdiscovery.py

@@ -1,28 +1,117 @@
+import logging
+
 from theHarvester.discovery.constants import MissingKey
-from theHarvester.lib.core import AsyncFetcher, Core
+from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse
+
+logger = logging.getLogger(__name__)
 
 
 class SearchDiscovery:
-    def __init__(self, word) -> None:
+    """Collect subdomains from ProjectDiscovery's passive DNS dataset."""
+
+    def __init__(self, word: str) -> None:
         self.word = word
-        self.key = Core.projectdiscovery_key()
-        if self.key is None:
+        self.totalhosts: set[str] = set()
+        self.proxy = False
+        self.hostname = 'https://dns.projectdiscovery.io'
+        self.key = self._get_api_key()
+        self.execution_status: str | None = None
+        self.stop_reason: str | None = None
+
+    def _get_api_key(self) -> str:
+        try:
+            key = Core.projectdiscovery_key()
+        except Exception as error:
+            raise MissingKey('ProjectDiscovery') from error
+        if not isinstance(key, str) or not key.strip():
             raise MissingKey('ProjectDiscovery')
-        self.total_results: list[str] = []
-        self.proxy: bool = False
+        return key
 
-    async def do_search(self):
-        url = f'https://dns.projectdiscovery.io/dns/{self.word}/subdomains'
-        response = await AsyncFetcher.fetch_all(
-            [url],
-            json=True,
-            headers={'User-Agent': Core.get_user_agent(), 'Authorization': self.key},
-            proxy=self.proxy,
-        )
-        self.total_results = [f'{domains}.{self.word}' for domains in response[0]['subdomains']]
+    async def do_search(self) -> None:
+        try:
+            url = f'{self.hostname}/dns/{self.word}/subdomains'
+            response = await AsyncFetcher.fetch_all(
+                [url],
+                headers={'User-Agent': Core.get_user_agent(), 'Authorization': self.key},
+                proxy=self.proxy,
+                json=True,
+                include_metadata=True,
+            )
 
-    async def get_hostnames(self):
-        return self.total_results
+            metadata = response[0] if response and isinstance(response[0], FetcherResponse) else None
+            if metadata is None:
+                self.execution_status = 'failed'
+                self.stop_reason = 'transport-error'
+                logger.info('No response from ProjectDiscovery for: %s', url)
+                return
+            if not 200 <= metadata.status < 300:
+                self.execution_status = 'rate-limited' if metadata.status == 429 else 'failed'
+                self.stop_reason = 'access-denied' if metadata.status in {401, 403} else f'http-{metadata.status}'
+                logger.info('ProjectDiscovery request failed with HTTP %s', metadata.status)
+                return
+
+            try:
+                data = metadata.body
+                if not isinstance(data, (dict, list)):
+                    self.execution_status = 'failed'
+                    self.stop_reason = 'invalid-response'
+                    logger.info('ProjectDiscovery returned malformed data')
+                    return
+
+                if isinstance(data, dict):
+                    if 'error' in data:
+                        error_message = data.get('message', data.get('error', 'Unknown error'))
+                        self.execution_status = 'failed'
+                        self.stop_reason = (
+                            'access-denied' if 'unauthorized' in str(error_message).casefold() else 'provider-error'
+                        )
+                        logger.info('ProjectDiscovery returned an error')
+                        return
+                    subdomains = data.get('subdomains', []) or data.get('data', []) or data.get('results', [])
+                else:
+                    subdomains = data
+
+                if not isinstance(subdomains, list):
+                    self.execution_status = 'failed'
+                    self.stop_reason = 'invalid-response'
+                    logger.info('ProjectDiscovery returned malformed subdomain data')
+                    return
+
+                malformed_items = False
+                for subdomain in subdomains:
+                    if isinstance(subdomain, str):
+                        label = subdomain
+                    elif isinstance(subdomain, dict):
+                        label = subdomain.get('subdomain', '') or subdomain.get('name', '')
+                        if not isinstance(label, str) or not label:
+                            malformed_items = True
+                            logger.info('ProjectDiscovery ignored a malformed subdomain item')
+                            continue
+                    else:
+                        malformed_items = True
+                        logger.info('ProjectDiscovery ignored a malformed subdomain item')
+                        continue
+                    self.totalhosts.add(f'{label}.{self.word}'.lower() if label else self.word.lower())
+
+                if malformed_items:
+                    self.execution_status = 'partial' if self.totalhosts else 'failed'
+                    self.stop_reason = 'invalid-response'
+                else:
+                    self.execution_status = 'completed'
+                    self.stop_reason = None if subdomains else 'no-results'
+            except Exception as error:
+                self.execution_status = 'partial' if self.totalhosts else 'failed'
+                self.stop_reason = 'invalid-response'
+                logger.info('Failed to parse ProjectDiscovery response: %s', type(error).__name__)
+        except MissingKey:
+            raise
+        except Exception as error:
+            self.execution_status = 'failed'
+            self.stop_reason = 'transport-error'
+            logger.info('ProjectDiscovery API error: %s', type(error).__name__)
+
+    async def get_hostnames(self) -> set[str]:
+        return self.totalhosts
 
     async def process(self, proxy: bool = False) -> None:
         self.proxy = proxy

theHarvester/lib/api/runs.py

@@ -75,7 +75,7 @@ async def list_runs(
 async def list_sources(_api_key: Annotated[str, Depends(get_api_key)]) -> SourceCatalogResponse:
     from theHarvester.lib.core import Core
 
-    provider_aliases = {'chaos': 'projectDiscovery', 'github-code': 'github', 'pentesttools': 'pentestTools'}
+    provider_aliases = {'github-code': 'github', 'pentesttools': 'pentestTools'}
     api_key_fields = Core.api_key_fields()
     provider_names = {provider.casefold(): provider for provider in api_key_fields}
 

theHarvester/lib/core.py

@@ -409,7 +409,6 @@ class Core:
             'brave',
             'censys',
             'certspotter',
-            'chaos',
             'commoncrawl',
             'criminalip',
             'crt-name',

theHarvester/lib/source_catalog.py

@@ -99,7 +99,6 @@ _SPECS = (
     _spec('builtwith', ResultRoute.SUBDOMAINS, ResultRoute.URLS),
     _spec('censys', ResultRoute.SUBDOMAINS, ResultRoute.EMAILS),
     _spec('certspotter', ResultRoute.SUBDOMAINS),
-    _spec('chaos', ResultRoute.SUBDOMAINS),
     _spec('commoncrawl', ResultRoute.SUBDOMAINS),
     _spec(
         'criminalip',