feat: add DNSDB passive discovery source

README.md

@@ -130,6 +130,7 @@ Read the **API key** column as follows:
 | `criminalip` | ✓ | No | ✓ | ✓ | No | No | No | ✓ |
 | `crtsh` | ✓ | No | No | No | No | No | No | No |
 | `dehashed` | No | No | ✓ | No | No | No | No | ✓ |
+| `dnsdb` | ✓ | No | No | No | No | No | No | ✓ |
 | `dnsdumpster` | ✓ | No | ✓ | No | No | No | No | ✓ |
 | `duckduckgo` | ✓ | ✓ | No | No | No | No | No | No |
 | `dymo` | ✓ | No | No | No | No | No | No | ✓ |

tests/discovery/test_dnsdb.py

@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+import logging
+
+import pytest
+
+from theHarvester.discovery import dnsdb
+from theHarvester.discovery.constants import MissingKey
+
+
+def _install_response(
+    monkeypatch: pytest.MonkeyPatch,
+    lines: tuple[bytes, ...],
+    *,
+    status: int = 200,
+) -> dict[str, object]:
+    requested: dict[str, object] = {}
+    lines_left = list(lines)
+
+    class FakeContent:
+        def __aiter__(self) -> FakeContent:
+            return self
+
+        async def __anext__(self) -> bytes:
+            if not lines_left:
+                raise StopAsyncIteration
+            return lines_left.pop(0)
+
+    class FakeResponse:
+        def __init__(self) -> None:
+            self.status = status
+
+        content = FakeContent()
+
+        async def __aenter__(self) -> FakeResponse:
+            return self
+
+        async def __aexit__(self, *_args: object) -> None:
+            return None
+
+    class FakeSession:
+        def __init__(self, **kwargs: object) -> None:
+            requested['session'] = kwargs
+
+        def get(self, url: str, **kwargs: object) -> FakeResponse:
+            requested['url'] = url
+            requested['request'] = kwargs
+            return FakeResponse()
+
+        async def __aenter__(self) -> FakeSession:
+            return self
+
+        async def __aexit__(self, *_args: object) -> None:
+            return None
+
+    monkeypatch.setattr(dnsdb.aiohttp, 'ClientSession', FakeSession)
+    return requested
+
+
+def test_blank_key_is_missing(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(dnsdb.Core, 'dnsdb_key', lambda: '  ')
+
+    with pytest.raises(MissingKey):
+        dnsdb.SearchDNSDB('example.com')
+
+
+@pytest.mark.asyncio
+async def test_process_collects_normalized_in_scope_rrset_owners(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(dnsdb.Core, 'dnsdb_key', lambda: 'dnsdb-test-key')
+    requested = _install_response(
+        monkeypatch,
+        (
+            b'{"cond":"begin"}\n',
+            b'{"obj":{"rrname":"API.Example.COM.","rrtype":"A"}}\n',
+            b'{"obj":{"rrname":"example.com.","rrtype":"NS"}}\n',
+            b'{"obj":{"rrname":"*.wild.example.com.","rrtype":"A"}}\n',
+            b'{"obj":{"rrname":"outside.test.","rrtype":"A"}}\n',
+            b'{"cond":"succeeded"}\n',
+        ),
+    )
+
+    search = dnsdb.SearchDNSDB(' Example.COM. ')
+    await search.process()
+
+    assert await search.get_hostnames() == {'api.example.com'}
+    assert requested['url'] == 'https://api.dnsdb.info/dnsdb/v2/lookup/rrset/name/*.example.com?limit=0'
+    assert requested['session']['headers'] == {
+        'Accept': 'application/x-ndjson',
+        'User-Agent': f'theHarvester/{dnsdb.__version__}',
+        'X-API-Key': 'dnsdb-test-key',
+    }
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ('last_line', 'expected_message'),
+    [
+        (b'{"cond":"limited"}\n', 'ended with limited'),
+        (b'not-json\n', 'malformed NDJSON'),
+    ],
+)
+async def test_process_preserves_partial_results(
+    monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+    last_line: bytes,
+    expected_message: str,
+) -> None:
+    caplog.set_level(logging.INFO, logger=dnsdb.__name__)
+    monkeypatch.setattr(dnsdb.Core, 'dnsdb_key', lambda: 'dnsdb-test-key')
+    _install_response(
+        monkeypatch,
+        (
+            b'{"cond":"begin"}\n',
+            b'{"obj":{"rrname":"first.example.com."}}\n',
+            last_line,
+        ),
+    )
+
+    search = dnsdb.SearchDNSDB('example.com')
+    await search.process()
+
+    assert await search.get_hostnames() == {'first.example.com'}
+    assert any(expected_message in message for message in caplog.messages)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ('status', 'expected_error'),
+    [
+        (401, PermissionError),
+        (429, ConnectionError),
+        (503, ConnectionError),
+    ],
+)
+async def test_process_exposes_http_failures(
+    monkeypatch: pytest.MonkeyPatch,
+    status: int,
+    expected_error: type[Exception],
+) -> None:
+    monkeypatch.setattr(dnsdb.Core, 'dnsdb_key', lambda: 'dnsdb-test-key')
+    _install_response(monkeypatch, (), status=status)
+
+    with pytest.raises(expected_error):
+        await dnsdb.SearchDNSDB('example.com').process()

tests/test_readme.py

@@ -120,8 +120,8 @@ def test_readme_matches_executable_source_contracts() -> None:
     documented = _documented_source_contracts(readme)
     executable = _executable_source_contracts()
 
-    assert len(executable) == 55
-    assert len(documented) == 55
+    assert len(executable) == 56
+    assert len(documented) == 56
     assert documented == executable
     assert {'securitytrails', 'shodaninternetdb'}.isdisjoint(documented)
 

theHarvester/__main__.py

@@ -29,6 +29,7 @@ from theHarvester.discovery import (
     commoncrawl,
     criminalip,
     crtsh,
+    dnsdb,
     dnssearch,
     duckduckgosearch,
     dymosearch,
@@ -652,6 +653,16 @@ async def start(rest_args: argparse.Namespace | None = None):
                         else:
                             show_default_error_message(engineitem, word, e)
 
+                elif engineitem == 'dnsdb':
+                    try:
+                        dnsdb_search = dnsdb.SearchDNSDB(word)
+                        stor_lst.append(store(dnsdb_search, engineitem, store_host=True))
+                    except MissingKey as e:
+                        if not args.quiet:
+                            output_logger.info(e)
+                    except Exception as e:
+                        show_default_error_message(engineitem, word, e)
+
                 elif engineitem == 'dnsdumpster':
                     try:
                         dnsdumpster_search = search_dnsdumpster.SearchDNSDumpster(word)

theHarvester/data/api-keys.yaml

@@ -25,6 +25,9 @@ apikeys:
   dehashed:
     key:
 
+  dnsdb:
+    key:
+
   dnsdumpster:
     key:
 

theHarvester/discovery/dnsdb.py

@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+import json
+import logging
+from urllib.parse import quote
+
+import aiohttp
+
+from theHarvester import __version__
+from theHarvester.discovery.constants import MissingKey
+from theHarvester.lib.core import Core
+
+logger = logging.getLogger(__name__)
+
+
+class SearchDNSDB:
+    """Collect RRset owner names from the DNSDB v2 streaming API.
+
+    API docs: https://docs.domaintools.com/api/dnsdb/lookups/rrset-lookups/
+    Streaming protocol: https://docs.domaintools.com/api/dnsdb/streaming-protocol/
+    """
+
+    BASE_URL = 'https://api.dnsdb.info/dnsdb/v2/lookup/rrset/name'
+
+    def __init__(self, target_domain: str) -> None:
+        self.target_domain = target_domain.strip().lower().rstrip('.').encode('idna').decode('ascii')
+        key = Core.dnsdb_key()
+        if not isinstance(key, str) or not key.strip():
+            raise MissingKey('dnsdb')
+        self.key = key
+        self.totalhosts: set[str] = set()
+        self.proxy: bool | str = False
+
+    def _hostname(self, value: object) -> str | None:
+        if not isinstance(value, str):
+            return None
+        value = value.strip().rstrip('.')
+        if value.startswith('_WILDCARD_.') or '*' in value or any(character.isspace() for character in value):
+            return None
+        try:
+            hostname = value.lower().encode('idna').decode('ascii')
+        except UnicodeError:
+            return None
+        if hostname != self.target_domain and hostname.endswith(f'.{self.target_domain}'):
+            return hostname
+        return None
+
+    async def do_search(self) -> None:
+        query = quote(f'*.{self.target_domain}', safe='*.')
+        url = f'{self.BASE_URL}/{query}?limit=0'
+        headers = {
+            'Accept': 'application/x-ndjson',
+            'User-Agent': f'theHarvester/{__version__}',
+            'X-API-Key': self.key,
+        }
+        timeout = aiohttp.ClientTimeout(total=120)
+        proxy_url = self.proxy if isinstance(self.proxy, str) and self.proxy else None
+
+        async with aiohttp.ClientSession(headers=headers, timeout=timeout) as session:
+            async with session.get(url, proxy=proxy_url) as response:
+                if response.status == 429:
+                    raise ConnectionError('DNSDB rate limit reached')
+                if response.status in {401, 403}:
+                    raise PermissionError('DNSDB authentication failed')
+                if response.status == 503:
+                    raise ConnectionError('DNSDB concurrent connection limit exceeded')
+                if response.status != 200:
+                    raise ConnectionError(f'DNSDB returned HTTP {response.status}')
+
+                first_record = True
+                async for line in response.content:
+                    try:
+                        record = json.loads(line)
+                    except (UnicodeDecodeError, json.JSONDecodeError):
+                        logger.info('DNSDB returned malformed NDJSON; partial results were preserved.')
+                        return
+                    if not isinstance(record, dict):
+                        logger.info('DNSDB returned an invalid stream record; partial results were preserved.')
+                        return
+                    if first_record:
+                        first_record = False
+                        if record.get('cond') != 'begin':
+                            logger.info('DNSDB stream did not begin correctly; no results were accepted.')
+                            return
+                        continue
+
+                    condition = record.get('cond')
+                    if condition in {'succeeded', 'limited', 'failed'}:
+                        if condition != 'succeeded':
+                            logger.info(f'DNSDB stream ended with {condition}; partial results were preserved.')
+                        return
+                    obj = record.get('obj')
+                    if isinstance(obj, dict) and (hostname := self._hostname(obj.get('rrname'))):
+                        self.totalhosts.add(hostname)
+
+                logger.info('DNSDB stream ended without a terminal condition; partial results were preserved.')
+
+    async def get_hostnames(self) -> set[str]:
+        return self.totalhosts
+
+    async def process(self, proxy: bool | str = False) -> None:
+        self.proxy = proxy
+        await self.do_search()

theHarvester/lib/core.py

@@ -43,6 +43,7 @@ class Core:
         'censys': ('id', 'secret'),
         'criminalip': ('key',),
         'dehashed': ('key',),
+        'dnsdb': ('key',),
         'dnsdumpster': ('key',),
         'dymo': ('key',),
         'fofa': ('key', 'email'),
@@ -136,6 +137,10 @@ class Core:
     def dehashed_key() -> str:
         return Core._api_key_value('dehashed')
 
+    @staticmethod
+    def dnsdb_key() -> str:
+        return Core._api_key_value('dnsdb')
+
     @staticmethod
     def dnsdumpster_key() -> str:
         return Core._api_key_value('dnsdumpster')
@@ -294,6 +299,7 @@ class Core:
             'criminalip',
             'crtsh',
             'dehashed',
+            'dnsdb',
             'dnsdumpster',
             'duckduckgo',
             'dymo',