diff --git a/README.md b/README.md index e22d06c7..5e7fcfb7 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,7 @@ Read the **API key** column as follows: | `criminalip` | ✓ | No | ✓ | ✓ | No | No | No | ✓ | | `crtsh` | ✓ | No | No | No | No | No | No | No | | `dehashed` | No | No | ✓ | No | No | No | No | ✓ | +| `dnsdb` | ✓ | No | No | No | No | No | No | ✓ | | `dnsdumpster` | ✓ | No | ✓ | No | No | No | No | ✓ | | `duckduckgo` | ✓ | ✓ | No | No | No | No | No | No | | `dymo` | ✓ | No | No | No | No | No | No | ✓ | diff --git a/tests/discovery/test_dnsdb.py b/tests/discovery/test_dnsdb.py new file mode 100644 index 00000000..8149f949 --- /dev/null +++ b/tests/discovery/test_dnsdb.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import logging + +import pytest + +from theHarvester.discovery import dnsdb +from theHarvester.discovery.constants import MissingKey + + +def _install_response( + monkeypatch: pytest.MonkeyPatch, + lines: tuple[bytes, ...], + *, + status: int = 200, +) -> dict[str, object]: + requested: dict[str, object] = {} + lines_left = list(lines) + + class FakeContent: + def __aiter__(self) -> FakeContent: + return self + + async def __anext__(self) -> bytes: + if not lines_left: + raise StopAsyncIteration + return lines_left.pop(0) + + class FakeResponse: + def __init__(self) -> None: + self.status = status + + content = FakeContent() + + async def __aenter__(self) -> FakeResponse: + return self + + async def __aexit__(self, *_args: object) -> None: + return None + + class FakeSession: + def __init__(self, **kwargs: object) -> None: + requested['session'] = kwargs + + def get(self, url: str, **kwargs: object) -> FakeResponse: + requested['url'] = url + requested['request'] = kwargs + return FakeResponse() + + async def __aenter__(self) -> FakeSession: + return self + + async def __aexit__(self, *_args: object) -> None: + return None + + monkeypatch.setattr(dnsdb.aiohttp, 'ClientSession', FakeSession) + return requested + + +def test_blank_key_is_missing(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(dnsdb.Core, 'dnsdb_key', lambda: ' ') + + with pytest.raises(MissingKey): + dnsdb.SearchDNSDB('example.com') + + +@pytest.mark.asyncio +async def test_process_collects_normalized_in_scope_rrset_owners(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(dnsdb.Core, 'dnsdb_key', lambda: 'dnsdb-test-key') + requested = _install_response( + monkeypatch, + ( + b'{"cond":"begin"}\n', + b'{"obj":{"rrname":"API.Example.COM.","rrtype":"A"}}\n', + b'{"obj":{"rrname":"example.com.","rrtype":"NS"}}\n', + b'{"obj":{"rrname":"*.wild.example.com.","rrtype":"A"}}\n', + b'{"obj":{"rrname":"outside.test.","rrtype":"A"}}\n', + b'{"cond":"succeeded"}\n', + ), + ) + + search = dnsdb.SearchDNSDB(' Example.COM. ') + await search.process() + + assert await search.get_hostnames() == {'api.example.com'} + assert requested['url'] == 'https://api.dnsdb.info/dnsdb/v2/lookup/rrset/name/*.example.com?limit=0' + assert requested['session']['headers'] == { + 'Accept': 'application/x-ndjson', + 'User-Agent': f'theHarvester/{dnsdb.__version__}', + 'X-API-Key': 'dnsdb-test-key', + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ('last_line', 'expected_message'), + [ + (b'{"cond":"limited"}\n', 'ended with limited'), + (b'not-json\n', 'malformed NDJSON'), + ], +) +async def test_process_preserves_partial_results( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + last_line: bytes, + expected_message: str, +) -> None: + caplog.set_level(logging.INFO, logger=dnsdb.__name__) + monkeypatch.setattr(dnsdb.Core, 'dnsdb_key', lambda: 'dnsdb-test-key') + _install_response( + monkeypatch, + ( + b'{"cond":"begin"}\n', + b'{"obj":{"rrname":"first.example.com."}}\n', + last_line, + ), + ) + + search = dnsdb.SearchDNSDB('example.com') + await search.process() + + assert await search.get_hostnames() == {'first.example.com'} + assert any(expected_message in message for message in caplog.messages) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ('status', 'expected_error'), + [ + (401, PermissionError), + (429, ConnectionError), + (503, ConnectionError), + ], +) +async def test_process_exposes_http_failures( + monkeypatch: pytest.MonkeyPatch, + status: int, + expected_error: type[Exception], +) -> None: + monkeypatch.setattr(dnsdb.Core, 'dnsdb_key', lambda: 'dnsdb-test-key') + _install_response(monkeypatch, (), status=status) + + with pytest.raises(expected_error): + await dnsdb.SearchDNSDB('example.com').process() diff --git a/tests/test_readme.py b/tests/test_readme.py index a4b31546..9251d00a 100644 --- a/tests/test_readme.py +++ b/tests/test_readme.py @@ -120,8 +120,8 @@ def test_readme_matches_executable_source_contracts() -> None: documented = _documented_source_contracts(readme) executable = _executable_source_contracts() - assert len(executable) == 55 - assert len(documented) == 55 + assert len(executable) == 56 + assert len(documented) == 56 assert documented == executable assert {'securitytrails', 'shodaninternetdb'}.isdisjoint(documented) diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 1343a7d5..ccaddbbe 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -29,6 +29,7 @@ from theHarvester.discovery import ( commoncrawl, criminalip, crtsh, + dnsdb, dnssearch, duckduckgosearch, dymosearch, @@ -652,6 +653,16 @@ async def start(rest_args: argparse.Namespace | None = None): else: show_default_error_message(engineitem, word, e) + elif engineitem == 'dnsdb': + try: + dnsdb_search = dnsdb.SearchDNSDB(word) + stor_lst.append(store(dnsdb_search, engineitem, store_host=True)) + except MissingKey as e: + if not args.quiet: + output_logger.info(e) + except Exception as e: + show_default_error_message(engineitem, word, e) + elif engineitem == 'dnsdumpster': try: dnsdumpster_search = search_dnsdumpster.SearchDNSDumpster(word) diff --git a/theHarvester/data/api-keys.yaml b/theHarvester/data/api-keys.yaml index 1f9f0f31..189a60d1 100644 --- a/theHarvester/data/api-keys.yaml +++ b/theHarvester/data/api-keys.yaml @@ -25,6 +25,9 @@ apikeys: dehashed: key: + dnsdb: + key: + dnsdumpster: key: diff --git a/theHarvester/discovery/dnsdb.py b/theHarvester/discovery/dnsdb.py new file mode 100644 index 00000000..62607649 --- /dev/null +++ b/theHarvester/discovery/dnsdb.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import json +import logging +from urllib.parse import quote + +import aiohttp + +from theHarvester import __version__ +from theHarvester.discovery.constants import MissingKey +from theHarvester.lib.core import Core + +logger = logging.getLogger(__name__) + + +class SearchDNSDB: + """Collect RRset owner names from the DNSDB v2 streaming API. + + API docs: https://docs.domaintools.com/api/dnsdb/lookups/rrset-lookups/ + Streaming protocol: https://docs.domaintools.com/api/dnsdb/streaming-protocol/ + """ + + BASE_URL = 'https://api.dnsdb.info/dnsdb/v2/lookup/rrset/name' + + def __init__(self, target_domain: str) -> None: + self.target_domain = target_domain.strip().lower().rstrip('.').encode('idna').decode('ascii') + key = Core.dnsdb_key() + if not isinstance(key, str) or not key.strip(): + raise MissingKey('dnsdb') + self.key = key + self.totalhosts: set[str] = set() + self.proxy: bool | str = False + + def _hostname(self, value: object) -> str | None: + if not isinstance(value, str): + return None + value = value.strip().rstrip('.') + if value.startswith('_WILDCARD_.') or '*' in value or any(character.isspace() for character in value): + return None + try: + hostname = value.lower().encode('idna').decode('ascii') + except UnicodeError: + return None + if hostname != self.target_domain and hostname.endswith(f'.{self.target_domain}'): + return hostname + return None + + async def do_search(self) -> None: + query = quote(f'*.{self.target_domain}', safe='*.') + url = f'{self.BASE_URL}/{query}?limit=0' + headers = { + 'Accept': 'application/x-ndjson', + 'User-Agent': f'theHarvester/{__version__}', + 'X-API-Key': self.key, + } + timeout = aiohttp.ClientTimeout(total=120) + proxy_url = self.proxy if isinstance(self.proxy, str) and self.proxy else None + + async with aiohttp.ClientSession(headers=headers, timeout=timeout) as session: + async with session.get(url, proxy=proxy_url) as response: + if response.status == 429: + raise ConnectionError('DNSDB rate limit reached') + if response.status in {401, 403}: + raise PermissionError('DNSDB authentication failed') + if response.status == 503: + raise ConnectionError('DNSDB concurrent connection limit exceeded') + if response.status != 200: + raise ConnectionError(f'DNSDB returned HTTP {response.status}') + + first_record = True + async for line in response.content: + try: + record = json.loads(line) + except (UnicodeDecodeError, json.JSONDecodeError): + logger.info('DNSDB returned malformed NDJSON; partial results were preserved.') + return + if not isinstance(record, dict): + logger.info('DNSDB returned an invalid stream record; partial results were preserved.') + return + if first_record: + first_record = False + if record.get('cond') != 'begin': + logger.info('DNSDB stream did not begin correctly; no results were accepted.') + return + continue + + condition = record.get('cond') + if condition in {'succeeded', 'limited', 'failed'}: + if condition != 'succeeded': + logger.info(f'DNSDB stream ended with {condition}; partial results were preserved.') + return + obj = record.get('obj') + if isinstance(obj, dict) and (hostname := self._hostname(obj.get('rrname'))): + self.totalhosts.add(hostname) + + logger.info('DNSDB stream ended without a terminal condition; partial results were preserved.') + + async def get_hostnames(self) -> set[str]: + return self.totalhosts + + async def process(self, proxy: bool | str = False) -> None: + self.proxy = proxy + await self.do_search() diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index 0e9a00a0..9939c69f 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -43,6 +43,7 @@ class Core: 'censys': ('id', 'secret'), 'criminalip': ('key',), 'dehashed': ('key',), + 'dnsdb': ('key',), 'dnsdumpster': ('key',), 'dymo': ('key',), 'fofa': ('key', 'email'), @@ -136,6 +137,10 @@ class Core: def dehashed_key() -> str: return Core._api_key_value('dehashed') + @staticmethod + def dnsdb_key() -> str: + return Core._api_key_value('dnsdb') + @staticmethod def dnsdumpster_key() -> str: return Core._api_key_value('dnsdumpster') @@ -294,6 +299,7 @@ class Core: 'criminalip', 'crtsh', 'dehashed', + 'dnsdb', 'dnsdumpster', 'duckduckgo', 'dymo',