diff --git a/CHANGELOG.md b/CHANGELOG.md index 26fa775f..b90ca292 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Removed the nonfunctional ThreatCrowd source because its service hostnames terminate at deleted AWS load balancers and return NXDOMAIN; OTX remains available through its separate adapter. ### Fixed +- Made DeHashed pagination honor the CLI limit, retain only normalized email and IP evidence, and discard raw breach rows; aligned LeakIX with its authenticated subdomain endpoint and documented rate-limit retry. - Added offline contracts for explicitly selected DNS and direct sources, retained normalized Pentest-Tools host and IP results, and hardened Shodan InternetDB, SubdomainFinder C99, and Windvane evidence boundaries. - Retained relevant GitLab project, profile, and website URLs in consolidated JSONL and SQLite results while excluding unrelated user URLs. - Removed BuiltWith's duplicate interesting-URL getter by allowing the shared collector to use either established getter spelling. diff --git a/README.md b/README.md index dc8ffcb3..5127ee5d 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ Open [http://127.0.0.1:5000/docs](http://127.0.0.1:5000/docs) for interactive Sw The service rate limit defaults to five requests per minute and can be changed with `--rate-limit`. The `/additional/*` routes require `THEHARVESTER_API_KEY` on the server and the same value in the `X-API-Key` request header. -The core `/query`, `/sources`, and `/dnsbrute` routes do not normally require authentication. When a `/query` selection includes `hibpverified` and its provider key is configured, the request requires `THEHARVESTER_API_KEY` in the `X-API-Key` header because it can access verified-domain account data. Keep the service bound to localhost. If you require remote access, add authentication, access controls, and TLS. +The core `/query`, `/sources`, and `/dnsbrute` routes do not normally require authentication. When a `/query` selection includes `dehashed`, `hibpverified`, or `leaklookup` and that source's provider key is configured, the request requires `THEHARVESTER_API_KEY` in the `X-API-Key` header because these sources can access breach-account data. Keep the service bound to localhost. If you require remote access, add authentication, access controls, and TLS. Docker Compose publishes port `5000` on every host interface unless you narrow the port mapping: @@ -146,7 +146,7 @@ Read the **API key** column as follows: | `commoncrawl` | ✓ | No | No | No | No | No | No | No | No | | `criminalip` | ✓ | No | ✓ | ✓ | No | No | No | No | ✓ | | `crtsh` | ✓ | No | No | No | No | No | No | No | No | -| `dehashed` | No | No | ✓ | No | No | No | No | No | ✓ | +| `dehashed` | No | ✓ | ✓ | No | No | No | No | No | ✓ | | `dnsdb` | ✓ | No | No | No | No | No | No | No | ✓ | | `dnsdumpster` | ✓ | No | ✓ | No | No | No | No | No | ✓ | | `duckduckgo` | ✓ | ✓ | No | No | No | No | No | No | No | @@ -162,7 +162,7 @@ Read the **API key** column as follows: | `hunter` | ✓ | ✓ | No | No | No | No | No | No | ✓ | | `hunterhow` | ✓ | No | No | No | No | No | No | No | ✓ | | `intelx` | ✓ | ✓ | No | No | ✓ | No | No | No | ✓ | -| `leakix` | ✓ | ✓ | No | No | No | No | No | No | Optional | +| `leakix` | ✓ | No | No | No | No | No | No | No | ✓ | | `leaklookup` | No | ✓ | No | No | No | No | ✓ | `POST /additional/leaks` response | ✓ | | `mojeek` | ✓ | ✓ | No | No | No | No | No | No | Optional | | `netlas` | ✓ | No | No | No | No | No | No | No | ✓ | diff --git a/tests/discovery/test_leakix.py b/tests/discovery/test_leakix.py index f63020e7..8fda2655 100644 --- a/tests/discovery/test_leakix.py +++ b/tests/discovery/test_leakix.py @@ -3,20 +3,124 @@ import logging import pytest from theHarvester.discovery import leakix +from theHarvester.discovery.constants import MissingKey +from theHarvester.lib.core import FetcherResponse + + +@pytest.mark.parametrize('key', [None, '', ' ']) +def test_missing_or_blank_key_fails_before_network(monkeypatch, key) -> None: + monkeypatch.setattr(leakix.Core, 'leakix_key', lambda: key) + + with pytest.raises(MissingKey): + leakix.SearchLeakix('example.com') @pytest.mark.asyncio -async def test_authentication_response_body_is_not_logged(monkeypatch, caplog) -> None: - monkeypatch.setattr(leakix.Core, 'leakix_key', lambda: None) +async def test_process_uses_documented_endpoint_and_normalizes_only_scoped_subdomains(monkeypatch) -> None: + monkeypatch.setattr(leakix.Core, 'leakix_key', lambda: 'test-key') + monkeypatch.setattr(leakix.Core, 'get_user_agent', lambda: 'test-agent') + requests = [] + + async def fake_fetch_all(urls, **kwargs): + requests.append((urls, kwargs)) + return [ + FetcherResponse( + body=[ + {'subdomain': 'API.Example.COM.'}, + {'subdomain': 'www.example.com'}, + {'subdomain': 'example.com.attacker.test'}, + {'subdomain': 7}, + {'hostname': 'undocumented.example.com'}, + ], + status=200, + headers={}, + ) + ] + + monkeypatch.setattr(leakix.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = leakix.SearchLeakix('example.com') + + await search.process(proxy=True) + + assert requests == [ + ( + ['https://leakix.net/api/subdomains/example.com'], + { + 'headers': {'User-Agent': 'test-agent', 'accept': 'application/json', 'api-key': 'test-key'}, + 'json': True, + 'proxy': True, + 'include_metadata': True, + }, + ) + ] + assert await search.get_hostnames() == {'api.example.com', 'www.example.com'} + assert await search.get_emails() == set() + + +@pytest.mark.asyncio +async def test_rate_limit_waits_for_provider_delay_and_retries_once(monkeypatch, caplog) -> None: + monkeypatch.setattr(leakix.Core, 'leakix_key', lambda: 'test-key') + monkeypatch.setattr(leakix.Core, 'get_user_agent', lambda: 'test-agent') + responses = iter( + [ + FetcherResponse( + body='provider-secret-limit-detail', + status=429, + headers={'x-limited-for': '0ms'}, + ), + FetcherResponse(body=[{'subdomain': 'api.example.com'}], status=200, headers={}), + ] + ) + calls = [] + sleeps = [] + + async def fake_fetch_all(*args, **kwargs): + calls.append((args, kwargs)) + return [next(responses)] + + async def fake_sleep(delay): + sleeps.append(delay) + + monkeypatch.setattr(leakix.AsyncFetcher, 'fetch_all', fake_fetch_all) + monkeypatch.setattr(leakix.asyncio, 'sleep', fake_sleep) + caplog.set_level(logging.INFO, logger=leakix.__name__) + search = leakix.SearchLeakix('example.com') + + await search.process() + + assert len(calls) == 2 + assert sleeps == [0.0] + assert await search.get_hostnames() == {'api.example.com'} + assert 'provider-secret-limit-detail' not in caplog.text + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ('response', 'expected_log'), + [ + (FetcherResponse(body='provider-secret-auth-detail', status=401, headers={}), 'HTTP 401'), + (FetcherResponse(body='provider-secret-auth-detail', status=403, headers={}), 'HTTP 403'), + (FetcherResponse(body={'subdomain': 'api.example.com'}, status=200, headers={}), 'malformed'), + (FetcherResponse(body=[], status=200, headers={}), None), + (None, 'request failed'), + ], +) +async def test_unusable_responses_fail_closed_without_logging_provider_detail( + monkeypatch, caplog, response, expected_log +) -> None: + monkeypatch.setattr(leakix.Core, 'leakix_key', lambda: 'test-key') monkeypatch.setattr(leakix.Core, 'get_user_agent', lambda: 'test-agent') async def fake_fetch_all(*args, **kwargs): - return ['Incorrect API Key: provider-secret-payload'] + return [response] monkeypatch.setattr(leakix.AsyncFetcher, 'fetch_all', fake_fetch_all) caplog.set_level(logging.INFO, logger=leakix.__name__) + search = leakix.SearchLeakix('example.com') - await leakix.SearchLeakix('example.com').process() + await search.process() - assert 'provider-secret-payload' not in caplog.text - assert 'requires authentication' in caplog.text + assert await search.get_hostnames() == set() + assert 'provider-secret-auth-detail' not in caplog.text + if expected_log is not None: + assert expected_log in caplog.text diff --git a/tests/discovery/test_rapiddns.py b/tests/discovery/test_rapiddns.py index 597bae1e..f6e35a81 100644 --- a/tests/discovery/test_rapiddns.py +++ b/tests/discovery/test_rapiddns.py @@ -137,12 +137,15 @@ async def test_rapiddns_evidence_reaches_existing_outputs( raise AssertionError('DNS resolution requires the explicit --dns-resolve flag') class FakeDehashed: - def __init__(self, _domain: str) -> None: - pass + def __init__(self, _domain: str, limit: int) -> None: + assert limit == 500 async def process(self, _proxy: bool) -> None: return None + async def get_emails(self) -> set[str]: + return {'user@example.com'} + async def get_ips(self) -> set[str]: return {'198.51.100.2'} @@ -304,8 +307,10 @@ async def test_rapiddns_evidence_reaches_existing_outputs( ) assert rest_results == legacy_rest_results assert set(rest_results[6]) == {'192.0.2.1', '198.51.100.2'} + assert rest_results[7] == ['user@example.com'] assert rest_results[8] == ['alias.example.com', 'api.example.com', 'broken.example.com'] assert stored[stored_before_rest:] == [ + ('email', ('user@example.com',), 'dehashed'), ('ip', ('198.51.100.2',), 'dehashed'), ('host', ('alias.example.com', 'api.example.com', 'broken.example.com'), 'rapiddns'), ('ip', ('192.0.2.1',), 'rapiddns'), @@ -314,6 +319,7 @@ async def test_rapiddns_evidence_reaches_existing_outputs( assert FakeSecurityScorecard.created == 1 assert completed_results[1].target == 'example.com' assert {'192.0.2.1', '198.51.100.2'} <= {value for kind, value in completed_results[1].results if kind == 'ip-address'} + assert ('email', 'user@example.com') in completed_results[1].results monkeypatch.setattr(sys, 'argv', ['theHarvester', '-d', 'example.com', '-b', 'rapiddns']) with pytest.raises(SystemExit) as no_file_exit: diff --git a/tests/discovery/test_search_dehashed.py b/tests/discovery/test_search_dehashed.py index f58ea03f..8fef644b 100644 --- a/tests/discovery/test_search_dehashed.py +++ b/tests/discovery/test_search_dehashed.py @@ -3,63 +3,142 @@ import logging import pytest from theHarvester.discovery import search_dehashed +from theHarvester.discovery.constants import MissingKey from theHarvester.discovery.search_dehashed import SearchDehashed +from theHarvester.lib.core import FetcherResponse + + +@pytest.mark.parametrize('key', [None, '', ' ']) +def test_missing_or_blank_key_fails_before_network(monkeypatch, key) -> None: + monkeypatch.setattr(search_dehashed.Core, 'dehashed_key', lambda: key) + + with pytest.raises(MissingKey): + SearchDehashed('example.com') @pytest.mark.asyncio -async def test_process_does_not_output_credentials(monkeypatch, capsys) -> None: - search = SearchDehashed.__new__(SearchDehashed) - search.data = [{'email': 'user@example.com', 'password': 'secret-password'}] +async def test_process_honors_limit_and_retains_only_normalized_evidence(monkeypatch) -> None: + monkeypatch.setattr(search_dehashed.Core, 'dehashed_key', lambda: 'test-key') + monkeypatch.setattr(search_dehashed.Core, 'get_user_agent', lambda: 'test-agent') + payloads = [] + first_page = [ + { + 'email': ' User@Example.COM ', + 'ip_address': '192.0.2.1', + 'password': 'provider-secret-password', + 'hashed_password': 'provider-secret-hash', + } + for _ in range(100) + ] + second_page = [ + {'email': 'admin@example.com', 'ip_address': '2001:0db8::1', 'database_name': 'private-breach'} for _ in range(20) + ] + responses = iter( + [ + FetcherResponse(body={'entries': first_page}, status=200, headers={}), + FetcherResponse(body={'entries': second_page}, status=200, headers={}), + ] + ) - async def do_search() -> None: - return None + async def fake_post_fetch(url, **kwargs): + payloads.append((url, kwargs)) + return next(responses) - monkeypatch.setattr(search, 'do_search', do_search) - await search.process() + monkeypatch.setattr(search_dehashed.AsyncFetcher, 'post_fetch', fake_post_fetch) + search = SearchDehashed('example.com', limit=120) - assert 'secret-password' not in capsys.readouterr().out - assert await search.get_emails() == {'user@example.com'} + await search.process(proxy='http://proxy.example:8080') + + assert [request['json_body']['size'] for _, request in payloads] == [100, 20] + assert all(request['proxy'] == 'http://proxy.example:8080' for _, request in payloads) + assert all(request['include_metadata'] is True for _, request in payloads) + assert await search.get_emails() == {'user@example.com', 'admin@example.com'} + assert await search.get_ips() == {'192.0.2.1', '2001:db8::1'} + assert 'provider-secret-password' not in repr(vars(search)) + assert 'provider-secret-hash' not in repr(vars(search)) + assert 'private-breach' not in repr(vars(search)) @pytest.mark.asyncio async def test_non_json_response_body_is_not_logged(monkeypatch, caplog) -> None: - class Response: - status = 200 + async def fake_post_fetch(*args, **kwargs): + return FetcherResponse(body='provider-secret-payload', status=200, headers={}) - async def __aenter__(self): - return self - - async def __aexit__(self, *args): - return None - - async def json(self): - raise ValueError - - async def text(self): - return 'provider-secret-payload' - - class Session: - def __init__(self, **kwargs): - pass - - async def __aenter__(self): - return self - - async def __aexit__(self, *args): - return None - - def post(self, *args, **kwargs): - return Response() - - monkeypatch.setattr(search_dehashed.aiohttp, 'ClientSession', Session) + monkeypatch.setattr(search_dehashed.Core, 'dehashed_key', lambda: 'test-key') + monkeypatch.setattr(search_dehashed.Core, 'get_user_agent', lambda: 'test-agent') + monkeypatch.setattr(search_dehashed.AsyncFetcher, 'post_fetch', fake_post_fetch) caplog.set_level(logging.INFO, logger=search_dehashed.__name__) - search = SearchDehashed.__new__(SearchDehashed) - search.word = 'example.com' - search.api = 'https://provider.example' - search.headers = {} - search.proxy = False - search.data = [] + search = SearchDehashed('example.com') await search.do_search() assert 'provider-secret-payload' not in caplog.text + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ('response', 'expected_log'), + [ + (FetcherResponse(body='provider-secret-auth-detail', status=401, headers={}), 'HTTP 401'), + (FetcherResponse(body='provider-secret-auth-detail', status=403, headers={}), 'HTTP 403'), + (FetcherResponse(body={'entries': []}, status=200, headers={}), None), + ], +) +async def test_authorization_and_empty_responses_fail_closed_without_provider_detail( + monkeypatch, caplog, response, expected_log +) -> None: + monkeypatch.setattr(search_dehashed.Core, 'dehashed_key', lambda: 'test-key') + monkeypatch.setattr(search_dehashed.Core, 'get_user_agent', lambda: 'test-agent') + + async def fake_post_fetch(*args, **kwargs): + return response + + monkeypatch.setattr(search_dehashed.AsyncFetcher, 'post_fetch', fake_post_fetch) + caplog.set_level(logging.INFO, logger=search_dehashed.__name__) + search = SearchDehashed('example.com') + + await search.process() + + assert await search.get_emails() == set() + assert await search.get_ips() == set() + assert 'provider-secret-auth-detail' not in caplog.text + if expected_log is not None: + assert expected_log in caplog.text + + +@pytest.mark.asyncio +async def test_rate_limit_retries_once_and_preserves_earlier_page(monkeypatch, caplog) -> None: + monkeypatch.setattr(search_dehashed.Core, 'dehashed_key', lambda: 'test-key') + monkeypatch.setattr(search_dehashed.Core, 'get_user_agent', lambda: 'test-agent') + responses = iter( + [ + FetcherResponse( + body={'entries': [{'email': 'first@example.com'} for _ in range(100)]}, + status=200, + headers={}, + ), + FetcherResponse(body='provider-secret-limit-detail', status=429, headers={'retry-after': '0'}), + FetcherResponse(body={'entries': [{'email': 'second@example.com'}]}, status=200, headers={}), + ] + ) + payloads = [] + sleeps = [] + + async def fake_post_fetch(url, **kwargs): + payloads.append(kwargs['json_body']) + return next(responses) + + async def fake_sleep(delay): + sleeps.append(delay) + + monkeypatch.setattr(search_dehashed.AsyncFetcher, 'post_fetch', fake_post_fetch) + monkeypatch.setattr(search_dehashed.asyncio, 'sleep', fake_sleep) + caplog.set_level(logging.INFO, logger=search_dehashed.__name__) + search = SearchDehashed('example.com', limit=200) + + await search.process() + + assert [payload['page'] for payload in payloads] == [1, 2, 2] + assert sleeps == [0.0] + assert await search.get_emails() == {'first@example.com', 'second@example.com'} + assert 'provider-secret-limit-detail' not in caplog.text diff --git a/tests/lib/test_core.py b/tests/lib/test_core.py index 80e37976..95ceeb48 100644 --- a/tests/lib/test_core.py +++ b/tests/lib/test_core.py @@ -24,6 +24,7 @@ def test_email_capability_expands_to_email_sources() -> None: "baidu", "brave", "censys", + "dehashed", "duckduckgo", "github-code", "gitlab", @@ -31,7 +32,6 @@ def test_email_capability_expands_to_email_sources() -> None: "hudsonrock", "hunter", "intelx", - "leakix", "leaklookup", "mojeek", "rocketreach", diff --git a/tests/lib/test_source_catalog.py b/tests/lib/test_source_catalog.py index 9161a301..a913f1e9 100644 --- a/tests/lib/test_source_catalog.py +++ b/tests/lib/test_source_catalog.py @@ -55,9 +55,7 @@ def test_subdomain_route_drives_subdomain_capability() -> None: def test_source_specs_describe_consolidated_routes_not_getter_presence() -> None: - assert SOURCE_SPECS['gitlab'].routes == frozenset( - {ResultRoute.SUBDOMAINS, ResultRoute.EMAILS, ResultRoute.URLS} - ) + assert SOURCE_SPECS['gitlab'].routes == frozenset({ResultRoute.SUBDOMAINS, ResultRoute.EMAILS, ResultRoute.URLS}) assert SOURCE_SPECS['haveibeenpwned'].routes == frozenset({ResultRoute.BREACHES}) assert SOURCE_SPECS['hibpverified'].routes == frozenset({ResultRoute.EMAILS, ResultRoute.BREACHES}) assert SOURCE_SPECS['leaklookup'].routes == frozenset({ResultRoute.EMAILS, ResultRoute.BREACHES}) @@ -79,6 +77,14 @@ def test_pentesttools_declares_its_normalized_subdomain_and_ip_routes() -> None: assert SOURCE_SPECS['pentesttools'].routes == frozenset({ResultRoute.SUBDOMAINS, ResultRoute.IPS}) +def test_dehashed_declares_its_normalized_email_and_ip_routes() -> None: + assert SOURCE_SPECS['dehashed'].routes == frozenset({ResultRoute.EMAILS, ResultRoute.IPS}) + + +def test_leakix_declares_only_its_documented_subdomain_route() -> None: + assert SOURCE_SPECS['leakix'].routes == frozenset({ResultRoute.SUBDOMAINS}) + + def test_unavailable_venacus_source_is_not_selectable() -> None: assert 'venacus' not in Core.get_supportedengines() assert 'venacus' not in SOURCE_SPECS diff --git a/tests/test_readme.py b/tests/test_readme.py index f2bfd267..bde9409e 100644 --- a/tests/test_readme.py +++ b/tests/test_readme.py @@ -19,7 +19,7 @@ ROUTE_COLUMNS = { ResultRoute.PEOPLE: 'People', ResultRoute.BREACHES: 'Breaches', } -OPTIONAL_API_KEY_SOURCES = {'hackertarget', 'leakix', 'mojeek', 'windvane'} +OPTIONAL_API_KEY_SOURCES = {'hackertarget', 'mojeek', 'windvane'} API_KEY_SOURCE_ALIASES = { 'github': {'github-code'}, 'pentestTools': {'pentesttools'}, diff --git a/tests/test_rest_api.py b/tests/test_rest_api.py index c90cb9cd..203ae858 100644 --- a/tests/test_rest_api.py +++ b/tests/test_rest_api.py @@ -241,6 +241,43 @@ def test_query_requires_operator_auth_for_configured_leaklookup(monkeypatch) -> assert response.status_code == 401 +@pytest.mark.parametrize('source', ['dehashed', 'emails']) +def test_query_requires_operator_auth_for_configured_dehashed(monkeypatch, source) -> None: + async def unexpected_start(*_args, **_kwargs): + raise AssertionError('collection must not start without operator authentication') + + monkeypatch.setenv('THEHARVESTER_API_KEY', 'operator-secret') + monkeypatch.setattr(api.__main__.Core, 'dehashed_key', lambda: 'provider-secret') + monkeypatch.setattr(api.__main__, 'start', unexpected_start) + + response = TestClient(api.app).get(f'/query?domain=example.test&source={source}') + + assert response.status_code == 401 + + +@pytest.mark.parametrize('dehashed_key', [None, '', ' ']) +def test_query_skips_operator_auth_when_dehashed_key_is_blank(monkeypatch, dehashed_key) -> None: + captured: list[Namespace] = [] + + async def fake_start( + args: Namespace, + *, + persist_completed_result: bool = False, + include_breaches: bool = False, + ): + captured.append(args) + return ([], [], [], [], [], [], [], [], [], []) + + monkeypatch.delenv('THEHARVESTER_API_KEY', raising=False) + monkeypatch.setattr(api.__main__.Core, 'dehashed_key', lambda: dehashed_key) + monkeypatch.setattr(api.__main__, 'start', fake_start) + + response = TestClient(api.app).get('/query?domain=example.test&source=dehashed') + + assert response.status_code == 200 + assert captured[0].source == 'dehashed' + + @pytest.mark.parametrize('leaklookup_key', [None, '', ' ']) def test_query_skips_operator_auth_when_credentialed_provider_keys_are_blank(monkeypatch, leaklookup_key) -> None: captured: list[Namespace] = [] diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 35847fe9..bf9bf9ea 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -709,7 +709,7 @@ async def start( elif engineitem == 'dehashed': try: - dehashed_search = search_dehashed.SearchDehashed(word) + dehashed_search = search_dehashed.SearchDehashed(word, limit=limit) stor_lst.append( store( dehashed_search, @@ -909,6 +909,9 @@ async def start( engineitem, ) ) + except MissingKey as e: + if not args.quiet: + output_logger.info(e) except Exception as e: show_default_error_message(engineitem, word, e) diff --git a/theHarvester/discovery/leakix.py b/theHarvester/discovery/leakix.py index 136a03a0..46cdd041 100644 --- a/theHarvester/discovery/leakix.py +++ b/theHarvester/discovery/leakix.py @@ -1,117 +1,85 @@ -import json as _stdlib_json +import asyncio import logging -from types import ModuleType -from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.discovery.constants import MissingKey +from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse +from theHarvester.lib.hostnames import normalize_scoped_hostname logger = logging.getLogger(__name__) -json: ModuleType = _stdlib_json -try: - import ujson as _ujson - - json = _ujson -except ImportError: - pass -except Exception: - pass - class SearchLeakix: - """Class uses LeakIX API to search for domain leaks and subdomains - Note: LeakIX requires API key for most endpoints - """ + """Find subdomains through LeakIX's documented subdomain endpoint.""" - def __init__(self, word) -> None: + def __init__(self, word: str) -> None: self.word = word - self.totalhosts: set = set() - self.totalemails: set = set() + self.api_key = (Core.leakix_key() or '').strip() + if not self.api_key: + raise MissingKey('LeakIX') + self.totalhosts: set[str] = set() self.proxy = False - self.hostname = 'https://leakix.net' + self.url = f'https://leakix.net/api/subdomains/{word}' + + async def _fetch(self) -> FetcherResponse | None: + responses = await AsyncFetcher.fetch_all( + [self.url], + headers={ + 'User-Agent': Core.get_user_agent(), + 'accept': 'application/json', + 'api-key': self.api_key, + }, + json=True, + proxy=self.proxy, + include_metadata=True, + ) + response = responses[0] if responses else None + return response if isinstance(response, FetcherResponse) else None @staticmethod - def _safe_parse_json(payload: object) -> list: - # If already a list, return it; if string, try parse; else return [] - if isinstance(payload, list): - return payload - if isinstance(payload, str): - try: - result = json.loads(payload) - return result if isinstance(result, list) else [result] if isinstance(result, dict) else [] - except Exception: - return [] - return [] + def _limited_for_seconds(value: str | None) -> float | None: + if value is None: + return None + try: + delay = float(value[:-2]) / 1000 if value.endswith('ms') else float(value.removesuffix('s')) + except ValueError: + return None + return delay if 0 <= delay <= 60 else None async def do_search(self) -> None: try: - headers = { - 'User-agent': Core.get_user_agent(), - 'accept': 'application/json', - } + response = await self._fetch() + if response is not None and response.status == 429: + delay = self._limited_for_seconds(response.headers.get('x-limited-for')) + if delay is not None: + logger.info(f'LeakIX rate limited; retrying once in {delay:g} seconds') + await asyncio.sleep(delay) + response = await self._fetch() + except (OSError, RuntimeError, ValueError): + logger.info('LeakIX request failed') + return - # Add API key if available - api_key = Core.leakix_key() - if api_key: - headers['api-key'] = api_key + if response is None: + logger.info('LeakIX request failed') + return + if not 200 <= response.status < 300: + logger.info(f'LeakIX request failed with HTTP {response.status}') + return + if not isinstance(response.body, list): + logger.info('LeakIX returned a malformed response') + return - search_queries = [ - f'{self.hostname}/api/subdomains/{self.word}', - f'{self.hostname}/host/{self.word}', - ] + for item in response.body: + if not isinstance(item, dict): + continue + normalized = normalize_scoped_hostname(item.get('subdomain'), self.word) + if normalized: + self.totalhosts.add(normalized) - for query_url in search_queries: - try: - response = await AsyncFetcher.fetch_all([query_url], headers=headers, proxy=self.proxy) - - if not response or not isinstance(response, list) or not response[0]: - continue - - # Check if the response is an error message - if isinstance(response[0], str) and ( - 'Incorrect API Key' in response[0] - or 'unauthorized' in response[0].lower() - or 'error' in response[0].lower() - ): - logger.info('LeakIX API requires authentication') - continue - - try: - data = self._safe_parse_json(response[0]) - - for item in data: - if isinstance(item, dict): - # Extract hostnames from different fields - hostname = item.get('hostname', '') or item.get('host', '') or item.get('domain', '') - if hostname and (hostname.endswith(f'.{self.word}') or hostname == self.word): - self.totalhosts.add(hostname.lower()) - - # Extract emails if available - email = item.get('email', '') or item.get('username', '') - if email and '@' in email and self.word in email: - self.totalemails.add(email.lower()) - - # Check for subdomains in other fields - for field in ['subdomain', 'target', 'service_name']: - value = item.get(field, '') - if value and isinstance(value, str): - if value.endswith(f'.{self.word}') or value == self.word: - self.totalhosts.add(value.lower()) - - except Exception as e: - logger.info(f'Failed to parse LeakIX response: {e}') - - except Exception as e: - logger.info(f'LeakIX API error for {query_url}: {e}') - continue - - except Exception as e: - logger.info(f'LeakIX API error: {e}') - - async def get_hostnames(self) -> set: + async def get_hostnames(self) -> set[str]: return self.totalhosts - async def get_emails(self) -> set: - return self.totalemails + async def get_emails(self) -> set[str]: + return set() async def process(self, proxy: bool = False) -> None: self.proxy = proxy diff --git a/theHarvester/discovery/search_dehashed.py b/theHarvester/discovery/search_dehashed.py index 17da8461..77c42927 100644 --- a/theHarvester/discovery/search_dehashed.py +++ b/theHarvester/discovery/search_dehashed.py @@ -1,95 +1,114 @@ import asyncio import logging -import random - -import aiohttp +from ipaddress import ip_address +from typing import Any from theHarvester.discovery.constants import MissingKey -from theHarvester.lib.core import Core +from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse logger = logging.getLogger(__name__) class SearchDehashed: - def __init__(self, word) -> None: + def __init__(self, word: str, limit: int = 500) -> None: self.word = word - self.key = Core.dehashed_key() - if self.key is None: + self.key = (Core.dehashed_key() or '').strip() + if not self.key: raise MissingKey('Dehashed') - self.api = 'https://api.dehashed.com/v2/search' self.headers = { 'Dehashed-Api-Key': self.key, 'User-Agent': Core.get_user_agent(), } - self.results = '' - self.data: list[dict] = [] + self.limit = max(limit, 0) + self.emails: set[str] = set() + self.ips: set[str] = set() self.proxy: bool = False + async def _fetch_page(self, payload: dict[str, Any]) -> Any: + response = await AsyncFetcher.post_fetch( + self.api, + headers=self.headers, + json_body=payload, + proxy=self.proxy, + include_metadata=True, + ) + if isinstance(response, FetcherResponse) and response.status == 429: + retry_after = response.headers.get('retry-after') or response.headers.get('Retry-After') + try: + delay = float(retry_after) if retry_after is not None else -1 + except ValueError: + delay = -1 + if 0 <= delay <= 60: + logger.info(f'\t[!] Dehashed rate limited; retrying once in {delay:g} seconds') + await asyncio.sleep(delay) + response = await AsyncFetcher.post_fetch( + self.api, + headers=self.headers, + json_body=payload, + proxy=self.proxy, + include_metadata=True, + ) + return response + + def _retain_evidence(self, entries: list[object]) -> None: + for entry in entries: + if not isinstance(entry, dict): + continue + email = entry.get('email') + if isinstance(email, str): + normalized_email = email.strip().lower() + local, separator, domain = normalized_email.partition('@') + if local and separator and domain and '@' not in domain: + self.emails.add(normalized_email) + address = entry.get('ip_address') + if isinstance(address, str): + try: + self.ips.add(str(ip_address(address.strip()))) + except ValueError: + continue + async def do_search(self) -> None: logger.info(f'\t[+] Performing Dehashed search for: {self.word}') page = 1 - size = 100 - while True: + remaining = self.limit + while remaining > 0: + size = min(100, remaining) payload = {'query': self.word, 'page': page, 'size': size, 'wildcard': False, 'regex': False, 'de_dupe': False} - try: - # Resolve proxy URL if enabled - proxy_url = None - if isinstance(self.proxy, str) and self.proxy: - proxy_url = self.proxy - elif isinstance(self.proxy, bool) and self.proxy: - try: - proxies = Core.proxy_list() - if proxies: - proxy_url = str(random.choice(proxies)) - except Exception: - proxy_url = None - - timeout = aiohttp.ClientTimeout(total=120) - async with aiohttp.ClientSession(headers=self.headers, timeout=timeout) as session: - async with session.post(self.api, json=payload, proxy=proxy_url) as response: - if response.status == 401: - raise Exception('Unauthorized. Check Dehashed API key.') - if response.status == 403: - raise Exception('Forbidden. API key is not allowed.') - try: - data = await response.json() - except Exception: - raise ValueError('Unexpected response format') - - entries = data.get('entries', []) + response = await self._fetch_page(payload) + if not isinstance(response, FetcherResponse): + logger.info('\t[!] Dehashed request failed') + break + if not 200 <= response.status < 300: + logger.info(f'\t[!] Dehashed request failed with HTTP {response.status}') + break + data = response.body + if not isinstance(data, dict) or not isinstance(entries := data.get('entries'), list): + logger.info('\t[!] Dehashed returned a malformed response') + break if not entries: break - - self.data.extend(entries) - logger.info(f'\t[+] Page {page} - Retrieved {len(entries)} entries.') - + retained_entries = entries[:remaining] + self._retain_evidence(retained_entries) + remaining -= len(retained_entries) + logger.info(f'\t[+] Page {page} - Retrieved {len(retained_entries)} entries.') if len(entries) < size: break page += 1 - await asyncio.sleep(0.5) - except Exception as e: - logger.info(f'\t[!] Dehashed error: {e}') + except (OSError, RuntimeError, ValueError): + logger.info('\t[!] Dehashed request failed') break async def process(self, proxy: bool = False) -> None: self.proxy = proxy await self.do_search() - async def get_emails(self) -> set: - emails = set() - for entry in self.data: - if entry.get('email'): - emails.add(entry['email']) - return emails + async def get_emails(self) -> set[str]: + return self.emails - async def get_hostnames(self) -> set: + async def get_hostnames(self) -> set[str]: return set() - async def get_ips(self) -> set: - ips = set() - for entry in self.data: - if entry.get('ip_address'): - ips.add(entry['ip_address']) - return ips + async def get_ips(self) -> set[str]: + return self.ips diff --git a/theHarvester/lib/api/api.py b/theHarvester/lib/api/api.py index 3e3a5793..cfd048e2 100644 --- a/theHarvester/lib/api/api.py +++ b/theHarvester/lib/api/api.py @@ -401,10 +401,15 @@ async def query( try: # Validate sources selected_sources = __main__.Core.expand_source_selection(','.join(source)) - credentialed_breach_source = ( - 'hibpverified' in selected_sources and bool((__main__.Core.hibpverified_key() or '').strip()) - ) or ('leaklookup' in selected_sources and bool((__main__.Core.leaklookup_key() or '').strip())) - if credentialed_breach_source: + credentialed_source = any( + source_name in selected_sources and bool((key_getter() or '').strip()) + for source_name, key_getter in ( + ('dehashed', __main__.Core.dehashed_key), + ('hibpverified', __main__.Core.hibpverified_key), + ('leaklookup', __main__.Core.leaklookup_key), + ) + ) + if credentialed_source: get_api_key(x_api_key) supported_engines = __main__.Core.get_supportedengines() for s in selected_sources: diff --git a/theHarvester/lib/source_catalog.py b/theHarvester/lib/source_catalog.py index 776db32a..ef8fb25b 100644 --- a/theHarvester/lib/source_catalog.py +++ b/theHarvester/lib/source_catalog.py @@ -83,7 +83,7 @@ _SPECS = ( activity=ActivityClass.DIRECT, ), _spec('crtsh', ResultRoute.SUBDOMAINS), - _spec('dehashed', ResultRoute.IPS), + _spec('dehashed', ResultRoute.EMAILS, ResultRoute.IPS), _spec('dnsdb', ResultRoute.SUBDOMAINS), _spec('dnsdumpster', ResultRoute.SUBDOMAINS, ResultRoute.IPS), _spec('duckduckgo', ResultRoute.SUBDOMAINS, ResultRoute.EMAILS), @@ -99,7 +99,7 @@ _SPECS = ( _spec('hunter', ResultRoute.SUBDOMAINS, ResultRoute.EMAILS), _spec('hunterhow', ResultRoute.SUBDOMAINS), _spec('intelx', ResultRoute.SUBDOMAINS, ResultRoute.EMAILS, ResultRoute.INTERESTING_URLS), - _spec('leakix', ResultRoute.SUBDOMAINS, ResultRoute.EMAILS), + _spec('leakix', ResultRoute.SUBDOMAINS), _spec('leaklookup', ResultRoute.EMAILS, ResultRoute.BREACHES), _spec('mojeek', ResultRoute.SUBDOMAINS, ResultRoute.EMAILS), _spec('netlas', ResultRoute.SUBDOMAINS),