diff --git a/CHANGELOG.md b/CHANGELOG.md index c47b94fe..de92f432 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added authenticated REST routes for listing completed enumeration runs and retrieving their normalized evidence. +- Added an authenticated HIBP verified-domain source for CLI and conditionally authenticated REST queries that retains normalized account emails and stable breach names without retaining the raw account mapping. - Added keyless Shodan Certificate Transparency hostname discovery with bounded requests and offline response contracts. - Added transactional SQLite storage and loading for completed full-pipeline runs without changing legacy result rows. - Added deterministic JSONL report companions finalized after selected one-shot actions complete. @@ -18,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added root contributor and security policies, structured issue forms, repository agent guidance, discovery terminology, and an operator-focused documentation wiki ([d090a29a](https://github.com/laramies/theHarvester/commit/d090a29a), [7c491ef5](https://github.com/laramies/theHarvester/commit/7c491ef5), [8b9d420b](https://github.com/laramies/theHarvester/commit/8b9d420b)). ### Changed +- Included HIBP verified-domain in `all` and matching capability selectors like every other P0 source, with REST operator authentication applied after source expansion when its provider key is configured. - Allowed REST `/query` requests to select discovery sources by result capability, matching the CLI's union semantics while preserving explicit source selection. - Expanded Common Crawl discovery to use every unique crawl ending within one year of the newest catalog entry, validate catalog endpoints, batch requests, cap each query at 100 pages, and enforce the CLI result limit across page requests ([249ce64b](https://github.com/laramies/theHarvester/commit/249ce64b), [70470cd8](https://github.com/laramies/theHarvester/commit/70470cd8)). - Completed bounded pagination for Wayback Archive and Cert Spotter, including continuation handling, truncation diagnostics, and preservation of partial results on provider failures ([df6ff2c9](https://github.com/laramies/theHarvester/commit/df6ff2c9), [f85a08ff](https://github.com/laramies/theHarvester/commit/f85a08ff)). diff --git a/README.md b/README.md index 6da597ce..abb16443 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ Open [http://127.0.0.1:5000/docs](http://127.0.0.1:5000/docs) for interactive Sw | Route | Purpose | | --- | --- | | `GET /sources` | List registered discovery sources. | -| `GET /query` | Return ASNs, interesting URLs, Twitter/LinkedIn fields, Trello URLs, IPs, emails, and hosts as JSON. | +| `GET /query` | Return consolidated discovery results, including emails and breach names, as JSON. | | `GET /dnsbrute` | Run DNS brute force for a domain. | | `POST /additional/breaches` | Return Have I Been Pwned breach data. | | `POST /additional/leaks` | Return Leak-Lookup data. | @@ -104,7 +104,7 @@ Open [http://127.0.0.1:5000/docs](http://127.0.0.1:5000/docs) for interactive Sw The service rate limit defaults to five requests per minute and can be changed with `--rate-limit`. The `/additional/*` routes require `THEHARVESTER_API_KEY` on the server and the same value in the `X-API-Key` request header. -The core `/query`, `/sources`, and `/dnsbrute` routes do not require authentication. Keep the service bound to localhost. If you require remote access, add authentication, access controls, and TLS. +The core `/query`, `/sources`, and `/dnsbrute` routes do not normally require authentication. When a `/query` selection includes `hibpverified` and its provider key is configured, the request requires `THEHARVESTER_API_KEY` in the `X-API-Key` header because it can access verified-domain account data. Keep the service bound to localhost. If you require remote access, add authentication, access controls, and TLS. Docker Compose publishes port `5000` on every host interface unless you narrow the port mapping: @@ -153,6 +153,7 @@ Read the **API key** column as follows: | `gitlab` | ✓ | ✓ | No | No | No | No | No | No | No | | `hackertarget` | ✓ | No | No | No | No | No | No | No | Optional | | `haveibeenpwned` | No | No | No | No | No | No | ✓ | `POST /additional/breaches` response | No | +| `hibpverified` | No | ✓ | No | No | No | No | ✓ | No | ✓ | | `hudsonrock` | ✓ | ✓ | ✓ | No | No | No | No | No | No | | `hunter` | ✓ | ✓ | No | No | No | No | No | No | ✓ | | `hunterhow` | ✓ | No | No | No | No | No | No | No | ✓ | @@ -191,6 +192,8 @@ Read the **API key** column as follows: Provider pricing is intentionally omitted because plans and quotas change frequently. See [Configuration and API Keys](docs/wiki/Configuration-and-API-Keys.md) and each provider's current documentation. +`haveibeenpwned` remains the keyless public breach catalogue. `hibpverified` is a separate authenticated source for HIBP's `breachedDomain` endpoint. It participates in `all` and matching capability selectors just like every other P0 source, and skips normally when its provider key is absent. REST selections that include it require the operator `X-API-Key` when the provider key is configured and return normalized emails plus stable breach names. A live run requires a user-owned paid HIBP API key and a user-owned domain verified in that account; routine tests use offline responses. + The runtime registry also reports the legacy identifiers `linkedin`, `linkedin_links`, `netcraft`, `omnisint`, `sublist3r`, and `zoomeyeapi`. These identifiers have no active CLI handlers. The table does not present them as usable sources. ## Configuration diff --git a/docs/wiki/Configuration-and-API-Keys.md b/docs/wiki/Configuration-and-API-Keys.md index 40de3ae2..3e45d314 100644 --- a/docs/wiki/Configuration-and-API-Keys.md +++ b/docs/wiki/Configuration-and-API-Keys.md @@ -28,6 +28,9 @@ apikeys: github: key: your-github-token + hibpverified: + key: your-hibp-api-key + tomba: key: your-tomba-key secret: your-tomba-secret @@ -39,6 +42,8 @@ The [README source matrix](https://github.com/laramies/theHarvester/blob/dev/REA Provider pricing, quotas, and terms change frequently. Check the provider's current documentation for these details. +`hibpverified` queries [HIBP's authenticated verified-domain endpoint](https://haveibeenpwned.com/API/v3#BreachedDomain) only when explicitly named, either alone or in a combination such as `breaches,hibpverified`. Capability selectors and `all` exclude it. Live use requires a user-owned paid HIBP API key and a user-owned domain verified in that account. REST queries selecting it also require the operator `X-API-Key`; the keyless `haveibeenpwned` source continues to query only the public breach catalogue. + ## Proxies Edit `~/.theHarvester/proxies.yaml` using `host:port` entries: diff --git a/docs/wiki/Rest-API.md b/docs/wiki/Rest-API.md index 0cbe3503..175eb61f 100644 --- a/docs/wiki/Rest-API.md +++ b/docs/wiki/Rest-API.md @@ -68,8 +68,20 @@ curl -sG http://127.0.0.1:5000/query \ ``` A completed `/query` also retains its normalized terminal record in the local -SQLite database. The response fields remain unchanged, and no JSON, XML, or -JSONL report file is written unless `filename` is supplied. +SQLite database. No JSON, XML, or JSONL report file is written unless `filename` +is supplied. + +HIBP verified-domain participates in `all`, `emails`, and `breaches` selections. +When its provider key is configured, any selection that includes it also requires +the operator API key: + +```bash +curl -sG http://127.0.0.1:5000/query \ + -H "X-API-Key: $THEHARVESTER_API_KEY" \ + --data-urlencode "domain=$VERIFIED_DOMAIN" \ + --data-urlencode 'source=hibpverified' \ + | jq '{emails, breaches}' +``` Completed-run routes require the operator API key because retained evidence can contain sensitive results: @@ -112,7 +124,7 @@ These routes may also require provider credentials in the request body or local ## Security boundary -`THEHARVESTER_API_KEY` protects `/additional/*` and `/runs*`. It does not authenticate `/query`, `/sources`, or `/dnsbrute`. +`THEHARVESTER_API_KEY` protects `/additional/*`, `/runs*`, and `/query` selections that include a configured `hibpverified` source. Other `/query` requests, `/sources`, and `/dnsbrute` remain unauthenticated. Keep the default localhost binding. If you require remote access, add authentication, network allowlists, TLS, request logging, and an appropriate rate limit. diff --git a/tests/discovery/test_hibpverified.py b/tests/discovery/test_hibpverified.py new file mode 100644 index 00000000..c8b12675 --- /dev/null +++ b/tests/discovery/test_hibpverified.py @@ -0,0 +1,192 @@ +import json +import sys +from pathlib import Path +from typing import Any + +import pytest + +from theHarvester import __main__ as theharvester_main +from theHarvester.discovery import hibpverified +from theHarvester.discovery.constants import MissingKey +from theHarvester.lib.completed_result import CompletedResult +from theHarvester.lib.core import FetcherResponse + + +@pytest.mark.parametrize('api_key', [None, '', ' ']) +def test_verified_domain_source_requires_its_api_key( + monkeypatch: pytest.MonkeyPatch, + api_key: str | None, +) -> None: + monkeypatch.setattr(hibpverified.Core, 'hibpverified_key', lambda: api_key) + + with pytest.raises(MissingKey, match='HIBP verified domain'): + hibpverified.SearchHibpVerified('example.com') + + +def test_verified_domain_source_handles_existing_config_without_new_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(hibpverified.Core, 'api_keys', lambda: {}) + + with pytest.raises(MissingKey, match='HIBP verified domain'): + hibpverified.SearchHibpVerified('example.com') + + +@pytest.mark.asyncio +async def test_verified_domain_source_returns_emails_and_stable_breach_names( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def fake_fetch_all(urls: list[str], **kwargs: Any) -> list[FetcherResponse]: + assert urls == ['https://haveibeenpwned.com/api/v3/breachedDomain/example.com'] + assert kwargs['headers']['hibp-api-key'] == 'secret' + assert kwargs['json'] is True + assert kwargs['include_metadata'] is True + return [ + FetcherResponse( + body={ + 'alice': ['ExampleBreachA', 'ExampleBreachB'], + 'bob': ['ExampleBreach'], + }, + status=200, + headers={}, + ) + ] + + monkeypatch.setattr(hibpverified.Core, 'hibpverified_key', lambda: 'secret') + monkeypatch.setattr(hibpverified.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = hibpverified.SearchHibpVerified('example.com') + + await search.process() + + assert await search.get_emails() == {'alice@example.com', 'bob@example.com'} + assert await search.get_breach_names() == {'ExampleBreach', 'ExampleBreachA', 'ExampleBreachB'} + + +@pytest.mark.asyncio +@pytest.mark.parametrize('payload', [None, [], {'alice': 'ExampleBreach'}, {'': ['ExampleBreach']}, {'alice': [None]}]) +async def test_verified_domain_source_rejects_malformed_responses( + monkeypatch: pytest.MonkeyPatch, + payload: Any, +) -> None: + async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: + return [FetcherResponse(body=payload, status=200, headers={})] + + monkeypatch.setattr(hibpverified.Core, 'hibpverified_key', lambda: 'secret') + monkeypatch.setattr(hibpverified.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = hibpverified.SearchHibpVerified('example.com') + + await search.process() + + assert await search.get_emails() == set() + assert await search.get_breach_names() == set() + + +@pytest.mark.asyncio +async def test_verified_domain_source_attributes_unverified_domain( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: + return [FetcherResponse(body={'statusCode': 403}, status=403, headers={})] + + monkeypatch.setattr(hibpverified.Core, 'hibpverified_key', lambda: 'secret') + monkeypatch.setattr(hibpverified.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = hibpverified.SearchHibpVerified('example.com') + + with caplog.at_level('INFO', logger=hibpverified.__name__): + await search.process() + + assert 'HIBP verified-domain target is not verified for this API key (HTTP 403)' in caplog.text + + +@pytest.mark.asyncio +async def test_verified_domain_source_treats_not_found_as_valid_empty( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: + return [FetcherResponse(body={'statusCode': 404}, status=404, headers={})] + + monkeypatch.setattr(hibpverified.Core, 'hibpverified_key', lambda: 'secret') + monkeypatch.setattr(hibpverified.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = hibpverified.SearchHibpVerified('example.com') + + with caplog.at_level('INFO', logger=hibpverified.__name__): + await search.process() + + assert await search.get_emails() == set() + assert await search.get_breach_names() == set() + assert 'failed' not in caplog.text + + +@pytest.mark.asyncio +async def test_verified_domain_source_attributes_rate_limit_without_retry( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + calls = 0 + + async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[FetcherResponse]: + nonlocal calls + calls += 1 + return [FetcherResponse(body={'statusCode': 429}, status=429, headers={})] + + monkeypatch.setattr(hibpverified.Core, 'hibpverified_key', lambda: 'secret') + monkeypatch.setattr(hibpverified.AsyncFetcher, 'fetch_all', fake_fetch_all) + search = hibpverified.SearchHibpVerified('example.com') + + with caplog.at_level('INFO', logger=hibpverified.__name__): + await search.process() + + assert calls == 1 + assert 'HIBP verified-domain request was rate limited (HTTP 429)' in caplog.text + + +@pytest.mark.asyncio +async def test_verified_domain_results_reach_completed_jsonl_and_sqlite_handoff( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + completed_results: list[CompletedResult] = [] + + class FakeStash: + async def do_init(self) -> None: + return None + + async def store_all(self, *_args: object) -> None: + return None + + async def store_completed_result(self, result: CompletedResult) -> None: + completed_results.append(result) + + class FakeHibpVerified: + def __init__(self, domain: str) -> None: + assert domain == 'example.com' + + async def process(self, _proxy: bool) -> None: + return None + + async def get_emails(self) -> set[str]: + return {'alice@example.com'} + + async def get_breach_names(self) -> set[str]: + return {'ExampleBreach'} + + report = tmp_path / 'hibp-verified-report' + monkeypatch.setattr(theharvester_main.stash, 'StashManager', FakeStash) + monkeypatch.setattr(theharvester_main.hibpverified, 'SearchHibpVerified', FakeHibpVerified) + monkeypatch.setattr( + sys, + 'argv', + ['theHarvester', '-d', 'example.com', '-b', 'hibpverified', '-f', str(report)], + ) + + with pytest.raises(SystemExit) as exit_info: + await theharvester_main.start() + + assert exit_info.value.code == 0 + assert completed_results[0].results == ( + ('breach', 'ExampleBreach'), + ('email', 'alice@example.com'), + ) + records = [json.loads(line) for line in report.with_suffix('.jsonl').read_text().splitlines()] + assert {'type': 'breach', 'value': 'ExampleBreach'} in records + assert {'type': 'email', 'value': 'alice@example.com'} in records diff --git a/tests/lib/test_core.py b/tests/lib/test_core.py index 4760f041..534d4b02 100644 --- a/tests/lib/test_core.py +++ b/tests/lib/test_core.py @@ -26,6 +26,7 @@ def test_email_capability_expands_to_email_sources() -> None: "duckduckgo", "github-code", "gitlab", + "hibpverified", "hudsonrock", "hunter", "intelx", @@ -65,8 +66,12 @@ def test_multiple_capabilities_form_a_union() -> None: ] -def test_breach_capability_selects_haveibeenpwned() -> None: - assert Core.expand_source_selection('breaches') == ['haveibeenpwned'] +def test_breach_capability_includes_every_matching_source() -> None: + assert Core.expand_source_selection('breaches') == ['haveibeenpwned', 'hibpverified'] + + +def test_named_source_can_be_combined_with_a_capability() -> None: + assert Core.expand_source_selection('breaches,hibpverified') == ['haveibeenpwned', 'hibpverified'] def test_all_preserves_every_supported_source() -> None: diff --git a/tests/lib/test_source_catalog.py b/tests/lib/test_source_catalog.py index 7373c9a6..aaf8f7c8 100644 --- a/tests/lib/test_source_catalog.py +++ b/tests/lib/test_source_catalog.py @@ -57,6 +57,7 @@ def test_subdomain_route_drives_subdomain_capability() -> None: def test_source_specs_describe_consolidated_routes_not_getter_presence() -> None: assert SOURCE_SPECS['gitlab'].routes == frozenset({ResultRoute.SUBDOMAINS, ResultRoute.EMAILS}) assert SOURCE_SPECS['haveibeenpwned'].routes == frozenset({ResultRoute.BREACHES}) + assert SOURCE_SPECS['hibpverified'].routes == frozenset({ResultRoute.EMAILS, ResultRoute.BREACHES}) assert SOURCE_SPECS['urlscan'].routes == frozenset( { ResultRoute.SUBDOMAINS, diff --git a/tests/test_readme.py b/tests/test_readme.py index 35697dc6..a88be9df 100644 --- a/tests/test_readme.py +++ b/tests/test_readme.py @@ -80,8 +80,8 @@ def test_readme_matches_declared_source_contracts() -> None: declared = _declared_source_contracts() assert '| Source | Subdomains | Emails | IPs | ASNs | URLs / links | People | Breaches |' in readme - assert len(declared) == 55 - assert len(documented) == 55 + assert len(declared) == 56 + assert len(documented) == 56 assert documented == declared assert {'securitytrails', 'shodaninternetdb'}.isdisjoint(documented) diff --git a/tests/test_rest_api.py b/tests/test_rest_api.py index 0bf418e7..243a6f18 100644 --- a/tests/test_rest_api.py +++ b/tests/test_rest_api.py @@ -1,17 +1,31 @@ from argparse import Namespace +import pytest from fastapi.testclient import TestClient from theHarvester.lib.api import api from theHarvester.lib.core import Core +@pytest.fixture(autouse=True) +def reset_rate_limiter() -> None: + api.limiter.reset() + yield + api.limiter.reset() + + def test_query_expands_source_capability(monkeypatch) -> None: captured: list[tuple[Namespace, bool]] = [] - async def fake_start(args: Namespace, *, persist_completed_result: bool = False): + async def fake_start( + args: Namespace, + *, + persist_completed_result: bool = False, + include_breaches: bool = False, + ): captured.append((args, persist_completed_result)) - return ([], [], [], [], [], [], [], [], []) + assert include_breaches is True + return ([], [], [], [], [], [], [], [], [], []) monkeypatch.setattr(api.__main__, 'start', fake_start) @@ -25,16 +39,23 @@ def test_query_expands_source_capability(monkeypatch) -> None: def test_query_unions_capabilities_and_explicit_sources(monkeypatch) -> None: captured: list[tuple[Namespace, bool]] = [] - async def fake_start(args: Namespace, *, persist_completed_result: bool = False): + async def fake_start( + args: Namespace, + *, + persist_completed_result: bool = False, + include_breaches: bool = False, + ): captured.append((args, persist_completed_result)) - return ([], [], [], [], [], [], [], [], []) + assert include_breaches is True + return ([], [], [], [], [], [], [], [], [], []) monkeypatch.setattr(api.__main__, 'start', fake_start) response = TestClient(api.app).get('/query?domain=example.test&source=emails&source=certspotter') assert response.status_code == 200 - assert captured[0][0].source == ','.join(Core.expand_source_selection('emails,certspotter')) + expected_sources = Core.expand_source_selection('emails,certspotter') + assert captured[0][0].source == ','.join(expected_sources) assert captured[0][1] is True @@ -48,3 +69,109 @@ def test_query_rejects_unknown_source_or_capability(monkeypatch) -> None: assert response.status_code == 400 assert response.json()['detail'].startswith("Source 'unknown' is not supported") + + +@pytest.mark.parametrize('source', ['hibpverified', 'breaches', 'emails', 'all']) +def test_query_requires_operator_key_when_selection_includes_verified_hibp(monkeypatch, source: str) -> None: + async def unexpected_start( + _args: Namespace, + *, + persist_completed_result: bool = False, + include_breaches: bool = False, + ): + raise AssertionError('enumeration must not start') + + monkeypatch.delenv('THEHARVESTER_API_KEY', raising=False) + monkeypatch.setattr(api.__main__.Core, 'hibpverified_key', lambda: 'provider-secret') + monkeypatch.setattr(api.__main__, 'start', unexpected_start) + + response = TestClient(api.app).get(f'/query?domain=example.test&source={source}') + + assert response.status_code == 503 + assert response.json()['detail'] == 'THEHARVESTER_API_KEY is not configured' + + +def test_authenticated_query_returns_verified_hibp_emails_and_breaches(monkeypatch) -> None: + captured: list[Namespace] = [] + + async def fake_start( + args: Namespace, + *, + persist_completed_result: bool = False, + include_breaches: bool = False, + ): + captured.append(args) + assert persist_completed_result is True + assert include_breaches is True + return ([], [], [], [], [], [], [], ['alice@example.test'], [], ['ExampleBreach']) + + monkeypatch.setenv('THEHARVESTER_API_KEY', 'operator-secret') + monkeypatch.setattr(api.__main__.Core, 'hibpverified_key', lambda: 'provider-secret') + monkeypatch.setattr(api.__main__, 'start', fake_start) + + response = TestClient(api.app).get( + '/query?domain=example.test&source=hibpverified', + headers={'X-API-Key': 'operator-secret'}, + ) + + assert response.status_code == 200 + assert captured[0].source == 'hibpverified' + assert response.json()['emails'] == ['alice@example.test'] + assert response.json()['breaches'] == ['ExampleBreach'] + + +def test_authenticated_query_includes_verified_hibp_from_capability_selection(monkeypatch) -> None: + captured: list[Namespace] = [] + + async def fake_start( + args: Namespace, + *, + persist_completed_result: bool = False, + include_breaches: bool = False, + ): + captured.append(args) + assert include_breaches is True + return ([], [], [], [], [], [], [], [], [], []) + + monkeypatch.setenv('THEHARVESTER_API_KEY', 'operator-secret') + monkeypatch.setattr(api.__main__.Core, 'hibpverified_key', lambda: 'provider-secret') + monkeypatch.setattr(api.__main__, 'start', fake_start) + + response = TestClient(api.app).get( + '/query?domain=example.test&source=breaches', + headers={'X-API-Key': 'operator-secret'}, + ) + + assert response.status_code == 200 + assert captured[0].source == 'haveibeenpwned,hibpverified' + + +def test_query_skips_operator_auth_when_verified_hibp_provider_key_is_absent(monkeypatch) -> None: + captured: list[Namespace] = [] + + async def fake_start( + args: Namespace, + *, + persist_completed_result: bool = False, + include_breaches: bool = False, + ): + captured.append(args) + return ([], [], [], [], [], [], [], [], [], []) + + monkeypatch.delenv('THEHARVESTER_API_KEY', raising=False) + monkeypatch.setattr(api.__main__.Core, 'hibpverified_key', lambda: None) + monkeypatch.setattr(api.__main__, 'start', fake_start) + + response = TestClient(api.app).get('/query?domain=example.test&source=breaches') + + assert response.status_code == 200 + assert captured[0].source == 'haveibeenpwned,hibpverified' + + +def test_sources_advertises_authenticated_verified_hibp(monkeypatch) -> None: + monkeypatch.setattr(api.__main__.Core, 'get_supportedengines', lambda: ['crtsh', 'hibpverified']) + + response = TestClient(api.app).get('/sources') + + assert response.status_code == 200 + assert response.json() == {'sources': ['crtsh', 'hibpverified']} diff --git a/tests/test_rest_terminal_evidence.py b/tests/test_rest_terminal_evidence.py index 8fb6a70c..e97ec352 100644 --- a/tests/test_rest_terminal_evidence.py +++ b/tests/test_rest_terminal_evidence.py @@ -8,9 +8,15 @@ from theHarvester.lib.api import api def test_query_requests_completed_result_persistence(monkeypatch) -> None: persistence_flags: list[bool] = [] - async def fake_start(args: Namespace, *, persist_completed_result: bool = False): + async def fake_start( + args: Namespace, + *, + persist_completed_result: bool = False, + include_breaches: bool = False, + ): persistence_flags.append(persist_completed_result) - return ([], [], [], [], [], [], [], [], []) + assert include_breaches is True + return ([], [], [], [], [], [], [], [], [], []) monkeypatch.setattr(api.__main__, 'start', fake_start) diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 22a6ac66..96f50898 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -42,6 +42,7 @@ from theHarvester.discovery import ( gitlabsearch, hackertarget, haveibeenpwned, + hibpverified, hudsonrocksearch, huntersearch, intelxsearch, @@ -139,7 +140,12 @@ def sanitize_filename(filename: str) -> str: return filename -async def start(rest_args: argparse.Namespace | None = None, *, persist_completed_result: bool = False): +async def start( + rest_args: argparse.Namespace | None = None, + *, + persist_completed_result: bool = False, + include_breaches: bool = False, +): """Main program function""" parser = argparse.ArgumentParser( description='theHarvester is used to gather open source intelligence (OSINT) on a company or domain.' @@ -233,7 +239,7 @@ async def start(rest_args: argparse.Namespace | None = None, *, persist_complete help="""Comma-separated sources or capability selectors: subdomains, emails, ips, asns, urls, people, breaches, or all. Sources: baidu, bevigil, brave, bufferoverun, builtwith, censys, certspotter, chaos, commoncrawl, criminalip, crtsh, dehashed, dnsdumpster, duckduckgo, dymo, fofa, fullhunt, github-code, - gitlab, hackertarget, haveibeenpwned, hudsonrock, hunter, hunterhow, intelx, leakix, leaklookup, mojeek, netlas, onyphe, otx, pentesttools, + gitlab, hackertarget, haveibeenpwned, hibpverified, hudsonrock, hunter, hunterhow, intelx, leakix, leaklookup, mojeek, netlas, onyphe, otx, pentesttools, projectdiscovery, rapiddns, robtex, rocketreach, securityscorecard, securityTrails, sherlockeye, shodan, shodanct, shodanInternetDB, subdomaincenter, subdomainfinderc99, thc, tomba, urlscan, venacus, virustotal, waybackarchive, whoisxml, windvane, yahoo, zoomeye""", ) @@ -743,6 +749,16 @@ async def start(rest_args: argparse.Namespace | None = None, *, persist_complete except Exception as e: show_default_error_message(engineitem, word, e) + elif engineitem == 'hibpverified': + try: + hibp_search = hibpverified.SearchHibpVerified(word) + stor_lst.append(store(hibp_search, engineitem)) + except MissingKey as error: + if not args.quiet: + output_logger.info(f'A Missing Key error occurred in hibpverified: {error}') + except Exception as error: + show_default_error_message(engineitem, word, error) + elif engineitem == 'hudsonrock': try: hudsonrock_search = hudsonrocksearch.SearchHudsonRock(word) @@ -1357,7 +1373,7 @@ async def start(rest_args: argparse.Namespace | None = None, *, persist_complete all_hosts = sorted_unique(all_hosts) if persist_completed_result: await persist_result(finish_completed_result()) - return ( + result = ( total_asns, interesting_urls, twitter_people_list_tracker, @@ -1368,6 +1384,7 @@ async def start(rest_args: argparse.Namespace | None = None, *, persist_complete all_emails, all_hosts, ) + return (*result, sorted_unique(all_breaches)) if include_breaches else result # Check to see if all_emails and all_hosts are defined. try: all_emails @@ -1911,7 +1928,7 @@ async def start(rest_args: argparse.Namespace | None = None, *, persist_complete if rest_args is not None: all_hosts = sorted_unique(all_hosts) - return ( + result = ( total_asns, interesting_urls, twitter_people_list_tracker, @@ -1922,6 +1939,7 @@ async def start(rest_args: argparse.Namespace | None = None, *, persist_complete all_emails, all_hosts, ) + return (*result, sorted_unique(all_breaches)) if include_breaches else result sys.exit(0) diff --git a/theHarvester/data/api-keys.yaml b/theHarvester/data/api-keys.yaml index 16f696c0..15910d21 100644 --- a/theHarvester/data/api-keys.yaml +++ b/theHarvester/data/api-keys.yaml @@ -44,6 +44,9 @@ apikeys: hackertarget: key: + hibpverified: + key: + hunter: key: diff --git a/theHarvester/discovery/hibpverified.py b/theHarvester/discovery/hibpverified.py new file mode 100644 index 00000000..00336e07 --- /dev/null +++ b/theHarvester/discovery/hibpverified.py @@ -0,0 +1,70 @@ +import logging + +from theHarvester.discovery.constants import MissingKey +from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse + +logger = logging.getLogger(__name__) + + +class SearchHibpVerified: + def __init__(self, word: str) -> None: + self.word = word.strip().lower().rstrip('.') + self.api_key = (Core.hibpverified_key() or '').strip() + if not self.api_key: + raise MissingKey('HIBP verified domain') + self.base_url = 'https://haveibeenpwned.com/api/v3' + self.headers = {'hibp-api-key': self.api_key, 'user-agent': 'theHarvester'} + self.emails: set[str] = set() + self.breach_names: set[str] = set() + + async def process(self, proxy: bool = False) -> None: + try: + responses = await AsyncFetcher.fetch_all( + [f'{self.base_url}/breachedDomain/{self.word}'], + headers=self.headers, + json=True, + proxy=proxy, + include_metadata=True, + ) + except (OSError, RuntimeError, ValueError): + logger.info('HIBP verified-domain request failed') + return + + response = responses[0] if responses and isinstance(responses[0], FetcherResponse) else None + if response is None: + logger.info('HIBP verified-domain request failed') + return + if response.status == 403: + logger.info('HIBP verified-domain target is not verified for this API key (HTTP 403)') + return + if response.status == 404: + return + if response.status == 429: + logger.info('HIBP verified-domain request was rate limited (HTTP 429)') + return + if response.status != 200: + logger.info(f'HIBP verified-domain request failed with HTTP {response.status}') + return + if not isinstance(response.body, dict): + logger.info('HIBP verified-domain returned malformed account data') + return + if not all( + isinstance(alias, str) + and alias.strip() + and '@' not in alias + and not any(character.isspace() for character in alias) + and isinstance(breaches, list) + and all(isinstance(breach, str) and breach.strip() for breach in breaches) + for alias, breaches in response.body.items() + ): + logger.info('HIBP verified-domain returned malformed account data') + return + for alias, breaches in response.body.items(): + self.emails.add(f'{alias.strip()}@{self.word}') + self.breach_names.update(breach.strip() for breach in breaches) + + async def get_emails(self) -> set[str]: + return self.emails + + async def get_breach_names(self) -> set[str]: + return self.breach_names diff --git a/theHarvester/lib/api/api.py b/theHarvester/lib/api/api.py index 61698cc2..2a7bcf04 100644 --- a/theHarvester/lib/api/api.py +++ b/theHarvester/lib/api/api.py @@ -52,6 +52,7 @@ class QueryResponse(BaseModel): ips: list[str] = Field(default_factory=list, description='List of IPs') emails: list[str] = Field(default_factory=list, description='List of emails') hosts: list[str] = Field(default_factory=list, description='List of hosts') + breaches: list[str] = Field(default_factory=list, description='List of breach names') class ErrorResponse(BaseModel): @@ -362,6 +363,7 @@ async def query( domain: Annotated[str, Query(min_length=3, description='Domain to be harvested')], dns_server: Annotated[str, Query(description='DNS server to use for lookup')] = '', user_agent: Annotated[str | None, Header()] = None, + x_api_key: Annotated[str | None, Header(alias='X-API-Key')] = None, dns_brute: Annotated[bool, Query(description='Perform a DNS brute force on the domain')] = False, dns_lookup: Annotated[bool, Query(description='Enable DNS server lookup')] = False, dns_resolve: Annotated[ @@ -389,6 +391,8 @@ async def query( try: # Validate sources selected_sources = __main__.Core.expand_source_selection(','.join(source)) + if 'hibpverified' in selected_sources and __main__.Core.hibpverified_key() is not None: + get_api_key(x_api_key) supported_engines = __main__.Core.get_supportedengines() for s in selected_sources: if s not in supported_engines: @@ -414,6 +418,7 @@ async def query( aips, aemails, ahosts, + abreaches, ) = await __main__.start( argparse.Namespace( dns_brute=dns_brute, @@ -434,6 +439,7 @@ async def query( screenshot='', ), persist_completed_result=True, + include_breaches=True, ) # Return the results using the Pydantic model @@ -448,6 +454,7 @@ async def query( 'ips': aips, 'emails': aemails, 'hosts': ahosts, + 'breaches': abreaches, } ) except HTTPException as e: diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index aff59580..ecfa6e5b 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -58,6 +58,7 @@ class Core: 'fullhunt': ('key',), 'github': ('key',), 'hackertarget': ('key',), + 'hibpverified': ('key',), 'hunter': ('key',), 'hunterhow': ('key',), 'intelx': ('key',), @@ -168,6 +169,10 @@ class Core: def hackertarget_key() -> str: return Core._api_key_value('hackertarget') + @staticmethod + def hibpverified_key() -> str | None: + return Core.api_keys().get('hibpverified', {}).get('key') + @staticmethod def hunter_key() -> str: return Core._api_key_value('hunter') @@ -307,6 +312,7 @@ class Core: 'gitlab', 'hackertarget', 'haveibeenpwned', + 'hibpverified', 'hudsonrock', 'hunter', 'hunterhow', diff --git a/theHarvester/lib/source_catalog.py b/theHarvester/lib/source_catalog.py index 91e4a531..bf6d9c0a 100644 --- a/theHarvester/lib/source_catalog.py +++ b/theHarvester/lib/source_catalog.py @@ -43,7 +43,10 @@ class SourceSpec: def _spec(name: str, *routes: ResultRoute) -> SourceSpec: - return SourceSpec(name=name, routes=frozenset(routes)) + return SourceSpec( + name=name, + routes=frozenset(routes), + ) _SPECS = ( @@ -69,6 +72,7 @@ _SPECS = ( _spec('gitlab', ResultRoute.SUBDOMAINS, ResultRoute.EMAILS), _spec('hackertarget', ResultRoute.SUBDOMAINS), _spec('haveibeenpwned', ResultRoute.BREACHES), + _spec('hibpverified', ResultRoute.EMAILS, ResultRoute.BREACHES), _spec('hudsonrock', ResultRoute.SUBDOMAINS, ResultRoute.EMAILS, ResultRoute.IPS), _spec('hunter', ResultRoute.SUBDOMAINS, ResultRoute.EMAILS), _spec('hunterhow', ResultRoute.SUBDOMAINS),