From 6bb4ba6fd505e393114aae7bdb055e2dade8065e Mon Sep 17 00:00:00 2001 From: NotoriousRebel <36310667+NotoriousRebel@users.noreply.github.com> Date: Thu, 16 Jul 2026 19:33:43 -0400 Subject: [PATCH 1/5] test: characterize configuration contract (#50) --- tests/lib/test_configuration_contract.py | 144 +++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 tests/lib/test_configuration_contract.py diff --git a/tests/lib/test_configuration_contract.py b/tests/lib/test_configuration_contract.py new file mode 100644 index 00000000..60bf10fc --- /dev/null +++ b/tests/lib/test_configuration_contract.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, NamedTuple + +import pytest +import yaml + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + from types import ModuleType + + from theHarvester.lib.core import Core + + +class ConfigurationEnvironment(NamedTuple): + core: type[Core] + module: ModuleType + directories: list[Path] + + +@pytest.fixture +def configuration_environment( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> ConfigurationEnvironment: + monkeypatch.setenv('HOME', str(tmp_path)) + import theHarvester.lib.core as core_module + + directories = [tmp_path / name for name in ('user', 'system', 'local')] + for directory in directories: + directory.mkdir() + monkeypatch.setattr(core_module, 'CONFIG_DIRS', directories) + return ConfigurationEnvironment(core_module.Core, core_module, directories) + + +@pytest.mark.parametrize( + ('present_indexes', 'expected_key'), + [ + ((0, 1, 2), 'user-key'), + ((1, 2), 'system-key'), + ((2,), 'local-key'), + ], +) +def test_api_keys_uses_first_available_configuration( + configuration_environment: ConfigurationEnvironment, + present_indexes: tuple[int, ...], + expected_key: str, +) -> None: + core = configuration_environment.core + configuration_dirs = configuration_environment.directories + keys = ('user-key', 'system-key', 'local-key') + for index in present_indexes: + (configuration_dirs[index] / 'api-keys.yaml').write_text( + f'apikeys:\n brave:\n key: {keys[index]}\n', + encoding='utf-8', + ) + + assert core.api_keys() == {'brave': {'key': expected_key}} + + +def test_missing_api_keys_uses_and_creates_bundled_default( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + configuration_environment: ConfigurationEnvironment, +) -> None: + core = configuration_environment.core + core_module = configuration_environment.module + configuration_dirs = configuration_environment.directories + bundled_directory = tmp_path / 'bundled' + bundled_directory.mkdir() + bundled_content = 'apikeys:\n brave:\n key: bundled-key\n' + (bundled_directory / 'api-keys.yaml').write_text(bundled_content, encoding='utf-8') + monkeypatch.setattr(core_module, 'DATA_DIR', bundled_directory) + + assert ( + core.api_keys(), + (configuration_dirs[0] / 'api-keys.yaml').read_text(encoding='utf-8'), + ) == ( + {'brave': {'key': 'bundled-key'}}, + bundled_content, + ) + + +@pytest.mark.parametrize( + ('filename', 'configuration_reader'), + [ + ('api-keys.yaml', 'api_keys'), + ('proxies.yaml', 'proxy_list'), + ], +) +def test_malformed_configuration_raises_yaml_error( + configuration_environment: ConfigurationEnvironment, + filename: str, + configuration_reader: str, +) -> None: + core = configuration_environment.core + configuration_dirs = configuration_environment.directories + (configuration_dirs[0] / filename).write_text('key: [unterminated\n', encoding='utf-8') + reader: Callable[[], dict[str, object]] = getattr(core, configuration_reader) + + with pytest.raises(yaml.YAMLError): + reader() + + +def test_provider_accessors_return_single_and_multi_field_credentials( + configuration_environment: ConfigurationEnvironment, +) -> None: + core = configuration_environment.core + configuration_dirs = configuration_environment.directories + (configuration_dirs[0] / 'api-keys.yaml').write_text( + 'apikeys:\n brave:\n key: brave-key\n censys:\n id: censys-id\n secret: censys-secret\n', + encoding='utf-8', + ) + + assert (core.brave_key(), core.censys_key()) == ( + 'brave-key', + ('censys-id', 'censys-secret'), + ) + + +@pytest.mark.parametrize( + ('contents', 'expected'), + [ + ( + 'http: [proxy.local:8080]\nsocks5: [socks.local:1080]\n', + { + 'http': ['http://proxy.local:8080'], + 'socks5': ['socks5://socks.local:1080'], + }, + ), + ('http:\n', {'http': [], 'socks5': []}), + ], +) +def test_proxy_lookup_normalizes_configured_addresses( + configuration_environment: ConfigurationEnvironment, + contents: str, + expected: dict[str, list[str]], +) -> None: + core = configuration_environment.core + configuration_dirs = configuration_environment.directories + (configuration_dirs[0] / 'proxies.yaml').write_text(contents, encoding='utf-8') + + assert core.proxy_list() == expected From e70d345172c0148f63bc8157b874f5edd3e43e3e Mon Sep 17 00:00:00 2001 From: NotoriousRebel <36310667+NotoriousRebel@users.noreply.github.com> Date: Thu, 16 Jul 2026 20:06:13 -0400 Subject: [PATCH 2/5] fix: defer proxy configuration loading (#51) --- tests/lib/test_configuration_contract.py | 49 +++++++++++++++++++++++- theHarvester/lib/core.py | 10 ++++- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/tests/lib/test_configuration_contract.py b/tests/lib/test_configuration_contract.py index 60bf10fc..7c0e3eba 100644 --- a/tests/lib/test_configuration_contract.py +++ b/tests/lib/test_configuration_contract.py @@ -1,5 +1,10 @@ from __future__ import annotations +import os +import subprocess +import sys +import textwrap +from pathlib import Path from typing import TYPE_CHECKING, NamedTuple import pytest @@ -7,7 +12,6 @@ import yaml if TYPE_CHECKING: from collections.abc import Callable - from pathlib import Path from types import ModuleType from theHarvester.lib.core import Core @@ -19,6 +23,49 @@ class ConfigurationEnvironment(NamedTuple): directories: list[Path] +def test_importing_core_does_not_access_configuration_files(tmp_path: Path) -> None: + script = textwrap.dedent( + """ + import sys + + configuration_accesses: list[str] = [] + + def record_configuration_access(event: str, arguments: tuple[object, ...]) -> None: + if event != 'open': + return + path = arguments[0] + if isinstance(path, str) and path.endswith(('api-keys.yaml', 'proxies.yaml')): + configuration_accesses.append(path) + + sys.addaudithook(record_configuration_access) + + import theHarvester.lib.core + + assert configuration_accesses == [], configuration_accesses + + first_proxy_list = theHarvester.lib.core.AsyncFetcher().proxy_list + access_count = len(configuration_accesses) + second_proxy_list = theHarvester.lib.core.AsyncFetcher().proxy_list + + assert second_proxy_list is first_proxy_list + assert len(configuration_accesses) == access_count + """ + ) + environment = os.environ.copy() + environment['HOME'] = str(tmp_path) + + result = subprocess.run( + [sys.executable, '-c', script], + capture_output=True, + check=False, + cwd=Path(__file__).parents[2], + env=environment, + text=True, + ) + + assert result.returncode == 0, result.stderr + + @pytest.fixture def configuration_environment( monkeypatch: pytest.MonkeyPatch, diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index f9e9e7d4..f62d015d 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -413,7 +413,15 @@ class Core: class AsyncFetcher: - proxy_list = Core.proxy_list() + _proxy_list: ClassVar[dict | None] = None + + @property + def proxy_list(self) -> dict: + proxy_list = self.__class__._proxy_list + if proxy_list is None: + proxy_list = Core.proxy_list() + self.__class__._proxy_list = proxy_list + return proxy_list @staticmethod def _default_headers(headers: dict[str, str] | None = None) -> dict[str, str]: From 1e6f0a22a92fb1b6394278e491a6856792c77e24 Mon Sep 17 00:00:00 2001 From: NotoriousRebel <36310667+NotoriousRebel@users.noreply.github.com> Date: Thu, 16 Jul 2026 20:35:18 -0400 Subject: [PATCH 3/5] refactor: add credential configuration adapters (#53) --- tests/lib/test_configuration_contract.py | 15 +++++--- tests/test_bravesearch.py | 49 ++++++++++++++++++++++++ theHarvester/discovery/bravesearch.py | 11 ++++-- theHarvester/lib/configuration.py | 21 ++++++++++ theHarvester/lib/core.py | 4 -- 5 files changed, 87 insertions(+), 13 deletions(-) create mode 100644 tests/test_bravesearch.py create mode 100644 theHarvester/lib/configuration.py diff --git a/tests/lib/test_configuration_contract.py b/tests/lib/test_configuration_contract.py index 7c0e3eba..58838093 100644 --- a/tests/lib/test_configuration_contract.py +++ b/tests/lib/test_configuration_contract.py @@ -10,6 +10,8 @@ from typing import TYPE_CHECKING, NamedTuple import pytest import yaml +from theHarvester.lib.configuration import FileSystemCredentialAdapter + if TYPE_CHECKING: from collections.abc import Callable from types import ModuleType @@ -89,7 +91,7 @@ def configuration_environment( ((2,), 'local-key'), ], ) -def test_api_keys_uses_first_available_configuration( +def test_api_keys_and_filesystem_credentials_use_first_available_configuration( configuration_environment: ConfigurationEnvironment, present_indexes: tuple[int, ...], expected_key: str, @@ -103,7 +105,10 @@ def test_api_keys_uses_first_available_configuration( encoding='utf-8', ) - assert core.api_keys() == {'brave': {'key': expected_key}} + assert (core.api_keys(), FileSystemCredentialAdapter().get('brave')) == ( + {'brave': {'key': expected_key}}, + expected_key, + ) def test_missing_api_keys_uses_and_creates_bundled_default( @@ -156,12 +161,12 @@ def test_provider_accessors_return_single_and_multi_field_credentials( core = configuration_environment.core configuration_dirs = configuration_environment.directories (configuration_dirs[0] / 'api-keys.yaml').write_text( - 'apikeys:\n brave:\n key: brave-key\n censys:\n id: censys-id\n secret: censys-secret\n', + 'apikeys:\n bevigil:\n key: bevigil-key\n censys:\n id: censys-id\n secret: censys-secret\n', encoding='utf-8', ) - assert (core.brave_key(), core.censys_key()) == ( - 'brave-key', + assert (core.bevigil_key(), core.censys_key()) == ( + 'bevigil-key', ('censys-id', 'censys-secret'), ) diff --git a/tests/test_bravesearch.py b/tests/test_bravesearch.py new file mode 100644 index 00000000..35a8495c --- /dev/null +++ b/tests/test_bravesearch.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from typing import Any + +import pytest + +from theHarvester.discovery.bravesearch import SearchBrave +from theHarvester.discovery.constants import MissingKey +from theHarvester.lib.configuration import InMemoryCredentialAdapter +from theHarvester.lib.core import AsyncFetcher + + +@pytest.mark.asyncio +async def test_brave_collects_with_in_memory_credentials(monkeypatch: pytest.MonkeyPatch) -> None: + request_headers: list[dict[str, str]] = [] + + async def fetch(*, headers: dict[str, str], **_kwargs: Any) -> dict[str, Any]: + request_headers.append(headers) + return { + 'web': { + 'results': [ + { + 'title': 'Documentation', + 'description': 'Example documentation', + 'url': 'https://docs.example.com', + } + ] + } + } + + monkeypatch.setattr(AsyncFetcher, 'fetch', fetch) + search = SearchBrave( + 'example.com', + 1, + credential_adapter=InMemoryCredentialAdapter({'brave': {'key': 'memory-key'}}), + ) + + await search.process() + + assert set(await search.get_hostnames()) == {'docs.example.com'} + assert request_headers + assert {headers['X-Subscription-Token'] for headers in request_headers} == {'memory-key'} + + +def test_brave_rejects_empty_in_memory_credentials() -> None: + credentials = InMemoryCredentialAdapter({'brave': {'key': ''}}) + + with pytest.raises(MissingKey, match='Brave Search'): + SearchBrave('example.com', 1, credential_adapter=credentials) diff --git a/theHarvester/discovery/bravesearch.py b/theHarvester/discovery/bravesearch.py index 82afc7bc..fe9e75e5 100644 --- a/theHarvester/discovery/bravesearch.py +++ b/theHarvester/discovery/bravesearch.py @@ -1,20 +1,23 @@ import asyncio import logging +from typing import Any from urllib.parse import quote from theHarvester.discovery.constants import MissingKey, get_delay -from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.configuration import CredentialAdapter, FileSystemCredentialAdapter +from theHarvester.lib.core import AsyncFetcher from theHarvester.parsers import myparser logger = logging.getLogger(__name__) class SearchBrave: - def __init__(self, word, limit): + def __init__(self, word: str, limit: int, credential_adapter: CredentialAdapter | None = None) -> None: self.word = word - self.results = [] + self.results: list[dict[str, Any]] = [] self.totalresults = '' - self.api_key = Core.brave_key() + credentials = credential_adapter if credential_adapter is not None else FileSystemCredentialAdapter() + self.api_key = credentials.get('brave') if self.api_key is None or self.api_key == '': raise MissingKey('Brave Search') self.server = 'https://api.search.brave.com/res/v1/web/search' diff --git a/theHarvester/lib/configuration.py b/theHarvester/lib/configuration.py new file mode 100644 index 00000000..f425efd7 --- /dev/null +++ b/theHarvester/lib/configuration.py @@ -0,0 +1,21 @@ +from collections.abc import Mapping +from dataclasses import dataclass + +from theHarvester.lib.core import Core + + +class FileSystemCredentialAdapter: + @staticmethod + def get(provider: str, field: str = 'key') -> str: + return Core.api_keys()[provider][field] + + +@dataclass(frozen=True) +class InMemoryCredentialAdapter: + credentials: Mapping[str, Mapping[str, str]] + + def get(self, provider: str, field: str = 'key') -> str: + return self.credentials[provider][field] + + +type CredentialAdapter = FileSystemCredentialAdapter | InMemoryCredentialAdapter diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index f62d015d..3c3a55d4 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -112,10 +112,6 @@ class Core: def bitbucket_key() -> str: return Core._api_key_value('bitbucket') - @staticmethod - def brave_key() -> str: - return Core._api_key_value('brave') - @staticmethod def bufferoverun_key() -> str: return Core._api_key_value('bufferoverun') From bb0d7e2d5ea611de257e2f1d86613d6c9352edb6 Mon Sep 17 00:00:00 2001 From: NotoriousRebel <36310667+NotoriousRebel@users.noreply.github.com> Date: Mon, 20 Jul 2026 23:19:12 -0400 Subject: [PATCH 4/5] docs: explain configuration boundaries --- theHarvester/discovery/bravesearch.py | 9 +++++++++ theHarvester/lib/configuration.py | 10 ++++++++++ theHarvester/lib/core.py | 2 ++ 3 files changed, 21 insertions(+) diff --git a/theHarvester/discovery/bravesearch.py b/theHarvester/discovery/bravesearch.py index fe9e75e5..cfdc42a5 100644 --- a/theHarvester/discovery/bravesearch.py +++ b/theHarvester/discovery/bravesearch.py @@ -12,6 +12,15 @@ logger = logging.getLogger(__name__) class SearchBrave: + """Search Brave while allowing credentials to be supplied without file access. + + Provider API: + https://api-dashboard.search.brave.com/app/documentation/web-search/query + + Filesystem credentials remain the production default; injection keeps tests and + embedded use independent of operator configuration files. + """ + def __init__(self, word: str, limit: int, credential_adapter: CredentialAdapter | None = None) -> None: self.word = word self.results: list[dict[str, Any]] = [] diff --git a/theHarvester/lib/configuration.py b/theHarvester/lib/configuration.py index f425efd7..d90c5f8a 100644 --- a/theHarvester/lib/configuration.py +++ b/theHarvester/lib/configuration.py @@ -1,3 +1,9 @@ +"""Credential boundaries that keep provider code independent of configuration files. + +Production access retains Core's existing file precedence. In-memory access lets +tests and embedded callers provide credentials without filesystem or global state. +""" + from collections.abc import Mapping from dataclasses import dataclass @@ -5,6 +11,8 @@ from theHarvester.lib.core import Core class FileSystemCredentialAdapter: + """Read production credentials through Core's existing file precedence.""" + @staticmethod def get(provider: str, field: str = 'key') -> str: return Core.api_keys()[provider][field] @@ -12,6 +20,8 @@ class FileSystemCredentialAdapter: @dataclass(frozen=True) class InMemoryCredentialAdapter: + """Provide credentials directly for isolated tests and programmatic callers.""" + credentials: Mapping[str, Mapping[str, str]] def get(self, provider: str, field: str = 'key') -> str: diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index 3c3a55d4..a2709c33 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -413,6 +413,8 @@ class AsyncFetcher: @property def proxy_list(self) -> dict: + """Load and cache proxies on first use instead of during module import.""" + proxy_list = self.__class__._proxy_list if proxy_list is None: proxy_list = Core.proxy_list() From 77d7e46032da6a57ba654bf9877f211c7184e168 Mon Sep 17 00:00:00 2001 From: NotoriousRebel <36310667+NotoriousRebel@users.noreply.github.com> Date: Tue, 28 Jul 2026 20:13:42 -0400 Subject: [PATCH 5/5] fix: preserve Brave credential accessor --- tests/lib/test_configuration_contract.py | 13 +++++++++++++ theHarvester/lib/core.py | 4 ++++ 2 files changed, 17 insertions(+) diff --git a/tests/lib/test_configuration_contract.py b/tests/lib/test_configuration_contract.py index 58838093..5eb5c0dd 100644 --- a/tests/lib/test_configuration_contract.py +++ b/tests/lib/test_configuration_contract.py @@ -171,6 +171,19 @@ def test_provider_accessors_return_single_and_multi_field_credentials( ) +def test_brave_key_returns_configured_value( + configuration_environment: ConfigurationEnvironment, +) -> None: + core = configuration_environment.core + configuration_dirs = configuration_environment.directories + (configuration_dirs[0] / 'api-keys.yaml').write_text( + 'apikeys:\n brave:\n key: brave-key\n', + encoding='utf-8', + ) + + assert core.brave_key() == 'brave-key' + + @pytest.mark.parametrize( ('contents', 'expected'), [ diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index a2709c33..0e9a00a0 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -112,6 +112,10 @@ class Core: def bitbucket_key() -> str: return Core._api_key_value('bitbucket') + @staticmethod + def brave_key() -> str: + return Core._api_key_value('brave') + @staticmethod def bufferoverun_key() -> str: return Core._api_key_value('bufferoverun')