From a599e8e13e2657de62ac4271cf2150398308b1ad Mon Sep 17 00:00:00 2001 From: OsirisTechnicalInstitute <213735593+OsirisTechnicalInstitute@users.noreply.github.com> Date: Mon, 5 Oct 2026 04:27:21 +0000 Subject: [PATCH 1/3] feat(discovery): add otilabs source Subdomains and live host addresses from the OTI Labs Domain Intelligence API (RapidAPI key). Co-Authored-By: Claude Opus 5.5 --- README.md | 3 +- tests/discovery/test_otilabs.py | 197 +++++++++++++++++++++++++++++ tests/test_readme.py | 3 +- theHarvester/data/api-keys.yaml | 3 + theHarvester/discovery/otilabs.py | 102 +++++++++++++++ theHarvester/lib/core.py | 5 + theHarvester/lib/source_catalog.py | 1 + theHarvester/lib/source_runner.py | 2 + 8 files changed, 314 insertions(+), 2 deletions(-) create mode 100644 tests/discovery/test_otilabs.py create mode 100644 theHarvester/discovery/otilabs.py diff --git a/README.md b/README.md index 90bc1566..88d81e44 100644 --- a/README.md +++ b/README.md @@ -223,7 +223,7 @@ Result types in this table always appear in this order: `subdomains`, `emails`, The `shodan` source contributes subdomains. Shodan host enrichment through `-s` or `--shodan` is a separate action and is not a source result route.
-View all 60 discovery sources +View all 61 discovery sources | Source | Returns | Activity | API key | | --- | --- | :---: | :---: | @@ -262,6 +262,7 @@ The `shodan` source contributes subdomains. Shodan host enrichment through `-s` | [`mojeek`](https://www.mojeek.com/services/search/web-search-api/) | subdomains, emails | P0 | Optional | | [`netlas`](https://netlas.io/) | subdomains only | P0 | Required | | [`onyphe`](https://www.onyphe.io/) | subdomains, ips, asns | P0 | Required | +| [`otilabs`](https://oti-labs.com/domain-intelligence-api) | subdomains, ips | P0 | Required | | [`otx`](https://otx.alienvault.com/) | subdomains, ips | P0 | No | | [`pentesttools`](https://pentest-tools.com/) | subdomains, ips | P1 | Required | | [`projectdiscovery`](https://chaos.projectdiscovery.io/) | subdomains only | P0 | Required | diff --git a/tests/discovery/test_otilabs.py b/tests/discovery/test_otilabs.py new file mode 100644 index 00000000..1d0aba58 --- /dev/null +++ b/tests/discovery/test_otilabs.py @@ -0,0 +1,197 @@ +from __future__ import annotations + +import contextlib +from typing import TYPE_CHECKING, Any + +import pytest + +from theHarvester.discovery import otilabs +from theHarvester.discovery.constants import MissingKey +from theHarvester.lib.core import FetcherResponse + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + +class ProviderSession: + def __init__(self) -> None: + self.kwargs: dict[str, Any] = {} + self.exited = False + + +@pytest.fixture(autouse=True) +def provider_session(monkeypatch: pytest.MonkeyPatch) -> ProviderSession: + session = ProviderSession() + + @contextlib.asynccontextmanager + async def fake_open_session(**kwargs: Any) -> AsyncIterator[object]: + session.kwargs = kwargs + try: + yield session + finally: + session.exited = True + + monkeypatch.setattr(otilabs.AsyncFetcher, 'open_session', fake_open_session) + monkeypatch.setattr(otilabs.Core, 'otilabs_key', staticmethod(lambda: 'test-key')) + return session + + +def respond(monkeypatch: pytest.MonkeyPatch, response: Any) -> list[dict[str, Any]]: + calls: list[dict[str, Any]] = [] + + async def fake_fetch(**kwargs: Any) -> Any: + calls.append(kwargs) + return response + + monkeypatch.setattr(otilabs.AsyncFetcher, 'fetch', fake_fetch) + return calls + + +def body(**overrides: Any) -> dict[str, Any]: + payload: dict[str, Any] = { + 'count': 4, + 'live_count': 2, + 'returned': 4, + 'subdomains': ['WWW.Example.COM.', 'api.example.com', 'old.example.com', 'cdn.outside.test'], + 'live': [ + {'host': 'www.example.com', 'ip': '192.0.2.10'}, + {'host': 'api.example.com', 'ip': 'edge.provider.test'}, + ], + 'pools': [], + 'sources_used': ['certspotter: 3 found', 'dns-brute: 1 found'], + } + payload.update(overrides) + return payload + + +@pytest.mark.provider_contract('otilabs') +@pytest.mark.asyncio +async def test_scoped_hosts_and_live_addresses_are_returned( + monkeypatch: pytest.MonkeyPatch, + provider_session: ProviderSession, +) -> None: + calls = respond(monkeypatch, FetcherResponse(body(), 200, {})) + search = otilabs.SearchOTILabs('example.com') + report = await search.process(proxy=True) + + assert report is None + assert await search.get_hostnames() == {'www.example.com', 'api.example.com', 'old.example.com'} + assert await search.get_ips() == {'192.0.2.10'} + assert [call['url'] for call in calls] == [ + 'https://domain-intelligence-api.p.rapidapi.com/domain/example.com/subdomains' + ] + assert calls[0]['session'] is provider_session + assert calls[0]['params'] == {'wait': '1'} + assert calls[0]['json'] is True and calls[0]['include_metadata'] is True + assert provider_session.kwargs['headers'] == { + 'X-RapidAPI-Key': 'test-key', + 'X-RapidAPI-Host': 'domain-intelligence-api.p.rapidapi.com', + } + assert provider_session.kwargs['proxy'] is True + assert provider_session.exited is True + + +@pytest.mark.parametrize('key', [None, '', ' ']) +def test_missing_or_blank_key_fails_closed(monkeypatch: pytest.MonkeyPatch, key: str | None) -> None: + monkeypatch.setattr(otilabs.Core, 'otilabs_key', staticmethod(lambda: key)) + with pytest.raises(MissingKey): + otilabs.SearchOTILabs('example.com') + + +@pytest.mark.parametrize( + ('response', 'status', 'reason'), + [ + (None, 'failed', 'transport-error'), + (FetcherResponse({}, 401, {}), 'failed', 'access-denied'), + (FetcherResponse({}, 403, {}), 'failed', 'access-denied'), + (FetcherResponse({}, 429, {}), 'rate-limited', 'http-429'), + (FetcherResponse({}, 503, {}), 'failed', 'http-503'), + (FetcherResponse([], 200, {}), 'failed', 'invalid-response'), + (FetcherResponse({'subdomains': 'many'}, 200, {}), 'failed', 'invalid-response'), + (FetcherResponse(body(live='many'), 200, {}), 'failed', 'invalid-response'), + ], +) +@pytest.mark.asyncio +async def test_provider_failures_are_truthful( + monkeypatch: pytest.MonkeyPatch, + provider_session: ProviderSession, + response: FetcherResponse | None, + status: str, + reason: str, +) -> None: + respond(monkeypatch, response) + search = otilabs.SearchOTILabs('example.com') + report = await search.process() + + assert report is not None + assert report.status == status + assert report.stop_reason == reason + assert provider_session.exited is True + + +@pytest.mark.asyncio +async def test_transport_exception_closes_the_session( + monkeypatch: pytest.MonkeyPatch, + provider_session: ProviderSession, +) -> None: + async def failing_fetch(**_kwargs: Any) -> Any: + raise RuntimeError('connection reset') + + monkeypatch.setattr(otilabs.AsyncFetcher, 'fetch', failing_fetch) + search = otilabs.SearchOTILabs('example.com') + report = await search.process() + + assert report is not None + assert (report.status, report.stop_reason) == ('failed', 'transport-error') + assert provider_session.exited is True + + +@pytest.mark.asyncio +async def test_malformed_items_keep_valid_results(monkeypatch: pytest.MonkeyPatch) -> None: + payload = body( + subdomains=['ok.example.com', 7], + live=[{'host': 'ok.example.com', 'ip': '192.0.2.20'}, 'bad', {'ip': '192.0.2.30'}], + ) + respond(monkeypatch, FetcherResponse(payload, 200, {})) + search = otilabs.SearchOTILabs('example.com') + report = await search.process() + + assert report is not None + assert (report.status, report.stop_reason) == ('failed', 'invalid-response') + assert await search.get_hostnames() == {'ok.example.com'} + assert await search.get_ips() == {'192.0.2.20'} + + +@pytest.mark.asyncio +async def test_pending_live_check_is_reported_as_partial(monkeypatch: pytest.MonkeyPatch) -> None: + payload = body(live_count=None, live=[], sources_used=['certspotter: 3 found', 'liveness: enriching']) + payload.pop('live') + respond(monkeypatch, FetcherResponse(payload, 200, {})) + search = otilabs.SearchOTILabs('example.com') + report = await search.process() + + assert report is not None + assert (report.status, report.stop_reason) == ('partial', 'enrichment-pending') + assert await search.get_hostnames() == {'www.example.com', 'api.example.com', 'old.example.com'} + assert await search.get_ips() == set() + + +@pytest.mark.asyncio +async def test_capped_list_is_reported_as_partial(monkeypatch: pytest.MonkeyPatch) -> None: + respond(monkeypatch, FetcherResponse(body(count=2500, returned=4), 200, {})) + search = otilabs.SearchOTILabs('example.com') + report = await search.process() + + assert report is not None + assert (report.status, report.stop_reason) == ('partial', 'result-cap') + + +@pytest.mark.asyncio +async def test_empty_result_completes_normally(monkeypatch: pytest.MonkeyPatch) -> None: + respond(monkeypatch, FetcherResponse(body(count=0, live_count=0, returned=0, subdomains=[], live=[]), 200, {})) + search = otilabs.SearchOTILabs('example.com') + report = await search.process() + + assert report is None + assert await search.get_hostnames() == set() + assert await search.get_ips() == set() diff --git a/tests/test_readme.py b/tests/test_readme.py index 6fd7d3c0..989ffd94 100644 --- a/tests/test_readme.py +++ b/tests/test_readme.py @@ -54,6 +54,7 @@ SOURCE_PROVIDER_LINKS = { 'mojeek': 'https://www.mojeek.com/services/search/web-search-api/', 'netlas': 'https://netlas.io/', 'onyphe': 'https://www.onyphe.io/', + 'otilabs': 'https://oti-labs.com/domain-intelligence-api', 'otx': 'https://otx.alienvault.com/', 'pentesttools': 'https://pentest-tools.com/', 'projectdiscovery': 'https://chaos.projectdiscovery.io/', @@ -154,7 +155,7 @@ def test_readme_matches_declared_source_contracts() -> None: assert _source_matrix(readme).count('| Source | Returns | Activity | API key |') == 1 assert 'Credentials |' not in _source_matrix(readme) - assert len(declared) == 60 + assert len(declared) == 61 assert len(documented) == len(declared) assert documented == declared source_links = _documented_source_links(readme) diff --git a/theHarvester/data/api-keys.yaml b/theHarvester/data/api-keys.yaml index 81f03de2..e004cea6 100644 --- a/theHarvester/data/api-keys.yaml +++ b/theHarvester/data/api-keys.yaml @@ -74,6 +74,9 @@ apikeys: onyphe: key: + otilabs: + key: + pentestTools: key: diff --git a/theHarvester/discovery/otilabs.py b/theHarvester/discovery/otilabs.py new file mode 100644 index 00000000..7ca227f9 --- /dev/null +++ b/theHarvester/discovery/otilabs.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import logging +from ipaddress import ip_address + +from theHarvester.discovery.constants import MissingKey +from theHarvester.discovery.provider_response import provider_http_error +from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse +from theHarvester.lib.hostnames import normalize_scoped_hostname +from theHarvester.lib.source_execution import SourceExecutionReport + +logger = logging.getLogger(__name__) + + +class SearchOTILabs: + """Search the OTI Labs Domain Intelligence API for subdomains and live host addresses. + + One request to the subdomains endpoint returns the names the API found in certificate + transparency logs and passive DNS, and the hosts that resolved when it last checked them. + ``wait=1`` asks the API to wait for all of its upstream sources before answering. + """ + + API_HOST = 'domain-intelligence-api.p.rapidapi.com' + + def __init__(self, word: str) -> None: + self.word = word + self.key = Core.otilabs_key() + if not isinstance(self.key, str) or not self.key.strip(): + raise MissingKey('otilabs') + self.totalhosts: set[str] = set() + self.totalips: set[str] = set() + self.proxy = False + + async def do_search(self) -> SourceExecutionReport | None: + headers = {'X-RapidAPI-Key': self.key, 'X-RapidAPI-Host': self.API_HOST} + async with AsyncFetcher.open_session(headers=headers, proxy=self.proxy, request_timeout=60) as session: + response = await AsyncFetcher.fetch( + session=session, + url=f'https://{self.API_HOST}/domain/{self.word}/subdomains', + params={'wait': '1'}, + json=True, + include_metadata=True, + ) + if error := provider_http_error(response): + return SourceExecutionReport(*error) + assert isinstance(response, FetcherResponse) + body = response.body + if not isinstance(body, dict) or not isinstance(body.get('subdomains'), list): + return SourceExecutionReport('failed', 'invalid-response') + live = body.get('live', []) + if not isinstance(live, list): + return SourceExecutionReport('failed', 'invalid-response') + + malformed = False + for name in body['subdomains']: + if not isinstance(name, str): + malformed = True + continue + if hostname := normalize_scoped_hostname(name, self.word): + self.totalhosts.add(hostname) + for item in live: + if not isinstance(item, dict) or not isinstance(item.get('host'), str): + malformed = True + continue + hostname = normalize_scoped_hostname(item['host'], self.word) + if hostname is None: + continue + self.totalhosts.add(hostname) + address = item.get('ip') + if isinstance(address, str): + try: + self.totalips.add(str(ip_address(address))) + except ValueError: + pass # a CNAME target rather than an address + if malformed: + return SourceExecutionReport('failed', 'invalid-response') + + sources = body.get('sources_used') + pending = isinstance(sources, list) and any( + isinstance(source, str) and source.endswith(': enriching') for source in sources + ) + if pending or body.get('live_count', 0) is None: + # The API answered before its live check finished, so a later run can return more addresses. + return SourceExecutionReport('partial', 'enrichment-pending') + count, returned = body.get('count'), body.get('returned') + if isinstance(count, int) and isinstance(returned, int) and count > returned: + return SourceExecutionReport('partial', 'result-cap') + return None + + async def get_hostnames(self) -> set[str]: + return self.totalhosts + + async def get_ips(self) -> set[str]: + return self.totalips + + async def process(self, proxy: bool = False) -> SourceExecutionReport | None: + self.proxy = proxy + try: + return await self.do_search() + except Exception as error: + logger.info('OTI Labs search failed: %s', type(error).__name__) + return SourceExecutionReport('failed', 'transport-error') diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index 1626d995..3a20f222 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -222,6 +222,7 @@ class Core: 'mojeek': ('key',), 'netlas': ('key',), 'onyphe': ('key',), + 'otilabs': ('key',), 'pentestTools': ('key',), 'projectDiscovery': ('key',), 'rocketreach': ('key',), @@ -391,6 +392,10 @@ class Core: def onyphe_key() -> str: return Core._api_key_value('onyphe') + @staticmethod + def otilabs_key() -> str: + return Core._api_key_value('otilabs') + @staticmethod def pentest_tools_key() -> str: return Core._api_key_value('pentestTools') diff --git a/theHarvester/lib/source_catalog.py b/theHarvester/lib/source_catalog.py index d95210f0..57dd280a 100644 --- a/theHarvester/lib/source_catalog.py +++ b/theHarvester/lib/source_catalog.py @@ -159,6 +159,7 @@ _SPECS = ( _spec('mojeek', ResultRoute.SUBDOMAINS, ResultRoute.EMAILS), _spec('netlas', ResultRoute.SUBDOMAINS), _spec('onyphe', ResultRoute.SUBDOMAINS, ResultRoute.IPS, ResultRoute.ASNS), + _spec('otilabs', ResultRoute.SUBDOMAINS, ResultRoute.IPS), _spec('otx', ResultRoute.SUBDOMAINS, ResultRoute.IPS), _spec( 'pentesttools', diff --git a/theHarvester/lib/source_runner.py b/theHarvester/lib/source_runner.py index 433e7969..feee70e7 100644 --- a/theHarvester/lib/source_runner.py +++ b/theHarvester/lib/source_runner.py @@ -41,6 +41,7 @@ from theHarvester.discovery import ( mojeek, netlas, onyphe, + otilabs, otxsearch, pentesttools, projectdiscovery, @@ -154,6 +155,7 @@ SOURCE_FACTORIES: dict[str, SourceFactory] = { 'mojeek': lambda request: mojeek.SearchMojeek(request.target, request.limit), 'netlas': lambda request: netlas.SearchNetlas(request.target, request.limit), 'onyphe': lambda request: onyphe.SearchOnyphe(request.target, request.limit), + 'otilabs': lambda request: otilabs.SearchOTILabs(request.target), 'otx': lambda request: otxsearch.SearchOtx(request.target), 'pentesttools': lambda request: pentesttools.SearchPentestTools(request.target), 'projectdiscovery': lambda request: projectdiscovery.SearchDiscovery(request.target), From e819792d064c9e14da3a04c6e89d453d74a805b2 Mon Sep 17 00:00:00 2001 From: OsirisTechnicalInstitute <213735593+OsirisTechnicalInstitute@users.noreply.github.com> Date: Mon, 5 Oct 2026 23:51:16 +0000 Subject: [PATCH 2/3] fix(discovery): classify otilabs as P1 DNS activity The API brute-forces common names and resolves what it finds through public resolvers, so the source is excluded from -b all and runs only when selected explicitly. Adds otilabs to the non-passive coverage in test_core and the CLI orchestration test, and asserts the activity class and selection in its contract module. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- tests/discovery/test_otilabs.py | 13 ++++++++++--- tests/lib/test_core.py | 3 ++- tests/test_all_source_orchestration.py | 2 ++ theHarvester/discovery/otilabs.py | 2 ++ theHarvester/lib/source_catalog.py | 7 ++++++- 6 files changed, 23 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 88d81e44..129415a7 100644 --- a/README.md +++ b/README.md @@ -262,7 +262,7 @@ The `shodan` source contributes subdomains. Shodan host enrichment through `-s` | [`mojeek`](https://www.mojeek.com/services/search/web-search-api/) | subdomains, emails | P0 | Optional | | [`netlas`](https://netlas.io/) | subdomains only | P0 | Required | | [`onyphe`](https://www.onyphe.io/) | subdomains, ips, asns | P0 | Required | -| [`otilabs`](https://oti-labs.com/domain-intelligence-api) | subdomains, ips | P0 | Required | +| [`otilabs`](https://oti-labs.com/domain-intelligence-api) | subdomains, ips | P1 | Required | | [`otx`](https://otx.alienvault.com/) | subdomains, ips | P0 | No | | [`pentesttools`](https://pentest-tools.com/) | subdomains, ips | P1 | Required | | [`projectdiscovery`](https://chaos.projectdiscovery.io/) | subdomains only | P0 | Required | diff --git a/tests/discovery/test_otilabs.py b/tests/discovery/test_otilabs.py index 1d0aba58..696e4657 100644 --- a/tests/discovery/test_otilabs.py +++ b/tests/discovery/test_otilabs.py @@ -8,6 +8,7 @@ import pytest from theHarvester.discovery import otilabs from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import FetcherResponse +from theHarvester.lib.source_catalog import ActivityClass, activity_classes_for_selection, get_source_spec, resolve_sources if TYPE_CHECKING: from collections.abc import AsyncIterator @@ -77,9 +78,7 @@ async def test_scoped_hosts_and_live_addresses_are_returned( assert report is None assert await search.get_hostnames() == {'www.example.com', 'api.example.com', 'old.example.com'} assert await search.get_ips() == {'192.0.2.10'} - assert [call['url'] for call in calls] == [ - 'https://domain-intelligence-api.p.rapidapi.com/domain/example.com/subdomains' - ] + assert [call['url'] for call in calls] == ['https://domain-intelligence-api.p.rapidapi.com/domain/example.com/subdomains'] assert calls[0]['session'] is provider_session assert calls[0]['params'] == {'wait': '1'} assert calls[0]['json'] is True and calls[0]['include_metadata'] is True @@ -195,3 +194,11 @@ async def test_empty_result_completes_normally(monkeypatch: pytest.MonkeyPatch) assert report is None assert await search.get_hostnames() == set() assert await search.get_ips() == set() + + +def test_source_is_p1_dns_activity_and_runs_only_when_selected() -> None: + assert get_source_spec('otilabs').activity is ActivityClass.DNS + assert activity_classes_for_selection(['otilabs']) == (ActivityClass.DNS,) + assert 'otilabs' not in resolve_sources('all') + assert resolve_sources('otilabs') == ['otilabs'] + assert resolve_sources('OTILabs') == ['otilabs'] diff --git a/tests/lib/test_core.py b/tests/lib/test_core.py index 2fc95b19..63ea9757 100644 --- a/tests/lib/test_core.py +++ b/tests/lib/test_core.py @@ -106,6 +106,7 @@ def test_all_selects_only_passive_catalog_sources() -> None: name: spec.activity for name, spec in SOURCE_SPECS.items() if spec.activity is not ActivityClass.PASSIVE } == { "criminalip": ActivityClass.DIRECT, + "otilabs": ActivityClass.DNS, "pentesttools": ActivityClass.DNS, "shodan": ActivityClass.DNS, "shodanInternetDB": ActivityClass.DNS, @@ -115,7 +116,7 @@ def test_all_selects_only_passive_catalog_sources() -> None: @pytest.mark.parametrize( 'source', - ['criminalip', 'pentesttools', 'shodan', 'shodanInternetDB', 'subdomainfinderc99'], + ['criminalip', 'otilabs', 'pentesttools', 'shodan', 'shodanInternetDB', 'subdomainfinderc99'], ) def test_non_passive_sources_run_only_when_explicitly_selected(source: str) -> None: assert source not in resolve_sources('all') diff --git a/tests/test_all_source_orchestration.py b/tests/test_all_source_orchestration.py index 36ba2915..a67c43df 100644 --- a/tests/test_all_source_orchestration.py +++ b/tests/test_all_source_orchestration.py @@ -15,6 +15,7 @@ from theHarvester.lib.source_catalog import SOURCE_SPECS, ActivityClass, ResultR NON_PASSIVE_SOURCES = ( 'criminalip', + 'otilabs', 'pentesttools', 'shodan', 'shodanInternetDB', @@ -217,6 +218,7 @@ async def test_explicit_non_passive_source_is_scheduled_once( else: module, constructor_name = { 'criminalip': (source_runner.criminalip, 'SearchCriminalIP'), + 'otilabs': (source_runner.otilabs, 'SearchOTILabs'), 'pentesttools': (source_runner.pentesttools, 'SearchPentestTools'), 'shodanInternetDB': (source_runner.shodan_internetdb, 'SearchShodanInternetDB'), 'subdomainfinderc99': (source_runner.subdomainfinderc99, 'SearchSubdomainfinderc99'), diff --git a/theHarvester/discovery/otilabs.py b/theHarvester/discovery/otilabs.py index 7ca227f9..c997fd7e 100644 --- a/theHarvester/discovery/otilabs.py +++ b/theHarvester/discovery/otilabs.py @@ -17,6 +17,8 @@ class SearchOTILabs: One request to the subdomains endpoint returns the names the API found in certificate transparency logs and passive DNS, and the hosts that resolved when it last checked them. + To build that list the API also brute-forces common names and resolves what it finds through + public DNS resolvers, so the source is P1 (DNS) and runs only when selected explicitly. ``wait=1`` asks the API to wait for all of its upstream sources before answering. """ diff --git a/theHarvester/lib/source_catalog.py b/theHarvester/lib/source_catalog.py index 57dd280a..c51fbf95 100644 --- a/theHarvester/lib/source_catalog.py +++ b/theHarvester/lib/source_catalog.py @@ -159,7 +159,12 @@ _SPECS = ( _spec('mojeek', ResultRoute.SUBDOMAINS, ResultRoute.EMAILS), _spec('netlas', ResultRoute.SUBDOMAINS), _spec('onyphe', ResultRoute.SUBDOMAINS, ResultRoute.IPS, ResultRoute.ASNS), - _spec('otilabs', ResultRoute.SUBDOMAINS, ResultRoute.IPS), + _spec( + 'otilabs', + ResultRoute.SUBDOMAINS, + ResultRoute.IPS, + activity=ActivityClass.DNS, + ), _spec('otx', ResultRoute.SUBDOMAINS, ResultRoute.IPS), _spec( 'pentesttools', From adc33a83d0f97d339cddcf172a9bd0f62692a111 Mon Sep 17 00:00:00 2001 From: OsirisTechnicalInstitute <213735593+OsirisTechnicalInstitute@users.noreply.github.com> Date: Wed, 7 Oct 2026 00:19:07 +0000 Subject: [PATCH 3/3] fix(discovery): classify otilabs as P0 again P1 describes DNS operations theHarvester runs itself, not work a provider does on its side (per review), so otilabs is passive and included in -b all. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- tests/discovery/test_otilabs.py | 13 +++---------- tests/lib/test_core.py | 3 +-- tests/test_all_source_orchestration.py | 2 -- theHarvester/discovery/otilabs.py | 2 -- theHarvester/lib/source_catalog.py | 7 +------ 6 files changed, 6 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 129415a7..88d81e44 100644 --- a/README.md +++ b/README.md @@ -262,7 +262,7 @@ The `shodan` source contributes subdomains. Shodan host enrichment through `-s` | [`mojeek`](https://www.mojeek.com/services/search/web-search-api/) | subdomains, emails | P0 | Optional | | [`netlas`](https://netlas.io/) | subdomains only | P0 | Required | | [`onyphe`](https://www.onyphe.io/) | subdomains, ips, asns | P0 | Required | -| [`otilabs`](https://oti-labs.com/domain-intelligence-api) | subdomains, ips | P1 | Required | +| [`otilabs`](https://oti-labs.com/domain-intelligence-api) | subdomains, ips | P0 | Required | | [`otx`](https://otx.alienvault.com/) | subdomains, ips | P0 | No | | [`pentesttools`](https://pentest-tools.com/) | subdomains, ips | P1 | Required | | [`projectdiscovery`](https://chaos.projectdiscovery.io/) | subdomains only | P0 | Required | diff --git a/tests/discovery/test_otilabs.py b/tests/discovery/test_otilabs.py index 696e4657..1d0aba58 100644 --- a/tests/discovery/test_otilabs.py +++ b/tests/discovery/test_otilabs.py @@ -8,7 +8,6 @@ import pytest from theHarvester.discovery import otilabs from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import FetcherResponse -from theHarvester.lib.source_catalog import ActivityClass, activity_classes_for_selection, get_source_spec, resolve_sources if TYPE_CHECKING: from collections.abc import AsyncIterator @@ -78,7 +77,9 @@ async def test_scoped_hosts_and_live_addresses_are_returned( assert report is None assert await search.get_hostnames() == {'www.example.com', 'api.example.com', 'old.example.com'} assert await search.get_ips() == {'192.0.2.10'} - assert [call['url'] for call in calls] == ['https://domain-intelligence-api.p.rapidapi.com/domain/example.com/subdomains'] + assert [call['url'] for call in calls] == [ + 'https://domain-intelligence-api.p.rapidapi.com/domain/example.com/subdomains' + ] assert calls[0]['session'] is provider_session assert calls[0]['params'] == {'wait': '1'} assert calls[0]['json'] is True and calls[0]['include_metadata'] is True @@ -194,11 +195,3 @@ async def test_empty_result_completes_normally(monkeypatch: pytest.MonkeyPatch) assert report is None assert await search.get_hostnames() == set() assert await search.get_ips() == set() - - -def test_source_is_p1_dns_activity_and_runs_only_when_selected() -> None: - assert get_source_spec('otilabs').activity is ActivityClass.DNS - assert activity_classes_for_selection(['otilabs']) == (ActivityClass.DNS,) - assert 'otilabs' not in resolve_sources('all') - assert resolve_sources('otilabs') == ['otilabs'] - assert resolve_sources('OTILabs') == ['otilabs'] diff --git a/tests/lib/test_core.py b/tests/lib/test_core.py index 63ea9757..2fc95b19 100644 --- a/tests/lib/test_core.py +++ b/tests/lib/test_core.py @@ -106,7 +106,6 @@ def test_all_selects_only_passive_catalog_sources() -> None: name: spec.activity for name, spec in SOURCE_SPECS.items() if spec.activity is not ActivityClass.PASSIVE } == { "criminalip": ActivityClass.DIRECT, - "otilabs": ActivityClass.DNS, "pentesttools": ActivityClass.DNS, "shodan": ActivityClass.DNS, "shodanInternetDB": ActivityClass.DNS, @@ -116,7 +115,7 @@ def test_all_selects_only_passive_catalog_sources() -> None: @pytest.mark.parametrize( 'source', - ['criminalip', 'otilabs', 'pentesttools', 'shodan', 'shodanInternetDB', 'subdomainfinderc99'], + ['criminalip', 'pentesttools', 'shodan', 'shodanInternetDB', 'subdomainfinderc99'], ) def test_non_passive_sources_run_only_when_explicitly_selected(source: str) -> None: assert source not in resolve_sources('all') diff --git a/tests/test_all_source_orchestration.py b/tests/test_all_source_orchestration.py index a67c43df..36ba2915 100644 --- a/tests/test_all_source_orchestration.py +++ b/tests/test_all_source_orchestration.py @@ -15,7 +15,6 @@ from theHarvester.lib.source_catalog import SOURCE_SPECS, ActivityClass, ResultR NON_PASSIVE_SOURCES = ( 'criminalip', - 'otilabs', 'pentesttools', 'shodan', 'shodanInternetDB', @@ -218,7 +217,6 @@ async def test_explicit_non_passive_source_is_scheduled_once( else: module, constructor_name = { 'criminalip': (source_runner.criminalip, 'SearchCriminalIP'), - 'otilabs': (source_runner.otilabs, 'SearchOTILabs'), 'pentesttools': (source_runner.pentesttools, 'SearchPentestTools'), 'shodanInternetDB': (source_runner.shodan_internetdb, 'SearchShodanInternetDB'), 'subdomainfinderc99': (source_runner.subdomainfinderc99, 'SearchSubdomainfinderc99'), diff --git a/theHarvester/discovery/otilabs.py b/theHarvester/discovery/otilabs.py index c997fd7e..7ca227f9 100644 --- a/theHarvester/discovery/otilabs.py +++ b/theHarvester/discovery/otilabs.py @@ -17,8 +17,6 @@ class SearchOTILabs: One request to the subdomains endpoint returns the names the API found in certificate transparency logs and passive DNS, and the hosts that resolved when it last checked them. - To build that list the API also brute-forces common names and resolves what it finds through - public DNS resolvers, so the source is P1 (DNS) and runs only when selected explicitly. ``wait=1`` asks the API to wait for all of its upstream sources before answering. """ diff --git a/theHarvester/lib/source_catalog.py b/theHarvester/lib/source_catalog.py index c51fbf95..57dd280a 100644 --- a/theHarvester/lib/source_catalog.py +++ b/theHarvester/lib/source_catalog.py @@ -159,12 +159,7 @@ _SPECS = ( _spec('mojeek', ResultRoute.SUBDOMAINS, ResultRoute.EMAILS), _spec('netlas', ResultRoute.SUBDOMAINS), _spec('onyphe', ResultRoute.SUBDOMAINS, ResultRoute.IPS, ResultRoute.ASNS), - _spec( - 'otilabs', - ResultRoute.SUBDOMAINS, - ResultRoute.IPS, - activity=ActivityClass.DNS, - ), + _spec('otilabs', ResultRoute.SUBDOMAINS, ResultRoute.IPS), _spec('otx', ResultRoute.SUBDOMAINS, ResultRoute.IPS), _spec( 'pentesttools',