Merge pull request #2631 from OsirisTechnicalInstitute/feature/otilabs-source

README.md

@@ -223,7 +223,7 @@ Result types in this table always appear in this order: `subdomains`, `emails`,
 The `shodan` source contributes subdomains. Shodan host enrichment through `-s` or `--shodan` is a separate action and is not a source result route.
 
 <details>
-<summary><strong>View all 60 discovery sources</strong></summary>
+<summary><strong>View all 61 discovery sources</strong></summary>
 
 | Source | Returns | Activity | API key |
 | --- | --- | :---: | :---: |
@@ -262,6 +262,7 @@ The `shodan` source contributes subdomains. Shodan host enrichment through `-s`
 | [`mojeek`](https://www.mojeek.com/services/search/web-search-api/) | subdomains, emails | P0 | Optional |
 | [`netlas`](https://netlas.io/) | subdomains only | P0 | Required |
 | [`onyphe`](https://www.onyphe.io/) | subdomains, ips, asns | P0 | Required |
+| [`otilabs`](https://oti-labs.com/domain-intelligence-api) | subdomains, ips | P0 | Required |
 | [`otx`](https://otx.alienvault.com/) | subdomains, ips | P0 | No |
 | [`pentesttools`](https://pentest-tools.com/) | subdomains, ips | P1 | Required |
 | [`projectdiscovery`](https://chaos.projectdiscovery.io/) | subdomains only | P0 | Required |

tests/discovery/test_otilabs.py

@@ -0,0 +1,197 @@
+from __future__ import annotations
+
+import contextlib
+from typing import TYPE_CHECKING, Any
+
+import pytest
+
+from theHarvester.discovery import otilabs
+from theHarvester.discovery.constants import MissingKey
+from theHarvester.lib.core import FetcherResponse
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+
+
+class ProviderSession:
+    def __init__(self) -> None:
+        self.kwargs: dict[str, Any] = {}
+        self.exited = False
+
+
+@pytest.fixture(autouse=True)
+def provider_session(monkeypatch: pytest.MonkeyPatch) -> ProviderSession:
+    session = ProviderSession()
+
+    @contextlib.asynccontextmanager
+    async def fake_open_session(**kwargs: Any) -> AsyncIterator[object]:
+        session.kwargs = kwargs
+        try:
+            yield session
+        finally:
+            session.exited = True
+
+    monkeypatch.setattr(otilabs.AsyncFetcher, 'open_session', fake_open_session)
+    monkeypatch.setattr(otilabs.Core, 'otilabs_key', staticmethod(lambda: 'test-key'))
+    return session
+
+
+def respond(monkeypatch: pytest.MonkeyPatch, response: Any) -> list[dict[str, Any]]:
+    calls: list[dict[str, Any]] = []
+
+    async def fake_fetch(**kwargs: Any) -> Any:
+        calls.append(kwargs)
+        return response
+
+    monkeypatch.setattr(otilabs.AsyncFetcher, 'fetch', fake_fetch)
+    return calls
+
+
+def body(**overrides: Any) -> dict[str, Any]:
+    payload: dict[str, Any] = {
+        'count': 4,
+        'live_count': 2,
+        'returned': 4,
+        'subdomains': ['WWW.Example.COM.', 'api.example.com', 'old.example.com', 'cdn.outside.test'],
+        'live': [
+            {'host': 'www.example.com', 'ip': '192.0.2.10'},
+            {'host': 'api.example.com', 'ip': 'edge.provider.test'},
+        ],
+        'pools': [],
+        'sources_used': ['certspotter: 3 found', 'dns-brute: 1 found'],
+    }
+    payload.update(overrides)
+    return payload
+
+
+@pytest.mark.provider_contract('otilabs')
+@pytest.mark.asyncio
+async def test_scoped_hosts_and_live_addresses_are_returned(
+    monkeypatch: pytest.MonkeyPatch,
+    provider_session: ProviderSession,
+) -> None:
+    calls = respond(monkeypatch, FetcherResponse(body(), 200, {}))
+    search = otilabs.SearchOTILabs('example.com')
+    report = await search.process(proxy=True)
+
+    assert report is None
+    assert await search.get_hostnames() == {'www.example.com', 'api.example.com', 'old.example.com'}
+    assert await search.get_ips() == {'192.0.2.10'}
+    assert [call['url'] for call in calls] == [
+        'https://domain-intelligence-api.p.rapidapi.com/domain/example.com/subdomains'
+    ]
+    assert calls[0]['session'] is provider_session
+    assert calls[0]['params'] == {'wait': '1'}
+    assert calls[0]['json'] is True and calls[0]['include_metadata'] is True
+    assert provider_session.kwargs['headers'] == {
+        'X-RapidAPI-Key': 'test-key',
+        'X-RapidAPI-Host': 'domain-intelligence-api.p.rapidapi.com',
+    }
+    assert provider_session.kwargs['proxy'] is True
+    assert provider_session.exited is True
+
+
+@pytest.mark.parametrize('key', [None, '', '  '])
+def test_missing_or_blank_key_fails_closed(monkeypatch: pytest.MonkeyPatch, key: str | None) -> None:
+    monkeypatch.setattr(otilabs.Core, 'otilabs_key', staticmethod(lambda: key))
+    with pytest.raises(MissingKey):
+        otilabs.SearchOTILabs('example.com')
+
+
+@pytest.mark.parametrize(
+    ('response', 'status', 'reason'),
+    [
+        (None, 'failed', 'transport-error'),
+        (FetcherResponse({}, 401, {}), 'failed', 'access-denied'),
+        (FetcherResponse({}, 403, {}), 'failed', 'access-denied'),
+        (FetcherResponse({}, 429, {}), 'rate-limited', 'http-429'),
+        (FetcherResponse({}, 503, {}), 'failed', 'http-503'),
+        (FetcherResponse([], 200, {}), 'failed', 'invalid-response'),
+        (FetcherResponse({'subdomains': 'many'}, 200, {}), 'failed', 'invalid-response'),
+        (FetcherResponse(body(live='many'), 200, {}), 'failed', 'invalid-response'),
+    ],
+)
+@pytest.mark.asyncio
+async def test_provider_failures_are_truthful(
+    monkeypatch: pytest.MonkeyPatch,
+    provider_session: ProviderSession,
+    response: FetcherResponse | None,
+    status: str,
+    reason: str,
+) -> None:
+    respond(monkeypatch, response)
+    search = otilabs.SearchOTILabs('example.com')
+    report = await search.process()
+
+    assert report is not None
+    assert report.status == status
+    assert report.stop_reason == reason
+    assert provider_session.exited is True
+
+
+@pytest.mark.asyncio
+async def test_transport_exception_closes_the_session(
+    monkeypatch: pytest.MonkeyPatch,
+    provider_session: ProviderSession,
+) -> None:
+    async def failing_fetch(**_kwargs: Any) -> Any:
+        raise RuntimeError('connection reset')
+
+    monkeypatch.setattr(otilabs.AsyncFetcher, 'fetch', failing_fetch)
+    search = otilabs.SearchOTILabs('example.com')
+    report = await search.process()
+
+    assert report is not None
+    assert (report.status, report.stop_reason) == ('failed', 'transport-error')
+    assert provider_session.exited is True
+
+
+@pytest.mark.asyncio
+async def test_malformed_items_keep_valid_results(monkeypatch: pytest.MonkeyPatch) -> None:
+    payload = body(
+        subdomains=['ok.example.com', 7],
+        live=[{'host': 'ok.example.com', 'ip': '192.0.2.20'}, 'bad', {'ip': '192.0.2.30'}],
+    )
+    respond(monkeypatch, FetcherResponse(payload, 200, {}))
+    search = otilabs.SearchOTILabs('example.com')
+    report = await search.process()
+
+    assert report is not None
+    assert (report.status, report.stop_reason) == ('failed', 'invalid-response')
+    assert await search.get_hostnames() == {'ok.example.com'}
+    assert await search.get_ips() == {'192.0.2.20'}
+
+
+@pytest.mark.asyncio
+async def test_pending_live_check_is_reported_as_partial(monkeypatch: pytest.MonkeyPatch) -> None:
+    payload = body(live_count=None, live=[], sources_used=['certspotter: 3 found', 'liveness: enriching'])
+    payload.pop('live')
+    respond(monkeypatch, FetcherResponse(payload, 200, {}))
+    search = otilabs.SearchOTILabs('example.com')
+    report = await search.process()
+
+    assert report is not None
+    assert (report.status, report.stop_reason) == ('partial', 'enrichment-pending')
+    assert await search.get_hostnames() == {'www.example.com', 'api.example.com', 'old.example.com'}
+    assert await search.get_ips() == set()
+
+
+@pytest.mark.asyncio
+async def test_capped_list_is_reported_as_partial(monkeypatch: pytest.MonkeyPatch) -> None:
+    respond(monkeypatch, FetcherResponse(body(count=2500, returned=4), 200, {}))
+    search = otilabs.SearchOTILabs('example.com')
+    report = await search.process()
+
+    assert report is not None
+    assert (report.status, report.stop_reason) == ('partial', 'result-cap')
+
+
+@pytest.mark.asyncio
+async def test_empty_result_completes_normally(monkeypatch: pytest.MonkeyPatch) -> None:
+    respond(monkeypatch, FetcherResponse(body(count=0, live_count=0, returned=0, subdomains=[], live=[]), 200, {}))
+    search = otilabs.SearchOTILabs('example.com')
+    report = await search.process()
+
+    assert report is None
+    assert await search.get_hostnames() == set()
+    assert await search.get_ips() == set()

tests/test_readme.py

@@ -54,6 +54,7 @@ SOURCE_PROVIDER_LINKS = {
     'mojeek': 'https://www.mojeek.com/services/search/web-search-api/',
     'netlas': 'https://netlas.io/',
     'onyphe': 'https://www.onyphe.io/',
+    'otilabs': 'https://oti-labs.com/domain-intelligence-api',
     'otx': 'https://otx.alienvault.com/',
     'pentesttools': 'https://pentest-tools.com/',
     'projectdiscovery': 'https://chaos.projectdiscovery.io/',
@@ -154,7 +155,7 @@ def test_readme_matches_declared_source_contracts() -> None:
 
     assert _source_matrix(readme).count('| Source | Returns | Activity | API key |') == 1
     assert 'Credentials |' not in _source_matrix(readme)
-    assert len(declared) == 60
+    assert len(declared) == 61
     assert len(documented) == len(declared)
     assert documented == declared
     source_links = _documented_source_links(readme)

theHarvester/data/api-keys.yaml

@@ -74,6 +74,9 @@ apikeys:
   onyphe:
     key:
 
+  otilabs:
+    key:
+
   pentestTools:
     key:
 

theHarvester/discovery/otilabs.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+import logging
+from ipaddress import ip_address
+
+from theHarvester.discovery.constants import MissingKey
+from theHarvester.discovery.provider_response import provider_http_error
+from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse
+from theHarvester.lib.hostnames import normalize_scoped_hostname
+from theHarvester.lib.source_execution import SourceExecutionReport
+
+logger = logging.getLogger(__name__)
+
+
+class SearchOTILabs:
+    """Search the OTI Labs Domain Intelligence API for subdomains and live host addresses.
+
+    One request to the subdomains endpoint returns the names the API found in certificate
+    transparency logs and passive DNS, and the hosts that resolved when it last checked them.
+    ``wait=1`` asks the API to wait for all of its upstream sources before answering.
+    """
+
+    API_HOST = 'domain-intelligence-api.p.rapidapi.com'
+
+    def __init__(self, word: str) -> None:
+        self.word = word
+        self.key = Core.otilabs_key()
+        if not isinstance(self.key, str) or not self.key.strip():
+            raise MissingKey('otilabs')
+        self.totalhosts: set[str] = set()
+        self.totalips: set[str] = set()
+        self.proxy = False
+
+    async def do_search(self) -> SourceExecutionReport | None:
+        headers = {'X-RapidAPI-Key': self.key, 'X-RapidAPI-Host': self.API_HOST}
+        async with AsyncFetcher.open_session(headers=headers, proxy=self.proxy, request_timeout=60) as session:
+            response = await AsyncFetcher.fetch(
+                session=session,
+                url=f'https://{self.API_HOST}/domain/{self.word}/subdomains',
+                params={'wait': '1'},
+                json=True,
+                include_metadata=True,
+            )
+        if error := provider_http_error(response):
+            return SourceExecutionReport(*error)
+        assert isinstance(response, FetcherResponse)
+        body = response.body
+        if not isinstance(body, dict) or not isinstance(body.get('subdomains'), list):
+            return SourceExecutionReport('failed', 'invalid-response')
+        live = body.get('live', [])
+        if not isinstance(live, list):
+            return SourceExecutionReport('failed', 'invalid-response')
+
+        malformed = False
+        for name in body['subdomains']:
+            if not isinstance(name, str):
+                malformed = True
+                continue
+            if hostname := normalize_scoped_hostname(name, self.word):
+                self.totalhosts.add(hostname)
+        for item in live:
+            if not isinstance(item, dict) or not isinstance(item.get('host'), str):
+                malformed = True
+                continue
+            hostname = normalize_scoped_hostname(item['host'], self.word)
+            if hostname is None:
+                continue
+            self.totalhosts.add(hostname)
+            address = item.get('ip')
+            if isinstance(address, str):
+                try:
+                    self.totalips.add(str(ip_address(address)))
+                except ValueError:
+                    pass  # a CNAME target rather than an address
+        if malformed:
+            return SourceExecutionReport('failed', 'invalid-response')
+
+        sources = body.get('sources_used')
+        pending = isinstance(sources, list) and any(
+            isinstance(source, str) and source.endswith(': enriching') for source in sources
+        )
+        if pending or body.get('live_count', 0) is None:
+            # The API answered before its live check finished, so a later run can return more addresses.
+            return SourceExecutionReport('partial', 'enrichment-pending')
+        count, returned = body.get('count'), body.get('returned')
+        if isinstance(count, int) and isinstance(returned, int) and count > returned:
+            return SourceExecutionReport('partial', 'result-cap')
+        return None
+
+    async def get_hostnames(self) -> set[str]:
+        return self.totalhosts
+
+    async def get_ips(self) -> set[str]:
+        return self.totalips
+
+    async def process(self, proxy: bool = False) -> SourceExecutionReport | None:
+        self.proxy = proxy
+        try:
+            return await self.do_search()
+        except Exception as error:
+            logger.info('OTI Labs search failed: %s', type(error).__name__)
+            return SourceExecutionReport('failed', 'transport-error')

theHarvester/lib/core.py

@@ -222,6 +222,7 @@ class Core:
         'mojeek': ('key',),
         'netlas': ('key',),
         'onyphe': ('key',),
+        'otilabs': ('key',),
         'pentestTools': ('key',),
         'projectDiscovery': ('key',),
         'rocketreach': ('key',),
@@ -391,6 +392,10 @@ class Core:
     def onyphe_key() -> str:
         return Core._api_key_value('onyphe')
 
+    @staticmethod
+    def otilabs_key() -> str:
+        return Core._api_key_value('otilabs')
+
     @staticmethod
     def pentest_tools_key() -> str:
         return Core._api_key_value('pentestTools')

theHarvester/lib/source_catalog.py

@@ -159,6 +159,7 @@ _SPECS = (
     _spec('mojeek', ResultRoute.SUBDOMAINS, ResultRoute.EMAILS),
     _spec('netlas', ResultRoute.SUBDOMAINS),
     _spec('onyphe', ResultRoute.SUBDOMAINS, ResultRoute.IPS, ResultRoute.ASNS),
+    _spec('otilabs', ResultRoute.SUBDOMAINS, ResultRoute.IPS),
     _spec('otx', ResultRoute.SUBDOMAINS, ResultRoute.IPS),
     _spec(
         'pentesttools',

theHarvester/lib/source_runner.py

@@ -41,6 +41,7 @@ from theHarvester.discovery import (
     mojeek,
     netlas,
     onyphe,
+    otilabs,
     otxsearch,
     pentesttools,
     projectdiscovery,
@@ -154,6 +155,7 @@ SOURCE_FACTORIES: dict[str, SourceFactory] = {
     'mojeek': lambda request: mojeek.SearchMojeek(request.target, request.limit),
     'netlas': lambda request: netlas.SearchNetlas(request.target, request.limit),
     'onyphe': lambda request: onyphe.SearchOnyphe(request.target, request.limit),
+    'otilabs': lambda request: otilabs.SearchOTILabs(request.target),
     'otx': lambda request: otxsearch.SearchOtx(request.target),
     'pentesttools': lambda request: pentesttools.SearchPentestTools(request.target),
     'projectdiscovery': lambda request: projectdiscovery.SearchDiscovery(request.target),