feat: support REST source capability selectors (#2480)

CHANGELOG.md

@@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added root contributor and security policies, structured issue forms, repository agent guidance, discovery terminology, and an operator-focused documentation wiki ([d090a29a](https://github.com/laramies/theHarvester/commit/d090a29a), [7c491ef5](https://github.com/laramies/theHarvester/commit/7c491ef5), [8b9d420b](https://github.com/laramies/theHarvester/commit/8b9d420b)).
 
 ### Changed
+- Allowed REST `/query` requests to select discovery sources by result capability, matching the CLI's union semantics while preserving explicit source selection.
 - Expanded Common Crawl discovery to use every unique crawl ending within one year of the newest catalog entry, validate catalog endpoints, batch requests, cap each query at 100 pages, and enforce the CLI result limit across page requests ([249ce64b](https://github.com/laramies/theHarvester/commit/249ce64b), [70470cd8](https://github.com/laramies/theHarvester/commit/70470cd8)).
 - Completed bounded pagination for Wayback Archive and Cert Spotter, including continuation handling, truncation diagnostics, and preservation of partial results on provider failures ([df6ff2c9](https://github.com/laramies/theHarvester/commit/df6ff2c9), [f85a08ff](https://github.com/laramies/theHarvester/commit/f85a08ff)).
 - Routed operator messages and diagnostics through logging, preserved host logging policy and existing handlers, and configured logging for the standalone API example ([8a7b8b71](https://github.com/laramies/theHarvester/commit/8a7b8b71)).

docs/wiki/Rest-API.md

@@ -52,6 +52,19 @@ curl -sG http://127.0.0.1:5000/query \
   | jq
 ```
 
+The `source` parameter also accepts the same capability selectors as the CLI:
+`subdomains`, `emails`, `ips`, `asns`, `urls`, `people`, and `breaches`.
+Repeat `source` to combine capabilities with explicit source names. Selection is
+a union and does not filter fields returned by a selected source.
+
+```bash
+curl -sG http://127.0.0.1:5000/query \
+  --data-urlencode 'domain=example.com' \
+  --data-urlencode 'source=emails' \
+  --data-urlencode 'source=certspotter' \
+  | jq
+```
+
 ## Additional API routes
 
 The following `POST /additional/*` routes provide optional breach, leak, security-score, and technology-stack lookups:

tests/test_rest_api.py

@@ -0,0 +1,48 @@
+from argparse import Namespace
+
+from fastapi.testclient import TestClient
+
+from theHarvester.lib.api import api
+from theHarvester.lib.core import Core
+
+
+def test_query_expands_source_capability(monkeypatch) -> None:
+    captured: list[Namespace] = []
+
+    async def fake_start(args: Namespace):
+        captured.append(args)
+        return ([], [], [], [], [], [], [], [], [])
+
+    monkeypatch.setattr(api.__main__, 'start', fake_start)
+
+    response = TestClient(api.app).get('/query?domain=example.test&source=subdomains')
+
+    assert response.status_code == 200
+    assert captured[0].source == ','.join(Core.expand_source_selection('subdomains'))
+
+
+def test_query_unions_capabilities_and_explicit_sources(monkeypatch) -> None:
+    captured: list[Namespace] = []
+
+    async def fake_start(args: Namespace):
+        captured.append(args)
+        return ([], [], [], [], [], [], [], [], [])
+
+    monkeypatch.setattr(api.__main__, 'start', fake_start)
+
+    response = TestClient(api.app).get('/query?domain=example.test&source=emails&source=certspotter')
+
+    assert response.status_code == 200
+    assert captured[0].source == ','.join(Core.expand_source_selection('emails,certspotter'))
+
+
+def test_query_rejects_unknown_source_or_capability(monkeypatch) -> None:
+    async def unexpected_start(_args: Namespace):
+        raise AssertionError('enumeration must not start')
+
+    monkeypatch.setattr(api.__main__, 'start', unexpected_start)
+
+    response = TestClient(api.app).get('/query?domain=example.test&source=unknown')
+
+    assert response.status_code == 400
+    assert response.json()['detail'].startswith("Source 'unknown' is not supported")

theHarvester/lib/api/api.py

@@ -285,7 +285,10 @@ async def dnsbrute(
 @limiter.limit(API_RATE_LIMIT)
 async def query(
     request: Request,
-    source: Annotated[list[str], Query(description='Data sources to query (comma separated with no space)')],
+    source: Annotated[
+        list[str],
+        Query(description='Data sources or capability selectors to query; repeated values form a union'),
+    ],
     domain: Annotated[str, Query(min_length=3, description='Domain to be harvested')],
     dns_server: Annotated[str, Query(description='DNS server to use for lookup')] = '',
     user_agent: Annotated[str | None, Header()] = None,
@@ -315,8 +318,9 @@ async def query(
 
     try:
         # Validate sources
+        selected_sources = __main__.Core.expand_source_selection(','.join(source))
         supported_engines = __main__.Core.get_supportedengines()
-        for s in source:
+        for s in selected_sources:
             if s not in supported_engines:
                 raise HTTPException(
                     status_code=status.HTTP_400_BAD_REQUEST,
@@ -350,7 +354,7 @@ async def query(
                 limit=limit,
                 proxies=proxies,
                 shodan=shodan,
-                source=','.join(source),
+                source=','.join(selected_sources),
                 start=start,
                 take_over=take_over,
                 wordlist=wordlist,