Merge pull request #2475 from NotoriousRebel/codex/remove-dead-threatcrowd-source

Remove nonfunctional ThreatCrowd source

CHANGELOG.md

@@ -29,6 +29,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Updated CI and container maintenance pins, including `actions/checkout`, `astral-sh/setup-uv`, `astral-sh/ruff-action`, `github/codeql-action`, StepSecurity Harden-Runner, Docker actions, and the Python base image.
 - Expanded offline regression coverage for discovery providers, configuration contracts, logging, output, documentation, workflow policy, and scope boundaries.
 
+### Removed
+- Removed the nonfunctional ThreatCrowd source because its service hostnames terminate at deleted AWS load balancers and return NXDOMAIN; OTX remains available through its separate adapter.
+
 ### Fixed
 - Made RapidDNS, Robtex, and Subdomain Center HTTP failures report the source and response status before parsing.
 - Fixed GitHub code-search fragment limits, boundary separation, and malformed-page termination with offline provider tests.

README.md

@@ -177,7 +177,6 @@ Read the **API key** column as follows:
 | `subdomaincenter` | ✓ | No | No | No | No | No | No | No |
 | `subdomainfinderc99` | ✓ | No | No | No | No | No | No | No |
 | `thc` | ✓ | No | No | No | No | No | No | No |
-| `threatcrowd` | ✓ | No | ✓ | No | No | No | No | No |
 | `tomba` | ✓ | ✓ | No | No | No | No | No | ✓ |
 | `urlscan` | ✓ | No | ✓ | ✓ | ✓ | No | No | No |
 | `venacus` | No | ✓ | ✓ | No | ✓ | ✓ | No | ✓ |

tests/discovery/test_threatcrowd.py

@@ -1,74 +0,0 @@
-import json
-import logging
-from typing import Any
-
-import pytest
-
-from theHarvester.discovery import threatcrowd
-
-
-_REPORT = {
-    'response_code': '1',
-    'subdomains': ['api.example.com', 'api.example.com', 'outside.test'],
-    'resolutions': [
-        {'ip_address': '192.0.2.1'},
-        {'ip_address': '2001:0db8::1'},
-        {'ip_address': '999.0.0.1'},
-        '2001:db8::2',
-        'NXDOMAIN',
-        {'ip_address': 1234},
-        1234,
-    ],
-}
-
-
-@pytest.mark.asyncio
-@pytest.mark.parametrize('payload', [_REPORT, json.dumps(_REPORT)])
-async def test_process_retains_only_valid_ip_addresses(monkeypatch: pytest.MonkeyPatch, payload: object) -> None:
-    async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[object]:
-        return [payload]
-
-    monkeypatch.setattr(threatcrowd.AsyncFetcher, 'fetch_all', fake_fetch_all)
-    search = threatcrowd.SearchThreatcrowd('example.com')
-
-    await search.process()
-
-    assert await search.get_hostnames() == {'api.example.com'}
-    assert await search.get_ips() == {'192.0.2.1', '2001:db8::1', '2001:db8::2'}
-
-
-@pytest.mark.asyncio
-@pytest.mark.parametrize('payload', ['', 'not-json', '[]', '{"subdomains":"wrong-shape"}'])
-async def test_process_handles_empty_or_malformed_payloads(
-    monkeypatch: pytest.MonkeyPatch,
-    payload: str,
-) -> None:
-    async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[str]:
-        return [payload]
-
-    monkeypatch.setattr(threatcrowd.AsyncFetcher, 'fetch_all', fake_fetch_all)
-    search = threatcrowd.SearchThreatcrowd('example.com')
-
-    await search.process()
-
-    assert await search.get_hostnames() == set()
-    assert await search.get_ips() == set()
-
-
-@pytest.mark.asyncio
-async def test_process_attributes_provider_errors(
-    monkeypatch: pytest.MonkeyPatch,
-    caplog: pytest.LogCaptureFixture,
-) -> None:
-    async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[dict[str, str]]:
-        return [{'response_code': '0'}]
-
-    monkeypatch.setattr(threatcrowd.AsyncFetcher, 'fetch_all', fake_fetch_all)
-    search = threatcrowd.SearchThreatcrowd('example.com')
-
-    with caplog.at_level(logging.INFO, logger=threatcrowd.__name__):
-        await search.process()
-
-    assert await search.get_hostnames() == set()
-    assert await search.get_ips() == set()
-    assert 'ThreatCrowd API returned error code' in caplog.text

tests/lib/test_source_catalog.py

@@ -27,6 +27,12 @@ def test_source_specs_cover_supported_sources() -> None:
     assert set(SOURCE_SPECS) <= set(Core.get_supportedengines())
 
 
+def test_dead_threatcrowd_source_is_not_selectable() -> None:
+    assert 'threatcrowd' not in Core.get_supportedengines()
+    assert 'threatcrowd' not in SOURCE_SPECS
+    assert 'threatcrowd' not in _scheduled_source_names()
+
+
 def test_subdomain_route_drives_subdomain_capability() -> None:
     spec = SourceSpec(
         name='example',

tests/test_readme.py

@@ -79,8 +79,8 @@ def test_readme_matches_declared_source_contracts() -> None:
     declared = _declared_source_contracts()
 
     assert '| Source | Subdomains | Emails | IPs | ASNs | URLs / links | People |' in readme
-    assert len(declared) == 56
-    assert len(documented) == 56
+    assert len(declared) == 55
+    assert len(documented) == 55
     assert documented == declared
     assert {'securitytrails', 'shodaninternetdb'}.isdisjoint(documented)
 

theHarvester/__main__.py

@@ -69,7 +69,6 @@ from theHarvester.discovery import (
     subdomainfinderc99,
     takeover,
     thc,
-    threatcrowd,
     tombasearch,
     urlscan,
     venacussearch,
@@ -236,7 +235,7 @@ async def start(rest_args: argparse.Namespace | None = None):
                             builtwith, censys, certspotter, chaos, commoncrawl, criminalip, crtsh, dehashed, dnsdumpster, duckduckgo, dymo, fofa, fullhunt, github-code,
                             gitlab, hackertarget, haveibeenpwned, hudsonrock, hunter, hunterhow, intelx, leakix, leaklookup, mojeek, netlas, onyphe, otx, pentesttools,
                             projectdiscovery, rapiddns, robtex, rocketreach, securityscorecard, securityTrails, sherlockeye, shodan, shodanInternetDB, subdomaincenter,
-                            subdomainfinderc99, thc, threatcrowd, tomba, urlscan, venacus, virustotal, waybackarchive, whoisxml, windvane, yahoo, zoomeye""",
+                            subdomainfinderc99, thc, tomba, urlscan, venacus, virustotal, waybackarchive, whoisxml, windvane, yahoo, zoomeye""",
     )
 
     # determines if the filename is coming from rest api or user
@@ -1126,18 +1125,6 @@ async def start(rest_args: argparse.Namespace | None = None):
                     except Exception as e:
                         show_default_error_message(engineitem, word, e)
 
-                elif engineitem == 'threatcrowd':
-                    try:
-                        threatcrowd_search = threatcrowd.SearchThreatcrowd(word)
-                        stor_lst.append(
-                            store(
-                                threatcrowd_search,
-                                engineitem,
-                            )
-                        )
-                    except Exception as e:
-                        show_default_error_message(engineitem, word, e)
-
                 elif engineitem == 'tomba':
                     try:
                         tomba_search = tombasearch.SearchTomba(word, limit, start)

theHarvester/discovery/threatcrowd.py

@@ -1,106 +0,0 @@
-import json as _stdlib_json
-import logging
-from ipaddress import ip_address
-from types import ModuleType
-
-from theHarvester.lib.core import AsyncFetcher, Core
-
-logger = logging.getLogger(__name__)
-
-json: ModuleType = _stdlib_json
-try:
-    import ujson as _ujson
-
-    json = _ujson
-except ImportError:
-    pass
-except Exception:
-    pass
-
-
-class SearchThreatcrowd:
-    """Class uses ThreatCrowd API to gather domain intelligence and subdomains"""
-
-    def __init__(self, word) -> None:
-        self.word = word
-        self.totalhosts: set = set()
-        self.totalips: set = set()
-        self.proxy = False
-        self.hostname = 'http://ci-www.threatcrowd.org'
-
-    @staticmethod
-    def _safe_parse_json(payload: object) -> dict:
-        if isinstance(payload, dict):
-            return payload
-        if isinstance(payload, str):
-            try:
-                return json.loads(payload)
-            except Exception:
-                return {}
-        return {}
-
-    async def do_search(self) -> None:
-        try:
-            headers = {'User-agent': Core.get_user_agent()}
-
-            # ThreatCrowd domain report API
-            url = f'{self.hostname}/searchApi/v2/domain/report/?domain={self.word}'
-
-            response = await AsyncFetcher.fetch_all([url], headers=headers, proxy=self.proxy)
-
-            if not response or not isinstance(response, list) or not response[0]:
-                logger.info(f'No response from ThreatCrowd API for: {self.word}')
-                return
-
-            try:
-                data = self._safe_parse_json(response[0])
-
-                if isinstance(data, dict):
-                    # Check response code - '1' means success in ThreatCrowd API
-                    response_code = data.get('response_code', '')
-                    if response_code and response_code != '1':
-                        logger.info(f'ThreatCrowd API returned error code: {response_code}')
-                        return
-
-                    # Extract subdomains - direct list in response
-                    subdomains = data.get('subdomains', [])
-                    if isinstance(subdomains, list):
-                        for subdomain in subdomains:
-                            if isinstance(subdomain, str) and subdomain.strip():
-                                # ThreatCrowd returns full subdomains, not relative ones
-                                clean_subdomain = subdomain.strip().lower()
-                                if clean_subdomain.endswith(f'.{self.word}') or clean_subdomain == self.word:
-                                    self.totalhosts.add(clean_subdomain)
-
-                    # Extract IPs if available (from resolutions)
-                    resolutions = data.get('resolutions', [])
-                    if isinstance(resolutions, list):
-                        for resolution in resolutions:
-                            if isinstance(resolution, dict):
-                                value = resolution.get('ip_address')
-                            elif isinstance(resolution, str):
-                                value = resolution
-                            else:
-                                continue
-                            if not isinstance(value, str):
-                                continue
-                            try:
-                                self.totalips.add(str(ip_address(value.strip())))
-                            except ValueError:
-                                continue
-
-            except Exception as e:
-                logger.info(f'Failed to parse ThreatCrowd response: {e}')
-
-        except Exception as e:
-            logger.info(f'ThreatCrowd API error: {e}')
-
-    async def get_hostnames(self) -> set:
-        return self.totalhosts
-
-    async def get_ips(self) -> set:
-        return self.totalips
-
-    async def process(self, proxy: bool = False) -> None:
-        self.proxy = proxy
-        await self.do_search()

theHarvester/lib/core.py

@@ -346,7 +346,6 @@ class Core:
             'subdomainfinderc99',
             'sublist3r',
             'thc',
-            'threatcrowd',
             'tomba',
             'urlscan',
             'venacus',

theHarvester/lib/source_catalog.py

@@ -91,7 +91,6 @@ _SPECS = (
     _spec('subdomaincenter', ResultRoute.SUBDOMAINS),
     _spec('subdomainfinderc99', ResultRoute.SUBDOMAINS),
     _spec('thc', ResultRoute.SUBDOMAINS),
-    _spec('threatcrowd', ResultRoute.SUBDOMAINS, ResultRoute.IPS),
     _spec('tomba', ResultRoute.SUBDOMAINS, ResultRoute.EMAILS),
     _spec('urlscan', ResultRoute.SUBDOMAINS, ResultRoute.IPS, ResultRoute.ASNS, ResultRoute.INTERESTING_URLS),
     _spec('venacus', ResultRoute.EMAILS, ResultRoute.IPS, ResultRoute.PEOPLE, ResultRoute.INTERESTING_URLS),