diff --git a/CHANGELOG.md b/CHANGELOG.md index 3db6c90e..c38da5bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated CI and container maintenance pins, including `actions/checkout`, `astral-sh/setup-uv`, `astral-sh/ruff-action`, `github/codeql-action`, StepSecurity Harden-Runner, Docker actions, and the Python base image. - Expanded offline regression coverage for discovery providers, configuration contracts, logging, output, documentation, workflow policy, and scope boundaries. +### Removed +- Removed the nonfunctional ThreatCrowd source because its service hostnames terminate at deleted AWS load balancers and return NXDOMAIN; OTX remains available through its separate adapter. + ### Fixed - Made RapidDNS, Robtex, and Subdomain Center HTTP failures report the source and response status before parsing. - Fixed GitHub code-search fragment limits, boundary separation, and malformed-page termination with offline provider tests. diff --git a/README.md b/README.md index 60fe490d..b0c2b4f7 100644 --- a/README.md +++ b/README.md @@ -177,7 +177,6 @@ Read the **API key** column as follows: | `subdomaincenter` | ✓ | No | No | No | No | No | No | No | | `subdomainfinderc99` | ✓ | No | No | No | No | No | No | No | | `thc` | ✓ | No | No | No | No | No | No | No | -| `threatcrowd` | ✓ | No | ✓ | No | No | No | No | No | | `tomba` | ✓ | ✓ | No | No | No | No | No | ✓ | | `urlscan` | ✓ | No | ✓ | ✓ | ✓ | No | No | No | | `venacus` | No | ✓ | ✓ | No | ✓ | ✓ | No | ✓ | diff --git a/tests/discovery/test_threatcrowd.py b/tests/discovery/test_threatcrowd.py deleted file mode 100644 index 4c6b5771..00000000 --- a/tests/discovery/test_threatcrowd.py +++ /dev/null @@ -1,74 +0,0 @@ -import json -import logging -from typing import Any - -import pytest - -from theHarvester.discovery import threatcrowd - - -_REPORT = { - 'response_code': '1', - 'subdomains': ['api.example.com', 'api.example.com', 'outside.test'], - 'resolutions': [ - {'ip_address': '192.0.2.1'}, - {'ip_address': '2001:0db8::1'}, - {'ip_address': '999.0.0.1'}, - '2001:db8::2', - 'NXDOMAIN', - {'ip_address': 1234}, - 1234, - ], -} - - -@pytest.mark.asyncio -@pytest.mark.parametrize('payload', [_REPORT, json.dumps(_REPORT)]) -async def test_process_retains_only_valid_ip_addresses(monkeypatch: pytest.MonkeyPatch, payload: object) -> None: - async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[object]: - return [payload] - - monkeypatch.setattr(threatcrowd.AsyncFetcher, 'fetch_all', fake_fetch_all) - search = threatcrowd.SearchThreatcrowd('example.com') - - await search.process() - - assert await search.get_hostnames() == {'api.example.com'} - assert await search.get_ips() == {'192.0.2.1', '2001:db8::1', '2001:db8::2'} - - -@pytest.mark.asyncio -@pytest.mark.parametrize('payload', ['', 'not-json', '[]', '{"subdomains":"wrong-shape"}']) -async def test_process_handles_empty_or_malformed_payloads( - monkeypatch: pytest.MonkeyPatch, - payload: str, -) -> None: - async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[str]: - return [payload] - - monkeypatch.setattr(threatcrowd.AsyncFetcher, 'fetch_all', fake_fetch_all) - search = threatcrowd.SearchThreatcrowd('example.com') - - await search.process() - - assert await search.get_hostnames() == set() - assert await search.get_ips() == set() - - -@pytest.mark.asyncio -async def test_process_attributes_provider_errors( - monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, -) -> None: - async def fake_fetch_all(*_args: Any, **_kwargs: Any) -> list[dict[str, str]]: - return [{'response_code': '0'}] - - monkeypatch.setattr(threatcrowd.AsyncFetcher, 'fetch_all', fake_fetch_all) - search = threatcrowd.SearchThreatcrowd('example.com') - - with caplog.at_level(logging.INFO, logger=threatcrowd.__name__): - await search.process() - - assert await search.get_hostnames() == set() - assert await search.get_ips() == set() - assert 'ThreatCrowd API returned error code' in caplog.text diff --git a/tests/lib/test_source_catalog.py b/tests/lib/test_source_catalog.py index 75c4c4f1..2a126042 100644 --- a/tests/lib/test_source_catalog.py +++ b/tests/lib/test_source_catalog.py @@ -27,6 +27,12 @@ def test_source_specs_cover_supported_sources() -> None: assert set(SOURCE_SPECS) <= set(Core.get_supportedengines()) +def test_dead_threatcrowd_source_is_not_selectable() -> None: + assert 'threatcrowd' not in Core.get_supportedengines() + assert 'threatcrowd' not in SOURCE_SPECS + assert 'threatcrowd' not in _scheduled_source_names() + + def test_subdomain_route_drives_subdomain_capability() -> None: spec = SourceSpec( name='example', diff --git a/tests/test_readme.py b/tests/test_readme.py index d0298220..9ccf4a89 100644 --- a/tests/test_readme.py +++ b/tests/test_readme.py @@ -79,8 +79,8 @@ def test_readme_matches_declared_source_contracts() -> None: declared = _declared_source_contracts() assert '| Source | Subdomains | Emails | IPs | ASNs | URLs / links | People |' in readme - assert len(declared) == 56 - assert len(documented) == 56 + assert len(declared) == 55 + assert len(documented) == 55 assert documented == declared assert {'securitytrails', 'shodaninternetdb'}.isdisjoint(documented) diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 67b81c9d..ede0b102 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -69,7 +69,6 @@ from theHarvester.discovery import ( subdomainfinderc99, takeover, thc, - threatcrowd, tombasearch, urlscan, venacussearch, @@ -236,7 +235,7 @@ async def start(rest_args: argparse.Namespace | None = None): builtwith, censys, certspotter, chaos, commoncrawl, criminalip, crtsh, dehashed, dnsdumpster, duckduckgo, dymo, fofa, fullhunt, github-code, gitlab, hackertarget, haveibeenpwned, hudsonrock, hunter, hunterhow, intelx, leakix, leaklookup, mojeek, netlas, onyphe, otx, pentesttools, projectdiscovery, rapiddns, robtex, rocketreach, securityscorecard, securityTrails, sherlockeye, shodan, shodanInternetDB, subdomaincenter, - subdomainfinderc99, thc, threatcrowd, tomba, urlscan, venacus, virustotal, waybackarchive, whoisxml, windvane, yahoo, zoomeye""", + subdomainfinderc99, thc, tomba, urlscan, venacus, virustotal, waybackarchive, whoisxml, windvane, yahoo, zoomeye""", ) # determines if the filename is coming from rest api or user @@ -1126,18 +1125,6 @@ async def start(rest_args: argparse.Namespace | None = None): except Exception as e: show_default_error_message(engineitem, word, e) - elif engineitem == 'threatcrowd': - try: - threatcrowd_search = threatcrowd.SearchThreatcrowd(word) - stor_lst.append( - store( - threatcrowd_search, - engineitem, - ) - ) - except Exception as e: - show_default_error_message(engineitem, word, e) - elif engineitem == 'tomba': try: tomba_search = tombasearch.SearchTomba(word, limit, start) diff --git a/theHarvester/discovery/threatcrowd.py b/theHarvester/discovery/threatcrowd.py deleted file mode 100644 index c2ecdf83..00000000 --- a/theHarvester/discovery/threatcrowd.py +++ /dev/null @@ -1,106 +0,0 @@ -import json as _stdlib_json -import logging -from ipaddress import ip_address -from types import ModuleType - -from theHarvester.lib.core import AsyncFetcher, Core - -logger = logging.getLogger(__name__) - -json: ModuleType = _stdlib_json -try: - import ujson as _ujson - - json = _ujson -except ImportError: - pass -except Exception: - pass - - -class SearchThreatcrowd: - """Class uses ThreatCrowd API to gather domain intelligence and subdomains""" - - def __init__(self, word) -> None: - self.word = word - self.totalhosts: set = set() - self.totalips: set = set() - self.proxy = False - self.hostname = 'http://ci-www.threatcrowd.org' - - @staticmethod - def _safe_parse_json(payload: object) -> dict: - if isinstance(payload, dict): - return payload - if isinstance(payload, str): - try: - return json.loads(payload) - except Exception: - return {} - return {} - - async def do_search(self) -> None: - try: - headers = {'User-agent': Core.get_user_agent()} - - # ThreatCrowd domain report API - url = f'{self.hostname}/searchApi/v2/domain/report/?domain={self.word}' - - response = await AsyncFetcher.fetch_all([url], headers=headers, proxy=self.proxy) - - if not response or not isinstance(response, list) or not response[0]: - logger.info(f'No response from ThreatCrowd API for: {self.word}') - return - - try: - data = self._safe_parse_json(response[0]) - - if isinstance(data, dict): - # Check response code - '1' means success in ThreatCrowd API - response_code = data.get('response_code', '') - if response_code and response_code != '1': - logger.info(f'ThreatCrowd API returned error code: {response_code}') - return - - # Extract subdomains - direct list in response - subdomains = data.get('subdomains', []) - if isinstance(subdomains, list): - for subdomain in subdomains: - if isinstance(subdomain, str) and subdomain.strip(): - # ThreatCrowd returns full subdomains, not relative ones - clean_subdomain = subdomain.strip().lower() - if clean_subdomain.endswith(f'.{self.word}') or clean_subdomain == self.word: - self.totalhosts.add(clean_subdomain) - - # Extract IPs if available (from resolutions) - resolutions = data.get('resolutions', []) - if isinstance(resolutions, list): - for resolution in resolutions: - if isinstance(resolution, dict): - value = resolution.get('ip_address') - elif isinstance(resolution, str): - value = resolution - else: - continue - if not isinstance(value, str): - continue - try: - self.totalips.add(str(ip_address(value.strip()))) - except ValueError: - continue - - except Exception as e: - logger.info(f'Failed to parse ThreatCrowd response: {e}') - - except Exception as e: - logger.info(f'ThreatCrowd API error: {e}') - - async def get_hostnames(self) -> set: - return self.totalhosts - - async def get_ips(self) -> set: - return self.totalips - - async def process(self, proxy: bool = False) -> None: - self.proxy = proxy - await self.do_search() diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index a6fd4f87..ca67c4a6 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -346,7 +346,6 @@ class Core: 'subdomainfinderc99', 'sublist3r', 'thc', - 'threatcrowd', 'tomba', 'urlscan', 'venacus', diff --git a/theHarvester/lib/source_catalog.py b/theHarvester/lib/source_catalog.py index 6b954b8e..06995ea2 100644 --- a/theHarvester/lib/source_catalog.py +++ b/theHarvester/lib/source_catalog.py @@ -91,7 +91,6 @@ _SPECS = ( _spec('subdomaincenter', ResultRoute.SUBDOMAINS), _spec('subdomainfinderc99', ResultRoute.SUBDOMAINS), _spec('thc', ResultRoute.SUBDOMAINS), - _spec('threatcrowd', ResultRoute.SUBDOMAINS, ResultRoute.IPS), _spec('tomba', ResultRoute.SUBDOMAINS, ResultRoute.EMAILS), _spec('urlscan', ResultRoute.SUBDOMAINS, ResultRoute.IPS, ResultRoute.ASNS, ResultRoute.INTERESTING_URLS), _spec('venacus', ResultRoute.EMAILS, ResultRoute.IPS, ResultRoute.PEOPLE, ResultRoute.INTERESTING_URLS),