diff --git a/CHANGELOG.md b/CHANGELOG.md index da613f3f..f9f7af00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Replaced Shodan's synchronous Python SDK with cancellable async Host API requests that honor configured proxies, query every unique resolved IPv4, paginate target-bound hostname and TLS-certificate searches without an adapter-specific result cap, retain successful partial results, and add no source-local deadline. Shodan now stores one canonical `shodan-host` result per IP with every normalized TCP or UDP service and scoped certificate CN/SAN metadata in native JSONL, SQLite, API, and HarvestView details instead of an escaped JSON value. - Migrated BuiltWith to the current v23 Domain API with privacy-preserving request controls, nested result parsing, and truthful partial or failed outcomes. +- Migrated Censys discovery from the deprecated Python Search SDK to the Censys Platform API, using a Personal Access Token and optional organization ID. - Removed the transport-wide delay before reading ready HTTP responses, bounded Wayback Archive to 30 seconds and Common Crawl to 120 seconds, kept both sources within the requested result limit, and made long-source progress visible in verbose mode. Common Crawl now requests one 50-record page at a time instead of bursting page batches. - Made Baidu, crt.sh, HackerTarget, Have I Been Pwned, Mojeek, OTX, and Robtex report blocked, malformed, or transport failures truthfully. Also fixed HackerTarget CSV parsing and Robtex AAAA results. - Hardened BufferOver, ProjectDiscovery, DNSDumpster, ONYPHE, and URLScan parsing and result attribution, including scoped typed results and bounded URLScan pagination. diff --git a/docs/wiki/Configuration-and-API-Keys.md b/docs/wiki/Configuration-and-API-Keys.md index 9d7159ce..75fc28fd 100644 --- a/docs/wiki/Configuration-and-API-Keys.md +++ b/docs/wiki/Configuration-and-API-Keys.md @@ -22,8 +22,8 @@ Keep the complete generated template and fill only the providers you intend to u ```yaml apikeys: censys: - id: your-censys-id - secret: your-censys-secret + token: your-censys-personal-access-token + organization_id: your-censys-organization-id github: key: your-github-token @@ -45,6 +45,8 @@ The [README source matrix](https://github.com/laramies/theHarvester/blob/dev/REA Provider pricing, quotas, and terms change frequently. Check the provider's current documentation for these details. +`censys.token` is a Censys Platform Personal Access Token. Set `organization_id` when searches should use an entitled organization. This source uses the Global Search API, which is unavailable to Free accounts because they are limited to asset lookups. The retired Search API ID and secret fields are not accepted. + `hibpverified` queries [HIBP's authenticated verified-domain endpoint](https://haveibeenpwned.com/API/v3#BreachedDomain). It is selected by its name, the `breaches` capability, and `all`. Without a configured HIBP API key it is skipped like other unavailable keyed sources. Live use requires a user-owned paid HIBP API key and a user-owned domain verified in that account. The keyless `haveibeenpwned` source continues to query only the public breach catalogue. `routeviews.key` is optional. RouteViews provides authenticated API keys to verified PeeringDB users. `--routeviews` uses the authenticated endpoint and documented 10-request-per-second allowance when the key is configured; otherwise it uses guest access at one request per second. If RouteViews rejects a configured key, the action fails without retrying as a guest; remove the key to select guest access. RouteViews does not document this as a paid subscription. diff --git a/pyproject.toml b/pyproject.toml index 198fb3a7..505cfdb3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,6 @@ dependencies = [ "aiomultiprocess==0.9.1", "aiosqlite==0.22.1", "beautifulsoup4==4.15.0", - "censys==2.2.19", "certifi==2026.6.17", "dnspython==2.8.0", "fastapi==0.138.1", diff --git a/tests/discovery/test_censys.py b/tests/discovery/test_censys.py index c3ce9ad4..a2471052 100644 --- a/tests/discovery/test_censys.py +++ b/tests/discovery/test_censys.py @@ -1,5 +1,7 @@ +import asyncio import sys import types +from pathlib import Path import pytest @@ -11,93 +13,249 @@ if 'aiohttp_socks' not in sys.modules: def from_url(*_args, **_kwargs): return None - setattr(aiohttp_socks_stub, 'ProxyConnector', _ProxyConnector) + aiohttp_socks_stub.ProxyConnector = _ProxyConnector sys.modules['aiohttp_socks'] = aiohttp_socks_stub from theHarvester.discovery import censysearch from theHarvester.discovery.constants import MissingKey - - -class _FakeQuery: - def __init__(self, pages): - self.pages = pages - - def __iter__(self): - return iter(self.pages) +from theHarvester.lib.core import FetcherResponse @pytest.mark.asyncio -async def test_missing_key_raises(monkeypatch) -> None: - monkeypatch.setattr(censysearch.Core, 'censys_key', lambda: (None, None)) +async def test_missing_platform_token_raises(monkeypatch) -> None: + monkeypatch.setattr(censysearch.Core, 'censys_key', lambda: (None, 'org-id')) - with pytest.raises(MissingKey): + with pytest.raises(MissingKey, match='Personal Access Token'): + censysearch.SearchCensys('example.com') + + +def test_legacy_search_api_credentials_fail_closed(monkeypatch) -> None: + monkeypatch.setattr( + censysearch.Core, + 'api_keys', + staticmethod(lambda: {'censys': {'id': 'legacy-id', 'secret': 'legacy-secret'}}), + ) + + with pytest.raises(MissingKey, match='Personal Access Token'): censysearch.SearchCensys('example.com') @pytest.mark.asyncio -async def test_search_uses_documented_pagination_and_fields(monkeypatch) -> None: - monkeypatch.setattr(censysearch.Core, 'censys_key', lambda: ('id', 'secret')) - - calls = {} - - class _FakeCensysCerts: - def __init__(self, api_id, api_secret, user_agent): - calls['init'] = {'api_id': api_id, 'api_secret': api_secret, 'user_agent': user_agent} - - def search(self, **kwargs): - calls['search'] = kwargs - return _FakeQuery( - [ - [ - {'names': ['a.example.com'], 'parsed': {'subject': {'email_address': 'admin@example.com'}}}, - {'names': ['b.example.com'], 'parsed': {'subject': {'email_address': ['ops@example.com']}}}, +async def test_search_calls_platform_api_directly_and_follows_page_tokens(monkeypatch) -> None: + monkeypatch.setattr(censysearch.Core, 'censys_key', lambda: ('platform-token', 'org-id')) + calls: list[dict[str, object]] = [] + responses = [ + FetcherResponse( + { + 'result': { + 'hits': [ + { + 'certificate_v1': { + 'resource': { + 'names': ['a.example.com'], + 'parsed': {'subject': {'email_address': ['admin@example.com']}}, + } + } + }, + {'host_v1': {'resource': {'ip': '192.0.2.1'}}}, ], - [ - {'names': ['c.example.com'], 'parsed': {'subject': {'email_address': None}}}, + 'next_page_token': 'next-page', + } + }, + 200, + {}, + ), + FetcherResponse( + { + 'result': { + 'hits': [ + { + 'certificate_v1': { + 'resource': { + 'names': ['b.example.com'], + 'parsed': {'subject': {'email_address': 'ops@example.com'}}, + } + } + } ], - ] - ) + 'next_page_token': '', + } + }, + 200, + {}, + ), + ] - monkeypatch.setattr(censysearch, 'CensysCerts', _FakeCensysCerts) + async def fake_post_fetch(url: str, **kwargs): + calls.append({'url': url, **kwargs}) + return responses.pop(0) + monkeypatch.setattr(censysearch.AsyncFetcher, 'post_fetch', fake_post_fetch) search = censysearch.SearchCensys('example.com', limit=250) - await search.process() - assert calls['init']['api_id'] == 'id' - assert calls['init']['api_secret'] == 'secret' - assert calls['search']['query'] == 'names: example.com' - assert calls['search']['per_page'] == 100 - assert calls['search']['pages'] == 3 - assert calls['search']['fields'] == ['names', 'parsed.subject.email_address'] - assert await search.get_hostnames() == {'a.example.com', 'b.example.com', 'c.example.com'} + await search.process(proxy=True) + + assert [call['url'] for call in calls] == [ + 'https://api.platform.censys.io/v3/global/search/query', + 'https://api.platform.censys.io/v3/global/search/query', + ] + assert calls[0]['headers'] == { + 'Accept': 'application/json', + 'Authorization': 'Bearer platform-token', + } + assert calls[0]['params'] == {'organization_id': 'org-id'} + assert calls[0]['json_body'] == { + 'query': 'cert.names: "example.com"', + 'fields': ['cert.names', 'cert.parsed.subject.email_address'], + 'page_size': 100, + } + assert calls[1]['json_body'] == { + **calls[0]['json_body'], + 'page_token': 'next-page', + } + assert calls[0]['json'] is True + assert calls[0]['proxy'] is True + assert calls[0]['include_metadata'] is True + assert await search.get_hostnames() == {'a.example.com', 'b.example.com'} assert await search.get_emails() == {'admin@example.com', 'ops@example.com'} + assert search.execution_status == 'completed' @pytest.mark.asyncio -async def test_search_respects_limit_across_page_data(monkeypatch) -> None: - monkeypatch.setattr(censysearch.Core, 'censys_key', lambda: ('id', 'secret')) +async def test_search_uses_free_wallet_and_respects_result_limit(monkeypatch) -> None: + monkeypatch.setattr(censysearch.Core, 'censys_key', lambda: ('platform-token', None)) + calls: list[dict[str, object]] = [] - class _FakeCensysCerts: - def __init__(self, api_id, api_secret, user_agent): - del api_id, api_secret, user_agent - - def search(self, **kwargs): - del kwargs - return _FakeQuery( - [ - [ - {'names': ['1.example.com']}, - {'names': ['2.example.com']}, - {'names': ['3.example.com']}, - {'names': ['4.example.com']}, - {'names': ['5.example.com']}, - ] - ] - ) - - monkeypatch.setattr(censysearch, 'CensysCerts', _FakeCensysCerts) + async def fake_post_fetch(_url: str, **kwargs): + calls.append(kwargs) + return FetcherResponse( + { + 'result': { + 'hits': [{'certificate_v1': {'resource': {'names': [f'{index}.example.com']}}} for index in range(1, 5)], + 'next_page_token': 'ignored-after-limit', + } + }, + 200, + {}, + ) + monkeypatch.setattr(censysearch.AsyncFetcher, 'post_fetch', fake_post_fetch) search = censysearch.SearchCensys('example.com', limit=3) + await search.process() + assert calls[0]['params'] == '' + assert calls[0]['json_body']['page_size'] == 3 assert await search.get_hostnames() == {'1.example.com', '2.example.com', '3.example.com'} + assert len(calls) == 1 + + +@pytest.mark.asyncio +async def test_search_accepts_missing_terminal_page_token(monkeypatch) -> None: + monkeypatch.setattr(censysearch.Core, 'censys_key', lambda: ('platform-token', None)) + + async def fake_post_fetch(*_args, **_kwargs): + return FetcherResponse({'result': {'hits': []}}, 200, {}) + + monkeypatch.setattr(censysearch.AsyncFetcher, 'post_fetch', fake_post_fetch) + search = censysearch.SearchCensys('example.com') + + await search.process() + + assert search.execution_status == 'completed' + assert search.stop_reason == 'no-results' + + +@pytest.mark.asyncio +async def test_malformed_limit_hit_is_partial_when_it_contains_evidence(monkeypatch) -> None: + monkeypatch.setattr(censysearch.Core, 'censys_key', lambda: ('platform-token', None)) + + async def fake_post_fetch(*_args, **_kwargs): + return FetcherResponse( + { + 'result': { + 'hits': [{'certificate_v1': {'resource': {'names': ['a.example.com'], 'parsed': 'bad'}}}], + 'next_page_token': 'unused', + } + }, + 200, + {}, + ) + + monkeypatch.setattr(censysearch.AsyncFetcher, 'post_fetch', fake_post_fetch) + search = censysearch.SearchCensys('example.com', limit=1) + + await search.process() + + assert await search.get_hostnames() == {'a.example.com'} + assert search.execution_status == 'partial' + assert search.stop_reason == 'invalid-response' + + +@pytest.mark.parametrize( + ('response', 'expected_status', 'expected_reason'), + [ + (FetcherResponse(None, 401, {}), 'failed', 'access-denied'), + (FetcherResponse(None, 429, {}), 'rate-limited', 'http-429'), + (FetcherResponse({'result': {'hits': 'bad'}}, 200, {}), 'failed', 'invalid-response'), + (None, 'failed', 'transport-error'), + ], +) +@pytest.mark.asyncio +async def test_search_reports_provider_failures_truthfully( + monkeypatch, response: FetcherResponse | None, expected_status: str, expected_reason: str +) -> None: + monkeypatch.setattr(censysearch.Core, 'censys_key', lambda: ('platform-token', None)) + + async def fake_post_fetch(*_args, **_kwargs): + return response + + monkeypatch.setattr(censysearch.AsyncFetcher, 'post_fetch', fake_post_fetch) + search = censysearch.SearchCensys('example.com') + + await search.process() + + assert search.execution_status == expected_status + assert search.stop_reason == expected_reason + + +@pytest.mark.asyncio +async def test_search_keeps_event_loop_responsive_and_propagates_cancellation(monkeypatch) -> None: + monkeypatch.setattr(censysearch.Core, 'censys_key', lambda: ('platform-token', None)) + request_started = asyncio.Event() + release_request = asyncio.Event() + + async def fake_post_fetch(*_args, **_kwargs): + request_started.set() + await release_request.wait() + + monkeypatch.setattr(censysearch.AsyncFetcher, 'post_fetch', fake_post_fetch) + task = asyncio.create_task(censysearch.SearchCensys('example.com').process()) + await request_started.wait() + callback_ran = asyncio.Event() + asyncio.get_running_loop().call_soon(callback_ran.set) + + await asyncio.wait_for(callback_ran.wait(), timeout=0.1) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + +@pytest.mark.asyncio +async def test_search_classifies_transport_exceptions(monkeypatch) -> None: + monkeypatch.setattr(censysearch.Core, 'censys_key', lambda: ('platform-token', None)) + + async def fake_post_fetch(*_args, **_kwargs): + raise OSError('offline') + + monkeypatch.setattr(censysearch.AsyncFetcher, 'post_fetch', fake_post_fetch) + search = censysearch.SearchCensys('example.com') + + await search.process() + + assert search.execution_status == 'failed' + assert search.stop_reason == 'transport-error' + + +def test_deprecated_censys_sdk_is_not_a_runtime_dependency() -> None: + assert '"censys==' not in Path('pyproject.toml').read_text() diff --git a/tests/lib/test_api_v1.py b/tests/lib/test_api_v1.py index 562e1d0b..5af0f780 100644 --- a/tests/lib/test_api_v1.py +++ b/tests/lib/test_api_v1.py @@ -381,6 +381,8 @@ def test_source_catalog_exposes_shared_action_activities(tmp_path, monkeypatch) assert response.status_code == 200 catalog = response.json() assert catalog['sources'] + censys = next(source for source in catalog['sources'] if source['name'] == 'censys') + assert censys['credentials'] == ['api-token'] assert catalog['actions'] == [ {'name': 'api-scan', 'activity': 'P2'}, {'name': 'dns-brute', 'activity': 'P1'}, diff --git a/tests/lib/test_configuration_contract.py b/tests/lib/test_configuration_contract.py index 5eb5c0dd..5cad140b 100644 --- a/tests/lib/test_configuration_contract.py +++ b/tests/lib/test_configuration_contract.py @@ -161,13 +161,13 @@ def test_provider_accessors_return_single_and_multi_field_credentials( core = configuration_environment.core configuration_dirs = configuration_environment.directories (configuration_dirs[0] / 'api-keys.yaml').write_text( - 'apikeys:\n bevigil:\n key: bevigil-key\n censys:\n id: censys-id\n secret: censys-secret\n', + 'apikeys:\n bevigil:\n key: bevigil-key\n censys:\n token: censys-token\n organization_id: censys-org\n', encoding='utf-8', ) assert (core.bevigil_key(), core.censys_key()) == ( 'bevigil-key', - ('censys-id', 'censys-secret'), + ('censys-token', 'censys-org'), ) diff --git a/tests/lib/test_core.py b/tests/lib/test_core.py index 2d115a4e..f9af1977 100644 --- a/tests/lib/test_core.py +++ b/tests/lib/test_core.py @@ -315,7 +315,7 @@ def test_user_agent_policy_separates_provider_and_browser_identities() -> None: ("accessor_name", "expected"), [ ("bevigil_key", "bevigil-key"), - ("censys_key", ("censys-id", "censys-secret")), + ("censys_key", ("censys-token", "censys-org")), ("fofa_key", ("fofa-key", "fofa-email")), ("routeviews_key", "routeviews-key"), ("tomba_key", ("tomba-key", "tomba-secret")), @@ -328,7 +328,7 @@ def test_api_key_accessors_read_configured_values(monkeypatch, accessor_name: st staticmethod( lambda: { 'bevigil': {'key': 'bevigil-key'}, - 'censys': {'id': 'censys-id', 'secret': 'censys-secret'}, + 'censys': {'token': 'censys-token', 'organization_id': 'censys-org'}, 'fofa': {'key': 'fofa-key', 'email': 'fofa-email'}, 'routeviews': {'key': 'routeviews-key'}, 'tomba': {'key': 'tomba-key', 'secret': 'tomba-secret'}, diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 4dd7f04f..290ccf3d 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -1142,7 +1142,7 @@ async def start( except MissingKey as mk: record_missing_credentials(engineitem) if not args.quiet: - output_logger.info(f'Censys API key is missing or invalid: {mk}') + output_logger.info(f'Censys Platform credentials are missing or invalid: {mk}') except ConnectionError as ce: if not args.quiet: output_logger.info(f'Network error while querying Censys: {ce}') diff --git a/theHarvester/data/api-keys.yaml b/theHarvester/data/api-keys.yaml index f2d3bd65..c457ae19 100644 --- a/theHarvester/data/api-keys.yaml +++ b/theHarvester/data/api-keys.yaml @@ -13,8 +13,8 @@ apikeys: key: censys: - id: - secret: + token: + organization_id: criminalip: key: diff --git a/theHarvester/discovery/censysearch.py b/theHarvester/discovery/censysearch.py index c7dfd4d7..1fb11e16 100644 --- a/theHarvester/discovery/censysearch.py +++ b/theHarvester/discovery/censysearch.py @@ -1,32 +1,32 @@ -import logging -from math import ceil +from typing import Any -from censys.common import __version__ -from censys.common.exceptions import ( - CensysRateLimitExceededException, - CensysUnauthorizedException, -) -from censys.search import CensysCerts - -from theHarvester import __version__ as thehavester_version from theHarvester.discovery.constants import MissingKey -from theHarvester.lib.core import Core - -logger = logging.getLogger(__name__) +from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse class SearchCensys: MAX_RESULTS_PER_PAGE = 100 + SERVER = 'https://api.platform.censys.io/v3/global/search/query' - def __init__(self, domain, limit: int = 500) -> None: + def __init__(self, domain: str, limit: int = 500) -> None: self.word = domain - self.key = Core.censys_key() - if self.key[0] is None or self.key[1] is None: - raise MissingKey('Censys ID and/or Secret') + token, self.organization_id = Core.censys_key() + if not isinstance(token, str) or not token.strip(): + raise MissingKey('Censys Personal Access Token') + self.token = token.strip() self.totalhosts: set[str] = set() self.emails: set[str] = set() self.limit = limit self.proxy = False + self.execution_status: str | None = None + self.stop_reason: str | None = None + + def _has_results(self) -> bool: + return bool(self.totalhosts or self.emails) + + def _stop(self, status: str, reason: str) -> None: + self.execution_status = 'partial' if self._has_results() else status + self.stop_reason = reason @staticmethod def _normalize_emails(email_address: object) -> set[str]: @@ -36,43 +36,116 @@ class SearchCensys: return {email for email in email_address if isinstance(email, str)} return set() - async def do_search(self) -> None: - try: - cert_search = CensysCerts( - api_id=self.key[0], - api_secret=self.key[1], - user_agent=f'censys-python/{__version__} (theHarvester/{thehavester_version}); +https://github.com/laramies/theHarvester)', - ) - except CensysUnauthorizedException: - raise MissingKey('Censys ID and/or Secret') + def _parse_hit(self, hit: object) -> bool: + if not isinstance(hit, dict): + return True + certificate = hit.get('certificate_v1') + if certificate is None: + return False + if not isinstance(certificate, dict) or not isinstance(certificate.get('resource'), dict): + return True + resource: dict[str, Any] = certificate['resource'] + names = resource.get('names', []) + if not isinstance(names, list): + return True + self.totalhosts.update(name for name in names if isinstance(name, str)) + parsed = resource.get('parsed', {}) + if not isinstance(parsed, dict): + return True + subject = parsed.get('subject', {}) + if not isinstance(subject, dict): + return True + self.emails.update(self._normalize_emails(subject.get('email_address'))) + return False + async def do_search(self) -> None: if self.limit <= 0: + self.execution_status = 'completed' + self.stop_reason = 'no-results' return - query = f'names: {self.word}' - try: - response = cert_search.search( - query=query, - per_page=min(self.limit, self.MAX_RESULTS_PER_PAGE), - pages=ceil(self.limit / self.MAX_RESULTS_PER_PAGE), - fields=['names', 'parsed.subject.email_address'], - ) - records_seen = 0 - for cert_page in response: - for cert in cert_page: - if records_seen >= self.limit: - return - self.totalhosts.update(cert.get('names', [])) - email_address = cert.get('parsed', {}).get('subject', {}).get('email_address') - self.emails.update(self._normalize_emails(email_address)) - records_seen += 1 - except CensysRateLimitExceededException: - logger.info('Censys rate limit exceeded') + headers = {'Accept': 'application/json', 'Authorization': f'Bearer {self.token}'} + params = ( + {'organization_id': self.organization_id.strip()} + if isinstance(self.organization_id, str) and self.organization_id.strip() + else '' + ) + page_token: str | None = None + seen_tokens: set[str] = set() + records_seen = 0 + malformed = False - async def get_hostnames(self) -> set: + while records_seen < self.limit: + body = { + 'query': f'cert.names: "{self.word}"', + 'fields': ['cert.names', 'cert.parsed.subject.email_address'], + 'page_size': min(self.MAX_RESULTS_PER_PAGE, self.limit - records_seen), + } + if page_token is not None: + body['page_token'] = page_token + try: + response = await AsyncFetcher.post_fetch( + self.SERVER, + headers=headers, + params=params, + json=True, + proxy=self.proxy, + include_metadata=True, + json_body=body, + ) + except Exception: + self._stop('failed', 'transport-error') + return + if not isinstance(response, FetcherResponse): + self._stop('failed', 'transport-error') + return + if response.status == 429: + self._stop('rate-limited', 'http-429') + return + if response.status in {401, 403}: + self._stop('failed', 'access-denied') + return + if not 200 <= response.status < 300: + self._stop('failed', f'http-{response.status}') + return + if not isinstance(response.body, dict) or not isinstance(response.body.get('result'), dict): + self._stop('failed', 'invalid-response') + return + result = response.body['result'] + hits = result.get('hits') + next_page_token = result.get('next_page_token') + if not isinstance(hits, list) or (next_page_token is not None and not isinstance(next_page_token, str)): + self._stop('failed', 'invalid-response') + return + + for hit in hits: + if records_seen >= self.limit: + break + malformed = self._parse_hit(hit) or malformed + records_seen += 1 + if records_seen >= self.limit: + if malformed: + self._stop('failed', 'invalid-response') + else: + self.execution_status = 'completed' + return + if not next_page_token: + if malformed: + self._stop('failed', 'invalid-response') + else: + self.execution_status = 'completed' + self.stop_reason = None if self._has_results() else 'no-results' + return + if next_page_token in seen_tokens: + self._stop('failed', 'repeated-cursor') + return + seen_tokens.add(next_page_token) + page_token = next_page_token + + async def get_hostnames(self) -> set[str]: return self.totalhosts - async def get_emails(self) -> set: + async def get_emails(self) -> set[str]: return self.emails async def process(self, proxy: bool = False) -> None: diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index 638d9cfd..3752d137 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -159,7 +159,7 @@ class Core: 'brave': ('key',), 'bufferoverun': ('key',), 'builtwith': ('key',), - 'censys': ('id', 'secret'), + 'censys': ('token',), 'criminalip': ('key',), 'dehashed': ('key',), 'dnsdb': ('key',), @@ -245,8 +245,9 @@ class Core: return Core._api_key_value('builtwith') @staticmethod - def censys_key() -> tuple: - return Core._api_key_value('censys') + def censys_key() -> tuple[object, object]: + credentials = Core.api_keys().get('censys', {}) + return credentials.get('token'), credentials.get('organization_id') @staticmethod def criminalip_key() -> str: diff --git a/uv.lock b/uv.lock index 2466d206..e530d1f3 100644 --- a/uv.lock +++ b/uv.lock @@ -219,15 +219,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ba/16/9826f089383c593cdfc4a6e5aca94d9e91ae1692c57af82c3b2aa5e810f7/anyio-4.14.0-py3-none-any.whl", hash = "sha256:dd9b7a2a9799ed6552fde617b2c5df02b7fdd7d88392fc48101e51bae46164d9", size = 123506, upload-time = "2026-06-15T22:00:47.595Z" }, ] -[[package]] -name = "argcomplete" -version = "3.6.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/38/61/0b9ae6399dd4a58d8c1b1dc5a27d6f2808023d0b5dd3104bb99f45a33ff6/argcomplete-3.6.3.tar.gz", hash = "sha256:62e8ed4fd6a45864acc8235409461b72c9a28ee785a2011cc5eb78318786c89c", size = 73754, upload-time = "2025-10-20T03:33:34.741Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/74/f5/9373290775639cb67a2fce7f629a1c240dce9f12fe927bc32b2736e16dfc/argcomplete-3.6.3-py3-none-any.whl", hash = "sha256:f5007b3a600ccac5d25bbce33089211dfd49eab4a7718da3f10e3082525a92ce", size = 43846, upload-time = "2025-10-20T03:33:33.021Z" }, -] - [[package]] name = "ast-serialize" version = "0.5.0" @@ -277,15 +268,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, ] -[[package]] -name = "backoff" -version = "2.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/47/d7/5bbeb12c44d7c4f2fb5b56abce497eb5ed9f34d85701de869acedd602619/backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba", size = 17001, upload-time = "2022-10-05T19:19:32.061Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/df/73/b6e24bd22e6720ca8ee9a85a0c4a2971af8497d8f3193fa05390cbd46e09/backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8", size = 15148, upload-time = "2022-10-05T19:19:30.546Z" }, -] - [[package]] name = "beautifulsoup4" version = "4.15.0" @@ -299,22 +281,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl", hash = "sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9", size = 109924, upload-time = "2026-06-07T16:44:21.566Z" }, ] -[[package]] -name = "censys" -version = "2.2.19" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "argcomplete" }, - { name = "backoff" }, - { name = "requests" }, - { name = "rich" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/41/aa/ed0d0faf4f7015bac902cdad929f487f9baefd224ab6fa9aba5635dd5d60/censys-2.2.19.tar.gz", hash = "sha256:9202e17c2583d4b3d0af32a5be161ddb505edd390a9ca909f2e7470d4af19a97", size = 62101, upload-time = "2025-12-11T16:08:47.421Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/66/4b/96c1ebc5f8534c18527c81b4df9cdb2a96151010f540ecbb4c9c2af4fee4/censys-2.2.19-py3-none-any.whl", hash = "sha256:eaad49779a3bbe290e244222563a48e78ca33777fbbdf6d329d8b52223fc1084", size = 80582, upload-time = "2025-12-11T16:08:46.368Z" }, -] - [[package]] name = "certifi" version = "2026.6.17" @@ -851,27 +817,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/2c/0f1e93c636720e8a3eb59af2bfda99d98b55891e1c53bc30c2e0e865f01b/lxml-6.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:58bb955caba94e467d2a96da17660d2d704e0675894cba21ab8a775b8621fd1c", size = 3817223, upload-time = "2026-05-19T19:22:56.823Z" }, ] -[[package]] -name = "markdown-it-py" -version = "4.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mdurl" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, -] - -[[package]] -name = "mdurl" -version = "0.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, -] - [[package]] name = "multidict" version = "6.7.1" @@ -1512,19 +1457,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/67/f3/6cd296376653270ac1b423bb30bd70942d9916b6978c6f40472d6ac038e7/retrying-1.4.2-py3-none-any.whl", hash = "sha256:bbc004aeb542a74f3569aeddf42a2516efefcdaff90df0eb38fbfbf19f179f59", size = 10859, upload-time = "2025-08-03T03:35:23.829Z" }, ] -[[package]] -name = "rich" -version = "15.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markdown-it-py" }, - { name = "pygments" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, -] - [[package]] name = "ruff" version = "0.16.1" @@ -1642,7 +1574,6 @@ dependencies = [ { name = "aiomultiprocess" }, { name = "aiosqlite" }, { name = "beautifulsoup4" }, - { name = "censys" }, { name = "certifi" }, { name = "dnspython" }, { name = "fastapi" }, @@ -1686,7 +1617,6 @@ requires-dist = [ { name = "aiomultiprocess", specifier = "==0.9.1" }, { name = "aiosqlite", specifier = "==0.22.1" }, { name = "beautifulsoup4", specifier = "==4.15.0" }, - { name = "censys", specifier = "==2.2.19" }, { name = "certifi", specifier = "==2026.6.17" }, { name = "dnspython", specifier = "==2.8.0" }, { name = "fastapi", specifier = "==0.138.1" },