diff --git a/tests/lib/test_core.py b/tests/lib/test_core.py index f81996bd..2fc95b19 100644 --- a/tests/lib/test_core.py +++ b/tests/lib/test_core.py @@ -2,6 +2,7 @@ from __future__ import annotations import asyncio import logging +import ssl import stat from concurrent.futures import ThreadPoolExecutor from pathlib import Path @@ -452,6 +453,103 @@ async def test_fetch_reused_session_uses_a_stable_explicit_ssl_policy(monkeypatc assert ssl_policies[0] is ssl_policies[1] +@pytest.mark.asyncio +async def test_fetch_reused_session_defers_default_ssl_policy_to_the_connector(monkeypatch) -> None: + session = DummySession() + + def fail_if_context_built(_verify=True): + raise AssertionError('a borrowed session must not rebuild an SSL context per request') + + monkeypatch.setattr(AsyncFetcher, '_ssl_context', staticmethod(fail_if_context_built)) + + await AsyncFetcher.fetch(session=session, url='https://example.com/one', verify=None) + + assert session.requests == [('GET', 'https://example.com/one', {})] + + +@pytest.mark.asyncio +async def test_fetch_reused_session_forwards_caller_headers(monkeypatch) -> None: + session = DummySession() + + await AsyncFetcher.fetch(session=session, url='https://example.com/api', headers={'X-Api-Key': 'test-key'}) + + _method, _url, request_options = session.requests[0] + assert request_options['headers']['X-Api-Key'] == 'test-key' + + +@pytest.mark.asyncio +async def test_fetch_reused_session_without_headers_keeps_session_headers(monkeypatch) -> None: + session = DummySession() + + await AsyncFetcher.fetch(session=session, url='https://example.com/api') + + assert 'headers' not in session.requests[0][2] + + +@pytest.mark.asyncio +async def test_post_fetch_reused_session_forwards_caller_headers(monkeypatch) -> None: + session = DummySession() + + await AsyncFetcher.post_fetch( + 'https://example.com/api', + headers={'X-Api-Key': 'test-key'}, + session=session, + json=True, + json_body={'query': 'example.com'}, + ) + + _method, _url, request_options = session.requests[0] + assert request_options['headers']['X-Api-Key'] == 'test-key' + assert request_options['json'] == {'query': 'example.com'} + + +@pytest.mark.asyncio +async def test_post_fetch_sends_the_default_empty_body(monkeypatch) -> None: + session = DummySession() + + await AsyncFetcher.post_fetch('https://example.com/api', session=session, json=True) + + _method, _url, request_options = session.requests[0] + assert request_options['data'] == '' + + +@pytest.mark.asyncio +async def test_post_fetch_passes_non_json_string_data_through(monkeypatch) -> None: + session = DummySession() + + await AsyncFetcher.post_fetch('https://example.com/api', data='field=value', session=session) + + _method, _url, request_options = session.requests[0] + assert request_options['data'] == 'field=value' + + +@pytest.mark.asyncio +async def test_tcp_connector_honors_disabled_verification() -> None: + connector = await AsyncFetcher._create_connector(None, None, False) + try: + assert connector._ssl is False + finally: + await connector.close() + + +@pytest.mark.asyncio +async def test_tcp_connector_defaults_to_a_verifying_context() -> None: + connector = await AsyncFetcher._create_connector(None, None, None) + try: + assert isinstance(connector._ssl, ssl.SSLContext) + finally: + await connector.close() + + +@pytest.mark.asyncio +async def test_create_session_with_verification_disabled_builds_an_unverified_connector() -> None: + session = await AsyncFetcher.create_session(verify=False) + try: + assert session.connector._ssl is False + finally: + await session.close() + + @pytest.mark.asyncio async def test_open_session_owns_one_proxy_and_cookie_policy(monkeypatch: pytest.MonkeyPatch) -> None: reset_dummy_sessions() diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index e82b9b2a..758fb538 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -526,7 +526,12 @@ class AsyncFetcher: @staticmethod def _normalize_data(data: str | dict[str, Any]) -> str | dict[str, Any]: - return json_loader.loads(data) if isinstance(data, str) else data + if isinstance(data, str) and data: + try: + return json_loader.loads(data) + except ValueError: + return data + return data @classmethod def _resolve_proxy(cls, proxy: str | bool | None) -> tuple[str | None, str | None]: @@ -779,7 +784,9 @@ class AsyncFetcher: return connector else: # Use default TCP connector for HTTP proxies or no proxy - return aiohttp.TCPConnector(ssl=ssl_context or ssl.create_default_context(cafile=certifi.where())) + return aiohttp.TCPConnector( + ssl=ssl_context if ssl_context is not None else ssl.create_default_context(cafile=certifi.where()) + ) @classmethod async def post_fetch( @@ -796,6 +803,7 @@ class AsyncFetcher: session: aiohttp.ClientSession | None = None, response_byte_limit: int | None = None, ) -> Any: + caller_headers = headers headers = cls._default_headers(headers) # By default, timeout is 5 minutes, changed to 12-minutes # results are well worth the wait @@ -817,6 +825,8 @@ class AsyncFetcher: } if params != '': request_kwargs['params'] = params + if caller_headers is not None: + request_kwargs['headers'] = headers return await cls._request( session, 'POST', @@ -855,7 +865,14 @@ class AsyncFetcher: """ try: owns_session = session is None - ssl_arg = cls._ssl_context(verify) if owns_session or not isinstance(verify, bool) else verify + if owns_session: + ssl_arg = cls._ssl_context(verify) + elif isinstance(verify, bool): + ssl_arg = verify + else: + # A borrowed session already owns its TLS policy; defer to the + # session connector instead of rebuilding a context per request. + ssl_arg = None proxy_url, proxy_type = cls._resolve_proxy(proxy) client_timeout = cls._request_timeout(request_timeout) req_headers = cls._default_headers(headers) @@ -871,9 +888,9 @@ class AsyncFetcher: assert session is not None try: - request_kwargs: dict[str, Any] = { - 'ssl': ssl_arg, - } + request_kwargs: dict[str, Any] = {} + if ssl_arg is not None: + request_kwargs['ssl'] = ssl_arg # For HTTP proxies, pass the proxy parameter; for SOCKS5, the connector handles it if proxy_url and proxy_type == 'http': request_kwargs['proxy'] = proxy_url @@ -881,6 +898,8 @@ class AsyncFetcher: request_kwargs['allow_redirects'] = follow_redirects if params != '': request_kwargs['params'] = params + if not owns_session and headers is not None: + request_kwargs['headers'] = req_headers return await cls._request( session, method, @@ -1102,6 +1121,7 @@ class AsyncFetcher: url=url, params=params, json=json, + headers=headers, include_metadata=include_metadata, ) for url in urls