From 66e1beee7c67ab353fbe8a330177e72f5a57bdfa Mon Sep 17 00:00:00 2001 From: NotoriousRebel <36310667+NotoriousRebel@users.noreply.github.com> Date: Wed, 5 Aug 2026 18:31:36 -0400 Subject: [PATCH] feat: preserve shared HTTP response metadata --- tests/lib/test_core.py | 138 ++++++++++++++++++++++++++++++++++++++- theHarvester/lib/core.py | 78 +++++++++++++++++++--- 2 files changed, 205 insertions(+), 11 deletions(-) diff --git a/tests/lib/test_core.py b/tests/lib/test_core.py index 5d964361..1fcdf7d3 100644 --- a/tests/lib/test_core.py +++ b/tests/lib/test_core.py @@ -9,7 +9,7 @@ import pytest import yaml import theHarvester.lib.core as core_module -from theHarvester.lib.core import CONFIG_DIRS, DATA_DIR, AsyncFetcher, Core +from theHarvester.lib.core import CONFIG_DIRS, DATA_DIR, AsyncFetcher, Core, FetcherResponse from theHarvester.lib.output import configure_logging @@ -132,10 +132,21 @@ def test_read_config_copies_default_to_home(name: str, capsys): assert file.exists() +_DEFAULT_JSON = object() + + class DummyResponse: - def __init__(self, text_value: str = 'response-text', json_value: Any = None): + def __init__( + self, + text_value: str = 'response-text', + json_value: Any = _DEFAULT_JSON, + status: int = 200, + headers: dict[str, str] | None = None, + ): self.text_value = text_value - self.json_value = {'ok': True} if json_value is None else json_value + self.json_value = {'ok': True} if json_value is _DEFAULT_JSON else json_value + self.status = status + self.headers = headers or {} async def __aenter__(self): return self @@ -147,6 +158,8 @@ class DummyResponse: return self.text_value async def json(self): + if isinstance(self.json_value, Exception): + raise self.json_value return self.json_value @@ -259,6 +272,108 @@ async def test_fetch_creates_session_with_default_headers(monkeypatch) -> None: ] +@pytest.mark.asyncio +async def test_fetch_can_include_buffered_response_metadata(monkeypatch) -> None: + reset_dummy_sessions() + monkeypatch.setattr(core_module.aiohttp, 'ClientSession', DummySession) + monkeypatch.setattr(core_module.ssl, 'create_default_context', lambda cafile=None: 'ssl-context') + monkeypatch.setattr(core_module.certifi, 'where', lambda: '/tmp/cacert.pem') + monkeypatch.setattr(core_module.asyncio, 'sleep', fake_sleep) + + def request_with_metadata(self, method: str, url: str, **kwargs): + self.requests.append((method, url, kwargs)) + return DummyResponse( + text_value='rate limited', + status=429, + headers={'Retry-After': '60', 'X-RateLimit-Remaining': '0'}, + ) + + monkeypatch.setattr(DummySession, 'request', request_with_metadata) + + result = await AsyncFetcher.fetch(url='https://example.com', include_metadata=True) + + assert result == FetcherResponse( + body='rate limited', + status=429, + headers={'retry-after': '60', 'x-ratelimit-remaining': '0'}, + ) + + +@pytest.mark.asyncio +async def test_fetch_metadata_distinguishes_transport_failure(monkeypatch) -> None: + async def failed_request(*_args: Any, **_kwargs: Any) -> str: + raise OSError('network unavailable') + + monkeypatch.setattr(AsyncFetcher, '_request', failed_request) + + result = await AsyncFetcher.fetch(session=DummySession(), url='https://example.com', include_metadata=True) + + assert result is None + + +@pytest.mark.asyncio +async def test_fetch_metadata_preserves_non_json_error_body(monkeypatch) -> None: + reset_dummy_sessions() + monkeypatch.setattr(core_module.aiohttp, 'ClientSession', DummySession) + monkeypatch.setattr(core_module.ssl, 'create_default_context', lambda cafile=None: 'ssl-context') + monkeypatch.setattr(core_module.certifi, 'where', lambda: '/tmp/cacert.pem') + monkeypatch.setattr(core_module.asyncio, 'sleep', fake_sleep) + + def request_with_invalid_json(self, method: str, url: str, **kwargs): + self.requests.append((method, url, kwargs)) + return DummyResponse(text_value='upstream error', json_value=ValueError(), status=502) + + monkeypatch.setattr(DummySession, 'request', request_with_invalid_json) + + result = await AsyncFetcher.fetch(url='https://example.com', json=True, include_metadata=True) + + assert result == FetcherResponse(body='upstream error', status=502, headers={}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ('text_value', 'expected_body'), + [('', ''), ('null', None)], +) +async def test_fetch_metadata_distinguishes_empty_json_from_null( + monkeypatch, + text_value: str, + expected_body: Any, +) -> None: + reset_dummy_sessions() + monkeypatch.setattr(core_module.aiohttp, 'ClientSession', DummySession) + monkeypatch.setattr(core_module.ssl, 'create_default_context', lambda cafile=None: 'ssl-context') + monkeypatch.setattr(core_module.certifi, 'where', lambda: '/tmp/cacert.pem') + monkeypatch.setattr(core_module.asyncio, 'sleep', fake_sleep) + + def request_with_json(self, method: str, url: str, **kwargs): + self.requests.append((method, url, kwargs)) + return DummyResponse(text_value=text_value, json_value=None) + + monkeypatch.setattr(DummySession, 'request', request_with_json) + + result = await AsyncFetcher.fetch(url='https://example.com', json=True, include_metadata=True) + + assert result == FetcherResponse(body=expected_body, status=200, headers={}) + + +@pytest.mark.asyncio +async def test_fetch_all_propagates_metadata_opt_in(monkeypatch) -> None: + seen: list[bool] = [] + + async def fake_fetch(*_args: Any, include_metadata: bool = False, **_kwargs: Any) -> FetcherResponse: + seen.append(include_metadata) + return FetcherResponse(body='limited', status=429, headers={'retry-after': '60'}) + + monkeypatch.setattr(core_module.aiohttp, 'ClientSession', DummySession) + monkeypatch.setattr(AsyncFetcher, 'fetch', fake_fetch) + + results = await AsyncFetcher.fetch_all(['https://one.example', 'https://two.example'], include_metadata=True) + + assert seen == [True, True] + assert [result.status for result in results] == [429, 429] + + @pytest.mark.asyncio async def test_fetch_uses_http_proxy_when_enabled(monkeypatch) -> None: reset_dummy_sessions() @@ -307,6 +422,23 @@ async def test_post_fetch_decodes_string_payload_and_posts_params(monkeypatch) - ] +@pytest.mark.asyncio +async def test_post_fetch_can_include_response_metadata(monkeypatch) -> None: + reset_dummy_sessions() + monkeypatch.setattr(core_module.aiohttp, 'ClientSession', DummySession) + monkeypatch.setattr(core_module.asyncio, 'sleep', fake_sleep) + + def request_with_metadata(self, method: str, url: str, **kwargs): + self.requests.append((method, url, kwargs)) + return DummyResponse(text_value='unavailable', status=503, headers={'Retry-After': '30'}) + + monkeypatch.setattr(DummySession, 'request', request_with_metadata) + + result = await AsyncFetcher.post_fetch('https://example.com/api', data='{}', include_metadata=True) + + assert result == FetcherResponse(body='unavailable', status=503, headers={'retry-after': '30'}) + + @pytest.mark.asyncio async def test_post_fetch_proxy_branch_uses_get_with_http_proxy(monkeypatch) -> None: reset_dummy_sessions() diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index 52eefbc9..a6fd4f87 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -5,6 +5,7 @@ import contextlib import logging import random import ssl +from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar @@ -33,6 +34,13 @@ CONFIG_DIRS = [ ] +@dataclass(frozen=True) +class FetcherResponse: + body: Any + status: int + headers: dict[str, str] + + class Core: quiet: bool = False _API_KEY_FIELDS: ClassVar[dict[str, tuple[str, ...]]] = { @@ -490,9 +498,34 @@ class AsyncFetcher: return aiohttp.ClientSession(headers=headers, timeout=client_timeout, connector=connector) @staticmethod - async def _read_response(response: aiohttp.ClientResponse, *, json: bool, delay: int) -> Any: + async def _read_response( + response: aiohttp.ClientResponse, + *, + json: bool, + delay: int, + include_metadata: bool = False, + ) -> Any: await asyncio.sleep(delay) - return await response.text() if json is False else await response.json() + if json is False: + body = await response.text() + elif include_metadata: + text_body = await response.text() + if not text_body.strip(): + body = text_body + else: + try: + body = await response.json() + except (aiohttp.ContentTypeError, ValueError): + body = text_body + else: + body = await response.json() + if not include_metadata: + return body + return FetcherResponse( + body=body, + status=response.status, + headers={name.lower(): value for name, value in response.headers.items()}, + ) @classmethod async def _request( @@ -504,15 +537,26 @@ class AsyncFetcher: json: bool = False, delay: int = 5, request_timeout: int | None = None, + include_metadata: bool = False, **request_kwargs: Any, ) -> Any: if request_timeout: async with asyncio.timeout(request_timeout): async with session.request(method.upper(), url, **request_kwargs) as response: - return await cls._read_response(response, json=json, delay=delay) + return await cls._read_response( + response, + json=json, + delay=delay, + include_metadata=include_metadata, + ) async with session.request(method.upper(), url, **request_kwargs) as response: - return await cls._read_response(response, json=json, delay=delay) + return await cls._read_response( + response, + json=json, + delay=delay, + include_metadata=include_metadata, + ) @staticmethod def _get_random_proxy(proxy_dict: dict) -> tuple[str | None, str | None]: @@ -555,6 +599,7 @@ class AsyncFetcher: params: Sized = '', json: bool = False, proxy: bool = False, + include_metadata: bool = False, ): headers = cls._default_headers(headers) timeout = cls._request_timeout(720) @@ -575,6 +620,7 @@ class AsyncFetcher: proxy=proxy_url if proxy_type == 'http' else None, json=json, delay=5, + include_metadata=include_metadata, ) else: async with await cls._build_session(headers, timeout, proxy_url, proxy_type, sslcontext) as session: @@ -585,6 +631,7 @@ class AsyncFetcher: proxy=proxy_url if proxy_type == 'http' else None, json=json, delay=5, + include_metadata=include_metadata, ) elif params == '': async with await cls._build_session(headers, timeout) as session: @@ -595,6 +642,7 @@ class AsyncFetcher: data=cls._normalize_data(data), json=json, delay=3, + include_metadata=include_metadata, ) else: async with await cls._build_session(headers, timeout) as session: @@ -607,9 +655,10 @@ class AsyncFetcher: params=params, json=json, delay=3, + include_metadata=include_metadata, ) except (aiohttp.ClientError, TimeoutError, OSError, ssl.SSLError, UnicodeDecodeError, ValueError): - return '' + return None if include_metadata else '' @classmethod async def fetch( @@ -624,6 +673,7 @@ class AsyncFetcher: verify: bool | None = None, follow_redirects: bool | None = None, request_timeout: int | None = None, + include_metadata: bool = False, ) -> Any: """Generic HTTP request helper. - If a session is not provided, one will be created and closed automatically. @@ -665,13 +715,14 @@ class AsyncFetcher: json=json, delay=5, request_timeout=request_timeout, + include_metadata=include_metadata, **request_kwargs, ) finally: if owns_session: await session.close() except (aiohttp.ClientError, TimeoutError, OSError, ssl.SSLError, UnicodeDecodeError, ValueError): - return '' + return None if include_metadata else '' @staticmethod async def takeover_fetch(session, url: str, proxy: str | None = None) -> tuple[Any, Any] | str: @@ -714,6 +765,7 @@ class AsyncFetcher: json: bool = False, takeover: bool = False, proxy: bool = False, + include_metadata: bool = False, ) -> list: # By default, timeout is 5 minutes; 60 seconds should suffice headers = cls._default_headers(headers) @@ -747,13 +799,18 @@ class AsyncFetcher: url, json=json, proxy=proxy_url, + include_metadata=include_metadata, ) for url, (proxy_url, proxy_type) in zip(urls, proxy_data, strict=False) ] ) ) else: - return list(await asyncio.gather(*[AsyncFetcher.fetch(session, url, json=json) for url in urls])) + return list( + await asyncio.gather( + *[AsyncFetcher.fetch(session, url, json=json, include_metadata=include_metadata) for url in urls] + ) + ) else: # Indicates the request has certain params async with aiohttp.ClientSession(headers=headers, timeout=timeout) as session: @@ -768,13 +825,18 @@ class AsyncFetcher: params, json, proxy=proxy_url, + include_metadata=include_metadata, ) for url, (proxy_url, proxy_type) in zip(urls, proxy_data, strict=False) ] ) ) else: - return list(await asyncio.gather(*[AsyncFetcher.fetch(session, url, params, json) for url in urls])) + return list( + await asyncio.gather( + *[AsyncFetcher.fetch(session, url, params, json, include_metadata=include_metadata) for url in urls] + ) + ) def show_default_error_message(engine_name: str, word: str, error) -> None: