WEBDIENSTE/theHarvester
1
0
Fork 0
mirror of https://github.com/laramies/theHarvester.git synced 2026-09-28 20:44:53 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2471 from NotoriousRebel/codex/http-response-metadata-v2

Browse Source 
Preserve shared HTTP response metadata
This commit is contained in:
Matt
2026-08-05 18:43:00 -04:00
committed by GitHub
parent 38983bd76a 66e1beee7c
commit abb065c95a
2 changed files with 205 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/lib/test_core.py
+135 -3
View File
@@ -9,7 +9,7 @@ import pytest
import yaml
import theHarvester.lib.core as core_module
from theHarvester.lib.core import CONFIG_DIRS, DATA_DIR, AsyncFetcher, Core
from theHarvester.lib.core import CONFIG_DIRS, DATA_DIR, AsyncFetcher, Core, FetcherResponse
from theHarvester.lib.output import configure_logging
@@ -132,10 +132,21 @@ def test_read_config_copies_default_to_home(name: str, capsys):
assert file.exists()
_DEFAULT_JSON = object()
class DummyResponse:
def __init__(self, text_value: str = 'response-text', json_value: Any = None):
def __init__(
self,
text_value: str = 'response-text',
json_value: Any = _DEFAULT_JSON,
status: int = 200,
headers: dict[str, str] | None = None,
):
self.text_value = text_value
self.json_value = {'ok': True} if json_value is None else json_value
self.json_value = {'ok': True} if json_value is _DEFAULT_JSON else json_value
self.status = status
self.headers = headers or {}
async def __aenter__(self):
return self
@@ -147,6 +158,8 @@ class DummyResponse:
return self.text_value
async def json(self):
if isinstance(self.json_value, Exception):
raise self.json_value
return self.json_value
@@ -259,6 +272,108 @@ async def test_fetch_creates_session_with_default_headers(monkeypatch) -> None:
]
@pytest.mark.asyncio
async def test_fetch_can_include_buffered_response_metadata(monkeypatch) -> None:
reset_dummy_sessions()
monkeypatch.setattr(core_module.aiohttp, 'ClientSession', DummySession)
monkeypatch.setattr(core_module.ssl, 'create_default_context', lambda cafile=None: 'ssl-context')
monkeypatch.setattr(core_module.certifi, 'where', lambda: '/tmp/cacert.pem')
monkeypatch.setattr(core_module.asyncio, 'sleep', fake_sleep)
def request_with_metadata(self, method: str, url: str, **kwargs):
self.requests.append((method, url, kwargs))
return DummyResponse(
text_value='rate limited',
status=429,
headers={'Retry-After': '60', 'X-RateLimit-Remaining': '0'},
)
monkeypatch.setattr(DummySession, 'request', request_with_metadata)
result = await AsyncFetcher.fetch(url='https://example.com', include_metadata=True)
assert result == FetcherResponse(
body='rate limited',
status=429,
headers={'retry-after': '60', 'x-ratelimit-remaining': '0'},
)
@pytest.mark.asyncio
async def test_fetch_metadata_distinguishes_transport_failure(monkeypatch) -> None:
async def failed_request(*_args: Any, **_kwargs: Any) -> str:
raise OSError('network unavailable')
monkeypatch.setattr(AsyncFetcher, '_request', failed_request)
result = await AsyncFetcher.fetch(session=DummySession(), url='https://example.com', include_metadata=True)
assert result is None
@pytest.mark.asyncio
async def test_fetch_metadata_preserves_non_json_error_body(monkeypatch) -> None:
reset_dummy_sessions()
monkeypatch.setattr(core_module.aiohttp, 'ClientSession', DummySession)
monkeypatch.setattr(core_module.ssl, 'create_default_context', lambda cafile=None: 'ssl-context')
monkeypatch.setattr(core_module.certifi, 'where', lambda: '/tmp/cacert.pem')
monkeypatch.setattr(core_module.asyncio, 'sleep', fake_sleep)
def request_with_invalid_json(self, method: str, url: str, **kwargs):
self.requests.append((method, url, kwargs))
return DummyResponse(text_value='upstream error', json_value=ValueError(), status=502)
monkeypatch.setattr(DummySession, 'request', request_with_invalid_json)
result = await AsyncFetcher.fetch(url='https://example.com', json=True, include_metadata=True)
assert result == FetcherResponse(body='upstream error', status=502, headers={})
@pytest.mark.asyncio
@pytest.mark.parametrize(
('text_value', 'expected_body'),
[('', ''), ('null', None)],
)
async def test_fetch_metadata_distinguishes_empty_json_from_null(
monkeypatch,
text_value: str,
expected_body: Any,
) -> None:
reset_dummy_sessions()
monkeypatch.setattr(core_module.aiohttp, 'ClientSession', DummySession)
monkeypatch.setattr(core_module.ssl, 'create_default_context', lambda cafile=None: 'ssl-context')
monkeypatch.setattr(core_module.certifi, 'where', lambda: '/tmp/cacert.pem')
monkeypatch.setattr(core_module.asyncio, 'sleep', fake_sleep)
def request_with_json(self, method: str, url: str, **kwargs):
self.requests.append((method, url, kwargs))
return DummyResponse(text_value=text_value, json_value=None)
monkeypatch.setattr(DummySession, 'request', request_with_json)
result = await AsyncFetcher.fetch(url='https://example.com', json=True, include_metadata=True)
assert result == FetcherResponse(body=expected_body, status=200, headers={})
@pytest.mark.asyncio
async def test_fetch_all_propagates_metadata_opt_in(monkeypatch) -> None:
seen: list[bool] = []
async def fake_fetch(*_args: Any, include_metadata: bool = False, **_kwargs: Any) -> FetcherResponse:
seen.append(include_metadata)
return FetcherResponse(body='limited', status=429, headers={'retry-after': '60'})
monkeypatch.setattr(core_module.aiohttp, 'ClientSession', DummySession)
monkeypatch.setattr(AsyncFetcher, 'fetch', fake_fetch)
results = await AsyncFetcher.fetch_all(['https://one.example', 'https://two.example'], include_metadata=True)
assert seen == [True, True]
assert [result.status for result in results] == [429, 429]
@pytest.mark.asyncio
async def test_fetch_uses_http_proxy_when_enabled(monkeypatch) -> None:
reset_dummy_sessions()
@@ -307,6 +422,23 @@ async def test_post_fetch_decodes_string_payload_and_posts_params(monkeypatch) -
]
@pytest.mark.asyncio
async def test_post_fetch_can_include_response_metadata(monkeypatch) -> None:
reset_dummy_sessions()
monkeypatch.setattr(core_module.aiohttp, 'ClientSession', DummySession)
monkeypatch.setattr(core_module.asyncio, 'sleep', fake_sleep)
def request_with_metadata(self, method: str, url: str, **kwargs):
self.requests.append((method, url, kwargs))
return DummyResponse(text_value='unavailable', status=503, headers={'Retry-After': '30'})
monkeypatch.setattr(DummySession, 'request', request_with_metadata)
result = await AsyncFetcher.post_fetch('https://example.com/api', data='{}', include_metadata=True)
assert result == FetcherResponse(body='unavailable', status=503, headers={'retry-after': '30'})
@pytest.mark.asyncio
async def test_post_fetch_proxy_branch_uses_get_with_http_proxy(monkeypatch) -> None:
reset_dummy_sessions()
theHarvester/lib/core.py
+70 -8
View File
@@ -5,6 +5,7 @@ import contextlib
import logging
import random
import ssl
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING, Any, ClassVar
@@ -33,6 +34,13 @@ CONFIG_DIRS = [
]
@dataclass(frozen=True)
class FetcherResponse:
body: Any
status: int
headers: dict[str, str]
class Core:
quiet: bool = False
_API_KEY_FIELDS: ClassVar[dict[str, tuple[str, ...]]] = {
@@ -490,9 +498,34 @@ class AsyncFetcher:
return aiohttp.ClientSession(headers=headers, timeout=client_timeout, connector=connector)
@staticmethod
async def _read_response(response: aiohttp.ClientResponse, *, json: bool, delay: int) -> Any:
async def _read_response(
response: aiohttp.ClientResponse,
*,
json: bool,
delay: int,
include_metadata: bool = False,
) -> Any:
await asyncio.sleep(delay)
return await response.text() if json is False else await response.json()
if json is False:
body = await response.text()
elif include_metadata:
text_body = await response.text()
if not text_body.strip():
body = text_body
else:
try:
body = await response.json()
except (aiohttp.ContentTypeError, ValueError):
body = text_body
else:
body = await response.json()
if not include_metadata:
return body
return FetcherResponse(
body=body,
status=response.status,
headers={name.lower(): value for name, value in response.headers.items()},
)
@classmethod
async def _request(
@@ -504,15 +537,26 @@ class AsyncFetcher:
json: bool = False,
delay: int = 5,
request_timeout: int | None = None,
include_metadata: bool = False,
**request_kwargs: Any,
) -> Any:
if request_timeout:
async with asyncio.timeout(request_timeout):
async with session.request(method.upper(), url, **request_kwargs) as response:
return await cls._read_response(response, json=json, delay=delay)
return await cls._read_response(
response,
json=json,
delay=delay,
include_metadata=include_metadata,
)
async with session.request(method.upper(), url, **request_kwargs) as response:
return await cls._read_response(response, json=json, delay=delay)
return await cls._read_response(
response,
json=json,
delay=delay,
include_metadata=include_metadata,
)
@staticmethod
def _get_random_proxy(proxy_dict: dict) -> tuple[str | None, str | None]:
@@ -555,6 +599,7 @@ class AsyncFetcher:
params: Sized = '',
json: bool = False,
proxy: bool = False,
include_metadata: bool = False,
):
headers = cls._default_headers(headers)
timeout = cls._request_timeout(720)
@@ -575,6 +620,7 @@ class AsyncFetcher:
proxy=proxy_url if proxy_type == 'http' else None,
json=json,
delay=5,
include_metadata=include_metadata,
)
else:
async with await cls._build_session(headers, timeout, proxy_url, proxy_type, sslcontext) as session:
@@ -585,6 +631,7 @@ class AsyncFetcher:
proxy=proxy_url if proxy_type == 'http' else None,
json=json,
delay=5,
include_metadata=include_metadata,
)
elif params == '':
async with await cls._build_session(headers, timeout) as session:
@@ -595,6 +642,7 @@ class AsyncFetcher:
data=cls._normalize_data(data),
json=json,
delay=3,
include_metadata=include_metadata,
)
else:
async with await cls._build_session(headers, timeout) as session:
@@ -607,9 +655,10 @@ class AsyncFetcher:
params=params,
json=json,
delay=3,
include_metadata=include_metadata,
)
except (aiohttp.ClientError, TimeoutError, OSError, ssl.SSLError, UnicodeDecodeError, ValueError):
return ''
return None if include_metadata else ''
@classmethod
async def fetch(
@@ -624,6 +673,7 @@ class AsyncFetcher:
verify: bool | None = None,
follow_redirects: bool | None = None,
request_timeout: int | None = None,
include_metadata: bool = False,
) -> Any:
"""Generic HTTP request helper.
- If a session is not provided, one will be created and closed automatically.
@@ -665,13 +715,14 @@ class AsyncFetcher:
json=json,
delay=5,
request_timeout=request_timeout,
include_metadata=include_metadata,
**request_kwargs,
)
finally:
if owns_session:
await session.close()
except (aiohttp.ClientError, TimeoutError, OSError, ssl.SSLError, UnicodeDecodeError, ValueError):
return ''
return None if include_metadata else ''
@staticmethod
async def takeover_fetch(session, url: str, proxy: str | None = None) -> tuple[Any, Any] | str:
@@ -714,6 +765,7 @@ class AsyncFetcher:
json: bool = False,
takeover: bool = False,
proxy: bool = False,
include_metadata: bool = False,
) -> list:
# By default, timeout is 5 minutes; 60 seconds should suffice
headers = cls._default_headers(headers)
@@ -747,13 +799,18 @@ class AsyncFetcher:
url,
json=json,
proxy=proxy_url,
include_metadata=include_metadata,
)
for url, (proxy_url, proxy_type) in zip(urls, proxy_data, strict=False)
]
)
)
else:
return list(await asyncio.gather(*[AsyncFetcher.fetch(session, url, json=json) for url in urls]))
return list(
await asyncio.gather(
*[AsyncFetcher.fetch(session, url, json=json, include_metadata=include_metadata) for url in urls]
)
)
else:
# Indicates the request has certain params
async with aiohttp.ClientSession(headers=headers, timeout=timeout) as session:
@@ -768,13 +825,18 @@ class AsyncFetcher:
params,
json,
proxy=proxy_url,
include_metadata=include_metadata,
)
for url, (proxy_url, proxy_type) in zip(urls, proxy_data, strict=False)
]
)
)
else:
return list(await asyncio.gather(*[AsyncFetcher.fetch(session, url, params, json) for url in urls]))
return list(
await asyncio.gather(
*[AsyncFetcher.fetch(session, url, params, json, include_metadata=include_metadata) for url in urls]
)
)
def show_default_error_message(engine_name: str, word: str, error) -> None: