mirror of
https://github.com/laramies/theHarvester.git
synced 2026-08-22 13:52:22 +02:00
1388 lines
54 KiB
Python
1388 lines
54 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import inspect
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
import sys
|
|
import time
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from uuid import UUID
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
|
|
|
|
class _FakeScreenshotBatch:
|
|
async def reachable_targets(self, targets: list[str]) -> list[tuple[str, str]]:
|
|
reachable: list[tuple[str, str]] = []
|
|
for subject in targets:
|
|
final_url, status = await self.visit(subject)
|
|
if status:
|
|
reachable.append((subject, final_url))
|
|
return reachable
|
|
|
|
async def capture_targets(self, targets, record) -> None:
|
|
async def capture(subject: str, final_url: str) -> tuple[str, str, Path]:
|
|
output_path = self.screenshot_path(subject)
|
|
return subject, await self.take_screenshot(final_url, output_path=output_path), output_path
|
|
|
|
tasks = [asyncio.create_task(capture(*target)) for target in targets]
|
|
try:
|
|
for task in asyncio.as_completed(tasks):
|
|
await record(*await task)
|
|
except asyncio.CancelledError:
|
|
for task in tasks:
|
|
task.cancel()
|
|
outcomes = await asyncio.gather(*tasks, return_exceptions=True)
|
|
for outcome in outcomes:
|
|
if isinstance(outcome, tuple):
|
|
await record(*outcome)
|
|
raise
|
|
finally:
|
|
for task in tasks:
|
|
task.cancel()
|
|
await asyncio.gather(*tasks, return_exceptions=True)
|
|
|
|
|
|
def test_run_paths_use_one_expanded_database_and_artifact_root(tmp_path, monkeypatch) -> None:
|
|
from theHarvester.lib.api.run_store import RunStore
|
|
|
|
monkeypatch.setenv('HOME', str(tmp_path))
|
|
monkeypatch.setenv('THEHARVESTER_RUN_DB', '~/state/runs.sqlite')
|
|
monkeypatch.delenv('THEHARVESTER_RUN_ARTIFACTS', raising=False)
|
|
|
|
store = RunStore()
|
|
|
|
assert store.database == tmp_path / 'state' / 'runs.sqlite'
|
|
assert store.artifact_directory('run-id') == tmp_path / 'state' / 'run-artifacts' / 'run-id'
|
|
|
|
|
|
def test_run_store_does_not_change_caller_owned_directory_permissions(tmp_path) -> None:
|
|
from theHarvester.lib.api.run_store import RunStore
|
|
|
|
tmp_path.chmod(0o755)
|
|
asyncio.run(RunStore(tmp_path / 'runs.sqlite').initialize())
|
|
|
|
assert os.stat(tmp_path).st_mode & 0o777 == 0o755
|
|
|
|
|
|
def test_run_history_is_bounded_without_loading_completed_evidence(tmp_path, monkeypatch) -> None:
|
|
from theHarvester.lib.api.run_models import RunRequest
|
|
from theHarvester.lib.api.run_store import RunStore
|
|
|
|
async def fail_load(*_args, **_kwargs):
|
|
raise AssertionError('run summaries must not hydrate terminal evidence')
|
|
|
|
async def scenario():
|
|
store = RunStore(tmp_path / 'runs.sqlite')
|
|
for index in range(4):
|
|
await store.create(RunRequest(target=f'{index}.example.test', sources=['crtsh']))
|
|
monkeypatch.setattr(store.results, 'load_run', fail_load)
|
|
return await store.list_runs(limit=2, offset=1)
|
|
|
|
history = asyncio.run(scenario())
|
|
|
|
assert len(history) == 2
|
|
assert [run['target'] for run in history] == ['2.example.test', '1.example.test']
|
|
assert all(run['result_count'] == 0 for run in history)
|
|
|
|
|
|
def test_run_history_breaks_timestamp_ties_by_run_id(tmp_path) -> None:
|
|
from theHarvester.lib.database import RunLifecycleStore
|
|
|
|
async def scenario() -> list[str]:
|
|
store = RunLifecycleStore(tmp_path / 'runs.sqlite')
|
|
await store.initialize()
|
|
for run_id in ('run-a', 'run-c', 'run-b'):
|
|
await store.create(
|
|
run_id=run_id,
|
|
target='example.test',
|
|
status='completed',
|
|
origin='imported',
|
|
created_at='2026-08-09T12:00:00+00:00',
|
|
request_json='{}',
|
|
)
|
|
first = await store.list_records(limit=2, offset=0)
|
|
second = await store.list_records(limit=2, offset=2)
|
|
return [str(run['run_id']) for run in first + second]
|
|
|
|
assert asyncio.run(scenario()) == ['run-c', 'run-b', 'run-a']
|
|
|
|
|
|
def test_api_lifespan_disposes_shared_sqlite_engines(tmp_path, monkeypatch) -> None:
|
|
from theHarvester.lib.api import api
|
|
|
|
disposed = False
|
|
|
|
async def no_op() -> None:
|
|
return None
|
|
|
|
async def dispose() -> None:
|
|
nonlocal disposed
|
|
disposed = True
|
|
|
|
monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
|
|
monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled')
|
|
monkeypatch.setattr(api, 'start_worker', no_op)
|
|
monkeypatch.setattr(api, 'stop_worker', no_op)
|
|
monkeypatch.setattr(api, 'dispose_sqlite_databases', dispose)
|
|
|
|
with TestClient(api.app):
|
|
pass
|
|
|
|
assert disposed is True
|
|
|
|
|
|
def test_api_lifespan_disposes_schedule_sqlite_engines(tmp_path, monkeypatch) -> None:
|
|
from theHarvester.lib.api import api
|
|
|
|
disposed = False
|
|
|
|
async def no_op() -> None:
|
|
return None
|
|
|
|
async def dispose() -> None:
|
|
nonlocal disposed
|
|
disposed = True
|
|
|
|
monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
|
|
monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled')
|
|
monkeypatch.setattr(api, 'start_worker', no_op)
|
|
monkeypatch.setattr(api, 'stop_worker', no_op)
|
|
monkeypatch.setattr(api, 'dispose_schedule_databases', dispose)
|
|
|
|
with TestClient(api.app):
|
|
pass
|
|
|
|
assert disposed is True
|
|
|
|
|
|
def test_static_assets_are_resolved_from_the_installed_module(tmp_path, monkeypatch) -> None:
|
|
from theHarvester.lib.api import api
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
|
|
assert api.STATIC_DIRECTORY == Path(api.__file__).resolve().parent / 'static'
|
|
|
|
|
|
def test_explicit_run_database_keeps_screenshot_artifacts_attached(tmp_path, monkeypatch) -> None:
|
|
from theHarvester.lib.api.run_store import RunStore
|
|
|
|
monkeypatch.delenv('THEHARVESTER_RUN_DB', raising=False)
|
|
monkeypatch.delenv('THEHARVESTER_RUN_ARTIFACTS', raising=False)
|
|
|
|
async def scenario():
|
|
store = RunStore(tmp_path / 'state' / 'runs.sqlite')
|
|
imported = await store.import_evidence(
|
|
{
|
|
'run_id': '4a6e5a15-fae5-462c-a34b-122ced6bb86d',
|
|
'target': 'example.test',
|
|
'status': 'complete',
|
|
'started_at': '2026-08-09T12:00:00+00:00',
|
|
'completed_at': '2026-08-09T12:01:00+00:00',
|
|
'results': [{'type': 'hostname', 'value': 'owned.example.test', 'actions': []}],
|
|
'source_executions': [],
|
|
'action_executions': [
|
|
{
|
|
'action': 'screenshot',
|
|
'status': 'completed',
|
|
'duration_ms': 1,
|
|
'result_count': 0,
|
|
'error_type': None,
|
|
'stop_reason': None,
|
|
}
|
|
],
|
|
'artifacts': [
|
|
{
|
|
'action': 'screenshot',
|
|
'kind': 'screenshot',
|
|
'subject': {'kind': 'hostname', 'value': 'owned.example.test'},
|
|
'file': {
|
|
'path': 'screenshots/owned.example.test.png',
|
|
'media_type': 'image/png',
|
|
'size_bytes': 3,
|
|
'sha256': '0' * 64,
|
|
},
|
|
'created_at': '2026-08-09T12:01:00+00:00',
|
|
}
|
|
],
|
|
},
|
|
'evidence.jsonl',
|
|
)
|
|
screenshot_dir = store.artifact_directory(imported['run_id']) / 'screenshots'
|
|
screenshot_dir.mkdir(parents=True)
|
|
(screenshot_dir / 'owned.example.test.png').write_bytes(b'png')
|
|
return await store.get(imported['run_id'])
|
|
|
|
run = asyncio.run(scenario())
|
|
|
|
assert run is not None
|
|
assert run['evidence']['run_id'] == run['run_id']
|
|
assert run['request']['source_run_id'] == '4a6e5a15-fae5-462c-a34b-122ced6bb86d'
|
|
assert [screenshot['name'] for screenshot in run['screenshots']] == ['owned.example.test.png']
|
|
|
|
|
|
def test_api_lifecycle_and_terminal_evidence_share_the_sqlalchemy_database(tmp_path) -> None:
|
|
from theHarvester.lib.api import run_store as run_store_module
|
|
from theHarvester.lib.api.run_models import RunRequest
|
|
from theHarvester.lib.api.run_store import RunStore
|
|
|
|
database = tmp_path / 'stash.sqlite'
|
|
|
|
async def scenario() -> str:
|
|
store = RunStore(database)
|
|
queued = await store.create(RunRequest(target='example.test', sources=['crtsh']))
|
|
await store.claim_next()
|
|
await store.finish(
|
|
queued['run_id'],
|
|
{
|
|
'run_id': '4a6e5a15-fae5-462c-a34b-122ced6bb86d',
|
|
'target': 'example.test',
|
|
'status': 'complete',
|
|
'started_at': '2026-08-09T12:00:00+00:00',
|
|
'completed_at': '2026-08-09T12:01:00+00:00',
|
|
'results': [{'type': 'hostname', 'value': 'api.example.test', 'sources': ['crtsh']}],
|
|
'source_executions': [
|
|
{
|
|
'source': 'crtsh',
|
|
'status': 'completed',
|
|
'duration_ms': 1,
|
|
'result_count': 1,
|
|
'error_type': None,
|
|
'stop_reason': None,
|
|
}
|
|
],
|
|
},
|
|
'',
|
|
)
|
|
return queued['run_id']
|
|
|
|
lifecycle_run_id = asyncio.run(scenario())
|
|
|
|
with sqlite3.connect(database) as db:
|
|
evidence_run_id = db.execute(
|
|
'SELECT evidence_run_id FROM run_records WHERE run_id = ?',
|
|
(lifecycle_run_id,),
|
|
).fetchone()[0]
|
|
stored_target = db.execute('SELECT target FROM runs WHERE run_id = ?', (evidence_run_id,)).fetchone()[0]
|
|
schema_version = db.execute('PRAGMA user_version').fetchone()[0]
|
|
assert evidence_run_id == lifecycle_run_id
|
|
assert stored_target == 'example.test'
|
|
assert schema_version == 8
|
|
assert 'import aiosqlite' not in inspect.getsource(run_store_module)
|
|
|
|
|
|
def test_run_store_distinguishes_no_evidence_from_a_broken_evidence_link(tmp_path) -> None:
|
|
from uuid import uuid4
|
|
|
|
from theHarvester.lib.api.run_models import RunRequest
|
|
from theHarvester.lib.api.run_store import RunStore
|
|
from theHarvester.lib.database import ResultStoreError
|
|
|
|
database = tmp_path / 'runs.sqlite'
|
|
store = RunStore(database)
|
|
queued = asyncio.run(store.create(RunRequest(target='example.test', sources=['crtsh'])))
|
|
assert asyncio.run(store.load_completed_result(queued['run_id'])) is None
|
|
|
|
with sqlite3.connect(database) as db:
|
|
db.execute(
|
|
'UPDATE run_records SET evidence_run_id = ? WHERE run_id = ?',
|
|
(str(uuid4()), queued['run_id']),
|
|
)
|
|
|
|
with pytest.raises(ResultStoreError, match='Attached run evidence does not exist'):
|
|
asyncio.run(store.load_completed_result(queued['run_id']))
|
|
|
|
|
|
def test_child_execution_passes_the_configured_database_to_core(tmp_path, monkeypatch) -> None:
|
|
from theHarvester import __main__ as main_module
|
|
from theHarvester.lib.api.run_models import RunRequest
|
|
from theHarvester.lib.api.run_store import RunStore
|
|
from theHarvester.lib.api.run_worker import _child_execute
|
|
from theHarvester.lib.completed_result import CompletedResult
|
|
|
|
database = tmp_path / 'state' / 'runs.sqlite'
|
|
seen_database = None
|
|
seen_run_id = None
|
|
|
|
async def fake_start(_args, **kwargs):
|
|
nonlocal seen_database, seen_run_id
|
|
seen_database = kwargs.get('result_database')
|
|
seen_run_id = kwargs.get('completed_run_id')
|
|
now = datetime.now(UTC)
|
|
result = CompletedResult.finish(
|
|
run_id=seen_run_id,
|
|
target='example.test',
|
|
started_at=now,
|
|
completed_at=now,
|
|
groups={},
|
|
)
|
|
return (result,)
|
|
|
|
async def scenario() -> None:
|
|
store = RunStore(database)
|
|
queued = await store.create(RunRequest(target='example.test', sources=['crtsh']))
|
|
await store.claim_next()
|
|
monkeypatch.setattr(main_module, 'start', fake_start)
|
|
await _child_execute(queued['run_id'], database)
|
|
|
|
asyncio.run(scenario())
|
|
|
|
assert seen_database == database
|
|
with sqlite3.connect(database) as db:
|
|
lifecycle_run_id = db.execute('SELECT run_id FROM run_records').fetchone()[0]
|
|
assert str(seen_run_id) == lifecycle_run_id
|
|
|
|
|
|
def test_child_screenshot_run_persists_downloadable_artifact_metadata(tmp_path, monkeypatch) -> None:
|
|
from theHarvester import __main__ as main_module
|
|
from theHarvester.lib.api.run_artifacts import read_child_evidence
|
|
from theHarvester.lib.api.run_models import RunRequest
|
|
from theHarvester.lib.api.run_store import RunStore
|
|
from theHarvester.lib.api.run_worker import _child_execute
|
|
|
|
class FakeScreenShotter(_FakeScreenshotBatch):
|
|
slash = '/'
|
|
|
|
def __init__(self, output: str) -> None:
|
|
self.output = output
|
|
|
|
def verify_path(self) -> bool:
|
|
return True
|
|
|
|
async def visit(self, host: str) -> tuple[str, str]:
|
|
return f'https://{host}', 'reachable'
|
|
|
|
def screenshot_path(self, url: str) -> Path:
|
|
return Path(self.output) / f'{url.removeprefix("https://")}.png'
|
|
|
|
async def take_screenshot(self, url: str, *, output_path: Path | None = None) -> str:
|
|
captured_url = url if url.startswith('https://') else f'https://{url}'
|
|
(output_path or self.screenshot_path(captured_url)).write_bytes(b'png')
|
|
return captured_url
|
|
|
|
database = tmp_path / 'state' / 'runs.sqlite'
|
|
monkeypatch.setenv('THEHARVESTER_RUN_ARTIFACTS', str(tmp_path / 'artifacts'))
|
|
monkeypatch.setattr(main_module, 'ScreenShotter', FakeScreenShotter)
|
|
|
|
async def scenario():
|
|
store = RunStore(database)
|
|
queued = await store.create(RunRequest(target='api.example.test', sources=[], screenshot=True))
|
|
await store.claim_next()
|
|
await _child_execute(queued['run_id'], database)
|
|
evidence, error = read_child_evidence(store.artifact_directory(queued['run_id']))
|
|
assert error is None
|
|
assert evidence is not None
|
|
await store.finish(queued['run_id'], evidence, '')
|
|
return await store.get(queued['run_id']), store.artifact_directory(queued['run_id'])
|
|
|
|
run, artifact_dir = asyncio.run(scenario())
|
|
|
|
assert run is not None
|
|
assert run['action_executions'][0]['action'] == 'screenshot'
|
|
assert run['action_executions'][0]['status'] == 'completed'
|
|
assert run['results'] == [{'type': 'hostname', 'value': 'api.example.test', 'sources': [], 'actions': []}]
|
|
assert [screenshot['name'] for screenshot in run['screenshots']] == ['api.example.test.png']
|
|
assert (artifact_dir / 'screenshots' / 'api.example.test.png').read_bytes() == b'png'
|
|
|
|
|
|
def test_child_screenshot_cancellation_reuses_the_checkpointed_evidence(tmp_path, monkeypatch) -> None:
|
|
from theHarvester import __main__ as main_module
|
|
from theHarvester.lib import source_runner
|
|
from theHarvester.lib.api.run_artifacts import read_child_evidence
|
|
from theHarvester.lib.api.run_models import RunRequest
|
|
from theHarvester.lib.api.run_store import RunStore
|
|
from theHarvester.lib.api.run_worker import _child_execute
|
|
|
|
first_captured = asyncio.Event()
|
|
|
|
class FakeScreenShotter(_FakeScreenshotBatch):
|
|
slash = '/'
|
|
|
|
def __init__(self, output: str) -> None:
|
|
self.output = output
|
|
|
|
def verify_path(self) -> bool:
|
|
return True
|
|
|
|
async def visit(self, host: str) -> tuple[str, str]:
|
|
return f'https://{host}', 'reachable'
|
|
|
|
async def take_screenshot(self, url: str, *, output_path: Path | None = None) -> str:
|
|
assert output_path is not None
|
|
if 'first.' in url:
|
|
output_path.write_bytes(b'png') # noqa: ASYNC240 - tiny in-memory screenshot fixture
|
|
first_captured.set()
|
|
return url
|
|
await first_captured.wait()
|
|
raise asyncio.CancelledError
|
|
|
|
def screenshot_path(self, url: str) -> Path:
|
|
return Path(self.output) / f'{url.removeprefix("https://")}.png'
|
|
|
|
class TwoHostSource:
|
|
def __init__(self, _word: str) -> None:
|
|
pass
|
|
|
|
async def process(self, _proxy: bool) -> None:
|
|
return None
|
|
|
|
async def get_hostnames(self) -> set[str]:
|
|
return {'first.example.test', 'second.example.test'}
|
|
|
|
database = tmp_path / 'runs.sqlite'
|
|
monkeypatch.setenv('THEHARVESTER_RUN_ARTIFACTS', str(tmp_path / 'artifacts'))
|
|
monkeypatch.setattr(source_runner.crtsh, 'SearchCrtsh', TwoHostSource)
|
|
monkeypatch.setattr(main_module, 'ScreenShotter', FakeScreenShotter)
|
|
|
|
async def scenario():
|
|
store = RunStore(database)
|
|
queued = await store.create(RunRequest(target='example.test', sources=['crtsh'], screenshot=True))
|
|
await store.claim_next()
|
|
await _child_execute(queued['run_id'], database)
|
|
evidence, error = read_child_evidence(store.artifact_directory(queued['run_id']))
|
|
assert error is None
|
|
assert evidence is not None
|
|
await store.fail(queued['run_id'], 'cancelled', '', cancelled=True, evidence=evidence)
|
|
return await store.get(queued['run_id'])
|
|
|
|
run = asyncio.run(scenario())
|
|
|
|
assert run is not None
|
|
assert run['status'] == 'cancelled'
|
|
assert run['action_executions'][0]['status'] == 'partial'
|
|
assert [screenshot['target'] for screenshot in run['screenshots']] == ['first.example.test']
|
|
|
|
|
|
def test_sqlite_import_preserves_run_ids_and_is_idempotent(tmp_path, monkeypatch) -> None:
|
|
from theHarvester.lib.api import run_store
|
|
from theHarvester.lib.api.run_store import RunStore
|
|
from theHarvester.lib.completed_result import CompletedResult
|
|
from theHarvester.lib.database import ResultStore, dispose_sqlite_databases
|
|
|
|
source_database = tmp_path / 'source.sqlite'
|
|
destination_database = tmp_path / 'destination.sqlite'
|
|
now = datetime.now(UTC)
|
|
first = CompletedResult.finish(
|
|
target='first.example.test',
|
|
started_at=now,
|
|
completed_at=now,
|
|
groups={'hostname': ['api.first.example.test']},
|
|
)
|
|
second = CompletedResult.finish(
|
|
target='second.example.test',
|
|
started_at=now,
|
|
completed_at=now,
|
|
groups={'email': ['security@second.example.test']},
|
|
)
|
|
list_calls: list[tuple[int | None, int]] = []
|
|
original_list_runs = ResultStore.list_runs
|
|
|
|
async def track_source_batches(self, *, limit=50, offset=0):
|
|
if Path(self.database) == source_database.resolve():
|
|
list_calls.append((limit, offset))
|
|
return await original_list_runs(self, limit=limit, offset=offset)
|
|
|
|
monkeypatch.setattr(run_store, 'DATABASE_IMPORT_BATCH_SIZE', 1)
|
|
monkeypatch.setattr(ResultStore, 'list_runs', track_source_batches)
|
|
|
|
async def scenario():
|
|
source = ResultStore(source_database)
|
|
await source.initialize()
|
|
await source.save_run(first)
|
|
await source.save_run(second)
|
|
await dispose_sqlite_databases()
|
|
destination = RunStore(destination_database)
|
|
imported = await destination.import_database(source_database, 'source.sqlite')
|
|
first_import_calls = list(list_calls)
|
|
list_calls.clear()
|
|
repeated = await destination.import_database(source_database, 'source.sqlite')
|
|
history = await destination.list_runs()
|
|
return imported, repeated, history, first_import_calls
|
|
|
|
imported, repeated, history, first_import_calls = asyncio.run(scenario())
|
|
|
|
expected_ids = sorted((str(first.run_id), str(second.run_id)))
|
|
assert imported == {'filename': 'source.sqlite', 'imported_run_ids': expected_ids, 'skipped_run_ids': []}
|
|
assert repeated == {'filename': 'source.sqlite', 'imported_run_ids': [], 'skipped_run_ids': expected_ids}
|
|
assert sorted(run['run_id'] for run in history) == expected_ids
|
|
assert first_import_calls == [(1, 0), (1, 1), (1, 2), (1, 0), (1, 1), (1, 2)]
|
|
|
|
|
|
def test_orphan_recovery_reattaches_partial_checkpoint_and_leaves_queued_work(tmp_path, monkeypatch) -> None:
|
|
from theHarvester.lib.api.run_artifacts import ensure_private_directory, write_child_evidence
|
|
from theHarvester.lib.api.run_models import RunRequest
|
|
from theHarvester.lib.api.run_store import RunStore
|
|
from theHarvester.lib.completed_result import CompletedResult
|
|
|
|
monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
|
|
monkeypatch.setenv('THEHARVESTER_RUN_ARTIFACTS', str(tmp_path / 'artifacts'))
|
|
|
|
async def scenario():
|
|
store = RunStore()
|
|
for target in ('first.example', 'second.example', 'third.example'):
|
|
await store.create(RunRequest(target=target, sources=['crtsh']))
|
|
cancelling = await store.claim_next()
|
|
assert cancelling is not None
|
|
await store.cancel(cancelling['run_id'])
|
|
artifact_dir = store.artifact_directory(cancelling['run_id'])
|
|
ensure_private_directory(artifact_dir)
|
|
now = datetime.now(UTC)
|
|
checkpoint = CompletedResult.finish(
|
|
target='first.example',
|
|
started_at=now,
|
|
completed_at=now,
|
|
groups={'email': ['saved@first.example']},
|
|
)
|
|
write_child_evidence(artifact_dir, checkpoint, partial=True)
|
|
running = await store.claim_next()
|
|
assert running is not None
|
|
persisted = CompletedResult.finish(
|
|
run_id=UUID(running['run_id']),
|
|
target=str(running['target']),
|
|
started_at=now,
|
|
completed_at=now,
|
|
groups={'hostname': [f'api.{running["target"]}']},
|
|
)
|
|
await store.results.save_run(persisted)
|
|
await store.recover_orphans()
|
|
return await store.get(cancelling['run_id']), await store.get(running['run_id']), await store.list_runs()
|
|
|
|
cancelling, running, history = asyncio.run(scenario())
|
|
|
|
assert cancelling is not None
|
|
assert cancelling['status'] == 'failed'
|
|
assert cancelling['evidence_status'] == 'partial'
|
|
assert cancelling['results'] == [{'type': 'email', 'value': 'saved@first.example', 'sources': [], 'actions': []}]
|
|
assert running is not None
|
|
assert running['status'] == 'failed'
|
|
assert running['results'] == [{'type': 'hostname', 'value': f'api.{running["target"]}', 'sources': [], 'actions': []}]
|
|
assert {run['target']: run['status'] for run in history}['third.example'] == 'queued'
|
|
|
|
|
|
def test_orphan_recovery_prefers_immutable_persisted_evidence_over_a_newer_checkpoint(tmp_path, monkeypatch) -> None:
|
|
from theHarvester.lib.api.run_artifacts import ensure_private_directory, write_child_evidence
|
|
from theHarvester.lib.api.run_models import RunRequest
|
|
from theHarvester.lib.api.run_store import RunStore
|
|
from theHarvester.lib.completed_result import CompletedResult
|
|
|
|
monkeypatch.setenv('THEHARVESTER_RUN_ARTIFACTS', str(tmp_path / 'artifacts'))
|
|
|
|
async def scenario():
|
|
store = RunStore(tmp_path / 'runs.sqlite')
|
|
queued = await store.create(RunRequest(target='expected.example.test', sources=['crtsh']))
|
|
claimed = await store.claim_next()
|
|
assert claimed is not None
|
|
await store.cancel(queued['run_id'])
|
|
now = datetime.now(UTC)
|
|
await store.results.save_run(
|
|
CompletedResult.finish(
|
|
run_id=UUID(queued['run_id']),
|
|
target='expected.example.test',
|
|
started_at=now,
|
|
completed_at=now,
|
|
groups={'hostname': ['persisted.expected.example.test']},
|
|
)
|
|
)
|
|
artifact_dir = store.artifact_directory(queued['run_id'])
|
|
ensure_private_directory(artifact_dir)
|
|
write_child_evidence(
|
|
artifact_dir,
|
|
CompletedResult.finish(
|
|
target='expected.example.test',
|
|
started_at=now,
|
|
completed_at=now,
|
|
groups={'hostname': ['checkpoint.expected.example.test']},
|
|
),
|
|
partial=True,
|
|
)
|
|
|
|
await store.recover_orphans()
|
|
return await store.get(queued['run_id'])
|
|
|
|
recovered = asyncio.run(scenario())
|
|
|
|
assert recovered is not None
|
|
assert recovered['status'] == 'failed'
|
|
assert recovered['evidence_status'] == 'complete'
|
|
assert recovered['results'] == [
|
|
{'type': 'hostname', 'value': 'persisted.expected.example.test', 'sources': [], 'actions': []}
|
|
]
|
|
|
|
|
|
def test_orphan_recovery_does_not_attach_same_id_evidence_for_another_target(tmp_path) -> None:
|
|
from theHarvester.lib.api.run_models import RunRequest
|
|
from theHarvester.lib.api.run_store import RunStore
|
|
from theHarvester.lib.completed_result import CompletedResult
|
|
|
|
async def scenario():
|
|
store = RunStore(tmp_path / 'runs.sqlite')
|
|
queued = await store.create(RunRequest(target='expected.example', sources=['crtsh']))
|
|
await store.claim_next()
|
|
now = datetime.now(UTC)
|
|
await store.results.save_run(
|
|
CompletedResult.finish(
|
|
run_id=UUID(queued['run_id']),
|
|
target='different.example',
|
|
started_at=now,
|
|
completed_at=now,
|
|
groups={'hostname': ['api.different.example']},
|
|
)
|
|
)
|
|
await store.recover_orphans()
|
|
return await store.get(queued['run_id'])
|
|
|
|
run = asyncio.run(scenario())
|
|
|
|
assert run is not None
|
|
assert run['status'] == 'failed'
|
|
assert run['results'] == []
|
|
|
|
|
|
def test_orphan_recovery_rejects_checkpoint_for_another_target(tmp_path, monkeypatch) -> None:
|
|
from theHarvester.lib.api.run_artifacts import ensure_private_directory, write_child_evidence
|
|
from theHarvester.lib.api.run_models import RunRequest
|
|
from theHarvester.lib.api.run_store import RunStore
|
|
from theHarvester.lib.completed_result import CompletedResult
|
|
|
|
monkeypatch.setenv('THEHARVESTER_RUN_ARTIFACTS', str(tmp_path / 'artifacts'))
|
|
|
|
async def scenario():
|
|
store = RunStore(tmp_path / 'runs.sqlite')
|
|
queued = await store.create(RunRequest(target='expected.example', sources=['crtsh']))
|
|
await store.claim_next()
|
|
artifact_dir = store.artifact_directory(queued['run_id'])
|
|
ensure_private_directory(artifact_dir)
|
|
now = datetime.now(UTC)
|
|
write_child_evidence(
|
|
artifact_dir,
|
|
CompletedResult.finish(
|
|
target='different.example',
|
|
started_at=now,
|
|
completed_at=now,
|
|
groups={'hostname': ['api.different.example']},
|
|
),
|
|
partial=True,
|
|
)
|
|
await store.recover_orphans()
|
|
return await store.get(queued['run_id'])
|
|
|
|
run = asyncio.run(scenario())
|
|
|
|
assert run is not None
|
|
assert run['status'] == 'failed'
|
|
assert run['results'] == []
|
|
assert 'does not match run target' in run['error']
|
|
|
|
|
|
@pytest.mark.parametrize('operation', ['finish', 'fail'])
|
|
def test_terminal_run_rejects_evidence_for_another_target(tmp_path, operation) -> None:
|
|
from fastapi import HTTPException
|
|
|
|
from theHarvester.lib.api.run_models import RunRequest
|
|
from theHarvester.lib.api.run_store import RunStore
|
|
from theHarvester.lib.completed_result import CompletedResult
|
|
|
|
async def scenario():
|
|
store = RunStore(tmp_path / 'runs.sqlite')
|
|
queued = await store.create(RunRequest(target='expected.example', sources=['crtsh']))
|
|
await store.claim_next()
|
|
now = datetime.now(UTC)
|
|
evidence = CompletedResult.finish(
|
|
target='different.example',
|
|
started_at=now,
|
|
completed_at=now,
|
|
groups={'hostname': ['api.different.example']},
|
|
).evidence_dict()
|
|
terminal = (
|
|
store.finish(queued['run_id'], evidence, '')
|
|
if operation == 'finish'
|
|
else store.fail(queued['run_id'], 'failed', '', evidence=evidence)
|
|
)
|
|
with pytest.raises(HTTPException, match='Evidence target does not match run target'):
|
|
await terminal
|
|
with pytest.raises(LookupError):
|
|
await store.results.load_run(UUID(queued['run_id']))
|
|
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_existing_evidence_does_not_bypass_checkpoint_target_validation(tmp_path) -> None:
|
|
from fastapi import HTTPException
|
|
|
|
from theHarvester.lib.api.run_models import RunRequest
|
|
from theHarvester.lib.api.run_store import RunStore
|
|
from theHarvester.lib.completed_result import CompletedResult
|
|
|
|
async def scenario():
|
|
store = RunStore(tmp_path / 'runs.sqlite')
|
|
queued = await store.create(RunRequest(target='expected.example.test', sources=['crtsh']))
|
|
await store.claim_next()
|
|
now = datetime.now(UTC)
|
|
await store.results.save_run(
|
|
CompletedResult.finish(
|
|
run_id=UUID(queued['run_id']),
|
|
target='expected.example.test',
|
|
started_at=now,
|
|
completed_at=now,
|
|
groups={'hostname': ['persisted.expected.example.test']},
|
|
)
|
|
)
|
|
evidence = CompletedResult.finish(
|
|
target='different.example.test',
|
|
started_at=now,
|
|
completed_at=now,
|
|
groups={},
|
|
).evidence_dict()
|
|
with pytest.raises(HTTPException, match='Evidence target does not match run target'):
|
|
await store.fail(queued['run_id'], 'failed', '', evidence=evidence)
|
|
|
|
asyncio.run(scenario())
|
|
|
|
|
|
@pytest.mark.parametrize(('operation', 'expected_status'), [('finish', 'cancelled'), ('fail', 'failed')])
|
|
def test_terminal_run_without_checkpoint_attaches_existing_evidence(tmp_path, operation, expected_status) -> None:
|
|
from theHarvester.lib.api.run_models import RunRequest
|
|
from theHarvester.lib.api.run_store import RunStore
|
|
from theHarvester.lib.completed_result import CompletedResult
|
|
|
|
async def scenario():
|
|
store = RunStore(tmp_path / 'runs.sqlite')
|
|
queued = await store.create(RunRequest(target='expected.example.test', sources=['crtsh']))
|
|
await store.claim_next()
|
|
await store.cancel(queued['run_id'])
|
|
now = datetime.now(UTC)
|
|
await store.results.save_run(
|
|
CompletedResult.finish(
|
|
run_id=UUID(queued['run_id']),
|
|
target='expected.example.test',
|
|
started_at=now,
|
|
completed_at=now,
|
|
groups={'hostname': ['persisted.expected.example.test']},
|
|
)
|
|
)
|
|
if operation == 'finish':
|
|
await store.finish(queued['run_id'], None, '')
|
|
else:
|
|
await store.fail(queued['run_id'], 'failed', '')
|
|
return await store.get(queued['run_id'])
|
|
|
|
run = asyncio.run(scenario())
|
|
|
|
assert run is not None
|
|
assert run['status'] == expected_status
|
|
assert run['evidence_status'] == 'complete'
|
|
assert run['results'] == [{'type': 'hostname', 'value': 'persisted.expected.example.test', 'sources': [], 'actions': []}]
|
|
|
|
|
|
def test_worker_lease_serializes_execution_owners(tmp_path) -> None:
|
|
from theHarvester.lib.api.run_store import RunStore
|
|
|
|
async def scenario() -> tuple[bool, bool, bool]:
|
|
store = RunStore(tmp_path / 'runs.sqlite')
|
|
first = await store.acquire_worker_lease('worker-a')
|
|
second = await store.acquire_worker_lease('worker-b')
|
|
await store.release_worker_lease('worker-a')
|
|
replacement = await store.acquire_worker_lease('worker-b')
|
|
return first, second, replacement
|
|
|
|
assert asyncio.run(scenario()) == (True, False, True)
|
|
|
|
|
|
def test_submission_fails_closed_when_worker_supervisor_stops(tmp_path, monkeypatch) -> None:
|
|
from theHarvester.lib.api import api, run_worker
|
|
|
|
async def no_op() -> None:
|
|
return None
|
|
|
|
monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key')
|
|
monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
|
|
monkeypatch.setattr(api, 'start_worker', no_op)
|
|
monkeypatch.setattr(api, 'stop_worker', no_op)
|
|
monkeypatch.setattr(run_worker, 'worker_enabled', lambda: True)
|
|
monkeypatch.setattr(run_worker, '_worker_task', type('StoppedTask', (), {'done': lambda self: True})())
|
|
|
|
with TestClient(api.app) as client:
|
|
response = client.post(
|
|
'/api/v1/runs',
|
|
headers={'X-API-Key': 'test-key'},
|
|
json={'target': 'example.com', 'sources': ['crtsh']},
|
|
)
|
|
history = client.get('/api/v1/runs', headers={'X-API-Key': 'test-key'})
|
|
|
|
assert response.status_code == 503
|
|
assert response.json()['detail'] == 'theHarvester execution worker is unavailable'
|
|
assert history.json() == []
|
|
|
|
|
|
def test_authenticated_operator_can_queue_direct_activity_for_selected_target(tmp_path, monkeypatch) -> None:
|
|
from theHarvester.lib.api import api, run_worker
|
|
|
|
async def no_op() -> None:
|
|
return None
|
|
|
|
monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key')
|
|
monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
|
|
monkeypatch.setattr(api, 'start_worker', no_op)
|
|
monkeypatch.setattr(api, 'stop_worker', no_op)
|
|
monkeypatch.setattr(run_worker, 'worker_enabled', lambda: True)
|
|
monkeypatch.setattr(run_worker, '_worker_task', type('RunningTask', (), {'done': lambda self: False})())
|
|
|
|
with TestClient(api.app) as client:
|
|
response = client.post(
|
|
'/api/v1/runs',
|
|
headers={'X-API-Key': 'test-key'},
|
|
json={'target': '192.0.2.8', 'sources': ['criminalip']},
|
|
)
|
|
|
|
assert response.status_code == 201
|
|
assert response.json()['target'] == '192.0.2.8'
|
|
assert response.json()['activities'] == ['P2']
|
|
|
|
|
|
def test_authenticated_operator_can_queue_screenshot_only_run(tmp_path, monkeypatch) -> None:
|
|
from theHarvester.lib.api import api, run_worker
|
|
|
|
async def no_op() -> None:
|
|
return None
|
|
|
|
monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key')
|
|
monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
|
|
monkeypatch.setattr(api, 'start_worker', no_op)
|
|
monkeypatch.setattr(api, 'stop_worker', no_op)
|
|
monkeypatch.setattr(run_worker, 'worker_enabled', lambda: True)
|
|
monkeypatch.setattr(run_worker, '_worker_task', type('RunningTask', (), {'done': lambda self: False})())
|
|
|
|
with TestClient(api.app) as client:
|
|
response = client.post(
|
|
'/api/v1/runs',
|
|
headers={'X-API-Key': 'test-key'},
|
|
json={'target': 'api.example.com', 'sources': [], 'screenshot': True},
|
|
)
|
|
|
|
assert response.status_code == 201
|
|
assert response.json()['target'] == 'api.example.com'
|
|
assert response.json()['sources'] == []
|
|
assert response.json()['activities'] == ['P2']
|
|
|
|
|
|
def test_authenticated_operator_can_queue_routeviews_only_ip_run(tmp_path, monkeypatch) -> None:
|
|
from theHarvester.lib.api import api, run_worker
|
|
|
|
async def no_op() -> None:
|
|
return None
|
|
|
|
monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key')
|
|
monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
|
|
monkeypatch.setattr(api, 'start_worker', no_op)
|
|
monkeypatch.setattr(api, 'stop_worker', no_op)
|
|
monkeypatch.setattr(run_worker, 'worker_enabled', lambda: True)
|
|
monkeypatch.setattr(run_worker, '_worker_task', type('RunningTask', (), {'done': lambda self: False})())
|
|
|
|
with TestClient(api.app) as client:
|
|
response = client.post(
|
|
'/api/v1/runs',
|
|
headers={'X-API-Key': 'test-key'},
|
|
json={'target': '192.0.2.7', 'sources': [], 'routeviews': True},
|
|
)
|
|
|
|
assert response.status_code == 201
|
|
assert response.json()['activities'] == ['P0']
|
|
assert response.json()['request']['routeviews'] is True
|
|
|
|
|
|
def test_dns_brute_run_accepts_operator_resolver_list(tmp_path, monkeypatch) -> None:
|
|
from theHarvester.lib.api import api, run_worker
|
|
|
|
async def no_op() -> None:
|
|
return None
|
|
|
|
monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key')
|
|
monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
|
|
monkeypatch.setattr(api, 'start_worker', no_op)
|
|
monkeypatch.setattr(api, 'stop_worker', no_op)
|
|
monkeypatch.setattr(run_worker, 'worker_enabled', lambda: True)
|
|
monkeypatch.setattr(run_worker, '_worker_task', type('RunningTask', (), {'done': lambda self: False})())
|
|
|
|
with TestClient(api.app) as client:
|
|
response = client.post(
|
|
'/api/v1/runs',
|
|
headers={'X-API-Key': 'test-key'},
|
|
json={
|
|
'target': 'dev.api.example.com',
|
|
'sources': [],
|
|
'dns_brute': True,
|
|
'dns_resolvers': ['192.0.2.53'],
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 201
|
|
assert response.json()['activities'] == ['P1']
|
|
assert response.json()['request']['dns_resolvers'] == ['192.0.2.53']
|
|
|
|
|
|
def test_dns_brute_child_uses_operator_resolver_list(tmp_path, monkeypatch) -> None:
|
|
from theHarvester import __main__ as main_module
|
|
from theHarvester.lib.api import run_worker
|
|
from theHarvester.lib.api.run_models import RunRequest
|
|
from theHarvester.lib.api.run_store import RunStore
|
|
from theHarvester.lib.completed_result import CompletedResult
|
|
|
|
received_options = []
|
|
|
|
async def fake_start(options, **_kwargs):
|
|
received_options.append(options)
|
|
now = datetime.now(UTC)
|
|
return (CompletedResult.finish(target=options.domain, started_at=now, completed_at=now, groups={}),)
|
|
|
|
monkeypatch.setattr(main_module, 'start', fake_start)
|
|
|
|
async def scenario() -> None:
|
|
store = RunStore(tmp_path / 'runs.sqlite')
|
|
created = await store.create(
|
|
RunRequest(
|
|
target='dev.api.example.com',
|
|
sources=[],
|
|
dns_brute=True,
|
|
dns_resolvers=['192.0.2.53'],
|
|
)
|
|
)
|
|
assert await store.claim_next() is not None
|
|
await run_worker._child_execute(created['run_id'], store.database)
|
|
|
|
asyncio.run(scenario())
|
|
|
|
assert received_options[0].source == ''
|
|
assert received_options[0].dns_brute is True
|
|
assert received_options[0].dns_resolve == ''
|
|
assert received_options[0].dns_resolvers == ('192.0.2.53',)
|
|
|
|
|
|
def test_routeviews_child_receives_explicit_action_without_source_limit_controls(tmp_path, monkeypatch) -> None:
|
|
from theHarvester import __main__ as main_module
|
|
from theHarvester.lib.api import run_worker
|
|
from theHarvester.lib.api.run_models import RunRequest
|
|
from theHarvester.lib.api.run_store import RunStore
|
|
from theHarvester.lib.completed_result import CompletedResult
|
|
|
|
received_options = []
|
|
|
|
async def fake_start(options, **_kwargs):
|
|
received_options.append(options)
|
|
now = datetime.now(UTC)
|
|
return (CompletedResult.finish(target=options.domain, started_at=now, completed_at=now, groups={}),)
|
|
|
|
monkeypatch.setattr(main_module, 'start', fake_start)
|
|
|
|
async def scenario() -> None:
|
|
store = RunStore(tmp_path / 'runs.sqlite')
|
|
created = await store.create(RunRequest(target='192.0.2.7', sources=[], routeviews=True, limit=9_999))
|
|
assert await store.claim_next() is not None
|
|
await run_worker._child_execute(created['run_id'], store.database)
|
|
|
|
asyncio.run(scenario())
|
|
|
|
assert received_options[0].source == ''
|
|
assert received_options[0].routeviews is True
|
|
assert received_options[0].limit == 9_999
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
('field', 'value', 'option'),
|
|
[
|
|
('shodan', True, '--shodan'),
|
|
('dns_resolve', True, '--dns-resolve'),
|
|
('dns_lookup', True, '--dns-lookup'),
|
|
('dns_brute', True, '--dns-brute'),
|
|
('dns_recursive_depth', 1, '--dns-recursive-depth'),
|
|
('takeover', True, '--take-over'),
|
|
('screenshot', True, '--screenshot'),
|
|
('vhost', True, '--vhost'),
|
|
],
|
|
)
|
|
def test_no_hosts_rejects_hostname_dependent_actions(field: str, value: object, option: str) -> None:
|
|
from pydantic import ValidationError
|
|
|
|
from theHarvester.lib.api.run_models import RunRequest
|
|
|
|
with pytest.raises(ValidationError, match=rf'--no-hosts cannot be combined with: {option}'):
|
|
RunRequest(target='example.test', sources=['bufferoverun'], no_hosts=True, **{field: value})
|
|
|
|
|
|
def test_no_hosts_allows_target_only_api_scan() -> None:
|
|
from theHarvester.lib.api.run_models import RunRequest
|
|
|
|
request = RunRequest(target='example.test', sources=[], no_hosts=True, api_scan=True)
|
|
|
|
assert request.no_hosts is True
|
|
assert request.api_scan is True
|
|
|
|
|
|
def test_no_hosts_conflict_precedes_virtual_host_input_validation() -> None:
|
|
from pydantic import ValidationError
|
|
|
|
from theHarvester.lib.api.run_models import RunRequest
|
|
|
|
with pytest.raises(ValidationError, match=r'--no-hosts cannot be combined with: --vhost'):
|
|
RunRequest(target='example.test', sources=[], no_hosts=True, vhost=True)
|
|
|
|
|
|
def test_no_hosts_child_preserves_the_run_request(tmp_path, monkeypatch) -> None:
|
|
from theHarvester import __main__ as main_module
|
|
from theHarvester.lib.api import run_worker
|
|
from theHarvester.lib.api.run_models import RunRequest
|
|
from theHarvester.lib.api.run_store import RunStore
|
|
from theHarvester.lib.completed_result import CompletedResult
|
|
|
|
received_options = []
|
|
|
|
async def fake_start(options, **_kwargs):
|
|
received_options.append(options)
|
|
now = datetime.now(UTC)
|
|
return (CompletedResult.finish(target=options.domain, started_at=now, completed_at=now, groups={}),)
|
|
|
|
monkeypatch.setattr(main_module, 'start', fake_start)
|
|
|
|
async def scenario() -> None:
|
|
store = RunStore(tmp_path / 'runs.sqlite')
|
|
created = await store.create(RunRequest(target='example.test', sources=['bufferoverun'], no_hosts=True))
|
|
assert await store.claim_next() is not None
|
|
await run_worker._child_execute(created['run_id'], store.database)
|
|
|
|
asyncio.run(scenario())
|
|
|
|
assert received_options[0].source == 'bufferoverun'
|
|
assert received_options[0].no_hosts is True
|
|
|
|
|
|
def test_virtual_host_request_normalizes_a_self_contained_target_action() -> None:
|
|
from pydantic import ValidationError
|
|
|
|
from theHarvester.lib.api.run_models import RunRequest
|
|
|
|
request = RunRequest(
|
|
target='Example.Test.',
|
|
sources=[],
|
|
vhost_endpoint='https://192.0.2.8',
|
|
vhost_candidates=['ADMIN.Example.Test.'],
|
|
)
|
|
|
|
assert request.vhost is True
|
|
assert request.vhost_endpoint == 'https://192.0.2.8:443/'
|
|
assert request.vhost_candidates == ['admin.example.test']
|
|
assert (
|
|
RunRequest(
|
|
target='example.test',
|
|
sources=['crtsh'],
|
|
vhost_endpoint='http://192.0.2.9',
|
|
).vhost
|
|
is True
|
|
)
|
|
assert (
|
|
RunRequest(
|
|
target='example.test',
|
|
sources=['crtsh'],
|
|
vhost_candidates=['admin.example.test'],
|
|
).vhost
|
|
is True
|
|
)
|
|
with pytest.raises(ValidationError, match='discovery source'):
|
|
RunRequest(target='example.test', sources=[], vhost_endpoint='https://192.0.2.8')
|
|
with pytest.raises(ValidationError, match='discovery source'):
|
|
RunRequest(target='example.test', sources=[], vhost_candidates=['admin.example.test'])
|
|
with pytest.raises(ValidationError, match='discovery source'):
|
|
RunRequest(target='example.test', sources=[], vhost=True)
|
|
with pytest.raises(ValidationError, match='hostname target'):
|
|
RunRequest(target='192.0.2.8', sources=['crtsh'], vhost=True)
|
|
with pytest.raises(ValidationError, match='direct transport'):
|
|
RunRequest(target='example.test', sources=['crtsh'], vhost=True, proxies=True)
|
|
|
|
|
|
def test_virtual_host_child_receives_bounded_controls_and_persistence_identity(tmp_path, monkeypatch) -> None:
|
|
from theHarvester import __main__ as main_module
|
|
from theHarvester.lib.api import run_worker
|
|
from theHarvester.lib.api.run_models import RunRequest
|
|
from theHarvester.lib.api.run_store import RunStore
|
|
from theHarvester.lib.completed_result import CompletedResult
|
|
|
|
captured = None
|
|
captured_database = None
|
|
captured_run_id = None
|
|
|
|
async def fake_start(options, **kwargs):
|
|
nonlocal captured, captured_database, captured_run_id
|
|
captured = options
|
|
captured_database = kwargs.get('result_database')
|
|
captured_run_id = kwargs.get('completed_run_id')
|
|
now = datetime.now(UTC)
|
|
return (
|
|
CompletedResult.finish(
|
|
run_id=captured_run_id,
|
|
target=options.domain,
|
|
started_at=now,
|
|
completed_at=now,
|
|
groups={},
|
|
),
|
|
)
|
|
|
|
monkeypatch.setattr(main_module, 'start', fake_start)
|
|
|
|
async def scenario() -> tuple[Path, str]:
|
|
store = RunStore(tmp_path / 'runs.sqlite')
|
|
created = await store.create(
|
|
RunRequest(
|
|
target='example.test',
|
|
sources=[],
|
|
vhost_endpoint='https://192.0.2.8',
|
|
vhost_candidates=['admin.example.test'],
|
|
vhost_request_limit=12,
|
|
vhost_runtime_seconds=9,
|
|
vhost_timeout_seconds=2,
|
|
vhost_concurrency=1,
|
|
vhost_insecure=True,
|
|
)
|
|
)
|
|
assert await store.claim_next() is not None
|
|
await run_worker._child_execute(created['run_id'], store.database)
|
|
return store.database, created['run_id']
|
|
|
|
database, run_id = asyncio.run(scenario())
|
|
|
|
assert captured is not None
|
|
assert captured.vhost is True
|
|
assert captured.vhost_endpoint == 'https://192.0.2.8:443/'
|
|
assert captured.vhost_candidates == ('admin.example.test',)
|
|
assert captured.vhost_request_limit == 12
|
|
assert captured.vhost_runtime_seconds == 9
|
|
assert captured.vhost_timeout_seconds == 2
|
|
assert captured.vhost_concurrency == 1
|
|
assert captured.vhost_insecure is True
|
|
assert captured_database == database
|
|
assert str(captured_run_id) == run_id
|
|
|
|
|
|
def test_api_scan_child_uses_operator_endpoint_paths(tmp_path, monkeypatch) -> None:
|
|
from theHarvester import __main__ as main_module
|
|
from theHarvester.lib.api import run_worker
|
|
from theHarvester.lib.api.run_models import RunRequest
|
|
from theHarvester.lib.api.run_store import RunStore
|
|
from theHarvester.lib.completed_result import CompletedResult
|
|
|
|
received_wordlists: list[Path] = []
|
|
|
|
async def fake_start(options, **_kwargs):
|
|
received_wordlists.append(Path(options.wordlist))
|
|
now = datetime.now(UTC)
|
|
return (CompletedResult.finish(target=options.domain, started_at=now, completed_at=now, groups={}),)
|
|
|
|
monkeypatch.setattr(main_module, 'start', fake_start)
|
|
|
|
async def scenario() -> None:
|
|
store = RunStore(tmp_path / 'runs.sqlite')
|
|
created = await store.create(
|
|
RunRequest(
|
|
target='api.example.test',
|
|
sources=[],
|
|
api_scan=True,
|
|
api_scan_paths=['/api/v2', '/health'],
|
|
)
|
|
)
|
|
assert await store.claim_next() is not None
|
|
await run_worker._child_execute(created['run_id'], store.database)
|
|
|
|
asyncio.run(scenario())
|
|
|
|
assert received_wordlists[0].read_text(encoding='utf-8') == '/api/v2\n/health\n'
|
|
|
|
|
|
def test_running_cancellation_terminates_child_and_retains_partial_evidence(tmp_path, monkeypatch) -> None:
|
|
from theHarvester.lib.api import api, run_worker
|
|
|
|
monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key')
|
|
monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
|
|
monkeypatch.setenv('THEHARVESTER_RUN_ARTIFACTS', str(tmp_path / 'artifacts'))
|
|
monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'enabled')
|
|
|
|
async def slow_process(_run_id, _database, artifact_dir):
|
|
artifact_dir.mkdir(parents=True, exist_ok=True)
|
|
(artifact_dir / 'evidence.json').write_text(
|
|
json.dumps(
|
|
{
|
|
'target': 'example.test',
|
|
'status': 'partial',
|
|
'results': [{'type': 'email', 'value': 'saved@example.test'}],
|
|
}
|
|
),
|
|
encoding='utf-8',
|
|
)
|
|
return await asyncio.create_subprocess_exec(
|
|
sys.executable,
|
|
'-c',
|
|
'import time; time.sleep(60)',
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
|
|
monkeypatch.setattr(run_worker, '_process_factory', slow_process)
|
|
headers = {'X-API-Key': 'test-key'}
|
|
with TestClient(api.app) as client:
|
|
submitted = client.post(
|
|
'/api/v1/runs',
|
|
headers=headers,
|
|
json={'target': 'example.test', 'sources': ['crtsh'], 'deadline_seconds': 60},
|
|
).json()
|
|
deadline = time.monotonic() + 3
|
|
while time.monotonic() < deadline:
|
|
detail = client.get(f'/api/v1/runs/{submitted["run_id"]}', headers=headers).json()
|
|
if detail['status'] == 'running':
|
|
break
|
|
time.sleep(0.02)
|
|
requested = client.post(f'/api/v1/runs/{submitted["run_id"]}/cancel', headers=headers)
|
|
while time.monotonic() < deadline:
|
|
detail = client.get(f'/api/v1/runs/{submitted["run_id"]}', headers=headers).json()
|
|
if detail['status'] == 'cancelled':
|
|
break
|
|
time.sleep(0.02)
|
|
|
|
assert requested.json()['status'] == 'cancelling'
|
|
assert detail['status'] == 'cancelled'
|
|
assert detail['completed_at'] is not None
|
|
assert detail['evidence_status'] == 'partial'
|
|
assert detail['results'] == [{'type': 'email', 'value': 'saved@example.test', 'sources': [], 'actions': []}]
|
|
|
|
|
|
def test_whole_run_deadline_terminates_child_and_retains_partial_evidence(tmp_path, monkeypatch) -> None:
|
|
from theHarvester.lib.api import run_worker
|
|
from theHarvester.lib.api.run_models import RunRequest
|
|
from theHarvester.lib.api.run_store import RunStore
|
|
|
|
monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
|
|
monkeypatch.setenv('THEHARVESTER_RUN_ARTIFACTS', str(tmp_path / 'artifacts'))
|
|
monkeypatch.setattr(run_worker, '_worker_stop', None)
|
|
|
|
async def slow_process(_run_id, _database, artifact_dir):
|
|
artifact_dir.mkdir(parents=True, exist_ok=True)
|
|
(artifact_dir / 'evidence.json').write_text(
|
|
json.dumps(
|
|
{
|
|
'target': 'example.test',
|
|
'status': 'partial',
|
|
'results': [{'type': 'email', 'value': 'saved@example.test'}],
|
|
}
|
|
),
|
|
encoding='utf-8',
|
|
)
|
|
return await asyncio.create_subprocess_exec(
|
|
sys.executable,
|
|
'-c',
|
|
'import time; time.sleep(60)',
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
|
|
monkeypatch.setattr(run_worker, '_process_factory', slow_process)
|
|
|
|
async def scenario():
|
|
store = RunStore()
|
|
await store.create(RunRequest(target='example.test', sources=['crtsh']))
|
|
run = await store.claim_next()
|
|
assert run is not None
|
|
run['request']['deadline_seconds'] = 0
|
|
await run_worker._execute_claimed(store, run)
|
|
return await store.get(run['run_id'])
|
|
|
|
detail = asyncio.run(scenario())
|
|
|
|
assert detail is not None
|
|
assert detail['status'] == 'failed'
|
|
assert 'deadline' in detail['error']
|
|
assert detail['evidence_status'] == 'partial'
|
|
assert detail['results'] == [{'type': 'email', 'value': 'saved@example.test', 'sources': [], 'actions': []}]
|
|
|
|
|
|
def test_worker_fails_run_without_attaching_child_evidence_for_another_target(tmp_path, monkeypatch) -> None:
|
|
from theHarvester.lib.api import run_worker
|
|
from theHarvester.lib.api.run_models import RunRequest
|
|
from theHarvester.lib.api.run_store import RunStore
|
|
|
|
monkeypatch.setenv('THEHARVESTER_RUN_ARTIFACTS', str(tmp_path / 'artifacts'))
|
|
monkeypatch.setattr(run_worker, '_worker_stop', None)
|
|
|
|
async def mismatched_process(_run_id, _database, artifact_dir):
|
|
artifact_dir.mkdir(parents=True, exist_ok=True)
|
|
(artifact_dir / 'evidence.json').write_text(
|
|
json.dumps(
|
|
{
|
|
'target': 'different.example',
|
|
'status': 'complete',
|
|
'results': [{'type': 'email', 'value': 'saved@different.example'}],
|
|
}
|
|
),
|
|
encoding='utf-8',
|
|
)
|
|
return await asyncio.create_subprocess_exec(
|
|
sys.executable,
|
|
'-c',
|
|
'pass',
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
|
|
monkeypatch.setattr(run_worker, '_process_factory', mismatched_process)
|
|
|
|
async def scenario():
|
|
store = RunStore(tmp_path / 'runs.sqlite')
|
|
await store.create(RunRequest(target='expected.example', sources=['crtsh'], dns_resolve=True))
|
|
run = await store.claim_next()
|
|
assert run is not None
|
|
assert run['request']['deadline_seconds'] is None
|
|
await run_worker._execute_claimed(store, run)
|
|
return await store.get(run['run_id'])
|
|
|
|
detail = asyncio.run(scenario())
|
|
|
|
assert detail is not None
|
|
assert detail['status'] == 'failed'
|
|
assert detail['results'] == []
|
|
assert 'does not match run target' in detail['error']
|
|
|
|
|
|
@pytest.mark.parametrize('source_workers', [0, -1, True, 1.5])
|
|
def test_run_request_requires_positive_source_workers(source_workers: object) -> None:
|
|
from pydantic import ValidationError
|
|
|
|
from theHarvester.lib.api.run_models import RunRequest
|
|
|
|
with pytest.raises(ValidationError):
|
|
RunRequest(target='example.test', sources=['crtsh'], source_workers=source_workers)
|
|
|
|
|
|
def test_source_workers_are_preserved_from_rest_request_to_child(tmp_path, monkeypatch) -> None:
|
|
from theHarvester import __main__ as main_module
|
|
from theHarvester.lib.api import run_worker
|
|
from theHarvester.lib.api.run_models import RunRequest
|
|
from theHarvester.lib.api.run_store import RunStore
|
|
from theHarvester.lib.completed_result import CompletedResult
|
|
|
|
received_options = []
|
|
|
|
async def fake_start(options, **_kwargs):
|
|
received_options.append(options)
|
|
now = datetime.now(UTC)
|
|
return (CompletedResult.finish(target=options.domain, started_at=now, completed_at=now, groups={}),)
|
|
|
|
monkeypatch.setattr(main_module, 'start', fake_start)
|
|
|
|
async def scenario() -> None:
|
|
store = RunStore(tmp_path / 'runs.sqlite')
|
|
created = await store.create(RunRequest(target='example.test', sources=['crtsh'], source_workers=7))
|
|
assert await store.claim_next() is not None
|
|
await run_worker._child_execute(created['run_id'], store.database)
|
|
|
|
asyncio.run(scenario())
|
|
|
|
assert received_options[0].source_workers == 7
|