From c91456531cc2c8fb4b4c877ae4049092c49da1be Mon Sep 17 00:00:00 2001 From: NotoriousRebel <36310667+NotoriousRebel@users.noreply.github.com> Date: Sun, 16 Aug 2026 14:30:02 -0400 Subject: [PATCH] feat: export completed runs as SQLite --- CHANGELOG.md | 1 + README.md | 1 + docs/images/harvestview-architecture.svg | 4 +- docs/wiki/Rest-API.md | 11 ++ docs/wiki/Results-and-Local-Data.md | 2 +- tests/e2e/test_harvestview.py | 115 +++++++++++++---- tests/lib/test_api_v1.py | 117 ++++++++++++++++++ tests/lib/test_harvestview_ui.py | 5 +- tests/test_readme.py | 12 +- theHarvester/lib/api/run_models.py | 14 +++ theHarvester/lib/api/run_store.py | 4 + theHarvester/lib/api/runs.py | 26 ++++ .../lib/api/static/harvestview/app.js | 28 ++++- .../lib/api/static/harvestview/index.html | 1 + theHarvester/lib/database.py | 29 +++++ 15 files changed, 335 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f52c965e..ba819463 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Added authenticated REST and HarvestView export of every completed run as a portable SQLite database without queue, cancellation, worker-lease, or legacy-observation state. - Added a catalog-derived offline provider-contract gate that fails on missing, unknown, or duplicate source coverage while keeping live checks outside routine CI. - Added sourced ASN organization attribution from URLScan, ONYPHE, and Shodan, linked to the exact hostname or IP evidence and retained in SQLite, JSONL, the API, CLI output, and HarvestView without claiming ownership or scope. - Added bounded RouteViews routing enrichment for exact discovered IPs with sourced ASN attribution, or explicit ASN, IP, and CIDR targets, retaining typed origin, BGP-route, and RPKI evidence as external relationships without expanding active scope. diff --git a/README.md b/README.md index c454abde..61b70583 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,7 @@ docker compose down | `POST /api/v1/runs/{run_id}/cancel` | Cancel queued or running work. | | `POST /api/v1/runs/import` | Import JSONL evidence without executing discovery. | | `POST /api/v1/runs/import-database` | Import completed runs from a theHarvester SQLite database. | +| `GET /api/v1/runs/export-database` | Export all completed run evidence as a portable SQLite database. | | `GET /api/v1/runs/{run_id}/export` | Export normalized evidence as JSONL. | HarvestView can start a screenshot or DNS brute-force run directly from a hostname result. These actions create a separate run record for that hostname and leave the parent evidence unchanged. Resolver addresses may be entered directly or loaded from a text file with one IP address per line. Ordinary DNS actions accept one or more resolvers; recursive DNS requires exactly three. diff --git a/docs/images/harvestview-architecture.svg b/docs/images/harvestview-architecture.svg index b81aa1d1..a048f334 100644 --- a/docs/images/harvestview-architecture.svg +++ b/docs/images/harvestview-architecture.svg @@ -96,7 +96,7 @@ Inspect and export history · assessment · route tabs screenshots · source outcomes · logs - review child actions · export JSONL + review actions · JSONL / SQLite export @@ -106,4 +106,4 @@ durable evidence boundary provider credentials remain server-side - \ No newline at end of file + diff --git a/docs/wiki/Rest-API.md b/docs/wiki/Rest-API.md index 71f0a68a..a127ca8f 100644 --- a/docs/wiki/Rest-API.md +++ b/docs/wiki/Rest-API.md @@ -32,6 +32,7 @@ Treat the runtime OpenAPI document as the exact request and response reference. | `POST /api/v1/runs/{run_id}/cancel` | Cancel queued work or request cancellation of running work. | | `POST /api/v1/runs/import` | Import a JSONL result file without executing discovery. | | `POST /api/v1/runs/import-database` | Import completed runs from a theHarvester SQLite database. | +| `GET /api/v1/runs/export-database` | Export all completed run evidence as a portable SQLite database. | | `GET /api/v1/runs/{run_id}/export` | Export normalized results as JSONL. | | `GET /api/v1/runs/{run_id}/screenshots/{name}` | Retrieve one managed screenshot. | @@ -169,6 +170,16 @@ curl -s "http://127.0.0.1:5000/api/v1/runs/import-database?filename=stash.sqlite The server checks the SQLite header, integrity, schema, and each completed run before copying it. Original run IDs are preserved. Exact duplicates are skipped, while a reused ID with different evidence is rejected. Close the source process or checkpoint its WAL before uploading the database. Screenshot metadata is imported, but screenshot files must be copied separately. The default upload ceiling is 1 GiB and can be changed with `THEHARVESTER_MAX_DATABASE_IMPORT_BYTES`. +Export every completed run as a consistent database that can be imported elsewhere: + +```bash +curl -s "http://127.0.0.1:5000/api/v1/runs/export-database" \ + -H "X-API-Key: $THEHARVESTER_API_KEY" \ + -o theharvester-completed-runs.sqlite +``` + +The export is rebuilt from canonical completed evidence, so it excludes queue state, cancellation state, worker leases, and legacy observations. It includes screenshot metadata but not screenshot files. The server checkpoints and closes the temporary database before download; no manual WAL handling is required. + Export one normalized result set in the same streamable format: ```bash diff --git a/docs/wiki/Results-and-Local-Data.md b/docs/wiki/Results-and-Local-Data.md index 08beaefc..4ee7e5d9 100644 --- a/docs/wiki/Results-and-Local-Data.md +++ b/docs/wiki/Results-and-Local-Data.md @@ -75,7 +75,7 @@ Two operational tables support the API without changing those six evidence conce ## API results -`GET /api/v1/runs/{run_id}` returns lifecycle state plus a normalized `results` array. Each result has `type`, `value`, `sources`, and `actions`. A `hostname` found through the `vhost` action has native endpoint observations; a `prefix` found through RouteViews has native origin, route, and RPKI observations with fixed external-relationship scope. Run-level source and action outcomes remain available in `source_executions` and `action_executions`, while file metadata is returned through `artifacts`. JSONL imports or exports one run, and SQLite import loads completed runs in bulk. Treat runtime `/docs`, `/redoc`, and OpenAPI as the exact request and response reference. +`GET /api/v1/runs/{run_id}` returns lifecycle state plus a normalized `results` array. Each result has `type`, `value`, `sources`, and `actions`. A `hostname` found through the `vhost` action has native endpoint observations; a `prefix` found through RouteViews has native origin, route, and RPKI observations with fixed external-relationship scope. Run-level source and action outcomes remain available in `source_executions` and `action_executions`, while file metadata is returned through `artifacts`. JSONL imports or exports one run. SQLite import and `GET /api/v1/runs/export-database` move completed runs in bulk without queue, cancellation, or worker-lease state. Treat runtime `/docs`, `/redoc`, and OpenAPI as the exact request and response reference. ## Handling and sharing diff --git a/tests/e2e/test_harvestview.py b/tests/e2e/test_harvestview.py index 5d685e63..daeb0fd8 100644 --- a/tests/e2e/test_harvestview.py +++ b/tests/e2e/test_harvestview.py @@ -1,10 +1,8 @@ from __future__ import annotations -import asyncio import json +import sqlite3 from collections import Counter -from concurrent.futures import ThreadPoolExecutor -from datetime import UTC, datetime from pathlib import Path import pytest @@ -1380,40 +1378,107 @@ def test_completed_empty_import_explains_terminal_outcome( def test_harvestview_imports_completed_runs_from_sqlite( - harvestview_server_url: str, + harvestview_server, page: Page, tmp_path: Path, ) -> None: - from theHarvester.lib.completed_result import CompletedResult - from theHarvester.lib.database import ResultStore, dispose_sqlite_databases - - database = tmp_path / 'completed-runs.sqlite' - now = datetime.now(UTC) - completed = CompletedResult.finish( - target='sqlite.example.test', - started_at=now, - completed_at=now, - groups={'hostname': ['api.sqlite.example.test']}, + harvestview_server_url = harvestview_server.url + source_run_id = '4ef278df-ce95-4120-9241-6e71dd96ad74' + evidence_file = tmp_path / 'completed-run.jsonl' + write_jsonl_evidence( + evidence_file, + { + 'run_id': source_run_id, + 'target': 'sqlite.example.test', + 'started_at': '2026-08-08T01:00:00Z', + 'completed_at': '2026-08-08T01:01:00Z', + 'status': 'complete', + 'source_executions': [{'source': 'crtsh', 'status': 'completed', 'duration_ms': 2, 'result_count': 1}], + 'action_executions': [{'action': 'vhost', 'status': 'completed', 'duration_ms': 1, 'result_count': 1}], + 'results': [ + { + 'type': 'hostname', + 'value': 'admin.sqlite.example.test', + 'sources': ['crtsh'], + 'actions': ['vhost'], + 'observations': [ + { + 'endpoint': 'https://192.0.2.8:443/', + 'http_host': 'admin.sqlite.example.test', + 'tls_server_name': 'admin.sqlite.example.test', + 'classification': 'distinct', + 'phase': 'body', + 'status': 401, + 'location': None, + 'body_sha256': 'a' * 64, + 'body_size': 12, + 'body_truncated': False, + 'context_phase': 'body', + 'context_status': 200, + 'context_location': None, + 'context_body_sha256': 'a' * 64, + 'context_body_size': 12, + 'context_body_truncated': False, + 'control_phase': 'body', + 'control_status': 200, + 'control_location': None, + 'control_body_sha256': 'a' * 64, + 'control_body_size': 12, + 'control_body_truncated': False, + 'confirmation_body_sha256': None, + 'tls_verified': True, + 'distinct_signals': ['status'], + 'reflection_normalized': False, + } + ], + } + ], + }, ) - async def prepare_database() -> None: - store = ResultStore(database) - await store.initialize() - await store.save_run(completed) - await dispose_sqlite_databases() - - with ThreadPoolExecutor(max_workers=1) as executor: - executor.submit(lambda: asyncio.run(prepare_database())).result() - page.goto(f'{harvestview_server_url}/') page.get_by_role('button', name='Import result file').first.click() - page.locator('#result-file').set_input_files(database) + page.locator('#result-file').set_input_files(evidence_file) page.locator('#submit-import-button').click() expect(page.locator('#detail-target')).to_have_text('sqlite.example.test') expect(page.locator('#run-count')).to_have_text('1') - expect(page.locator('#toast')).to_have_text('Imported 1 run from completed-runs.sqlite; 0 already present.') + expect(page.locator('#toast')).to_have_text('Imported completed-run.jsonl without executing discovery.') expect(page.get_by_role('button', name='Hostnames 1')).to_be_enabled() + imported_run_id = page.locator('#detail-run-id').inner_text() + with page.expect_download() as database_download: + page.get_by_role('button', name='Export database').click() + assert database_download.value.suggested_filename == 'theharvester-completed-runs.sqlite' + exported_database = Path(database_download.value.path()) + portable_database = tmp_path / database_download.value.suggested_filename + portable_database.write_bytes(exported_database.read_bytes()) + assert portable_database.read_bytes().startswith(b'SQLite format 3\x00') + with sqlite3.connect(portable_database) as connection: + exported_run_ids = {row[0] for row in connection.execute('SELECT run_id FROM runs')} + exported_tables = {row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type = 'table'")} + assert exported_run_ids == {imported_run_id} + assert {'run_records', 'run_worker_leases'}.isdisjoint(exported_tables) + + harvestview_server.stop() + database = Path(harvestview_server.environment['THEHARVESTER_RUN_DB']) + for database_file in (database, Path(f'{database}-wal'), Path(f'{database}-shm')): + database_file.unlink(missing_ok=True) + harvestview_server.start() + + page.goto(f'{harvestview_server_url}/') + expect(page.get_by_role('heading', name='No enumeration runs yet')).to_be_visible() + page.get_by_role('button', name='Import result file').first.click() + page.locator('#result-file').set_input_files(portable_database) + page.locator('#submit-import-button').click() + + expect(page.locator('#toast')).to_have_text('Imported 1 run from theharvester-completed-runs.sqlite; 0 already present.') + expect(page.locator('#detail-run-id')).to_have_text(imported_run_id) + expect(page.locator('#detail-target')).to_have_text('sqlite.example.test') + result_row = page.locator('.tabulator-row').first + expect(result_row).to_contain_text('admin.sqlite.example.test') + expect(result_row).to_contain_text('https://192.0.2.8:443/ · HTTP 401 · status') + expect(result_row).to_contain_text('crtsh') + expect(result_row).to_contain_text('vhost') def test_harvestview_can_import_and_analyze_fixture_evidence_through_the_real_ui( diff --git a/tests/lib/test_api_v1.py b/tests/lib/test_api_v1.py index 0c20607a..fc8950e1 100644 --- a/tests/lib/test_api_v1.py +++ b/tests/lib/test_api_v1.py @@ -265,6 +265,7 @@ def test_api_exposes_one_fresh_run_contract(tmp_path, monkeypatch) -> None: '/api/v1/runs', '/api/v1/runs/import', '/api/v1/runs/import-database', + '/api/v1/runs/export-database', '/api/v1/runs/{run_id}', '/api/v1/runs/{run_id}/cancel', '/api/v1/runs/{run_id}/export', @@ -441,6 +442,11 @@ def test_openapi_explains_scope_and_execution_controls(tmp_path, monkeypatch) -> assert 'endpoint paths' in properties['api_scan_paths']['description'] import_content = schema['paths']['/api/v1/runs/import']['post']['requestBody']['content'] assert set(import_content) == {'application/x-ndjson'} + database_export_content = schema['paths']['/api/v1/runs/export-database']['get']['responses']['200']['content'] + assert set(database_export_content) == {'application/vnd.sqlite3'} + assert database_export_content['application/vnd.sqlite3']['schema']['description'] == ( + 'Portable SQLite database containing every completed run and no API lifecycle state.' + ) export_content = schema['paths']['/api/v1/runs/{run_id}/export']['get']['responses']['200']['content'] assert set(export_content) == {'application/x-ndjson'} assert set(schema['components']['schemas']['NormalizedResult']['properties']) == { @@ -795,6 +801,117 @@ def test_api_database_import_exposes_completed_cli_runs(tmp_path, monkeypatch) - assert detail.json()['results'] == [{'type': 'hostname', 'value': 'api.imported.example.test', 'sources': [], 'actions': []}] +def test_api_database_export_round_trip_contains_only_completed_evidence(tmp_path, monkeypatch) -> None: + from theHarvester.lib.api import api + + source_database = tmp_path / 'source.sqlite' + destination_database = tmp_path / 'destination.sqlite' + exported_database = tmp_path / 'exported.sqlite' + headers = {'X-API-Key': 'test-key'} + monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key') + monkeypatch.setenv('THEHARVESTER_RUN_DB', str(source_database)) + monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled') + vhost_records = [json.loads(line) for line in _vhost_jsonl_result().splitlines()] + vhost_records[0]['source_executions'] = [ + { + 'source': 'crtsh', + 'status': 'completed', + 'duration_ms': 2, + 'result_count': 1, + 'error_type': None, + 'stop_reason': None, + } + ] + vhost_records[0]['action_executions'].append( + { + 'action': 'screenshot', + 'status': 'completed', + 'duration_ms': 1, + 'result_count': 0, + 'error_type': None, + 'stop_reason': None, + } + ) + vhost_records[0]['artifacts'] = [ + { + 'action': 'screenshot', + 'kind': 'screenshot', + 'subject': {'kind': 'hostname', 'value': 'admin.example.test'}, + 'file': { + 'path': 'screenshots/admin.example.test.png', + 'media_type': 'image/png', + 'size_bytes': 16, + 'sha256': '0' * 64, + }, + 'created_at': '2026-08-08T01:01:00Z', + } + ] + vhost_records[1]['sources'] = ['crtsh'] + vhost_jsonl = ''.join(json.dumps(record) + '\n' for record in vhost_records) + + with TestClient(api.app) as client: + first = client.post( + '/api/v1/runs/import', + params={'filename': 'vhost.jsonl'}, + headers=headers, + content=vhost_jsonl, + ) + second = client.post( + '/api/v1/runs/import', + params={'filename': 'network.jsonl'}, + headers=headers, + content=_network_jsonl_result(), + ) + source_details = { + run_id: client.get(f'/api/v1/runs/{run_id}', headers=headers).json() + for run_id in (first.json()['run_id'], second.json()['run_id']) + } + exported = client.get('/api/v1/runs/export-database', headers=headers) + + assert first.status_code == 201 + assert second.status_code == 201 + assert exported.status_code == 200 + assert exported.headers['content-type'] == 'application/vnd.sqlite3' + assert exported.headers['content-disposition'] == 'attachment; filename="theharvester-completed-runs.sqlite"' + assert exported.content.startswith(b'SQLite format 3\x00') + exported_database.write_bytes(exported.content) + with sqlite3.connect(exported_database) as connection: + tables = {row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type = 'table'")} + assert {'runs', 'results'} <= tables + assert {'run_records', 'run_worker_leases'}.isdisjoint(tables) + + monkeypatch.setenv('THEHARVESTER_RUN_DB', str(destination_database)) + with TestClient(api.app) as client: + reimported = client.post( + '/api/v1/runs/import-database', + params={'filename': 'theharvester-completed-runs.sqlite'}, + headers={**headers, 'Content-Type': 'application/vnd.sqlite3'}, + content=exported.content, + ) + imported_details = { + run_id: client.get(f'/api/v1/runs/{run_id}', headers=headers).json() + for run_id in reimported.json()['imported_run_ids'] + } + + assert reimported.status_code == 201 + assert reimported.json()['imported_run_ids'] == sorted((first.json()['run_id'], second.json()['run_id'])) + canonical_evidence_fields = ( + 'run_id', + 'target', + 'started_at', + 'completed_at', + 'evidence_status', + 'result_count', + 'source_executions', + 'action_executions', + 'artifacts', + 'results', + ) + assert { + run_id: {field: detail[field] for field in canonical_evidence_fields} for run_id, detail in imported_details.items() + } == {run_id: {field: detail[field] for field in canonical_evidence_fields} for run_id, detail in source_details.items()} + + def test_api_jsonl_round_trip_preserves_source_attribution(tmp_path, monkeypatch) -> None: from theHarvester.lib.api import api diff --git a/tests/lib/test_harvestview_ui.py b/tests/lib/test_harvestview_ui.py index 4ef18803..4d8a3fe5 100644 --- a/tests/lib/test_harvestview_ui.py +++ b/tests/lib/test_harvestview_ui.py @@ -83,7 +83,7 @@ def test_harvestview_has_an_operator_readable_shodan_host_route(tmp_path, monkey assert "title: 'Services'" in script.text -def test_harvestview_offers_jsonl_and_sqlite_imports_with_jsonl_export(tmp_path, monkeypatch) -> None: +def test_harvestview_offers_jsonl_and_sqlite_imports_and_exports(tmp_path, monkeypatch) -> None: from theHarvester.lib.api import api monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key') @@ -95,11 +95,14 @@ def test_harvestview_offers_jsonl_and_sqlite_imports_with_jsonl_export(tmp_path, script = client.get('/static/harvestview/app.js') assert 'accept=".jsonl,.sqlite,.sqlite3,.db,application/x-ndjson,application/vnd.sqlite3"' in root.text + assert 'id="export-database-button"' in root.text assert 'id="export-jsonl-button"' in root.text assert 'id="route-csv-button"' not in root.text assert 'id="export-json-button"' not in root.text assert 'id="export-csv-button"' not in root.text assert '/export' in script.text + assert '/api/v1/runs/export-database' in script.text + assert 'theharvester-completed-runs.sqlite' in script.text assert "fileKind === 'jsonl' ? '/api/v1/runs/import' : '/api/v1/runs/import-database'" in script.text assert '/exports/' not in script.text assert 'text/csv' not in script.text diff --git a/tests/test_readme.py b/tests/test_readme.py index de0a78f8..553bd444 100644 --- a/tests/test_readme.py +++ b/tests/test_readme.py @@ -129,7 +129,7 @@ def test_readme_architecture_diagrams_are_local_and_accessible() -> None: 'HarvestView run desk architecture', Path('docs/images/harvestview-architecture.svg'), 'harvestview-architecture', - ('Authenticated REST API', 'queued → running → terminal', 'Isolated run worker', 'export JSONL'), + ('Authenticated REST API', 'queued → running → terminal', 'Isolated run worker', 'JSONL / SQLite export'), ), ) @@ -193,6 +193,16 @@ def test_operator_docs_recommend_jsonl_and_assume_uv_is_available() -> None: assert all('`report.jsonl`' in page and 'automation' in page for page in (quick_start, workflows)) +def test_operator_docs_cover_portable_database_export() -> None: + readme = Path('README.md').read_text() + rest_api = Path('docs/wiki/Rest-API.md').read_text() + local_data = Path('docs/wiki/Results-and-Local-Data.md').read_text() + + assert all('/api/v1/runs/export-database' in page for page in (readme, rest_api, local_data)) + assert 'queue state, cancellation state, worker leases, and legacy observations' in rest_api + assert 'no manual WAL handling is required' in rest_api + + def test_readme_explains_jsonl_record_and_structured_evidence_parsing() -> None: readme = Path('README.md').read_text() diff --git a/theHarvester/lib/api/run_models.py b/theHarvester/lib/api/run_models.py index 68269caa..f2e41d77 100644 --- a/theHarvester/lib/api/run_models.py +++ b/theHarvester/lib/api/run_models.py @@ -642,6 +642,20 @@ DATABASE_IMPORT_REQUEST_OPENAPI = { 'content': {'application/vnd.sqlite3': {'schema': {'type': 'string', 'format': 'binary'}}}, } } +DATABASE_EXPORT_RESPONSES: dict[int | str, dict[str, Any]] = { + 200: { + 'description': 'Portable completed-run evidence as SQLite.', + 'content': { + 'application/vnd.sqlite3': { + 'schema': { + 'type': 'string', + 'format': 'binary', + 'description': 'Portable SQLite database containing every completed run and no API lifecycle state.', + } + }, + }, + } +} EXPORT_RESPONSES: dict[int | str, dict[str, Any]] = { 200: { 'description': 'Normalized run results as JSONL.', diff --git a/theHarvester/lib/api/run_store.py b/theHarvester/lib/api/run_store.py index 55050398..ef8e683b 100644 --- a/theHarvester/lib/api/run_store.py +++ b/theHarvester/lib/api/run_store.py @@ -373,6 +373,10 @@ class RunStore: 'skipped_run_ids': sorted(skipped_run_ids), } + async def export_database(self, destination: Path) -> None: + await self.initialize() + await self.results.export_database(destination) + async def list_runs(self, *, limit: int = 100, offset: int = 0) -> list[dict[str, Any]]: await self.initialize() return [await self._row(record) for record in await self.lifecycle.list_records(limit=limit, offset=offset)] diff --git a/theHarvester/lib/api/runs.py b/theHarvester/lib/api/runs.py index 4bdebe75..fa6be971 100644 --- a/theHarvester/lib/api/runs.py +++ b/theHarvester/lib/api/runs.py @@ -9,6 +9,7 @@ import anyio from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response, status from fastapi.responses import FileResponse from pydantic import ValidationError +from starlette.background import BackgroundTask from theHarvester.lib.api.auth import get_api_key from theHarvester.lib.source_catalog import ACTION_ACTIVITIES, SOURCE_SPECS, SourceSpec, get_source_spec, resolve_sources @@ -16,6 +17,7 @@ from theHarvester.lib.source_catalog import ACTION_ACTIVITIES, SOURCE_SPECS, Sou from . import run_worker from .run_evidence import parse_jsonl_import from .run_models import ( + DATABASE_EXPORT_RESPONSES, DATABASE_IMPORT_REQUEST_OPENAPI, EXPORT_RESPONSES, IMPORT_REQUEST_OPENAPI, @@ -37,6 +39,11 @@ MAX_RUN_REQUEST_BYTES = 64 * 1024 DEFAULT_MAX_DATABASE_IMPORT_BYTES = 1024 * 1024 * 1024 +def _remove_database_export(path: Path) -> None: + for candidate in (path, Path(f'{path}-wal'), Path(f'{path}-shm')): + candidate.unlink(missing_ok=True) + + async def _read_limited_body(request: Request, limit: int, detail: str) -> bytes: content_length = request.headers.get('content-length') if content_length and content_length.isdigit() and int(content_length) > limit: @@ -225,6 +232,25 @@ async def import_database( await anyio.Path(temporary_path).unlink(missing_ok=True) +@router.get('/runs/export-database', response_class=FileResponse, responses=DATABASE_EXPORT_RESPONSES) +async def export_database(_api_key: Annotated[str, Depends(get_api_key)]) -> FileResponse: + descriptor, temporary_name = tempfile.mkstemp(prefix='theharvester-export-', suffix='.sqlite') + os.close(descriptor) + temporary_path = Path(temporary_name) + await anyio.Path(temporary_path).chmod(0o600) + try: + await RunStore().export_database(temporary_path) + except BaseException: + _remove_database_export(temporary_path) + raise + return FileResponse( + temporary_path, + media_type='application/vnd.sqlite3', + filename='theharvester-completed-runs.sqlite', + background=BackgroundTask(_remove_database_export, temporary_path), + ) + + @router.get('/runs/{run_id}', response_model_exclude_unset=True) async def get_run(run_id: str, _api_key: Annotated[str, Depends(get_api_key)]) -> RunDetail: run = await RunStore().get(run_id) diff --git a/theHarvester/lib/api/static/harvestview/app.js b/theHarvester/lib/api/static/harvestview/app.js index 4cd9a553..4bfacfac 100644 --- a/theHarvester/lib/api/static/harvestview/app.js +++ b/theHarvester/lib/api/static/harvestview/app.js @@ -31,7 +31,8 @@ }; const nodes = { - themeButton: $('#theme-button'), importButton: $('#import-button'), newRunButton: $('#new-run-button'), + themeButton: $('#theme-button'), importButton: $('#import-button'), exportDatabase: $('#export-database-button'), + newRunButton: $('#new-run-button'), loading: $('#loading-state'), empty: $('#empty-state'), detail: $('#run-detail'), workspaceError: $('#workspace-error'), workspaceErrorMessage: $('#workspace-error-message'), retryWorkspace: $('#retry-workspace-button'), @@ -1249,17 +1250,33 @@ toast(state.detail.status === 'cancelled' ? 'Queued enumeration cancelled.' : 'Cancellation requested.'); } - async function downloadServerExport() { + async function downloadServerFile(path, fallbackFilename, failureMessage) { try { - const response = await api(`/api/v1/runs/${encodeURIComponent(state.selectedId)}/export`); + const response = await api(path); const disposition = response.headers.get('Content-Disposition') || ''; const match = disposition.match(/filename="([^"]+)"/); - downloadBlob(await response.blob(), match?.[1] || 'harvestview-results.jsonl'); + downloadBlob(await response.blob(), match?.[1] || fallbackFilename); } catch (error) { - toast(`Could not export results: ${error.message}. Keep the run open and try again.`, true); + toast(`${failureMessage}: ${error.message}`, true); } } + function downloadServerExport() { + return downloadServerFile( + `/api/v1/runs/${encodeURIComponent(state.selectedId)}/export`, + 'harvestview-results.jsonl', + 'Could not export results', + ); + } + + function downloadDatabase() { + return downloadServerFile( + '/api/v1/runs/export-database', + 'theharvester-completed-runs.sqlite', + 'Could not export the database', + ); + } + function downloadBlob(blob, filename) { const url = URL.createObjectURL(blob); const link = Object.assign(document.createElement('a'), {href: url, download: filename}); @@ -1303,6 +1320,7 @@ nodes.retryWorkspace.addEventListener('click', start); nodes.newRunButton.addEventListener('click', openNewRun); nodes.importButton.addEventListener('click', openImport); + nodes.exportDatabase.addEventListener('click', downloadDatabase); nodes.historySearch.addEventListener('input', renderHistory); nodes.newRunForm.addEventListener('submit', submitRun); nodes.resultActionForm.addEventListener('submit', submitResultAction); diff --git a/theHarvester/lib/api/static/harvestview/index.html b/theHarvester/lib/api/static/harvestview/index.html index 1392e019..e3fb1860 100644 --- a/theHarvester/lib/api/static/harvestview/index.html +++ b/theHarvester/lib/api/static/harvestview/index.html @@ -25,6 +25,7 @@ diff --git a/theHarvester/lib/database.py b/theHarvester/lib/database.py index 5ac9d638..36386eac 100644 --- a/theHarvester/lib/database.py +++ b/theHarvester/lib/database.py @@ -76,6 +76,11 @@ logger = logging.getLogger(__name__) SCHEMA_VERSION = 8 _DEFAULT_DATABASE = Path('~/.local/share/theHarvester/stash.sqlite').expanduser() +_PORTABLE_DATABASE_DROP_STATEMENTS = ( + 'DROP TABLE IF EXISTS legacy_observations', + 'DROP TABLE IF EXISTS run_records', + 'DROP TABLE IF EXISTS run_worker_leases', +) _LEGACY_RESULT_KIND_RENAMES = { 'api-endpoint': 'url', @@ -507,6 +512,16 @@ def _row_count(result: Any) -> int: return int(result.rowcount) +def _finalize_portable_database(path: Path) -> None: + with sqlite3.connect(path) as connection: + connection.execute('PRAGMA wal_checkpoint(TRUNCATE)') + connection.execute('PRAGMA journal_mode = DELETE') + for statement in _PORTABLE_DATABASE_DROP_STATEMENTS: + connection.execute(statement) + connection.commit() + connection.execute('VACUUM') + + class RunLifecycleStore: """Persist API run state in the same SQLite database as terminal evidence.""" @@ -1118,6 +1133,20 @@ class ResultStore: for run, result_count in rows ] + async def export_database(self, destination: Path) -> None: + """Write every completed run to a portable, importable SQLite database.""" + await self.initialize() + exported = ResultStore(destination) + try: + await exported.initialize() + summaries = await self.list_runs(limit=None) + for summary in summaries: + await exported.save_run(await self.load_run(UUID(str(summary['run_id'])))) + finally: + await exported.dispose() + await asyncio.to_thread(_finalize_portable_database, destination) + await exported.validate_import_database() + async def validate_import_database(self) -> None: engine = create_async_engine(URL.create('sqlite+aiosqlite', database=self.database)) try: