theHarvester/tests/lib/test_api_v1.py

from __future__ import annotations

import asyncio
import json

import pytest
from fastapi.testclient import TestClient


def _jsonl_result(
    *,
    target: str = 'example.test',
    finding_type: str = 'email',
    value: str = 'a@example.test',
    finding_fields: dict[str, object] | None = None,
    summary_fields: dict[str, object] | None = None,
) -> str:
    summary = {
        'type': 'summary',
        'run_id': '9f9b4383-6cc4-4f3f-80a4-c8d21930dc2d',
        'target': target,
        'started_at': '2026-08-08T01:00:00Z',
        'completed_at': '2026-08-08T01:01:00Z',
        'evidence_status': 'complete',
        'result_count': 1,
        'counts': {finding_type: 1},
    }
    summary.update(summary_fields or {})
    return '\n'.join(
        (
            json.dumps(summary),
            json.dumps({'type': finding_type, 'value': value, 'sources': [], **(finding_fields or {})}),
            '',
        )
    )


@pytest.mark.parametrize(
    ('finding_type', 'finding_fields'),
    [
        ('made-up-kind', {}),
        ('api-endpoint', {}),
        ('interesting-url', {}),
        ('ip-address', {}),
        ('linkedin-link', {}),
        ('subdomain', {}),
        ('hostname', {'dns_status': 'made-up-status'}),
    ],
)
def test_api_rejects_jsonl_findings_outside_the_contract(
    tmp_path,
    monkeypatch,
    finding_type: str,
    finding_fields: dict[str, object],
) -> None:
    from theHarvester.lib.api import api

    monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key')
    monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
    monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled')

    with TestClient(api.app) as client:
        response = client.post(
            '/api/v1/runs/import',
            params={'filename': 'invalid.jsonl'},
            headers={'X-API-Key': 'test-key'},
            content=_jsonl_result(finding_type=finding_type, finding_fields=finding_fields),
        )

    assert response.status_code == 400


def test_api_exposes_one_fresh_run_contract(tmp_path, monkeypatch) -> None:
    from theHarvester.lib.api import api

    monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key')
    monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
    monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled')

    with TestClient(api.app, base_url='http://127.0.0.1', client=('127.0.0.1', 50000)) as client:
        schema = client.get('/openapi.json').json()
        paths = set(schema['paths'])
        old_responses = [
            client.get('/query?domain=example.test&source=crtsh'),
            client.get('/sources'),
            client.get('/dnsbrute?domain=example.test'),
            client.get('/runs'),
            client.post('/additional/all', json={'domain': 'example.test'}),
        ]

    assert paths == {
        '/api/v1/sources',
        '/api/v1/runs',
        '/api/v1/runs/import',
        '/api/v1/runs/import-database',
        '/api/v1/runs/{run_id}',
        '/api/v1/runs/{run_id}/cancel',
        '/api/v1/runs/{run_id}/export',
        '/api/v1/runs/{run_id}/screenshots/{name}',
    }
    assert all(response.status_code == 404 for response in old_responses)


def test_screenshot_route_serves_only_a_run_owned_png(tmp_path, monkeypatch) -> None:
    from theHarvester.lib.api import api

    monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key')
    monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
    monkeypatch.setenv('THEHARVESTER_RUN_ARTIFACTS', str(tmp_path / 'artifacts'))
    monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled')
    headers = {'X-API-Key': 'test-key'}

    with TestClient(api.app, client=('127.0.0.2', 50000)) as client:
        imported = client.post(
            '/api/v1/runs/import',
            params={'filename': 'smoke.jsonl'},
            headers=headers,
            content=_jsonl_result(
                finding_type='hostname',
                value='owned.example.test',
                summary_fields={
                    'action_executions': [
                        {
                            'action': 'screenshot',
                            'status': 'completed',
                            'duration_ms': 1,
                            'result_count': 0,
                            'error_type': None,
                            'stop_reason': None,
                        }
                    ],
                    'artifacts': [
                        {
                            'action': 'screenshot',
                            'kind': 'screenshot',
                            'subject': {'kind': 'hostname', 'value': 'owned.example.test'},
                            'file': {
                                'path': 'screenshots/owned.example.test.png',
                                'media_type': 'image/png',
                                'size_bytes': 16,
                                'sha256': '0' * 64,
                            },
                            'created_at': '2026-08-08T01:01:00Z',
                        }
                    ],
                },
            ),
        )
        assert imported.status_code == 201
        run_id = imported.json()['run_id']
        screenshot_dir = tmp_path / 'artifacts' / run_id / 'screenshots'
        screenshot_dir.mkdir(parents=True)
        (screenshot_dir / 'owned.example.test.png').write_bytes(b'owned screenshot')
        (screenshot_dir / 'unrecorded.png').write_bytes(b'unrecorded screenshot')
        outside = tmp_path / 'outside.png'
        outside.write_bytes(b'outside screenshot')
        (screenshot_dir / 'linked.png').symlink_to(outside)

        owned = client.get(f'/api/v1/runs/{run_id}/screenshots/owned.example.test.png', headers=headers)
        unrecorded = client.get(f'/api/v1/runs/{run_id}/screenshots/unrecorded.png', headers=headers)
        linked = client.get(f'/api/v1/runs/{run_id}/screenshots/linked.png', headers=headers)
        traversal = client.get(f'/api/v1/runs/{run_id}/screenshots/%2E%2E%2Foutside.png', headers=headers)

    assert owned.status_code == 200
    assert owned.content == b'owned screenshot'
    assert unrecorded.status_code == 404
    assert linked.status_code == 404
    assert traversal.status_code == 404


def test_openapi_names_the_public_response_shapes(tmp_path, monkeypatch) -> None:
    from theHarvester.lib.api import api

    monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key')
    monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
    monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled')

    with TestClient(api.app) as client:
        schema = client.get('/openapi.json').json()

    paths = schema['paths']
    assert paths['/api/v1/sources']['get']['responses']['200']['content']['application/json']['schema'] == {
        '$ref': '#/components/schemas/SourceCatalogResponse'
    }
    assert paths['/api/v1/runs']['get']['responses']['200']['content']['application/json']['schema']['items'] == {
        '$ref': '#/components/schemas/RunSummary'
    }
    for path, method in (
        ('/api/v1/runs', 'post'),
        ('/api/v1/runs/import', 'post'),
        ('/api/v1/runs/{run_id}', 'get'),
        ('/api/v1/runs/{run_id}/cancel', 'post'),
    ):
        assert paths[path][method]['responses']['201' if path in {'/api/v1/runs', '/api/v1/runs/import'} else '200']['content'][
            'application/json'
        ]['schema'] == {'$ref': '#/components/schemas/RunDetail'}


def test_source_catalog_exposes_shared_action_activities(tmp_path, monkeypatch) -> None:
    from theHarvester.lib.api import api

    monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key')
    monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
    monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled')

    with TestClient(api.app) as client:
        response = client.get('/api/v1/sources', headers={'X-API-Key': 'test-key'})

    assert response.status_code == 200
    catalog = response.json()
    assert catalog['sources']
    assert catalog['actions'] == [
        {'name': 'api-scan', 'activity': 'P2'},
        {'name': 'dns-brute', 'activity': 'P1'},
        {'name': 'dns-lookup', 'activity': 'P1'},
        {'name': 'dns-recursive', 'activity': 'P1'},
        {'name': 'dns-resolve', 'activity': 'P1'},
        {'name': 'screenshot', 'activity': 'P2'},
        {'name': 'shodan', 'activity': 'P0'},
        {'name': 'takeover', 'activity': 'P2'},
    ]


def test_openapi_explains_scope_and_execution_controls(tmp_path, monkeypatch) -> None:
    from theHarvester.lib.api import api

    monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key')
    monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
    monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled')

    with TestClient(api.app) as client:
        schema = client.get('/openapi.json').json()

    request_body = schema['paths']['/api/v1/runs']['post']['requestBody']
    properties = request_body['content']['application/json']['schema']['properties']

    assert request_body['required'] is True
    assert 'union' in properties['sources']['description']
    assert 'do not filter' in properties['sources']['description']
    assert '/24' in properties['dns_lookup']['description']
    assert 'whole run' in properties['deadline_seconds']['description']
    assert 'three resolver' in properties['dns_recursive_query_limit']['description']
    assert 'discovery sources' in properties['proxies']['description']
    assert 'configured proxies' in properties['takeover']['description']
    assert 'take_over' not in properties
    assert 'endpoint paths' in properties['api_scan_paths']['description']
    import_content = schema['paths']['/api/v1/runs/import']['post']['requestBody']['content']
    assert set(import_content) == {'application/x-ndjson'}
    export_content = schema['paths']['/api/v1/runs/{run_id}/export']['get']['responses']['200']['content']
    assert set(export_content) == {'application/x-ndjson'}
    assert set(schema['components']['schemas']['NormalizedResult']['properties']) == {
        'type',
        'value',
        'sources',
        'actions',
    }
    assert export_content['application/x-ndjson']['schema']['description'] == (
        'UTF-8 JSONL with one summary followed by normalized findings.'
    )

    def references(value):
        if isinstance(value, dict):
            if '$ref' in value:
                yield value['$ref']
            for child in value.values():
                yield from references(child)
        elif isinstance(value, list):
            for child in value:
                yield from references(child)

    components = schema['components']['schemas']
    for reference in references(schema):
        assert reference.startswith('#/components/schemas/')
        assert reference.removeprefix('#/components/schemas/') in components


def test_run_detail_exposes_one_normalized_evidence_surface(tmp_path, monkeypatch) -> None:
    from theHarvester.lib.api import api

    monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key')
    monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
    monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled')

    with TestClient(api.app) as client:
        imported = client.post(
            '/api/v1/runs/import',
            params={'filename': 'result.jsonl'},
            headers={'X-API-Key': 'test-key'},
            content=_jsonl_result(),
        )

    assert imported.status_code == 201
    assert 'evidence' not in imported.json()


def test_api_scan_can_run_without_discovery_sources(tmp_path, monkeypatch) -> None:
    from pydantic import ValidationError

    from theHarvester.lib.api.run_models import RunRequest

    request = RunRequest(target='example.test', sources=[], api_scan=True, api_scan_paths=['/api/v2', '/health'])

    assert request.api_scan is True
    assert request.api_scan_paths == ['/api/v2', '/health']
    with pytest.raises(ValidationError):
        RunRequest(target='example.test', sources=[], api_scan=True, api_scan_paths=['https://other.example/api'])


def test_fresh_api_uses_catalog_takeover_name_and_rejects_unknown_fields() -> None:
    from pydantic import ValidationError

    from theHarvester.lib.api.run_models import RunRequest

    request = RunRequest(target='example.test', sources=[], takeover=True)

    assert request.takeover is True
    with pytest.raises(ValidationError):
        RunRequest(target='example.test', sources=[], take_over=True)


@pytest.mark.parametrize(
    ('evidence_status', 'execution_status'),
    [('complete', 'failed'), ('partial', 'completed')],
)
def test_api_rejects_evidence_status_that_disagrees_with_executions(
    tmp_path,
    monkeypatch,
    evidence_status,
    execution_status,
) -> None:
    from theHarvester.lib.api import api

    monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key')
    monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
    monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled')
    payload = _jsonl_result(
        summary_fields={
            'evidence_status': evidence_status,
            'source_executions': [
                {
                    'source': 'crtsh',
                    'status': execution_status,
                    'duration_ms': 1,
                    'result_count': 0,
                    'error_type': 'RuntimeError',
                    'stop_reason': 'provider-error',
                }
            ],
        },
        finding_fields={'sources': []},
    )

    with TestClient(api.app) as client:
        response = client.post(
            '/api/v1/runs/import',
            params={'filename': 'inconsistent.jsonl'},
            headers={'X-API-Key': 'test-key'},
            content=payload,
        )

    assert response.status_code == 400
    assert response.json()['detail'] == 'Evidence status does not match its execution outcomes'


def test_api_preserves_sparse_failed_status_without_executions(tmp_path, monkeypatch) -> None:
    from theHarvester.lib.api import api

    database = tmp_path / 'runs.sqlite'
    monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key')
    monkeypatch.setenv('THEHARVESTER_RUN_DB', str(database))
    monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled')
    payload = _jsonl_result(summary_fields={'evidence_status': 'failed'})

    with TestClient(api.app) as client:
        imported = client.post(
            '/api/v1/runs/import',
            params={'filename': 'failed.jsonl'},
            headers={'X-API-Key': 'test-key'},
            content=payload,
        )
        exported = client.get(f'/api/v1/runs/{imported.json()["run_id"]}/export', headers={'X-API-Key': 'test-key'})

    assert imported.status_code == 201
    assert imported.json()['evidence_status'] == 'failed'
    assert json.loads(exported.text.splitlines()[0])['evidence_status'] == 'failed'


def test_api_import_and_export_accept_only_jsonl(tmp_path, monkeypatch) -> None:
    from theHarvester.lib.api import api

    monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key')
    monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
    monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled')
    headers = {'X-API-Key': 'test-key'}

    with TestClient(api.app, client=('127.0.0.3', 50000)) as client:
        rejected = client.post(
            '/api/v1/runs/import',
            params={'filename': 'legacy.json'},
            headers=headers,
            content='{"target":"example.test"}',
        )
        imported = client.post(
            '/api/v1/runs/import',
            params={'filename': 'result.jsonl'},
            headers={**headers, 'Content-Type': 'application/x-ndjson'},
            content=_jsonl_result(),
        )
        exported = client.get(f'/api/v1/runs/{imported.json()["run_id"]}/export', headers=headers)
        reimported = client.post(
            '/api/v1/runs/import',
            params={'filename': 'round-trip.jsonl'},
            headers={**headers, 'Content-Type': 'application/x-ndjson'},
            content=exported.content,
        )
        old_json = client.get(f'/api/v1/runs/{imported.json()["run_id"]}/exports/json', headers=headers)
        old_csv = client.get(f'/api/v1/runs/{imported.json()["run_id"]}/exports/csv', headers=headers)

    assert rejected.status_code == 400
    assert rejected.json()['detail'] == 'Choose a .jsonl result file'
    assert imported.status_code == 201
    assert exported.status_code == 200
    assert exported.headers['content-type'] == 'application/x-ndjson'
    assert exported.headers['content-disposition'].endswith('.jsonl"')
    records = [json.loads(line) for line in exported.text.splitlines()]
    assert 'schema' not in records[0]
    assert 'schema_version' not in records[0]
    assert records[0]['type'] == 'summary'
    assert records[0]['target'] == 'example.test'
    assert records[1] == {'sources': [], 'type': 'email', 'value': 'a@example.test'}
    assert reimported.status_code == 201
    assert reimported.json()['results'] == [{'type': 'email', 'value': 'a@example.test', 'sources': [], 'actions': []}]
    assert old_json.status_code == 404
    assert old_csv.status_code == 404


def test_api_jsonl_round_trip_preserves_canonical_url_sources(tmp_path, monkeypatch) -> None:
    from theHarvester.lib.api import api

    monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key')
    monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
    monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled')
    headers = {'X-API-Key': 'test-key', 'Content-Type': 'application/x-ndjson'}
    sources = ['builtwith', 'gitlab', 'rocketreach']
    executions = [
        {
            'source': source,
            'status': 'completed',
            'duration_ms': 1,
            'result_count': 1,
            'error_type': None,
            'stop_reason': None,
        }
        for source in sources
    ]
    payload = _jsonl_result(
        finding_type='url',
        value='https://example.test/profile',
        finding_fields={'sources': sources},
        summary_fields={'source_executions': executions},
    )

    with TestClient(api.app) as client:
        imported = client.post(
            '/api/v1/runs/import',
            params={'filename': 'urls.jsonl'},
            headers=headers,
            content=payload,
        )
        run_id = imported.json()['run_id']
        detail = client.get(f'/api/v1/runs/{run_id}', headers={'X-API-Key': 'test-key'})
        exported = client.get(f'/api/v1/runs/{run_id}/export', headers={'X-API-Key': 'test-key'})

    assert imported.status_code == 201
    assert detail.json()['results'] == [
        {
            'type': 'url',
            'value': 'https://example.test/profile',
            'sources': sources,
            'actions': [],
        }
    ]
    records = [json.loads(line) for line in exported.text.splitlines()]
    assert records[1] == {'sources': sources, 'type': 'url', 'value': 'https://example.test/profile'}


def test_api_database_import_rejects_non_sqlite_content(tmp_path, monkeypatch) -> None:
    from theHarvester.lib.api import api

    monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key')
    monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
    monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled')

    with TestClient(api.app) as client:
        response = client.post(
            '/api/v1/runs/import-database',
            params={'filename': 'results.sqlite'},
            headers={'X-API-Key': 'test-key', 'Content-Type': 'application/vnd.sqlite3'},
            content=b'not a sqlite database',
        )

    assert response.status_code == 400
    assert response.json()['detail'] == 'Uploaded file is not a SQLite database'


def test_api_database_import_exposes_completed_cli_runs(tmp_path, monkeypatch) -> None:
    from datetime import UTC, datetime

    from theHarvester.lib.api import api
    from theHarvester.lib.completed_result import CompletedResult
    from theHarvester.lib.database import ResultStore, dispose_sqlite_databases

    source_database = tmp_path / 'source.sqlite'
    destination_database = tmp_path / 'destination.sqlite'
    now = datetime.now(UTC)
    completed = CompletedResult.finish(
        target='imported.example.test',
        started_at=now,
        completed_at=now,
        groups={'hostname': ['api.imported.example.test']},
    )

    async def seed() -> None:
        store = ResultStore(source_database)
        await store.initialize()
        await store.save_run(completed)
        await dispose_sqlite_databases()

    asyncio.run(seed())
    monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key')
    monkeypatch.setenv('THEHARVESTER_RUN_DB', str(destination_database))
    monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled')

    with TestClient(api.app) as client:
        imported = client.post(
            '/api/v1/runs/import-database',
            params={'filename': 'source.sqlite'},
            headers={'X-API-Key': 'test-key', 'Content-Type': 'application/vnd.sqlite3'},
            content=source_database.read_bytes(),
        )
        detail = client.get(
            f'/api/v1/runs/{completed.run_id}',
            headers={'X-API-Key': 'test-key'},
        )

    assert imported.status_code == 201
    assert imported.json()['imported_run_ids'] == [str(completed.run_id)]
    assert detail.status_code == 200
    assert detail.json()['results'] == [{'type': 'hostname', 'value': 'api.imported.example.test', 'sources': [], 'actions': []}]


def test_api_jsonl_round_trip_preserves_source_attribution(tmp_path, monkeypatch) -> None:
    from theHarvester.lib.api import api

    source_execution = {
        'source': 'crtsh',
        'status': 'completed',
        'duration_ms': 0,
        'result_count': 1,
        'error_type': None,
        'stop_reason': None,
    }
    monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key')
    monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
    monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled')
    headers = {'X-API-Key': 'test-key'}

    with TestClient(api.app, client=('127.0.0.4', 50000)) as client:
        imported = client.post(
            '/api/v1/runs/import',
            params={'filename': 'complete.jsonl'},
            headers=headers,
            content=_jsonl_result(
                finding_fields={'sources': ['crtsh']},
                summary_fields={'source_executions': [source_execution]},
            ),
        )
        exported = client.get(f'/api/v1/runs/{imported.json()["run_id"]}/export', headers=headers)
        reimported = client.post(
            '/api/v1/runs/import',
            params={'filename': 'complete-round-trip.jsonl'},
            headers=headers,
            content=exported.content,
        )

    summary = json.loads(exported.text.splitlines()[0])
    assert imported.json()['evidence_status'] == 'complete'
    assert imported.json()['source_executions'] == [source_execution]
    assert imported.json()['results'] == [{'type': 'email', 'value': 'a@example.test', 'sources': ['crtsh'], 'actions': []}]
    assert summary['run_id'] == imported.json()['run_id']
    assert summary['evidence_status'] == 'complete'
    assert summary['source_executions'] == [source_execution]
    assert json.loads(exported.text.splitlines()[1])['sources'] == ['crtsh']
    assert reimported.json()['evidence_status'] == 'complete'
    assert reimported.json()['request']['source_run_id'] == imported.json()['run_id']
    assert reimported.json()['source_executions'] == [source_execution]


def test_api_jsonl_export_uses_evidence_timestamps_not_lifecycle_timestamps(tmp_path, monkeypatch) -> None:
    from theHarvester.lib.api import api
    from theHarvester.lib.api.run_models import RunRequest
    from theHarvester.lib.api.run_store import RunStore

    monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key')
    monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
    monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled')
    store = RunStore()
    queued = asyncio.run(store.create(RunRequest(target='example.test', sources=['crtsh'])))
    asyncio.run(store.claim_next())
    asyncio.run(
        store.finish(
            queued['run_id'],
            {
                'run_id': '3e7cf0c1-214b-4429-80ba-058b2cb68b06',
                'target': 'example.test',
                'status': 'complete',
                'started_at': '2026-08-07T01:00:00Z',
                'completed_at': '2026-08-07T01:01:00Z',
                'results': [],
                'source_executions': [],
            },
            '',
        )
    )

    with TestClient(api.app, client=('127.0.0.16', 50000)) as client:
        response = client.get(
            f'/api/v1/runs/{queued["run_id"]}/export',
            headers={'X-API-Key': 'test-key'},
        )

    summary = json.loads(response.text.splitlines()[0])
    completed = asyncio.run(store.load_completed_result(queued['run_id']))
    assert completed is not None
    assert response.text == completed.jsonl()
    assert summary['started_at'] == '2026-08-07T01:00:00Z'
    assert summary['completed_at'] == '2026-08-07T01:01:00Z'


def test_api_jsonl_export_uses_lifecycle_timestamps_for_sparse_partial_evidence(tmp_path, monkeypatch) -> None:
    from theHarvester.lib.api import api
    from theHarvester.lib.api.run_models import RunRequest
    from theHarvester.lib.api.run_store import RunStore

    monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key')
    monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
    monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled')
    store = RunStore()
    queued = asyncio.run(store.create(RunRequest(target='example.test', sources=['crtsh'])))
    asyncio.run(store.claim_next())
    asyncio.run(
        store.fail(
            queued['run_id'],
            'Provider process exited.',
            '',
            evidence={
                'run_id': 'eb470313-d813-4d81-bd75-c1221a8bc00e',
                'target': 'example.test',
                'status': 'partial',
                'results': [],
                'source_executions': [],
            },
        )
    )

    with TestClient(api.app, client=('127.0.0.17', 50000)) as client:
        exported = client.get(
            f'/api/v1/runs/{queued["run_id"]}/export',
            headers={'X-API-Key': 'test-key'},
        )
        reimported = client.post(
            '/api/v1/runs/import',
            params={'filename': 'partial.jsonl'},
            headers={'X-API-Key': 'test-key'},
            content=exported.content,
        )

    summary = json.loads(exported.text.splitlines()[0])
    assert exported.status_code == 200
    assert isinstance(summary['started_at'], str)
    assert isinstance(summary['completed_at'], str)
    assert reimported.status_code == 201
    assert summary['evidence_status'] == 'partial'
    assert reimported.json()['evidence_status'] == 'partial'


def test_api_jsonl_round_trip_uses_canonical_hostname_and_ip_kinds(tmp_path, monkeypatch) -> None:
    from theHarvester.lib.api import api

    monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key')
    monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
    monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled')
    headers = {'X-API-Key': 'test-key'}

    for client_ip, finding_type, value, run_id in (
        ('127.0.0.5', 'hostname', 'www.example.test', '0f17b751-dd31-46da-968f-31580e233b72'),
        ('127.0.0.6', 'ip', '192.0.2.1', 'f7419165-d78c-4aef-9023-e9686f864ff0'),
    ):
        with TestClient(api.app, client=(client_ip, 50000)) as client:
            imported = client.post(
                '/api/v1/runs/import',
                params={'filename': f'{finding_type}.jsonl'},
                headers=headers,
                content=_jsonl_result(finding_type=finding_type, value=value, summary_fields={'run_id': run_id}),
            )
            exported = client.get(f'/api/v1/runs/{imported.json()["run_id"]}/export', headers=headers)
            reimported = client.post(
                '/api/v1/runs/import',
                params={'filename': f'{finding_type}-round-trip.jsonl'},
                headers=headers,
                content=exported.content,
            )

        assert imported.json()['results'] == [{'type': finding_type, 'value': value, 'sources': [], 'actions': []}]
        assert json.loads(exported.text.splitlines()[1]) == {'sources': [], 'type': finding_type, 'value': value}
        assert reimported.json()['results'] == [{'type': finding_type, 'value': value, 'sources': [], 'actions': []}]


def test_api_jsonl_round_trip_preserves_execution_outcomes_and_action_origins(tmp_path, monkeypatch) -> None:
    from theHarvester.lib.api import api

    monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key')
    monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
    monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled')
    headers = {'X-API-Key': 'test-key'}
    payload = _jsonl_result(
        finding_type='hostname',
        value='api.example.test',
        finding_fields={'sources': ['crtsh', 'crtsh'], 'actions': ['dns-brute', 'dns-brute']},
        summary_fields={
            'evidence_status': 'partial',
            'source_executions': [
                {
                    'source': 'crtsh',
                    'status': 'completed',
                    'duration_ms': 1,
                    'result_count': 1,
                    'error_type': None,
                    'stop_reason': None,
                },
                {
                    'source': 'certspotter',
                    'status': 'rate-limited',
                    'duration_ms': 2,
                    'result_count': 0,
                    'error_type': None,
                    'stop_reason': 'http-429',
                },
            ],
            'action_executions': [
                {
                    'action': 'dns-brute',
                    'status': 'partial',
                    'duration_ms': 3,
                    'result_count': 1,
                    'error_type': 'TimeoutError',
                    'stop_reason': 'query-errors',
                }
            ],
        },
    )

    with TestClient(api.app, client=('127.0.0.18', 50000)) as client:
        imported = client.post(
            '/api/v1/runs/import',
            params={'filename': 'attributed.jsonl'},
            headers=headers,
            content=payload,
        )
        exported = client.get(f'/api/v1/runs/{imported.json()["run_id"]}/export', headers=headers)
        reimported = client.post(
            '/api/v1/runs/import',
            params={'filename': 'attributed-round-trip.jsonl'},
            headers=headers,
            content=exported.content,
        )

    assert imported.status_code == 201
    assert exported.status_code == 200
    assert reimported.status_code == 201
    assert reimported.json()['evidence_status'] == 'partial'
    assert reimported.json()['source_executions'] == imported.json()['source_executions']
    assert reimported.json()['action_executions'] == imported.json()['action_executions']
    assert reimported.json()['results'] == [
        {
            'type': 'hostname',
            'value': 'api.example.test',
            'sources': ['crtsh'],
            'actions': ['dns-brute'],
        }
    ]


def test_api_rejects_non_string_jsonl_timestamps(tmp_path, monkeypatch) -> None:
    from theHarvester.lib.api import api

    monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key')
    monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
    monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled')

    with TestClient(api.app, client=('127.0.0.7', 50000)) as client:
        response = client.post(
            '/api/v1/runs/import',
            params={'filename': 'invalid.jsonl'},
            headers={'X-API-Key': 'test-key'},
            content=_jsonl_result(summary_fields={'completed_at': {'not': 'a timestamp'}}),
        )

    assert response.status_code == 400
    assert response.json()['detail'] == 'JSONL summary must contain an ISO-8601 UTC completed_at'


def test_api_rejects_invalid_jsonl_summary_identity_and_timestamps(tmp_path, monkeypatch) -> None:
    from theHarvester.lib.api import api

    monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key')
    monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
    monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled')
    cases = (
        ({'run_id': None}, None, 'JSONL summary must contain a UUID run_id'),
        ({}, 'run_id', 'JSONL summary must contain a UUID run_id'),
        ({'run_id': 'not-a-uuid'}, None, 'JSONL summary must contain a UUID run_id'),
        ({}, 'started_at', 'JSONL summary must contain an ISO-8601 UTC started_at'),
        ({}, 'completed_at', 'JSONL summary must contain an ISO-8601 UTC completed_at'),
        ({'started_at': 'not-a-time'}, None, 'JSONL summary must contain an ISO-8601 UTC started_at'),
        (
            {'completed_at': '2026-08-08T02:01:00+01:00'},
            None,
            'JSONL summary must contain an ISO-8601 UTC completed_at',
        ),
        (
            {'started_at': '2026-08-08T03:00:00Z', 'completed_at': '2026-08-08T02:00:00Z'},
            None,
            'JSONL summary completed_at must not be earlier than started_at',
        ),
    )

    for index, (updates, removed_field, detail) in enumerate(cases, start=8):
        records = [json.loads(line) for line in _jsonl_result().splitlines()]
        records[0].update(updates)
        if removed_field:
            records[0].pop(removed_field)
        content = ''.join(json.dumps(record) + '\n' for record in records)
        with TestClient(api.app, client=(f'127.0.0.{index}', 50000)) as client:
            response = client.post(
                '/api/v1/runs/import',
                params={'filename': 'invalid.jsonl'},
                headers={'X-API-Key': 'test-key'},
                content=content,
            )

        assert response.status_code == 400
        assert response.json()['detail'] == detail


def test_api_refuses_to_export_before_evidence_exists(tmp_path, monkeypatch) -> None:
    from theHarvester.lib.api import api
    from theHarvester.lib.api.run_models import RunRequest
    from theHarvester.lib.api.run_store import RunStore

    monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key')
    monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
    monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled')
    run = asyncio.run(RunStore().create(RunRequest(target='example.test', sources=['crtsh'])))

    with TestClient(api.app) as client:
        response = client.get(f'/api/v1/runs/{run["run_id"]}/export', headers={'X-API-Key': 'test-key'})

    assert response.status_code == 409
    assert response.json()['detail'] == 'No run evidence is available to export'