diff --git a/README.md b/README.md index 78c7678b..27e12dbf 100644 --- a/README.md +++ b/README.md @@ -25,11 +25,11 @@ Providers control their own availability, quotas, and response formats, so indiv ### Discovery routes and enrichment -![theHarvester discovery routes and enrichment](docs/images/run-evidence-architecture.svg) +[![theHarvester discovery routes and enrichment](docs/images/run-evidence-architecture.svg)](docs/images/run-evidence-architecture.svg) ### HarvestView run desk -![HarvestView run desk architecture](docs/images/harvestview-architecture.svg) +[![HarvestView run desk architecture](docs/images/harvestview-architecture.svg)](docs/images/harvestview-architecture.svg) ## Quick start diff --git a/docs/images/harvestview-architecture.svg b/docs/images/harvestview-architecture.svg index a048f334..566eaf20 100644 --- a/docs/images/harvestview-architecture.svg +++ b/docs/images/harvestview-architecture.svg @@ -1,109 +1,158 @@ - - HarvestView run desk architecture - HarvestView validates local browser requests through the authenticated API, records one durable run, executes it in an isolated worker, and returns normalized evidence to the route workbench. - - - - - - - + + HarvestView run desk architecture + A loopback-only HarvestView workflow sends a reviewed plan through the authenticated REST API and durable run control to an isolated worker, preserves normalized evidence separately from lifecycle state, supports inspection and child actions, and imports or exports JSONL and SQLite evidence. + + + + + + - - HARVESTVIEW · LOCAL OPERATOR WORKFLOW - One run desk, one durable lifecycle - The browser reviews scope; the API validates; an isolated worker executes; evidence remains available after terminal state. + + HARVESTVIEW · LOCAL OPERATOR WORKFLOW + One run desk, one durable evidence path + The browser reviews scope; the API validates; an isolated worker executes; completed evidence remains portable. - - - LOCAL HOST · 127.0.0.1 + + + LOCALHOST · 127.0.0.1:5000 - - - - - - + + + + + + + - - - BROWSER - Create or import - new run · JSONL · SQLite - source and action review - P0 · P1 · P2 authorization + + + BROWSER + Plan and authorize + target · sources + actions · P0/P1/P2 + review before submit - - - API GATE - Authenticated REST API - derived HttpOnly browser session - X-API-Key for API clients - target · scope · options · limits + + + API GATE + Authenticated REST API + HttpOnly · X-API-Key + target · scope · limits + create · list + detail · cancel - - - CONTROL - Durable run control - queued → running → terminal - cancel request · source status - one record per finite run + + + EVIDENCE + Canonical evidence + results · provenance + source/action outcomes + artifacts · status + complete · partial · failed - - - WORKER - Isolated run worker - claim · checkpoint · cancel - bounded output · child cleanup + + + WORKBENCH + Inspect and act + history · assessment + route tabs · search + filters · copy · logs + screenshots · child runs - - - ENGINE - theHarvester engine - cataloged sources · explicit actions - DNS · Shodan · RouteViews · vhost + + + CONTROL + Durable run lifecycle + queued · running + cancelling · cancelled + completed · failed + leases · deadlines - - - EVIDENCE - Evidence & artifacts - CompletedResult · route records - SQLite · screenshots · logs - partial evidence remains inspectable + + + WORKER + Isolated run worker + claim · execute + checkpoint · cancel + 58 sources + 10 explicit actions - - - WORKBENCH - Inspect and export - history · assessment · route tabs - screenshots · source outcomes · logs - review actions · JSONL / SQLite export + + + INTERCHANGE + Import and export + JSONL · one run + SQLite · completed set + screenshot files separate - - - control and lifecycle handoff - - normalized evidence handoff - - durable evidence boundary - provider credentials remain server-side - + Provider credentials stay server-side. Lifecycle state never replaces terminal evidence status. + + + + run control + + evidence handoff + + import path + diff --git a/docs/images/run-evidence-architecture.svg b/docs/images/run-evidence-architecture.svg index 61da2c73..d4c0e6cf 100644 --- a/docs/images/run-evidence-architecture.svg +++ b/docs/images/run-evidence-architecture.svg @@ -1,137 +1,173 @@ - - Discovery routes and enrichment - theHarvester source selection and adapters feed passive, DNS, and direct actions before normalization into identity, contact, network, and action evidence routes. - - - - - - - + + theHarvester discovery and evidence architecture + Seven result capabilities and 58 cataloged discovery sources join ten explicit P0, P1, and P2 actions in one normalized evidence contract, which routes identity, network, and action findings to terminal, JSONL, SQLite, REST, HarvestView, screenshots, JSON, and XML outputs. + + + + + + - - DISCOVERY · ENRICHMENT · EVIDENCE - Sources become attributable evidence - Catalog policy selects the work; one normalized result preserves routes, producers, outcomes, and partial evidence. + + DISCOVERY · ENRICHMENT · ATTRIBUTABLE EVIDENCE + From source selection to durable evidence + Every selected source and action reports its own outcome before results are deduplicated, attributed, and exported. - - - COLLECT - - - ACTIONS - - - NORMALIZE & ROUTE + + + SELECT + EXECUTE + + + EVIDENCE ROUTES - - - - - - - - - + + + + + + + + - - - CATALOG - Source selection - subdomains · emails · IPs - ASNs · URLs · people - breaches · explicit names - activity · keys · routes + + + SCOPE + Scope and source + selection + 7 result capabilities + names · all · no-hosts - - - SOURCES - Discovery adapters - search · certificates · DNS - code · archives · threat intel - normalized observations - truthful source outcomes + + + SOURCES + 58 discovery adapters + search · CT · DNS data + code · archives + threat intelligence + activity · keys · outcomes - - - P0 - Passive enrichment - Shodan host detail - RouteViews routes · origins · RPKI + + + NORMALIZE + CompletedResult evidence contract + dedupe · provenance · source and action outcomes + complete · partial · failed evidence · artifacts - - - P1 - DNS interaction - resolve · reverse lookup - brute force · recursive discovery + + + P0 + Provider + enrichment + shodan + routeviews - - - P2 - Direct interaction - vhost · screenshots · takeover - API path discovery + + + P1 + DNS interaction + dns-resolve + dns-lookup + dns-brute + dns-recursive - - - NORMALIZED - CompletedResult - dedupe · provenance · outcomes - artifacts · evidence status - partial evidence stays attributable + + + P2 + Direct action + api-scan + screenshot + takeover + vhost - - - IDENTITY - Identity & contact routes - hostname / subdomain - email · URL - person · person-link · breach - vhost observations stay on hostname + + + IDENTITY + Identity and contact + subdomains · emails · urls + people · person links · breaches - - - NETWORK - Network & action routes - IP · ASN · prefix - Shodan host · takeover - scope extension - external relationship · other - JSONL · SQLite · REST · HarvestView + + + NETWORK + Network and action evidence + ips · asns · prefixes + Shodan hosts · takeover + recursive DNS · vhost evidence + scope extension · external relations - - - data handoff - - normalized evidence - - P1 · DNS interaction - - P2 · direct interaction - P0 sources and enrichments remain passive provider requests - \ No newline at end of file + + + OUTPUT + Durable outputs + Terminal · JSONL · SQLite · REST + HarvestView · screenshots · JSON/XML + + + + evidence flow + + P0 provider + + P1 DNS + + P2 direct + diff --git a/tests/test_readme.py b/tests/test_readme.py index acfa28aa..460b7282 100644 --- a/tests/test_readme.py +++ b/tests/test_readme.py @@ -6,7 +6,7 @@ from pathlib import Path import yaml -from theHarvester.lib.source_catalog import SOURCE_SPECS +from theHarvester.lib.source_catalog import ACTION_ACTIVITIES, RESULT_CAPABILITIES, SOURCE_SPECS OPTIONAL_API_KEY_SOURCES = {'hackertarget', 'mojeek', 'windvane'} API_KEY_SOURCE_ALIASES = { @@ -199,24 +199,34 @@ def test_readme_architecture_diagrams_are_local_and_accessible() -> None: 'theHarvester discovery routes and enrichment', Path('docs/images/run-evidence-architecture.svg'), 'run-evidence-architecture', - ('subdomains · emails · IPs', 'Shodan host detail', 'RouteViews routes', 'vhost · screenshots · takeover'), + ('58 discovery adapters', 'CompletedResult evidence contract', 'Terminal · JSONL · SQLite · REST'), ), ( 'HarvestView run desk architecture', Path('docs/images/harvestview-architecture.svg'), 'harvestview-architecture', - ('Authenticated REST API', 'queued → running → terminal', 'Isolated run worker', 'JSONL / SQLite export'), + ('Authenticated REST API', 'queued · running', 'Isolated run worker', 'JSONL · one run'), ), ) for alt, svg, slug, expected_text in diagrams: svg_text = svg.read_text() - assert f'![{alt}]({svg})' in readme + assert f'[![{alt}]({svg})]({svg})' in readme assert 'role="img"' in svg_text assert f'' in svg_text assert f'<desc id="{slug}-desc">' in svg_text + assert 'viewBox="0 0 960 640"' in svg_text + assert '@media (prefers-color-scheme: light)' in svg_text + assert "font: 600 16px 'Geist'" in svg_text assert all(text in svg_text for text in expected_text) + run_diagram = diagrams[0][1].read_text() + harvestview_diagram = diagrams[1][1].read_text() + assert all(capability in run_diagram for capability in RESULT_CAPABILITIES) + assert all(action in run_diagram for action in ACTION_ACTIVITIES) + assert f'{len(SOURCE_SPECS)} discovery adapters' in run_diagram + assert f'{len(ACTION_ACTIVITIES)} explicit actions' in harvestview_diagram + def test_wiki_diagrams_are_local_accessible_and_used_deliberately() -> None: raw_root = 'https://raw.githubusercontent.com/laramies/theHarvester/dev/'