diff --git a/README.md b/README.md index 50fe88b0..d9f9fcc6 100644 --- a/README.md +++ b/README.md @@ -20,14 +20,24 @@ It is built for the early reconnaissance stage of authorized security assessment Source availability, quotas, and response formats are controlled by third parties and can change independently of theHarvester. -![theHarvester run evidence architecture](docs/images/run-evidence-architecture.svg) - -Open the diagram in a dedicated browser view with [`docs/diagrams/run-evidence-architecture.html`](docs/diagrams/run-evidence-architecture.html). - ## Package versions [![Packaging status](https://repology.org/badge/vertical-allrepos/theharvester.svg)](https://repology.org/project/theharvester/versions) +## Architecture at a glance + +### Discovery routes and enrichment + +![theHarvester discovery routes and enrichment](docs/images/run-evidence-architecture.svg) + +Open the diagram in a dedicated browser view with [`docs/diagrams/run-evidence-architecture.html`](docs/diagrams/run-evidence-architecture.html). + +### HarvestView run desk + +![HarvestView run desk architecture](docs/images/harvestview-architecture.svg) + +Open the diagram in a dedicated browser view with [`docs/diagrams/harvestview-architecture.html`](docs/diagrams/harvestview-architecture.html). + ## Quick start theHarvester requires Python 3.12 or newer. From a source checkout: diff --git a/docs/diagrams/harvestview-architecture.html b/docs/diagrams/harvestview-architecture.html new file mode 100644 index 00000000..cd0039eb --- /dev/null +++ b/docs/diagrams/harvestview-architecture.html @@ -0,0 +1,131 @@ + + + + + + HarvestView run desk architecture + + + + +
+

Architecture · HarvestView

+

One local run desk, one durable lifecycle

+ + HarvestView run desk architecture + HarvestView validates local browser requests through the authenticated API, records one durable run, executes it in an isolated worker, and returns normalized evidence to the route workbench. + + + + + + + + + HARVESTVIEW · LOCAL OPERATOR WORKFLOW + One run desk, one durable lifecycle + The browser reviews scope; the API validates; an isolated worker executes; evidence remains available after terminal state. + + + + LOCAL HOST · 127.0.0.1 + + + + + + + + + + + BROWSER + Create or import + new run · JSONL · SQLite + source and action review + P0 · P1 · P2 authorization + + + + API GATE + Authenticated REST API + derived HttpOnly browser session + X-API-Key for API clients + target · scope · options · limits + + + + CONTROL + Durable run control + queued → running → terminal + cancel request · source status + one record per finite run + + + + WORKER + Isolated run worker + claim · checkpoint · cancel + bounded output · child cleanup + + + + ENGINE + theHarvester engine + cataloged sources · explicit actions + DNS · Shodan · RouteViews · vhost + + + + EVIDENCE + Evidence & artifacts + CompletedResult · route records + SQLite · screenshots · logs + partial evidence remains inspectable + + + + WORKBENCH + Inspect and export + history · assessment · route tabs + screenshots · source outcomes · logs + review child actions · export JSONL + + + + control and lifecycle handoff + + normalized evidence handoff + + durable evidence boundary + provider credentials remain server-side + +
+ + diff --git a/docs/diagrams/run-evidence-architecture.html b/docs/diagrams/run-evidence-architecture.html index 842f0afd..41ae772b 100644 --- a/docs/diagrams/run-evidence-architecture.html +++ b/docs/diagrams/run-evidence-architecture.html @@ -3,20 +3,157 @@ - One run, one evidence model + Discovery routes and enrichment + -theHarvester run architecture: operator interfaces feed a validated run contract and finite execution pipeline, which produces normalized evidence for durable storage and operator views. +
+

Architecture · theHarvester

+

Discovery routes and enrichment

+ + Discovery routes and enrichment + theHarvester source selection and adapters feed passive, DNS, and direct actions before normalization into identity, contact, network, and action evidence routes. + + + + + + + + + DISCOVERY · ENRICHMENT · EVIDENCE + Sources become attributable evidence + Catalog policy selects the work; one normalized result preserves routes, producers, outcomes, and partial evidence. + + + + COLLECT + + + ACTIONS + + + NORMALIZE & ROUTE + + + + + + + + + + + + + + CATALOG + Source selection + subdomains · emails · IPs + ASNs · URLs · people + breaches · explicit names + activity · keys · routes + + + + SOURCES + Discovery adapters + search · certificates · DNS + code · archives · threat intel + normalized observations + truthful source outcomes + + + + P0 + Passive enrichment + Shodan host detail + RouteViews routes · origins · RPKI + + + + P1 + DNS interaction + resolve · reverse lookup + brute force · recursive discovery + + + + P2 + Direct interaction + vhost · screenshots · takeover + API path discovery + + + + NORMALIZED + CompletedResult + dedupe · provenance · outcomes + artifacts · evidence status + partial evidence stays attributable + + + + IDENTITY + Identity & contact routes + hostname / subdomain + email · URL + person · person-link · breach + vhost observations stay on hostname + + + + NETWORK + Network & action routes + IP · ASN · prefix + Shodan host · takeover + scope extension + external relationship · other + JSONL · SQLite · REST · HarvestView + + + + data handoff + + normalized evidence + + P1 · DNS interaction + + P2 · direct interaction + P0 sources and enrichments remain passive provider requests + +
diff --git a/docs/images/harvestview-architecture.svg b/docs/images/harvestview-architecture.svg new file mode 100644 index 00000000..b81aa1d1 --- /dev/null +++ b/docs/images/harvestview-architecture.svg @@ -0,0 +1,109 @@ + + + HarvestView run desk architecture + HarvestView validates local browser requests through the authenticated API, records one durable run, executes it in an isolated worker, and returns normalized evidence to the route workbench. + + + + + + + + + + HARVESTVIEW · LOCAL OPERATOR WORKFLOW + One run desk, one durable lifecycle + The browser reviews scope; the API validates; an isolated worker executes; evidence remains available after terminal state. + + + + LOCAL HOST · 127.0.0.1 + + + + + + + + + + + BROWSER + Create or import + new run · JSONL · SQLite + source and action review + P0 · P1 · P2 authorization + + + + API GATE + Authenticated REST API + derived HttpOnly browser session + X-API-Key for API clients + target · scope · options · limits + + + + CONTROL + Durable run control + queued → running → terminal + cancel request · source status + one record per finite run + + + + WORKER + Isolated run worker + claim · checkpoint · cancel + bounded output · child cleanup + + + + ENGINE + theHarvester engine + cataloged sources · explicit actions + DNS · Shodan · RouteViews · vhost + + + + EVIDENCE + Evidence & artifacts + CompletedResult · route records + SQLite · screenshots · logs + partial evidence remains inspectable + + + + WORKBENCH + Inspect and export + history · assessment · route tabs + screenshots · source outcomes · logs + review child actions · export JSONL + + + + control and lifecycle handoff + + normalized evidence handoff + + durable evidence boundary + provider credentials remain server-side + \ No newline at end of file diff --git a/docs/images/run-evidence-architecture.svg b/docs/images/run-evidence-architecture.svg index fc69a776..61da2c73 100644 --- a/docs/images/run-evidence-architecture.svg +++ b/docs/images/run-evidence-architecture.svg @@ -1,97 +1,137 @@ - - One run, one evidence model - theHarvester architecture showing CLI, HarvestView, and REST requests moving through a validated run contract, the source catalog, and a finite enumeration pipeline, then into one normalized CompletedResult that feeds durable JSONL and SQLite evidence plus terminal, API, and HarvestView views. - - - - - - + + Discovery routes and enrichment + theHarvester source selection and adapters feed passive, DNS, and direct actions before normalization into identity, contact, network, and action evidence routes. + + + + + + + - - THEHARVESTER ARCHITECTURE - One run, one evidence model - Every interface and activity class converges on the same normalized result. + + DISCOVERY · ENRICHMENT · EVIDENCE + Sources become attributable evidence + Catalog policy selects the work; one normalized result preserves routes, producers, outcomes, and partial evidence. - - - CONTROL + + + COLLECT + + + ACTIONS + + + NORMALIZE & ROUTE - - - EXECUTION + + + + + + + + + - - - EVIDENCE + + + CATALOG + Source selection + subdomains · emails · IPs + ASNs · URLs · people + breaches · explicit names + activity · keys · routes - - - - - - + + + SOURCES + Discovery adapters + search · certificates · DNS + code · archives · threat intel + normalized observations + truthful source outcomes - - ENTRY POINTS - Operator interfaces - CLI · HarvestView · REST - one finite run + + + P0 + Passive enrichment + Shodan host detail + RouteViews routes · origins · RPKI - - SHARED INPUT - Validated run contract - target · sources · limits · activity controls + + + P1 + DNS interaction + resolve · reverse lookup + brute force · recursive discovery - - REGISTRY - Source catalog - routes · keys · class + + + P2 + Direct interaction + vhost · screenshots · takeover + API path discovery - - FINITE WORK - Enumeration pipeline - - Selected sources - P0 · P1 · P2 by catalog policy - Explicit actions - RouteViews · DNS · direct + + + NORMALIZED + CompletedResult + dedupe · provenance · outcomes + artifacts · evidence status + partial evidence stays attributable - - NORMALIZED RECORD - CompletedResult - findings · origins · outcomes - artifacts · evidence status - partial evidence stays attributable + + + IDENTITY + Identity & contact routes + hostname / subdomain + email · URL + person · person-link · breach + vhost observations stay on hostname - - OUTPUTS - Durable · JSONL · SQLite - Views · terminal · API - HarvestView + + + NETWORK + Network & action routes + IP · ASN · prefix + Shodan host · takeover + scope extension + external relationship · other + JSONL · SQLite · REST · HarvestView - - - internal handoff - - evidence boundary - - operator output - P0 provider · P1 DNS · P2 direct interaction - + + + data handoff + + normalized evidence + + P1 · DNS interaction + + P2 · direct interaction + P0 sources and enrichments remain passive provider requests + \ No newline at end of file diff --git a/tests/test_readme.py b/tests/test_readme.py index 431af910..e43f6774 100644 --- a/tests/test_readme.py +++ b/tests/test_readme.py @@ -116,24 +116,36 @@ def test_wiki_navigation_and_readme_links_resolve() -> None: assert all(Path(target).is_file() for target in readme_wiki_links) -def test_readme_architecture_diagram_is_local_and_accessible() -> None: +def test_readme_architecture_diagrams_are_local_and_accessible() -> None: readme = Path('README.md').read_text() - html = Path('docs/diagrams/run-evidence-architecture.html') - svg = Path('docs/images/run-evidence-architecture.svg') - html_text = html.read_text() - svg_text = svg.read_text() + diagrams = ( + ( + 'theHarvester discovery routes and enrichment', + Path('docs/diagrams/run-evidence-architecture.html'), + Path('docs/images/run-evidence-architecture.svg'), + 'run-evidence-architecture', + ('subdomains · emails · IPs', 'Shodan host detail', 'RouteViews routes', 'vhost · screenshots · takeover'), + ), + ( + 'HarvestView run desk architecture', + Path('docs/diagrams/harvestview-architecture.html'), + Path('docs/images/harvestview-architecture.svg'), + 'harvestview-architecture', + ('Authenticated REST API', 'queued → running → terminal', 'Isolated run worker', 'export JSONL'), + ), + ) - assert f'![theHarvester run evidence architecture]({svg})' in readme - assert f'[`{html}`]({html})' in readme - assert 'src="../images/run-evidence-architecture.svg"' in html_text - assert '' in svg_text - assert '' in svg_text - assert 'P0 · P1 · P2 by catalog policy' in svg_text - assert 'Durable · JSONL · SQLite' in svg_text - assert 'Views · terminal · API' in svg_text - assert '>HarvestView' in svg_text + for alt, html, svg, slug, expected_text in diagrams: + html_text = html.read_text() + svg_text = svg.read_text() + assert f'![{alt}]({svg})' in readme + assert f'[`{html}`]({html})' in readme + assert '' in svg_text + assert f'' in svg_text + assert all(text in svg_text for text in expected_text) def test_virtual_host_wiki_examples_match_the_structured_result_contract() -> None: