docs: make README diagrams readable

This commit is contained in:
NotoriousRebel
2026-08-17 00:19:56 -04:00
parent d940ba25ba
commit 4c35f1c54f
4 changed files with 322 additions and 227 deletions
+2 -2
View File
@@ -25,11 +25,11 @@ Providers control their own availability, quotas, and response formats, so indiv
### Discovery routes and enrichment
![theHarvester discovery routes and enrichment](docs/images/run-evidence-architecture.svg)
[![theHarvester discovery routes and enrichment](docs/images/run-evidence-architecture.svg)](docs/images/run-evidence-architecture.svg)
### HarvestView run desk
![HarvestView run desk architecture](docs/images/harvestview-architecture.svg)
[![HarvestView run desk architecture](docs/images/harvestview-architecture.svg)](docs/images/harvestview-architecture.svg)
## Quick start
+146 -97
View File
@@ -1,109 +1,158 @@
<?xml version="1.0" encoding="UTF-8"?>
<svg viewBox="0 0 1280 720" xmlns="http://www.w3.org/2000/svg" role="img" aria-labelledby="harvestview-architecture-title harvestview-architecture-desc">
<title id="harvestview-architecture-title">HarvestView run desk architecture</title>
<desc id="harvestview-architecture-desc">HarvestView validates local browser requests through the authenticated API, records one durable run, executes it in an isolated worker, and returns normalized evidence to the route workbench.</desc>
<defs>
<style>@import url('https://fonts.googleapis.com/css2?family=Instrument+Serif:ital@0;1&amp;family=Geist:wght@400;500;600&amp;family=Geist+Mono:wght@400;500;600&amp;display=swap');</style>
<style>
.title { font: 400 28px "Instrument Serif", Georgia, serif; fill: #0e1d22; }
.subtitle { font: 400 12px "Geist", system-ui, sans-serif; fill: #4f5e63; }
.eyebrow { font: 600 8px "Geist Mono", ui-monospace, monospace; letter-spacing: .16em; fill: #4f5e63; }
.node-title { font: 600 12px "Geist", system-ui, sans-serif; fill: #0e1d22; }
.node-copy { font: 400 8px "Geist Mono", ui-monospace, monospace; fill: #4f5e63; }
.node-copy-strong { font: 500 8px "Geist Mono", ui-monospace, monospace; fill: #34464c; }
.boundary { fill: rgba(14,29,34,.018); stroke: #889ca3; stroke-width: 1; stroke-dasharray: 8 8; }
.node { fill: #fdfaf4; stroke: #889ca3; stroke-width: 1; }
.node-muted { fill: #ede7dd; stroke: #889ca3; stroke-width: 1; }
.node-focal { fill: #bcede4; stroke: #009685; stroke-width: 1.2; }
.tag { fill: #f5efe7; stroke: #889ca3; stroke-width: .8; }
.tag-teal { fill: #bcede4; stroke: #009685; stroke-width: .8; }
.tag-text { font: 600 8px "Geist Mono", ui-monospace, monospace; fill: #34464c; text-anchor: middle; }
.tag-text-teal { fill: #00594c; }
.connector { fill: none; stroke: #4f5e63; stroke-width: 1.2; }
.connector-accent { fill: none; stroke: #009685; stroke-width: 1.2; }
.legend { font: 400 8px "Geist Mono", ui-monospace, monospace; fill: #4f5e63; }
</style>
<marker id="harvestview-architecture-arrow" markerWidth="8" markerHeight="6" refX="7" refY="3" orient="auto"><polygon points="0 0, 8 3, 0 6" fill="#4f5e63"/></marker>
<marker id="harvestview-architecture-arrow-accent" markerWidth="8" markerHeight="6" refX="7" refY="3" orient="auto"><polygon points="0 0, 8 3, 0 6" fill="#009685"/></marker>
<marker id="harvestview-architecture-arrow-link" markerWidth="8" markerHeight="6" refX="7" refY="3" orient="auto"><polygon points="0 0, 8 3, 0 6" fill="#00548c"/></marker>
</defs>
<svg viewBox="0 0 960 640" xmlns="http://www.w3.org/2000/svg" role="img" aria-labelledby="harvestview-architecture-title harvestview-architecture-desc">
<title id="harvestview-architecture-title">HarvestView run desk architecture</title>
<desc id="harvestview-architecture-desc">A loopback-only HarvestView workflow sends a reviewed plan through the authenticated REST API and durable run control to an isolated worker, preserves normalized evidence separately from lifecycle state, supports inspection and child actions, and imports or exports JSONL and SQLite evidence.</desc>
<defs>
<style>
@import url('https://fonts.googleapis.com/css2?family=Instrument+Serif:ital@0;1&amp;family=Geist:wght@400;500;600&amp;family=Geist+Mono:wght@400;500;600&amp;display=swap');
:root {
--paper: #2d3142;
--paper-2: #393e53;
--ink: #f5f5f5;
--muted: #bfc0c0;
--soft: #8e98ac;
--rule: rgba(245,245,245,.16);
--rule-strong: rgba(245,245,245,.28);
--boundary-fill: rgba(245,245,245,.025);
--accent: #f08a59;
--accent-tint: rgba(240,138,89,.10);
--link: #6a95d8;
--link-tint: rgba(106,149,216,.10);
}
@media (prefers-color-scheme: light) {
:root {
--paper: #f5f5f5;
--paper-2: #ececec;
--ink: #2d3142;
--muted: #4f5d75;
--soft: #7a8399;
--rule: rgba(45,49,66,.14);
--rule-strong: rgba(45,49,66,.28);
--boundary-fill: rgba(45,49,66,.02);
--accent: #b84f24;
--accent-tint: rgba(235,108,54,.10);
--link: #2e5aa8;
--link-tint: rgba(46,90,168,.08);
}
}
.bg { fill: var(--paper); }
.title { font: 400 28px 'Instrument Serif', Georgia, serif; fill: var(--ink); }
.subtitle { font: 400 12px 'Geist', system-ui, sans-serif; fill: var(--muted); }
.eyebrow { font: 600 8px 'Geist Mono', ui-monospace, monospace; letter-spacing: .16em; fill: var(--soft); }
.boundary { fill: var(--boundary-fill); stroke: var(--rule-strong); stroke-width: 1; stroke-dasharray: 8 8; }
.boundary-mask { fill: var(--paper); }
.node { fill: var(--paper); stroke: var(--rule-strong); stroke-width: 1; }
.node-muted { fill: var(--paper-2); stroke: var(--rule-strong); stroke-width: 1; }
.node-link { fill: var(--link-tint); stroke: var(--link); stroke-width: 1; }
.node-focal { fill: var(--accent-tint); stroke: var(--accent); stroke-width: 1.2; }
.tag { fill: var(--paper-2); stroke: var(--rule-strong); stroke-width: .8; }
.tag-link { fill: var(--link-tint); stroke: var(--link); stroke-width: .8; }
.tag-accent { fill: var(--accent-tint); stroke: var(--accent); stroke-width: .8; }
.tag-text { font: 600 8px 'Geist Mono', ui-monospace, monospace; letter-spacing: .08em; fill: var(--muted); text-anchor: middle; }
.tag-text-link { fill: var(--link); }
.tag-text-accent { fill: var(--accent); }
.node-title { font: 600 16px 'Geist', system-ui, sans-serif; fill: var(--ink); }
.node-copy { font: 400 12px 'Geist Mono', ui-monospace, monospace; fill: var(--muted); }
.node-copy-strong { font: 500 12px 'Geist Mono', ui-monospace, monospace; fill: var(--ink); }
.connector { fill: none; stroke: var(--muted); stroke-width: 1.2; }
.connector-link { fill: none; stroke: var(--link); stroke-width: 1.2; }
.connector-accent { fill: none; stroke: var(--accent); stroke-width: 1.2; }
.dashed { stroke-dasharray: 4 4; }
.marker-muted { fill: var(--muted); }
.marker-link { fill: var(--link); }
.marker-accent { fill: var(--accent); }
.legend { font: 500 12px 'Geist Mono', ui-monospace, monospace; fill: var(--muted); }
</style>
<marker id="harvestview-arrow" markerWidth="8" markerHeight="8" refX="8" refY="4" orient="auto"><polygon points="0 0, 8 4, 0 8" class="marker-muted"/></marker>
<marker id="harvestview-arrow-link" markerWidth="8" markerHeight="8" refX="8" refY="4" orient="auto"><polygon points="0 0, 8 4, 0 8" class="marker-link"/></marker>
<marker id="harvestview-arrow-accent" markerWidth="8" markerHeight="8" refX="8" refY="4" orient="auto"><polygon points="0 0, 8 4, 0 8" class="marker-accent"/></marker>
</defs>
<rect width="1280" height="720" fill="#f5efe7"/>
<text x="40" y="52" class="eyebrow">HARVESTVIEW · LOCAL OPERATOR WORKFLOW</text>
<text x="40" y="88" class="title">One run desk, one durable lifecycle</text>
<text x="40" y="112" class="subtitle">The browser reviews scope; the API validates; an isolated worker executes; evidence remains available after terminal state.</text>
<rect width="960" height="640" class="bg"/>
<text x="40" y="32" class="eyebrow">HARVESTVIEW · LOCAL OPERATOR WORKFLOW</text>
<text x="40" y="64" class="title">One run desk, one durable evidence path</text>
<text x="40" y="88" class="subtitle">The browser reviews scope; the API validates; an isolated worker executes; completed evidence remains portable.</text>
<rect x="40" y="132" width="1200" height="436" rx="8" class="boundary"/>
<rect x="56" y="136" width="184" height="16" fill="#f5efe7"/>
<text x="60" y="148" class="eyebrow">LOCAL HOST · 127.0.0.1</text>
<rect x="24" y="120" width="912" height="408" rx="8" class="boundary"/>
<rect x="40" y="124" width="204" height="16" rx="4" class="boundary-mask"/>
<text x="48" y="136" class="eyebrow">LOCALHOST · 127.0.0.1:5000</text>
<line x1="224" y1="292" x2="264" y2="292" class="connector" marker-end="url(#harvestview-architecture-arrow)"/>
<line x1="464" y1="292" x2="504" y2="292" class="connector" marker-end="url(#harvestview-architecture-arrow)"/>
<line x1="604" y1="364" x2="604" y2="412" class="connector" marker-end="url(#harvestview-architecture-arrow)"/>
<line x1="704" y1="472" x2="744" y2="472" class="connector" marker-end="url(#harvestview-architecture-arrow)"/>
<line x1="844" y1="412" x2="844" y2="364" class="connector-accent" marker-end="url(#harvestview-architecture-arrow-accent)"/>
<line x1="944" y1="292" x2="984" y2="292" class="connector" marker-end="url(#harvestview-architecture-arrow)"/>
<line x1="220" y1="224" x2="244" y2="224" class="connector" marker-end="url(#harvestview-arrow)"/>
<line x1="444" y1="192" x2="468" y2="192" class="connector-link dashed" marker-end="url(#harvestview-arrow-link)"/>
<line x1="344" y1="296" x2="344" y2="336" class="connector" marker-end="url(#harvestview-arrow)"/>
<line x1="444" y1="408" x2="468" y2="408" class="connector" marker-end="url(#harvestview-arrow)"/>
<line x1="568" y1="336" x2="568" y2="296" class="connector-accent" marker-end="url(#harvestview-arrow-accent)"/>
<line x1="668" y1="224" x2="692" y2="224" class="connector" marker-end="url(#harvestview-arrow)"/>
<line x1="806" y1="296" x2="806" y2="336" class="connector-link" marker-end="url(#harvestview-arrow-link)"/>
<rect x="64" y="220" width="160" height="144" rx="8" class="node"/>
<rect x="76" y="232" width="72" height="20" rx="4" class="tag"/>
<text x="112" y="246" class="tag-text">BROWSER</text>
<text x="80" y="276" class="node-title">Create or import</text>
<text x="80" y="300" class="node-copy-strong">new run · JSONL · SQLite</text>
<text x="80" y="320" class="node-copy-strong">source and action review</text>
<text x="80" y="344" class="node-copy">P0 · P1 · P2 authorization</text>
<rect x="40" y="152" width="180" height="144" rx="8" class="node"/>
<rect x="52" y="164" width="72" height="20" rx="4" class="tag"/>
<text x="88" y="180" class="tag-text">BROWSER</text>
<text x="56" y="216" class="node-title">Plan and authorize</text>
<text x="56" y="240" class="node-copy-strong">target · sources</text>
<text x="56" y="256" class="node-copy-strong">actions · P0/P1/P2</text>
<text x="56" y="272" class="node-copy">review before submit</text>
<rect x="264" y="220" width="200" height="144" rx="8" class="node"/>
<rect x="276" y="232" width="96" height="20" rx="4" class="tag"/>
<text x="324" y="246" class="tag-text">API GATE</text>
<text x="280" y="276" class="node-title">Authenticated REST API</text>
<text x="280" y="300" class="node-copy-strong">derived HttpOnly browser session</text>
<text x="280" y="320" class="node-copy-strong">X-API-Key for API clients</text>
<text x="280" y="344" class="node-copy">target · scope · options · limits</text>
<rect x="244" y="152" width="200" height="144" rx="8" class="node-link"/>
<rect x="256" y="164" width="88" height="20" rx="4" class="tag-link"/>
<text x="300" y="180" class="tag-text tag-text-link">API GATE</text>
<text x="260" y="216" class="node-title">Authenticated REST API</text>
<text x="260" y="240" class="node-copy-strong">HttpOnly · X-API-Key</text>
<text x="260" y="256" class="node-copy-strong">target · scope · limits</text>
<text x="260" y="272" class="node-copy">create · list</text>
<text x="260" y="288" class="node-copy">detail · cancel</text>
<rect x="504" y="220" width="200" height="144" rx="8" class="node-muted"/>
<rect x="516" y="232" width="96" height="20" rx="4" class="tag"/>
<text x="564" y="246" class="tag-text">CONTROL</text>
<text x="520" y="276" class="node-title">Durable run control</text>
<text x="520" y="300" class="node-copy-strong">queued → running → terminal</text>
<text x="520" y="320" class="node-copy-strong">cancel request · source status</text>
<text x="520" y="344" class="node-copy">one record per finite run</text>
<rect x="468" y="152" width="200" height="144" rx="8" class="node-focal"/>
<rect x="480" y="164" width="88" height="20" rx="4" class="tag-accent"/>
<text x="524" y="180" class="tag-text tag-text-accent">EVIDENCE</text>
<text x="484" y="216" class="node-title">Canonical evidence</text>
<text x="484" y="240" class="node-copy-strong">results · provenance</text>
<text x="484" y="256" class="node-copy-strong">source/action outcomes</text>
<text x="484" y="272" class="node-copy">artifacts · status</text>
<text x="484" y="288" class="node-copy">complete · partial · failed</text>
<rect x="504" y="412" width="200" height="120" rx="8" class="node"/>
<rect x="516" y="424" width="80" height="20" rx="4" class="tag"/>
<text x="556" y="438" class="tag-text">WORKER</text>
<text x="520" y="468" class="node-title">Isolated run worker</text>
<text x="520" y="492" class="node-copy-strong">claim · checkpoint · cancel</text>
<text x="520" y="512" class="node-copy">bounded output · child cleanup</text>
<rect x="692" y="152" width="228" height="144" rx="8" class="node"/>
<rect x="704" y="164" width="112" height="20" rx="4" class="tag"/>
<text x="760" y="180" class="tag-text">WORKBENCH</text>
<text x="708" y="216" class="node-title">Inspect and act</text>
<text x="708" y="240" class="node-copy-strong">history · assessment</text>
<text x="708" y="256" class="node-copy-strong">route tabs · search</text>
<text x="708" y="272" class="node-copy">filters · copy · logs</text>
<text x="708" y="288" class="node-copy">screenshots · child runs</text>
<rect x="744" y="412" width="200" height="120" rx="8" class="node"/>
<rect x="756" y="424" width="80" height="20" rx="4" class="tag"/>
<text x="796" y="438" class="tag-text">ENGINE</text>
<text x="760" y="468" class="node-title">theHarvester engine</text>
<text x="760" y="492" class="node-copy-strong">cataloged sources · explicit actions</text>
<text x="760" y="512" class="node-copy">DNS · Shodan · RouteViews · vhost</text>
<rect x="244" y="336" width="200" height="144" rx="8" class="node-muted"/>
<rect x="256" y="348" width="88" height="20" rx="4" class="tag"/>
<text x="300" y="364" class="tag-text">CONTROL</text>
<text x="260" y="392" class="node-title">Durable run lifecycle</text>
<text x="260" y="420" class="node-copy-strong">queued · running</text>
<text x="260" y="436" class="node-copy-strong">cancelling · cancelled</text>
<text x="260" y="452" class="node-copy">completed · failed</text>
<text x="260" y="468" class="node-copy">leases · deadlines</text>
<rect x="744" y="220" width="200" height="144" rx="8" class="node-focal"/>
<rect x="756" y="232" width="96" height="20" rx="4" class="tag-teal"/>
<text x="804" y="246" class="tag-text tag-text-teal">EVIDENCE</text>
<text x="760" y="276" class="node-title">Evidence &amp; artifacts</text>
<text x="760" y="300" class="node-copy-strong">CompletedResult · route records</text>
<text x="760" y="320" class="node-copy-strong">SQLite · screenshots · logs</text>
<text x="760" y="344" class="node-copy">partial evidence remains inspectable</text>
<rect x="468" y="336" width="200" height="144" rx="8" class="node"/>
<rect x="480" y="348" width="80" height="20" rx="4" class="tag"/>
<text x="520" y="364" class="tag-text">WORKER</text>
<text x="484" y="392" class="node-title">Isolated run worker</text>
<text x="484" y="420" class="node-copy-strong">claim · execute</text>
<text x="484" y="436" class="node-copy-strong">checkpoint · cancel</text>
<text x="484" y="452" class="node-copy">58 sources</text>
<text x="484" y="468" class="node-copy">10 explicit actions</text>
<rect x="984" y="220" width="232" height="144" rx="8" class="node"/>
<rect x="996" y="232" width="112" height="20" rx="4" class="tag"/>
<text x="1052" y="246" class="tag-text">WORKBENCH</text>
<text x="1000" y="276" class="node-title">Inspect and export</text>
<text x="1000" y="300" class="node-copy-strong">history · assessment · route tabs</text>
<text x="1000" y="320" class="node-copy-strong">screenshots · source outcomes · logs</text>
<text x="1000" y="340" class="node-copy">review actions · JSONL / SQLite export</text>
<rect x="692" y="336" width="228" height="144" rx="8" class="node-link"/>
<rect x="704" y="348" width="104" height="20" rx="4" class="tag-link"/>
<text x="756" y="364" class="tag-text tag-text-link">INTERCHANGE</text>
<text x="708" y="392" class="node-title">Import and export</text>
<text x="708" y="420" class="node-copy-strong">JSONL · one run</text>
<text x="708" y="436" class="node-copy-strong">SQLite · completed set</text>
<text x="708" y="452" class="node-copy">screenshot files separate</text>
<line x1="40" y1="636" x2="1240" y2="636" stroke="rgba(14,29,34,.14)" stroke-width=".8"/>
<line x1="48" y1="676" x2="88" y2="676" class="connector" marker-end="url(#harvestview-architecture-arrow)"/>
<text x="104" y="680" class="legend">control and lifecycle handoff</text>
<line x1="352" y1="676" x2="392" y2="676" class="connector-accent" marker-end="url(#harvestview-architecture-arrow-accent)"/>
<text x="408" y="680" class="legend">normalized evidence handoff</text>
<rect x="676" y="664" width="20" height="20" rx="4" class="tag-teal"/>
<text x="708" y="680" class="legend">durable evidence boundary</text>
<text x="1240" y="680" class="legend" text-anchor="end">provider credentials remain server-side</text>
</svg>
<text x="48" y="512" class="node-copy">Provider credentials stay server-side. Lifecycle state never replaces terminal evidence status.</text>
<line x1="40" y1="584" x2="920" y2="584" stroke="var(--rule)" stroke-width=".8"/>
<line x1="48" y1="616" x2="80" y2="616" class="connector" marker-end="url(#harvestview-arrow)"/>
<text x="92" y="620" class="legend">run control</text>
<line x1="248" y1="616" x2="280" y2="616" class="connector-accent" marker-end="url(#harvestview-arrow-accent)"/>
<text x="292" y="620" class="legend">evidence handoff</text>
<line x1="500" y1="616" x2="532" y2="616" class="connector-link dashed" marker-end="url(#harvestview-arrow-link)"/>
<text x="544" y="620" class="legend">import path</text>
</svg>

Before

Width:  |  Height:  |  Size: 8.2 KiB

After

Width:  |  Height:  |  Size: 10 KiB

+160 -124
View File
@@ -1,137 +1,173 @@
<?xml version="1.0" encoding="UTF-8"?>
<svg viewBox="0 0 1280 720" xmlns="http://www.w3.org/2000/svg" role="img" aria-labelledby="run-evidence-architecture-title run-evidence-architecture-desc">
<title id="run-evidence-architecture-title">Discovery routes and enrichment</title>
<desc id="run-evidence-architecture-desc">theHarvester source selection and adapters feed passive, DNS, and direct actions before normalization into identity, contact, network, and action evidence routes.</desc>
<defs>
<style>@import url('https://fonts.googleapis.com/css2?family=Instrument+Serif:ital@0;1&amp;family=Geist:wght@400;500;600&amp;family=Geist+Mono:wght@400;500;600&amp;display=swap');</style>
<style>
.title { font: 400 28px "Instrument Serif", Georgia, serif; fill: #0e1d22; }
.subtitle { font: 400 12px "Geist", system-ui, sans-serif; fill: #4f5e63; }
.eyebrow { font: 600 8px "Geist Mono", ui-monospace, monospace; letter-spacing: .16em; fill: #4f5e63; }
.node-title { font: 600 12px "Geist", system-ui, sans-serif; fill: #0e1d22; }
.node-copy { font: 400 8px "Geist Mono", ui-monospace, monospace; fill: #4f5e63; }
.node-copy-strong { font: 500 8px "Geist Mono", ui-monospace, monospace; fill: #34464c; }
.zone { fill: rgba(14,29,34,.018); stroke: rgba(14,29,34,.14); stroke-width: .8; }
.node { fill: #fdfaf4; stroke: #889ca3; stroke-width: 1; }
.node-muted { fill: #ede7dd; stroke: #889ca3; stroke-width: 1; }
.node-focal { fill: #bcede4; stroke: #009685; stroke-width: 1.2; }
.node-p1 { fill: #fde4bb; stroke: #723f00; stroke-width: 1; }
.node-p2 { fill: #ffddd7; stroke: #a91515; stroke-width: 1; }
.tag { fill: #f5efe7; stroke: #889ca3; stroke-width: .8; }
.tag-teal { fill: #bcede4; stroke: #009685; stroke-width: .8; }
.tag-p1 { fill: #fde4bb; stroke: #723f00; stroke-width: .8; }
.tag-p2 { fill: #ffddd7; stroke: #a91515; stroke-width: .8; }
.tag-text { font: 600 8px "Geist Mono", ui-monospace, monospace; fill: #34464c; text-anchor: middle; }
.tag-text-teal { fill: #00594c; }
.tag-text-p1 { fill: #723f00; }
.tag-text-p2 { fill: #a91515; }
.connector { fill: none; stroke: #4f5e63; stroke-width: 1.2; }
.legend { font: 400 8px "Geist Mono", ui-monospace, monospace; fill: #4f5e63; }
</style>
<marker id="run-evidence-architecture-arrow" markerWidth="8" markerHeight="6" refX="7" refY="3" orient="auto"><polygon points="0 0, 8 3, 0 6" fill="#4f5e63"/></marker>
<marker id="run-evidence-architecture-arrow-accent" markerWidth="8" markerHeight="6" refX="7" refY="3" orient="auto"><polygon points="0 0, 8 3, 0 6" fill="#009685"/></marker>
<marker id="run-evidence-architecture-arrow-link" markerWidth="8" markerHeight="6" refX="7" refY="3" orient="auto"><polygon points="0 0, 8 3, 0 6" fill="#00548c"/></marker>
</defs>
<svg viewBox="0 0 960 640" xmlns="http://www.w3.org/2000/svg" role="img" aria-labelledby="run-evidence-architecture-title run-evidence-architecture-desc">
<title id="run-evidence-architecture-title">theHarvester discovery and evidence architecture</title>
<desc id="run-evidence-architecture-desc">Seven result capabilities and 58 cataloged discovery sources join ten explicit P0, P1, and P2 actions in one normalized evidence contract, which routes identity, network, and action findings to terminal, JSONL, SQLite, REST, HarvestView, screenshots, JSON, and XML outputs.</desc>
<defs>
<style>
@import url('https://fonts.googleapis.com/css2?family=Instrument+Serif:ital@0;1&amp;family=Geist:wght@400;500;600&amp;family=Geist+Mono:wght@400;500;600&amp;display=swap');
:root {
--paper: #2d3142;
--paper-2: #393e53;
--ink: #f5f5f5;
--muted: #bfc0c0;
--soft: #8e98ac;
--rule: rgba(245,245,245,.16);
--rule-strong: rgba(245,245,245,.28);
--zone-fill: rgba(245,245,245,.025);
--accent: #f08a59;
--accent-tint: rgba(240,138,89,.10);
--link: #6a95d8;
--link-tint: rgba(106,149,216,.10);
}
@media (prefers-color-scheme: light) {
:root {
--paper: #f5f5f5;
--paper-2: #ececec;
--ink: #2d3142;
--muted: #4f5d75;
--soft: #7a8399;
--rule: rgba(45,49,66,.14);
--rule-strong: rgba(45,49,66,.28);
--zone-fill: rgba(45,49,66,.02);
--accent: #b84f24;
--accent-tint: rgba(235,108,54,.10);
--link: #2e5aa8;
--link-tint: rgba(46,90,168,.08);
}
}
.bg { fill: var(--paper); }
.title { font: 400 28px 'Instrument Serif', Georgia, serif; fill: var(--ink); }
.subtitle { font: 400 12px 'Geist', system-ui, sans-serif; fill: var(--muted); }
.eyebrow { font: 600 8px 'Geist Mono', ui-monospace, monospace; letter-spacing: .16em; fill: var(--soft); }
.zone { fill: var(--zone-fill); stroke: var(--rule); stroke-width: .8; }
.zone-mask { fill: var(--paper); }
.node { fill: var(--paper); stroke: var(--rule-strong); stroke-width: 1; }
.node-muted { fill: var(--paper-2); stroke: var(--rule-strong); stroke-width: 1; }
.node-link { fill: var(--link-tint); stroke: var(--link); stroke-width: 1; }
.node-focal { fill: var(--accent-tint); stroke: var(--accent); stroke-width: 1.2; }
.tag { fill: var(--paper-2); stroke: var(--rule-strong); stroke-width: .8; }
.tag-link { fill: var(--link-tint); stroke: var(--link); stroke-width: .8; }
.tag-accent { fill: var(--accent-tint); stroke: var(--accent); stroke-width: .8; }
.tag-text { font: 600 8px 'Geist Mono', ui-monospace, monospace; letter-spacing: .08em; fill: var(--muted); text-anchor: middle; }
.tag-text-link { fill: var(--link); }
.tag-text-accent { fill: var(--accent); }
.node-title { font: 600 16px 'Geist', system-ui, sans-serif; fill: var(--ink); }
.node-copy { font: 400 12px 'Geist Mono', ui-monospace, monospace; fill: var(--muted); }
.node-copy-strong { font: 500 12px 'Geist Mono', ui-monospace, monospace; fill: var(--ink); }
.connector { fill: none; stroke: var(--muted); stroke-width: 1.2; }
.connector-link { fill: none; stroke: var(--link); stroke-width: 1.2; }
.connector-accent { fill: none; stroke: var(--accent); stroke-width: 1.2; }
.marker-muted { fill: var(--muted); }
.marker-link { fill: var(--link); }
.marker-accent { fill: var(--accent); }
.legend { font: 500 12px 'Geist Mono', ui-monospace, monospace; fill: var(--muted); }
</style>
<marker id="run-evidence-arrow" markerWidth="8" markerHeight="8" refX="8" refY="4" orient="auto"><polygon points="0 0, 8 4, 0 8" class="marker-muted"/></marker>
<marker id="run-evidence-arrow-link" markerWidth="8" markerHeight="8" refX="8" refY="4" orient="auto"><polygon points="0 0, 8 4, 0 8" class="marker-link"/></marker>
<marker id="run-evidence-arrow-accent" markerWidth="8" markerHeight="8" refX="8" refY="4" orient="auto"><polygon points="0 0, 8 4, 0 8" class="marker-accent"/></marker>
</defs>
<rect width="1280" height="720" fill="#f5efe7"/>
<text x="40" y="52" class="eyebrow">DISCOVERY · ENRICHMENT · EVIDENCE</text>
<text x="40" y="88" class="title">Sources become attributable evidence</text>
<text x="40" y="112" class="subtitle">Catalog policy selects the work; one normalized result preserves routes, producers, outcomes, and partial evidence.</text>
<rect width="960" height="640" class="bg"/>
<text x="40" y="32" class="eyebrow">DISCOVERY · ENRICHMENT · ATTRIBUTABLE EVIDENCE</text>
<text x="40" y="64" class="title">From source selection to durable evidence</text>
<text x="40" y="88" class="subtitle">Every selected source and action reports its own outcome before results are deduplicated, attributed, and exported.</text>
<rect x="40" y="132" width="432" height="420" rx="8" class="zone"/>
<rect x="56" y="136" width="72" height="16" fill="#f5efe7"/>
<text x="60" y="148" class="eyebrow">COLLECT</text>
<rect x="492" y="132" width="284" height="420" rx="8" class="zone"/>
<rect x="508" y="136" width="72" height="16" fill="#f5efe7"/>
<text x="512" y="148" class="eyebrow">ACTIONS</text>
<rect x="796" y="132" width="444" height="420" rx="8" class="zone"/>
<rect x="812" y="136" width="120" height="16" fill="#f5efe7"/>
<text x="816" y="148" class="eyebrow">NORMALIZE &amp; ROUTE</text>
<rect x="40" y="120" width="512" height="448" rx="8" class="zone"/>
<rect x="52" y="124" width="132" height="16" rx="4" class="zone-mask"/>
<text x="60" y="136" class="eyebrow">SELECT + EXECUTE</text>
<rect x="576" y="120" width="344" height="448" rx="8" class="zone"/>
<rect x="588" y="124" width="124" height="16" rx="4" class="zone-mask"/>
<text x="596" y="136" class="eyebrow">EVIDENCE ROUTES</text>
<line x1="224" y1="304" x2="264" y2="304" class="connector" marker-end="url(#run-evidence-architecture-arrow)"/>
<path d="M464 256 H480 Q488 256 488 248 V220 Q488 212 496 212 H516" class="connector" marker-end="url(#run-evidence-architecture-arrow)"/>
<path d="M464 304 H480 Q488 304 488 312 V324 Q488 332 496 332 H516" class="connector" marker-end="url(#run-evidence-architecture-arrow)"/>
<path d="M464 352 H472 Q480 352 480 360 V444 Q480 452 488 452 H516" class="connector" marker-end="url(#run-evidence-architecture-arrow)"/>
<path d="M752 212 H768 Q776 212 776 220 V268 Q776 276 784 276 H816" class="connector" marker-end="url(#run-evidence-architecture-arrow)"/>
<path d="M752 332 H768 Q776 332 776 324 V312 Q776 304 784 304 H816" class="connector" marker-end="url(#run-evidence-architecture-arrow)"/>
<path d="M752 452 H760 Q768 452 768 444 V340 Q768 332 776 332 H816" class="connector" marker-end="url(#run-evidence-architecture-arrow)"/>
<path d="M1016 280 H1020 Q1028 280 1028 272 V252 Q1028 244 1036 244 H1040" class="connector" marker-end="url(#run-evidence-architecture-arrow)"/>
<path d="M1016 328 H1020 Q1028 328 1028 336 V432 Q1028 440 1036 440 H1040" class="connector" marker-end="url(#run-evidence-architecture-arrow)"/>
<line x1="260" y1="216" x2="332" y2="216" class="connector" marker-end="url(#run-evidence-arrow)"/>
<line x1="432" y1="288" x2="432" y2="304" class="connector" marker-end="url(#run-evidence-arrow)"/>
<line x1="132" y1="424" x2="132" y2="400" class="connector-link" marker-end="url(#run-evidence-arrow-link)"/>
<line x1="296" y1="424" x2="296" y2="400" class="connector" marker-end="url(#run-evidence-arrow)"/>
<line x1="460" y1="424" x2="460" y2="400" class="connector" marker-end="url(#run-evidence-arrow)"/>
<path d="M532 328 H544 Q552 328 552 320 V216 Q552 208 560 208 H600" class="connector" marker-end="url(#run-evidence-arrow)"/>
<path d="M532 352 H584 Q592 352 592 360 H600" class="connector" marker-end="url(#run-evidence-arrow)"/>
<path d="M532 376 H560 Q568 376 568 384 V496 Q568 504 576 504 H600" class="connector" marker-end="url(#run-evidence-arrow)"/>
<rect x="64" y="224" width="160" height="160" rx="8" class="node-muted"/>
<rect x="76" y="236" width="64" height="20" rx="4" class="tag"/>
<text x="108" y="250" class="tag-text">CATALOG</text>
<text x="80" y="280" class="node-title">Source selection</text>
<text x="80" y="304" class="node-copy-strong">subdomains · emails · IPs</text>
<text x="80" y="324" class="node-copy-strong">ASNs · URLs · people</text>
<text x="80" y="344" class="node-copy-strong">breaches · explicit names</text>
<text x="80" y="368" class="node-copy">activity · keys · routes</text>
<rect x="60" y="144" width="200" height="144" rx="8" class="node"/>
<rect x="72" y="156" width="56" height="20" rx="4" class="tag"/>
<text x="100" y="172" class="tag-text">SCOPE</text>
<text x="76" y="208" class="node-title">Scope and source</text>
<text x="76" y="228" class="node-title">selection</text>
<text x="76" y="252" class="node-copy-strong">7 result capabilities</text>
<text x="76" y="272" class="node-copy">names · all · no-hosts</text>
<rect x="264" y="224" width="200" height="160" rx="8" class="node"/>
<rect x="276" y="236" width="72" height="20" rx="4" class="tag"/>
<text x="312" y="250" class="tag-text">SOURCES</text>
<text x="280" y="280" class="node-title">Discovery adapters</text>
<text x="280" y="304" class="node-copy-strong">search · certificates · DNS</text>
<text x="280" y="324" class="node-copy-strong">code · archives · threat intel</text>
<text x="280" y="348" class="node-copy">normalized observations</text>
<text x="280" y="368" class="node-copy">truthful source outcomes</text>
<rect x="332" y="144" width="200" height="144" rx="8" class="node"/>
<rect x="344" y="156" width="72" height="20" rx="4" class="tag"/>
<text x="380" y="172" class="tag-text">SOURCES</text>
<text x="348" y="208" class="node-title">58 discovery adapters</text>
<text x="348" y="232" class="node-copy-strong">search · CT · DNS data</text>
<text x="348" y="248" class="node-copy-strong">code · archives</text>
<text x="348" y="264" class="node-copy-strong">threat intelligence</text>
<text x="348" y="280" class="node-copy">activity · keys · outcomes</text>
<rect x="516" y="164" width="236" height="96" rx="8" class="node"/>
<rect x="528" y="176" width="40" height="20" rx="4" class="tag-teal"/>
<text x="548" y="190" class="tag-text tag-text-teal">P0</text>
<text x="584" y="192" class="node-title">Passive enrichment</text>
<text x="532" y="220" class="node-copy-strong">Shodan host detail</text>
<text x="532" y="240" class="node-copy-strong">RouteViews routes · origins · RPKI</text>
<rect x="60" y="304" width="472" height="96" rx="8" class="node-focal"/>
<rect x="72" y="316" width="104" height="20" rx="4" class="tag-accent"/>
<text x="124" y="332" class="tag-text tag-text-accent">NORMALIZE</text>
<text x="188" y="332" class="node-title">CompletedResult evidence contract</text>
<text x="76" y="364" class="node-copy-strong">dedupe · provenance · source and action outcomes</text>
<text x="76" y="384" class="node-copy">complete · partial · failed evidence · artifacts</text>
<rect x="516" y="284" width="236" height="96" rx="8" class="node-p1"/>
<rect x="528" y="296" width="40" height="20" rx="4" class="tag-p1"/>
<text x="548" y="310" class="tag-text tag-text-p1">P1</text>
<text x="584" y="312" class="node-title">DNS interaction</text>
<text x="532" y="340" class="node-copy-strong">resolve · reverse lookup</text>
<text x="532" y="360" class="node-copy-strong">brute force · recursive discovery</text>
<rect x="60" y="424" width="144" height="128" rx="8" class="node-link"/>
<rect x="72" y="436" width="40" height="20" rx="4" class="tag-link"/>
<text x="92" y="452" class="tag-text tag-text-link">P0</text>
<text x="76" y="480" class="node-title">Provider</text>
<text x="76" y="500" class="node-title">enrichment</text>
<text x="76" y="524" class="node-copy-strong">shodan</text>
<text x="76" y="544" class="node-copy-strong">routeviews</text>
<rect x="516" y="404" width="236" height="96" rx="8" class="node-p2"/>
<rect x="528" y="416" width="40" height="20" rx="4" class="tag-p2"/>
<text x="548" y="430" class="tag-text tag-text-p2">P2</text>
<text x="584" y="432" class="node-title">Direct interaction</text>
<text x="532" y="460" class="node-copy-strong">vhost · screenshots · takeover</text>
<text x="532" y="480" class="node-copy-strong">API path discovery</text>
<rect x="224" y="424" width="144" height="128" rx="8" class="node-muted"/>
<rect x="236" y="436" width="40" height="20" rx="4" class="tag"/>
<text x="256" y="452" class="tag-text">P1</text>
<text x="240" y="480" class="node-title">DNS interaction</text>
<text x="240" y="500" class="node-copy-strong">dns-resolve</text>
<text x="240" y="516" class="node-copy-strong">dns-lookup</text>
<text x="240" y="532" class="node-copy-strong">dns-brute</text>
<text x="240" y="548" class="node-copy-strong">dns-recursive</text>
<rect x="816" y="240" width="200" height="128" rx="8" class="node-focal"/>
<rect x="828" y="252" width="92" height="20" rx="4" class="tag-teal"/>
<text x="874" y="266" class="tag-text tag-text-teal">NORMALIZED</text>
<text x="832" y="300" class="node-title">CompletedResult</text>
<text x="832" y="324" class="node-copy-strong">dedupe · provenance · outcomes</text>
<text x="832" y="344" class="node-copy-strong">artifacts · evidence status</text>
<text x="832" y="360" class="node-copy">partial evidence stays attributable</text>
<rect x="388" y="424" width="144" height="128" rx="8" class="node-focal"/>
<rect x="400" y="436" width="40" height="20" rx="4" class="tag-accent"/>
<text x="420" y="452" class="tag-text tag-text-accent">P2</text>
<text x="404" y="480" class="node-title">Direct action</text>
<text x="404" y="500" class="node-copy-strong">api-scan</text>
<text x="404" y="516" class="node-copy-strong">screenshot</text>
<text x="404" y="532" class="node-copy-strong">takeover</text>
<text x="404" y="548" class="node-copy-strong">vhost</text>
<rect x="1040" y="164" width="200" height="160" rx="8" class="node"/>
<rect x="1052" y="176" width="104" height="20" rx="4" class="tag"/>
<text x="1104" y="190" class="tag-text">IDENTITY</text>
<text x="1056" y="220" class="node-title">Identity &amp; contact routes</text>
<text x="1056" y="244" class="node-copy-strong">hostname / subdomain</text>
<text x="1056" y="264" class="node-copy-strong">email · URL</text>
<text x="1056" y="284" class="node-copy-strong">person · person-link · breach</text>
<text x="1056" y="308" class="node-copy">vhost observations stay on hostname</text>
<rect x="600" y="152" width="296" height="112" rx="8" class="node"/>
<rect x="612" y="164" width="88" height="20" rx="4" class="tag"/>
<text x="656" y="180" class="tag-text">IDENTITY</text>
<text x="616" y="208" class="node-title">Identity and contact</text>
<text x="616" y="236" class="node-copy-strong">subdomains · emails · urls</text>
<text x="616" y="256" class="node-copy-strong">people · person links · breaches</text>
<rect x="1040" y="348" width="200" height="184" rx="8" class="node"/>
<rect x="1052" y="360" width="96" height="20" rx="4" class="tag"/>
<text x="1100" y="374" class="tag-text">NETWORK</text>
<text x="1056" y="404" class="node-title">Network &amp; action routes</text>
<text x="1056" y="428" class="node-copy-strong">IP · ASN · prefix</text>
<text x="1056" y="448" class="node-copy-strong">Shodan host · takeover</text>
<text x="1056" y="468" class="node-copy-strong">scope extension</text>
<text x="1056" y="488" class="node-copy-strong">external relationship · other</text>
<text x="1056" y="516" class="node-copy">JSONL · SQLite · REST · HarvestView</text>
<rect x="600" y="288" width="296" height="144" rx="8" class="node"/>
<rect x="612" y="300" width="88" height="20" rx="4" class="tag"/>
<text x="656" y="316" class="tag-text">NETWORK</text>
<text x="616" y="344" class="node-title">Network and action evidence</text>
<text x="616" y="372" class="node-copy-strong">ips · asns · prefixes</text>
<text x="616" y="392" class="node-copy-strong">Shodan hosts · takeover</text>
<text x="616" y="412" class="node-copy-strong">recursive DNS · vhost evidence</text>
<text x="616" y="428" class="node-copy">scope extension · external relations</text>
<line x1="40" y1="636" x2="1240" y2="636" stroke="rgba(14,29,34,.14)" stroke-width=".8"/>
<line x1="48" y1="676" x2="88" y2="676" class="connector" marker-end="url(#run-evidence-architecture-arrow)"/>
<text x="104" y="680" class="legend">data handoff</text>
<rect x="248" y="664" width="20" height="20" rx="4" class="tag-teal"/>
<text x="280" y="680" class="legend">normalized evidence</text>
<rect x="464" y="664" width="20" height="20" rx="4" class="tag-p1"/>
<text x="496" y="680" class="legend">P1 · DNS interaction</text>
<rect x="704" y="664" width="20" height="20" rx="4" class="tag-p2"/>
<text x="736" y="680" class="legend">P2 · direct interaction</text>
<text x="1240" y="680" class="legend" text-anchor="end">P0 sources and enrichments remain passive provider requests</text>
</svg>
<rect x="600" y="464" width="296" height="88" rx="8" class="node"/>
<rect x="612" y="476" width="72" height="20" rx="4" class="tag"/>
<text x="648" y="492" class="tag-text">OUTPUT</text>
<text x="700" y="492" class="node-title">Durable outputs</text>
<text x="616" y="520" class="node-copy-strong">Terminal · JSONL · SQLite · REST</text>
<text x="616" y="540" class="node-copy">HarvestView · screenshots · JSON/XML</text>
<line x1="40" y1="584" x2="920" y2="584" stroke="var(--rule)" stroke-width=".8"/>
<line x1="48" y1="616" x2="80" y2="616" class="connector" marker-end="url(#run-evidence-arrow)"/>
<text x="92" y="620" class="legend">evidence flow</text>
<rect x="240" y="604" width="20" height="20" rx="4" class="tag-link"/>
<text x="272" y="620" class="legend">P0 provider</text>
<rect x="432" y="604" width="20" height="20" rx="4" class="tag"/>
<text x="464" y="620" class="legend">P1 DNS</text>
<rect x="584" y="604" width="20" height="20" rx="4" class="tag-accent"/>
<text x="616" y="620" class="legend">P2 direct</text>
</svg>

Before

Width:  |  Height:  |  Size: 10 KiB

After

Width:  |  Height:  |  Size: 11 KiB

+14 -4
View File
@@ -6,7 +6,7 @@ from pathlib import Path
import yaml
from theHarvester.lib.source_catalog import SOURCE_SPECS
from theHarvester.lib.source_catalog import ACTION_ACTIVITIES, RESULT_CAPABILITIES, SOURCE_SPECS
OPTIONAL_API_KEY_SOURCES = {'hackertarget', 'mojeek', 'windvane'}
API_KEY_SOURCE_ALIASES = {
@@ -199,24 +199,34 @@ def test_readme_architecture_diagrams_are_local_and_accessible() -> None:
'theHarvester discovery routes and enrichment',
Path('docs/images/run-evidence-architecture.svg'),
'run-evidence-architecture',
('subdomains · emails · IPs', 'Shodan host detail', 'RouteViews routes', 'vhost · screenshots · takeover'),
('58 discovery adapters', 'CompletedResult evidence contract', 'Terminal · JSONL · SQLite · REST'),
),
(
'HarvestView run desk architecture',
Path('docs/images/harvestview-architecture.svg'),
'harvestview-architecture',
('Authenticated REST API', 'queued running → terminal', 'Isolated run worker', 'JSONL / SQLite export'),
('Authenticated REST API', 'queued · running', 'Isolated run worker', 'JSONL · one run'),
),
)
for alt, svg, slug, expected_text in diagrams:
svg_text = svg.read_text()
assert f'![{alt}]({svg})' in readme
assert f'[![{alt}]({svg})]({svg})' in readme
assert 'role="img"' in svg_text
assert f'<title id="{slug}-title">' in svg_text
assert f'<desc id="{slug}-desc">' in svg_text
assert 'viewBox="0 0 960 640"' in svg_text
assert '@media (prefers-color-scheme: light)' in svg_text
assert "font: 600 16px 'Geist'" in svg_text
assert all(text in svg_text for text in expected_text)
run_diagram = diagrams[0][1].read_text()
harvestview_diagram = diagrams[1][1].read_text()
assert all(capability in run_diagram for capability in RESULT_CAPABILITIES)
assert all(action in run_diagram for action in ACTION_ACTIVITIES)
assert f'{len(SOURCE_SPECS)} discovery adapters' in run_diagram
assert f'{len(ACTION_ACTIVITIES)} explicit actions' in harvestview_diagram
def test_wiki_diagrams_are_local_accessible_and_used_deliberately() -> None:
raw_root = 'https://raw.githubusercontent.com/laramies/theHarvester/dev/'