From 6fea00a7e0b8b813e4649549e30530714e17f027 Mon Sep 17 00:00:00 2001 From: NotoriousRebel <36310667+NotoriousRebel@users.noreply.github.com> Date: Mon, 17 Aug 2026 10:24:03 -0400 Subject: [PATCH] docs: add action evidence JSONL examples --- docs/wiki/Operator-Workflows.md | 33 ++++++++++++++- docs/wiki/Results-and-Local-Data.md | 65 +++++++++++++++++++++++++++++ tests/test_readme.py | 43 +++++++++++++++++++ 3 files changed, 140 insertions(+), 1 deletion(-) diff --git a/docs/wiki/Operator-Workflows.md b/docs/wiki/Operator-Workflows.md index 00ff94da..9314b3bf 100644 --- a/docs/wiki/Operator-Workflows.md +++ b/docs/wiki/Operator-Workflows.md @@ -44,7 +44,20 @@ uv run theHarvester -d "$AUTHORIZED_DOMAIN" -b crtsh -r resolvers.txt DNS requests disclose candidate names to each selected resolver. Candidates from all selected sources are normalized and deduplicated before one run-wide phase queries A, AAAA, and CNAME at most once per hostname and record type. The phase runs at most 20 hostname jobs concurrently with per-query resolver timeouts and no default query-count or phase-runtime ceiling. -Reverse DNS (`-n`) uses an independent run-wide job set. It deduplicates addresses across overlapping discovered `/24` ranges and runs at most 20 PTR jobs concurrently with per-query resolver timeouts and no default request-count or phase-runtime ceiling. +## Reverse DNS + +Network activity: provider-facing discovery followed by resolver-facing PTR queries. + +```bash +AUTHORIZED_DOMAIN='replace-with-a-domain-you-control' +uv run theHarvester \ + -d "$AUTHORIZED_DOMAIN" \ + -b rapiddns \ + --dns-lookup \ + -f report +``` + +Reverse DNS uses the `/24` network containing each discovered IPv4 address. It deduplicates addresses across overlapping ranges and runs at most 20 PTR jobs concurrently with per-query resolver timeouts. It has no default request-count or phase-runtime ceiling. ## Shodan enrichment @@ -72,6 +85,24 @@ uv run theHarvester -d "$AUTHORIZED_DOMAIN" -c DNS brute force actively tests candidate names. Do not run it against `example.com` or an unrelated third-party domain. +## Recursive DNS + +Network activity: provider-facing discovery followed by resolver-facing DNS queries for descendant names. + +Create `resolvers.txt` with exactly three distinct resolver IP addresses. Then set a depth and save the structured evidence: + +```bash +AUTHORIZED_DOMAIN='replace-with-a-domain-you-control' +uv run theHarvester \ + -d "$AUTHORIZED_DOMAIN" \ + -b crtsh \ + --dns-resolvers resolvers.txt \ + --dns-recursive-depth 1 \ + -f report +``` + +Recursive DNS uses source results as seed names. Set the depth explicitly; query and runtime caps still apply. JSONL keeps the discovered hostnames and addresses along with recursive finding, classification, and summary records. + ## Takeover checks Network activity: provider-facing discovery followed by target-facing checks. diff --git a/docs/wiki/Results-and-Local-Data.md b/docs/wiki/Results-and-Local-Data.md index 9844660b..ba5a9f87 100644 --- a/docs/wiki/Results-and-Local-Data.md +++ b/docs/wiki/Results-and-Local-Data.md @@ -30,6 +30,71 @@ The recommended automation output is `report.jsonl`. Its first record summarizes jq -c 'select(.type != "summary") | {type, value, sources, actions}' report.jsonl ``` +### Action evidence examples + +These synthetic reports use reserved names, ASNs, and addresses. Each block starts with a summary and passes the same JSONL parser that handles imports. The summary records action outcomes. Each finding's `actions` array identifies the action that produced it. + +
+DNS resolution, reverse lookup, and brute force + +Resolution contributes IP findings. Reverse lookup contributes hostnames, while brute force can contribute both hostnames and IP addresses. + +```jsonl +{"action_executions":[{"action":"dns-brute","duration_ms":25.0,"error_type":null,"result_count":2,"status":"completed","stop_reason":null},{"action":"dns-lookup","duration_ms":31.0,"error_type":null,"result_count":1,"status":"completed","stop_reason":null},{"action":"dns-resolve","duration_ms":18.0,"error_type":null,"result_count":1,"status":"completed","stop_reason":null}],"artifacts":[],"completed_at":"2026-08-17T12:01:00Z","counts":{"hostname":3,"ip":2},"evidence_status":"complete","result_count":5,"run_id":"123e4567-e89b-12d3-a456-426614174101","source_executions":[{"duration_ms":12.0,"error_type":null,"result_count":1,"source":"crtsh","status":"completed","stop_reason":null},{"duration_ms":15.0,"error_type":null,"result_count":1,"source":"rapiddns","status":"completed","stop_reason":null}],"started_at":"2026-08-17T12:00:00Z","target":"example.com","type":"summary"} +{"actions":["dns-brute"],"sources":[],"type":"hostname","value":"admin.example.com"} +{"sources":["crtsh"],"type":"hostname","value":"api.example.com"} +{"actions":["dns-lookup"],"sources":[],"type":"hostname","value":"ptr.example.com"} +{"actions":["dns-resolve"],"sources":["rapiddns"],"type":"ip","value":"192.0.2.10"} +{"actions":["dns-brute"],"sources":[],"type":"ip","value":"192.0.2.20"} +``` + +
+ +
+Recursive DNS + +Recursive DNS contributes ordinary hostname and IP findings. Its structured records keep the parent name, returned addresses, PTR values, depth, query count, and stop reason. The structured `value` fields contain JSON strings, so use `fromjson` when reading them with `jq`. + +```jsonl +{"action_executions":[{"action":"dns-recursive","duration_ms":40.0,"error_type":null,"result_count":5,"status":"completed","stop_reason":"depth-limit"}],"artifacts":[],"completed_at":"2026-08-17T12:01:00Z","counts":{"dns-recursive-classification":1,"dns-recursive-finding":1,"dns-recursive-summary":1,"hostname":2,"ip":1},"evidence_status":"complete","result_count":6,"run_id":"123e4567-e89b-12d3-a456-426614174102","source_executions":[{"duration_ms":12.0,"error_type":null,"result_count":1,"source":"crtsh","status":"completed","stop_reason":null}],"started_at":"2026-08-17T12:00:00Z","target":"example.com","type":"summary"} +{"actions":["dns-recursive"],"sources":[],"type":"dns-recursive-classification","value":"{\"addressability\":\"not-currently-addressable\",\"addresses\":[],\"cnames\":[\"missing.vendor.test\"],\"hostname\":\"unused.api.example.com\",\"parent\":\"api.example.com\",\"ptrs\":[\"legacy-ptr.example.net\"]}"} +{"actions":["dns-recursive"],"sources":[],"type":"dns-recursive-finding","value":"{\"addresses\":[\"192.0.2.21\"],\"hostname\":\"dev.api.example.com\",\"parent\":\"api.example.com\",\"ptrs\":[\"ptr.example.net\"]}"} +{"actions":["dns-recursive"],"sources":[],"type":"dns-recursive-summary","value":"{\"depth_reached\":1,\"query_count\":24,\"stop_reason\":\"depth-limit\",\"zero_yield_batches\":0}"} +{"sources":["crtsh"],"type":"hostname","value":"api.example.com"} +{"actions":["dns-recursive"],"sources":[],"type":"hostname","value":"dev.api.example.com"} +{"actions":["dns-recursive"],"sources":[],"type":"ip","value":"192.0.2.21"} +``` + +
+ +
+Shodan host enrichment + +A `shodan-host` finding uses the IP address as its value. JSONL keeps service details together rather than creating separate result kinds. + +```jsonl +{"action_executions":[{"action":"shodan","duration_ms":22.0,"error_type":null,"result_count":1,"status":"completed","stop_reason":null}],"artifacts":[],"completed_at":"2026-08-17T12:01:00Z","counts":{"ip":1,"shodan-host":1},"evidence_status":"complete","result_count":2,"run_id":"123e4567-e89b-12d3-a456-426614174103","source_executions":[{"duration_ms":15.0,"error_type":null,"result_count":1,"source":"rapiddns","status":"completed","stop_reason":null}],"started_at":"2026-08-17T12:00:00Z","target":"example.com","type":"summary"} +{"sources":["rapiddns"],"type":"ip","value":"192.0.2.10"} +{"actions":["shodan"],"details":{"asn":"AS64500","domains":["example.com"],"hostnames":["api.example.com"],"organization":"Example Network","services":[{"http":{"components":["nginx"],"server":"nginx","title":"Example"},"observed_at":"2026-08-17T11:58:00Z","port":443,"product":"nginx","transport":"tcp","version":"1.24.0"}]},"sources":[],"type":"shodan-host","value":"192.0.2.10"} +``` + +
+ +
+RouteViews routing evidence + +RouteViews keeps the ASN as a scalar finding and attaches origin, BGP route, and RPKI observations to a canonical prefix. The prefix remains an external relationship and does not expand target scope. + +```jsonl +{"action_executions":[{"action":"routeviews","duration_ms":55.0,"error_type":null,"result_count":1,"status":"completed","stop_reason":null}],"artifacts":[],"completed_at":"2026-08-17T12:01:00Z","counts":{"asn":1,"prefix":1},"evidence_status":"complete","result_count":2,"run_id":"123e4567-e89b-12d3-a456-426614174104","source_executions":[],"started_at":"2026-08-17T12:00:00Z","target":"AS64500","type":"summary"} +{"sources":[],"type":"asn","value":"AS64500"} +{"actions":["routeviews"],"observations":[{"action":"routeviews","collected_at":"2026-08-17T12:01:00Z","origin_asn":"AS64500","type":"observed-origin"},{"action":"routeviews","as_path":"64496 64500","collected_at":"2026-08-17T12:01:00Z","collector":"route-views.example","communities":"64496:100 64500:200","observed_at":"2026-08-17T11:59:00Z","origin_asn":"AS64500","peer_address":"2001:db8::1","peer_asn":"AS64496","type":"bgp-route"},{"action":"routeviews","collected_at":"2026-08-17T12:01:00Z","observed_at":"2026-08-17T11:59:00Z","origin_asn":"AS64500","state":"valid","type":"rpki-validation"}],"scope":"external-relationship","sources":[],"type":"prefix","value":"192.0.2.0/24"} +``` + +
+ +The [full virtual-host JSONL finding](Virtual-Host-Discovery#one-finding-multiple-endpoint-observations) shows how one confirmed hostname retains evidence from more than one endpoint. + The same `-f report` command creates `report.json` and `report.xml` compatibility reports. ## JSON and XML compatibility reports diff --git a/tests/test_readme.py b/tests/test_readme.py index cadc1cf3..8f1dcfe7 100644 --- a/tests/test_readme.py +++ b/tests/test_readme.py @@ -2,10 +2,12 @@ from __future__ import annotations import json import re +from collections import Counter from pathlib import Path import yaml +from theHarvester.lib.api.run_evidence import parse_jsonl_import from theHarvester.lib.completed_result import parse_result_jsonl from theHarvester.lib.source_catalog import ACTION_ACTIVITIES, RESULT_CAPABILITIES, SOURCE_SPECS @@ -317,6 +319,47 @@ def test_virtual_host_wiki_examples_match_the_structured_result_contract() -> No assert len(finding['observations']) == 2 +def test_wiki_action_jsonl_examples_match_the_result_contract() -> None: + page = Path('docs/wiki/Results-and-Local-Data.md').read_text() + examples = re.findall(r'```jsonl\n(.*?)\n```', page, flags=re.DOTALL) + + assert len(examples) == 4 + documented_actions: set[str] = set() + for example in examples: + summary, findings = parse_result_jsonl(example) + imported = parse_jsonl_import(example.encode()) + finding_counts = Counter(finding['type'] for finding in findings) + action_counts = Counter(action for finding in findings for action in finding.get('actions', [])) + source_counts = Counter(source for finding in findings for source in finding.get('sources', [])) + execution_counts = {execution['action']: execution['result_count'] for execution in summary['action_executions']} + source_execution_counts = {execution['source']: execution['result_count'] for execution in summary['source_executions']} + + assert summary['counts'] == dict(finding_counts) + assert summary['result_count'] == len(findings) + assert imported['run_id'] == summary['run_id'] + assert imported['results'] == findings + assert execution_counts == dict(action_counts) + assert source_execution_counts == dict(source_counts) + documented_actions.update(execution_counts) + + assert documented_actions == { + 'dns-brute', + 'dns-lookup', + 'dns-recursive', + 'dns-resolve', + 'routeviews', + 'shodan', + } + assert '[full virtual-host JSONL finding](Virtual-Host-Discovery#one-finding-multiple-endpoint-observations)' in page + + +def test_operator_workflows_include_reverse_and_recursive_dns_commands() -> None: + page = Path('docs/wiki/Operator-Workflows.md').read_text() + + assert '-b rapiddns \\\n --dns-lookup \\\n -f report' in page + assert '--dns-resolvers resolvers.txt \\\n --dns-recursive-depth 1 \\\n -f report' in page + + def test_readme_preserves_project_social_attribution() -> None: readme = Path('README.md').read_text() profiles = {