feat(subdomain): discovery guesses names as well as asking for them

Discovery was passive-only — subfinder, ctfr, crtname, assetfinder and certificate
SANs — so anything absent from a public source was absent from the scan, and every
later stage inherited the miss. It now runs two more sources after the passive merge,
carrying SubdomainSource.BRUTEFORCE and PERMUTATION through the same scoping and
merge every other provider uses.

Bruteforce is `dnsx -d <apex> -w <list>` over a wordlist shipped in the repo:
4,860 labels ranked by real-world frequency, regenerated by
scripts/fetch_subdomain_wordlist.py and committed like the port registry, so an
install with no egress still has one. The word budget reads from the top of that
ranking, which is why a smaller budget is simply the first N lines. Permutation is
alterx over the names already found, seeded apex-first because a name one label from
the apex has more siblings worth guessing than a five-label one.

Two things this needed to be worth running at all:

  - **A budget, not an idle watchdog.** A guessed name that does not exist produces
    no output, so the stall detector that protects the resolver killed both runs at
    exactly 90s the first time. Guessing is bounded by total time instead, floored at
    5 names/s against a measured 9, and says so when it stops early.
  - **Wildcard rejection.** The wildcard probe moved ahead of expansion: on a wildcard
    zone every guess answers, and storing 5,000 fake names would be worse than not
    guessing. Answers that resolve only to the wildcard address are dropped and
    counted in the source's note.

Both are skipped at passive intensity and say why — the stage still touches no
target, but a wordlist run does query the target's own nameservers. Bruteforce
defaults on at 1,000 words (110s measured, 2 hits on vulnweb.com, both attributed
correctly alongside their passive sources); permutations default off, because every
variant is another DNS query.

ProviderResult gained `note` so a source can come up short without being reported as
failed. alterx reads seeds from stdin — `-l <file>` answers "no input found".

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01DPueZXQMbjLXMokXeA8LFP

scripts/fetch_subdomain_wordlist.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+"""Regenerate tools/data/subdomains.txt from SecLists' DNS discovery list.
+
+The list is MIT-licensed and slow-moving, so the generated file is committed rather
+than fetched at build time — a scan on a box with no egress still gets a wordlist.
+Run this when you want a newer cut:
+
+    python3 scripts/fetch_subdomain_wordlist.py [--size 5000]
+
+Names are ranked by how often they appeared in a real DNS corpus, so the order is the
+budget: the resolver walks the file top to bottom and a smaller wordlist is simply the
+first N lines. Keep it that way.
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+import sys
+import urllib.request
+from pathlib import Path
+
+SOURCE = (
+    "https://raw.githubusercontent.com/danielmiessler/SecLists/master/"
+    "Discovery/DNS/subdomains-top1million-{size}.txt"
+)
+SIZES = (5000, 20000, 110000)
+OUTPUT = Path(__file__).resolve().parent.parent / "tools" / "data" / "subdomains.txt"
+
+# one DNS label: what dnsx will prefix onto the apex, nothing else
+_LABEL = re.compile(r"^[a-z0-9_](?:[a-z0-9_-]{0,61}[a-z0-9_])?$")
+
+
+def fetch(size: int) -> list[str]:
+    with urllib.request.urlopen(SOURCE.format(size=size), timeout=60) as response:  # noqa: S310
+        body = response.read().decode("utf-8", errors="replace")
+    seen: set[str] = set()
+    labels: list[str] = []
+    for raw in body.splitlines():
+        label = raw.strip().lower()
+        if not label or label in seen or not _LABEL.match(label):
+            continue
+        seen.add(label)
+        labels.append(label)
+    return labels
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--size", type=int, default=5000, choices=SIZES)
+    args = parser.parse_args()
+
+    labels = fetch(args.size)
+    if len(labels) < args.size // 2:
+        print(f"only {len(labels)} usable labels, refusing to write", file=sys.stderr)
+        return 1
+    OUTPUT.write_text("\n".join(labels) + "\n")
+    print(f"wrote {OUTPUT} ({len(labels)} labels)")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

stages/subdomain/config.py

@@ -7,6 +7,8 @@ from stages.config import StageConfig, threads, timeout
 from stages.subdomain.providers import PASSIVE_PROVIDERS
 
 PASSIVE_TOOLS: tuple[str, ...] = tuple(sorted(PASSIVE_PROVIDERS))
+# ranked by real-world frequency, so a smaller budget is simply the first N lines
+DEFAULT_WORDLIST = "/app/tools/data/subdomains.txt"
 # amass is deliberately not here: it never exits early, so it costs the whole
 # tool timeout on every scan for hosts the other sources already return
 DEFAULT_PASSIVE_TOOLS: list[str] = [
@@ -35,6 +37,43 @@ class SubdomainConfig(StageConfig):
         title="TLS certificate discovery",
         description="Pull subject alternative names from the target's certificates.",
     )
+    bruteforce: bool = Field(
+        default=True,
+        title="Bruteforce names",
+        description="Ask the target's nameservers for common names the public sources never listed.",
+    )
+    wordlist: str = Field(
+        default=DEFAULT_WORDLIST,
+        max_length=500,
+        title="Wordlist",
+        description="One label per line, ranked best first. The word budget below reads from the top.",
+    )
+    wordlist_limit: int = Field(
+        default=1000,
+        ge=100,
+        le=1_000_000,
+        title="Words to try",
+        description="Names tried per apex, from the top of the list. This is a time budget: the resolver clears about 9 a second.",
+    )
+    permutations: bool = Field(
+        default=False,
+        title="Permute discovered names",
+        description="Build variants of the names already found (api → api-dev, api2, api-staging) and resolve those too.",
+    )
+    permutation_seeds: int = Field(
+        default=250,
+        ge=1,
+        le=10_000,
+        title="Names to permute",
+        description="How many discovered names to build variants from. Variants grow with the square of this.",
+    )
+    permutation_limit: int = Field(
+        default=20_000,
+        ge=100,
+        le=500_000,
+        title="Variants to resolve",
+        description="Cap on generated variants. Each one is a DNS query, on top of the word budget.",
+    )
     dns_threads: int = threads(30, title="Resolver threads")
     dns_batch_size: int = Field(
         default=1000,

stages/subdomain/providers/base.py

@@ -44,6 +44,8 @@ class ProviderResult:
     source: SubdomainSource
     subdomains: set[str] = field(default_factory=set)
     raw_count: int = 0
+    # what the source wants said about the run when it did not fail
+    note: str | None = None
     error: str | None = None
     skipped: bool = False
     skip_reason: str | None = None

stages/subdomain/stage.py

@@ -1,16 +1,21 @@
 from __future__ import annotations
 
+import contextlib
+import os
 import random
 import statistics
+import tempfile
+import time
 import uuid
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from dataclasses import dataclass, field
+from pathlib import Path
 
 from sqlalchemy import delete
 
 from shared.enums.activity import ActivityEvent, ActivityLevel
 from shared.enums.api_key import APIProvider
-from shared.enums.scan import AssetKind, Phase, StageGroup, StageRole
+from shared.enums.scan import AssetKind, Intensity, Phase, StageGroup, StageRole
 from shared.enums.subdomain import SubdomainSource
 from shared.logging import get_logger
 from shared.models.subdomain import Subdomain
@@ -27,6 +32,7 @@ from stages.subdomain.providers import (
     ProviderResult,
     SubdomainProvider,
 )
+from tools.alterx import AlterxClient, AlterxError
 from tools.dnsx.client import DnsxClient, DnsxError
 
 logger = get_logger(__name__)
@@ -44,6 +50,25 @@ _MIN_RESOLVE_THREADS = 10
 _MIN_BATCHES_FOR_MEDIAN = 3
 _DEGRADED_RATIO = 0.5
 _SHUFFLE_SEED = 1
+# measured: dnsx -t 30 clears ~9 guessed names a second, so 5/s is the floor a
+# budget may assume. Guessing is bounded by total time, never by an idle watchdog.
+_GUESS_FLOOR_RATE = 5
+_GUESS_MIN_BUDGET = 300
+_MAX_LABEL = 63
+
+
+# a name close to the apex has more siblings worth guessing than a five-label one
+def _seed_rank(name: str) -> tuple[int, int, str]:
+    return (name.count("."), len(name), name)
+
+
+def _guess_budget(count: int) -> int:
+    return max(_GUESS_MIN_BUDGET, -(-count // _GUESS_FLOOR_RATE))
+
+
+def _is_wildcard(info: dict, wildcard_ips: set[str]) -> bool:
+    ips = set(info.get("ips") or ())
+    return bool(ips) and bool(wildcard_ips) and ips <= wildcard_ips
 
 
 @dataclass
@@ -95,8 +120,12 @@ class SubdomainStage(Stage):
     config_model = SubdomainConfig
 
     def should_run(self) -> bool:
-        return self.cfg.enabled and bool(
-            self.cfg.enabled_sources or self.cfg.tls_discovery
+        cfg = self.cfg
+        return cfg.enabled and bool(
+            cfg.enabled_sources
+            or cfg.tls_discovery
+            or cfg.bruteforce
+            or cfg.permutations
         )
 
     def run(self) -> StageResult:
@@ -122,6 +151,17 @@ class SubdomainStage(Stage):
         self._check_abort()
 
         merged = merge_and_filter(results, domain, resolved.included_subdomains)
+        # a guessed name that lands on the wildcard address is not a discovery, so the
+        # probe that decides that has to run before any name is guessed
+        wildcard_ips = self._wildcard_ips(domain, cfg)
+        extra = self._expand(
+            domain, cfg, sorted(merged, key=_seed_rank), wildcard_ips, activity
+        )
+        if extra:
+            merged = merge_and_filter(
+                [*results, *extra], domain, resolved.included_subdomains
+            )
+
         # the target itself is in scope — nothing downstream runs without it
         merged.setdefault(domain, set()).add(SubdomainSource.TARGET.value)
         excluded = {n for n in merged if matches_any(n, resolved.excluded_subdomains)}
@@ -134,7 +174,6 @@ class SubdomainStage(Stage):
 
         # excluded subdomains are stored but not resolved or processed further
         to_resolve = [n for n in merged if n not in excluded]
-        wildcard_ips = self._wildcard_ips(domain, cfg)
         state = self._resolve(to_resolve, cfg)
 
         active, ips_seen = self._persist(merged, state.records, wildcard_ips, excluded)
@@ -145,11 +184,181 @@ class SubdomainStage(Stage):
                 "active": active,
                 "ips": len(ips_seen),
                 "excluded": len(excluded),
+                **{r.source.value: len(r.subdomains) for r in extra if r.subdomains},
             },
             warnings=self._resolution_warnings(state),
             partial=state.lost,
         )
 
+    def _expand(
+        self,
+        domain: str,
+        cfg: SubdomainConfig,
+        seeds: list[str],
+        wildcard_ips: set[str],
+        activity: ActivityLogService,
+    ) -> list[ProviderResult]:
+        """Names no public source listed: guessed from a wordlist, then built from what was found."""
+        passive = self.ctx.resolved.intensity == Intensity.PASSIVE.value
+        out: list[ProviderResult] = []
+        if cfg.bruteforce:
+            out.append(self._bruteforce(domain, cfg, wildcard_ips, passive=passive))
+        if cfg.permutations:
+            out.append(self._permute(cfg, seeds, wildcard_ips, passive=passive))
+        for result in out:
+            self._check_abort()
+            self._log_provider(activity, result)
+        return out
+
+    @staticmethod
+    def _skipped(source: SubdomainSource, reason: str) -> ProviderResult:
+        return ProviderResult(source=source, skipped=True, skip_reason=reason)
+
+    @contextlib.contextmanager
+    def _wordlist(self, cfg: SubdomainConfig):
+        """The word budget is the first N lines, because the file is ranked best first."""
+        path = Path(cfg.wordlist)
+        if not path.is_file():
+            yield None
+            return
+        words: list[str] = []
+        with path.open(encoding="utf-8", errors="replace") as handle:
+            for line in handle:
+                word = line.strip().lower()
+                # a DNS label is 63 characters; anything longer is not a word list
+                if word and not word.startswith("#") and len(word) <= _MAX_LABEL:
+                    words.append(word)
+                if len(words) >= cfg.wordlist_limit:
+                    break
+        if not words:
+            yield None
+            return
+        fd, name = tempfile.mkstemp(prefix="wordlist_", suffix=".txt")
+        try:
+            with os.fdopen(fd, "w") as handle:
+                handle.write("\n".join(words) + "\n")
+            yield name, len(words)
+        finally:
+            with contextlib.suppress(OSError):
+                Path(name).unlink(missing_ok=True)
+
+    def _bruteforce(
+        self,
+        domain: str,
+        cfg: SubdomainConfig,
+        wildcard_ips: set[str],
+        *,
+        passive: bool,
+    ) -> ProviderResult:
+        source = SubdomainSource.BRUTEFORCE
+        if passive:
+            return self._skipped(
+                source, "a passive scan does not query the target's nameservers"
+            )
+        client = self._client(cfg)
+        if client is None:
+            return self._skipped(source, "dnsx is not installed on this instance")
+
+        with self._wordlist(cfg) as prepared:
+            if prepared is None:
+                return self._skipped(source, f"no wordlist at {cfg.wordlist}")
+            path, tried = prepared
+            self.emit_progress(f"trying {tried:,} names against {domain}")
+            start = time.monotonic()
+            found: set[str] = set()
+            wildcards = 0
+            with client.stream_brute(
+                domain, path, timeout=_guess_budget(tried)
+            ) as stream:
+                for record in stream.records:
+                    parsed = self._record(record)
+                    if parsed is None:
+                        continue
+                    name, info = parsed
+                    if _is_wildcard(info, wildcard_ips):
+                        wildcards += 1
+                        continue
+                    found.add(name)
+                    self._check_abort()
+
+            cut_short = stream.timed_out
+
+        notes = []
+        if wildcards:
+            notes.append(
+                f"{wildcards:,} of {wildcards + len(found):,} answers were the "
+                "wildcard address and were dropped"
+            )
+        if cut_short:
+            notes.append(f"stopped at the {_guess_budget(tried):,}s budget")
+        return ProviderResult(
+            source=source,
+            subdomains=found,
+            raw_count=len(found),
+            note="; ".join(notes) or None,
+            duration_seconds=round(time.monotonic() - start, 2),
+        )
+
+    def _permute(
+        self,
+        cfg: SubdomainConfig,
+        seeds: list[str],
+        wildcard_ips: set[str],
+        *,
+        passive: bool,
+    ) -> ProviderResult:
+        source = SubdomainSource.PERMUTATION
+        if passive:
+            return self._skipped(
+                source, "a passive scan does not query the target's nameservers"
+            )
+        if not seeds:
+            return self._skipped(
+                source, "nothing was discovered to build variants from"
+            )
+        try:
+            client = AlterxClient(
+                limit=cfg.permutation_limit,
+                recorder=self.ctx.recorder,
+                extra_args=self.ctx.resolved.tool_args("alterx"),
+            )
+        except AlterxError:
+            return self._skipped(source, "alterx is not installed on this instance")
+
+        start = time.monotonic()
+        candidates = client.permute(seeds[: cfg.permutation_seeds])
+        if not candidates:
+            return ProviderResult(source=source, duration_seconds=0.0)
+        self.emit_progress(f"resolving {len(candidates):,} name variants")
+        client = self._client(cfg)
+        if client is None:
+            return self._skipped(source, "dnsx is not installed on this instance")
+        found: set[str] = set()
+        # same contract as bruteforce: a variant that does not exist says nothing back
+        with client.stream_query(
+            candidates,
+            record_types=["a", "aaaa", "cname"],
+            timeout=_guess_budget(len(candidates)),
+        ) as stream:
+            for record in stream.records:
+                parsed = self._record(record)
+                if parsed is None:
+                    continue
+                name, info = parsed
+                if info.get("active") and not _is_wildcard(info, wildcard_ips):
+                    found.add(name)
+                self._check_abort()
+        notes = [f"{len(candidates):,} variants resolved"]
+        if stream.timed_out:
+            notes.append("stopped at the budget")
+        return ProviderResult(
+            source=source,
+            subdomains=found,
+            raw_count=len(found),
+            note="; ".join(notes),
+            duration_seconds=round(time.monotonic() - start, 2),
+        )
+
     def _wildcard_ips(self, domain: str, cfg: SubdomainConfig) -> set[str]:
         probe = f"{uuid.uuid4().hex[:12]}.{domain}"
         info = self._resolve([probe], cfg).records.get(probe)
@@ -242,9 +451,7 @@ class SubdomainStage(Stage):
         self.emit_progress(message, source=source)
 
     def _client(self, cfg: SubdomainConfig) -> DnsxClient | None:
-        threads = min(
-            max(cfg.dns_threads, _MIN_RESOLVE_THREADS), _MAX_RESOLVE_THREADS
-        )
+        threads = min(max(cfg.dns_threads, _MIN_RESOLVE_THREADS), _MAX_RESOLVE_THREADS)
         try:
             return DnsxClient(
                 timeout=max(120, cfg.tool_timeout(self.ctx.resolved.intensity)),

tools/alterx/__init__.py

@@ -0,0 +1,3 @@
+from tools.alterx.client import AlterxClient, AlterxError
+
+__all__ = ["AlterxClient", "AlterxError"]

tools/alterx/client.py

@@ -0,0 +1,72 @@
+"""alterx CLI client — permutation candidates generated from names already discovered."""
+
+from __future__ import annotations
+
+from shared.logging import get_logger
+from tools.runner import CLIToolRunner, OutputFormat, ToolNotFoundError
+from tools.runner.models import CommandRecorder
+
+logger = get_logger(__name__)
+
+ALTERX_BINARY = "alterx"
+DEFAULT_TIMEOUT = 600
+
+
+class AlterxError(Exception):
+    pass
+
+
+class AlterxClient:
+    def __init__(
+        self,
+        *,
+        limit: int,
+        enrich: bool = True,
+        patterns: list[str] | None = None,
+        recorder: CommandRecorder | None = None,
+        extra_args: list[str] | None = None,
+    ) -> None:
+        self.limit = max(1, limit)
+        self.enrich = enrich
+        self.patterns = patterns or []
+        self.recorder = recorder
+        self.extra_args = extra_args or []
+
+        try:
+            self._runner = CLIToolRunner(ALTERX_BINARY, default_timeout=DEFAULT_TIMEOUT)
+        except ToolNotFoundError as e:
+            raise AlterxError(str(e)) from e
+
+    def permute(self, names: list[str]) -> list[str]:
+        """Candidate hostnames built from the seeds. Generates, never resolves."""
+        if not names:
+            return []
+        args = ["-limit", str(self.limit)]
+        if self.enrich:
+            # words mined from the seeds themselves, which is what beats a blind list
+            args.append("-enrich")
+        for pattern in self.patterns:
+            args += ["-p", pattern]
+
+        result = self._runner.run(
+            args=args,
+            input_data=names,
+            # alterx reads its seeds from stdin; -l with a file path yields "no input found"
+            use_stdin=True,
+            use_output_file=False,
+            output_format=OutputFormat.PLAIN,
+            silent=True,
+            silent_flag="-silent",
+            recorder=self.recorder,
+            tool=ALTERX_BINARY,
+            extra_args=self.extra_args,
+        )
+        seeds = set(names)
+        out: list[str] = []
+        for line in result.output_lines:
+            candidate = line.strip().lower()
+            if candidate and candidate not in seeds:
+                out.append(candidate)
+            if len(out) >= self.limit:
+                break
+        return out

tools/dnsx/client.py

@@ -105,6 +105,7 @@ class DnsxClient:
         record_types: list[str] | None = None,
         *,
         idle_timeout: int | None = None,
+        timeout: int = 0,
     ) -> Iterator[StreamOutcome]:
         """Resolve in bulk, streaming records as they land so a stall never zeroes the run."""
         args = self._build_base_args()
@@ -118,7 +119,7 @@ class DnsxClient:
             json_flag="-json",
             silent=True,
             silent_flag="-silent",
-            timeout=0,
+            timeout=timeout,
             idle_timeout=idle_timeout,
             recorder=self.recorder,
             tool=DNSX_BINARY,
@@ -126,6 +127,33 @@ class DnsxClient:
         ) as stream:
             yield stream
 
+    @contextlib.contextmanager
+    def stream_brute(
+        self,
+        domain: str,
+        wordlist: str,
+        *,
+        timeout: int,
+    ) -> Iterator[StreamOutcome]:
+        """Bruteforce one apex from a wordlist, streaming only the names that answer.
+
+        Bounded by a total budget, never an idle watchdog: a guessed name that does not
+        exist produces no output, so silence here is the normal case, not a stall.
+        """
+        args = self._build_base_args()
+        args += ["-d", domain, "-w", wordlist, "-a", "-aaaa", "-resp"]
+        with self._runner.stream_json(
+            args=args,
+            json_flag="-json",
+            silent=True,
+            silent_flag="-silent",
+            timeout=timeout,
+            recorder=self.recorder,
+            tool=DNSX_BINARY,
+            extra_args=self.extra_args,
+        ) as stream:
+            yield stream
+
     def resolve(
         self,
         targets: str | list[str],

worker/Dockerfile

@@ -30,6 +30,7 @@ ENV PATH="/usr/local/go/bin:/root/go/bin:${PATH}"
 RUN go install -v github.com/projectdiscovery/dnsx/cmd/dnsx@latest \
     && go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest \
     && go install -v github.com/projectdiscovery/tlsx/cmd/tlsx@latest \
+    && go install -v github.com/projectdiscovery/alterx/cmd/alterx@latest \
     && go install -v github.com/tomnomnom/assetfinder@latest \
     && go install -v github.com/owasp-amass/amass/v4/...@master