mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-21 07:41:28 +02:00
Yogesh OjhaandClaude Opus 4.8
dd7ff0f6f1
The vulnerability stage handed every http_assets row to nuclei, ordered by URL, with one 90-minute clock cutting wherever it landed. Template-spray then gave every host the first slice of the corpus in load order. On the two largest local scans that was 23.4M and 10.7M planned requests, of which 3.9% and 9% were sent, and no host recorded what it got. One GeoServer CVE was found 11 times on 11 hostnames of one server. Introduce one planner, shared/services/scan_surface/, that decides what every vulnerability scanner receives. It writes scan_surface_items per scan and is a pure function of the tables the scan already produced plus the context; nothing in it names a nuclei flag. Vocabulary lives in shared/definitions/scan_surface.py, mirrored by config/scan-surface.ts. - Inputs are typed by protocol, because nuclei rewrites input per protocol: roots for HTTP templates (any status), bare names for DNS, host:port for TLS and network, parameterised requests for fuzzing. Path and query are stripped from non-DAST input; values are normalised to one spelling. - Web assets that are one backend fold into one origin on exact agreement of address, certificate, body hash, status, title, canary and favicon — never a similarity score. Measured: 1,812 web assets to 1,607 origins on gov.np. - The corpus is spent in tiers: one-request checks on every asset, the universal set on each origin, product-matched checks through a curated tech map, the blind sweep last. Each tier is a ranked host-spray batch with its own coverage row; the budget cut lands between batches and every origin is scanned, covered or not scanned. - A finding on a representative is replayed onto each cluster member with one request, writing a real observed row or splitting the member out. Never an inferred row. - dast_scan is a new stage (off by default, after endpoint_probe): nuclei -dast over request items and exposure checks under directories, then dalfox behind a reflection probe. dalfox v3.2.2 is pinned in the worker image. Verified live on testasp.vulnweb.com: the tiers ran in order, nuclei -dast found two LFIs, dalfox executed under the reflection gate. All suites green. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01R7sBu1jBa4e2CXLS1KEsNr
64 lines
1.8 KiB
Python
64 lines
1.8 KiB
Python
"""One spelling per target: nuclei deduplicates its input byte for byte."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from urllib.parse import urlsplit
|
|
|
|
from shared.utils.net import bracketed, host_port
|
|
|
|
DEFAULT_PORTS: dict[str, int] = {"http": 80, "https": 443}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Root:
|
|
scheme: str
|
|
host: str
|
|
port: int
|
|
|
|
@property
|
|
def value(self) -> str:
|
|
return root_value(self.scheme, self.host, self.port)
|
|
|
|
|
|
def root_value(scheme: str, host: str, port: int | None) -> str:
|
|
"""`scheme://host[:port]`, lowercase, default port omitted, no path."""
|
|
scheme = (scheme or "https").lower()
|
|
host = (host or "").lower().strip("[]")
|
|
literal = bracketed(host)
|
|
if port and port != DEFAULT_PORTS.get(scheme):
|
|
return f"{scheme}://{literal}:{port}"
|
|
return f"{scheme}://{literal}"
|
|
|
|
|
|
def parse_root(
|
|
url: str, *, scheme: str | None = None, port: int | None = None
|
|
) -> Root | None:
|
|
"""The origin a URL names, or None when it names none."""
|
|
text = (url or "").strip()
|
|
if not text:
|
|
return None
|
|
if "://" not in text:
|
|
text = f"{scheme or 'https'}://{text}"
|
|
parsed = urlsplit(text)
|
|
host = (parsed.hostname or "").lower()
|
|
if not host:
|
|
return None
|
|
scheme_value = (parsed.scheme or scheme or "https").lower()
|
|
try:
|
|
port_value = parsed.port
|
|
except ValueError:
|
|
port_value = None
|
|
port_value = port_value or port or DEFAULT_PORTS.get(scheme_value)
|
|
if not port_value:
|
|
return None
|
|
return Root(scheme=scheme_value, host=host, port=int(port_value))
|
|
|
|
|
|
def service_value(host: str, port: int) -> str:
|
|
"""`host:port`, the form ssl and network checks take."""
|
|
return host_port((host or "").lower(), int(port))
|
|
|
|
|
|
__all__ = ["DEFAULT_PORTS", "Root", "parse_root", "root_value", "service_value"]
|