mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-20 15:21:30 +02:00
Yogesh OjhaandClaude Opus 4.8
dd7ff0f6f1
The vulnerability stage handed every http_assets row to nuclei, ordered by URL, with one 90-minute clock cutting wherever it landed. Template-spray then gave every host the first slice of the corpus in load order. On the two largest local scans that was 23.4M and 10.7M planned requests, of which 3.9% and 9% were sent, and no host recorded what it got. One GeoServer CVE was found 11 times on 11 hostnames of one server. Introduce one planner, shared/services/scan_surface/, that decides what every vulnerability scanner receives. It writes scan_surface_items per scan and is a pure function of the tables the scan already produced plus the context; nothing in it names a nuclei flag. Vocabulary lives in shared/definitions/scan_surface.py, mirrored by config/scan-surface.ts. - Inputs are typed by protocol, because nuclei rewrites input per protocol: roots for HTTP templates (any status), bare names for DNS, host:port for TLS and network, parameterised requests for fuzzing. Path and query are stripped from non-DAST input; values are normalised to one spelling. - Web assets that are one backend fold into one origin on exact agreement of address, certificate, body hash, status, title, canary and favicon — never a similarity score. Measured: 1,812 web assets to 1,607 origins on gov.np. - The corpus is spent in tiers: one-request checks on every asset, the universal set on each origin, product-matched checks through a curated tech map, the blind sweep last. Each tier is a ranked host-spray batch with its own coverage row; the budget cut lands between batches and every origin is scanned, covered or not scanned. - A finding on a representative is replayed onto each cluster member with one request, writing a real observed row or splitting the member out. Never an inferred row. - dast_scan is a new stage (off by default, after endpoint_probe): nuclei -dast over request items and exposure checks under directories, then dalfox behind a reflection probe. dalfox v3.2.2 is pinned in the worker image. Verified live on testasp.vulnweb.com: the tiers ran in order, nuclei -dast found two LFIs, dalfox executed under the reflection gate. All suites green. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01R7sBu1jBa4e2CXLS1KEsNr
81 lines
2.2 KiB
Python
81 lines
2.2 KiB
Python
"""What a web asset was seen running, as the tags the check library uses."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Iterable
|
|
|
|
from shared.definitions.scan_surface import (
|
|
TECH_IGNORED,
|
|
normalize_tech,
|
|
tags_for_tech,
|
|
)
|
|
|
|
_CPE_PARTS = 5
|
|
|
|
|
|
def _cpe_product(cpe: str) -> str | None:
|
|
parts = (cpe or "").split(":")
|
|
if len(parts) < _CPE_PARTS or parts[0] != "cpe":
|
|
return None
|
|
if parts[1] == "2.3":
|
|
return parts[4] or None
|
|
return parts[3] or None
|
|
|
|
|
|
def _software_names(software: Iterable) -> list[str]:
|
|
names: list[str] = []
|
|
for entry in software or ():
|
|
if isinstance(entry, dict):
|
|
name = entry.get("name") or entry.get("product")
|
|
else:
|
|
name = entry
|
|
if isinstance(name, str) and name.strip():
|
|
names.append(name)
|
|
return names
|
|
|
|
|
|
def host_tags(
|
|
*,
|
|
tech: Iterable[str],
|
|
cpe: Iterable[str],
|
|
webserver: str | None,
|
|
software: Iterable,
|
|
vocabulary: frozenset[str],
|
|
) -> tuple[list[str], list[str]]:
|
|
"""Tags the library carries for this software, and the names that map to none."""
|
|
found: set[str] = set()
|
|
unmapped: list[str] = []
|
|
seen: set[str] = set()
|
|
|
|
names: list[str] = [*tech, *_software_names(software)]
|
|
if webserver:
|
|
names.append(webserver)
|
|
for name in names:
|
|
key = normalize_tech(name)
|
|
if not key or key in seen:
|
|
continue
|
|
seen.add(key)
|
|
mapped = tags_for_tech(name)
|
|
if mapped is None:
|
|
compact = key.replace(" ", "")
|
|
if compact in vocabulary:
|
|
found.add(compact)
|
|
elif key in vocabulary:
|
|
found.add(key)
|
|
elif key not in TECH_IGNORED:
|
|
unmapped.append(name.split(":", 1)[0].strip())
|
|
continue
|
|
found.update(t for t in mapped if t in vocabulary)
|
|
|
|
for entry in cpe:
|
|
product = _cpe_product(str(entry))
|
|
if not product:
|
|
continue
|
|
token = normalize_tech(product.replace("_", " ")).replace(" ", "")
|
|
if token in vocabulary:
|
|
found.add(token)
|
|
return sorted(found), sorted(set(unmapped))
|
|
|
|
|
|
__all__ = ["host_tags"]
|