mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-19 23:01:29 +02:00
The vulnerability stage handed every http_assets row to nuclei, ordered by URL, with one 90-minute clock cutting wherever it landed. Template-spray then gave every host the first slice of the corpus in load order. On the two largest local scans that was 23.4M and 10.7M planned requests, of which 3.9% and 9% were sent, and no host recorded what it got. One GeoServer CVE was found 11 times on 11 hostnames of one server. Introduce one planner, shared/services/scan_surface/, that decides what every vulnerability scanner receives. It writes scan_surface_items per scan and is a pure function of the tables the scan already produced plus the context; nothing in it names a nuclei flag. Vocabulary lives in shared/definitions/scan_surface.py, mirrored by config/scan-surface.ts. - Inputs are typed by protocol, because nuclei rewrites input per protocol: roots for HTTP templates (any status), bare names for DNS, host:port for TLS and network, parameterised requests for fuzzing. Path and query are stripped from non-DAST input; values are normalised to one spelling. - Web assets that are one backend fold into one origin on exact agreement of address, certificate, body hash, status, title, canary and favicon — never a similarity score. Measured: 1,812 web assets to 1,607 origins on gov.np. - The corpus is spent in tiers: one-request checks on every asset, the universal set on each origin, product-matched checks through a curated tech map, the blind sweep last. Each tier is a ranked host-spray batch with its own coverage row; the budget cut lands between batches and every origin is scanned, covered or not scanned. - A finding on a representative is replayed onto each cluster member with one request, writing a real observed row or splitting the member out. Never an inferred row. - dast_scan is a new stage (off by default, after endpoint_probe): nuclei -dast over request items and exposure checks under directories, then dalfox behind a reflection probe. dalfox v3.2.2 is pinned in the worker image. Verified live on testasp.vulnweb.com: the tiers ran in order, nuclei -dast found two LFIs, dalfox executed under the reflection gate. All suites green. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01R7sBu1jBa4e2CXLS1KEsNr
148 lines
5.0 KiB
Python
148 lines
5.0 KiB
Python
"""What the vulnerability scanner was handed, read back for the UI."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections import Counter
|
|
from uuid import UUID
|
|
|
|
from sqlalchemy import func, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from shared.definitions.scan_surface import (
|
|
TIER_LABELS,
|
|
TIER_ORDER,
|
|
DropReason,
|
|
SurfaceClass,
|
|
)
|
|
from shared.definitions.vulnerabilities import CoverageStatus
|
|
from shared.models.scan_surface import (
|
|
AssetSurface,
|
|
ScanSurfaceItem,
|
|
SurfaceSummary,
|
|
SurfaceTierCount,
|
|
)
|
|
from shared.services.asset_query.scope import QueryScope, ScopeLike
|
|
|
|
_MAX_UNMAPPED = 12
|
|
_RAN = frozenset(
|
|
{
|
|
CoverageStatus.COMPLETED.value,
|
|
CoverageStatus.PARTIAL.value,
|
|
CoverageStatus.FAILED.value,
|
|
}
|
|
)
|
|
|
|
|
|
def _outcome(stamp) -> str:
|
|
return str(stamp or "").split("@", 1)[0]
|
|
|
|
|
|
class ScanSurfaceService:
|
|
def __init__(self, session: AsyncSession):
|
|
self.session = session
|
|
|
|
async def summary(self, scope: ScopeLike) -> SurfaceSummary:
|
|
scope = QueryScope.of(scope)
|
|
if not scope:
|
|
return SurfaceSummary()
|
|
rows = (
|
|
await self.session.execute(
|
|
select(
|
|
ScanSurfaceItem.class_,
|
|
ScanSurfaceItem.drop_reason,
|
|
ScanSurfaceItem.tiers_planned,
|
|
ScanSurfaceItem.tiers_done,
|
|
ScanSurfaceItem.unmapped_tech,
|
|
).where(scope.match(ScanSurfaceItem.scan_id))
|
|
)
|
|
).all()
|
|
if not rows:
|
|
return SurfaceSummary()
|
|
|
|
summary = SurfaceSummary(planned=True)
|
|
dropped: Counter[str] = Counter()
|
|
unmapped: Counter[str] = Counter()
|
|
tiers: dict[str, SurfaceTierCount] = {}
|
|
for class_, reason, planned, done, missing in rows:
|
|
unmapped.update(missing or [])
|
|
if class_ == SurfaceClass.ROOT.value:
|
|
summary.roots += 1
|
|
elif class_ == SurfaceClass.NAME.value:
|
|
summary.names += 1
|
|
elif class_ == SurfaceClass.SERVICE.value:
|
|
summary.services += 1
|
|
elif class_ == SurfaceClass.REQUEST.value:
|
|
summary.requests += 1
|
|
elif class_ == SurfaceClass.BASE.value:
|
|
summary.bases += 1
|
|
if reason == DropReason.COVERED_BY_ORIGIN.value:
|
|
summary.covered += 1
|
|
continue
|
|
if reason:
|
|
dropped[reason] += 1
|
|
continue
|
|
if class_ == SurfaceClass.ROOT.value:
|
|
summary.origins += 1
|
|
outcomes = dict(done or {})
|
|
for tier in planned or []:
|
|
count = tiers.setdefault(
|
|
tier, SurfaceTierCount(tier=tier, label=TIER_LABELS.get(tier, tier))
|
|
)
|
|
result = _outcome(outcomes.get(tier))
|
|
if result == CoverageStatus.COMPLETED.value:
|
|
count.scanned += 1
|
|
elif result in _RAN:
|
|
count.partial += 1
|
|
else:
|
|
count.not_scanned += 1
|
|
summary.dropped = dict(dropped)
|
|
summary.tiers = [tiers[t] for t in TIER_ORDER if t in tiers]
|
|
summary.unmapped_tech = [
|
|
name for name, _ in unmapped.most_common(_MAX_UNMAPPED)
|
|
]
|
|
return summary
|
|
|
|
async def for_asset(self, scan_id: UUID, asset_id: UUID) -> AssetSurface | None:
|
|
item = (
|
|
await self.session.execute(
|
|
select(ScanSurfaceItem).where(
|
|
ScanSurfaceItem.scan_id == scan_id,
|
|
ScanSurfaceItem.http_asset_id == asset_id,
|
|
ScanSurfaceItem.class_ == SurfaceClass.ROOT.value,
|
|
)
|
|
)
|
|
).scalar_one_or_none()
|
|
if item is None:
|
|
return None
|
|
representative = None
|
|
if item.representative_id is not None:
|
|
representative = await self.session.scalar(
|
|
select(ScanSurfaceItem.value).where(
|
|
ScanSurfaceItem.scan_id == scan_id,
|
|
ScanSurfaceItem.http_asset_id == item.representative_id,
|
|
ScanSurfaceItem.class_ == SurfaceClass.ROOT.value,
|
|
)
|
|
)
|
|
members = item.members
|
|
if item.representative_id is None and item.cluster_id is not None:
|
|
members = int(
|
|
await self.session.scalar(
|
|
select(func.count()).where(
|
|
ScanSurfaceItem.cluster_id == item.cluster_id
|
|
)
|
|
)
|
|
or 1
|
|
)
|
|
return AssetSurface(
|
|
state=item.state,
|
|
representative_id=item.representative_id,
|
|
representative_value=representative,
|
|
cluster_signals=list(item.cluster_signals or []),
|
|
members=members,
|
|
drop_reason=item.drop_reason,
|
|
tiers_planned=list(item.tiers_planned or []),
|
|
tiers_done=dict(item.tiers_done or {}),
|
|
tags=list(item.tags or []),
|
|
note=item.note,
|
|
)
|