mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-18 22:33:24 +02:00
Yogesh OjhaandClaude Opus 4.8
dd7ff0f6f1
The vulnerability stage handed every http_assets row to nuclei, ordered by URL, with one 90-minute clock cutting wherever it landed. Template-spray then gave every host the first slice of the corpus in load order. On the two largest local scans that was 23.4M and 10.7M planned requests, of which 3.9% and 9% were sent, and no host recorded what it got. One GeoServer CVE was found 11 times on 11 hostnames of one server. Introduce one planner, shared/services/scan_surface/, that decides what every vulnerability scanner receives. It writes scan_surface_items per scan and is a pure function of the tables the scan already produced plus the context; nothing in it names a nuclei flag. Vocabulary lives in shared/definitions/scan_surface.py, mirrored by config/scan-surface.ts. - Inputs are typed by protocol, because nuclei rewrites input per protocol: roots for HTTP templates (any status), bare names for DNS, host:port for TLS and network, parameterised requests for fuzzing. Path and query are stripped from non-DAST input; values are normalised to one spelling. - Web assets that are one backend fold into one origin on exact agreement of address, certificate, body hash, status, title, canary and favicon — never a similarity score. Measured: 1,812 web assets to 1,607 origins on gov.np. - The corpus is spent in tiers: one-request checks on every asset, the universal set on each origin, product-matched checks through a curated tech map, the blind sweep last. Each tier is a ranked host-spray batch with its own coverage row; the budget cut lands between batches and every origin is scanned, covered or not scanned. - A finding on a representative is replayed onto each cluster member with one request, writing a real observed row or splitting the member out. Never an inferred row. - dast_scan is a new stage (off by default, after endpoint_probe): nuclei -dast over request items and exposure checks under directories, then dalfox behind a reflection probe. dalfox v3.2.2 is pinned in the worker image. Verified live on testasp.vulnweb.com: the tiers ran in order, nuclei -dast found two LFIs, dalfox executed under the reflection gate. All suites green. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01R7sBu1jBa4e2CXLS1KEsNr
138 lines
4.6 KiB
Python
138 lines
4.6 KiB
Python
import uuid
|
|
from datetime import datetime
|
|
|
|
from pydantic import BaseModel
|
|
from pydantic import Field as PydanticField
|
|
from sqlalchemy import Column, Index
|
|
from sqlalchemy.types import JSON
|
|
from sqlmodel import Field, SQLModel
|
|
|
|
from shared.definitions.scan_surface import SurfaceClass, SurfaceState
|
|
from shared.utils.datetime import utc_now
|
|
|
|
|
|
def _json_list() -> Field:
|
|
return Field(default_factory=list, sa_column=Column(JSON, nullable=False))
|
|
|
|
|
|
def _json_dict() -> Field:
|
|
return Field(default_factory=dict, sa_column=Column(JSON, nullable=False))
|
|
|
|
|
|
class ScanSurfaceItem(SQLModel, table=True):
|
|
"""One thing a vulnerability scanner was handed, or deliberately not."""
|
|
|
|
__tablename__ = "scan_surface_items"
|
|
__table_args__ = (
|
|
Index("ix_scan_surface_scan_class", "scan_id", "class"),
|
|
Index("ix_scan_surface_scan_asset", "scan_id", "http_asset_id"),
|
|
Index("ix_scan_surface_target_value", "target_id", "value"),
|
|
)
|
|
|
|
id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True)
|
|
scan_id: uuid.UUID = Field(foreign_key="scans.id", index=True, ondelete="CASCADE")
|
|
target_id: uuid.UUID = Field(
|
|
foreign_key="targets.id", index=True, ondelete="CASCADE"
|
|
)
|
|
project_id: uuid.UUID = Field(foreign_key="projects.id", index=True)
|
|
|
|
class_: str = Field(
|
|
default=SurfaceClass.ROOT.value,
|
|
max_length=16,
|
|
sa_column_kwargs={"name": "class"},
|
|
)
|
|
value: str = Field(max_length=2000)
|
|
host: str | None = Field(default=None, max_length=500)
|
|
port: int | None = Field(default=None)
|
|
scheme: str | None = Field(default=None, max_length=8)
|
|
|
|
http_asset_id: uuid.UUID | None = Field(default=None)
|
|
endpoint_id: uuid.UUID | None = Field(default=None)
|
|
port_id: uuid.UUID | None = Field(default=None)
|
|
subdomain_id: uuid.UUID | None = Field(default=None)
|
|
|
|
cluster_id: uuid.UUID | None = Field(default=None, index=True)
|
|
representative_id: uuid.UUID | None = Field(default=None, index=True)
|
|
cluster_signals: list = _json_list()
|
|
members: int = Field(default=1)
|
|
|
|
drop_reason: str | None = Field(default=None, max_length=32)
|
|
rank: float = Field(default=0.0)
|
|
batch: int | None = Field(default=None)
|
|
guarded: bool = Field(default=False)
|
|
tags: list = _json_list()
|
|
unmapped_tech: list = _json_list()
|
|
|
|
tiers_planned: list = _json_list()
|
|
tiers_done: dict = _json_dict()
|
|
state: str = Field(default=SurfaceState.PLANNED.value, max_length=16)
|
|
note: str | None = Field(default=None, max_length=500)
|
|
|
|
created_at: datetime = Field(default_factory=utc_now)
|
|
|
|
|
|
class SurfaceItemRead(BaseModel):
|
|
id: uuid.UUID
|
|
class_: str = PydanticField(alias="class")
|
|
value: str
|
|
host: str | None = None
|
|
port: int | None = None
|
|
scheme: str | None = None
|
|
http_asset_id: uuid.UUID | None = None
|
|
cluster_id: uuid.UUID | None = None
|
|
representative_id: uuid.UUID | None = None
|
|
representative_value: str | None = None
|
|
cluster_signals: list[str] = PydanticField(default_factory=list)
|
|
members: int = 1
|
|
drop_reason: str | None = None
|
|
rank: float = 0.0
|
|
batch: int | None = None
|
|
guarded: bool = False
|
|
tags: list[str] = PydanticField(default_factory=list)
|
|
unmapped_tech: list[str] = PydanticField(default_factory=list)
|
|
tiers_planned: list[str] = PydanticField(default_factory=list)
|
|
tiers_done: dict[str, str] = PydanticField(default_factory=dict)
|
|
state: str
|
|
note: str | None = None
|
|
|
|
model_config = {"populate_by_name": True}
|
|
|
|
|
|
class SurfaceTierCount(BaseModel):
|
|
tier: str
|
|
label: str
|
|
scanned: int = 0
|
|
partial: int = 0
|
|
not_scanned: int = 0
|
|
|
|
|
|
class SurfaceSummary(BaseModel):
|
|
"""What the scanner was handed, per class, and how far each tier got."""
|
|
|
|
planned: bool = False
|
|
roots: int = 0
|
|
origins: int = 0
|
|
covered: int = 0
|
|
dropped: dict[str, int] = PydanticField(default_factory=dict)
|
|
names: int = 0
|
|
services: int = 0
|
|
requests: int = 0
|
|
bases: int = 0
|
|
tiers: list[SurfaceTierCount] = PydanticField(default_factory=list)
|
|
unmapped_tech: list[str] = PydanticField(default_factory=list)
|
|
|
|
|
|
class AssetSurface(BaseModel):
|
|
"""What the vulnerability scan did with one web asset."""
|
|
|
|
state: str
|
|
representative_id: uuid.UUID | None = None
|
|
representative_value: str | None = None
|
|
cluster_signals: list[str] = PydanticField(default_factory=list)
|
|
members: int = 1
|
|
drop_reason: str | None = None
|
|
tiers_planned: list[str] = PydanticField(default_factory=list)
|
|
tiers_done: dict[str, str] = PydanticField(default_factory=dict)
|
|
tags: list[str] = PydanticField(default_factory=list)
|
|
note: str | None = None
|