mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-20 23:31:31 +02:00
Yogesh OjhaandClaude Opus 4.8
dd7ff0f6f1
The vulnerability stage handed every http_assets row to nuclei, ordered by URL, with one 90-minute clock cutting wherever it landed. Template-spray then gave every host the first slice of the corpus in load order. On the two largest local scans that was 23.4M and 10.7M planned requests, of which 3.9% and 9% were sent, and no host recorded what it got. One GeoServer CVE was found 11 times on 11 hostnames of one server. Introduce one planner, shared/services/scan_surface/, that decides what every vulnerability scanner receives. It writes scan_surface_items per scan and is a pure function of the tables the scan already produced plus the context; nothing in it names a nuclei flag. Vocabulary lives in shared/definitions/scan_surface.py, mirrored by config/scan-surface.ts. - Inputs are typed by protocol, because nuclei rewrites input per protocol: roots for HTTP templates (any status), bare names for DNS, host:port for TLS and network, parameterised requests for fuzzing. Path and query are stripped from non-DAST input; values are normalised to one spelling. - Web assets that are one backend fold into one origin on exact agreement of address, certificate, body hash, status, title, canary and favicon — never a similarity score. Measured: 1,812 web assets to 1,607 origins on gov.np. - The corpus is spent in tiers: one-request checks on every asset, the universal set on each origin, product-matched checks through a curated tech map, the blind sweep last. Each tier is a ranked host-spray batch with its own coverage row; the budget cut lands between batches and every origin is scanned, covered or not scanned. - A finding on a representative is replayed onto each cluster member with one request, writing a real observed row or splitting the member out. Never an inferred row. - dast_scan is a new stage (off by default, after endpoint_probe): nuclei -dast over request items and exposure checks under directories, then dalfox behind a reflection probe. dalfox v3.2.2 is pinned in the worker image. Verified live on testasp.vulnweb.com: the tiers ran in order, nuclei -dast found two LFIs, dalfox executed under the reflection gate. All suites green. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01R7sBu1jBa4e2CXLS1KEsNr
212 lines
7.8 KiB
Python
212 lines
7.8 KiB
Python
import uuid
|
|
from datetime import datetime
|
|
from typing import Any
|
|
|
|
from pydantic import BaseModel
|
|
from sqlalchemy import BigInteger, Column, Computed, Text
|
|
from sqlalchemy.dialects.postgresql import TSVECTOR
|
|
from sqlalchemy.types import JSON
|
|
from sqlmodel import Field, SQLModel, UniqueConstraint
|
|
|
|
from shared.models.scan_surface import AssetSurface
|
|
from shared.utils.datetime import utc_now
|
|
|
|
HEADER_SEARCH_LIMIT = 20_000
|
|
BODY_SEARCH_LIMIT = 100_000
|
|
SEARCH_TSV_SQL = (
|
|
"setweight(to_tsvector('simple', "
|
|
f"left(coalesce(raw_response_header, ''), {HEADER_SEARCH_LIMIT})), 'A') || "
|
|
"setweight(to_tsvector('simple', "
|
|
f"left(coalesce(response_body, ''), {BODY_SEARCH_LIMIT})), 'B')"
|
|
)
|
|
|
|
|
|
def _json_list() -> Field:
|
|
return Field(default_factory=list, sa_column=Column(JSON, nullable=False))
|
|
|
|
|
|
def _text() -> Field:
|
|
return Field(default=None, sa_column=Column(Text, nullable=True))
|
|
|
|
|
|
def _json_optional_list() -> Field:
|
|
return Field(default=None, sa_column=Column(JSON(none_as_null=True), nullable=True))
|
|
|
|
|
|
class HttpAsset(SQLModel, table=True):
|
|
__tablename__ = "http_assets"
|
|
__table_args__ = (UniqueConstraint("scan_id", "url", name="uq_httpasset_scan_url"),)
|
|
|
|
id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True)
|
|
scan_id: uuid.UUID = Field(foreign_key="scans.id", index=True, ondelete="CASCADE")
|
|
target_id: uuid.UUID = Field(
|
|
foreign_key="targets.id", index=True, ondelete="CASCADE"
|
|
)
|
|
project_id: uuid.UUID = Field(foreign_key="projects.id", index=True)
|
|
|
|
# request
|
|
url: str = Field(max_length=2000)
|
|
final_url: str | None = Field(default=None, max_length=2000)
|
|
host: str = Field(max_length=500, index=True)
|
|
port: int = Field(default=0)
|
|
scheme: str = Field(default="https", max_length=8)
|
|
method: str | None = Field(default=None, max_length=10)
|
|
path: str | None = Field(default=None, max_length=2000)
|
|
|
|
# response
|
|
status_code: int | None = Field(default=None, index=True)
|
|
chain_status_codes: list = _json_list()
|
|
title: str | None = Field(default=None, max_length=1000)
|
|
webserver: str | None = Field(default=None, max_length=255)
|
|
content_type: str | None = Field(default=None, max_length=255)
|
|
content_length: int | None = Field(default=None)
|
|
location: str | None = Field(default=None, max_length=2000)
|
|
response_time: float | None = Field(default=None)
|
|
words: int | None = Field(default=None)
|
|
lines: int | None = Field(default=None)
|
|
|
|
# fingerprint
|
|
tech: list = _json_list()
|
|
cpe: list = _json_list()
|
|
favicon_hash: str | None = Field(default=None, max_length=64)
|
|
favicon_path: str | None = Field(default=None, max_length=2000)
|
|
content_hash: str | None = Field(default=None, max_length=80)
|
|
header_hash: str | None = Field(default=None, max_length=80)
|
|
jarm: str | None = Field(default=None, max_length=64)
|
|
supports_http2: bool = Field(default=False)
|
|
supports_pipeline: bool = Field(default=False)
|
|
|
|
# network
|
|
ip: str | None = Field(default=None, max_length=45, index=True)
|
|
a_records: list = _json_list()
|
|
aaaa_records: list = _json_list()
|
|
cname: str | None = Field(default=None, max_length=500)
|
|
asn: int | None = Field(default=None, sa_type=BigInteger)
|
|
asn_org: str | None = Field(default=None, max_length=255)
|
|
is_cdn: bool = Field(default=False, index=True)
|
|
cdn_name: str | None = Field(default=None, max_length=100)
|
|
cdn_type: str | None = Field(default=None, max_length=20)
|
|
waf: str | None = Field(default=None, max_length=100)
|
|
|
|
# tls
|
|
tls_version: str | None = Field(default=None, max_length=20)
|
|
tls_cipher: str | None = Field(default=None, max_length=100)
|
|
tls_subject_cn: str | None = Field(default=None, max_length=500)
|
|
tls_subject_dn: str | None = Field(default=None, max_length=1000)
|
|
tls_sans: list = _json_list()
|
|
tls_issuer: str | None = Field(default=None, max_length=500)
|
|
tls_issuer_cn: str | None = Field(default=None, max_length=500)
|
|
tls_issuer_org: str | None = Field(default=None, max_length=500)
|
|
tls_serial: str | None = Field(default=None, max_length=200)
|
|
tls_fingerprint: str | None = Field(default=None, max_length=80)
|
|
tls_not_before: datetime | None = Field(default=None)
|
|
tls_not_after: datetime | None = Field(default=None)
|
|
tls_expired: bool | None = Field(default=None)
|
|
tls_self_signed: bool | None = Field(default=None)
|
|
|
|
screenshot_path: str | None = Field(default=None, max_length=500)
|
|
screenshot_phash: int | None = Field(
|
|
default=None, sa_column=Column(BigInteger, nullable=True)
|
|
)
|
|
|
|
# raw capture (httpx -irr)
|
|
raw_request: str | None = _text()
|
|
raw_response_header: str | None = _text()
|
|
response_body: str | None = _text()
|
|
response_headers: dict = Field(
|
|
default_factory=dict, sa_column=Column(JSON, nullable=False)
|
|
)
|
|
body_preview: str | None = Field(default=None, max_length=512)
|
|
hygiene_issues: list | None = _json_optional_list()
|
|
hygiene_checked: list | None = _json_optional_list()
|
|
not_found: list | None = _json_optional_list()
|
|
search_tsv: Any | None = Field(
|
|
default=None,
|
|
sa_column=Column(TSVECTOR, Computed(SEARCH_TSV_SQL, persisted=True)),
|
|
)
|
|
|
|
discovered_at: datetime = Field(default_factory=utc_now)
|
|
created_at: datetime = Field(default_factory=utc_now)
|
|
|
|
|
|
class HttpAssetRead(BaseModel):
|
|
id: uuid.UUID
|
|
scan_id: uuid.UUID
|
|
target_id: uuid.UUID
|
|
url: str
|
|
final_url: str | None = None
|
|
host: str
|
|
port: int
|
|
scheme: str
|
|
method: str | None = None
|
|
path: str | None = None
|
|
status_code: int | None = None
|
|
chain_status_codes: list[int] = Field(default_factory=list)
|
|
title: str | None = None
|
|
webserver: str | None = None
|
|
content_type: str | None = None
|
|
content_length: int | None = None
|
|
location: str | None = None
|
|
response_time: float | None = None
|
|
words: int | None = None
|
|
lines: int | None = None
|
|
tech: list[str] = Field(default_factory=list)
|
|
cpe: list = Field(default_factory=list)
|
|
favicon_hash: str | None = None
|
|
content_hash: str | None = None
|
|
header_hash: str | None = None
|
|
jarm: str | None = None
|
|
supports_http2: bool = False
|
|
supports_pipeline: bool = False
|
|
ip: str | None = None
|
|
a_records: list[str] = Field(default_factory=list)
|
|
aaaa_records: list[str] = Field(default_factory=list)
|
|
cname: str | None = None
|
|
asn: int | None = None
|
|
asn_org: str | None = None
|
|
is_cdn: bool = False
|
|
cdn_name: str | None = None
|
|
cdn_type: str | None = None
|
|
waf: str | None = None
|
|
tls_version: str | None = None
|
|
tls_cipher: str | None = None
|
|
tls_subject_cn: str | None = None
|
|
tls_sans: list[str] = Field(default_factory=list)
|
|
tls_issuer: str | None = None
|
|
tls_issuer_org: str | None = None
|
|
tls_serial: str | None = None
|
|
tls_fingerprint: str | None = None
|
|
tls_not_before: datetime | None = None
|
|
tls_not_after: datetime | None = None
|
|
tls_expired: bool | None = None
|
|
tls_self_signed: bool | None = None
|
|
screenshot_path: str | None = None
|
|
body_preview: str | None = None
|
|
hygiene_issues: list[str] = Field(default_factory=list)
|
|
hygiene_checked: list[str] = Field(default_factory=list)
|
|
discovered_at: datetime
|
|
|
|
|
|
class HygieneVerdict(BaseModel):
|
|
key: str
|
|
failed: bool
|
|
evidence: str | None = None
|
|
|
|
|
|
class HttpAssetDetail(HttpAssetRead):
|
|
tls_subject_dn: str | None = None
|
|
tls_issuer_cn: str | None = None
|
|
raw_request: str | None = None
|
|
raw_response_header: str | None = None
|
|
response_body: str | None = None
|
|
response_headers: dict = Field(default_factory=dict)
|
|
hygiene: list[HygieneVerdict] = Field(default_factory=list)
|
|
surface: AssetSurface | None = None
|
|
|
|
|
|
class HttpAssetSummary(BaseModel):
|
|
total: int
|
|
by_status: dict[str, int] = Field(default_factory=dict)
|
|
by_tech: dict[str, int] = Field(default_factory=dict)
|
|
cdn: int = 0
|