mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-20 23:31:31 +02:00
Yogesh OjhaandClaude Opus 4.8
dd7ff0f6f1
The vulnerability stage handed every http_assets row to nuclei, ordered by URL, with one 90-minute clock cutting wherever it landed. Template-spray then gave every host the first slice of the corpus in load order. On the two largest local scans that was 23.4M and 10.7M planned requests, of which 3.9% and 9% were sent, and no host recorded what it got. One GeoServer CVE was found 11 times on 11 hostnames of one server. Introduce one planner, shared/services/scan_surface/, that decides what every vulnerability scanner receives. It writes scan_surface_items per scan and is a pure function of the tables the scan already produced plus the context; nothing in it names a nuclei flag. Vocabulary lives in shared/definitions/scan_surface.py, mirrored by config/scan-surface.ts. - Inputs are typed by protocol, because nuclei rewrites input per protocol: roots for HTTP templates (any status), bare names for DNS, host:port for TLS and network, parameterised requests for fuzzing. Path and query are stripped from non-DAST input; values are normalised to one spelling. - Web assets that are one backend fold into one origin on exact agreement of address, certificate, body hash, status, title, canary and favicon — never a similarity score. Measured: 1,812 web assets to 1,607 origins on gov.np. - The corpus is spent in tiers: one-request checks on every asset, the universal set on each origin, product-matched checks through a curated tech map, the blind sweep last. Each tier is a ranked host-spray batch with its own coverage row; the budget cut lands between batches and every origin is scanned, covered or not scanned. - A finding on a representative is replayed onto each cluster member with one request, writing a real observed row or splitting the member out. Never an inferred row. - dast_scan is a new stage (off by default, after endpoint_probe): nuclei -dast over request items and exposure checks under directories, then dalfox behind a reflection probe. dalfox v3.2.2 is pinned in the worker image. Verified live on testasp.vulnweb.com: the tiers ran in order, nuclei -dast found two LFIs, dalfox executed under the reflection gate. All suites green. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01R7sBu1jBa4e2CXLS1KEsNr
511 lines
11 KiB
Python
511 lines
11 KiB
Python
import shared.models._tztypes # patch datetime->timestamptz first
|
|
from shared.models.activity_log import ActivityLog, ActivityLogRead
|
|
from shared.models.ai import AiNarrative
|
|
from shared.models.api_key import (
|
|
APIKey,
|
|
APIKeyCreate,
|
|
APIKeyRead,
|
|
APIKeyUpdate,
|
|
ProviderInfo,
|
|
)
|
|
from shared.models.compare import (
|
|
ChangeField,
|
|
ChangeRow,
|
|
ChangeRows,
|
|
ComparableRun,
|
|
CoverageLine,
|
|
DimensionDelta,
|
|
DimensionVerdict,
|
|
RunDifference,
|
|
RunSide,
|
|
ScanComparison,
|
|
ScreenshotPair,
|
|
SettingDiff,
|
|
StageDiff,
|
|
)
|
|
from shared.models.connector import (
|
|
ActionRead,
|
|
ActionRequest,
|
|
AddTargetRequest,
|
|
CandidatePage,
|
|
CandidateRead,
|
|
Connector,
|
|
ConnectorAction,
|
|
ConnectorCandidate,
|
|
ConnectorCreate,
|
|
ConnectorCreated,
|
|
ConnectorHost,
|
|
ConnectorRead,
|
|
ConnectorScope,
|
|
ConnectorUpdate,
|
|
DiscoveredDomain,
|
|
FindingRecorded,
|
|
FindingReport,
|
|
HostFacts,
|
|
IngestItem,
|
|
IngestRequest,
|
|
IngestResult,
|
|
NoticeRead,
|
|
TargetAdded,
|
|
TargetOption,
|
|
)
|
|
from shared.models.crosslink import CrossLink, CrossLinkPeer
|
|
from shared.models.dns import (
|
|
DnsLookup,
|
|
DnsLookupRead,
|
|
DnsLookupSummary,
|
|
DnsRecord,
|
|
DnsRecordRead,
|
|
)
|
|
from shared.models.endpoint import (
|
|
Endpoint,
|
|
EndpointCoverage,
|
|
EndpointDetail,
|
|
EndpointFacets,
|
|
EndpointFilter,
|
|
EndpointPage,
|
|
EndpointRead,
|
|
EndpointSummary,
|
|
EndpointTree,
|
|
TreeNode,
|
|
)
|
|
from shared.models.export import Export, ExportCreate, ExportRead
|
|
from shared.models.http_asset import (
|
|
HttpAsset,
|
|
HttpAssetRead,
|
|
HttpAssetSummary,
|
|
)
|
|
from shared.models.instance_settings import (
|
|
InstanceSettings,
|
|
InstanceSettingsRead,
|
|
InstanceSettingsUpdate,
|
|
)
|
|
from shared.models.interest import (
|
|
InterestCatalog,
|
|
InterestDismissal,
|
|
InterestFilter,
|
|
InterestPage,
|
|
InterestRow,
|
|
InterestRule,
|
|
InterestRuleCreate,
|
|
InterestRuleRead,
|
|
InterestRuleUpdate,
|
|
InterestSignal,
|
|
InterestSummary,
|
|
RulePreview,
|
|
RuleSuggestion,
|
|
)
|
|
from shared.models.ip_address import (
|
|
IpAddress,
|
|
IpAddressRead,
|
|
IpAddressSummary,
|
|
TargetIpAddressRead,
|
|
)
|
|
from shared.models.ip_asn_range import IpAsnRange, IpCountryRange
|
|
from shared.models.note import (
|
|
Note,
|
|
NoteCount,
|
|
NoteCreate,
|
|
NoteRead,
|
|
NoteTag,
|
|
NoteUpdate,
|
|
)
|
|
from shared.models.notification import (
|
|
Notification,
|
|
NotificationCreate,
|
|
NotificationRead,
|
|
)
|
|
from shared.models.notification_channel import (
|
|
NotificationChannel,
|
|
NotificationChannelCreate,
|
|
NotificationChannelRead,
|
|
NotificationChannelTestResult,
|
|
NotificationChannelUpdate,
|
|
NotificationPreference,
|
|
)
|
|
from shared.models.organization import (
|
|
Organization,
|
|
OrganizationCreate,
|
|
OrganizationRead,
|
|
OrganizationSummary,
|
|
OrganizationUpdate,
|
|
)
|
|
from shared.models.port import Port, PortRead, PortSummary
|
|
from shared.models.project import Project, ProjectBase, ProjectCreate, ProjectRead
|
|
from shared.models.proxy import (
|
|
Proxy,
|
|
ProxyCreate,
|
|
ProxyEndpoint,
|
|
ProxyEndpointRead,
|
|
ProxyRead,
|
|
ProxyTestResult,
|
|
ProxyUpdate,
|
|
)
|
|
from shared.models.report import Report, ReportFont, ReportTemplate, ReportTheme
|
|
from shared.models.ripestat import (
|
|
RIPEStatAbuseContact,
|
|
RIPEStatAnnouncedPrefix,
|
|
RIPEStatASNNeighbour,
|
|
RIPEStatASOverview,
|
|
RIPEStatNetworkInfo,
|
|
RIPEStatPrefixOverview,
|
|
RIPEStatRelatedPrefix,
|
|
)
|
|
from shared.models.scan import (
|
|
Scan,
|
|
ScanCreate,
|
|
ScanRead,
|
|
)
|
|
from shared.models.scan_activity import ScanActivity, ScanActivityRead
|
|
from shared.models.scan_command import (
|
|
ScanCommand,
|
|
ScanCommandDetail,
|
|
ScanCommandRead,
|
|
)
|
|
from shared.models.scan_context import (
|
|
AuthConfig,
|
|
AuthHeader,
|
|
ScanContext,
|
|
ScanContextCreate,
|
|
ScanContextRead,
|
|
ScanContextUpdate,
|
|
)
|
|
from shared.models.scan_engine import (
|
|
EngineCatalog,
|
|
ScanEngine,
|
|
ScanEngineCreate,
|
|
ScanEngineRead,
|
|
ScanEngineUpdate,
|
|
)
|
|
from shared.models.scan_schedule import (
|
|
ScanSchedule,
|
|
ScanScheduleCreate,
|
|
ScanScheduleRead,
|
|
ScanScheduleUpdate,
|
|
ScheduleTargetRef,
|
|
)
|
|
from shared.models.scan_surface import (
|
|
AssetSurface,
|
|
ScanSurfaceItem,
|
|
SurfaceItemRead,
|
|
SurfaceSummary,
|
|
SurfaceTierCount,
|
|
)
|
|
from shared.models.software import (
|
|
NvdCpeMatch,
|
|
NvdCve,
|
|
SoftwareComponentRead,
|
|
SoftwareCoverage,
|
|
SoftwareCve,
|
|
SoftwareCveRead,
|
|
SoftwareFacet,
|
|
SoftwareFacets,
|
|
SoftwareFilter,
|
|
SoftwarePage,
|
|
)
|
|
from shared.models.subdomain import (
|
|
Subdomain,
|
|
SubdomainRead,
|
|
SubdomainSummary,
|
|
TargetSubdomainRead,
|
|
)
|
|
from shared.models.tag import Tag, TagBase, TagCreate, TagRead, TagSummary, TagUpdate
|
|
from shared.models.target import (
|
|
Target,
|
|
TargetBulkCreate,
|
|
TargetBulkCreateResponse,
|
|
TargetCreate,
|
|
TargetImportItem,
|
|
TargetImportRequest,
|
|
TargetImportResult,
|
|
TargetRead,
|
|
TargetType,
|
|
TargetUpdate,
|
|
TargetValidationRequest,
|
|
TargetValidationResponse,
|
|
)
|
|
from shared.models.target_seed import (
|
|
TargetSeed,
|
|
TargetSeedRead,
|
|
TargetSeedResult,
|
|
TargetSeedWrite,
|
|
)
|
|
from shared.models.user import User, UserBase, UserCreate, UserRead
|
|
from shared.models.viewdns import ViewDNSCache
|
|
from shared.models.vuln_template import (
|
|
TemplateFilter,
|
|
TemplateLibraryStats,
|
|
TemplatePage,
|
|
TemplateSelection,
|
|
VulnTemplate,
|
|
VulnTemplateRead,
|
|
VulnTemplateUpdate,
|
|
VulnTemplateUploadRequest,
|
|
)
|
|
from shared.models.vulnerability import (
|
|
ScanVulnerabilities,
|
|
Vulnerability,
|
|
VulnerabilityCoverage,
|
|
VulnerabilityFacets,
|
|
VulnerabilityFilter,
|
|
VulnerabilityPage,
|
|
VulnerabilityRead,
|
|
VulnerabilityTriage,
|
|
)
|
|
from shared.models.watch import (
|
|
ProgramWatch,
|
|
UserMark,
|
|
WatchCreate,
|
|
WatchEvent,
|
|
WatchHost,
|
|
WatchRead,
|
|
WatchUpdate,
|
|
)
|
|
from shared.models.wordlist import (
|
|
Wordlist,
|
|
WordlistFile,
|
|
WordlistRead,
|
|
WordlistUpdate,
|
|
WordlistUploadRequest,
|
|
WordlistUploadResult,
|
|
)
|
|
|
|
TargetRead.model_rebuild()
|
|
|
|
__all__ = [
|
|
"APIKey",
|
|
"APIKeyCreate",
|
|
"APIKeyRead",
|
|
"APIKeyUpdate",
|
|
"ActionRead",
|
|
"ActionRequest",
|
|
"ActivityLog",
|
|
"ActivityLogRead",
|
|
"AddTargetRequest",
|
|
"AiNarrative",
|
|
"AssetSurface",
|
|
"AuthConfig",
|
|
"AuthHeader",
|
|
"CandidatePage",
|
|
"CandidateRead",
|
|
"ChangeField",
|
|
"ChangeRow",
|
|
"ChangeRows",
|
|
"ComparableRun",
|
|
"Connector",
|
|
"ConnectorAction",
|
|
"ConnectorCandidate",
|
|
"ConnectorCreate",
|
|
"ConnectorCreated",
|
|
"ConnectorHost",
|
|
"ConnectorRead",
|
|
"ConnectorScope",
|
|
"ConnectorUpdate",
|
|
"CoverageLine",
|
|
"CrossLink",
|
|
"CrossLinkPeer",
|
|
"DimensionDelta",
|
|
"DimensionVerdict",
|
|
"DiscoveredDomain",
|
|
"DnsLookup",
|
|
"DnsLookupRead",
|
|
"DnsLookupSummary",
|
|
"DnsRecord",
|
|
"DnsRecordRead",
|
|
"Endpoint",
|
|
"EndpointCoverage",
|
|
"EndpointDetail",
|
|
"EndpointFacets",
|
|
"EndpointFilter",
|
|
"EndpointPage",
|
|
"EndpointRead",
|
|
"EndpointSummary",
|
|
"EndpointTree",
|
|
"EngineCatalog",
|
|
"Export",
|
|
"ExportCreate",
|
|
"ExportRead",
|
|
"FindingRecorded",
|
|
"FindingReport",
|
|
"HostFacts",
|
|
"HttpAsset",
|
|
"HttpAssetRead",
|
|
"HttpAssetSummary",
|
|
"IngestItem",
|
|
"IngestRequest",
|
|
"IngestResult",
|
|
"InstanceSettings",
|
|
"InstanceSettingsRead",
|
|
"InstanceSettingsUpdate",
|
|
"InterestCatalog",
|
|
"InterestDismissal",
|
|
"InterestFilter",
|
|
"InterestPage",
|
|
"InterestRow",
|
|
"InterestRule",
|
|
"InterestRuleCreate",
|
|
"InterestRuleRead",
|
|
"InterestRuleUpdate",
|
|
"InterestSignal",
|
|
"InterestSummary",
|
|
"IpAddress",
|
|
"IpAddressRead",
|
|
"IpAddressSummary",
|
|
"IpAsnRange",
|
|
"IpCountryRange",
|
|
"Note",
|
|
"NoteCount",
|
|
"NoteCreate",
|
|
"NoteRead",
|
|
"NoteTag",
|
|
"NoteUpdate",
|
|
"NoticeRead",
|
|
"Notification",
|
|
"NotificationChannel",
|
|
"NotificationChannelCreate",
|
|
"NotificationChannelRead",
|
|
"NotificationChannelTestResult",
|
|
"NotificationChannelUpdate",
|
|
"NotificationCreate",
|
|
"NotificationPreference",
|
|
"NotificationRead",
|
|
"NvdCpeMatch",
|
|
"NvdCve",
|
|
"Organization",
|
|
"OrganizationCreate",
|
|
"OrganizationRead",
|
|
"OrganizationSummary",
|
|
"OrganizationUpdate",
|
|
"Port",
|
|
"PortRead",
|
|
"PortSummary",
|
|
"ProgramWatch",
|
|
"Project",
|
|
"ProjectBase",
|
|
"ProjectCreate",
|
|
"ProjectRead",
|
|
"ProviderInfo",
|
|
"Proxy",
|
|
"ProxyCreate",
|
|
"ProxyEndpoint",
|
|
"ProxyEndpointRead",
|
|
"ProxyRead",
|
|
"ProxyTestResult",
|
|
"ProxyUpdate",
|
|
"RIPEStatASNNeighbour",
|
|
"RIPEStatASOverview",
|
|
"RIPEStatAbuseContact",
|
|
"RIPEStatAnnouncedPrefix",
|
|
"RIPEStatNetworkInfo",
|
|
"RIPEStatPrefixOverview",
|
|
"RIPEStatRelatedPrefix",
|
|
"Report",
|
|
"ReportFont",
|
|
"ReportTemplate",
|
|
"ReportTheme",
|
|
"RulePreview",
|
|
"RuleSuggestion",
|
|
"RunDifference",
|
|
"RunSide",
|
|
"Scan",
|
|
"ScanActivity",
|
|
"ScanActivityRead",
|
|
"ScanCommand",
|
|
"ScanCommandDetail",
|
|
"ScanCommandRead",
|
|
"ScanComparison",
|
|
"ScanContext",
|
|
"ScanContextCreate",
|
|
"ScanContextRead",
|
|
"ScanContextUpdate",
|
|
"ScanCreate",
|
|
"ScanEngine",
|
|
"ScanEngineCreate",
|
|
"ScanEngineRead",
|
|
"ScanEngineUpdate",
|
|
"ScanRead",
|
|
"ScanSchedule",
|
|
"ScanScheduleCreate",
|
|
"ScanScheduleRead",
|
|
"ScanScheduleUpdate",
|
|
"ScanSurfaceItem",
|
|
"ScanVulnerabilities",
|
|
"ScheduleTargetRef",
|
|
"ScreenshotPair",
|
|
"SettingDiff",
|
|
"SoftwareComponentRead",
|
|
"SoftwareCoverage",
|
|
"SoftwareCve",
|
|
"SoftwareCveRead",
|
|
"SoftwareFacet",
|
|
"SoftwareFacets",
|
|
"SoftwareFilter",
|
|
"SoftwarePage",
|
|
"StageDiff",
|
|
"Subdomain",
|
|
"SubdomainRead",
|
|
"SubdomainSummary",
|
|
"SurfaceItemRead",
|
|
"SurfaceSummary",
|
|
"SurfaceTierCount",
|
|
"Tag",
|
|
"TagBase",
|
|
"TagCreate",
|
|
"TagRead",
|
|
"TagSummary",
|
|
"TagUpdate",
|
|
"Target",
|
|
"TargetAdded",
|
|
"TargetBulkCreate",
|
|
"TargetBulkCreateResponse",
|
|
"TargetCreate",
|
|
"TargetImportItem",
|
|
"TargetImportRequest",
|
|
"TargetImportResult",
|
|
"TargetIpAddressRead",
|
|
"TargetOption",
|
|
"TargetRead",
|
|
"TargetSeed",
|
|
"TargetSeedRead",
|
|
"TargetSeedResult",
|
|
"TargetSeedWrite",
|
|
"TargetSubdomainRead",
|
|
"TargetType",
|
|
"TargetUpdate",
|
|
"TargetValidationRequest",
|
|
"TargetValidationResponse",
|
|
"TemplateFilter",
|
|
"TemplateLibraryStats",
|
|
"TemplatePage",
|
|
"TemplateSelection",
|
|
"TreeNode",
|
|
"User",
|
|
"UserBase",
|
|
"UserCreate",
|
|
"UserMark",
|
|
"UserRead",
|
|
"ViewDNSCache",
|
|
"VulnTemplate",
|
|
"VulnTemplateRead",
|
|
"VulnTemplateUpdate",
|
|
"VulnTemplateUploadRequest",
|
|
"Vulnerability",
|
|
"VulnerabilityCoverage",
|
|
"VulnerabilityFacets",
|
|
"VulnerabilityFilter",
|
|
"VulnerabilityPage",
|
|
"VulnerabilityRead",
|
|
"VulnerabilityTriage",
|
|
"WatchCreate",
|
|
"WatchEvent",
|
|
"WatchHost",
|
|
"WatchRead",
|
|
"WatchUpdate",
|
|
"Wordlist",
|
|
"WordlistFile",
|
|
"WordlistRead",
|
|
"WordlistUpdate",
|
|
"WordlistUploadRequest",
|
|
"WordlistUploadResult",
|
|
]
|