mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-21 07:41:28 +02:00
Yogesh OjhaandClaude Opus 5
56eb5999c4
Four implementations answered "what does this host share?" and two of them could only answer it for one run: the project-wide page had no Correlation tab at all, and its detail sheet passed the row's own scan_id, so "Related" silently meant "related inside this one target's last run". One engine now, shared/services/correlation/, parameterised by QueryScope. kinds.py holds the vocabulary as data; hubs.py discovers hubs with a GROUP BY over the whole scope instead of grouping a 3,000-row sample in Python, so a hub is no longer missed for falling outside the cut. asset_query/groups.py derives its dimensions from the same registry, and correlation-graph, related and correlation take WebAssetScope. New /surface/correlation page. The count is a promise, and three bugs broke it. Asset kinds joined http_assets and counted rows, so a host answering on 80 and 443 counted twice (cert.fingerprint said 80 where the search returned 41). asn excludes CDN-fronted rows but its token did not, 8779 against 8815; KindSpec.narrows appends -is:cdn. And carriers came from a window over the group set the caller asked about, so one row's own value read as 100% of its carriers and common swallowed it. All 28 hub counts across 14 kinds now equal their searches. The provider rules moved into the engine. generic_page, shared_edge and cert_covers gated cross links alone, so estate-wide the graph drew 404 Not Found across 5 targets, 403 Forbidden across 4 and a 548-byte 404 body across 5 as correlations. A hub is judged once its members load and is labelled with who wrote it; hubs load MAX_HUBS_PER_KIND + 40 deep and re-rank. Related takes the whole vocabulary, each kind carrying the phrase it is announced with and the query its button opens, which retires relationDsl and RELATION_LABELS. asn became a kind, out of the default set. A scope over several targets opens on what crosses them: 3,000 nodes over 21,690 correlating hosts is a hairball, the 30 crossing hubs are the finding. Measured on 63,844 host rows across 10 covering scans: ~1.1s cold for the estate, 70-200ms for a scan, cached after. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FwCCFgRBC8ut6SQpmzfGAa
347 lines
9.6 KiB
Python
347 lines
9.6 KiB
Python
import uuid
|
|
|
|
from pydantic import BaseModel, ConfigDict, Field
|
|
|
|
from shared.definitions.asset_query import MAX_QUERY_LENGTH
|
|
from shared.models.asset_query import QueryError
|
|
from shared.models.http_asset import HttpAssetRead
|
|
from shared.models.ip_address import IpAddressRead
|
|
from shared.models.port import PortRead
|
|
from shared.models.subdomain import Facet, SubdomainRelation
|
|
|
|
|
|
class SurfaceStat(BaseModel):
|
|
key: str
|
|
label: str
|
|
value: int
|
|
filter: str = ""
|
|
|
|
|
|
class AttentionItem(BaseModel):
|
|
key: str
|
|
label: str
|
|
count: int
|
|
filter: str
|
|
tone: str
|
|
|
|
|
|
class Bucket(BaseModel):
|
|
key: str
|
|
label: str
|
|
count: int
|
|
klass: str
|
|
|
|
|
|
class Tally(BaseModel):
|
|
name: str
|
|
count: int
|
|
|
|
|
|
class ClusterStat(BaseModel):
|
|
kind: str
|
|
reason: str
|
|
value: str
|
|
count: int
|
|
|
|
|
|
class SubdomainInsights(BaseModel):
|
|
surface: list[SurfaceStat] = Field(default_factory=list)
|
|
attention: list[AttentionItem] = Field(default_factory=list)
|
|
sources: list[Tally] = Field(default_factory=list)
|
|
single_source: int = 0
|
|
resolution: list[Bucket] = Field(default_factory=list)
|
|
status_reframe: list[Bucket] = Field(default_factory=list)
|
|
cert_buckets: list[Bucket] = Field(default_factory=list)
|
|
top_tech: list[Tally] = Field(default_factory=list)
|
|
tech_total: int = 0
|
|
top_asn: list[Tally] = Field(default_factory=list)
|
|
geography: list[Tally] = Field(default_factory=list)
|
|
geo_total: int = 0
|
|
clusters: list[ClusterStat] = Field(default_factory=list)
|
|
|
|
|
|
class SubdomainCorrelation(BaseModel):
|
|
primary_asset: HttpAssetRead | None = None
|
|
services: list[HttpAssetRead] = Field(default_factory=list)
|
|
ports: list[PortRead] = Field(default_factory=list)
|
|
ip_metas: list[IpAddressRead] = Field(default_factory=list)
|
|
related: list[SubdomainRelation] = Field(default_factory=list)
|
|
|
|
|
|
class IpGroupRead(BaseModel):
|
|
ip: str
|
|
version: int
|
|
targets: list[str] = Field(default_factory=list)
|
|
asn: int | None = None
|
|
asn_org: str | None = None
|
|
country: str | None = None
|
|
prefix: str | None = None
|
|
is_cdn: bool = False
|
|
cdn_name: str | None = None
|
|
is_alive: bool | None = None
|
|
ptr_hostnames: list[str] = Field(default_factory=list)
|
|
ports: list[PortRead] = Field(default_factory=list)
|
|
host_count: int = 0
|
|
hosts: list[str] = Field(default_factory=list)
|
|
port_count: int = 0
|
|
has_sensitive: bool = False
|
|
asset_count: int = 0
|
|
|
|
|
|
class IpGroupPage(BaseModel):
|
|
items: list[IpGroupRead] = Field(default_factory=list)
|
|
total: int = 0
|
|
total_capped: bool = False
|
|
error: QueryError | None = None
|
|
|
|
|
|
class IpGroupFilter(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
q: str | None = Field(default=None, max_length=MAX_QUERY_LENGTH)
|
|
exposure: list[str] = Field(default_factory=list, max_length=4)
|
|
asns: list[int] = Field(default_factory=list, max_length=100)
|
|
countries: list[str] = Field(default_factory=list, max_length=100)
|
|
ports: list[int] = Field(default_factory=list, max_length=200)
|
|
services: list[str] = Field(default_factory=list, max_length=200)
|
|
cdn: str = Field(default="any", max_length=10)
|
|
alive: str = Field(default="any", max_length=10)
|
|
version: int = Field(default=0, ge=0, le=6)
|
|
sensitive: bool = False
|
|
hosted: bool = False
|
|
open: bool = False
|
|
sort: str = Field(default="hosts", max_length=20)
|
|
order: str = Field(default="desc", max_length=4)
|
|
limit: int = Field(default=50, ge=1, le=200)
|
|
offset: int = Field(default=0, ge=0, le=100_000_000)
|
|
|
|
def has_facets(self) -> bool:
|
|
return bool(
|
|
self.exposure
|
|
or self.asns
|
|
or self.countries
|
|
or self.ports
|
|
or self.services
|
|
or self.cdn != "any"
|
|
or self.alive != "any"
|
|
or self.version
|
|
or self.sensitive
|
|
or self.hosted
|
|
or self.open
|
|
)
|
|
|
|
|
|
class IpFacets(BaseModel):
|
|
exposure: list[Facet] = Field(default_factory=list)
|
|
asn: list[Facet] = Field(default_factory=list)
|
|
country: list[Facet] = Field(default_factory=list)
|
|
port: list[Facet] = Field(default_factory=list)
|
|
service: list[Facet] = Field(default_factory=list)
|
|
|
|
|
|
class ServiceRead(BaseModel):
|
|
id: uuid.UUID
|
|
scan_id: uuid.UUID | None = None
|
|
target_id: uuid.UUID | None = None
|
|
target_value: str | None = None
|
|
ip: str
|
|
port: int
|
|
protocol: str
|
|
state: str
|
|
service_name: str | None = None
|
|
service_class: str
|
|
description: str = ""
|
|
registered: bool = False
|
|
source: str
|
|
is_http: bool = False
|
|
tls: bool = False
|
|
product: str | None = None
|
|
version: str | None = None
|
|
banner: str | None = None
|
|
asn: int | None = None
|
|
asn_org: str | None = None
|
|
country: str | None = None
|
|
prefix: str | None = None
|
|
is_cdn: bool = False
|
|
cdn_name: str | None = None
|
|
scan_policy: str | None = None
|
|
host_count: int = 0
|
|
hosts: list[str] = Field(default_factory=list)
|
|
web_count: int = 0
|
|
status_code: int | None = None
|
|
url: str | None = None
|
|
title: str | None = None
|
|
screenshot_path: str | None = None
|
|
is_sensitive: bool = False
|
|
is_new: bool = False
|
|
|
|
|
|
class ServicePage(BaseModel):
|
|
items: list[ServiceRead] = Field(default_factory=list)
|
|
total: int = 0
|
|
total_capped: bool = False
|
|
error: QueryError | None = None
|
|
|
|
|
|
class ServiceFilter(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
q: str | None = Field(default=None, max_length=MAX_QUERY_LENGTH)
|
|
classes: list[str] = Field(default_factory=list, max_length=10)
|
|
ports: list[int] = Field(default_factory=list, max_length=200)
|
|
services: list[str] = Field(default_factory=list, max_length=200)
|
|
sources: list[str] = Field(default_factory=list, max_length=10)
|
|
asns: list[int] = Field(default_factory=list, max_length=100)
|
|
countries: list[str] = Field(default_factory=list, max_length=100)
|
|
cdn: str = Field(default="any", max_length=10)
|
|
http: str = Field(default="any", max_length=10)
|
|
sensitive: bool = False
|
|
named: bool = False
|
|
new: bool = False
|
|
sort: str = Field(default="exposure", max_length=20)
|
|
order: str = Field(default="desc", max_length=4)
|
|
limit: int = Field(default=50, ge=1, le=200)
|
|
offset: int = Field(default=0, ge=0, le=100_000_000)
|
|
|
|
def has_facets(self) -> bool:
|
|
return bool(
|
|
self.classes
|
|
or self.ports
|
|
or self.services
|
|
or self.sources
|
|
or self.asns
|
|
or self.countries
|
|
or self.cdn != "any"
|
|
or self.http != "any"
|
|
or self.sensitive
|
|
or self.named
|
|
or self.new
|
|
)
|
|
|
|
|
|
class ServiceFacets(BaseModel):
|
|
klass: list[Facet] = Field(default_factory=list, serialization_alias="class")
|
|
port: list[Facet] = Field(default_factory=list)
|
|
service: list[Facet] = Field(default_factory=list)
|
|
source: list[Facet] = Field(default_factory=list)
|
|
asn: list[Facet] = Field(default_factory=list)
|
|
country: list[Facet] = Field(default_factory=list)
|
|
|
|
|
|
class ExposureBand(BaseModel):
|
|
key: str
|
|
label: str
|
|
count: int
|
|
addresses: int
|
|
query: str
|
|
|
|
|
|
class ExposureLine(BaseModel):
|
|
key: str
|
|
label: str
|
|
detail: str | None = None
|
|
count: int
|
|
query: str
|
|
|
|
|
|
class ScanExposure(BaseModel):
|
|
services: int = 0
|
|
addresses: int = 0
|
|
web_services: int = 0
|
|
non_web_services: int = 0
|
|
answering_http: int = 0
|
|
sensitive: int = 0
|
|
named: int = 0
|
|
passive_only: int = 0
|
|
nonstandard_web: int = 0
|
|
bands: list[ExposureBand] = Field(default_factory=list)
|
|
top_services: list[ExposureLine] = Field(default_factory=list)
|
|
coverage: list[ExposureLine] = Field(default_factory=list)
|
|
scanned: int = 0
|
|
|
|
|
|
class OriginEvidence(BaseModel):
|
|
kind: str
|
|
label: str
|
|
value: str
|
|
|
|
|
|
class OriginSample(BaseModel):
|
|
host: str
|
|
url: str
|
|
ip: str | None = None
|
|
port: int = 0
|
|
status_code: int | None = None
|
|
title: str | None = None
|
|
webserver: str | None = None
|
|
cdn_name: str | None = None
|
|
asn_org: str | None = None
|
|
screenshot_path: str | None = None
|
|
|
|
|
|
class OriginFinding(BaseModel):
|
|
kind: str
|
|
confidence: str
|
|
exposed: OriginSample
|
|
fronted: list[OriginSample] = Field(default_factory=list)
|
|
fronted_total: int = 0
|
|
evidence: list[OriginEvidence] = Field(default_factory=list)
|
|
open_ports: list[int] = Field(default_factory=list)
|
|
sensitive_ports: list[int] = Field(default_factory=list)
|
|
query: str = ""
|
|
|
|
|
|
class OriginExposure(BaseModel):
|
|
findings: list[OriginFinding] = Field(default_factory=list)
|
|
probed_addresses: int = 0
|
|
fronted_assets: int = 0
|
|
|
|
|
|
class CorrelationHost(BaseModel):
|
|
id: uuid.UUID
|
|
name: str
|
|
live: bool = False
|
|
status: int | None = None
|
|
title: str | None = None
|
|
hubs: int = 0
|
|
target: str = ""
|
|
|
|
|
|
class CorrelationHub(BaseModel):
|
|
"""One shared identity and the hosts that carry it."""
|
|
|
|
id: str
|
|
kind: str
|
|
value: str
|
|
label: str
|
|
count: int
|
|
targets: int = 1
|
|
share: float = 0.0
|
|
common: bool = False
|
|
platform: bool = False
|
|
platform_label: str = ""
|
|
query: str
|
|
members: list[int] = Field(default_factory=list)
|
|
|
|
|
|
class CorrelationKindStat(BaseModel):
|
|
key: str
|
|
label: str
|
|
help: str
|
|
default: bool = True
|
|
hubs: int = 0
|
|
total: int = 0
|
|
hosts: int = 0
|
|
common: int = 0
|
|
platform: int = 0
|
|
crossing: int = 0
|
|
|
|
|
|
class CorrelationGraph(BaseModel):
|
|
hosts: list[CorrelationHost] = Field(default_factory=list)
|
|
hubs: list[CorrelationHub] = Field(default_factory=list)
|
|
kinds: list[CorrelationKindStat] = Field(default_factory=list)
|
|
total_hosts: int = 0
|
|
estate_hosts: int = 0
|
|
shared_hosts: int = 0
|
|
targets_total: int = 1
|
|
truncated: bool = False
|