mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-21 07:41:28 +02:00
Yogesh OjhaandClaude Opus 5
913815acd4
Four scans against real estates (hackerone.com 40 names, bugcrowd.com 71, uber.com 11,203, shopify.com 26,235) show the correlation graph drawing the CDN's identities and the resolver reporting a fraction of the live surface as all of it. The resolver's silence is re-queried once. dnsx prints nothing both for a name that does not exist and for a query it dropped, and the loss is uniform, so `_retry_degraded`'s peer median never fires (and a single-batch estate has no peers). uber.com reported 135 of 11,203 names resolved; a plain dnsx re-run over a random 200 of the 11,068 it called dead answered 7, the same 7 three times over, and a DoH check of another 25 found 2 more. `_retry_silent` re-queries every unanswered name once (`dns_retry_silent`), writes through `_write_resolution` so the live view keeps up, counts recoveries as `recovered`, and reports partial when that pass still recovers names. A hub that belongs to a provider is not a correlation. bugcrowd.com drew six hubs and five of them were the CDN's: four Fastly anycast addresses and the `j.sni.global.fastly.net` CNAME every Fastly tenant is aliased to - four hubs over one pair of hosts, each restating what the CDN kind already says. Hubs now carry `platform`/`platform_label` from `ip_addresses.is_cdn` and from curated lists beside `is_shared_nameserver` (`SHARED_EDGE_HOSTS`, `PUBLIC_CA_NAMES`), the graph hides them behind "Hide provider", and `related()` drops them. Curated, never inferred from a suffix: `d3tqr1od3k1eh2.cloudfront.net` is one distribution, `j.sni.global.fastly.net` is the whole platform; a private issuer stays visible. Three more corrections in the same build. A share is read against the hosts carrying that kind, not every name in the scan, so shopify.com's 16-of-18 JARM cluster no longer reads as 0.5%, and `common` needs `MIN_ESTATE_FOR_COMMON` carriers. The 3,000-host cut is ranked by "answered, then resolved, then name" - alphabetically it held 19 of shopify.com's 116 web assets, and ranking took that scan from 35 hubs to 94 and its certificate correlation from 6 hosts to 66. A body under `MIN_BODY_BYTES` is not an identity: 11 hosts shared the SHA-256 of an empty body. Hidden hubs no longer consume the per-kind cap, chip counts equal what they draw, and the endpoint goes through `lead_cache` like its neighbours (3.3s to 18ms). Enrichment, which feeds all of it: - `sync_ranges` takes an advisory lock and a budget. Two concurrent scans on a fresh instance each pulled the same 200 MB and each truncated and reloaded 1.1M rows: 291s against 145s. A scan that finds the tables missing reports partial, and `app.tasks.ip_asn.refresh` fills the blanks through `backfill_addresses` once the feed lands. - No address is attributed from its target. The last-resort fill stamped the target's BGP holder and its WHOIS registrant country onto every address the range tables did not match, private ones included - a registrant country is not a geolocation and a target's network is not the operator of everything its scan found. It fired wherever that data exists, which is IP, range and ASN targets and any registry that publishes a country; only an ASN target now states its own sweep. - cdncheck sees every address, httpx only the ones that answered, so `ip_enrichment` folds `ip_addresses.is_cdn` back onto the hosts that resolve to it. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YYop31s6BqDUt4TBs5mbwa
340 lines
9.4 KiB
Python
340 lines
9.4 KiB
Python
import uuid
|
|
|
|
from pydantic import BaseModel, Field
|
|
|
|
from shared.definitions.asset_query import MAX_QUERY_LENGTH
|
|
from shared.models.asset_query import QueryError
|
|
from shared.models.http_asset import HttpAssetRead
|
|
from shared.models.ip_address import IpAddressRead
|
|
from shared.models.port import PortRead
|
|
from shared.models.subdomain import Facet, SubdomainRelation
|
|
|
|
|
|
class SurfaceStat(BaseModel):
|
|
key: str
|
|
label: str
|
|
value: int
|
|
filter: str = ""
|
|
|
|
|
|
class AttentionItem(BaseModel):
|
|
key: str
|
|
label: str
|
|
count: int
|
|
filter: str
|
|
tone: str
|
|
|
|
|
|
class Bucket(BaseModel):
|
|
key: str
|
|
label: str
|
|
count: int
|
|
klass: str
|
|
|
|
|
|
class Tally(BaseModel):
|
|
name: str
|
|
count: int
|
|
|
|
|
|
class ClusterStat(BaseModel):
|
|
kind: str
|
|
reason: str
|
|
value: str
|
|
count: int
|
|
|
|
|
|
class SubdomainInsights(BaseModel):
|
|
surface: list[SurfaceStat] = Field(default_factory=list)
|
|
attention: list[AttentionItem] = Field(default_factory=list)
|
|
sources: list[Tally] = Field(default_factory=list)
|
|
single_source: int = 0
|
|
resolution: list[Bucket] = Field(default_factory=list)
|
|
status_reframe: list[Bucket] = Field(default_factory=list)
|
|
cert_buckets: list[Bucket] = Field(default_factory=list)
|
|
top_tech: list[Tally] = Field(default_factory=list)
|
|
tech_total: int = 0
|
|
top_asn: list[Tally] = Field(default_factory=list)
|
|
geography: list[Tally] = Field(default_factory=list)
|
|
geo_total: int = 0
|
|
clusters: list[ClusterStat] = Field(default_factory=list)
|
|
|
|
|
|
class SubdomainCorrelation(BaseModel):
|
|
primary_asset: HttpAssetRead | None = None
|
|
services: list[HttpAssetRead] = Field(default_factory=list)
|
|
ports: list[PortRead] = Field(default_factory=list)
|
|
ip_metas: list[IpAddressRead] = Field(default_factory=list)
|
|
related: list[SubdomainRelation] = Field(default_factory=list)
|
|
|
|
|
|
class IpGroupRead(BaseModel):
|
|
ip: str
|
|
version: int
|
|
targets: list[str] = Field(default_factory=list)
|
|
asn: int | None = None
|
|
asn_org: str | None = None
|
|
country: str | None = None
|
|
prefix: str | None = None
|
|
is_cdn: bool = False
|
|
cdn_name: str | None = None
|
|
is_alive: bool | None = None
|
|
ptr_hostnames: list[str] = Field(default_factory=list)
|
|
ports: list[PortRead] = Field(default_factory=list)
|
|
host_count: int = 0
|
|
hosts: list[str] = Field(default_factory=list)
|
|
port_count: int = 0
|
|
has_sensitive: bool = False
|
|
asset_count: int = 0
|
|
|
|
|
|
class IpGroupPage(BaseModel):
|
|
items: list[IpGroupRead] = Field(default_factory=list)
|
|
total: int = 0
|
|
total_capped: bool = False
|
|
error: QueryError | None = None
|
|
|
|
|
|
class IpGroupFilter(BaseModel):
|
|
q: str | None = Field(default=None, max_length=MAX_QUERY_LENGTH)
|
|
exposure: list[str] = Field(default_factory=list, max_length=4)
|
|
asns: list[int] = Field(default_factory=list, max_length=100)
|
|
countries: list[str] = Field(default_factory=list, max_length=100)
|
|
ports: list[int] = Field(default_factory=list, max_length=200)
|
|
services: list[str] = Field(default_factory=list, max_length=200)
|
|
cdn: str = Field(default="any", max_length=10)
|
|
alive: str = Field(default="any", max_length=10)
|
|
version: int = Field(default=0, ge=0, le=6)
|
|
sensitive: bool = False
|
|
hosted: bool = False
|
|
open: bool = False
|
|
sort: str = Field(default="hosts", max_length=20)
|
|
order: str = Field(default="desc", max_length=4)
|
|
limit: int = Field(default=50, ge=1, le=200)
|
|
offset: int = Field(default=0, ge=0, le=100_000_000)
|
|
|
|
def has_facets(self) -> bool:
|
|
return bool(
|
|
self.exposure
|
|
or self.asns
|
|
or self.countries
|
|
or self.ports
|
|
or self.services
|
|
or self.cdn != "any"
|
|
or self.alive != "any"
|
|
or self.version
|
|
or self.sensitive
|
|
or self.hosted
|
|
or self.open
|
|
)
|
|
|
|
|
|
class IpFacets(BaseModel):
|
|
exposure: list[Facet] = Field(default_factory=list)
|
|
asn: list[Facet] = Field(default_factory=list)
|
|
country: list[Facet] = Field(default_factory=list)
|
|
port: list[Facet] = Field(default_factory=list)
|
|
service: list[Facet] = Field(default_factory=list)
|
|
|
|
|
|
class ServiceRead(BaseModel):
|
|
id: uuid.UUID
|
|
scan_id: uuid.UUID | None = None
|
|
target_id: uuid.UUID | None = None
|
|
target_value: str | None = None
|
|
ip: str
|
|
port: int
|
|
protocol: str
|
|
state: str
|
|
service_name: str | None = None
|
|
service_class: str
|
|
description: str = ""
|
|
registered: bool = False
|
|
source: str
|
|
is_http: bool = False
|
|
tls: bool = False
|
|
product: str | None = None
|
|
version: str | None = None
|
|
banner: str | None = None
|
|
asn: int | None = None
|
|
asn_org: str | None = None
|
|
country: str | None = None
|
|
prefix: str | None = None
|
|
is_cdn: bool = False
|
|
cdn_name: str | None = None
|
|
scan_policy: str | None = None
|
|
host_count: int = 0
|
|
hosts: list[str] = Field(default_factory=list)
|
|
web_count: int = 0
|
|
status_code: int | None = None
|
|
url: str | None = None
|
|
title: str | None = None
|
|
screenshot_path: str | None = None
|
|
is_sensitive: bool = False
|
|
is_new: bool = False
|
|
|
|
|
|
class ServicePage(BaseModel):
|
|
items: list[ServiceRead] = Field(default_factory=list)
|
|
total: int = 0
|
|
total_capped: bool = False
|
|
error: QueryError | None = None
|
|
|
|
|
|
class ServiceFilter(BaseModel):
|
|
q: str | None = Field(default=None, max_length=MAX_QUERY_LENGTH)
|
|
classes: list[str] = Field(default_factory=list, max_length=10)
|
|
ports: list[int] = Field(default_factory=list, max_length=200)
|
|
services: list[str] = Field(default_factory=list, max_length=200)
|
|
sources: list[str] = Field(default_factory=list, max_length=10)
|
|
asns: list[int] = Field(default_factory=list, max_length=100)
|
|
countries: list[str] = Field(default_factory=list, max_length=100)
|
|
cdn: str = Field(default="any", max_length=10)
|
|
http: str = Field(default="any", max_length=10)
|
|
sensitive: bool = False
|
|
named: bool = False
|
|
new: bool = False
|
|
sort: str = Field(default="exposure", max_length=20)
|
|
order: str = Field(default="desc", max_length=4)
|
|
limit: int = Field(default=50, ge=1, le=200)
|
|
offset: int = Field(default=0, ge=0, le=100_000_000)
|
|
|
|
def has_facets(self) -> bool:
|
|
return bool(
|
|
self.classes
|
|
or self.ports
|
|
or self.services
|
|
or self.sources
|
|
or self.asns
|
|
or self.countries
|
|
or self.cdn != "any"
|
|
or self.http != "any"
|
|
or self.sensitive
|
|
or self.named
|
|
or self.new
|
|
)
|
|
|
|
|
|
class ServiceFacets(BaseModel):
|
|
klass: list[Facet] = Field(default_factory=list, serialization_alias="class")
|
|
port: list[Facet] = Field(default_factory=list)
|
|
service: list[Facet] = Field(default_factory=list)
|
|
source: list[Facet] = Field(default_factory=list)
|
|
asn: list[Facet] = Field(default_factory=list)
|
|
country: list[Facet] = Field(default_factory=list)
|
|
|
|
|
|
class ExposureBand(BaseModel):
|
|
key: str
|
|
label: str
|
|
count: int
|
|
addresses: int
|
|
query: str
|
|
|
|
|
|
class ExposureLine(BaseModel):
|
|
key: str
|
|
label: str
|
|
detail: str | None = None
|
|
count: int
|
|
query: str
|
|
|
|
|
|
class ScanExposure(BaseModel):
|
|
services: int = 0
|
|
addresses: int = 0
|
|
web_services: int = 0
|
|
non_web_services: int = 0
|
|
answering_http: int = 0
|
|
sensitive: int = 0
|
|
named: int = 0
|
|
passive_only: int = 0
|
|
nonstandard_web: int = 0
|
|
bands: list[ExposureBand] = Field(default_factory=list)
|
|
top_services: list[ExposureLine] = Field(default_factory=list)
|
|
coverage: list[ExposureLine] = Field(default_factory=list)
|
|
scanned: int = 0
|
|
|
|
|
|
class OriginEvidence(BaseModel):
|
|
kind: str
|
|
label: str
|
|
value: str
|
|
|
|
|
|
class OriginSample(BaseModel):
|
|
host: str
|
|
url: str
|
|
ip: str | None = None
|
|
port: int = 0
|
|
status_code: int | None = None
|
|
title: str | None = None
|
|
webserver: str | None = None
|
|
cdn_name: str | None = None
|
|
asn_org: str | None = None
|
|
screenshot_path: str | None = None
|
|
|
|
|
|
class OriginFinding(BaseModel):
|
|
kind: str
|
|
confidence: str
|
|
exposed: OriginSample
|
|
fronted: list[OriginSample] = Field(default_factory=list)
|
|
fronted_total: int = 0
|
|
evidence: list[OriginEvidence] = Field(default_factory=list)
|
|
open_ports: list[int] = Field(default_factory=list)
|
|
sensitive_ports: list[int] = Field(default_factory=list)
|
|
query: str = ""
|
|
|
|
|
|
class OriginExposure(BaseModel):
|
|
findings: list[OriginFinding] = Field(default_factory=list)
|
|
probed_addresses: int = 0
|
|
fronted_assets: int = 0
|
|
|
|
|
|
class CorrelationHost(BaseModel):
|
|
id: uuid.UUID
|
|
name: str
|
|
live: bool = False
|
|
status: int | None = None
|
|
title: str | None = None
|
|
hubs: int = 0
|
|
|
|
|
|
class CorrelationHub(BaseModel):
|
|
"""One shared identity and the hosts that carry it."""
|
|
|
|
id: str
|
|
kind: str
|
|
value: str
|
|
label: str
|
|
count: int
|
|
share: float = 0.0
|
|
common: bool = False
|
|
platform: bool = False
|
|
platform_label: str = ""
|
|
query: str
|
|
members: list[int] = Field(default_factory=list)
|
|
|
|
|
|
class CorrelationKindStat(BaseModel):
|
|
key: str
|
|
label: str
|
|
help: str
|
|
default: bool = True
|
|
hubs: int = 0
|
|
hosts: int = 0
|
|
common: int = 0
|
|
platform: int = 0
|
|
|
|
|
|
class CorrelationGraph(BaseModel):
|
|
hosts: list[CorrelationHost] = Field(default_factory=list)
|
|
hubs: list[CorrelationHub] = Field(default_factory=list)
|
|
kinds: list[CorrelationKindStat] = Field(default_factory=list)
|
|
total_hosts: int = 0
|
|
estate_hosts: int = 0
|
|
shared_hosts: int = 0
|
|
truncated: bool = False
|