"""Everything a report can know, gathered once and cached.""" from __future__ import annotations from datetime import datetime from functools import cached_property from typing import Any from uuid import UUID from sqlalchemy import and_, case, cast, distinct, func, select, text from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.dialects.postgresql import array as pg_array from sqlalchemy.orm import Session, aliased from reports.data.models import ( Address, Certificate, DimensionCoverage, EndpointRow, Facet, Finding, Host, HygieneCount, HygieneHost, HygieneRollup, PostureRollup, PostureZone, Service, ) from shared.definitions import domain_posture as posture_defs from shared.definitions import hygiene as hygiene_defs from shared.definitions.ports import SENSITIVE_PORTS, ServiceClass from shared.definitions.reports import MAX_REPORT_ROWS, ReportScope from shared.definitions.surface import SURFACE_KINDS, SURFACE_ORDER, SurfaceDimension from shared.definitions.vulnerabilities import ( SUPPRESSED_STATES, Severity, severity_rank, ) from shared.enums.scan import ScanStatus from shared.models.domain_posture import DomainPosture from shared.models.endpoint import Endpoint from shared.models.http_asset import HttpAsset from shared.models.ip_address import IpAddress from shared.models.port import Port from shared.models.scan import Scan from shared.models.secret import Secret from shared.models.subdomain import Subdomain from shared.models.target import Target from shared.models.vulnerability import ( Vulnerability, VulnerabilityCoverage, VulnerabilityTriage, ) from shared.services.asset_query import predicates as preds from shared.utils.datetime import utc_now from shared.utils.net import is_registry_routable _DIM = SurfaceDimension _MAX_HOSTS_PER_IP = 6 _SERVER_ERROR = 500 _TABLE = { _DIM.WEB_ASSETS.value: Subdomain, _DIM.IPS.value: IpAddress, _DIM.SERVICES.value: Port, _DIM.ENDPOINTS.value: Endpoint, _DIM.VULNERABILITIES.value: Vulnerability, _DIM.SECRETS.value: Secret, } _KEY = { _DIM.WEB_ASSETS.value: (Subdomain.name,), _DIM.IPS.value: (IpAddress.ip,), _DIM.SERVICES.value: (Port.ip, Port.number, Port.protocol), _DIM.ENDPOINTS.value: (Endpoint.signature,), _DIM.VULNERABILITIES.value: (Vulnerability.fingerprint,), _DIM.SECRETS.value: (Secret.fingerprint,), } _LABEL = { _DIM.WEB_ASSETS.value: Subdomain.name, _DIM.IPS.value: IpAddress.ip, _DIM.SERVICES.value: Port.ip, _DIM.ENDPOINTS.value: Endpoint.url, _DIM.VULNERABILITIES.value: Vulnerability.matched_at, _DIM.SECRETS.value: Secret.value, } # the dimensions a report counts: those with a per-scan table above REPORT_DIMENSIONS: tuple[str, ...] = tuple(d for d in SURFACE_ORDER if d in _TABLE) class ReportSource: """Reads for one scan, or for a target's most recent covering run per dimension.""" def __init__( self, session: Session, *, scope: str, scan: Scan | None, target: Target, project_name: str = "", ) -> None: self.session = session self.scope = scope self.scan = scan self.target = target self.project_name = project_name self._memo: dict[str, Any] = {} # ---------- identity ---------- @property def subject(self) -> str: return self.target.target_value @property def subject_type(self) -> str: value = self.target.target_type return getattr(value, "value", str(value)).lower() @property def is_scan_scope(self) -> bool: return self.scope == ReportScope.SCAN.value @cached_property def observed_at(self) -> datetime | None: if self.scan is not None: return ( self.scan.completed_at or self.scan.started_at or self.scan.created_at ) stamps = [c.observed_at for c in self.coverage.values() if c.observed_at] return max(stamps) if stamps else None @property def observed_label(self) -> str: return self.observed_at.strftime("%d %b %Y") if self.observed_at else "—" # ---------- scan selection ---------- @cached_property def runs(self) -> list[Scan]: rows = ( self.session.execute( select(Scan) .where(Scan.target_id == self.target.id) .order_by(Scan.created_at.desc()) .limit(60) ) .scalars() .all() ) return list(rows) def scan_for(self, dimension: str) -> UUID | None: """The run whose rows this dimension is reported from.""" if self.is_scan_scope: return self.scan.id if self.scan else None key = f"scan_for:{dimension}" if key in self._memo: return self._memo[key] table = _TABLE[dimension] row = self.session.execute( select(table.scan_id, func.max(Scan.created_at).label("at")) .join(Scan, Scan.id == table.scan_id) .where(table.target_id == self.target.id) .group_by(table.scan_id) .order_by(func.max(Scan.created_at).desc()) .limit(1) ).first() found = row[0] if row else None self._memo[key] = found return found @cached_property def previous_scan(self) -> Scan | None: if self.scan is None: return None cutoff = self.scan.started_at or self.scan.created_at return ( self.session.execute( select(Scan) .where( Scan.target_id == self.target.id, Scan.id != self.scan.id, Scan.created_at < cutoff, Scan.status == ScanStatus.COMPLETED.value, ) .order_by(Scan.created_at.desc()) .limit(1) ) .scalars() .first() ) @property def cutoff(self) -> datetime | None: if self.scan is not None: return self.scan.started_at or self.scan.created_at return self.observed_at # ---------- coverage ---------- @cached_property def planned_stages(self) -> dict[str, dict]: if self.scan is None: return {} config = self.scan.execution_config or {} return { name: values for name, values in (config.get("stages") or {}).items() if (values or {}).get("enabled", True) } @cached_property def producing_stages(self) -> dict[str, set[str]]: """Dimension -> the enabled stages that produce it.""" try: from stages.registry import stages as stage_specs # noqa: PLC0415 except ImportError: return {} specs = {spec.name: spec for spec in stage_specs()} out: dict[str, set[str]] = {dim: set() for dim in REPORT_DIMENSIONS} for name in self.planned_stages: spec = specs.get(name) if spec is None: continue for dimension, kinds in SURFACE_KINDS.items(): if dimension in out and spec.produces & kinds: out[dimension].add(name) return out def _count(self, dimension: str, scan_id: UUID | None) -> int: if scan_id is None: return 0 table = _TABLE[dimension] query = select(func.count()).select_from(table).where(table.scan_id == scan_id) if dimension == _DIM.VULNERABILITIES.value: query = query.where(self._not_suppressed()) return int(self.session.execute(query).scalar() or 0) @cached_property def coverage(self) -> dict[str, DimensionCoverage]: out: dict[str, DimensionCoverage] = {} for dimension in REPORT_DIMENSIONS: scan_id = self.scan_for(dimension) count = self._count(dimension, scan_id) planned = bool(self.producing_stages.get(dimension)) covered = count > 0 or (planned and self.scan is not None) observed = None if scan_id is not None: run = next((r for r in self.runs if r.id == scan_id), self.scan) if run is not None: observed = run.completed_at or run.started_at or run.created_at entry = DimensionCoverage( dimension=dimension, covered=covered, count=count, observed_at=observed, scan_id=str(scan_id) if scan_id else "", ) if not covered: entry.note = "Not scanned" out[dimension] = entry self._attach_previous(out) return out def _attach_previous(self, coverage: dict[str, DimensionCoverage]) -> None: previous = self.previous_scan if previous is None: return covered_before = self._covered_by(previous) for dimension, entry in coverage.items(): if not entry.covered or dimension not in covered_before: continue entry.previous = self._count(dimension, previous.id) def _covered_by(self, scan: Scan) -> set[str]: """Rows are the only proof a previous run produced a dimension.""" return {d for d in REPORT_DIMENSIONS if self._count(d, scan.id)} @cached_property def covered_dimensions(self) -> frozenset[str]: return frozenset(d for d, c in self.coverage.items() if c.covered) def count_of(self, dimension: str) -> int: return self.coverage[dimension].count # ---------- baselines ---------- def _baseline_keys(self, dimension: str) -> set[tuple]: key = f"baseline:{dimension}" if key in self._memo: return self._memo[key] table = _TABLE[dimension] columns = _KEY[dimension] cutoff = self.cutoff found: set[tuple] = set() if cutoff is not None: scan_id = self.scan_for(dimension) query = ( select(*columns) .distinct() .where(table.target_id == self.target.id, table.discovered_at < cutoff) ) if scan_id is not None: query = query.where(table.scan_id != scan_id) found = {tuple(row) for row in self.session.execute(query)} self._memo[key] = found return found def has_baseline(self, dimension: str) -> bool: return bool(self._baseline_keys(dimension)) # ---------- vulnerabilities ---------- @staticmethod def _not_suppressed(): suppressed = ( select(VulnerabilityTriage.id) .where( VulnerabilityTriage.target_id == Vulnerability.target_id, VulnerabilityTriage.fingerprint == Vulnerability.fingerprint, VulnerabilityTriage.state.in_(SUPPRESSED_STATES), ) .exists() ) return ~suppressed @cached_property def triage(self) -> dict[str, tuple[str, str | None]]: rows = self.session.execute( select( VulnerabilityTriage.fingerprint, VulnerabilityTriage.state, VulnerabilityTriage.note, ).where(VulnerabilityTriage.target_id == self.target.id) ).all() return {row[0]: (row[1], row[2]) for row in rows} @cached_property def findings(self) -> list[Finding]: scan_id = self.scan_for(_DIM.VULNERABILITIES.value) if scan_id is None: return [] rows = ( self.session.execute( select(Vulnerability) .where(Vulnerability.scan_id == scan_id) .limit(MAX_REPORT_ROWS) ) .scalars() .all() ) baseline = self._baseline_keys(_DIM.VULNERABILITIES.value) known = self.has_baseline(_DIM.VULNERABILITIES.value) assets = self._asset_context({r.host for r in rows if r.host}) out: list[Finding] = [] for row in rows: state, note = self.triage.get(row.fingerprint, ("open", None)) if state in SUPPRESSED_STATES: continue asset = assets.get(row.host or "") out.append( Finding( id=str(row.id), fingerprint=row.fingerprint, template_id=row.template_id, name=row.template_name, severity=row.severity, scanner=row.scanner, protocol=row.protocol, matched_at=row.matched_at, host=row.host, ip=row.ip, port=row.port, url=row.url, description=row.description, impact=row.impact, remediation=row.remediation, references=list(row.references or []), tags=list(row.tags or []), cve_ids=list(row.cve_ids or []), cwe_ids=list(row.cwe_ids or []), cvss_score=row.cvss_score, cvss_metrics=row.cvss_metrics, epss_score=row.epss_score, is_kev=row.is_kev, state=state, note=note, is_new=known and (row.fingerprint,) not in baseline, extracted=[str(v) for v in (row.extracted_results or [])], request=row.request, response=row.response, curl=row.curl_command, matcher=row.matcher_name, discovered_at=row.discovered_at, screenshot=asset[0] if asset else None, asset_title=asset[1] if asset else None, asset_status=asset[2] if asset else None, asset_tech=list(asset[3]) if asset else [], ) ) out.sort( key=lambda f: ( severity_rank(f.severity), not f.is_kev, -(f.epss_score or 0), ) ) return out def _asset_context(self, hosts: set[str]) -> dict[str, tuple]: if not hosts: return {} scan_id = self.scan_for(_DIM.WEB_ASSETS.value) if scan_id is None: return {} rows = self.session.execute( select( Subdomain.name, Subdomain.screenshot_path, Subdomain.page_title, Subdomain.http_status, Subdomain.tech, ).where( Subdomain.scan_id == scan_id, Subdomain.name.in_(list(hosts)[:2000]) ) ).all() return {row[0]: (row[1], row[2], row[3], row[4] or []) for row in rows} @cached_property def coverage_rows(self) -> list[VulnerabilityCoverage]: scan_id = self.scan_for(_DIM.VULNERABILITIES.value) if scan_id is None: return [] return list( self.session.execute( select(VulnerabilityCoverage) .where(VulnerabilityCoverage.scan_id == scan_id) .order_by(VulnerabilityCoverage.started_at) ) .scalars() .all() ) @cached_property def severity_counts(self) -> dict[str, int]: counts = dict.fromkeys((s.value for s in Severity), 0) for finding in self.findings: counts[finding.severity] = counts.get(finding.severity, 0) + 1 return counts @cached_property def suppressed_count(self) -> int: scan_id = self.scan_for(_DIM.VULNERABILITIES.value) if scan_id is None: return 0 total = int( self.session.execute( select(func.count()) .select_from(Vulnerability) .where(Vulnerability.scan_id == scan_id) ).scalar() or 0 ) return max(0, total - len(self.findings)) # ---------- hosts ---------- @cached_property def host_rows(self) -> list[Host]: scan_id = self.scan_for(_DIM.WEB_ASSETS.value) if scan_id is None: return [] newest = ( select(HttpAsset) .where(HttpAsset.scan_id == scan_id) .distinct(HttpAsset.host) .order_by(HttpAsset.host, HttpAsset.discovered_at.desc()) .subquery() ) asset = aliased(HttpAsset, newest) rows = self.session.execute( select(Subdomain, asset) .outerjoin(asset, newest.c.host == Subdomain.name) .where(Subdomain.scan_id == scan_id) .order_by(Subdomain.http_status.is_(None), Subdomain.name) .limit(MAX_REPORT_ROWS) ).all() baseline = self._baseline_keys(_DIM.WEB_ASSETS.value) known = self.has_baseline(_DIM.WEB_ASSETS.value) by_host = self.findings_by_host out: list[Host] = [] for sub, asset in rows: worst = by_host.get(sub.name) out.append( Host( name=sub.name, status=sub.http_status, title=sub.page_title, url=sub.http_url, ips=list(sub.resolved_ips or []), cname=sub.cname, tech=list((asset.tech if asset else None) or sub.tech or []), webserver=sub.webserver, is_cdn=bool(asset.is_cdn if asset else sub.is_cdn), cdn_name=(asset.cdn_name if asset else None) or sub.cdn_name, cdn_type=asset.cdn_type if asset else None, waf=(asset.waf if asset else None) or sub.waf, asn=sub.asn or (asset.asn if asset else None), asn_org=sub.asn_org or (asset.asn_org if asset else None), tls_issuer=(asset.tls_issuer_org or asset.tls_issuer_cn) if asset else None, tls_not_after=(asset.tls_not_after if asset else None) or sub.tls_not_after, tls_expired=(asset.tls_expired if asset else None) if asset else sub.tls_expired, tls_self_signed=sub.tls_self_signed, screenshot=sub.screenshot_path, sources=list(sub.sources or []), is_new=known and (sub.name,) not in baseline, findings=worst[0] if worst else 0, worst=worst[1] if worst else None, ) ) return out @cached_property def findings_by_host(self) -> dict[str, tuple[int, str]]: out: dict[str, tuple[int, str]] = {} for finding in self.findings: key = finding.host or "" if not key: continue count, worst = out.get(key, (0, Severity.UNKNOWN.value)) best = ( worst if severity_rank(worst) <= severity_rank(finding.severity) else finding.severity ) out[key] = (count + 1, best) return out @cached_property def answered_hosts(self) -> list[Host]: return [h for h in self.host_rows if h.status] # ---------- addresses ---------- @cached_property def address_rows(self) -> list[Address]: scan_id = self.scan_for(_DIM.IPS.value) if scan_id is None: return [] rows = ( self.session.execute( select(IpAddress) .where(IpAddress.scan_id == scan_id) .order_by(IpAddress.ip) .limit(MAX_REPORT_ROWS) ) .scalars() .all() ) ports = self._ports_per_ip() hosts = self._hosts_per_ip() baseline = self._baseline_keys(_DIM.IPS.value) known = self.has_baseline(_DIM.IPS.value) return [ Address( ip=row.ip, version=row.version, asn=row.asn, asn_org=row.asn_org, country=row.country, prefix=row.prefix, is_cdn=row.is_cdn, cdn_name=row.cdn_name, cdn_type=row.cdn_type, scan_policy=row.scan_policy, scan_policy_reason=row.scan_policy_reason, ptr=list(row.ptr_hostnames or []), open_ports=ports.get(row.ip, 0), hosts=hosts.get(row.ip, []), is_new=known and (row.ip,) not in baseline, ) for row in rows ] def _ports_per_ip(self) -> dict[str, int]: scan_id = self.scan_for(_DIM.SERVICES.value) if scan_id is None: return {} rows = self.session.execute( select(Port.ip, func.count()) .where(Port.scan_id == scan_id) .group_by(Port.ip) ).all() return {row[0]: int(row[1]) for row in rows} def _hosts_per_ip(self) -> dict[str, list[str]]: scan_id = self.scan_for(_DIM.WEB_ASSETS.value) if scan_id is None: return {} rows = self.session.execute( select(HttpAsset.ip, HttpAsset.host) .where(HttpAsset.scan_id == scan_id, HttpAsset.ip.is_not(None)) .distinct() .limit(20000) ).all() out: dict[str, list[str]] = {} for ip, host in rows: bucket = out.setdefault(ip, []) if len(bucket) < _MAX_HOSTS_PER_IP and host not in bucket: bucket.append(host) return out # ---------- services ---------- @cached_property def service_rows(self) -> list[Service]: scan_id = self.scan_for(_DIM.SERVICES.value) if scan_id is None: return [] rows = self.session.execute( select(Port, IpAddress) .outerjoin( IpAddress, and_(IpAddress.scan_id == Port.scan_id, IpAddress.ip == Port.ip), ) .where(Port.scan_id == scan_id) .order_by(Port.ip, Port.number) .limit(MAX_REPORT_ROWS) ).all() hosts = self._hosts_per_ip() baseline = self._baseline_keys(_DIM.SERVICES.value) known = self.has_baseline(_DIM.SERVICES.value) return [ Service( ip=port.ip, port=port.number, protocol=port.protocol, service_name=port.service_name, service_class=port.service_class, product=port.product, version=port.version, banner=port.banner, is_http=port.is_http, tls=port.tls, source=port.source, sensitive=port.number in SENSITIVE_PORTS, hosts=hosts.get(port.ip, []), asn_org=address.asn_org if address else None, country=address.country if address else None, is_new=known and (port.ip, port.number, port.protocol) not in baseline, ) for port, address in rows ] @cached_property def sensitive_services(self) -> list[Service]: return [s for s in self.service_rows if s.sensitive] @cached_property def internal_estate(self) -> bool: """Every address answered is private.""" addresses = {s.ip for s in self.service_rows if s.ip} return bool(addresses) and not any(is_registry_routable(ip) for ip in addresses) # ---------- endpoints ---------- @cached_property def endpoint_rows(self) -> list[EndpointRow]: scan_id = self.scan_for(_DIM.ENDPOINTS.value) if scan_id is None: return [] rows = ( self.session.execute( select(Endpoint) .where(Endpoint.scan_id == scan_id) .order_by(Endpoint.host, Endpoint.path) .limit(MAX_REPORT_ROWS) ) .scalars() .all() ) baseline = self._baseline_keys(_DIM.ENDPOINTS.value) known = self.has_baseline(_DIM.ENDPOINTS.value) return [ EndpointRow( url=row.url, host=row.host, path=row.path, status=row.status_code, endpoint_class=row.endpoint_class, content_type=row.content_type, length=row.content_length, title=row.title, params=list(row.params or []), interest=list(row.interest or []), sources=list(row.sources or []), is_probed=row.is_probed, is_new=known and (row.signature,) not in baseline, ) for row in rows ] # ---------- hygiene ---------- @cached_property def hygiene(self) -> HygieneRollup | None: scan_id = self.scan_for(_DIM.WEB_ASSETS.value) if scan_id is None: return None issues = cast(Subdomain.hygiene_issues, JSONB) checked = cast(Subdomain.hygiene_checked, JSONB) warning_keys = pg_array(list(hygiene_defs.WARNING_KEYS)) info_keys = pg_array(list(hygiene_defs.INFO_KEYS)) columns = [ func.count() .filter(Subdomain.hygiene_checked.isnot(None)) .label("evaluated"), func.count() .filter( and_( Subdomain.http_status.isnot(None), Subdomain.hygiene_checked.is_(None), ) ) .label("pending"), func.count() .filter( and_( func.jsonb_array_length(checked) > 0, func.jsonb_array_length(issues) == 0, ) ) .label("clean"), func.count() .filter(func.jsonb_exists_any(issues, warning_keys)) .label("warning"), func.count().filter(func.jsonb_exists_any(issues, info_keys)).label("info"), ] for key in hygiene_defs.CHECK_KEYS: columns.append( func.count().filter(func.jsonb_exists(issues, key)).label(f"f_{key}") ) columns.append( func.count().filter(func.jsonb_exists(checked, key)).label(f"a_{key}") ) row = ( self.session.execute( select(*columns) .select_from(Subdomain) .where(Subdomain.scan_id == scan_id) ) .one() ._mapping ) host_rows = self.session.execute( select(Subdomain.name, Subdomain.hygiene_issues) .where( Subdomain.scan_id == scan_id, func.jsonb_exists_any(issues, warning_keys), ) .order_by( preds.hygiene_length(Subdomain.hygiene_issues).desc(), Subdomain.name ) .limit(MAX_REPORT_ROWS) ).all() labels = {c.key: c.label for c in hygiene_defs.CHECKS} order = hygiene_defs.CHECK_ORDER return HygieneRollup( evaluated=int(row["evaluated"]), pending=int(row["pending"]), clean=int(row["clean"]), warning=int(row["warning"]), info=int(row["info"]), checks=[ HygieneCount( key=key, failing=int(row[f"f_{key}"]), applicable=int(row[f"a_{key}"]), ) for key in hygiene_defs.CHECK_KEYS ], hosts=[ HygieneHost( name=name, checks=[ labels.get(k, k) for k in sorted(keys or [], key=lambda k: order.get(k, 99)) if k in hygiene_defs.WARNING_KEYS ], ) for name, keys in host_rows ], ) # ---------- domain posture ---------- @cached_property def domain_posture(self) -> PostureRollup | None: scan_id = self.scan.id if self.is_scan_scope and self.scan else None if scan_id is None: row = self.session.execute( select(DomainPosture.scan_id) .join(Scan, Scan.id == DomainPosture.scan_id) .where(DomainPosture.target_id == self.target.id) .group_by(DomainPosture.scan_id) .order_by(func.max(Scan.created_at).desc()) .limit(1) ).first() scan_id = row[0] if row else None if scan_id is None: return None rows = ( self.session.execute( select(DomainPosture) .where(DomainPosture.scan_id == scan_id) .order_by(DomainPosture.hosts.desc(), DomainPosture.zone) ) .scalars() .all() ) if not rows: return None warning_keys = set(posture_defs.WARNING_KEYS) spoof_keys = set(posture_defs.SPOOFABLE_KEYS) failing: dict[str, int] = dict.fromkeys(posture_defs.CHECK_KEYS, 0) applicable: dict[str, int] = dict.fromkeys(posture_defs.CHECK_KEYS, 0) warning = info = clean = spoofable = 0 labels = {c.key: c.label for c in posture_defs.CHECKS} order = posture_defs.CHECK_ORDER zones: list[PostureZone] = [] for r in rows: issues = set(r.posture_issues or []) for key in r.posture_checked or []: if key in applicable: applicable[key] += 1 for key in issues: if key in failing: failing[key] += 1 if issues & warning_keys: warning += 1 elif issues: info += 1 else: clean += 1 if issues & spoof_keys: spoofable += 1 zones.append( PostureZone( zone=r.zone, parent=r.parent, hosts=r.hosts, spf_all=r.spf_all, dmarc_policy=r.dmarc_policy, dnssec=r.dnssec, checks=[ labels.get(k, k) for k in sorted(issues, key=lambda k: order.get(k, 99)) ], ) ) return PostureRollup( evaluated=len(rows), zone_count=sum(1 for r in rows if r.parent is None), mail_hosts=sum(1 for r in rows if r.parent is not None), clean=clean, warning=warning, info=info, spoofable=spoofable, checks=[ HygieneCount(key=key, failing=failing[key], applicable=applicable[key]) for key in posture_defs.CHECK_KEYS ], zones=zones[:MAX_REPORT_ROWS], ) # ---------- certificates ---------- @cached_property def certificates(self) -> list[Certificate]: scan_id = self.scan_for(_DIM.WEB_ASSETS.value) if scan_id is None: return [] rows = self.session.execute( select( HttpAsset.host, HttpAsset.tls_subject_cn, HttpAsset.tls_issuer_org, HttpAsset.tls_issuer_cn, HttpAsset.tls_not_before, HttpAsset.tls_not_after, HttpAsset.tls_expired, HttpAsset.tls_self_signed, HttpAsset.tls_sans, HttpAsset.tls_version, ) .where(HttpAsset.scan_id == scan_id, HttpAsset.tls_not_after.is_not(None)) .distinct(HttpAsset.host) .order_by(HttpAsset.host, HttpAsset.id) .limit(MAX_REPORT_ROWS) ).all() now = self.observed_at or utc_now() out: list[Certificate] = [] for row in rows: expires = row[5] days = None if expires is not None: try: days = (expires - now).days except TypeError: days = None out.append( Certificate( host=row[0], subject=row[1], issuer=row[2] or row[3], not_before=row[4], not_after=expires, expired=row[6], self_signed=row[7], days_left=days, sans=len(row[8] or []), version=row[9], ) ) return out # ---------- rollups ---------- def _facet( self, column, scan_id: UUID | None, table, limit: int = 12 ) -> list[Facet]: if scan_id is None: return [] rows = self.session.execute( select(column, func.count()) .where(table.scan_id == scan_id, column.is_not(None), column != "") .group_by(column) .order_by(func.count().desc()) .limit(limit) ).all() return [Facet(name=str(row[0]), count=int(row[1])) for row in rows] @cached_property def countries(self) -> list[Facet]: return self._facet(IpAddress.country, self.scan_for(_DIM.IPS.value), IpAddress) @cached_property def networks(self) -> list[Facet]: return self._facet(IpAddress.asn_org, self.scan_for(_DIM.IPS.value), IpAddress) @cached_property def webservers(self) -> list[Facet]: return self._facet( HttpAsset.webserver, self.scan_for(_DIM.WEB_ASSETS.value), HttpAsset ) @cached_property def technologies(self) -> list[Facet]: scan_id = self.scan_for(_DIM.WEB_ASSETS.value) if scan_id is None: return [] rows = self.session.execute( text( "SELECT v AS value, count(DISTINCT a.host) AS c " "FROM http_assets a, LATERAL jsonb_array_elements_text(cast(a.tech AS jsonb)) v " "WHERE a.scan_id = :sid GROUP BY v ORDER BY c DESC, v ASC LIMIT 14" ), {"sid": str(scan_id)}, ).all() return [Facet(name=str(row[0]), count=int(row[1])) for row in rows] @cached_property def status_classes(self) -> list[Facet]: scan_id = self.scan_for(_DIM.WEB_ASSETS.value) if scan_id is None: return [] bucket = case( (Subdomain.http_status.between(200, 299), "2xx"), (Subdomain.http_status.between(300, 399), "3xx"), (Subdomain.http_status.between(400, 499), "4xx"), (Subdomain.http_status >= _SERVER_ERROR, "5xx"), else_="none", ) rows = self.session.execute( select(bucket, func.count()) .where(Subdomain.scan_id == scan_id) .group_by(bucket) .order_by(bucket) ).all() return [Facet(name=str(row[0]), count=int(row[1])) for row in rows] @cached_property def service_classes(self) -> list[Facet]: scan_id = self.scan_for(_DIM.SERVICES.value) if scan_id is None: return [] rows = self.session.execute( select(Port.service_class, func.count()) .where(Port.scan_id == scan_id) .group_by(Port.service_class) .order_by(func.count().desc()) ).all() order = [c.value for c in ServiceClass] facets = [Facet(name=str(row[0]), count=int(row[1])) for row in rows] facets.sort(key=lambda f: order.index(f.name) if f.name in order else 99) return facets @cached_property def top_services(self) -> list[Facet]: scan_id = self.scan_for(_DIM.SERVICES.value) if scan_id is None: return [] rows = self.session.execute( select(Port.service_name, func.count()) .where(Port.scan_id == scan_id, Port.service_name.is_not(None)) .group_by(Port.service_name) .order_by(func.count().desc()) .limit(12) ).all() return [Facet(name=str(row[0]), count=int(row[1])) for row in rows] @cached_property def cdn_split(self) -> tuple[int, int, int]: """Edge, cloud, direct, across resolving hosts.""" scan_id = self.scan_for(_DIM.WEB_ASSETS.value) if scan_id is None: return (0, 0, 0) rows = self.session.execute( select(HttpAsset.cdn_type, func.count(distinct(HttpAsset.host))) .where(HttpAsset.scan_id == scan_id) .group_by(HttpAsset.cdn_type) ).all() edge = cloud = direct = 0 for kind, count in rows: if kind in {"cdn", "waf"}: edge += int(count) elif kind == "cloud": cloud += int(count) else: direct += int(count) return (edge, cloud, direct) # ---------- run detail ---------- @cached_property def tools_used(self) -> list[str]: try: from stages.registry import stages as stage_specs # noqa: PLC0415 except ImportError: return [] specs = {spec.name: spec for spec in stage_specs()} names: set[str] = set() for stage in self.planned_stages: spec = specs.get(stage) if spec: names.update(spec.tools) return sorted(names) def trend(self, dimension: str, limit: int = 8) -> list[int]: table = _TABLE[dimension] rows = self.session.execute( select(table.scan_id, func.count(), func.max(Scan.created_at).label("at")) .join(Scan, Scan.id == table.scan_id) .where(table.target_id == self.target.id) .group_by(table.scan_id) .order_by(func.max(Scan.created_at).desc()) .limit(limit) ).all() return [int(row[1]) for row in reversed(rows)] def added_and_gone( self, dimension: str, limit: int = 40 ) -> tuple[list[str], list[str], int, int]: """What this run holds that the previous did not, and the other way round.""" previous = self.previous_scan scan_id = self.scan_for(dimension) if previous is None or scan_id is None: return ([], [], 0, 0) table = _TABLE[dimension] columns = _KEY[dimension] label = _LABEL[dimension] service = dimension == _DIM.SERVICES.value def read(scan: UUID) -> dict[tuple, str]: rows = self.session.execute( select(*columns, label).where(table.scan_id == scan).distinct() ) out: dict[tuple, str] = {} for row in rows: key = tuple(row[: len(columns)]) out[key] = f"{row[-1]}:{key[1]}" if service else str(row[-1]) return out current = read(scan_id) before = read(previous.id) added = [v for k, v in current.items() if k not in before] gone = [v for k, v in before.items() if k not in current] return (sorted(added)[:limit], sorted(gone)[:limit], len(added), len(gone)) def excluded(self) -> dict[str, list[str]]: config = (self.scan.execution_config or {}) if self.scan else {} return { "hosts": list(config.get("excluded_subdomains") or []), "addresses": list(config.get("excluded_ips") or []), "paths": list(config.get("excluded_paths") or []), } def has_any(self, *dimensions: str) -> bool: return any(self.count_of(d) for d in dimensions) @cached_property def origin_candidates(self) -> list[tuple[str, str, str, list[str]]]: """Addresses answering directly that share an identity with a CDN-fronted hostname.""" scan_id = self.scan_for(_DIM.WEB_ASSETS.value) if scan_id is None: return [] rows = self.session.execute( select( HttpAsset.host, HttpAsset.ip, HttpAsset.cdn_type, HttpAsset.content_hash, HttpAsset.tls_fingerprint, HttpAsset.favicon_hash, HttpAsset.status_code, ).where( HttpAsset.scan_id == scan_id, HttpAsset.status_code.is_not(None), ) ).all() fronted: dict[tuple[str, str], set[str]] = {} direct: dict[tuple[str, str], set[str]] = {} for host, ip, cdn_type, content, tls, favicon, _status in rows: for kind, value in (("body", content), ("tls", tls), ("favicon", favicon)): if not value: continue bucket = fronted if cdn_type in {"cdn", "waf"} else direct bucket.setdefault((kind, value), set()).add( host if bucket is fronted else (ip or host) ) out: list[tuple[str, str, str, list[str]]] = [] seen: set[str] = set() for key, addresses in direct.items(): hosts = fronted.get(key) if not hosts: continue for address in sorted(addresses): if address in seen: continue seen.add(address) out.append((address, key[0], key[1][:16], sorted(hosts)[:6])) return out[:40]