rengine/api/app/services/vulnerability.py

from __future__ import annotations

from datetime import datetime
from uuid import UUID

from sqlalchemy import Text, case, cast, distinct, func, not_, or_, select, text
from sqlalchemy.dialects.postgresql import JSONB, aggregate_order_by
from sqlalchemy.exc import DBAPIError
from sqlalchemy.ext.asyncio import AsyncSession

from app.services.asset_query import (
    NO_JIT,
    STATEMENT_TIMEOUT,
    QueryScope,
    QuerySyntaxError,
    ScopeLike,
    VulnQueryContext,
    build_leads,
    build_vuln_groups,
    compile_vuln_query,
    parse_query,
    query_error_for,
    vuln_corroborated,
    vuln_corroborated_ids,
    vuln_has_baseline,
    vuln_state,
    vuln_suppressed,
)
from app.services.surface_scope import baselined_targets
from app.services.target_names import target_names
from shared.definitions.asset_query import COUNT_CAP, VULN_QUERY
from shared.definitions.vulnerabilities import (
    ACTIONABLE_SEVERITIES,
    CUSTOM_ROOT,
    EPSS_HIGH,
    OFFICIAL_ROOT,
    PROTOCOL_LABELS,
    SCANNER_LABELS,
    SEVERITY_LABELS,
    SEVERITY_ORDER,
    VULN_STATE_LABELS,
    VULN_STATES,
    CorroborationBasis,
    CoverageStatus,
    Severity,
    VulnState,
)
from shared.logging import get_logger
from shared.models.asset_query import QueryError, QueryGroups, QueryLeads
from shared.models.http_asset import HttpAsset
from shared.models.ip_address import IpAddress
from shared.models.scan import Scan
from shared.models.subdomain import Subdomain
from shared.models.vulnerability import (
    AssetContext,
    BulkTriageResult,
    BulkTriageUpdate,
    Corroboration,
    CoverageRead,
    IssuePage,
    IssueRead,
    RankedFinding,
    ScanVulnerabilities,
    SeverityCount,
    TriageResult,
    TriageUpdate,
    Vulnerability,
    VulnerabilityCoverage,
    VulnerabilityFacets,
    VulnerabilityFilter,
    VulnerabilityPage,
    VulnerabilityRead,
    VulnerabilityTriage,
    VulnFacet,
    VulnHostRow,
)
from shared.services.asset_query import lead_cache
from shared.services.vuln_templates import sets_for
from shared.utils.datetime import utc_now

logger = get_logger(__name__)

_SAMPLE_HOSTS = 3
_TEMPLATE_ROOTS = (OFFICIAL_ROOT, CUSTOM_ROOT)

_FACET_LIMIT = 30
_TOP_FINDINGS = 6
_TOP_HOSTS = 6
_EVIDENCE_PREVIEW = 400

# fingerprints on this page an earlier scan of the same target already reported
_SEEN_SQL = """
SELECT DISTINCT e.fingerprint AS fingerprint
FROM vulnerabilities e
JOIN vulnerabilities cur ON cur.scan_id = ANY(:sids) AND cur.fingerprint = e.fingerprint
WHERE e.target_id = cur.target_id AND NOT (e.scan_id = ANY(:sids))
  AND e.discovered_at < cur.discovered_at AND e.fingerprint = ANY(:fps)
"""


def _rank_of(severity: str) -> int:
    try:
        return SEVERITY_ORDER.index(severity)
    except ValueError:
        return len(SEVERITY_ORDER)


def _corroboration_basis(row, other) -> tuple[str, list[str]] | None:
    shared = sorted(set(row.cve_ids or []) & set(other.cve_ids or []))
    if shared:
        return CorroborationBasis.CVE.value, shared
    shared = sorted(set(row.cwe_ids or []) & set(other.cwe_ids or []))
    if shared:
        return CorroborationBasis.CWE.value, shared
    return None


def _severity_rank():
    return case(
        {name: index for index, name in enumerate(SEVERITY_ORDER)},
        value=Vulnerability.severity,
        else_=len(SEVERITY_ORDER),
    )


class VulnerabilityService:
    def __init__(self, session: AsyncSession):
        self.session = session

    @staticmethod
    def _context(scope: QueryScope, now: datetime) -> VulnQueryContext:
        return VulnQueryContext(scope=scope, now=now)

    @staticmethod
    def _apply_filter(query, f: VulnerabilityFilter, scope: QueryScope):
        if f.severities:
            query = query.where(Vulnerability.severity.in_(f.severities))
        elif not f.include_info:
            query = query.where(Vulnerability.severity.in_(ACTIONABLE_SEVERITIES))
        if f.protocols:
            query = query.where(Vulnerability.protocol.in_(f.protocols))
        if f.templates:
            query = query.where(Vulnerability.template_id.in_(f.templates))
        if f.hosts:
            query = query.where(Vulnerability.host.in_(f.hosts))
        if f.scanners:
            query = query.where(Vulnerability.scanner.in_(f.scanners))
        if f.tags:
            query = query.where(
                or_(
                    *[
                        func.jsonb_exists(cast(Vulnerability.tags, JSONB), tag)
                        for tag in f.tags
                    ]
                )
            )
        if f.kev:
            query = query.where(Vulnerability.is_kev.is_(True))
        if f.cve:
            query = query.where(
                func.jsonb_array_length(cast(Vulnerability.cve_ids, JSONB)) > 0
            )
        if f.new:
            from app.services.asset_query import vuln_is_new  # noqa: PLC0415

            query = query.where(vuln_is_new(scope))
        if f.corroborated:
            query = query.where(vuln_corroborated(scope))
        if f.states:
            query = query.where(vuln_state(scope).in_(f.states))
        elif not f.include_suppressed:
            query = query.where(not_(vuln_suppressed(scope)))
        return query

    @staticmethod
    def _order(query, f: VulnerabilityFilter, scope: QueryScope):
        if f.sort == "risk":
            # the finding first: severity, then what makes one severity outrank another
            confirmed = vuln_corroborated_ids(scope).subquery()
            return query.outerjoin(
                confirmed, confirmed.c.id == Vulnerability.id
            ).order_by(
                _severity_rank().asc(),
                Vulnerability.is_kev.desc(),
                Vulnerability.exploit_score.desc(),
                confirmed.c.id.isnot(None).desc(),
                Vulnerability.epss_score.desc().nulls_last(),
                Vulnerability.cvss_score.desc().nulls_last(),
                Vulnerability.template_id.asc(),
                Vulnerability.matched_at.asc(),
            )
        column = {
            "severity": _severity_rank(),
            "name": Vulnerability.template_name,
            "template": Vulnerability.template_id,
            "host": Vulnerability.host,
            "seen": Vulnerability.discovered_at,
            "cvss": Vulnerability.cvss_score,
            "epss": Vulnerability.epss_score,
            "exploit": Vulnerability.exploit_score,
            "type": Vulnerability.protocol,
        }.get(f.sort, _severity_rank())
        primary = column.desc() if f.order == "desc" else column.asc()
        return query.order_by(
            primary.nulls_last(),
            Vulnerability.template_id.asc(),
            Vulnerability.matched_at.asc(),
        )

    def _scoped(self, scope: QueryScope, f: VulnerabilityFilter, columns=None):
        base = select(Vulnerability) if columns is None else select(*columns)
        base = base.where(scope.match(Vulnerability.scan_id))
        return self._apply_filter(base, f, scope)

    async def search(
        self, scope: ScopeLike, f: VulnerabilityFilter
    ) -> VulnerabilityPage:
        scope = QueryScope.of(scope)
        now = utc_now()
        base = self._scoped(scope, f)
        try:
            predicate = compile_vuln_query(
                parse_query(f.q, VULN_QUERY), self._context(scope, now)
            )
        except QuerySyntaxError as exc:
            return VulnerabilityPage(
                error=QueryError(
                    message=exc.message, hint=exc.hint, start=exc.start, end=exc.end
                )
            )
        if predicate is not None:
            base = base.where(predicate)

        await self.session.execute(text(STATEMENT_TIMEOUT))
        await self.session.execute(text(NO_JIT))
        try:
            counted = await self.session.scalar(
                select(func.count()).select_from(base.limit(COUNT_CAP + 1).subquery())
            )
            rows = (
                (
                    await self.session.execute(
                        self._order(base, f, scope).limit(f.limit).offset(f.offset)
                    )
                )
                .scalars()
                .all()
            )
        except DBAPIError as exc:
            await self.session.rollback()
            rejected = query_error_for(exc)
            if rejected is None:
                raise
            logger.info("vulnerability query rejected", error=str(exc.orig))
            return VulnerabilityPage(error=rejected)

        total = int(counted or 0)
        capped = total > COUNT_CAP
        page = VulnerabilityPage(
            total=min(total, COUNT_CAP) if capped else total, total_capped=capped
        )
        if not rows:
            return page

        fingerprints = [row.fingerprint for row in rows]
        baseline, seen = await self._seen_before(scope, fingerprints)
        states = await self._states(scope, fingerprints)
        spread = await self._spread(scope, [row.template_id for row in rows])
        assets = await self._assets(scope, rows)
        peers, colocated = await self._corroborators(scope, rows)
        names = await target_names(self.session, (row.target_id for row in rows))
        for row in rows:
            decision = states.get((row.target_id, row.fingerprint))
            page.items.append(
                self._to_read(
                    row,
                    state=decision[0] if decision else VulnState.OPEN.value,
                    note=decision[1] if decision else None,
                    is_new=row.target_id in baseline and row.fingerprint not in seen,
                    host_count=spread.get(row.template_id, 1),
                    asset=assets.get(row.id),
                    corroborated_by=peers.get(row.id, []),
                    colocated=colocated.get(row.id, 0),
                    evidence=False,
                    target_value=names.get(row.target_id),
                )
            )
        return page

    def _to_read(
        self,
        row: Vulnerability,
        *,
        state: str,
        note: str | None,
        is_new: bool,
        host_count: int,
        asset: AssetContext | None,
        evidence: bool,
        corroborated_by: list[Corroboration] | None = None,
        colocated: int = 0,
        target_value: str | None = None,
    ) -> VulnerabilityRead:
        return VulnerabilityRead(
            id=row.id,
            scan_id=row.scan_id,
            target_id=row.target_id,
            target_value=target_value,
            fingerprint=row.fingerprint,
            scanner=row.scanner,
            template_id=row.template_id,
            template_name=row.template_name,
            template_path=row.template_path,
            template_url=row.template_url,
            severity=row.severity,
            protocol=row.protocol,
            matcher_name=row.matcher_name,
            extractor_name=row.extractor_name,
            extracted_results=list(row.extracted_results or []),
            description=row.description,
            impact=row.impact,
            remediation=row.remediation,
            references=list(row.references or []),
            tags=list(row.tags or []),
            authors=list(row.authors or []),
            cve_ids=list(row.cve_ids or []),
            cwe_ids=list(row.cwe_ids or []),
            cvss_metrics=row.cvss_metrics,
            cvss_score=row.cvss_score,
            epss_score=row.epss_score,
            epss_percentile=row.epss_percentile,
            cpe=row.cpe,
            is_kev=row.is_kev,
            exploit_score=int(row.exploit_score or 0),
            intel_kinds=list(row.intel_kinds or []),
            kev_ransomware=bool(row.kev_ransomware),
            kev_due_date=row.kev_due_date,
            poc_count=row.poc_count,
            template_available=row.template_available,
            hackerone_reports=row.hackerone_reports,
            matched_at=row.matched_at,
            host=row.host,
            ip=row.ip,
            port=row.port,
            scheme=row.scheme,
            url=row.url,
            path=row.path,
            curl_command=row.curl_command,
            request=row.request if evidence else _preview(row.request),
            response=row.response if evidence else _preview(row.response),
            interaction=dict(row.interaction or {}),
            observed_at=row.observed_at,
            discovered_at=row.discovered_at,
            state=state,
            note=note,
            is_new=is_new,
            host_count=host_count,
            sets=_sets(row.tags, row.template_path),
            corroborated_by=corroborated_by or [],
            colocated=colocated,
            asset=asset,
        )

    async def issues(self, scope: ScopeLike, f: VulnerabilityFilter) -> IssuePage:
        """One row per check that fired, over the same scope and filter as search()."""
        scope = QueryScope.of(scope)
        now = utc_now()
        try:
            predicate = compile_vuln_query(
                parse_query(f.q, VULN_QUERY), self._context(scope, now)
            )
        except QuerySyntaxError as exc:
            return IssuePage(
                error=QueryError(
                    message=exc.message, hint=exc.hint, start=exc.start, end=exc.end
                )
            )
        rank = _severity_rank()
        state = vuln_state(scope)
        baseline = await self.session.scalar(select(vuln_has_baseline(scope)))
        confirmed = vuln_corroborated_ids(scope).subquery("vuln_confirmed")
        columns = [
            Vulnerability.template_id,
            func.count().label("findings"),
            func.count(distinct(Vulnerability.host)).label("hosts"),
            func.count(distinct(Vulnerability.ip)).label("addresses"),
            func.count(distinct(Vulnerability.matched_at)).label("locations"),
            func.min(rank).label("rank"),
            func.bool_or(Vulnerability.is_kev).label("kev"),
            func.max(Vulnerability.cvss_score).label("cvss"),
            func.max(Vulnerability.epss_score).label("epss"),
            func.min(Vulnerability.discovered_at).label("first_seen"),
            func.max(Vulnerability.discovered_at).label("last_seen"),
            func.min(Vulnerability.template_name).label("name"),
            func.min(cast(Vulnerability.id, Text)).label("sample"),
            func.array_agg(
                aggregate_order_by(distinct(Vulnerability.host), Vulnerability.host)
            ).label("host_list"),
            func.count().filter(confirmed.c.id.isnot(None)).label("corroborated"),
        ]
        if baseline:
            from app.services.asset_query import vuln_is_new  # noqa: PLC0415

            columns.append(func.count().filter(vuln_is_new(scope)).label("new"))
        for name in VULN_STATES:
            columns.append(func.count().filter(state == name).label(f"state_{name}"))
        base = self._scoped(scope, f, columns=columns).outerjoin(
            confirmed, confirmed.c.id == Vulnerability.id
        )
        if predicate is not None:
            base = base.where(predicate)
        grouped = base.group_by(Vulnerability.template_id)
        counted_sq = grouped.subquery()
        order = {
            "severity": counted_sq.c.rank,
            "name": counted_sq.c.name,
            "template": counted_sq.c.template_id,
            "host": counted_sq.c.hosts,
            "seen": counted_sq.c.first_seen,
            "cvss": counted_sq.c.cvss,
            "epss": counted_sq.c.epss,
            "findings": counted_sq.c.findings,
        }
        if f.sort == "risk":
            ordering = [
                counted_sq.c.rank.asc(),
                counted_sq.c.kev.desc(),
                counted_sq.c.corroborated.desc(),
                counted_sq.c.epss.desc().nulls_last(),
                counted_sq.c.cvss.desc().nulls_last(),
                counted_sq.c.hosts.desc(),
                counted_sq.c.template_id.asc(),
            ]
        else:
            column = order.get(f.sort, counted_sq.c.rank)
            primary = column.desc() if f.order == "desc" else column.asc()
            ordering = [primary.nulls_last(), counted_sq.c.template_id.asc()]

        await self.session.execute(text(STATEMENT_TIMEOUT))
        await self.session.execute(text(NO_JIT))
        try:
            total = int(
                await self.session.scalar(
                    select(func.count()).select_from(
                        select(counted_sq.c.template_id).limit(COUNT_CAP + 1).subquery()
                    )
                )
                or 0
            )
            rows = (
                await self.session.execute(
                    select(counted_sq)
                    .order_by(*ordering)
                    .limit(f.limit)
                    .offset(f.offset)
                )
            ).all()
        except DBAPIError as exc:
            await self.session.rollback()
            rejected = query_error_for(exc)
            if rejected is None:
                raise
            logger.info("vulnerability issues rejected", error=str(exc.orig))
            return IssuePage(error=rejected)

        capped = total > COUNT_CAP
        page = IssuePage(
            total=min(total, COUNT_CAP) if capped else total, total_capped=capped
        )
        if not rows:
            return page
        samples = {
            row.id: row
            for row in (
                await self.session.execute(
                    select(Vulnerability).where(
                        Vulnerability.id.in_([UUID(r.sample) for r in rows])
                    )
                )
            ).scalars()
        }
        for r in rows:
            sample = samples.get(UUID(r.sample))
            if sample is None:
                continue
            hosts = [h for h in (r.host_list or []) if h]
            page.items.append(
                IssueRead(
                    template_id=r.template_id,
                    template_name=sample.template_name,
                    template_url=sample.template_url,
                    scanner=sample.scanner,
                    severity=SEVERITY_ORDER[int(r.rank)]
                    if r.rank is not None and int(r.rank) < len(SEVERITY_ORDER)
                    else sample.severity,
                    protocol=sample.protocol,
                    tags=list(sample.tags or []),
                    sets=_sets(sample.tags, sample.template_path),
                    cve_ids=list(sample.cve_ids or []),
                    cwe_ids=list(sample.cwe_ids or []),
                    cvss_score=r.cvss,
                    epss_score=r.epss,
                    is_kev=bool(r.kev),
                    description=sample.description,
                    remediation=sample.remediation,
                    findings=int(r.findings),
                    hosts=int(r.hosts),
                    addresses=int(r.addresses),
                    locations=int(r.locations),
                    sample_hosts=hosts[:_SAMPLE_HOSTS],
                    corroborated=int(r.corroborated or 0),
                    new_count=int(getattr(r, "new", 0) or 0),
                    states={
                        name: int(getattr(r, f"state_{name}") or 0)
                        for name in VULN_STATES
                        if getattr(r, f"state_{name}")
                    },
                    first_seen=r.first_seen,
                    last_seen=r.last_seen,
                    sample_id=sample.id,
                )
            )
        return page

    async def get(self, scope: ScopeLike, vuln_id: UUID) -> VulnerabilityRead | None:
        scope = QueryScope.of(scope)
        row = await self.session.get(Vulnerability, vuln_id)
        if row is None or row.scan_id not in scope.ids:
            return None
        baseline, seen = await self._seen_before(scope, [row.fingerprint])
        states = await self._states(scope, [row.fingerprint])
        spread = await self._spread(scope, [row.template_id])
        assets = await self._assets(scope, [row])
        peers, colocated = await self._corroborators(scope, [row])
        decision = states.get((row.target_id, row.fingerprint))
        return self._to_read(
            row,
            state=decision[0] if decision else VulnState.OPEN.value,
            note=decision[1] if decision else None,
            is_new=row.target_id in baseline and row.fingerprint not in seen,
            host_count=spread.get(row.template_id, 1),
            asset=assets.get(row.id),
            corroborated_by=peers.get(row.id, []),
            colocated=colocated.get(row.id, 0),
            evidence=True,
        )

    async def _corroborators(
        self, scope: QueryScope, rows: list[Vulnerability]
    ) -> tuple[dict[UUID, list[Corroboration]], dict[UUID, int]]:
        """Peers that confirm the same weakness, and how many findings share each location."""
        locations = sorted({r.matched_at for r in rows})
        if not locations:
            return {}, {}
        found = await self.session.execute(
            select(
                Vulnerability.id,
                Vulnerability.matched_at,
                Vulnerability.template_id,
                Vulnerability.template_name,
                Vulnerability.severity,
                Vulnerability.scanner,
                Vulnerability.cve_ids,
                Vulnerability.cwe_ids,
            ).where(
                scope.match(Vulnerability.scan_id),
                Vulnerability.matched_at.in_(locations),
                not_(vuln_suppressed(scope)),
            )
        )
        at_location: dict[str, list] = {}
        for other in found.all():
            at_location.setdefault(other.matched_at, []).append(other)

        peers: dict[UUID, list[Corroboration]] = {}
        colocated: dict[UUID, int] = {}
        for row in rows:
            others = [o for o in at_location.get(row.matched_at, []) if o.id != row.id]
            colocated[row.id] = len(others)
            if row.severity == Severity.INFO.value:
                continue
            agreed = []
            for other in others:
                if other.severity == Severity.INFO.value:
                    continue
                if other.template_id == row.template_id:
                    continue
                basis = _corroboration_basis(row, other)
                if basis is None:
                    continue
                agreed.append(
                    Corroboration(
                        template_id=other.template_id,
                        template_name=other.template_name,
                        severity=other.severity,
                        scanner=other.scanner,
                        basis=basis[0],
                        shared=basis[1],
                    )
                )
            peers[row.id] = sorted(
                agreed,
                key=lambda c: (
                    _rank_of(c.severity),
                    c.scanner == row.scanner,
                    c.template_id,
                ),
            )
        return peers, colocated

    async def _seen_before(
        self, scope: QueryScope, fingerprints: list[str]
    ) -> tuple[set[UUID], set[str]]:
        baseline = await baselined_targets(self.session, Vulnerability, scope)
        if not baseline:
            return set(), set()
        rows = (
            await self.session.execute(
                text(_SEEN_SQL).bindparams(sids=list(scope.ids), fps=fingerprints)
            )
        ).scalars()
        return baseline, set(rows.all())

    async def _states(
        self, scope: QueryScope, fingerprints: list[str]
    ) -> dict[tuple[UUID, str], tuple[str, str | None]]:
        targets = select(Scan.target_id).where(Scan.id.in_(scope.ids))
        rows = await self.session.execute(
            select(
                VulnerabilityTriage.target_id,
                VulnerabilityTriage.fingerprint,
                VulnerabilityTriage.state,
                VulnerabilityTriage.note,
            ).where(
                VulnerabilityTriage.target_id.in_(targets),
                VulnerabilityTriage.fingerprint.in_(fingerprints),
            )
        )
        return {(target, fp): (state, note) for target, fp, state, note in rows.all()}

    async def _spread(self, scope: QueryScope, templates: list[str]) -> dict[str, int]:
        """How many hostnames each check fired on in this scan — the finding's reach."""
        rows = await self.session.execute(
            select(
                Vulnerability.template_id,
                func.count(
                    func.distinct(
                        func.coalesce(Vulnerability.host, Vulnerability.matched_at)
                    )
                ),
            )
            .where(
                scope.match(Vulnerability.scan_id),
                Vulnerability.template_id.in_(templates),
            )
            .group_by(Vulnerability.template_id)
        )
        return {template: int(n) for template, n in rows.all()}

    async def _assets(
        self, scope: QueryScope, rows: list[Vulnerability]
    ) -> dict[UUID, AssetContext]:
        hosts = sorted({r.host for r in rows if r.host})
        ips = sorted({r.ip for r in rows if r.ip})
        if not hosts and not ips:
            return {}

        by_host: dict[str, dict] = {}
        if hosts:
            found = await self.session.execute(
                select(
                    Subdomain.name,
                    Subdomain.http_status,
                    Subdomain.page_title,
                    Subdomain.webserver,
                    Subdomain.tech,
                    Subdomain.screenshot_path,
                    Subdomain.is_cdn,
                    Subdomain.cdn_name,
                    Subdomain.waf,
                ).where(scope.match(Subdomain.scan_id), Subdomain.name.in_(hosts))
            )
            for row in found:
                by_host[row.name] = dict(row._mapping)

        by_ip: dict[str, dict] = {}
        if ips:
            found = await self.session.execute(
                select(
                    IpAddress.ip,
                    IpAddress.asn,
                    IpAddress.asn_org,
                    IpAddress.country,
                    IpAddress.is_cdn,
                    IpAddress.cdn_name,
                ).where(scope.match(IpAddress.scan_id), IpAddress.ip.in_(ips))
            )
            for row in found:
                by_ip[row.ip] = dict(row._mapping)

        asset_ids = [r.http_asset_id for r in rows if r.http_asset_id]
        by_asset: dict[UUID, dict] = {}
        if asset_ids:
            found = await self.session.execute(
                select(
                    HttpAsset.id,
                    HttpAsset.url,
                    HttpAsset.status_code,
                    HttpAsset.title,
                    HttpAsset.webserver,
                    HttpAsset.tech,
                    HttpAsset.screenshot_path,
                    HttpAsset.is_cdn,
                    HttpAsset.cdn_name,
                    HttpAsset.waf,
                ).where(HttpAsset.id.in_(asset_ids))
            )
            for row in found:
                by_asset[row.id] = dict(row._mapping)

        out: dict[UUID, AssetContext] = {}
        for row in rows:
            asset = by_asset.get(row.http_asset_id) if row.http_asset_id else None
            host = by_host.get(row.host or "") or {}
            address = by_ip.get(row.ip or "") or {}
            if not asset and not host and not address:
                continue
            source = asset or host
            out[row.id] = AssetContext(
                host=row.host,
                url=(asset or {}).get("url") or row.url,
                status_code=source.get("status_code") or source.get("http_status"),
                title=source.get("title") or source.get("page_title"),
                webserver=source.get("webserver"),
                tech=list(source.get("tech") or []),
                screenshot_path=source.get("screenshot_path"),
                ip=row.ip,
                asn=address.get("asn"),
                asn_org=address.get("asn_org"),
                country=address.get("country"),
                is_cdn=bool(source.get("is_cdn") or address.get("is_cdn")),
                cdn_name=source.get("cdn_name") or address.get("cdn_name"),
                waf=source.get("waf"),
            )
        return out

    async def leads(self, scope: ScopeLike, f: VulnerabilityFilter) -> QueryLeads:
        scope = QueryScope.of(scope)

        async def _build() -> QueryLeads:
            now = utc_now()
            base = self._scoped(scope, f, columns=(Vulnerability.id,))
            ctx = self._context(scope, now)
            await self.session.execute(text(STATEMENT_TIMEOUT))
            await self.session.execute(text(NO_JIT))
            try:
                return await build_leads(
                    self.session,
                    base,
                    VULN_QUERY.examples,
                    lambda q: compile_vuln_query(parse_query(q, VULN_QUERY), ctx),
                    filtered=f.has_facets(),
                )
            except DBAPIError as exc:
                await self.session.rollback()
                logger.info("vulnerability leads failed", error=str(exc.orig))
                return QueryLeads()

        return await lead_cache.leads(
            self.session,
            dimension="vulnerabilities",
            scans=scope.ids,
            facets=lead_cache.facets_of(f),
            build=_build,
        )

    async def groups(
        self, scope: ScopeLike, f: VulnerabilityFilter, key: str
    ) -> QueryGroups:
        scope = QueryScope.of(scope)
        now = utc_now()
        base = self._scoped(scope, f, columns=(Vulnerability.id,))
        try:
            predicate = compile_vuln_query(
                parse_query(f.q, VULN_QUERY), self._context(scope, now)
            )
        except QuerySyntaxError:
            return QueryGroups(dimension=key)
        if predicate is not None:
            base = base.where(predicate)
        await self.session.execute(text(STATEMENT_TIMEOUT))
        await self.session.execute(text(NO_JIT))
        try:
            return await build_vuln_groups(self.session, base, key, scope)
        except DBAPIError as exc:
            await self.session.rollback()
            logger.info("vulnerability groups failed", error=str(exc.orig))
            return QueryGroups(dimension=key)

    async def facets(self, scope: ScopeLike) -> VulnerabilityFacets:
        scope = QueryScope.of(scope)
        reach = scope.match(Vulnerability.scan_id)
        n = func.count()

        async def tally(column, limit: int = _FACET_LIMIT):
            rows = await self.session.execute(
                select(column, n)
                .where(reach, column.isnot(None), cast(column, Text) != "")
                .group_by(column)
                .order_by(n.desc(), column)
                .limit(limit)
            )
            return [
                VulnFacet(name=str(value), count=int(count))
                for value, count in rows.all()
            ]

        async def tally_array(column, limit: int = _FACET_LIMIT):
            value = func.jsonb_array_elements_text(cast(column, JSONB)).column_valued(
                "v"
            )
            rows = await self.session.execute(
                select(value, n)
                .select_from(Vulnerability)
                .where(reach)
                .group_by(value)
                .order_by(n.desc(), value)
                .limit(limit)
            )
            return [VulnFacet(name=str(v), count=int(c)) for v, c in rows.all()]

        state = vuln_state(scope).label("state")
        state_rows = await self.session.execute(
            select(state, n).where(reach).group_by(state).order_by(n.desc())
        )
        severity = await tally(Vulnerability.severity, len(SEVERITY_ORDER))
        worst = (
            select(Vulnerability.template_id, func.min(_severity_rank()).label("rank"))
            .where(reach)
            .group_by(Vulnerability.template_id)
            .subquery()
        )
        issue_rows = await self.session.execute(
            select(worst.c.rank, func.count()).group_by(worst.c.rank)
        )
        issue_counts = {int(rank): int(count) for rank, count in issue_rows.all()}

        def by_severity(counts: dict[str, int]) -> list[VulnFacet]:
            return [
                VulnFacet(
                    name=name, count=counts[name], label=SEVERITY_LABELS.get(name)
                )
                for name in SEVERITY_ORDER
                if counts.get(name)
            ]

        return VulnerabilityFacets(
            severity=by_severity({f.name: f.count for f in severity}),
            issue_severity=by_severity(
                {
                    SEVERITY_ORDER[rank]: count
                    for rank, count in issue_counts.items()
                    if rank < len(SEVERITY_ORDER)
                }
            ),
            state=[
                VulnFacet(
                    name=str(value),
                    count=int(count),
                    label=VULN_STATE_LABELS.get(str(value)),
                )
                for value, count in state_rows.all()
            ],
            protocol=[
                VulnFacet(name=f.name, count=f.count, label=PROTOCOL_LABELS.get(f.name))
                for f in await tally(Vulnerability.protocol)
            ],
            template=await tally(Vulnerability.template_id),
            tag=await tally_array(Vulnerability.tags),
            host=await tally(Vulnerability.host),
            scanner=[
                VulnFacet(name=f.name, count=f.count, label=SCANNER_LABELS.get(f.name))
                for f in await tally(Vulnerability.scanner, 8)
            ],
        )

    async def coverage(self, scope: ScopeLike) -> list[CoverageRead]:
        scope = QueryScope.of(scope)
        rows = await self.session.execute(
            select(VulnerabilityCoverage)
            .where(scope.match(VulnerabilityCoverage.scan_id))
            .order_by(VulnerabilityCoverage.started_at)
        )
        return [
            CoverageRead(
                id=row.id,
                scanner=row.scanner,
                group=row.group,
                status=row.status,
                severities=list(row.severities or []),
                template_sets=list(row.template_sets or []),
                templates_selected=row.templates_selected,
                templates_loaded=row.templates_loaded,
                custom_templates=row.custom_templates,
                hosts_total=row.hosts_total,
                hosts_scanned=row.hosts_scanned,
                hosts_dropped=list(row.hosts_dropped or []),
                requests_sent=row.requests_sent,
                matched=row.matched,
                errors=row.errors,
                rate_limit=row.rate_limit,
                duration_seconds=row.duration_seconds,
                error=row.error,
            )
            for row in rows.scalars().all()
        ]

    async def overview(self, scope: ScopeLike) -> ScanVulnerabilities:
        scope = QueryScope.of(scope)
        coverage = await self.coverage(scope)
        ran = any(row.status != CoverageStatus.SKIPPED.value for row in coverage)
        reach = scope.match(Vulnerability.scan_id)
        live = not_(vuln_suppressed(scope))

        counts = (
            await self.session.execute(
                select(Vulnerability.severity, func.count())
                .where(reach, live)
                .group_by(Vulnerability.severity)
            )
        ).all()
        by_severity = {name: int(count) for name, count in counts}
        totals = (
            await self.session.execute(
                select(
                    func.count(),
                    func.count().filter(Vulnerability.is_kev.is_(True)),
                    func.count().filter(
                        func.jsonb_array_length(cast(Vulnerability.cve_ids, JSONB)) > 0
                    ),
                    func.count(func.distinct(Vulnerability.host)),
                    func.count(func.distinct(Vulnerability.template_id)),
                ).where(reach, live)
            )
        ).one()
        suppressed = await self.session.scalar(
            select(func.count()).where(reach, vuln_suppressed(scope))
        )
        new_count = await self._new_count(scope, live)

        total = int(totals[0] or 0)
        result = ScanVulnerabilities(
            ran=ran,
            total=total,
            issues=int(totals[4] or 0),
            actionable=sum(by_severity.get(s, 0) for s in ACTIONABLE_SEVERITIES),
            new_count=new_count,
            kev_count=int(totals[1] or 0),
            cve_count=int(totals[2] or 0),
            suppressed=int(suppressed or 0),
            affected_hosts=int(totals[3] or 0),
            by_severity=[
                SeverityCount(
                    severity=name,
                    label=SEVERITY_LABELS[name],
                    count=by_severity.get(name, 0),
                )
                for name in SEVERITY_ORDER
                if by_severity.get(name)
            ],
            coverage=coverage,
            scanned_hosts=sum(row.hosts_total for row in coverage),
            templates_run=max((_ran(row) for row in coverage), default=0) or None,
            requests_sent=_sum_or_none(row.requests_sent for row in coverage),
            duration_seconds=sum(row.duration_seconds or 0 for row in coverage) or None,
        )
        _headline(result)
        if total:
            result.top_findings = await self._top_findings(scope, live)
            result.top_hosts = await self._top_hosts(scope, live)
        return result

    async def _new_count(self, scope: QueryScope, live) -> int:
        from app.services.asset_query import vuln_is_new  # noqa: PLC0415

        baseline = await self.session.scalar(select(vuln_has_baseline(scope)))
        if not baseline:
            return 0
        return int(
            await self.session.scalar(
                select(func.count()).where(
                    scope.match(Vulnerability.scan_id), live, vuln_is_new(scope)
                )
            )
            or 0
        )

    async def _top_findings(self, scope: QueryScope, live) -> list[RankedFinding]:
        spread = (
            select(
                Vulnerability.template_id.label("template_id"),
                func.count(
                    func.distinct(
                        func.coalesce(Vulnerability.host, Vulnerability.matched_at)
                    )
                ).label("hosts"),
                func.min(_severity_rank()).label("rank"),
                func.min(Vulnerability.id.cast(Text)).label("sample"),
            )
            .where(scope.match(Vulnerability.scan_id), live)
            .group_by(Vulnerability.template_id)
            .subquery()
        )
        rows = await self.session.execute(
            select(Vulnerability, spread.c.hosts)
            .join(
                spread, Vulnerability.id == cast(spread.c.sample, Vulnerability.id.type)
            )
            .order_by(
                spread.c.rank.asc(),
                Vulnerability.is_kev.desc(),
                spread.c.hosts.desc(),
                Vulnerability.epss_score.desc().nulls_last(),
            )
            .limit(_TOP_FINDINGS)
        )
        items = list(rows.all())
        baseline: set[UUID] = set()
        seen: set[str] = set()
        if items:
            baseline, seen = await self._seen_before(
                scope, [row[0].fingerprint for row in items]
            )
        out = []
        for row, hosts in items:
            is_new = row.target_id in baseline and row.fingerprint not in seen
            signals = []
            if row.is_kev:
                signals.append("kev")
            if (row.epss_score or 0) >= EPSS_HIGH:
                signals.append("epss")
            if is_new:
                signals.append("new")
            out.append(
                RankedFinding(
                    id=row.id,
                    template_id=row.template_id,
                    name=row.template_name,
                    severity=row.severity,
                    host=row.host,
                    matched_at=row.matched_at,
                    host_count=int(hosts or 1),
                    is_kev=row.is_kev,
                    is_new=is_new,
                    cve_ids=list(row.cve_ids or []),
                    epss_score=row.epss_score,
                    signals=signals,
                )
            )
        return out

    async def _top_hosts(self, scope: QueryScope, live) -> list[VulnHostRow]:
        rank = _severity_rank()
        rows = await self.session.execute(
            select(
                Vulnerability.host,
                func.count().label("total"),
                func.min(rank).label("worst"),
                func.count().filter(Vulnerability.severity == Severity.CRITICAL.value),
                func.count().filter(Vulnerability.severity == Severity.HIGH.value),
                func.count().filter(Vulnerability.severity == Severity.MEDIUM.value),
            )
            .where(
                scope.match(Vulnerability.scan_id), live, Vulnerability.host.isnot(None)
            )
            .group_by(Vulnerability.host)
            .order_by(func.min(rank).asc(), func.count().desc())
            .limit(_TOP_HOSTS)
        )
        out = []
        for host, total, worst, critical, high, medium in rows.all():
            severity = (
                SEVERITY_ORDER[int(worst)] if worst is not None else Severity.INFO.value
            )
            counts = [
                SeverityCount(
                    severity=name, label=SEVERITY_LABELS[name], count=int(value)
                )
                for name, value in (
                    (Severity.CRITICAL.value, critical),
                    (Severity.HIGH.value, high),
                    (Severity.MEDIUM.value, medium),
                )
                if value
            ]
            out.append(
                VulnHostRow(
                    host=host,
                    total=int(total),
                    worst=severity,
                    counts=counts,
                    query=f'host="{host}"',
                )
            )
        return out

    async def triage(
        self, scope: ScopeLike, fingerprint: str, data: TriageUpdate, user_id: UUID
    ) -> TriageResult | None:
        scope = QueryScope.of(scope)
        row = await self.session.scalar(
            select(Vulnerability).where(
                scope.match(Vulnerability.scan_id),
                Vulnerability.fingerprint == fingerprint,
            )
        )
        if row is None:
            return None
        existing = await self.session.scalar(
            select(VulnerabilityTriage).where(
                VulnerabilityTriage.target_id == row.target_id,
                VulnerabilityTriage.fingerprint == fingerprint,
            )
        )
        now = utc_now()
        if existing is None:
            existing = VulnerabilityTriage(
                project_id=row.project_id,
                target_id=row.target_id,
                fingerprint=fingerprint,
                template_id=row.template_id,
                matched_at=row.matched_at,
            )
            self.session.add(existing)
        existing.state = data.state
        existing.note = data.note
        existing.decided_by = user_id
        existing.updated_at = now
        await self.session.commit()
        affected = await self.session.scalar(
            select(func.count()).where(
                Vulnerability.target_id == row.target_id,
                Vulnerability.fingerprint == fingerprint,
            )
        )
        return TriageResult(
            fingerprint=fingerprint,
            state=existing.state,
            note=existing.note,
            updated=int(affected or 0),
        )

    async def triage_many(
        self, scope: ScopeLike, data: BulkTriageUpdate, user_id: UUID
    ) -> BulkTriageResult:
        """One decision for every observation of the named findings or checks."""
        if not data.fingerprints and not data.template_ids:
            return BulkTriageResult(state=data.state)
        picks = []
        if data.fingerprints:
            picks.append(Vulnerability.fingerprint.in_(data.fingerprints))
        if data.template_ids:
            picks.append(Vulnerability.template_id.in_(data.template_ids))
        rows = (
            await self.session.execute(
                select(Vulnerability).where(
                    QueryScope.of(scope).match(Vulnerability.scan_id), or_(*picks)
                )
            )
        ).scalars()
        wanted = {(row.target_id, row.fingerprint): row for row in rows}
        if not wanted:
            return BulkTriageResult(state=data.state)
        targets = {key[0] for key in wanted}
        fingerprints = {key[1] for key in wanted}
        existing = {
            (row.target_id, row.fingerprint): row
            for row in (
                await self.session.execute(
                    select(VulnerabilityTriage).where(
                        VulnerabilityTriage.target_id.in_(targets),
                        VulnerabilityTriage.fingerprint.in_(fingerprints),
                    )
                )
            ).scalars()
        }
        now = utc_now()
        for key, sample in wanted.items():
            decision = existing.get(key)
            if decision is None:
                decision = VulnerabilityTriage(
                    project_id=sample.project_id,
                    target_id=key[0],
                    fingerprint=key[1],
                    template_id=sample.template_id,
                    matched_at=sample.matched_at,
                )
                self.session.add(decision)
            decision.state = data.state
            decision.note = data.note
            decision.decided_by = user_id
            decision.updated_at = now
        await self.session.commit()
        affected = await self.session.scalar(
            select(func.count()).where(
                Vulnerability.target_id.in_(targets),
                Vulnerability.fingerprint.in_(fingerprints),
            )
        )
        return BulkTriageResult(
            state=data.state,
            fingerprints=len(fingerprints),
            updated=int(affected or 0),
        )


def _sets(tags, path: str | None) -> list[str]:
    relative = path or ""
    for root in _TEMPLATE_ROOTS:
        if relative.startswith(root):
            relative = relative[len(root) :]
            break
    return sets_for(list(tags or []), relative)


def _preview(value: str | None) -> str | None:
    if not value:
        return None
    return value[:_EVIDENCE_PREVIEW]


def _sum_or_none(values) -> int | None:
    known = [v for v in values if v is not None]
    return sum(known) if known else None


def _ran(row) -> int:
    """What the scanner loaded, not what the plan asked for — a null is unknown, not zero."""
    if row.templates_loaded is not None:
        return row.templates_loaded
    return row.templates_selected or 0


def _headline(result: ScanVulnerabilities) -> None:
    """Outlier first: the finding leads, the total is the fallback."""
    counts = {item.severity: item.count for item in result.by_severity}
    if not result.ran:
        result.headline = "No vulnerability scan ran"
        result.headline_detail = "Enable it on the engine, or add it at launch."
        return
    checks = result.templates_run or 0
    hosts = result.scanned_hosts
    account = (
        f"{checks:,} checks ran against {hosts:,} {_plural('target', hosts)}"
        if checks
        else f"{hosts:,} {_plural('target', hosts)} tested"
    )
    if result.kev_count:
        n = result.kev_count
        result.headline = f"{n} known-exploited {'weakness is' if n == 1 else 'weaknesses are'} reachable"
        result.headline_detail = (
            "Listed as exploited in the wild. Treat as active risk."
        )
        result.headline_query = "is:kev"
        result.headline_tone = "destructive"
        return
    critical = counts.get(Severity.CRITICAL.value, 0)
    if critical:
        result.headline = f"{critical} critical {_plural('finding', critical)}"
        result.headline_detail = f"{account}."
        result.headline_query = "severity:critical"
        result.headline_tone = "destructive"
        return
    high = counts.get(Severity.HIGH.value, 0)
    if high:
        result.headline = f"{high} high-severity {_plural('finding', high)}"
        result.headline_detail = f"{account}."
        result.headline_query = "severity:high"
        result.headline_tone = "warning"
        return
    if result.new_count:
        n = result.new_count
        result.headline = f"{n} new {_plural('finding', n)} since the previous scan"
        result.headline_detail = f"{account}."
        result.headline_query = "is:new"
        result.headline_tone = "warning"
        return
    if result.total:
        result.headline = (
            f"{result.total} {_plural('finding', result.total)}, none above medium"
        )
        result.headline_detail = f"{account}."
        result.headline_query = ""
        result.headline_tone = "info"
        return
    result.headline = "No findings"
    result.headline_detail = f"{account} and matched nothing."
    result.headline_tone = "success"


def _plural(word: str, count: int) -> str:
    return word if count == 1 else f"{word}s"