mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-20 07:11:28 +02:00
Yogesh Ojha
f2e28d2f72
After the match, every inferred CVE row gets fixed_in: the nearest newer release of the same product this scan saw that does not carry the CVE, and how many assets run it. Each release's CVE set is read from the NVD corpus, uncapped, so a scan at the match cap is still judged whole. A version whose leading number sits far above the product's median is a misread and never the fix. is:fixable filters on it, and the row and the sheet name the release.
838 lines
26 KiB
Python
838 lines
26 KiB
Python
"""Match the software an asset reports against the NVD corpus. Nothing is sent to the asset."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import statistics
|
|
import uuid
|
|
from collections import defaultdict
|
|
from dataclasses import dataclass, field
|
|
from datetime import date
|
|
|
|
from sqlalchemy import text
|
|
from sqlalchemy.orm import Session
|
|
|
|
from shared.definitions.evidence import Evidence
|
|
from shared.definitions.software import (
|
|
LOW_CAVEATS,
|
|
MAX_MATCHES_PER_SCAN,
|
|
MEDIUM_CAVEATS,
|
|
OUTLIER_FACTOR,
|
|
OUTLIER_SPAN,
|
|
PRODUCTS_BY_KEY,
|
|
Caveat,
|
|
Confidence,
|
|
VersionSource,
|
|
)
|
|
from shared.definitions.threat_intel import (
|
|
SIGNALS_BY_KIND,
|
|
ExploitSignal,
|
|
exploit_score,
|
|
)
|
|
from shared.definitions.vulnerabilities import SUPPRESSED_STATES, Severity
|
|
from shared.enums.scan import ScanScope, ScanStatus
|
|
from shared.logging import get_logger
|
|
from shared.models.software import SoftwareComponentRead, SoftwareCoverage
|
|
from shared.services import locks
|
|
from shared.services.asset_query.lead_cache import bump_sync
|
|
from shared.utils.datetime import utc_now
|
|
from shared.utils.software import (
|
|
components_of,
|
|
normalize_product,
|
|
version_key,
|
|
version_kind,
|
|
)
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
LIKELY_EPSS = 0.088
|
|
_BACKFILL_CHUNK = 2000
|
|
_MAX_UNMAPPED_SHOWN = 25
|
|
MAX_EXPOSURES_REPORTED = 200
|
|
|
|
_HAD_PREVIOUS = "SELECT EXISTS (SELECT 1 FROM software_cves WHERE scan_id = :scan_id)"
|
|
|
|
_CORROBORATE = """
|
|
UPDATE software_new s SET evidence = :corroborated
|
|
FROM vulnerabilities v
|
|
WHERE v.scan_id = :scan_id
|
|
AND v.severity <> :info
|
|
AND s.host IS NOT NULL AND v.host = s.host
|
|
AND jsonb_exists(v.cve_ids::jsonb, s.cve)
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM vulnerability_triage t
|
|
WHERE t.target_id = v.target_id AND t.fingerprint = v.fingerprint
|
|
AND t.state = ANY(:suppressed))
|
|
"""
|
|
|
|
_NOT_STORED = """
|
|
NOT EXISTS (SELECT 1 FROM software_cves s
|
|
WHERE s.scan_id = :scan_id AND s.fingerprint = n.fingerprint)
|
|
"""
|
|
|
|
_NEWLY_EXPOSED = f"""
|
|
SELECT n.cve, n.host, n.ip, n.port, n.name, n.version, n.severity, n.is_kev,
|
|
n.kev_ransomware, n.exploit_score, n.evidence
|
|
FROM software_new n
|
|
WHERE {_NOT_STORED}
|
|
ORDER BY n.is_kev DESC, n.exploit_score DESC, n.cvss_score DESC NULLS LAST, n.cve, n.host
|
|
LIMIT :limit
|
|
""" # noqa: S608
|
|
|
|
_NEWLY_EXPOSED_COUNT = f"SELECT count(*) FROM software_new n WHERE {_NOT_STORED}" # noqa: S608
|
|
|
|
# ---------- merge ----------
|
|
_KEY_COLUMNS = ("id", "scan_id", "target_id", "project_id", "fingerprint", "cve")
|
|
_JSON_COLUMNS = ("intel_kinds", "caveats")
|
|
_MUTABLE_COLUMNS = (
|
|
"name",
|
|
"version",
|
|
"vendor",
|
|
"product",
|
|
"cpe",
|
|
"version_source",
|
|
"severity",
|
|
"cvss_score",
|
|
"epss_score",
|
|
"epss_percentile",
|
|
"is_kev",
|
|
"kev_ransomware",
|
|
"kev_due_date",
|
|
"exploit_score",
|
|
"intel_kinds",
|
|
"confidence",
|
|
"caveats",
|
|
"evidence",
|
|
"host",
|
|
"ip",
|
|
"port",
|
|
"url",
|
|
"http_asset_id",
|
|
"port_id",
|
|
)
|
|
_ROW_COLUMNS = (*_KEY_COLUMNS, *_MUTABLE_COLUMNS, "discovered_at", "created_at")
|
|
|
|
|
|
def _compared(column: str, alias: str) -> str:
|
|
cast = "::jsonb" if column in _JSON_COLUMNS else ""
|
|
return f"{alias}.{column}{cast}"
|
|
|
|
|
|
_MERGE_DELETE = """
|
|
DELETE FROM software_cves s
|
|
WHERE s.scan_id = :scan_id
|
|
AND NOT EXISTS (SELECT 1 FROM software_new n WHERE n.fingerprint = s.fingerprint)
|
|
"""
|
|
|
|
_MERGE_UPDATE = f"""
|
|
UPDATE software_cves s
|
|
SET {", ".join(f"{c} = n.{c}" for c in _MUTABLE_COLUMNS)}
|
|
FROM software_new n
|
|
WHERE s.scan_id = :scan_id
|
|
AND s.fingerprint = n.fingerprint
|
|
AND ({", ".join(_compared(c, "s") for c in _MUTABLE_COLUMNS)})
|
|
IS DISTINCT FROM
|
|
({", ".join(_compared(c, "n") for c in _MUTABLE_COLUMNS)})
|
|
""" # noqa: S608
|
|
|
|
_MERGE_INSERT = f"""
|
|
INSERT INTO software_cves ({", ".join(_ROW_COLUMNS)})
|
|
SELECT {", ".join(f"n.{c}" for c in _ROW_COLUMNS)}
|
|
FROM software_new n
|
|
WHERE {_NOT_STORED}
|
|
""" # noqa: S608
|
|
|
|
_TEMP = """
|
|
CREATE TEMP TABLE software_components (
|
|
ref uuid,
|
|
host varchar(500),
|
|
ip varchar(45),
|
|
port integer,
|
|
url varchar(2000),
|
|
http_asset_id uuid,
|
|
port_id uuid,
|
|
name varchar(120),
|
|
version varchar(64),
|
|
version_key varchar(160),
|
|
version_kind varchar(1),
|
|
vendor varchar(200),
|
|
product varchar(200),
|
|
source varchar(16),
|
|
distro varchar(40),
|
|
coarse boolean
|
|
) ON COMMIT DROP
|
|
"""
|
|
|
|
_BUILD = """
|
|
CREATE TEMP TABLE software_new ON COMMIT DROP AS
|
|
SELECT matched.*,
|
|
CASE WHEN json_array_length(matched.caveats) >= :low_caveats THEN cast(:low AS varchar)
|
|
WHEN json_array_length(matched.caveats) >= :medium_caveats THEN cast(:medium AS varchar)
|
|
ELSE cast(:high AS varchar) END AS confidence
|
|
FROM (
|
|
SELECT DISTINCT ON (fingerprint) * FROM (
|
|
SELECT
|
|
gen_random_uuid() AS id,
|
|
cast(:scan_id AS uuid) AS scan_id,
|
|
cast(:target_id AS uuid) AS target_id,
|
|
cast(:project_id AS uuid) AS project_id,
|
|
md5(
|
|
coalesce(c.host, c.ip, '') || '|' || coalesce(c.port::text, '') || '|' ||
|
|
m.vendor || '|' || m.product || '|' || c.version || '|' || m.cve
|
|
) AS fingerprint,
|
|
m.cve,
|
|
c.name,
|
|
c.version,
|
|
m.vendor,
|
|
m.product,
|
|
'cpe:2.3:a:' || m.vendor || ':' || m.product || ':' || c.version
|
|
|| ':*:*:*:*:*:*:*' AS cpe,
|
|
cast(c.source AS varchar) AS version_source,
|
|
cast(coalesce(v.severity, :unknown) AS varchar) AS severity,
|
|
v.cvss_score,
|
|
e.score AS epss_score,
|
|
e.percentile AS epss_percentile,
|
|
(k.cve IS NOT NULL) AS is_kev,
|
|
coalesce(k.known_ransomware, false) AS kev_ransomware,
|
|
k.due_date AS kev_due_date,
|
|
0 AS exploit_score,
|
|
'[]'::json AS intel_kinds,
|
|
cast(:inferred AS varchar) AS evidence,
|
|
(
|
|
SELECT coalesce(json_agg(x), '[]'::json) FROM (
|
|
SELECT :conditional AS x WHERE m.conditional
|
|
UNION ALL
|
|
SELECT :backport WHERE c.distro IS NOT NULL
|
|
UNION ALL
|
|
SELECT :fingerprint WHERE c.source = :fingerprint_source
|
|
UNION ALL
|
|
SELECT :coarse WHERE c.coarse
|
|
) caveat
|
|
) AS caveats,
|
|
c.host,
|
|
c.ip,
|
|
c.port,
|
|
c.url,
|
|
c.http_asset_id,
|
|
c.port_id,
|
|
cast(:now AS timestamptz) AS discovered_at,
|
|
cast(:now AS timestamptz) AS created_at
|
|
FROM software_components c
|
|
JOIN nvd_cpe_matches m
|
|
ON m.product = c.product
|
|
AND (c.vendor IS NULL OR m.vendor = c.vendor)
|
|
AND m.version_kind = c.version_kind
|
|
AND (m.exact_key IS NULL OR m.exact_key = c.version_key)
|
|
AND (m.start_key IS NULL OR c.version_key > m.start_key
|
|
OR (m.start_incl AND c.version_key = m.start_key))
|
|
AND (m.end_key IS NULL OR c.version_key < m.end_key
|
|
OR (m.end_incl AND c.version_key = m.end_key))
|
|
LEFT JOIN nvd_cves v ON v.cve = m.cve
|
|
LEFT JOIN epss_scores e ON e.cve = m.cve
|
|
LEFT JOIN kev_entries k ON k.cve = m.cve
|
|
) rows
|
|
ORDER BY fingerprint
|
|
LIMIT :cap
|
|
) matched
|
|
"""
|
|
|
|
_CARRIED = """
|
|
SELECT DISTINCT c.product, c.version, m.cve
|
|
FROM (SELECT DISTINCT product, vendor, version, version_key, version_kind
|
|
FROM software_components WHERE NOT coarse) c
|
|
JOIN nvd_cpe_matches m
|
|
ON m.product = c.product
|
|
AND (c.vendor IS NULL OR m.vendor = c.vendor)
|
|
AND m.version_kind = c.version_kind
|
|
AND (m.exact_key IS NULL OR m.exact_key = c.version_key)
|
|
AND (m.start_key IS NULL OR c.version_key > m.start_key
|
|
OR (m.start_incl AND c.version_key = m.start_key))
|
|
AND (m.end_key IS NULL OR c.version_key < m.end_key
|
|
OR (m.end_incl AND c.version_key = m.end_key))
|
|
"""
|
|
|
|
_FIXED_TEMP = """
|
|
CREATE TEMP TABLE software_fixed (
|
|
product varchar(200),
|
|
version varchar(64),
|
|
cve varchar(30),
|
|
fixed_in varchar(64),
|
|
fixed_in_assets integer
|
|
) ON COMMIT DROP
|
|
"""
|
|
|
|
_FIXED_SET = """
|
|
UPDATE software_cves s
|
|
SET fixed_in = f.fixed_in, fixed_in_assets = f.fixed_in_assets
|
|
FROM software_fixed f
|
|
WHERE s.scan_id = :scan_id
|
|
AND s.product = f.product AND s.version = f.version AND s.cve = f.cve
|
|
AND (s.fixed_in IS DISTINCT FROM f.fixed_in
|
|
OR s.fixed_in_assets IS DISTINCT FROM f.fixed_in_assets)
|
|
"""
|
|
|
|
_FIXED_CLEAR = """
|
|
UPDATE software_cves s
|
|
SET fixed_in = NULL, fixed_in_assets = NULL
|
|
WHERE s.scan_id = :scan_id AND s.fixed_in IS NOT NULL
|
|
AND NOT EXISTS (SELECT 1 FROM software_fixed f
|
|
WHERE f.product = s.product AND f.version = s.version
|
|
AND f.cve = s.cve)
|
|
"""
|
|
|
|
_COMPONENTS = """
|
|
SELECT id, host, ip, port, url, software
|
|
FROM http_assets
|
|
WHERE scan_id = :scan_id
|
|
AND json_array_length(software) > 0
|
|
"""
|
|
|
|
_PORT_COMPONENTS = """
|
|
SELECT id, ip, number, product, version
|
|
FROM ports
|
|
WHERE scan_id = :scan_id
|
|
AND product IS NOT NULL
|
|
AND version IS NOT NULL
|
|
AND version <> ''
|
|
"""
|
|
|
|
|
|
@dataclass
|
|
class _Component:
|
|
ref: uuid.UUID
|
|
host: str | None
|
|
ip: str | None
|
|
port: int | None
|
|
url: str | None
|
|
http_asset_id: uuid.UUID | None
|
|
port_id: uuid.UUID | None
|
|
name: str
|
|
version: str
|
|
version_key: str
|
|
version_kind: str
|
|
vendor: str | None
|
|
product: str
|
|
source: str
|
|
distro: str | None
|
|
coarse: bool
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Exposure:
|
|
"""One software CVE row the latest match wrote that the previous match did not hold."""
|
|
|
|
cve: str
|
|
host: str
|
|
name: str
|
|
version: str
|
|
severity: str
|
|
is_kev: bool
|
|
kev_ransomware: bool
|
|
exploit_score: int
|
|
evidence: str
|
|
target_id: uuid.UUID
|
|
project_id: uuid.UUID
|
|
|
|
|
|
@dataclass
|
|
class MatchResult:
|
|
coverage: SoftwareCoverage
|
|
exposed: list[Exposure] = field(default_factory=list)
|
|
exposed_total: int = 0
|
|
|
|
|
|
@dataclass
|
|
class RematchResult:
|
|
scans: int = 0
|
|
findings: int = 0
|
|
exposed: list[Exposure] = field(default_factory=list)
|
|
exposed_total: int = 0
|
|
|
|
|
|
_SEGMENTS_FOR_PRECISION = 2
|
|
|
|
|
|
def _is_coarse(version: str) -> bool:
|
|
"""A bare major version matches every release in its series."""
|
|
return len([p for p in version.split(".") if p]) < _SEGMENTS_FOR_PRECISION
|
|
|
|
|
|
def _resolve(name: str, version: str) -> tuple[str | None, str] | None:
|
|
spec = PRODUCTS_BY_KEY.get(normalize_product(name))
|
|
if spec is None or not version:
|
|
return None
|
|
return spec.vendor, spec.product
|
|
|
|
|
|
def _gather(
|
|
session: Session, scan_id: uuid.UUID
|
|
) -> tuple[list[_Component], list[tuple[str, str]]]:
|
|
found: list[_Component] = []
|
|
unmapped: list[tuple[str, str]] = []
|
|
|
|
for row in session.execute(text(_COMPONENTS), {"scan_id": str(scan_id)}):
|
|
for entry in row.software or []:
|
|
name = (entry or {}).get("name") or ""
|
|
version = (entry or {}).get("version") or ""
|
|
key = version_key(version)
|
|
if not name or not key:
|
|
continue
|
|
pair = _resolve(name, version)
|
|
if pair is None:
|
|
unmapped.append((name, entry.get("source") or ""))
|
|
continue
|
|
vendor, product = pair
|
|
found.append(
|
|
_Component(
|
|
ref=uuid.uuid4(),
|
|
host=row.host,
|
|
ip=row.ip,
|
|
port=row.port,
|
|
url=row.url,
|
|
http_asset_id=row.id,
|
|
port_id=None,
|
|
name=name,
|
|
version=version,
|
|
version_key=key,
|
|
version_kind=version_kind(key) or "n",
|
|
vendor=vendor,
|
|
product=product,
|
|
source=entry.get("source") or VersionSource.FINGERPRINT.value,
|
|
distro=entry.get("distro"),
|
|
coarse=_is_coarse(version),
|
|
)
|
|
)
|
|
|
|
for row in session.execute(text(_PORT_COMPONENTS), {"scan_id": str(scan_id)}):
|
|
key = version_key(row.version)
|
|
if not key:
|
|
continue
|
|
pair = _resolve(row.product, row.version)
|
|
if pair is None:
|
|
unmapped.append((row.product, VersionSource.BANNER.value))
|
|
continue
|
|
vendor, product = pair
|
|
found.append(
|
|
_Component(
|
|
ref=uuid.uuid4(),
|
|
host=None,
|
|
ip=row.ip,
|
|
port=row.number,
|
|
url=None,
|
|
http_asset_id=None,
|
|
port_id=row.id,
|
|
name=row.product,
|
|
version=row.version,
|
|
version_key=key,
|
|
version_kind=version_kind(key) or "n",
|
|
vendor=vendor,
|
|
product=product,
|
|
source=VersionSource.BANNER.value,
|
|
distro=None,
|
|
coarse=_is_coarse(row.version),
|
|
)
|
|
)
|
|
return found, unmapped
|
|
|
|
|
|
_LEADING = re.compile(r"\d+")
|
|
|
|
|
|
def _leading(version: str) -> int | None:
|
|
found = _LEADING.match(version)
|
|
return int(found.group()) if found else None
|
|
|
|
|
|
def _plausible(releases: dict[str, str]) -> list[str]:
|
|
"""Release keys in order, without a leading number far above the product's median."""
|
|
leads = [n for n in (_leading(v) for v in releases.values()) if n is not None]
|
|
median = statistics.median(leads) if leads else 0
|
|
kept = []
|
|
for key, version in releases.items():
|
|
lead = _leading(version)
|
|
if (
|
|
lead is not None
|
|
and median
|
|
and lead > median * OUTLIER_FACTOR
|
|
and lead - median > OUTLIER_SPAN
|
|
):
|
|
continue
|
|
kept.append(key)
|
|
return sorted(kept)
|
|
|
|
|
|
def fixed_releases(
|
|
components: list[_Component], carried: dict[tuple[str, str], set[str]]
|
|
) -> dict[tuple[str, str, str], tuple[str, int]]:
|
|
"""The nearest newer release in the scan that does not carry each CVE, and its asset count."""
|
|
versions: dict[str, dict[str, str]] = defaultdict(dict)
|
|
assets: dict[tuple[str, str], set] = defaultdict(set)
|
|
for c in components:
|
|
if c.coarse or c.version_kind != "n":
|
|
continue
|
|
versions[c.product].setdefault(c.version_key, c.version)
|
|
assets[(c.product, c.version_key)].add(c.http_asset_id or c.port_id or c.ref)
|
|
out: dict[tuple[str, str, str], tuple[str, int]] = {}
|
|
for product, releases in versions.items():
|
|
keys = _plausible(releases)
|
|
for index, key in enumerate(keys):
|
|
version = releases[key]
|
|
for cve in carried.get((product, version), ()):
|
|
for newer in keys[index + 1 :]:
|
|
candidate = releases[newer]
|
|
if cve not in carried.get((product, candidate), ()):
|
|
out[(product, version, cve)] = (
|
|
candidate,
|
|
len(assets[(product, newer)]),
|
|
)
|
|
break
|
|
return out
|
|
|
|
|
|
def _mark_fixed(
|
|
session: Session, scan_id: uuid.UUID, components: list[_Component]
|
|
) -> None:
|
|
carried: dict[tuple[str, str], set[str]] = defaultdict(set)
|
|
for product, version, cve in session.execute(text(_CARRIED)):
|
|
carried[(product, version)].add(cve)
|
|
fixed = fixed_releases(components, carried)
|
|
session.execute(text(_FIXED_TEMP))
|
|
if fixed:
|
|
session.execute(
|
|
text(
|
|
"INSERT INTO software_fixed (product, version, cve, fixed_in, "
|
|
"fixed_in_assets) VALUES (:product, :version, :cve, :fixed_in, :assets)"
|
|
),
|
|
[
|
|
{
|
|
"product": product,
|
|
"version": version,
|
|
"cve": cve,
|
|
"fixed_in": fixed_in,
|
|
"assets": count,
|
|
}
|
|
for (product, version, cve), (fixed_in, count) in fixed.items()
|
|
],
|
|
)
|
|
params = {"scan_id": str(scan_id)}
|
|
session.execute(text(_FIXED_SET), params)
|
|
session.execute(text(_FIXED_CLEAR), params)
|
|
|
|
|
|
def _signals(
|
|
is_kev: bool, ransomware: bool, due: date | None, epss: float | None
|
|
) -> list[str]:
|
|
kinds: list[str] = []
|
|
if is_kev:
|
|
kinds.append(ExploitSignal.KEV.value)
|
|
if ransomware:
|
|
kinds.append(ExploitSignal.RANSOMWARE.value)
|
|
if due is not None and due < utc_now().date():
|
|
kinds.append(ExploitSignal.OVERDUE.value)
|
|
if epss is not None and epss >= LIKELY_EPSS:
|
|
kinds.append(ExploitSignal.LIKELY.value)
|
|
return [k for k in kinds if k in SIGNALS_BY_KIND]
|
|
|
|
|
|
def _rank(session: Session) -> None:
|
|
"""One rank per CVE, applied to every row that carries it."""
|
|
rows = session.execute(
|
|
text(
|
|
"SELECT DISTINCT cve, is_kev, kev_ransomware, kev_due_date, epss_score "
|
|
"FROM software_new"
|
|
)
|
|
).all()
|
|
if not rows:
|
|
return
|
|
ranked = {
|
|
row.cve: _signals(
|
|
row.is_kev, row.kev_ransomware, row.kev_due_date, row.epss_score
|
|
)
|
|
for row in rows
|
|
}
|
|
payload = [
|
|
{"cve": cve, "score": exploit_score(kinds), "kinds": kinds}
|
|
for cve, kinds in ranked.items()
|
|
]
|
|
session.execute(
|
|
text(
|
|
"UPDATE software_new s SET exploit_score = v.score, "
|
|
"intel_kinds = v.kinds "
|
|
"FROM (SELECT unnest(cast(:cves AS varchar[])) AS cve, "
|
|
"unnest(cast(:scores AS int[])) AS score, "
|
|
"unnest(cast(:kinds AS json[])) AS kinds) v "
|
|
"WHERE s.cve = v.cve"
|
|
),
|
|
{
|
|
"cves": [p["cve"] for p in payload],
|
|
"scores": [p["score"] for p in payload],
|
|
"kinds": [json.dumps(p["kinds"]) for p in payload],
|
|
},
|
|
)
|
|
|
|
|
|
def match_scan(
|
|
session: Session,
|
|
*,
|
|
scan_id: uuid.UUID,
|
|
target_id: uuid.UUID,
|
|
project_id: uuid.UUID,
|
|
) -> MatchResult:
|
|
"""Rebuild this scan's inferred CVEs from the software its assets reported."""
|
|
params = {"scan_id": str(scan_id)}
|
|
session.execute(
|
|
text("SELECT pg_advisory_xact_lock(:key)"),
|
|
{"key": locks.software_match(scan_id)},
|
|
)
|
|
components, unmapped = _gather(session, scan_id)
|
|
had_previous = bool(session.execute(text(_HAD_PREVIOUS), params).scalar())
|
|
coverage = SoftwareCoverage(
|
|
components=len(components) + len(unmapped),
|
|
mapped=len(components),
|
|
unmapped=len(unmapped),
|
|
unmapped_names=_unmapped_read(unmapped),
|
|
)
|
|
if not components:
|
|
session.execute(
|
|
text("DELETE FROM software_cves WHERE scan_id = :scan_id"), params
|
|
)
|
|
session.commit()
|
|
return MatchResult(coverage=coverage)
|
|
|
|
session.execute(text(_TEMP))
|
|
session.execute(
|
|
text(
|
|
"INSERT INTO software_components (ref, host, ip, port, url, http_asset_id, "
|
|
"port_id, name, version, version_key, version_kind, vendor, product, source, distro, coarse) "
|
|
"VALUES (:ref, :host, :ip, :port, :url, :http_asset_id, :port_id, :name, "
|
|
":version, :version_key, :version_kind, :vendor, :product, :source, :distro, :coarse)"
|
|
),
|
|
[
|
|
{
|
|
"ref": str(c.ref),
|
|
"host": c.host,
|
|
"ip": c.ip,
|
|
"port": c.port,
|
|
"url": c.url,
|
|
"http_asset_id": str(c.http_asset_id) if c.http_asset_id else None,
|
|
"port_id": str(c.port_id) if c.port_id else None,
|
|
"name": c.name,
|
|
"version": c.version,
|
|
"version_key": c.version_key,
|
|
"version_kind": c.version_kind,
|
|
"vendor": c.vendor,
|
|
"product": c.product,
|
|
"source": c.source,
|
|
"distro": c.distro,
|
|
"coarse": c.coarse,
|
|
}
|
|
for c in components
|
|
],
|
|
)
|
|
session.execute(
|
|
text(_BUILD),
|
|
{
|
|
"scan_id": str(scan_id),
|
|
"target_id": str(target_id),
|
|
"project_id": str(project_id),
|
|
"now": utc_now(),
|
|
"unknown": Severity.UNKNOWN.value,
|
|
"inferred": Evidence.INFERRED.value,
|
|
"high": Confidence.HIGH.value,
|
|
"medium": Confidence.MEDIUM.value,
|
|
"low": Confidence.LOW.value,
|
|
"medium_caveats": MEDIUM_CAVEATS,
|
|
"low_caveats": LOW_CAVEATS,
|
|
"conditional": Caveat.CONDITIONAL.value,
|
|
"backport": Caveat.BACKPORT.value,
|
|
"fingerprint": Caveat.FINGERPRINT.value,
|
|
"fingerprint_source": VersionSource.FINGERPRINT.value,
|
|
"coarse": Caveat.COARSE.value,
|
|
"cap": MAX_MATCHES_PER_SCAN,
|
|
},
|
|
)
|
|
coverage.findings = int(
|
|
session.execute(text("SELECT count(*) FROM software_new")).scalar_one()
|
|
)
|
|
_rank(session)
|
|
session.execute(
|
|
text(_CORROBORATE),
|
|
{
|
|
**params,
|
|
"corroborated": Evidence.CORROBORATED.value,
|
|
"info": Severity.INFO.value,
|
|
"suppressed": list(SUPPRESSED_STATES),
|
|
},
|
|
)
|
|
exposed: list[Exposure] = []
|
|
exposed_total = 0
|
|
if had_previous:
|
|
exposed_total = int(
|
|
session.execute(text(_NEWLY_EXPOSED_COUNT), params).scalar() or 0
|
|
)
|
|
if exposed_total:
|
|
rows = session.execute(
|
|
text(_NEWLY_EXPOSED), {**params, "limit": MAX_EXPOSURES_REPORTED}
|
|
).all()
|
|
exposed = [
|
|
Exposure(
|
|
cve=r.cve,
|
|
host=r.host or r.ip or "",
|
|
name=r.name,
|
|
version=r.version,
|
|
severity=r.severity,
|
|
is_kev=bool(r.is_kev),
|
|
kev_ransomware=bool(r.kev_ransomware),
|
|
exploit_score=int(r.exploit_score or 0),
|
|
evidence=r.evidence,
|
|
target_id=target_id,
|
|
project_id=project_id,
|
|
)
|
|
for r in rows
|
|
]
|
|
session.execute(text(_MERGE_DELETE), params)
|
|
session.execute(text(_MERGE_UPDATE), params)
|
|
session.execute(text(_MERGE_INSERT), params)
|
|
_mark_fixed(session, scan_id, components)
|
|
session.commit()
|
|
|
|
coverage.matched = session.execute(
|
|
text(
|
|
"SELECT count(DISTINCT coalesce(http_asset_id::text, port_id::text)) "
|
|
"FROM software_cves WHERE scan_id = :scan_id"
|
|
),
|
|
params,
|
|
).scalar_one()
|
|
logger.info(
|
|
"software matched",
|
|
scan=str(scan_id),
|
|
components=len(components),
|
|
findings=coverage.findings,
|
|
newly_exposed=exposed_total,
|
|
)
|
|
return MatchResult(coverage=coverage, exposed=exposed, exposed_total=exposed_total)
|
|
|
|
|
|
def _unmapped_read(unmapped: list[tuple[str, str]]) -> list[SoftwareComponentRead]:
|
|
counts: dict[tuple[str, str], int] = {}
|
|
for name, source in unmapped:
|
|
counts[(name, source)] = counts.get((name, source), 0) + 1
|
|
ordered = sorted(counts.items(), key=lambda item: -item[1])[:_MAX_UNMAPPED_SHOWN]
|
|
return [
|
|
SoftwareComponentRead(
|
|
name=name,
|
|
version_source=source or VersionSource.FINGERPRINT.value,
|
|
mapped=False,
|
|
assets=count,
|
|
)
|
|
for (name, source), count in ordered
|
|
]
|
|
|
|
|
|
_LATEST_SCANS = """
|
|
SELECT DISTINCT ON (target_id) id, target_id, project_id
|
|
FROM scans
|
|
WHERE status = ANY(:settled)
|
|
AND scope <> :focused
|
|
ORDER BY target_id, coalesce(started_at, created_at) DESC
|
|
"""
|
|
|
|
|
|
def rematch_latest(session: Session) -> RematchResult:
|
|
"""Rebuild the covering run of every target against the current corpus."""
|
|
rows = session.execute(
|
|
text(_LATEST_SCANS),
|
|
{
|
|
"settled": [ScanStatus.COMPLETED.value, ScanStatus.CANCELLED.value],
|
|
"focused": ScanScope.FOCUSED.value,
|
|
},
|
|
).all()
|
|
out = RematchResult()
|
|
touched: list[uuid.UUID] = []
|
|
for row in rows:
|
|
try:
|
|
result = match_scan(
|
|
session,
|
|
scan_id=row.id,
|
|
target_id=row.target_id,
|
|
project_id=row.project_id,
|
|
)
|
|
except Exception:
|
|
session.rollback()
|
|
logger.warning("software rematch failed", scan=str(row.id), exc_info=True)
|
|
continue
|
|
out.scans += 1
|
|
out.findings += result.coverage.findings
|
|
out.exposed_total += result.exposed_total
|
|
out.exposed.extend(result.exposed)
|
|
touched.append(row.target_id)
|
|
if touched:
|
|
bump_sync(touched)
|
|
out.exposed.sort(key=lambda e: (not e.is_kev, -e.exploit_score, e.cve, e.host))
|
|
del out.exposed[MAX_EXPOSURES_REPORTED:]
|
|
return out
|
|
|
|
|
|
BACKFILL_SCANS_PER_TICK = 5
|
|
|
|
# NULL: not evaluated yet
|
|
_PENDING_SCANS = """
|
|
SELECT DISTINCT scan_id
|
|
FROM http_assets
|
|
WHERE software IS NULL
|
|
LIMIT :limit
|
|
"""
|
|
|
|
_ASSETS_TO_FILL = """
|
|
SELECT id, tech, webserver
|
|
FROM http_assets
|
|
WHERE scan_id = :scan_id
|
|
AND software IS NULL
|
|
"""
|
|
|
|
|
|
@dataclass
|
|
class BackfillResult:
|
|
rows: int = 0
|
|
components: int = 0
|
|
|
|
|
|
def pending_scans(
|
|
session: Session, *, limit: int = BACKFILL_SCANS_PER_TICK
|
|
) -> list[uuid.UUID]:
|
|
rows = session.execute(text(_PENDING_SCANS), {"limit": limit}).all()
|
|
return [row[0] for row in rows]
|
|
|
|
|
|
def backfill_scan(session: Session, scan_id: uuid.UUID) -> BackfillResult:
|
|
"""Read the components of every web asset in the scan that has none."""
|
|
rows = session.execute(text(_ASSETS_TO_FILL), {"scan_id": str(scan_id)}).all()
|
|
out = BackfillResult()
|
|
payload = []
|
|
for row in rows:
|
|
entries = components_of(list(row.tech or []), row.webserver)
|
|
payload.append({"id": str(row.id), "software": json.dumps(entries)})
|
|
out.rows += 1
|
|
out.components += len(entries)
|
|
for start in range(0, len(payload), _BACKFILL_CHUNK):
|
|
session.execute(
|
|
text(
|
|
"UPDATE http_assets SET software = v.software "
|
|
"FROM (SELECT unnest(cast(:ids AS uuid[])) AS id, "
|
|
"unnest(cast(:software AS json[])) AS software) v "
|
|
"WHERE http_assets.id = v.id"
|
|
),
|
|
{
|
|
"ids": [
|
|
item["id"] for item in payload[start : start + _BACKFILL_CHUNK]
|
|
],
|
|
"software": [
|
|
item["software"]
|
|
for item in payload[start : start + _BACKFILL_CHUNK]
|
|
],
|
|
},
|
|
)
|
|
session.commit()
|
|
return out
|