WEBDIENSTE/rengine
1
0
Fork 0
mirror of https://github.com/yogeshojha/rengine.git synced 2026-10-21 07:41:28 +02:00
Code Issues Packages Projects Releases Wiki Activity
Files
ca918eb356804b9b59ca37e13e4e7da42032f2ef
rengine/shared/services/software_match.py
T
Yogesh OjhaandClaude Opus 5 2b433a4e1e perf(software): the backfill converges and match_scan merges instead of rebuilding 
http_assets.software was never declared on the model, so the components the
probe parsed were dropped at insert and the five-minute backfill became the
only writer. Its pending test was an empty list beside a tech, and a
versionless technology yields an empty list, so those rows were queued for
ever. Each tick rebuilt the scan with a delete and a reinsert, and a tick
outran its own interval, so eight ran at once, deadlocked on the same scan
and held every worker slot.

The column is three-state: NULL is not evaluated, [] is evaluated with
nothing versioned. The probe writes it at insert. The backfill reads NULL
alone under an instance lock and matches only when it wrote a component.
match_scan builds the scan's rows in a temp table, applies confidence and
rank there, and merges them by fingerprint under a per-scan advisory lock, so
a row whose columns did not move is not written and finalize, the nightly
rematch and the backfill queue rather than deadlock. Backfill beat entries
expire after one interval. software_cves carries the result tables'
autovacuum thresholds. The dashboard store loads its cells beside the
overview rather than after it.

A full rematch of 441,044 rows now writes none of them. The dashboard
overview went from 13.7 s to 0.8 s and its software count from 13.6 s to
22 ms with no heap fetches.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CjhsosXbGgSkBU5z5yPYiz
2026-09-15 17:22:35 +05:30

704 lines
21 KiB
Python
Raw Blame History

"""Match the software an asset reports against the NVD corpus. Nothing is sent to the asset."""
from __future__ import annotations
import json
import uuid
from dataclasses import dataclass, field
from datetime import date
from sqlalchemy import text
from sqlalchemy.orm import Session
from shared.definitions.evidence import Evidence
from shared.definitions.software import (
LOW_CAVEATS,
MAX_MATCHES_PER_SCAN,
MEDIUM_CAVEATS,
PRODUCTS_BY_KEY,
Caveat,
Confidence,
VersionSource,
)
from shared.definitions.threat_intel import (
SIGNALS_BY_KIND,
ExploitSignal,
exploit_score,
)
from shared.definitions.vulnerabilities import SUPPRESSED_STATES, Severity
from shared.enums.scan import ScanScope, ScanStatus
from shared.logging import get_logger
from shared.models.software import SoftwareComponentRead, SoftwareCoverage
from shared.services import locks
from shared.services.asset_query.lead_cache import bump_sync
from shared.utils.datetime import utc_now
from shared.utils.software import (
components_of,
normalize_product,
version_key,
version_kind,
)
logger = get_logger(__name__)
LIKELY_EPSS = 0.088
_BACKFILL_CHUNK = 2000
_MAX_UNMAPPED_SHOWN = 25
MAX_EXPOSURES_REPORTED = 200
_HAD_PREVIOUS = "SELECT EXISTS (SELECT 1 FROM software_cves WHERE scan_id = :scan_id)"
_CORROBORATE = """
UPDATE software_new s SET evidence = :corroborated
FROM vulnerabilities v
WHERE v.scan_id = :scan_id
AND v.severity <> :info
AND s.host IS NOT NULL AND v.host = s.host
AND jsonb_exists(v.cve_ids::jsonb, s.cve)
AND NOT EXISTS (
SELECT 1 FROM vulnerability_triage t
WHERE t.target_id = v.target_id AND t.fingerprint = v.fingerprint
AND t.state = ANY(:suppressed))
"""
_NOT_STORED = """
NOT EXISTS (SELECT 1 FROM software_cves s
WHERE s.scan_id = :scan_id AND s.fingerprint = n.fingerprint)
"""
_NEWLY_EXPOSED = f"""
SELECT n.cve, n.host, n.ip, n.port, n.name, n.version, n.severity, n.is_kev,
n.kev_ransomware, n.exploit_score, n.evidence
FROM software_new n
WHERE {_NOT_STORED}
ORDER BY n.is_kev DESC, n.exploit_score DESC, n.cvss_score DESC NULLS LAST, n.cve, n.host
LIMIT :limit
""" # noqa: S608
_NEWLY_EXPOSED_COUNT = f"SELECT count(*) FROM software_new n WHERE {_NOT_STORED}" # noqa: S608
# ---------- merge ----------
_KEY_COLUMNS = ("id", "scan_id", "target_id", "project_id", "fingerprint", "cve")
_JSON_COLUMNS = ("intel_kinds", "caveats")
_MUTABLE_COLUMNS = (
"name",
"version",
"vendor",
"product",
"cpe",
"version_source",
"severity",
"cvss_score",
"epss_score",
"epss_percentile",
"is_kev",
"kev_ransomware",
"kev_due_date",
"exploit_score",
"intel_kinds",
"confidence",
"caveats",
"evidence",
"host",
"ip",
"port",
"url",
"http_asset_id",
"port_id",
)
_ROW_COLUMNS = (*_KEY_COLUMNS, *_MUTABLE_COLUMNS, "discovered_at", "created_at")
def _compared(column: str, alias: str) -> str:
cast = "::jsonb" if column in _JSON_COLUMNS else ""
return f"{alias}.{column}{cast}"
_MERGE_DELETE = """
DELETE FROM software_cves s
WHERE s.scan_id = :scan_id
AND NOT EXISTS (SELECT 1 FROM software_new n WHERE n.fingerprint = s.fingerprint)
"""
_MERGE_UPDATE = f"""
UPDATE software_cves s
SET {", ".join(f"{c} = n.{c}" for c in _MUTABLE_COLUMNS)}
FROM software_new n
WHERE s.scan_id = :scan_id
AND s.fingerprint = n.fingerprint
AND ({", ".join(_compared(c, "s") for c in _MUTABLE_COLUMNS)})
IS DISTINCT FROM
({", ".join(_compared(c, "n") for c in _MUTABLE_COLUMNS)})
""" # noqa: S608
_MERGE_INSERT = f"""
INSERT INTO software_cves ({", ".join(_ROW_COLUMNS)})
SELECT {", ".join(f"n.{c}" for c in _ROW_COLUMNS)}
FROM software_new n
WHERE {_NOT_STORED}
""" # noqa: S608
_TEMP = """
CREATE TEMP TABLE software_components (
ref uuid,
host varchar(500),
ip varchar(45),
port integer,
url varchar(2000),
http_asset_id uuid,
port_id uuid,
name varchar(120),
version varchar(64),
version_key varchar(160),
version_kind varchar(1),
vendor varchar(200),
product varchar(200),
source varchar(16),
distro varchar(40),
coarse boolean
) ON COMMIT DROP
"""
_BUILD = """
CREATE TEMP TABLE software_new ON COMMIT DROP AS
SELECT matched.*,
CASE WHEN json_array_length(matched.caveats) >= :low_caveats THEN cast(:low AS varchar)
WHEN json_array_length(matched.caveats) >= :medium_caveats THEN cast(:medium AS varchar)
ELSE cast(:high AS varchar) END AS confidence
FROM (
SELECT DISTINCT ON (fingerprint) * FROM (
SELECT
gen_random_uuid() AS id,
cast(:scan_id AS uuid) AS scan_id,
cast(:target_id AS uuid) AS target_id,
cast(:project_id AS uuid) AS project_id,
md5(
coalesce(c.host, c.ip, '') || '|' || coalesce(c.port::text, '') || '|' ||
m.vendor || '|' || m.product || '|' || c.version || '|' || m.cve
) AS fingerprint,
m.cve,
c.name,
c.version,
m.vendor,
m.product,
'cpe:2.3:a:' || m.vendor || ':' || m.product || ':' || c.version
|| ':*:*:*:*:*:*:*' AS cpe,
cast(c.source AS varchar) AS version_source,
cast(coalesce(v.severity, :unknown) AS varchar) AS severity,
v.cvss_score,
e.score AS epss_score,
e.percentile AS epss_percentile,
(k.cve IS NOT NULL) AS is_kev,
coalesce(k.known_ransomware, false) AS kev_ransomware,
k.due_date AS kev_due_date,
0 AS exploit_score,
'[]'::json AS intel_kinds,
cast(:inferred AS varchar) AS evidence,
(
SELECT coalesce(json_agg(x), '[]'::json) FROM (
SELECT :conditional AS x WHERE m.conditional
UNION ALL
SELECT :backport WHERE c.distro IS NOT NULL
UNION ALL
SELECT :fingerprint WHERE c.source = :fingerprint_source
UNION ALL
SELECT :coarse WHERE c.coarse
) caveat
) AS caveats,
c.host,
c.ip,
c.port,
c.url,
c.http_asset_id,
c.port_id,
cast(:now AS timestamptz) AS discovered_at,
cast(:now AS timestamptz) AS created_at
FROM software_components c
JOIN nvd_cpe_matches m
ON m.product = c.product
AND (c.vendor IS NULL OR m.vendor = c.vendor)
AND m.version_kind = c.version_kind
AND (m.exact_key IS NULL OR m.exact_key = c.version_key)
AND (m.start_key IS NULL OR c.version_key > m.start_key
OR (m.start_incl AND c.version_key = m.start_key))
AND (m.end_key IS NULL OR c.version_key < m.end_key
OR (m.end_incl AND c.version_key = m.end_key))
LEFT JOIN nvd_cves v ON v.cve = m.cve
LEFT JOIN epss_scores e ON e.cve = m.cve
LEFT JOIN kev_entries k ON k.cve = m.cve
) rows
ORDER BY fingerprint
LIMIT :cap
) matched
"""
_COMPONENTS = """
SELECT id, host, ip, port, url, software
FROM http_assets
WHERE scan_id = :scan_id
AND json_array_length(software) > 0
"""
_PORT_COMPONENTS = """
SELECT id, ip, number, product, version
FROM ports
WHERE scan_id = :scan_id
AND product IS NOT NULL
AND version IS NOT NULL
AND version <> ''
"""
@dataclass
class _Component:
ref: uuid.UUID
host: str | None
ip: str | None
port: int | None
url: str | None
http_asset_id: uuid.UUID | None
port_id: uuid.UUID | None
name: str
version: str
version_key: str
version_kind: str
vendor: str | None
product: str
source: str
distro: str | None
coarse: bool
@dataclass(frozen=True)
class Exposure:
"""One software CVE row the latest match wrote that the previous match did not hold."""
cve: str
host: str
name: str
version: str
severity: str
is_kev: bool
kev_ransomware: bool
exploit_score: int
evidence: str
target_id: uuid.UUID
project_id: uuid.UUID
@dataclass
class MatchResult:
coverage: SoftwareCoverage
exposed: list[Exposure] = field(default_factory=list)
exposed_total: int = 0
@dataclass
class RematchResult:
scans: int = 0
findings: int = 0
exposed: list[Exposure] = field(default_factory=list)
exposed_total: int = 0
_SEGMENTS_FOR_PRECISION = 2
def _is_coarse(version: str) -> bool:
"""A bare major version matches every release in its series."""
return len([p for p in version.split(".") if p]) < _SEGMENTS_FOR_PRECISION
def _resolve(name: str, version: str) -> tuple[str | None, str] | None:
spec = PRODUCTS_BY_KEY.get(normalize_product(name))
if spec is None or not version:
return None
return spec.vendor, spec.product
def _gather(
session: Session, scan_id: uuid.UUID
) -> tuple[list[_Component], list[tuple[str, str]]]:
found: list[_Component] = []
unmapped: list[tuple[str, str]] = []
for row in session.execute(text(_COMPONENTS), {"scan_id": str(scan_id)}):
for entry in row.software or []:
name = (entry or {}).get("name") or ""
version = (entry or {}).get("version") or ""
key = version_key(version)
if not name or not key:
continue
pair = _resolve(name, version)
if pair is None:
unmapped.append((name, entry.get("source") or ""))
continue
vendor, product = pair
found.append(
_Component(
ref=uuid.uuid4(),
host=row.host,
ip=row.ip,
port=row.port,
url=row.url,
http_asset_id=row.id,
port_id=None,
name=name,
version=version,
version_key=key,
version_kind=version_kind(key) or "n",
vendor=vendor,
product=product,
source=entry.get("source") or VersionSource.FINGERPRINT.value,
distro=entry.get("distro"),
coarse=_is_coarse(version),
)
)
for row in session.execute(text(_PORT_COMPONENTS), {"scan_id": str(scan_id)}):
key = version_key(row.version)
if not key:
continue
pair = _resolve(row.product, row.version)
if pair is None:
unmapped.append((row.product, VersionSource.BANNER.value))
continue
vendor, product = pair
found.append(
_Component(
ref=uuid.uuid4(),
host=None,
ip=row.ip,
port=row.number,
url=None,
http_asset_id=None,
port_id=row.id,
name=row.product,
version=row.version,
version_key=key,
version_kind=version_kind(key) or "n",
vendor=vendor,
product=product,
source=VersionSource.BANNER.value,
distro=None,
coarse=_is_coarse(row.version),
)
)
return found, unmapped
def _signals(
is_kev: bool, ransomware: bool, due: date | None, epss: float | None
) -> list[str]:
kinds: list[str] = []
if is_kev:
kinds.append(ExploitSignal.KEV.value)
if ransomware:
kinds.append(ExploitSignal.RANSOMWARE.value)
if due is not None and due < utc_now().date():
kinds.append(ExploitSignal.OVERDUE.value)
if epss is not None and epss >= LIKELY_EPSS:
kinds.append(ExploitSignal.LIKELY.value)
return [k for k in kinds if k in SIGNALS_BY_KIND]
def _rank(session: Session) -> None:
"""One rank per CVE, applied to every row that carries it."""
rows = session.execute(
text(
"SELECT DISTINCT cve, is_kev, kev_ransomware, kev_due_date, epss_score "
"FROM software_new"
)
).all()
if not rows:
return
ranked = {
row.cve: _signals(
row.is_kev, row.kev_ransomware, row.kev_due_date, row.epss_score
)
for row in rows
}
payload = [
{"cve": cve, "score": exploit_score(kinds), "kinds": kinds}
for cve, kinds in ranked.items()
]
session.execute(
text(
"UPDATE software_new s SET exploit_score = v.score, "
"intel_kinds = v.kinds "
"FROM (SELECT unnest(cast(:cves AS varchar[])) AS cve, "
"unnest(cast(:scores AS int[])) AS score, "
"unnest(cast(:kinds AS json[])) AS kinds) v "
"WHERE s.cve = v.cve"
),
{
"cves": [p["cve"] for p in payload],
"scores": [p["score"] for p in payload],
"kinds": [json.dumps(p["kinds"]) for p in payload],
},
)
def match_scan(
session: Session,
*,
scan_id: uuid.UUID,
target_id: uuid.UUID,
project_id: uuid.UUID,
) -> MatchResult:
"""Rebuild this scan's inferred CVEs from the software its assets reported."""
params = {"scan_id": str(scan_id)}
session.execute(
text("SELECT pg_advisory_xact_lock(:key)"),
{"key": locks.software_match(scan_id)},
)
components, unmapped = _gather(session, scan_id)
had_previous = bool(session.execute(text(_HAD_PREVIOUS), params).scalar())
coverage = SoftwareCoverage(
components=len(components) + len(unmapped),
mapped=len(components),
unmapped=len(unmapped),
unmapped_names=_unmapped_read(unmapped),
)
if not components:
session.execute(
text("DELETE FROM software_cves WHERE scan_id = :scan_id"), params
)
session.commit()
return MatchResult(coverage=coverage)
session.execute(text(_TEMP))
session.execute(
text(
"INSERT INTO software_components (ref, host, ip, port, url, http_asset_id, "
"port_id, name, version, version_key, version_kind, vendor, product, source, distro, coarse) "
"VALUES (:ref, :host, :ip, :port, :url, :http_asset_id, :port_id, :name, "
":version, :version_key, :version_kind, :vendor, :product, :source, :distro, :coarse)"
),
[
{
"ref": str(c.ref),
"host": c.host,
"ip": c.ip,
"port": c.port,
"url": c.url,
"http_asset_id": str(c.http_asset_id) if c.http_asset_id else None,
"port_id": str(c.port_id) if c.port_id else None,
"name": c.name,
"version": c.version,
"version_key": c.version_key,
"version_kind": c.version_kind,
"vendor": c.vendor,
"product": c.product,
"source": c.source,
"distro": c.distro,
"coarse": c.coarse,
}
for c in components
],
)
session.execute(
text(_BUILD),
{
"scan_id": str(scan_id),
"target_id": str(target_id),
"project_id": str(project_id),
"now": utc_now(),
"unknown": Severity.UNKNOWN.value,
"inferred": Evidence.INFERRED.value,
"high": Confidence.HIGH.value,
"medium": Confidence.MEDIUM.value,
"low": Confidence.LOW.value,
"medium_caveats": MEDIUM_CAVEATS,
"low_caveats": LOW_CAVEATS,
"conditional": Caveat.CONDITIONAL.value,
"backport": Caveat.BACKPORT.value,
"fingerprint": Caveat.FINGERPRINT.value,
"fingerprint_source": VersionSource.FINGERPRINT.value,
"coarse": Caveat.COARSE.value,
"cap": MAX_MATCHES_PER_SCAN,
},
)
coverage.findings = int(
session.execute(text("SELECT count(*) FROM software_new")).scalar_one()
)
_rank(session)
session.execute(
text(_CORROBORATE),
{
**params,
"corroborated": Evidence.CORROBORATED.value,
"info": Severity.INFO.value,
"suppressed": list(SUPPRESSED_STATES),
},
)
exposed: list[Exposure] = []
exposed_total = 0
if had_previous:
exposed_total = int(
session.execute(text(_NEWLY_EXPOSED_COUNT), params).scalar() or 0
)
if exposed_total:
rows = session.execute(
text(_NEWLY_EXPOSED), {**params, "limit": MAX_EXPOSURES_REPORTED}
).all()
exposed = [
Exposure(
cve=r.cve,
host=r.host or r.ip or "",
name=r.name,
version=r.version,
severity=r.severity,
is_kev=bool(r.is_kev),
kev_ransomware=bool(r.kev_ransomware),
exploit_score=int(r.exploit_score or 0),
evidence=r.evidence,
target_id=target_id,
project_id=project_id,
)
for r in rows
]
session.execute(text(_MERGE_DELETE), params)
session.execute(text(_MERGE_UPDATE), params)
session.execute(text(_MERGE_INSERT), params)
session.commit()
coverage.matched = session.execute(
text(
"SELECT count(DISTINCT coalesce(http_asset_id::text, port_id::text)) "
"FROM software_cves WHERE scan_id = :scan_id"
),
params,
).scalar_one()
logger.info(
"software matched",
scan=str(scan_id),
components=len(components),
findings=coverage.findings,
newly_exposed=exposed_total,
)
return MatchResult(coverage=coverage, exposed=exposed, exposed_total=exposed_total)
def _unmapped_read(unmapped: list[tuple[str, str]]) -> list[SoftwareComponentRead]:
counts: dict[tuple[str, str], int] = {}
for name, source in unmapped:
counts[(name, source)] = counts.get((name, source), 0) + 1
ordered = sorted(counts.items(), key=lambda item: -item[1])[:_MAX_UNMAPPED_SHOWN]
return [
SoftwareComponentRead(
name=name,
version_source=source or VersionSource.FINGERPRINT.value,
mapped=False,
assets=count,
)
for (name, source), count in ordered
]
_LATEST_SCANS = """
SELECT DISTINCT ON (target_id) id, target_id, project_id
FROM scans
WHERE status = ANY(:settled)
AND scope <> :focused
ORDER BY target_id, coalesce(started_at, created_at) DESC
"""
def rematch_latest(session: Session) -> RematchResult:
"""Rebuild the covering run of every target against the current corpus."""
rows = session.execute(
text(_LATEST_SCANS),
{
"settled": [ScanStatus.COMPLETED.value, ScanStatus.CANCELLED.value],
"focused": ScanScope.FOCUSED.value,
},
).all()
out = RematchResult()
touched: list[uuid.UUID] = []
for row in rows:
try:
result = match_scan(
session,
scan_id=row.id,
target_id=row.target_id,
project_id=row.project_id,
)
except Exception:
session.rollback()
logger.warning("software rematch failed", scan=str(row.id), exc_info=True)
continue
out.scans += 1
out.findings += result.coverage.findings
out.exposed_total += result.exposed_total
out.exposed.extend(result.exposed)
touched.append(row.target_id)
if touched:
bump_sync(touched)
out.exposed.sort(key=lambda e: (not e.is_kev, -e.exploit_score, e.cve, e.host))
del out.exposed[MAX_EXPOSURES_REPORTED:]
return out
BACKFILL_SCANS_PER_TICK = 5
# NULL: not evaluated yet
_PENDING_SCANS = """
SELECT DISTINCT scan_id
FROM http_assets
WHERE software IS NULL
LIMIT :limit
"""
_ASSETS_TO_FILL = """
SELECT id, tech, webserver
FROM http_assets
WHERE scan_id = :scan_id
AND software IS NULL
"""
@dataclass
class BackfillResult:
rows: int = 0
components: int = 0
def pending_scans(
session: Session, *, limit: int = BACKFILL_SCANS_PER_TICK
) -> list[uuid.UUID]:
rows = session.execute(text(_PENDING_SCANS), {"limit": limit}).all()
return [row[0] for row in rows]
def backfill_scan(session: Session, scan_id: uuid.UUID) -> BackfillResult:
"""Read the components of every web asset in the scan that has none."""
rows = session.execute(text(_ASSETS_TO_FILL), {"scan_id": str(scan_id)}).all()
out = BackfillResult()
payload = []
for row in rows:
entries = components_of(list(row.tech or []), row.webserver)
payload.append({"id": str(row.id), "software": json.dumps(entries)})
out.rows += 1
out.components += len(entries)
for start in range(0, len(payload), _BACKFILL_CHUNK):
session.execute(
text(
"UPDATE http_assets SET software = v.software "
"FROM (SELECT unnest(cast(:ids AS uuid[])) AS id, "
"unnest(cast(:software AS json[])) AS software) v "
"WHERE http_assets.id = v.id"
),
{
"ids": [
item["id"] for item in payload[start : start + _BACKFILL_CHUNK]
],
"software": [
item["software"]
for item in payload[start : start + _BACKFILL_CHUNK]
],
},
)
session.commit()
return out
Reference in New Issue View Git Blame Copy Permalink