mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-21 07:41:28 +02:00
Yogesh OjhaandClaude Opus 5
2b433a4e1e
http_assets.software was never declared on the model, so the components the probe parsed were dropped at insert and the five-minute backfill became the only writer. Its pending test was an empty list beside a tech, and a versionless technology yields an empty list, so those rows were queued for ever. Each tick rebuilt the scan with a delete and a reinsert, and a tick outran its own interval, so eight ran at once, deadlocked on the same scan and held every worker slot. The column is three-state: NULL is not evaluated, [] is evaluated with nothing versioned. The probe writes it at insert. The backfill reads NULL alone under an instance lock and matches only when it wrote a component. match_scan builds the scan's rows in a temp table, applies confidence and rank there, and merges them by fingerprint under a per-scan advisory lock, so a row whose columns did not move is not written and finalize, the nightly rematch and the backfill queue rather than deadlock. Backfill beat entries expire after one interval. software_cves carries the result tables' autovacuum thresholds. The dashboard store loads its cells beside the overview rather than after it. A full rematch of 441,044 rows now writes none of them. The dashboard overview went from 13.7 s to 0.8 s and its software count from 13.6 s to 22 ms with no heap fetches. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CjhsosXbGgSkBU5z5yPYiz
56 lines
1.6 KiB
Python
56 lines
1.6 KiB
Python
"""Advisory locks for instance-wide work that must not run twice at once."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import zlib
|
|
from collections.abc import Iterator
|
|
from contextlib import contextmanager
|
|
from typing import TYPE_CHECKING
|
|
|
|
from sqlalchemy import text
|
|
|
|
if TYPE_CHECKING:
|
|
from sqlalchemy.orm import Session
|
|
|
|
# bounty_platform() spans BOUNTY_SYNC .. BOUNTY_SYNC + 0xFFFF
|
|
BOUNTY_SYNC = 0x624F0001
|
|
BOUNTY_FEED = 0x624F0002
|
|
IP_RANGES = 0x624E0001
|
|
# secret_mining() spans SECRET_MINING .. SECRET_MINING + 0xFFFF
|
|
SECRET_MINING = 0x53450001
|
|
SOFTWARE_BACKFILL = 0x53570001
|
|
# software_match() spans SOFTWARE_MATCH .. SOFTWARE_MATCH + 0xFFFF
|
|
SOFTWARE_MATCH = 0x53580001
|
|
|
|
|
|
def bounty_platform(platform: str) -> int:
|
|
"""One lock per platform."""
|
|
return BOUNTY_SYNC + (zlib.crc32(platform.encode()) & 0xFFFF)
|
|
|
|
|
|
def secret_mining(scan_id: object) -> int:
|
|
"""One lock per scan: the stage, the backfill and a re-mine never overlap."""
|
|
return SECRET_MINING + (zlib.crc32(str(scan_id).encode()) & 0xFFFF)
|
|
|
|
|
|
def software_match(scan_id: object) -> int:
|
|
"""One lock per scan."""
|
|
return SOFTWARE_MATCH + (zlib.crc32(str(scan_id).encode()) & 0xFFFF)
|
|
|
|
|
|
@contextmanager
|
|
def sync_lock(session: Session, key: int) -> Iterator[bool]:
|
|
"""Session-level advisory lock. Yields False when another session holds it."""
|
|
held = bool(
|
|
session.execute(
|
|
text("SELECT pg_try_advisory_lock(:key)"), {"key": key}
|
|
).scalar_one()
|
|
)
|
|
try:
|
|
yield held
|
|
finally:
|
|
if held:
|
|
session.rollback()
|
|
session.execute(text("SELECT pg_advisory_unlock(:key)"), {"key": key})
|
|
session.commit()
|