mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-20 23:31:31 +02:00
Yogesh OjhaandClaude Opus 5
2b433a4e1e
http_assets.software was never declared on the model, so the components the probe parsed were dropped at insert and the five-minute backfill became the only writer. Its pending test was an empty list beside a tech, and a versionless technology yields an empty list, so those rows were queued for ever. Each tick rebuilt the scan with a delete and a reinsert, and a tick outran its own interval, so eight ran at once, deadlocked on the same scan and held every worker slot. The column is three-state: NULL is not evaluated, [] is evaluated with nothing versioned. The probe writes it at insert. The backfill reads NULL alone under an instance lock and matches only when it wrote a component. match_scan builds the scan's rows in a temp table, applies confidence and rank there, and merges them by fingerprint under a per-scan advisory lock, so a row whose columns did not move is not written and finalize, the nightly rematch and the backfill queue rather than deadlock. Backfill beat entries expire after one interval. software_cves carries the result tables' autovacuum thresholds. The dashboard store loads its cells beside the overview rather than after it. A full rematch of 441,044 rows now writes none of them. The dashboard overview went from 13.7 s to 0.8 s and its software count from 13.6 s to 22 ms with no heap fetches. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CjhsosXbGgSkBU5z5yPYiz
51 lines
1.7 KiB
Python
51 lines
1.7 KiB
Python
"""Evaluate the software of web assets whose components were never read."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from celery import shared_task
|
|
from sqlalchemy import select
|
|
|
|
from app.database import get_sync_session
|
|
from shared.logging import get_logger
|
|
from shared.models.scan import Scan
|
|
from shared.services import locks, software_match
|
|
from shared.services.asset_query.lead_cache import bump_sync
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
|
|
@shared_task(name="app.tasks.software.backfill", max_retries=0)
|
|
def backfill(limit: int = software_match.BACKFILL_SCANS_PER_TICK) -> dict:
|
|
scans = 0
|
|
rows = 0
|
|
matched = 0
|
|
with (
|
|
get_sync_session() as session,
|
|
locks.sync_lock(session, locks.SOFTWARE_BACKFILL) as held,
|
|
):
|
|
if not held:
|
|
return {"scans": 0, "rows": 0, "more": False, "skipped": "running"}
|
|
pending = software_match.pending_scans(session, limit=limit)
|
|
for scan_id in pending:
|
|
filled = software_match.backfill_scan(session, scan_id)
|
|
rows += filled.rows
|
|
scans += 1
|
|
if not filled.components:
|
|
continue
|
|
scan = session.execute(
|
|
select(Scan.target_id, Scan.project_id).where(Scan.id == scan_id)
|
|
).one_or_none()
|
|
if scan is None:
|
|
continue
|
|
software_match.match_scan(
|
|
session,
|
|
scan_id=scan_id,
|
|
target_id=scan.target_id,
|
|
project_id=scan.project_id,
|
|
)
|
|
bump_sync([scan.target_id])
|
|
matched += 1
|
|
if scans:
|
|
logger.info("software backfill", scans=scans, rows=rows, matched=matched)
|
|
return {"scans": scans, "rows": rows, "more": len(pending) >= limit}
|