mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-19 23:01:29 +02:00
Yogesh Ojha
6949492e11
- Store each run's row, first-seen and retired counts (scan_deltas, scan_retired), valid while the revision stamp matches scan_revisions; statement triggers on the seven result tables bump it. Finalize warms, a beat backfill fills the rest, a live run answers from its last count and queues a refresh. /scans 1.5 s -> 43 ms, dashboard overview 5-10 s -> 0.9 s. - lead_cache holds a live scope for 20 s with a stale copy, and every build is single-flight; search pages opt out. - Leads: lowercase subdomain names (CHECK) and LIKE for host:, a guard for empty JSON arrays, branches as MATERIALIZED CTEs read to the cap, every count in one statement. - Addresses and services: the derived CTE as one grouped pass, rows and count in one execution (page_rows), GIN lookups for page hosts, facets through grouping sets. - Correlation carriers counted by EXISTS; takeover latest-sighting in SQL, no silent cap. - Screenshot backfill partial index, nine unused low-cardinality indexes dropped, hostname helpers memoised, vhost names normalised.
66 lines
2.0 KiB
Python
66 lines
2.0 KiB
Python
"""Advisory locks for instance-wide work that must not run twice at once."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import zlib
|
|
from collections.abc import Iterator
|
|
from contextlib import contextmanager
|
|
from typing import TYPE_CHECKING
|
|
|
|
from sqlalchemy import text
|
|
|
|
if TYPE_CHECKING:
|
|
from sqlalchemy.orm import Session
|
|
|
|
# bounty_platform() spans BOUNTY_SYNC .. BOUNTY_SYNC + 0xFFFF
|
|
BOUNTY_SYNC = 0x624F0001
|
|
BOUNTY_FEED = 0x624F0002
|
|
# bounty_reports() spans BOUNTY_REPORTS .. BOUNTY_REPORTS + 0xFFFF
|
|
BOUNTY_REPORTS = 0x62520001
|
|
IP_RANGES = 0x624E0001
|
|
# secret_mining() spans SECRET_MINING .. SECRET_MINING + 0xFFFF
|
|
SECRET_MINING = 0x53450001
|
|
SOFTWARE_BACKFILL = 0x53570001
|
|
# software_match() spans SOFTWARE_MATCH .. SOFTWARE_MATCH + 0xFFFF
|
|
SOFTWARE_MATCH = 0x53580001
|
|
NEW_CHECKS_SWEEP = 0x4E430001
|
|
SCAN_ADMISSION = 0x53410001
|
|
SCAN_DELTAS_BACKFILL = 0x53440001
|
|
|
|
|
|
def bounty_platform(platform: str) -> int:
|
|
"""One lock per platform."""
|
|
return BOUNTY_SYNC + (zlib.crc32(platform.encode()) & 0xFFFF)
|
|
|
|
|
|
def bounty_reports(platform: str) -> int:
|
|
"""One lock per platform."""
|
|
return BOUNTY_REPORTS + (zlib.crc32(platform.encode()) & 0xFFFF)
|
|
|
|
|
|
def secret_mining(scan_id: object) -> int:
|
|
"""One lock per scan: the stage, the backfill and a re-mine never overlap."""
|
|
return SECRET_MINING + (zlib.crc32(str(scan_id).encode()) & 0xFFFF)
|
|
|
|
|
|
def software_match(scan_id: object) -> int:
|
|
"""One lock per scan."""
|
|
return SOFTWARE_MATCH + (zlib.crc32(str(scan_id).encode()) & 0xFFFF)
|
|
|
|
|
|
@contextmanager
|
|
def sync_lock(session: Session, key: int) -> Iterator[bool]:
|
|
"""Session-level advisory lock. Yields False when another session holds it."""
|
|
held = bool(
|
|
session.execute(
|
|
text("SELECT pg_try_advisory_lock(:key)"), {"key": key}
|
|
).scalar_one()
|
|
)
|
|
try:
|
|
yield held
|
|
finally:
|
|
if held:
|
|
session.rollback()
|
|
session.execute(text("SELECT pg_advisory_unlock(:key)"), {"key": key})
|
|
session.commit()
|