mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-19 23:01:29 +02:00
Yogesh Ojha
6949492e11
- Store each run's row, first-seen and retired counts (scan_deltas, scan_retired), valid while the revision stamp matches scan_revisions; statement triggers on the seven result tables bump it. Finalize warms, a beat backfill fills the rest, a live run answers from its last count and queues a refresh. /scans 1.5 s -> 43 ms, dashboard overview 5-10 s -> 0.9 s. - lead_cache holds a live scope for 20 s with a stale copy, and every build is single-flight; search pages opt out. - Leads: lowercase subdomain names (CHECK) and LIKE for host:, a guard for empty JSON arrays, branches as MATERIALIZED CTEs read to the cap, every count in one statement. - Addresses and services: the derived CTE as one grouped pass, rows and count in one execution (page_rows), GIN lookups for page hosts, facets through grouping sets. - Correlation carriers counted by EXISTS; takeover latest-sighting in SQL, no silent cap. - Screenshot backfill partial index, nine unused low-cardinality indexes dropped, hostname helpers memoised, vhost names normalised.
48 lines
1.8 KiB
Python
48 lines
1.8 KiB
Python
import uuid
|
|
from datetime import datetime
|
|
|
|
from sqlalchemy import BigInteger, Column, PrimaryKeyConstraint
|
|
from sqlmodel import Field, SQLModel
|
|
|
|
|
|
class ScanRevision(SQLModel, table=True):
|
|
"""Change counters per scan and dimension, bumped by triggers on the result tables."""
|
|
|
|
__tablename__ = "scan_revisions"
|
|
__table_args__ = (PrimaryKeyConstraint("scan_id", "dimension"),)
|
|
|
|
scan_id: uuid.UUID = Field(foreign_key="scans.id", ondelete="CASCADE")
|
|
dimension: str = Field(max_length=32)
|
|
# the scan's own rows
|
|
rows_rev: int = Field(default=0, sa_column=Column(BigInteger, nullable=False))
|
|
# the scan's own rows or an earlier row of its target
|
|
history_rev: int = Field(default=0, sa_column=Column(BigInteger, nullable=False))
|
|
|
|
|
|
class ScanDelta(SQLModel, table=True):
|
|
"""A scan's rows and the keys it was the first to report for its target, as of `history_rev`."""
|
|
|
|
__tablename__ = "scan_deltas"
|
|
__table_args__ = (PrimaryKeyConstraint("scan_id", "dimension"),)
|
|
|
|
scan_id: uuid.UUID = Field(foreign_key="scans.id", ondelete="CASCADE")
|
|
dimension: str = Field(max_length=32)
|
|
history_rev: int = Field(sa_column=Column(BigInteger, nullable=False))
|
|
first_seen: int
|
|
rows: int
|
|
first_at: datetime | None = None
|
|
|
|
|
|
class ScanRetired(SQLModel, table=True):
|
|
"""Keys the previous run held that a scan lacks, as of both runs' `rows_rev`."""
|
|
|
|
__tablename__ = "scan_retired"
|
|
__table_args__ = (PrimaryKeyConstraint("scan_id", "prev_scan_id", "dimension"),)
|
|
|
|
scan_id: uuid.UUID = Field(foreign_key="scans.id", ondelete="CASCADE")
|
|
prev_scan_id: uuid.UUID = Field(foreign_key="scans.id", ondelete="CASCADE")
|
|
dimension: str = Field(max_length=32)
|
|
rows_rev: int = Field(sa_column=Column(BigInteger, nullable=False))
|
|
prev_rows_rev: int = Field(sa_column=Column(BigInteger, nullable=False))
|
|
retired: int
|