mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-20 07:11:28 +02:00
Yogesh OjhaandClaude Opus 5
7781259c62
POST /exports saves what the table was showing — dimension, scope, the filter body, the format — and queues a run. The row is the recipe, so rerun runs it again against today's rows rather than handing back yesterday's file, which is the point of keeping it. The scope resolves when the run starts, not when it was saved: a scan export reads that scan, a target or project export re-picks the newest covering scan of each target, so a saved export follows the estate. That needed a sync resolver, since surface_scope is async and the api is not on the worker's path. A bundle writes one file per dimension into a zip. Expiry is its own status rather than a failure wearing the wrong label, and a reaper fails a run the worker lost, which reports still lack. Streaming takes a connection of its own. yield_per opens a server side cursor, and the progress commit — or the per chunk join — executed on the session would invalidate it mid-file; the first large export failed with "named cursor isn't valid anymore" and nothing smaller ever would have. The export therefore reads committed rows, so its promise test moves to the durable fixture. Proven end to end: a filtered CSV that matches the table, 7,784 URLs one per line, a five file zip, a project wide export across 2,184 addresses, and a rerun. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LYZfaNjq3Ym8GPvWYuHisS
70 lines
2.2 KiB
Python
70 lines
2.2 KiB
Python
"""The scans a dimension reads, resolved from sync code the way the api resolves them."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import TYPE_CHECKING
|
|
|
|
from sqlalchemy import func, select
|
|
|
|
from shared.definitions.surface import SurfaceDimension
|
|
from shared.models.endpoint import Endpoint
|
|
from shared.models.ip_address import IpAddress
|
|
from shared.models.port import Port
|
|
from shared.models.scan import Scan
|
|
from shared.models.software import SoftwareCve
|
|
from shared.models.subdomain import Subdomain
|
|
from shared.models.vulnerability import Vulnerability
|
|
from shared.services.asset_query import QueryScope
|
|
from shared.services.scan_scope import census_only, covers
|
|
|
|
if TYPE_CHECKING:
|
|
from uuid import UUID
|
|
|
|
from sqlalchemy.orm import Session
|
|
|
|
TABLES = {
|
|
SurfaceDimension.WEB_ASSETS.value: Subdomain,
|
|
SurfaceDimension.ENDPOINTS.value: Endpoint,
|
|
SurfaceDimension.SERVICES.value: Port,
|
|
SurfaceDimension.IPS.value: IpAddress,
|
|
SurfaceDimension.VULNERABILITIES.value: Vulnerability,
|
|
SurfaceDimension.SOFTWARE.value: SoftwareCve,
|
|
}
|
|
|
|
|
|
def _started():
|
|
return func.coalesce(Scan.started_at, Scan.created_at)
|
|
|
|
|
|
def _picks(session: Session, project_id: UUID, dimension: str, target_id=None):
|
|
"""The newest scan of each target that actually ran this dimension."""
|
|
statement = (
|
|
select(Scan.id, Scan.target_id)
|
|
.where(
|
|
Scan.project_id == project_id,
|
|
census_only(),
|
|
covers(TABLES[dimension], dimension),
|
|
)
|
|
.distinct(Scan.target_id)
|
|
.order_by(Scan.target_id, _started().desc())
|
|
)
|
|
if target_id is not None:
|
|
statement = statement.where(Scan.target_id == target_id)
|
|
return session.execute(statement).all()
|
|
|
|
|
|
def project_scope(session: Session, project_id: UUID, dimension: str) -> QueryScope:
|
|
return QueryScope(
|
|
tuple(row.id for row in _picks(session, project_id, dimension)),
|
|
project_id=project_id,
|
|
)
|
|
|
|
|
|
def target_scope(
|
|
session: Session, project_id: UUID, target_id: UUID, dimension: str
|
|
) -> QueryScope:
|
|
return QueryScope(
|
|
tuple(row.id for row in _picks(session, project_id, dimension, target_id)),
|
|
project_id=project_id,
|
|
)
|