mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-21 07:41:28 +02:00
Yogesh OjhaandClaude Opus 5
3075ad3ddf
The five search services build their statement with pure, session-free helpers — _context, _apply_filter, _order, _scoped — and then enrich the page asynchronously. Only the enrichment needs the api. Moving the builders to shared/services/surface_query lets the worker compile the same filtered set the table shows, which an export has to do and could not reach from worker/app. Web assets gains the _scoped it never had: its base statement was inlined in search, leads, seeds and renders, so a predicate could drift between the rows a query returns and the rows its leads count. The four now read one definition. No behaviour change. Every route over the moved builders — search, leads, groups, renders and facets across all five dimensions — was exercised against a live scan. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LYZfaNjq3Ym8GPvWYuHisS
103 lines
3.3 KiB
Python
103 lines
3.3 KiB
Python
"""Web asset statement building, shared by the api's search and the worker's export."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import TYPE_CHECKING
|
|
|
|
from sqlalchemy import cast, func, or_, select
|
|
from sqlalchemy.dialects.postgresql import JSONB
|
|
from sqlalchemy.dialects.postgresql import array as pg_array
|
|
|
|
from shared.definitions.asset_query import HOST_QUERY
|
|
from shared.models.port import Port
|
|
from shared.models.subdomain import Subdomain
|
|
from shared.services.asset_query import (
|
|
QueryContext,
|
|
QueryScope,
|
|
ScopeLike,
|
|
compile_query,
|
|
parse_query,
|
|
)
|
|
from shared.services.asset_query import (
|
|
predicates as preds,
|
|
)
|
|
|
|
if TYPE_CHECKING:
|
|
from datetime import datetime
|
|
from uuid import UUID
|
|
|
|
from shared.models.subdomain import SubdomainFilter
|
|
|
|
|
|
def context(scope: QueryScope, now: datetime) -> QueryContext:
|
|
return QueryContext(scope=scope, now=now)
|
|
|
|
|
|
def apply_filter(query, f: SubdomainFilter, now: datetime, scope: ScopeLike):
|
|
if f.statuses:
|
|
query = query.where(or_(*[preds.status_class(s) for s in f.statuses]))
|
|
if f.tech:
|
|
query = query.where(
|
|
func.jsonb_exists_any(cast(Subdomain.tech, JSONB), pg_array(f.tech))
|
|
)
|
|
if f.sources:
|
|
query = query.where(
|
|
func.jsonb_exists_any(cast(Subdomain.sources, JSONB), pg_array(f.sources))
|
|
)
|
|
if f.cert:
|
|
query = query.where(or_(*[preds.cert_state(c, now) for c in f.cert]))
|
|
if f.hygiene:
|
|
query = query.where(preds.hygiene(f.hygiene))
|
|
if f.services:
|
|
query = query.where(preds.port_match(Port.service_name.in_(f.services), scope))
|
|
if f.cdn == "yes":
|
|
query = query.where(Subdomain.is_cdn.is_(True))
|
|
elif f.cdn == "no":
|
|
query = query.where(Subdomain.is_cdn.is_(False))
|
|
if f.waf == "present":
|
|
query = query.where(Subdomain.waf.isnot(None))
|
|
elif f.waf == "none":
|
|
query = query.where(Subdomain.waf.is_(None))
|
|
if f.live:
|
|
query = query.where(preds.live())
|
|
if f.screenshot:
|
|
query = query.where(Subdomain.screenshot_path.isnot(None))
|
|
if f.new:
|
|
query = query.where(preds.is_new(scope))
|
|
if f.issues:
|
|
query = query.where(preds.issues(now))
|
|
return query
|
|
|
|
|
|
def order(query, f: SubdomainFilter):
|
|
col = {
|
|
"status": Subdomain.http_status,
|
|
"size": Subdomain.content_length,
|
|
"time": Subdomain.response_time,
|
|
"title": Subdomain.page_title,
|
|
"cert": Subdomain.tls_not_after,
|
|
"discovered": Subdomain.discovered_at,
|
|
"ip": cast(Subdomain.resolved_ips, JSONB).op("->>")(0),
|
|
}.get(f.sort, Subdomain.name)
|
|
primary = col.desc() if f.order == "desc" else col.asc()
|
|
return query.order_by(primary.nulls_last(), Subdomain.name.asc())
|
|
|
|
|
|
def scoped(
|
|
project_id: UUID,
|
|
scope: QueryScope,
|
|
f: SubdomainFilter,
|
|
now: datetime,
|
|
columns=None,
|
|
):
|
|
"""The one definition of the filtered host set. Was inlined four times."""
|
|
base = select(Subdomain) if columns is None else select(*columns)
|
|
base = base.where(
|
|
Subdomain.project_id == project_id, scope.match(Subdomain.scan_id)
|
|
)
|
|
return apply_filter(base, f, now, scope)
|
|
|
|
|
|
def compiled(scope: QueryScope, f: SubdomainFilter, now: datetime):
|
|
return compile_query(parse_query(f.q, HOST_QUERY), context(scope, now))
|