mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-20 23:31:31 +02:00
Yogesh OjhaandClaude Opus 5
2375f1f1e4
The unit of exposure is now a service — (ip, port) — not a bare open port. Pipeline (EXPANSION): L0 produce hosts, L1 cdn_check + passive_ports, L2 port_scan, L3 http_probe, L4 screenshot/waf_detect/service_fingerprint. - cdn_check applies to every target type and records cdn_type, so a domain scan finally knows which addresses are CDN-fronted. cdn/waf is an edge you do not own (edge ports only); cloud is your own machine (scanned in full). - port_scan decides and records a per-address scan_policy (full/web/skip) honouring the scan context's excluded_ips, and offers port profiles, exclusions, scan type, a tarpit threshold and an address budget. - passive_ports reads Shodan internetdb through naabu -passive: keyless, touches_target=False, so a passive-intensity scan still gets exposure data. - http_probe seeds its host x port matrix from open ports and folds every live response back onto the port (is_http, tls, product). - service_fingerprint names the software behind non-web ports with a pure Python banner prober (tools/banner/), including a SOCKS5 CONNECT so it honours the context proxy. naabu -sV is unimplemented. - shared/definitions/ports.py is the one vocabulary; ip_inventory and port_inventory are the only writers of ip_addresses/ports. - open_ports_found is measured from the table (DERIVED_COUNTS): four stages write ports, so summing stage results double-counted. - naabu failures no longer report zero open ports as success. Surfaces: a Services tab (SERVICE_QUERY grammar, 16 fields, 18 counted leads, 9 group dimensions, detail sheet) built on the shared results shell, and an outlier-first Exposure panel on the Overview. Every count is a promise — leads, groups and panel pivots were verified to equal the row count they land on. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BnC2xMMzV6WQnpDp5zcXoU
66 lines
2.3 KiB
Python
66 lines
2.3 KiB
Python
import uuid
|
|
from collections.abc import Iterable
|
|
from typing import TYPE_CHECKING
|
|
|
|
from sqlalchemy import text
|
|
|
|
from shared.enums.scan import (
|
|
ACTIVITY_TERMINAL_STATUSES,
|
|
ScanActivityStatus,
|
|
ScanStatus,
|
|
)
|
|
from shared.models.scan_activity import ScanActivity
|
|
|
|
if TYPE_CHECKING:
|
|
from sqlalchemy.orm import Session
|
|
|
|
# rollup SSOT — a new headline metric needs an entry here + a scans column (migration)
|
|
# + shared.definitions.notifications._SCAN_COUNT_LABELS; other count keys stay stage-only.
|
|
COUNT_TO_COLUMN = {
|
|
"subdomains": "subdomains_found",
|
|
"ips": "ips_found",
|
|
"open_ports": "open_ports_found",
|
|
"http_assets": "http_assets_found",
|
|
"vulnerabilities": "vulnerabilities_found",
|
|
"endpoints": "endpoints_found",
|
|
}
|
|
|
|
|
|
# counts several stages write into the same rows; summing their results double-counts,
|
|
# so these are measured from the table instead
|
|
DERIVED_COUNTS: dict[str, str] = {
|
|
"open_ports_found": "SELECT count(*) FROM ports WHERE scan_id = :sid",
|
|
}
|
|
|
|
|
|
def derived_counts(session: "Session", scan_id: uuid.UUID) -> dict[str, int]:
|
|
return {
|
|
column: int(session.execute(text(sql).bindparams(sid=scan_id)).scalar() or 0)
|
|
for column, sql in DERIVED_COUNTS.items()
|
|
}
|
|
|
|
|
|
def aggregate_status(activities: Iterable[ScanActivity]) -> str:
|
|
# in-flight/empty -> RUNNING sentinel (don't finalize); any failed stage -> failed
|
|
statuses = [a.status for a in activities]
|
|
if not statuses:
|
|
return ScanStatus.RUNNING.value
|
|
if any(s not in ACTIVITY_TERMINAL_STATUSES for s in statuses):
|
|
return ScanStatus.RUNNING.value
|
|
if any(s == ScanActivityStatus.ABORTED.value for s in statuses):
|
|
return ScanStatus.CANCELLED.value
|
|
if any(s == ScanActivityStatus.FAILED.value for s in statuses):
|
|
return ScanStatus.FAILED.value
|
|
return ScanStatus.COMPLETED.value
|
|
|
|
|
|
def aggregate_counts(activities: Iterable[ScanActivity]) -> dict[str, int]:
|
|
"""Sum per-stage result counts into the scan rollup columns."""
|
|
totals: dict[str, int] = dict.fromkeys(COUNT_TO_COLUMN.values(), 0)
|
|
for activity in activities:
|
|
for key, column in COUNT_TO_COLUMN.items():
|
|
value = (activity.result or {}).get(key)
|
|
if isinstance(value, int):
|
|
totals[column] += value
|
|
return totals
|