mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-20 23:31:31 +02:00
Yogesh OjhaandClaude Opus 5
6e49f13818
GET /dashboard/overview was 842 ms and half of it was one statement: 493 ms of row_number() OVER (PARTITION BY target_id, signature ORDER BY discovered_at) across all 534,765 endpoints in the project, run on every load to answer which scan first saw each thing. The same shape runs for all five dimensions. DISTINCT ON asks the same question and can ride ix_endpoints_target_signature, so Postgres sorts incrementally within each group instead of sorting the project (measured 511 ms -> 360 ms on the raw statement, 842 ms -> 683 ms end to end, byte-identical output). Rejected on the way: a covering index on (target_id, signature, discovered_at, scan_id) removes the sort entirely but costs 72 MB for another 1.4x, and raising work_mem does nothing because the plan already sorts incrementally rather than spilling. This is a floor, not a fix. The work is O(every row the project has ever recorded) on every dashboard load, and it grows without bound. The honest answer is a first-sighting flag written with the row — Class A in the project's own derivation rule — or a cached per-scan count, since a finished scan's answer never changes. Both are their own piece of work. GET /surface/overview re-fetched the project's targets inside each of the five coverage() calls and re-resolved the same scan picks for the exposures count. The service is constructed per request, so the answers cannot go stale within one: they are memoised on the instance. 213 ms -> 188 ms. Four tests on what the number means rather than how it is computed: a first scan contributes nothing new, only the scan that first saw a host is credited, an unchanged host is never counted twice across three scans, and newness is judged per target so a fresh target contributes nothing. Mutation-tested — crediting the last sighting instead of the first fails three of them. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013dKUV96WkQQETZBMoAG98C
84 lines
2.8 KiB
Python
84 lines
2.8 KiB
Python
"""A scan is credited with a row only if it was the first to see it."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import timedelta
|
|
|
|
import pytest
|
|
|
|
from app.services.dashboard_overview import DashboardOverviewService
|
|
from shared.definitions.surface import SurfaceDimension
|
|
|
|
pytestmark = pytest.mark.api
|
|
|
|
WEB = SurfaceDimension.WEB_ASSETS.value
|
|
|
|
|
|
def _metric(overview, key: str):
|
|
return next(m for m in overview.surface if m.key == key)
|
|
|
|
|
|
async def test_a_first_scan_contributes_nothing_new(estate, now):
|
|
await estate.scan("example.com", "only", at=now)
|
|
await estate.hosts("only", ["a.example.com", "b.example.com"], at=now)
|
|
|
|
out = await DashboardOverviewService(estate.session).overview(
|
|
estate.project_id, "7d"
|
|
)
|
|
|
|
web = _metric(out, WEB)
|
|
assert web.value == 2
|
|
assert web.new_in_window == 0, "no baseline, so nothing counts as new"
|
|
|
|
|
|
async def test_only_the_scan_that_first_saw_a_host_is_credited(estate, now):
|
|
old = now - timedelta(days=3)
|
|
await estate.scan("example.com", "first", at=old)
|
|
await estate.hosts("first", ["a.example.com", "b.example.com"], at=old)
|
|
await estate.scan("example.com", "second", at=now)
|
|
await estate.hosts(
|
|
"second", ["a.example.com", "b.example.com", "c.example.com"], at=now
|
|
)
|
|
|
|
out = await DashboardOverviewService(estate.session).overview(
|
|
estate.project_id, "7d"
|
|
)
|
|
|
|
web = _metric(out, WEB)
|
|
assert web.value == 3, "the newest covering scan holds three hosts"
|
|
assert web.new_in_window == 1, "only c.example.com is new"
|
|
|
|
|
|
async def test_a_repeat_finding_is_never_counted_twice(estate, now):
|
|
"""Three scans, one unchanged host: the first sighting is credited once."""
|
|
for days, name in ((10, "one"), (5, "two"), (0, "three")):
|
|
await estate.scan("example.com", name, at=now - timedelta(days=days))
|
|
await estate.hosts(name, ["a.example.com"], at=now - timedelta(days=days))
|
|
|
|
out = await DashboardOverviewService(estate.session).overview(
|
|
estate.project_id, "30d"
|
|
)
|
|
|
|
assert _metric(out, WEB).new_in_window == 0, (
|
|
"the only scan that first saw it has no baseline of its own"
|
|
)
|
|
|
|
|
|
async def test_newness_is_judged_per_target(estate, now):
|
|
old = now - timedelta(days=3)
|
|
await estate.scan("one.com", "one-first", at=old)
|
|
await estate.hosts("one-first", ["a.one.com"], at=old)
|
|
await estate.scan("one.com", "one-second", at=now)
|
|
await estate.hosts("one-second", ["a.one.com", "b.one.com"], at=now)
|
|
await estate.scan("two.com", "two-only", at=now)
|
|
await estate.hosts("two-only", ["a.two.com", "b.two.com"], at=now)
|
|
|
|
out = await DashboardOverviewService(estate.session).overview(
|
|
estate.project_id, "7d"
|
|
)
|
|
|
|
web = _metric(out, WEB)
|
|
assert web.value == 4
|
|
assert web.targets_covered == 2
|
|
assert web.new_in_window == 1, "two.com has no baseline and contributes nothing"
|