mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-20 07:11:28 +02:00
Yogesh OjhaandClaude Opus 5
56eb5999c4
Four implementations answered "what does this host share?" and two of them could only answer it for one run: the project-wide page had no Correlation tab at all, and its detail sheet passed the row's own scan_id, so "Related" silently meant "related inside this one target's last run". One engine now, shared/services/correlation/, parameterised by QueryScope. kinds.py holds the vocabulary as data; hubs.py discovers hubs with a GROUP BY over the whole scope instead of grouping a 3,000-row sample in Python, so a hub is no longer missed for falling outside the cut. asset_query/groups.py derives its dimensions from the same registry, and correlation-graph, related and correlation take WebAssetScope. New /surface/correlation page. The count is a promise, and three bugs broke it. Asset kinds joined http_assets and counted rows, so a host answering on 80 and 443 counted twice (cert.fingerprint said 80 where the search returned 41). asn excludes CDN-fronted rows but its token did not, 8779 against 8815; KindSpec.narrows appends -is:cdn. And carriers came from a window over the group set the caller asked about, so one row's own value read as 100% of its carriers and common swallowed it. All 28 hub counts across 14 kinds now equal their searches. The provider rules moved into the engine. generic_page, shared_edge and cert_covers gated cross links alone, so estate-wide the graph drew 404 Not Found across 5 targets, 403 Forbidden across 4 and a 548-byte 404 body across 5 as correlations. A hub is judged once its members load and is labelled with who wrote it; hubs load MAX_HUBS_PER_KIND + 40 deep and re-rank. Related takes the whole vocabulary, each kind carrying the phrase it is announced with and the query its button opens, which retires relationDsl and RELATION_LABELS. asn became a kind, out of the default set. A scope over several targets opens on what crosses them: 3,000 nodes over 21,690 correlating hosts is a hairball, the 30 crossing hubs are the finding. Measured on 63,844 host rows across 10 covering scans: ~1.1s cold for the estate, 70-200ms for a scan, cached after. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FwCCFgRBC8ut6SQpmzfGAa
96 lines
3.0 KiB
Python
96 lines
3.0 KiB
Python
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from app.services.subdomain import _RELATION_CAP, SubdomainService
|
|
from shared.definitions.correlation import MIN_ESTATE_FOR_COMMON
|
|
from shared.enums.ip import IpSource
|
|
from shared.models.ip_address import IpAddress
|
|
|
|
pytestmark = pytest.mark.api
|
|
|
|
|
|
async def _related(estate, scan: str, name: str) -> dict:
|
|
rows = await SubdomainService(estate.session).related(
|
|
estate.project_id, estate.scans[scan], name
|
|
)
|
|
return {row.kind: row for row in rows}
|
|
|
|
|
|
async def test_a_shared_address_is_a_relation(estate, now):
|
|
await estate.scan("example.com", "run", at=now)
|
|
await estate.hosts(
|
|
"run", ["a.example.com", "b.example.com"], at=now, ips=["10.0.0.1"]
|
|
)
|
|
await estate.hosts("run", ["c.example.com"], at=now, ips=["10.0.0.9"])
|
|
|
|
rows = await _related(estate, "run", "a.example.com")
|
|
|
|
assert rows["ip"].hosts == ["b.example.com"]
|
|
assert rows["ip"].total == 1
|
|
|
|
|
|
async def test_the_count_is_the_truth_not_the_length_of_the_list(estate, now):
|
|
await estate.scan("example.com", "run", at=now)
|
|
names = [f"h{i}.example.com" for i in range(_RELATION_CAP + 40)]
|
|
await estate.hosts("run", names, at=now, cname="edge.example.net")
|
|
await estate.hosts(
|
|
"run",
|
|
[f"other{i}.example.com" for i in range(_RELATION_CAP * 3)],
|
|
at=now,
|
|
cname="other.example.net",
|
|
)
|
|
|
|
rows = await _related(estate, "run", names[0])
|
|
|
|
assert len(rows["cname"].hosts) == _RELATION_CAP
|
|
assert rows["cname"].total == len(names) - 1
|
|
|
|
|
|
async def test_a_value_the_whole_estate_shares_is_not_a_relation(estate, now):
|
|
await estate.scan("example.com", "run", at=now)
|
|
names = [f"h{i}.example.com" for i in range(MIN_ESTATE_FOR_COMMON + 10)]
|
|
await estate.hosts("run", names, at=now, cname="everything.example.net")
|
|
|
|
rows = await _related(estate, "run", names[0])
|
|
|
|
assert "cname" not in rows
|
|
|
|
|
|
async def test_a_small_scan_suppresses_nothing(estate, now):
|
|
await estate.scan("example.com", "run", at=now)
|
|
names = [f"h{i}.example.com" for i in range(4)]
|
|
await estate.hosts("run", names, at=now, cname="edge.example.net")
|
|
|
|
rows = await _related(estate, "run", names[0])
|
|
|
|
assert rows["cname"].total == 3
|
|
|
|
|
|
async def test_a_shared_edge_is_the_provider_not_a_relation(estate, now):
|
|
await estate.scan("example.com", "run", at=now)
|
|
await estate.hosts(
|
|
"run",
|
|
["a.example.com", "b.example.com"],
|
|
at=now,
|
|
ips=["151.101.2.132"],
|
|
cname="j.sni.global.fastly.net",
|
|
)
|
|
sid = estate.scans["run"]
|
|
estate.session.add(
|
|
IpAddress(
|
|
project_id=estate.project_id,
|
|
scan_id=sid,
|
|
target_id=await estate._target_of(sid),
|
|
ip="151.101.2.132",
|
|
version=4,
|
|
source=IpSource.DNS_RESOLUTION.value,
|
|
is_cdn=True,
|
|
cdn_name="fastly",
|
|
discovered_at=now,
|
|
)
|
|
)
|
|
await estate.session.flush()
|
|
|
|
assert await _related(estate, "run", "a.example.com") == {}
|