mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-21 07:41:28 +02:00
Yogesh OjhaandClaude Opus 5
913815acd4
Four scans against real estates (hackerone.com 40 names, bugcrowd.com 71, uber.com 11,203, shopify.com 26,235) show the correlation graph drawing the CDN's identities and the resolver reporting a fraction of the live surface as all of it. The resolver's silence is re-queried once. dnsx prints nothing both for a name that does not exist and for a query it dropped, and the loss is uniform, so `_retry_degraded`'s peer median never fires (and a single-batch estate has no peers). uber.com reported 135 of 11,203 names resolved; a plain dnsx re-run over a random 200 of the 11,068 it called dead answered 7, the same 7 three times over, and a DoH check of another 25 found 2 more. `_retry_silent` re-queries every unanswered name once (`dns_retry_silent`), writes through `_write_resolution` so the live view keeps up, counts recoveries as `recovered`, and reports partial when that pass still recovers names. A hub that belongs to a provider is not a correlation. bugcrowd.com drew six hubs and five of them were the CDN's: four Fastly anycast addresses and the `j.sni.global.fastly.net` CNAME every Fastly tenant is aliased to - four hubs over one pair of hosts, each restating what the CDN kind already says. Hubs now carry `platform`/`platform_label` from `ip_addresses.is_cdn` and from curated lists beside `is_shared_nameserver` (`SHARED_EDGE_HOSTS`, `PUBLIC_CA_NAMES`), the graph hides them behind "Hide provider", and `related()` drops them. Curated, never inferred from a suffix: `d3tqr1od3k1eh2.cloudfront.net` is one distribution, `j.sni.global.fastly.net` is the whole platform; a private issuer stays visible. Three more corrections in the same build. A share is read against the hosts carrying that kind, not every name in the scan, so shopify.com's 16-of-18 JARM cluster no longer reads as 0.5%, and `common` needs `MIN_ESTATE_FOR_COMMON` carriers. The 3,000-host cut is ranked by "answered, then resolved, then name" - alphabetically it held 19 of shopify.com's 116 web assets, and ranking took that scan from 35 hubs to 94 and its certificate correlation from 6 hosts to 66. A body under `MIN_BODY_BYTES` is not an identity: 11 hosts shared the SHA-256 of an empty body. Hidden hubs no longer consume the per-kind cap, chip counts equal what they draw, and the endpoint goes through `lead_cache` like its neighbours (3.3s to 18ms). Enrichment, which feeds all of it: - `sync_ranges` takes an advisory lock and a budget. Two concurrent scans on a fresh instance each pulled the same 200 MB and each truncated and reloaded 1.1M rows: 291s against 145s. A scan that finds the tables missing reports partial, and `app.tasks.ip_asn.refresh` fills the blanks through `backfill_addresses` once the feed lands. - No address is attributed from its target. The last-resort fill stamped the target's BGP holder and its WHOIS registrant country onto every address the range tables did not match, private ones included - a registrant country is not a geolocation and a target's network is not the operator of everything its scan found. It fired wherever that data exists, which is IP, range and ASN targets and any registry that publishes a country; only an ASN target now states its own sweep. - cdncheck sees every address, httpx only the ones that answered, so `ip_enrichment` folds `ip_addresses.is_cdn` back onto the hosts that resolve to it. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YYop31s6BqDUt4TBs5mbwa
93 lines
2.9 KiB
Python
93 lines
2.9 KiB
Python
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from app.services.subdomain import _RELATION_CAP, SubdomainService
|
|
from shared.definitions.correlation import MIN_ESTATE_FOR_COMMON
|
|
from shared.enums.ip import IpSource
|
|
from shared.models.ip_address import IpAddress
|
|
|
|
pytestmark = pytest.mark.api
|
|
|
|
|
|
async def _related(estate, scan: str, name: str) -> dict:
|
|
rows = await SubdomainService(estate.session).related(
|
|
estate.project_id, estate.scans[scan], name
|
|
)
|
|
return {row.kind: row for row in rows}
|
|
|
|
|
|
async def test_a_shared_address_is_a_relation(estate, now):
|
|
await estate.scan("example.com", "run", at=now)
|
|
await estate.hosts(
|
|
"run", ["a.example.com", "b.example.com"], at=now, ips=["10.0.0.1"]
|
|
)
|
|
await estate.hosts("run", ["c.example.com"], at=now, ips=["10.0.0.9"])
|
|
|
|
rows = await _related(estate, "run", "a.example.com")
|
|
|
|
assert rows["ip"].hosts == ["b.example.com"]
|
|
assert rows["ip"].total == 1
|
|
|
|
|
|
async def test_the_count_is_the_truth_not_the_length_of_the_list(estate, now):
|
|
await estate.scan("example.com", "run", at=now)
|
|
names = [f"h{i}.example.com" for i in range(_RELATION_CAP + 40)]
|
|
await estate.hosts("run", names, at=now, cname="edge.example.net")
|
|
await estate.hosts(
|
|
"run", [f"other{i}.example.com" for i in range(_RELATION_CAP * 3)], at=now
|
|
)
|
|
|
|
rows = await _related(estate, "run", names[0])
|
|
|
|
assert len(rows["cname"].hosts) == _RELATION_CAP
|
|
assert rows["cname"].total == len(names) - 1
|
|
|
|
|
|
async def test_a_value_the_whole_estate_shares_is_not_a_relation(estate, now):
|
|
await estate.scan("example.com", "run", at=now)
|
|
names = [f"h{i}.example.com" for i in range(MIN_ESTATE_FOR_COMMON + 10)]
|
|
await estate.hosts("run", names, at=now, cname="everything.example.net")
|
|
|
|
rows = await _related(estate, "run", names[0])
|
|
|
|
assert "cname" not in rows
|
|
|
|
|
|
async def test_a_small_scan_suppresses_nothing(estate, now):
|
|
await estate.scan("example.com", "run", at=now)
|
|
names = [f"h{i}.example.com" for i in range(4)]
|
|
await estate.hosts("run", names, at=now, cname="edge.example.net")
|
|
|
|
rows = await _related(estate, "run", names[0])
|
|
|
|
assert rows["cname"].total == 3
|
|
|
|
|
|
async def test_a_shared_edge_is_the_provider_not_a_relation(estate, now):
|
|
await estate.scan("example.com", "run", at=now)
|
|
await estate.hosts(
|
|
"run",
|
|
["a.example.com", "b.example.com"],
|
|
at=now,
|
|
ips=["151.101.2.132"],
|
|
cname="j.sni.global.fastly.net",
|
|
)
|
|
sid = estate.scans["run"]
|
|
estate.session.add(
|
|
IpAddress(
|
|
project_id=estate.project_id,
|
|
scan_id=sid,
|
|
target_id=await estate._target_of(sid),
|
|
ip="151.101.2.132",
|
|
version=4,
|
|
source=IpSource.DNS_RESOLUTION.value,
|
|
is_cdn=True,
|
|
cdn_name="fastly",
|
|
discovered_at=now,
|
|
)
|
|
)
|
|
await estate.session.flush()
|
|
|
|
assert await _related(estate, "run", "a.example.com") == {}
|