mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-20 07:11:28 +02:00
Yogesh OjhaandClaude Sonnet 5
7a960f1de1
urllib.parse.SplitResult.port raises ValueError when the authority carries a non-numeric port; nine call sites read it unguarded, so one malformed URL (a stray quote, a client typo) turned into an unhandled exception in the middle of a scan, a report, an interest rule or a proxy dial. shared/utils/net.py:url_port makes the read itself the failure boundary; unreadable_port names the bogus text so the naabu and banner proxy paths can report what they refused rather than retrying against a host string that dropped its port silently. shared/definitions/ports.py:SCHEME_PORTS replaces four private copies of the same http/https default-port map (endpoints.py, endpoint_judge, scan_surface/normalize, nuclei's parser, dalfox's parser). Also renames live_hosts to answered_hosts in the report source and the interest context, and lifts the asset query's inline is:web predicate into predicates.answered() — the existing distinction between is:live (2xx) and is:web (answered at all) was already the rule; live_hosts was the one name still claiming the stronger one. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VckaDx6mBB4TsG76u61gvv
106 lines
3.0 KiB
Python
106 lines
3.0 KiB
Python
"""What every provider is handed: the scan, its rules, and the estate it sits in."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime
|
|
from functools import cached_property
|
|
|
|
from sqlalchemy import func, select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from shared.models.interest import InterestRule
|
|
from shared.models.scan import Scan
|
|
from shared.models.subdomain import Subdomain
|
|
from shared.services.ai.config import AIConfig
|
|
from shared.utils.datetime import utc_now
|
|
|
|
MAX_JUDGE_HOSTS = 1500
|
|
SHAPE_KEEP = 3
|
|
|
|
|
|
@dataclass
|
|
class HostRow:
|
|
id: object
|
|
name: str
|
|
http_status: int | None
|
|
page_title: str | None
|
|
tech: list
|
|
webserver: str | None
|
|
is_cdn: bool
|
|
asn: int | None
|
|
asn_org: str | None
|
|
favicon_hash: str | None
|
|
|
|
@property
|
|
def shape(self) -> tuple:
|
|
return (
|
|
self.http_status,
|
|
(self.page_title or "").strip().lower()[:120],
|
|
tuple(sorted(t.lower() for t in (self.tech or []))[:6]),
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class InterestContext:
|
|
session: Session
|
|
scan: Scan
|
|
rules: list[InterestRule] = field(default_factory=list)
|
|
ai: AIConfig | None = None
|
|
now: datetime = field(default_factory=utc_now)
|
|
|
|
@cached_property
|
|
def host_total(self) -> int:
|
|
return int(
|
|
self.session.execute(
|
|
select(func.count(Subdomain.id)).where(
|
|
Subdomain.scan_id == self.scan.id,
|
|
Subdomain.is_excluded.is_(False),
|
|
)
|
|
).scalar()
|
|
or 0
|
|
)
|
|
|
|
@cached_property
|
|
def answered_hosts(self) -> list[HostRow]:
|
|
rows = (
|
|
self.session.execute(
|
|
select(
|
|
Subdomain.id,
|
|
Subdomain.name,
|
|
Subdomain.http_status,
|
|
Subdomain.page_title,
|
|
Subdomain.tech,
|
|
Subdomain.webserver,
|
|
Subdomain.is_cdn,
|
|
Subdomain.asn,
|
|
Subdomain.asn_org,
|
|
Subdomain.favicon_hash,
|
|
)
|
|
.where(
|
|
Subdomain.scan_id == self.scan.id,
|
|
Subdomain.is_excluded.is_(False),
|
|
Subdomain.http_status.isnot(None),
|
|
)
|
|
.order_by(Subdomain.name)
|
|
)
|
|
.mappings()
|
|
.all()
|
|
)
|
|
return [HostRow(**dict(r)) for r in rows]
|
|
|
|
def judgeable(self, limit: int = MAX_JUDGE_HOSTS) -> list[HostRow]:
|
|
"""Bulk parked pages are one answer, not five hundred."""
|
|
seen: dict[tuple, int] = {}
|
|
kept: list[HostRow] = []
|
|
for row in self.answered_hosts:
|
|
shape = row.shape
|
|
count = seen.get(shape, 0)
|
|
seen[shape] = count + 1
|
|
if count >= SHAPE_KEEP:
|
|
continue
|
|
kept.append(row)
|
|
if len(kept) >= limit:
|
|
break
|
|
return kept
|