Files
rengine/shared/utils/validation.py
T
Yogesh OjhaandClaude Sonnet 5 8fac962556 fix(celery): dispatch the canvas one step at a time, not nested
A nested chord canvas put the whole remaining pipeline into every
stage message, duplicated per sibling per level: 33 MB per message,
39 of them filled a 1 GB Redis, and the worker was OOM-killed
deserializing them. Every later launch then failed with "command not
allowed when used memory > 'maxmemory'".

shared/services/orchestrator/canvas.py now builds one step at a time;
a step's chord body is app.tasks.scan.run_scan_step(steps, index+1),
which dispatches the next step once the current one settles. Messages
are ~1 KB now; tests/pipeline/test_canvas_shape.py caps them.

Also, from an end-to-end review of the pipeline:

- The reaper re-queues a PENDING scan with no launch anywhere in the
  queue, so a lost or failed send_task no longer strands it forever.
  api create() now dispatches off the event loop and records a failed
  dispatch the way create_batch already did.
- Cancel no longer revokes queued stage tasks. A cancelled run used to
  walk every remaining step recording one aborted row per stage, and
  whether finalize ran depended on a race with the revoke broadcast.
  The step body now finalizes a cancelled run directly, and the api
  also dispatches finalize so a run with no live canvas still settles.
- The reaper only skips scans that actually have a message queued,
  read off the scans queue, instead of skipping every scan whenever
  the queue holds anything.
- Every beat tick carries expires, so a worker that comes back after
  being down does not replay hours of stacked ticks.
- worker-default and worker-beat restart on their own.
- _run_process kills the tool in a finally; a soft time limit raised
  inside proc.wait used to leave the tool running past the stage.
- on_worker_ready crashed on a missing Consumer attribute, so the
  worker's IP-range and threat-feed warm-up never ran.
- Worker logging now goes through the root logger; it was configured
  on a "rengine.worker" logger nothing writes to, so INFO logs (task
  starts, step hand-overs) never appeared in the container output.
- _normalize_url no longer raises on an unparseable value, which had
  been killing the whole bounty feed sync on one bad scope entry.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0181JfVNatLVWnDbtGWRWPov
2026-09-22 19:31:15 +05:30

228 lines
6.5 KiB
Python

import ipaddress
import re
from urllib.parse import urlsplit, urlunsplit
import validators
from shared.enums.target import TargetType
HEX_COLOR_LENGTH = 7 # #RRGGBB
MAX_NAME_LEN = 120
def clean_name(v: str, *, max_len: int = MAX_NAME_LEN) -> str:
v = (v or "").strip()
if not v:
msg = "Name must not be empty."
raise ValueError(msg)
if len(v) > max_len:
msg = f"Name must be {max_len} characters or fewer."
raise ValueError(msg)
return v
def clean_optional_name(v: str | None, *, max_len: int = MAX_NAME_LEN) -> str | None:
"""An update that omits the name leaves it alone."""
return None if v is None else clean_name(v, max_len=max_len)
def validate_domain(value: str) -> bool:
return validators.domain(value) is True
def validate_ip(value: str) -> bool:
try:
ipaddress.ip_address(value)
return True
except ValueError:
return False
def validate_ip_range(value: str) -> bool:
try:
ipaddress.ip_network(value, strict=False)
return "/" in value
except ValueError:
return False
MAX_ASN = 4294967295
_ASN_SHAPE = re.compile(r"^AS(\d+)$")
def validate_asn(value: str) -> bool:
match = _ASN_SHAPE.match(value.upper())
return bool(match) and 0 < int(match.group(1)) <= MAX_ASN
WEB_SCHEMES = ("http://", "https://")
_SCHEME = re.compile(r"^[A-Za-z][A-Za-z0-9+.\-]*://")
def validate_url(value: str) -> bool:
return value.lower().startswith(WEB_SCHEMES) and validators.url(value) is True
def _lower_host(netloc: str) -> str:
"""Case-fold the host, leaving any userinfo as written."""
userinfo, sep, host = netloc.rpartition("@")
return f"{userinfo}{sep}{host.lower()}"
def _normalize_url(value: str) -> str:
try:
parts = urlsplit(value)
except ValueError:
return value
return urlunsplit(
parts._replace(scheme=parts.scheme.lower(), netloc=_lower_host(parts.netloc))
)
def normalize_target_value(value: str) -> str:
"""Normalized target value. Host names are case-folded so one host is one target."""
v = (value or "").strip()
if not v:
return v
if _SCHEME.match(v):
return _normalize_url(v)
v = v.removeprefix("*.")
return v.rstrip("/").rstrip(".").lower()
TARGET_FORMAT_HINT = "Enter a domain, IP address, CIDR range, URL or ASN."
_MAX_ECHO = 120
def unrecognised_target(value: str) -> str:
"""Wording for a target value the scanner will not take."""
shown = (value or "").strip()[:_MAX_ECHO]
return _refusal(normalize_target_value(shown), shown) or (
f"Unrecognised target: {shown}. {TARGET_FORMAT_HINT}"
)
def _refusal(value: str, shown: str) -> str | None:
"""The reason, when the value has the right shape but is out of bounds."""
if _ASN_SHAPE.match(value.upper()):
return f"{shown} is out of range. An AS number runs from 1 to {MAX_ASN}."
try:
block = ipaddress.ip_network(value, strict=False)
except ValueError:
return None
if block.prefixlen == 0:
return f"{shown} covers every address. Enter a narrower range."
return (
f"{shown} is out of scope. Loopback, link-local, multicast and reserved "
"address space are not scanned."
)
def _routable(address) -> bool:
return not (
address.is_unspecified
or address.is_loopback
or address.is_link_local
or address.is_multicast
or address.is_reserved
)
def scannable_address(value: str) -> bool:
"""Address space a scan may be aimed at. Private ranges stay in for corporate estates."""
try:
return _routable(ipaddress.ip_address(value))
except ValueError:
return False
def _scannable(value: str, target_type: TargetType) -> bool:
"""Address space a scan may be aimed at. Private ranges stay in for corporate estates."""
if target_type is TargetType.IP_RANGE:
block = ipaddress.ip_network(value, strict=False)
return block.prefixlen > 0 and _routable(block.network_address)
if target_type is TargetType.IP:
return scannable_address(value)
return True
def validate_target(target_value: str) -> TargetType | None:
value = normalize_target_value(target_value)
if not value:
return None
_validators = [
(TargetType.IP_RANGE, validate_ip_range),
(TargetType.IP, validate_ip),
(TargetType.URL, validate_url),
(TargetType.ASN, validate_asn),
(TargetType.DOMAIN, validate_domain),
]
for target_type, validator_func in _validators:
if validator_func(value):
return target_type if _scannable(value, target_type) else None
return None
def normalize_query(query: str, target_type: TargetType) -> str:
match target_type:
case TargetType.DOMAIN:
return normalize_domain(query)
case TargetType.URL:
return normalize_domain(extract_domain_from_url(query))
case TargetType.ASN:
return str(extract_asn_number(query))
case _:
return query.strip()
def normalize_domain(domain: str) -> str:
domain = domain.strip().lower()
domain = re.sub(r"^https?://", "", domain)
domain = domain.split("/")[0].split("?")[0].split("#")[0].split(":")[0]
return domain.rstrip(".")
def extract_asn_number(query: str) -> int:
cleaned = re.sub(r"^[Aa][Ss]", "", query.strip())
return int(cleaned)
def extract_domain_from_url(url: str) -> str:
return normalize_domain(url)
def validate_hex_color(color: str) -> str:
color = color.strip()
if not color.startswith("#"):
msg = "Color must start with #"
raise ValueError(msg)
if len(color) != HEX_COLOR_LENGTH:
msg = f"Color must be {HEX_COLOR_LENGTH} characters in the form #RRGGBB"
raise ValueError(msg)
hex_pattern = r"^#[0-9A-Fa-f]{6}$"
if not re.match(hex_pattern, color):
msg = "Color must use hexadecimal digits 0-9 and A-F"
raise ValueError(msg)
return color.upper()
_HOST_RE = re.compile(
r"^(?=.{1,253}$)([a-z0-9_](?:[a-z0-9_-]{0,62}[a-z0-9_])?\.)+[a-z0-9][a-z0-9-]{0,62}$"
)
def normalize_host(raw: str) -> str | None:
"""A bare host name, wildcard label and trailing dot stripped. None when it is not one."""
if not raw:
return None
name = raw.strip().lower().rstrip(".")
if name.startswith("*."):
name = name[2:]
if not name or "." not in name or " " in name or "/" in name or "@" in name:
return None
if not _HOST_RE.match(name):
return None
return name