mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-20 07:11:28 +02:00
Yogesh Ojha
e035ca24d9
A new AI Detection stage in the Analysis group runs julius v1.4.19 over every web asset root and names model servers, AI gateways, AI applications, cloud AI endpoints and MCP servers, with the models they list. julius's MCP probe runs alone against /mcp and /api/mcp. Batches of eight roots are paced to the scan's rate, AI ports are probed first, and a minute budget and an asset cap are reported when they cut. Results live on http_assets.ai_* and fold onto hosts and onto services per ip:port, so an IP target with no host rows still carries them. Web asset and service rows show the service, the host and service grammars take ai: and ai.model:, the dashboard and target page count services per product through /ports/ai, two built-in exposure rules flag AI services and unauthenticated model lists, and compare watches ai_services. The AI default ports join WEB_PORTS. julius ignores -f without a positional target, so targets go on stdin.
459 lines
16 KiB
Python
459 lines
16 KiB
Python
from __future__ import annotations
|
|
|
|
import re
|
|
from dataclasses import dataclass
|
|
from datetime import datetime
|
|
|
|
from sqlalchemy import (
|
|
and_,
|
|
cast,
|
|
exists,
|
|
false,
|
|
func,
|
|
literal,
|
|
or_,
|
|
select,
|
|
true,
|
|
union_all,
|
|
)
|
|
from sqlalchemy.dialects.postgresql import INET, JSONB
|
|
from sqlalchemy.dialects.postgresql import array as pg_array
|
|
|
|
from shared.definitions.ai_services import CATEGORY_LABELS as AI_CATEGORIES
|
|
from shared.definitions.asset_query import FLAGS, HOST_QUERY, FieldType, Op
|
|
from shared.definitions.domain_posture import QUERY_VALUES as POSTURE_VALUES
|
|
from shared.definitions.hygiene import QUERY_VALUES as HYGIENE_VALUES
|
|
from shared.models.endpoint import Endpoint
|
|
from shared.models.http_asset import HttpAsset
|
|
from shared.models.interest import InterestSignal
|
|
from shared.models.port import Port
|
|
from shared.models.subdomain import Subdomain
|
|
from shared.models.vulnerability import Vulnerability
|
|
|
|
from . import predicates as preds
|
|
from .ast import Compare, Or, QuerySyntaxError, Term
|
|
from .scope import QueryScope
|
|
from .terms import (
|
|
date_match,
|
|
folded_match,
|
|
int_coerce,
|
|
json_array_match,
|
|
negate,
|
|
number_match,
|
|
scaled_coerce,
|
|
string_match,
|
|
target_match,
|
|
tri_state,
|
|
)
|
|
from .values import asn_number, like, network, render_hash, status_range, tsquery
|
|
from .walk import as_compare, free_text_fields, walker
|
|
|
|
_IPV4_RE = re.compile(r"^[0-9]{1,3}(\.[0-9]{1,3}){3}$")
|
|
_HEADER_WEIGHT = "A"
|
|
_BODY_WEIGHT = "B"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class QueryContext:
|
|
scope: QueryScope
|
|
now: datetime
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Compiled:
|
|
where: object | None = None
|
|
asset: object | None = None
|
|
|
|
def flatten(self, ctx: QueryContext):
|
|
parts = [self.where] if self.where is not None else []
|
|
if self.asset is not None:
|
|
parts.append(preds.asset_match(ctx.scope, self.asset))
|
|
return and_(*parts) if parts else true()
|
|
|
|
|
|
def _ip(cmp: Compare):
|
|
branches = []
|
|
for raw in cmp.values:
|
|
cidr = network(raw)
|
|
if cidr is not None:
|
|
branches.append(_within(str(cidr)))
|
|
elif cmp.op is Op.EQ or _IPV4_RE.match(raw):
|
|
branches.append(func.jsonb_exists(cast(Subdomain.resolved_ips, JSONB), raw))
|
|
else:
|
|
branches.append(preds.ip_text().ilike(like(raw), escape="\\"))
|
|
matched = or_(*branches)
|
|
return negate(matched) if cmp.op is Op.NE else matched
|
|
|
|
|
|
def _within(cidr: str):
|
|
element = (
|
|
func.jsonb_array_elements_text(cast(Subdomain.resolved_ips, JSONB))
|
|
.table_valued("value")
|
|
.alias("resolved_ip")
|
|
)
|
|
return exists(
|
|
select(1)
|
|
.select_from(element)
|
|
.where(preds.inet_of(element.c.value).op("<<=")(cast(literal(cidr), INET)))
|
|
)
|
|
|
|
|
|
def _tsv(cmp: Compare, weight: str):
|
|
branches = []
|
|
for raw in cmp.values:
|
|
query = tsquery(raw, weight, prefix=not cmp.quoted)
|
|
if query:
|
|
branches.append(
|
|
HttpAsset.search_tsv.op("@@")(func.to_tsquery("simple", query))
|
|
)
|
|
if not branches:
|
|
return false()
|
|
matched = or_(*branches)
|
|
return negate(matched) if cmp.op is Op.NE else matched
|
|
|
|
|
|
def _header(cmp: Compare):
|
|
if cmp.sub:
|
|
name = cmp.sub.lower().replace("-", "_")
|
|
return string_match(
|
|
cast(HttpAsset.response_headers, JSONB).op("->>")(name), cmp
|
|
)
|
|
if cmp.quoted:
|
|
return string_match(HttpAsset.raw_response_header, cmp)
|
|
return _tsv(cmp, _HEADER_WEIGHT)
|
|
|
|
|
|
def _cert(cmp: Compare, ctx: QueryContext):
|
|
matched = or_(*[preds.cert_state(v.lower(), ctx.now) for v in cmp.values])
|
|
return negate(matched) if cmp.op is Op.NE else matched
|
|
|
|
|
|
def _status(cmp: Compare):
|
|
branches = []
|
|
numeric: list[str] = []
|
|
for raw in cmp.values:
|
|
bounds = status_range(raw)
|
|
if bounds is not None:
|
|
branches.append(
|
|
and_(
|
|
Subdomain.http_status >= bounds[0],
|
|
Subdomain.http_status <= bounds[1],
|
|
)
|
|
)
|
|
elif raw.lower() == "none":
|
|
branches.append(Subdomain.http_status.is_(None))
|
|
else:
|
|
numeric.append(raw)
|
|
if numeric:
|
|
positive = Compare(
|
|
name=cmp.name,
|
|
op=Op.MATCH if cmp.op is Op.NE else cmp.op,
|
|
values=tuple(numeric),
|
|
quoted=cmp.quoted,
|
|
sub=cmp.sub,
|
|
start=cmp.start,
|
|
end=cmp.end,
|
|
)
|
|
branches.append(number_match(Subdomain.http_status, positive, int_coerce(cmp)))
|
|
matched = or_(*branches)
|
|
if cmp.op is Op.NE:
|
|
return or_(Subdomain.http_status.is_(None), negate(matched))
|
|
return matched
|
|
|
|
|
|
def _flag(cmp: Compare, ctx: QueryContext):
|
|
branches = []
|
|
for raw in cmp.values:
|
|
name = raw.lower()
|
|
builder = _FLAG_BUILDERS.get(name)
|
|
if builder is None:
|
|
msg = f"Unknown flag {raw!r}."
|
|
hint = f"Try one of: {', '.join(FLAGS)}"
|
|
raise QuerySyntaxError(msg, cmp.start, cmp.end, hint)
|
|
branches.append(builder(ctx))
|
|
return or_(*branches)
|
|
|
|
|
|
_FLAG_BUILDERS = {
|
|
"live": lambda _ctx: preds.live(),
|
|
"web": lambda _ctx: preds.answered(),
|
|
"new": lambda ctx: preds.is_new(ctx.scope),
|
|
"resolved": lambda _ctx: preds.resolved(),
|
|
"auth": lambda _ctx: preds.auth(),
|
|
"cdn": lambda _ctx: Subdomain.is_cdn.is_(True),
|
|
"cloud": lambda ctx: preds.asset_match(ctx.scope, HttpAsset.cdn_type == "cloud"),
|
|
"waf": lambda _ctx: Subdomain.waf.isnot(None),
|
|
"screenshot": lambda _ctx: Subdomain.screenshot_path.isnot(None),
|
|
"important": lambda _ctx: Subdomain.is_important.is_(True),
|
|
"wildcard": lambda _ctx: Subdomain.is_wildcard.is_(True),
|
|
"issue": lambda ctx: preds.issues(ctx.now, ctx.scope),
|
|
"sensitive": lambda ctx: preds.sensitive(ctx.scope),
|
|
"http2": lambda ctx: preds.asset_match(
|
|
ctx.scope, HttpAsset.supports_http2.is_(True)
|
|
),
|
|
"redirect": lambda _ctx: and_(
|
|
Subdomain.final_url.isnot(None), Subdomain.final_url != Subdomain.http_url
|
|
),
|
|
"vulnerable": lambda ctx: preds.host_vuln(ctx.scope),
|
|
"exposed": lambda _ctx: preds.interesting(),
|
|
"interesting": lambda _ctx: preds.interesting(),
|
|
"kev": lambda ctx: preds.host_vuln(ctx.scope, Vulnerability.is_kev.is_(True)),
|
|
}
|
|
|
|
|
|
def _cdn(cmp: Compare):
|
|
state = tri_state(cmp)
|
|
if state is None:
|
|
return string_match(Subdomain.cdn_name, cmp)
|
|
return Subdomain.is_cdn.is_(state)
|
|
|
|
|
|
def _waf(cmp: Compare):
|
|
state = tri_state(cmp)
|
|
if state is None:
|
|
return string_match(Subdomain.waf, cmp)
|
|
return Subdomain.waf.isnot(None) if state else Subdomain.waf.is_(None)
|
|
|
|
|
|
def _issuer(cmp: Compare):
|
|
return or_(
|
|
string_match(HttpAsset.tls_issuer, cmp),
|
|
string_match(HttpAsset.tls_issuer_org, cmp),
|
|
string_match(HttpAsset.tls_issuer_cn, cmp),
|
|
)
|
|
|
|
|
|
def _url():
|
|
return func.coalesce(Subdomain.final_url, Subdomain.http_url)
|
|
|
|
|
|
def _endpoint_count(scope: QueryScope):
|
|
return (
|
|
select(func.count())
|
|
.select_from(Endpoint)
|
|
.where(scope.match(Endpoint.scan_id), Endpoint.host == Subdomain.name)
|
|
.correlate(Subdomain)
|
|
.scalar_subquery()
|
|
)
|
|
|
|
|
|
def _interest_kind(cmp: Compare):
|
|
matched = preds.interest_signal(
|
|
InterestSignal.kind.in_([v.lower() for v in cmp.values])
|
|
)
|
|
return negate(matched) if cmp.op is Op.NE else matched
|
|
|
|
|
|
def _interest_source(cmp: Compare):
|
|
matched = preds.interest_signal(
|
|
InterestSignal.source.in_([v.lower() for v in cmp.values])
|
|
)
|
|
return negate(matched) if cmp.op is Op.NE else matched
|
|
|
|
|
|
def _interest_band(cmp: Compare):
|
|
matched = Subdomain.interest_band.in_([v.lower() for v in cmp.values])
|
|
return negate(matched) if cmp.op is Op.NE else matched
|
|
|
|
|
|
def _hygiene(cmp: Compare):
|
|
for raw in cmp.values:
|
|
if raw.lower() not in HYGIENE_VALUES:
|
|
msg = f"Unknown hygiene check {raw!r}."
|
|
hint = f"Try one of: {', '.join(HYGIENE_VALUES)}"
|
|
raise QuerySyntaxError(msg, cmp.start, cmp.end, hint)
|
|
matched = preds.hygiene(list(cmp.values))
|
|
return negate(matched) if cmp.op is Op.NE else matched
|
|
|
|
|
|
def _posture(cmp: Compare):
|
|
for raw in cmp.values:
|
|
if raw.lower() not in POSTURE_VALUES:
|
|
msg = f"Unknown posture check {raw!r}."
|
|
hint = f"Try one of: {', '.join(POSTURE_VALUES)}"
|
|
raise QuerySyntaxError(msg, cmp.start, cmp.end, hint)
|
|
matched = preds.posture(list(cmp.values))
|
|
return negate(matched) if cmp.op is Op.NE else matched
|
|
|
|
|
|
def _ai(cmp: Compare, ctx: QueryContext):
|
|
services = cast(Subdomain.ai_services, JSONB)
|
|
state = tri_state(cmp)
|
|
if state is not None:
|
|
present = func.coalesce(func.jsonb_array_length(services), 0) > 0
|
|
matched = present if state else ~present
|
|
return negate(matched) if cmp.op is Op.NE else matched
|
|
if cmp.op in (Op.RE, Op.NRE):
|
|
return json_array_match(Subdomain.ai_services, cmp)
|
|
values = [v.lower() for v in cmp.values]
|
|
categories = [v for v in values if v in AI_CATEGORIES]
|
|
keys = [v for v in values if v not in AI_CATEGORIES]
|
|
parts = []
|
|
if keys:
|
|
parts.append(func.jsonb_exists_any(services, pg_array(keys)))
|
|
if categories:
|
|
parts.append(
|
|
preds.asset_match(ctx.scope, HttpAsset.ai_category.in_(categories))
|
|
)
|
|
matched = or_(*parts)
|
|
return negate(matched) if cmp.op is Op.NE else matched
|
|
|
|
|
|
def _ai_model(cmp: Compare):
|
|
state = tri_state(cmp)
|
|
if state is None:
|
|
return json_array_match(HttpAsset.ai_models, cmp)
|
|
present = func.coalesce(
|
|
func.jsonb_array_length(cast(HttpAsset.ai_models, JSONB)), 0
|
|
)
|
|
matched = present > 0 if state else present == 0
|
|
return negate(matched) if cmp.op is Op.NE else matched
|
|
|
|
|
|
_SUBDOMAIN_BUILDERS = {
|
|
"target": lambda c, _ctx: target_match(Subdomain.target_id, c),
|
|
"host": lambda c, _ctx: folded_match(Subdomain.name, c),
|
|
"url": lambda c, _ctx: string_match(_url(), c),
|
|
"cname": lambda c, _ctx: string_match(Subdomain.cname, c),
|
|
"source": lambda c, _ctx: json_array_match(Subdomain.sources, c),
|
|
"discovered": lambda c, ctx: date_match(
|
|
Subdomain.discovered_at, c, ctx.now, future=False
|
|
),
|
|
"status": lambda c, _ctx: _status(c),
|
|
"title": lambda c, _ctx: string_match(Subdomain.page_title, c),
|
|
"server": lambda c, _ctx: string_match(Subdomain.webserver, c),
|
|
"tech": lambda c, _ctx: json_array_match(Subdomain.tech, c),
|
|
"content_type": lambda c, _ctx: string_match(Subdomain.content_type, c),
|
|
"size": lambda c, _ctx: number_match(
|
|
Subdomain.content_length, c, scaled_coerce(c, FieldType.BYTES)
|
|
),
|
|
"time": lambda c, _ctx: number_match(
|
|
Subdomain.response_time, c, scaled_coerce(c, FieldType.DURATION)
|
|
),
|
|
"paths": lambda c, ctx: number_match(_endpoint_count(ctx.scope), c, int_coerce(c)),
|
|
"favicon": lambda c, _ctx: string_match(Subdomain.favicon_hash, c),
|
|
"screenshot": lambda c, _ctx: preds.renders_like(
|
|
Subdomain.screenshot_phash, render_hash(c.values[0], c.start, c.end)
|
|
),
|
|
"ip": lambda c, _ctx: _ip(c),
|
|
"asn": lambda c, _ctx: number_match(
|
|
Subdomain.asn, c, lambda raw: asn_number(raw, c.start, c.end)
|
|
),
|
|
"org": lambda c, _ctx: string_match(Subdomain.asn_org, c),
|
|
"cdn": lambda c, _ctx: _cdn(c),
|
|
"waf": lambda c, _ctx: _waf(c),
|
|
"port": lambda c, ctx: preds.port_match(
|
|
number_match(Port.number, c, int_coerce(c)), ctx.scope
|
|
),
|
|
"service": lambda c, ctx: preds.port_match(
|
|
string_match(Port.service_name, c), ctx.scope
|
|
),
|
|
"cert": _cert,
|
|
"cert.expires": lambda c, ctx: date_match(
|
|
Subdomain.tls_not_after, c, ctx.now, future=True
|
|
),
|
|
"vuln": lambda c, ctx: preds.host_vuln(
|
|
ctx.scope, string_match(Vulnerability.severity, c)
|
|
),
|
|
"exposure": lambda c, _ctx: _interest_kind(c),
|
|
"flagged": lambda c, _ctx: _interest_source(c),
|
|
"exposure_band": lambda c, _ctx: _interest_band(c),
|
|
"hygiene": lambda c, _ctx: _hygiene(c),
|
|
"posture": lambda c, _ctx: _posture(c),
|
|
"ai": _ai,
|
|
"cve": lambda c, ctx: preds.host_vuln(
|
|
ctx.scope, json_array_match(Vulnerability.cve_ids, c)
|
|
),
|
|
"is": _flag,
|
|
}
|
|
|
|
_ASSET_BUILDERS = {
|
|
"ai.model": lambda c, _ctx: _ai_model(c),
|
|
"path": lambda c, _ctx: string_match(HttpAsset.path, c),
|
|
"words": lambda c, _ctx: number_match(HttpAsset.words, c, int_coerce(c)),
|
|
"lines": lambda c, _ctx: number_match(HttpAsset.lines, c, int_coerce(c)),
|
|
"redirect": lambda c, _ctx: string_match(HttpAsset.location, c),
|
|
"content_hash": lambda c, _ctx: string_match(HttpAsset.content_hash, c),
|
|
"tracking": lambda c, _ctx: json_array_match(HttpAsset.tracking_ids, c),
|
|
"body": lambda c, _ctx: _tsv(c, _BODY_WEIGHT),
|
|
"header": lambda c, _ctx: _header(c),
|
|
"cert.cn": lambda c, _ctx: string_match(HttpAsset.tls_subject_cn, c),
|
|
"cert.san": lambda c, _ctx: json_array_match(HttpAsset.tls_sans, c),
|
|
"cert.issuer": lambda c, _ctx: _issuer(c),
|
|
"tls.version": lambda c, _ctx: string_match(HttpAsset.tls_version, c),
|
|
"jarm": lambda c, _ctx: string_match(HttpAsset.jarm, c),
|
|
"cert.fingerprint": lambda c, _ctx: string_match(HttpAsset.tls_fingerprint, c),
|
|
"header_hash": lambda c, _ctx: string_match(HttpAsset.header_hash, c),
|
|
}
|
|
|
|
_FREE_TEXT_BUILDERS = {
|
|
"cdn": lambda c, _ctx: string_match(Subdomain.cdn_name, c),
|
|
"waf": lambda c, _ctx: string_match(Subdomain.waf, c),
|
|
"url": lambda c, _ctx: string_match(Subdomain.final_url, c),
|
|
"favicon": lambda c, _ctx: Subdomain.favicon_hash == c.values[0],
|
|
}
|
|
|
|
|
|
def compile_compare(cmp: Compare, ctx: QueryContext) -> Compiled:
|
|
builder = _SUBDOMAIN_BUILDERS.get(cmp.name)
|
|
if builder is not None:
|
|
return Compiled(where=builder(cmp, ctx))
|
|
builder = _ASSET_BUILDERS.get(cmp.name)
|
|
if builder is not None:
|
|
return Compiled(asset=builder(cmp, ctx))
|
|
msg = f"Field {cmp.name!r} cannot be searched."
|
|
raise QuerySyntaxError(msg, cmp.start, cmp.end)
|
|
|
|
|
|
def compile_term(term: Term, ctx: QueryContext):
|
|
"""One id lookup per free-text field."""
|
|
branches = []
|
|
assets = []
|
|
for field in free_text_fields(HOST_QUERY):
|
|
cmp = as_compare(term, field)
|
|
override = _FREE_TEXT_BUILDERS.get(field)
|
|
if override is not None:
|
|
branches.append(override(cmp, ctx))
|
|
elif field in _ASSET_BUILDERS:
|
|
assets.append(_ASSET_BUILDERS[field](cmp, ctx))
|
|
else:
|
|
branches.append(_SUBDOMAIN_BUILDERS[field](cmp, ctx))
|
|
if assets:
|
|
branches.append(preds.asset_match(ctx.scope, or_(*assets)))
|
|
if not branches:
|
|
return false()
|
|
reach = ctx.scope.match(Subdomain.scan_id)
|
|
reachable = union_all(
|
|
*[
|
|
select(Subdomain.id).where(reach, branch).correlate(None)
|
|
for branch in branches
|
|
]
|
|
).subquery()
|
|
return Subdomain.id.in_(select(reachable.c.id))
|
|
|
|
|
|
def _compile_or(node: Or, ctx: QueryContext, walk):
|
|
"""Fold sibling asset predicates into one semijoin."""
|
|
parts, assets = [], []
|
|
for part in node.parts:
|
|
if isinstance(part, Compare):
|
|
compiled = compile_compare(part, ctx)
|
|
if compiled.where is None and compiled.asset is not None:
|
|
assets.append(compiled.asset)
|
|
continue
|
|
parts.append(compiled.flatten(ctx))
|
|
else:
|
|
parts.append(walk(part, ctx))
|
|
if assets:
|
|
parts.append(preds.asset_match(ctx.scope, or_(*assets)))
|
|
return or_(*parts) if parts else true()
|
|
|
|
|
|
compile_query = walker(
|
|
HOST_QUERY,
|
|
_SUBDOMAIN_BUILDERS,
|
|
term=compile_term,
|
|
compare=lambda cmp, ctx: compile_compare(cmp, ctx).flatten(ctx),
|
|
fold_or=_compile_or,
|
|
)
|