mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-21 15:51:30 +02:00
Yogesh OjhaandClaude Opus 5
cb1412ce90
reNgine 2 answered this with one global keyword list matched by `icontains`, and rebuilt that query once per table row — `get_is_interesting` was a SerializerMethodField, so a 5,000-host page ran the whole unindexable OR-chain 5,000 times. It was also dead until the user saved the settings form once, since `get_interesting_subdomains` returns none() with no custom_type row. The rule survives; everything around it is rebuilt. An interest rule IS a saved HOST_QUERY query, so there is no second matcher to keep in sync, and the keyword box compiles to `host:[…] or title:[…]` behind the same grammar. Signals persist to interest_signals, roll up in SQL onto subdomains.interest_score/band/kinds so sorting and `is:interesting` stay index-backed, and the scan carries a signature of the rules it was labelled against — a stale read refreshes in the background, so editing a keyword still re-labels every past scan without a read-time query. interest/ is a top-level package mounted into api and worker like stages/ and reports/, with providers auto-discovered from disk. Three ship: rules the keyword lists and 19 saved queries correlation what stands out from the estate, which no keyword can see ai a judgement, labelled as one everywhere it appears Correlation is the half a rule cannot express: an ASN carrying under 5% of the surface, software or a favicon on three hosts or fewer, the one asset answering directly while the rest sits behind a CDN. All gated on a minimum estate size, because rarity means nothing without a norm. On gov.cy: 226 flagged of 1,379 in 0.4s, 29 of them critical. AI sees hostname, status, page title and technology for hosts that answered, and nothing else — never a response body — deduped to three samples per identical response shape. It may answer only in a closed schema, a host it was not given is dropped rather than stored, and its sentence must state what was observed, never what is exploitable. Page titles are attacker-controlled, so they travel as JSON data with strip_control and a cap, never interpolated into the instruction, and nothing it returns triggers an action. Every AI signal stores its model and prompt version, and the UI never lets it read as an observation. Then the judgement graduates: suggest.py shows the model what it flagged and asks for the query that would catch it again without a model. Each proposal is compiled, counted against the scan and dropped if it errors, matches nothing or already exists; nothing is stored until a person presses Add, after which it runs on every keyless install with no cost. Alerts are delta-only — hosts the target has never flagged before, and only those in a high band or matched by a rule the user asked to be notified about. Four bugs found on the way. `dr` matched drones.gov.cy, so shipped hostname rules are anchored on label boundaries and false positives went from 6 hosts to 0. predicates.interest_signal aliased InterestSignal while its condition used the unaliased class, uncorrelating the EXISTS so every interest: field returned the unfiltered total. The correlation aggregates needed explicit output aliases or a bare `t` collided with the CTE and printed a UUID inside a sentence. And dismiss removed the signals while leaving the denormalised rank, so the row stayed in the list. The asset query compiler moves to shared/ because the worker has no api/ and has to compile a rule at finalize; api keeps a shim that aliases every submodule into sys.modules, so no existing import changes. The rule library lives at /interest, not in the Arsenal. The Arsenal is tools. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01NVh1CGtjDA33fupXjLGbk2
276 lines
8.3 KiB
Python
276 lines
8.3 KiB
Python
import uuid
|
|
from datetime import datetime
|
|
|
|
from pydantic import BaseModel
|
|
from pydantic import Field as PField
|
|
from sqlalchemy import Column, Index
|
|
from sqlalchemy.types import JSON
|
|
from sqlmodel import Field, SQLModel, UniqueConstraint
|
|
|
|
from shared.definitions.interest import (
|
|
MAX_EVIDENCE,
|
|
MAX_REASON,
|
|
MAX_RULE_NAME,
|
|
InterestBand,
|
|
InterestSource,
|
|
RuleMode,
|
|
)
|
|
from shared.utils.datetime import utc_now
|
|
|
|
|
|
class InterestRule(SQLModel, table=True):
|
|
__tablename__ = "interest_rules"
|
|
__table_args__ = (
|
|
Index("ix_interest_rules_project_enabled", "project_id", "enabled"),
|
|
)
|
|
|
|
id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True, index=True)
|
|
project_id: uuid.UUID | None = Field(
|
|
default=None, foreign_key="projects.id", index=True, ondelete="CASCADE"
|
|
)
|
|
name: str = Field(max_length=MAX_RULE_NAME)
|
|
description: str | None = Field(default=None, max_length=300)
|
|
mode: str = Field(default=RuleMode.QUERY.value, max_length=16)
|
|
query: str = Field(default="", max_length=2000)
|
|
keywords: list = Field(default_factory=list, sa_column=Column(JSON, nullable=False))
|
|
keyword_fields: list = Field(
|
|
default_factory=list, sa_column=Column(JSON, nullable=False)
|
|
)
|
|
live_only: bool = Field(default=False)
|
|
kind: str = Field(max_length=40)
|
|
weight: int | None = Field(default=None)
|
|
enabled: bool = Field(default=True, index=True)
|
|
builtin: bool = Field(default=False, index=True)
|
|
notify: bool = Field(default=False)
|
|
created_by: uuid.UUID | None = Field(default=None, foreign_key="users.id")
|
|
created_at: datetime = Field(default_factory=utc_now)
|
|
updated_at: datetime = Field(default_factory=utc_now)
|
|
|
|
|
|
class InterestSignal(SQLModel, table=True):
|
|
__tablename__ = "interest_signals"
|
|
__table_args__ = (
|
|
UniqueConstraint(
|
|
"scan_id", "subdomain_id", "source", "key", name="uq_interest_signal"
|
|
),
|
|
Index("ix_interest_signals_scan_score", "scan_id", "weight"),
|
|
)
|
|
|
|
id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True, index=True)
|
|
scan_id: uuid.UUID = Field(foreign_key="scans.id", index=True, ondelete="CASCADE")
|
|
target_id: uuid.UUID = Field(
|
|
foreign_key="targets.id", index=True, ondelete="CASCADE"
|
|
)
|
|
project_id: uuid.UUID = Field(foreign_key="projects.id", index=True)
|
|
subdomain_id: uuid.UUID = Field(
|
|
foreign_key="subdomains.id", index=True, ondelete="CASCADE"
|
|
)
|
|
host: str = Field(max_length=500, index=True)
|
|
|
|
source: str = Field(max_length=16, index=True)
|
|
key: str = Field(max_length=120)
|
|
kind: str = Field(max_length=40, index=True)
|
|
weight: int = Field(default=0)
|
|
label: str = Field(default="", max_length=MAX_RULE_NAME)
|
|
reason: str = Field(default="", max_length=MAX_REASON)
|
|
evidence: str | None = Field(default=None, max_length=MAX_EVIDENCE)
|
|
rule_id: uuid.UUID | None = Field(default=None, index=True)
|
|
model: str | None = Field(default=None, max_length=80)
|
|
prompt_version: str | None = Field(default=None, max_length=16)
|
|
created_at: datetime = Field(default_factory=utc_now)
|
|
|
|
|
|
class InterestDismissal(SQLModel, table=True):
|
|
__tablename__ = "interest_dismissals"
|
|
__table_args__ = (
|
|
UniqueConstraint("target_id", "host", "kind", name="uq_interest_dismissal"),
|
|
)
|
|
|
|
id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True, index=True)
|
|
target_id: uuid.UUID = Field(
|
|
foreign_key="targets.id", index=True, ondelete="CASCADE"
|
|
)
|
|
project_id: uuid.UUID = Field(foreign_key="projects.id", index=True)
|
|
host: str = Field(max_length=500, index=True)
|
|
kind: str = Field(default="", max_length=40)
|
|
note: str | None = Field(default=None, max_length=300)
|
|
created_by: uuid.UUID | None = Field(default=None, foreign_key="users.id")
|
|
created_at: datetime = Field(default_factory=utc_now)
|
|
|
|
|
|
class InterestRuleCreate(BaseModel):
|
|
name: str = PField(min_length=1, max_length=MAX_RULE_NAME)
|
|
description: str | None = None
|
|
mode: str = RuleMode.QUERY.value
|
|
query: str = ""
|
|
keywords: list[str] = PField(default_factory=list)
|
|
keyword_fields: list[str] = PField(default_factory=list)
|
|
live_only: bool = False
|
|
kind: str
|
|
weight: int | None = None
|
|
enabled: bool = True
|
|
notify: bool = False
|
|
project_id: uuid.UUID | None = None
|
|
|
|
|
|
class InterestRuleUpdate(BaseModel):
|
|
name: str | None = None
|
|
description: str | None = None
|
|
query: str | None = None
|
|
keywords: list[str] | None = None
|
|
keyword_fields: list[str] | None = None
|
|
live_only: bool | None = None
|
|
kind: str | None = None
|
|
weight: int | None = None
|
|
enabled: bool | None = None
|
|
notify: bool | None = None
|
|
|
|
|
|
class InterestRuleRead(BaseModel):
|
|
id: uuid.UUID
|
|
project_id: uuid.UUID | None = None
|
|
name: str
|
|
description: str | None = None
|
|
mode: str
|
|
query: str
|
|
keywords: list[str] = PField(default_factory=list)
|
|
keyword_fields: list[str] = PField(default_factory=list)
|
|
live_only: bool = False
|
|
kind: str
|
|
kind_label: str = ""
|
|
weight: int
|
|
enabled: bool
|
|
builtin: bool
|
|
notify: bool
|
|
updated_at: datetime
|
|
matches: int | None = None
|
|
error: str | None = None
|
|
|
|
|
|
class SignalRead(BaseModel):
|
|
source: str
|
|
kind: str
|
|
kind_label: str
|
|
label: str
|
|
reason: str
|
|
evidence: str | None = None
|
|
weight: int
|
|
rule_id: uuid.UUID | None = None
|
|
model: str | None = None
|
|
judgement: bool = False
|
|
|
|
|
|
class InterestRow(BaseModel):
|
|
subdomain_id: uuid.UUID
|
|
host: str
|
|
score: int
|
|
band: str
|
|
kinds: list[str] = PField(default_factory=list)
|
|
signals: list[SignalRead] = PField(default_factory=list)
|
|
sources: list[str] = PField(default_factory=list)
|
|
http_status: int | None = None
|
|
page_title: str | None = None
|
|
tech: list[str] = PField(default_factory=list)
|
|
webserver: str | None = None
|
|
resolved_ips: list[str] = PField(default_factory=list)
|
|
asn: int | None = None
|
|
asn_org: str | None = None
|
|
is_cdn: bool = False
|
|
screenshot_path: str | None = None
|
|
is_new: bool = False
|
|
dismissed: bool = False
|
|
|
|
|
|
class InterestSummary(BaseModel):
|
|
total: int = 0
|
|
bands: dict[str, int] = PField(default_factory=dict)
|
|
sources: dict[str, int] = PField(default_factory=dict)
|
|
kinds: dict[str, int] = PField(default_factory=dict)
|
|
dismissed: int = 0
|
|
judged_hosts: int = 0
|
|
judged_at: datetime | None = None
|
|
model: str | None = None
|
|
ai_available: bool = False
|
|
ai_enabled: bool = False
|
|
stale: bool = False
|
|
|
|
|
|
class InterestFilter(BaseModel):
|
|
q: str | None = PField(default=None, max_length=200)
|
|
bands: list[str] = PField(default_factory=list, max_length=8)
|
|
sources: list[str] = PField(default_factory=list, max_length=8)
|
|
kinds: list[str] = PField(default_factory=list, max_length=40)
|
|
sort: str = "score"
|
|
order: str = "desc"
|
|
limit: int = PField(default=50, ge=1, le=200)
|
|
offset: int = PField(default=0, ge=0, le=100_000)
|
|
|
|
|
|
class InterestPage(BaseModel):
|
|
rows: list[InterestRow] = PField(default_factory=list)
|
|
total: int = 0
|
|
summary: InterestSummary = PField(default_factory=InterestSummary)
|
|
|
|
|
|
class KindEntry(BaseModel):
|
|
key: str
|
|
label: str
|
|
help: str
|
|
weight: int
|
|
tone: str
|
|
|
|
|
|
class SourceEntry(BaseModel):
|
|
key: str
|
|
label: str
|
|
help: str
|
|
judgement: bool = False
|
|
|
|
|
|
class BandEntry(BaseModel):
|
|
key: str
|
|
label: str
|
|
tone: str
|
|
floor: int
|
|
|
|
|
|
class InterestCatalog(BaseModel):
|
|
kinds: list[KindEntry] = PField(default_factory=list)
|
|
sources: list[SourceEntry] = PField(default_factory=list)
|
|
bands: list[BandEntry] = PField(default_factory=list)
|
|
modes: dict[str, str] = PField(default_factory=dict)
|
|
keyword_fields: dict[str, str] = PField(default_factory=dict)
|
|
max_score: int = 0
|
|
providers: list[str] = PField(default_factory=list)
|
|
|
|
|
|
class RulePreview(BaseModel):
|
|
matches: int = 0
|
|
capped: bool = False
|
|
error: str | None = None
|
|
sample: list[str] = PField(default_factory=list)
|
|
|
|
|
|
class RuleSuggestion(BaseModel):
|
|
name: str
|
|
kind: str
|
|
kind_label: str = ""
|
|
query: str
|
|
reason: str = ""
|
|
matches: int = 0
|
|
|
|
|
|
class DismissRequest(BaseModel):
|
|
host: str = PField(min_length=1, max_length=500)
|
|
target_id: uuid.UUID
|
|
kind: str = ""
|
|
note: str | None = None
|
|
|
|
|
|
class JudgeRequest(BaseModel):
|
|
force: bool = False
|
|
|
|
|
|
DEFAULT_BAND = InterestBand.NOTABLE.value
|
|
DEFAULT_SOURCE = InterestSource.RULE.value
|