mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-20 07:11:28 +02:00
Yogesh OjhaandClaude Opus 5
cb1412ce90
reNgine 2 answered this with one global keyword list matched by `icontains`, and rebuilt that query once per table row — `get_is_interesting` was a SerializerMethodField, so a 5,000-host page ran the whole unindexable OR-chain 5,000 times. It was also dead until the user saved the settings form once, since `get_interesting_subdomains` returns none() with no custom_type row. The rule survives; everything around it is rebuilt. An interest rule IS a saved HOST_QUERY query, so there is no second matcher to keep in sync, and the keyword box compiles to `host:[…] or title:[…]` behind the same grammar. Signals persist to interest_signals, roll up in SQL onto subdomains.interest_score/band/kinds so sorting and `is:interesting` stay index-backed, and the scan carries a signature of the rules it was labelled against — a stale read refreshes in the background, so editing a keyword still re-labels every past scan without a read-time query. interest/ is a top-level package mounted into api and worker like stages/ and reports/, with providers auto-discovered from disk. Three ship: rules the keyword lists and 19 saved queries correlation what stands out from the estate, which no keyword can see ai a judgement, labelled as one everywhere it appears Correlation is the half a rule cannot express: an ASN carrying under 5% of the surface, software or a favicon on three hosts or fewer, the one asset answering directly while the rest sits behind a CDN. All gated on a minimum estate size, because rarity means nothing without a norm. On gov.cy: 226 flagged of 1,379 in 0.4s, 29 of them critical. AI sees hostname, status, page title and technology for hosts that answered, and nothing else — never a response body — deduped to three samples per identical response shape. It may answer only in a closed schema, a host it was not given is dropped rather than stored, and its sentence must state what was observed, never what is exploitable. Page titles are attacker-controlled, so they travel as JSON data with strip_control and a cap, never interpolated into the instruction, and nothing it returns triggers an action. Every AI signal stores its model and prompt version, and the UI never lets it read as an observation. Then the judgement graduates: suggest.py shows the model what it flagged and asks for the query that would catch it again without a model. Each proposal is compiled, counted against the scan and dropped if it errors, matches nothing or already exists; nothing is stored until a person presses Add, after which it runs on every keyless install with no cost. Alerts are delta-only — hosts the target has never flagged before, and only those in a high band or matched by a rule the user asked to be notified about. Four bugs found on the way. `dr` matched drones.gov.cy, so shipped hostname rules are anchored on label boundaries and false positives went from 6 hosts to 0. predicates.interest_signal aliased InterestSignal while its condition used the unaliased class, uncorrelating the EXISTS so every interest: field returned the unfiltered total. The correlation aggregates needed explicit output aliases or a bare `t` collided with the CTE and printed a UUID inside a sentence. And dismiss removed the signals while leaving the denormalised rank, so the row stayed in the list. The asset query compiler moves to shared/ because the worker has no api/ and has to compile a rule at finalize; api keeps a shim that aliases every submodule into sys.modules, so no existing import changes. The rule library lives at /interest, not in the Arsenal. The Arsenal is tools. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01NVh1CGtjDA33fupXjLGbk2
352 lines
7.8 KiB
Python
352 lines
7.8 KiB
Python
import shared.models._tztypes # patch datetime->timestamptz first
|
|
from shared.models.activity_log import ActivityLog, ActivityLogRead
|
|
from shared.models.ai import AiNarrative
|
|
from shared.models.api_key import (
|
|
APIKey,
|
|
APIKeyCreate,
|
|
APIKeyRead,
|
|
APIKeyUpdate,
|
|
ProviderInfo,
|
|
)
|
|
from shared.models.dns import (
|
|
DnsLookup,
|
|
DnsLookupRead,
|
|
DnsLookupSummary,
|
|
DnsRecord,
|
|
DnsRecordRead,
|
|
)
|
|
from shared.models.endpoint import (
|
|
Endpoint,
|
|
EndpointCoverage,
|
|
EndpointDetail,
|
|
EndpointFacets,
|
|
EndpointFilter,
|
|
EndpointPage,
|
|
EndpointRead,
|
|
EndpointSummary,
|
|
EndpointTree,
|
|
TreeNode,
|
|
)
|
|
from shared.models.http_asset import (
|
|
HttpAsset,
|
|
HttpAssetRead,
|
|
HttpAssetSummary,
|
|
)
|
|
from shared.models.instance_settings import (
|
|
InstanceSettings,
|
|
InstanceSettingsRead,
|
|
InstanceSettingsUpdate,
|
|
)
|
|
from shared.models.interest import (
|
|
InterestCatalog,
|
|
InterestDismissal,
|
|
InterestFilter,
|
|
InterestPage,
|
|
InterestRow,
|
|
InterestRule,
|
|
InterestRuleCreate,
|
|
InterestRuleRead,
|
|
InterestRuleUpdate,
|
|
InterestSignal,
|
|
InterestSummary,
|
|
RulePreview,
|
|
RuleSuggestion,
|
|
)
|
|
from shared.models.ip_address import (
|
|
IpAddress,
|
|
IpAddressRead,
|
|
IpAddressSummary,
|
|
TargetIpAddressRead,
|
|
)
|
|
from shared.models.ip_asn_range import IpAsnRange, IpCountryRange
|
|
from shared.models.notification import (
|
|
Notification,
|
|
NotificationCreate,
|
|
NotificationRead,
|
|
)
|
|
from shared.models.notification_channel import (
|
|
NotificationChannel,
|
|
NotificationChannelCreate,
|
|
NotificationChannelRead,
|
|
NotificationChannelTestResult,
|
|
NotificationChannelUpdate,
|
|
NotificationPreference,
|
|
)
|
|
from shared.models.organization import (
|
|
Organization,
|
|
OrganizationCreate,
|
|
OrganizationRead,
|
|
OrganizationSummary,
|
|
OrganizationUpdate,
|
|
)
|
|
from shared.models.port import Port, PortRead, PortSummary
|
|
from shared.models.project import Project, ProjectBase, ProjectCreate, ProjectRead
|
|
from shared.models.proxy import (
|
|
Proxy,
|
|
ProxyCreate,
|
|
ProxyEndpoint,
|
|
ProxyEndpointRead,
|
|
ProxyRead,
|
|
ProxyTestResult,
|
|
ProxyUpdate,
|
|
)
|
|
from shared.models.report import Report, ReportFont, ReportTemplate, ReportTheme
|
|
from shared.models.ripestat import (
|
|
RIPEStatAbuseContact,
|
|
RIPEStatAnnouncedPrefix,
|
|
RIPEStatASNNeighbour,
|
|
RIPEStatASOverview,
|
|
RIPEStatNetworkInfo,
|
|
RIPEStatPrefixOverview,
|
|
RIPEStatRelatedPrefix,
|
|
)
|
|
from shared.models.scan import (
|
|
Scan,
|
|
ScanCreate,
|
|
ScanRead,
|
|
)
|
|
from shared.models.scan_activity import ScanActivity, ScanActivityRead
|
|
from shared.models.scan_command import (
|
|
ScanCommand,
|
|
ScanCommandDetail,
|
|
ScanCommandRead,
|
|
)
|
|
from shared.models.scan_context import (
|
|
AuthConfig,
|
|
AuthHeader,
|
|
ScanContext,
|
|
ScanContextCreate,
|
|
ScanContextRead,
|
|
ScanContextUpdate,
|
|
)
|
|
from shared.models.scan_engine import (
|
|
EngineCatalog,
|
|
ScanEngine,
|
|
ScanEngineCreate,
|
|
ScanEngineRead,
|
|
ScanEngineUpdate,
|
|
)
|
|
from shared.models.scan_schedule import (
|
|
ScanSchedule,
|
|
ScanScheduleCreate,
|
|
ScanScheduleRead,
|
|
ScanScheduleUpdate,
|
|
ScheduleTargetRef,
|
|
)
|
|
from shared.models.subdomain import (
|
|
Subdomain,
|
|
SubdomainRead,
|
|
SubdomainSummary,
|
|
TargetSubdomainRead,
|
|
)
|
|
from shared.models.tag import Tag, TagBase, TagCreate, TagRead, TagSummary, TagUpdate
|
|
from shared.models.target import (
|
|
Target,
|
|
TargetBulkCreate,
|
|
TargetBulkCreateResponse,
|
|
TargetCreate,
|
|
TargetImportItem,
|
|
TargetImportRequest,
|
|
TargetImportResult,
|
|
TargetRead,
|
|
TargetType,
|
|
TargetUpdate,
|
|
TargetValidationRequest,
|
|
TargetValidationResponse,
|
|
)
|
|
from shared.models.user import User, UserBase, UserCreate, UserRead
|
|
from shared.models.viewdns import ViewDNSCache
|
|
from shared.models.vuln_template import (
|
|
TemplateFilter,
|
|
TemplateLibraryStats,
|
|
TemplatePage,
|
|
TemplateSelection,
|
|
VulnTemplate,
|
|
VulnTemplateRead,
|
|
VulnTemplateUpdate,
|
|
VulnTemplateUploadRequest,
|
|
)
|
|
from shared.models.vulnerability import (
|
|
ScanVulnerabilities,
|
|
Vulnerability,
|
|
VulnerabilityCoverage,
|
|
VulnerabilityFacets,
|
|
VulnerabilityFilter,
|
|
VulnerabilityPage,
|
|
VulnerabilityRead,
|
|
VulnerabilityTriage,
|
|
)
|
|
from shared.models.wordlist import (
|
|
Wordlist,
|
|
WordlistFile,
|
|
WordlistRead,
|
|
WordlistUpdate,
|
|
WordlistUploadRequest,
|
|
WordlistUploadResult,
|
|
)
|
|
|
|
TargetRead.model_rebuild()
|
|
|
|
__all__ = [
|
|
"APIKey",
|
|
"APIKeyCreate",
|
|
"APIKeyRead",
|
|
"APIKeyUpdate",
|
|
"ActivityLog",
|
|
"ActivityLogRead",
|
|
"AiNarrative",
|
|
"AuthConfig",
|
|
"AuthHeader",
|
|
"DnsLookup",
|
|
"DnsLookupRead",
|
|
"DnsLookupSummary",
|
|
"DnsRecord",
|
|
"DnsRecordRead",
|
|
"Endpoint",
|
|
"EndpointCoverage",
|
|
"EndpointDetail",
|
|
"EndpointFacets",
|
|
"EndpointFilter",
|
|
"EndpointPage",
|
|
"EndpointRead",
|
|
"EndpointSummary",
|
|
"EndpointTree",
|
|
"EngineCatalog",
|
|
"HttpAsset",
|
|
"HttpAssetRead",
|
|
"HttpAssetSummary",
|
|
"InstanceSettings",
|
|
"InstanceSettingsRead",
|
|
"InstanceSettingsUpdate",
|
|
"InterestCatalog",
|
|
"InterestDismissal",
|
|
"InterestFilter",
|
|
"InterestPage",
|
|
"InterestRow",
|
|
"InterestRule",
|
|
"InterestRuleCreate",
|
|
"InterestRuleRead",
|
|
"InterestRuleUpdate",
|
|
"InterestSignal",
|
|
"InterestSummary",
|
|
"IpAddress",
|
|
"IpAddressRead",
|
|
"IpAddressSummary",
|
|
"IpAsnRange",
|
|
"IpCountryRange",
|
|
"Notification",
|
|
"NotificationChannel",
|
|
"NotificationChannelCreate",
|
|
"NotificationChannelRead",
|
|
"NotificationChannelTestResult",
|
|
"NotificationChannelUpdate",
|
|
"NotificationCreate",
|
|
"NotificationPreference",
|
|
"NotificationRead",
|
|
"Organization",
|
|
"OrganizationCreate",
|
|
"OrganizationRead",
|
|
"OrganizationSummary",
|
|
"OrganizationUpdate",
|
|
"Port",
|
|
"PortRead",
|
|
"PortSummary",
|
|
"Project",
|
|
"ProjectBase",
|
|
"ProjectCreate",
|
|
"ProjectRead",
|
|
"ProviderInfo",
|
|
"Proxy",
|
|
"ProxyCreate",
|
|
"ProxyEndpoint",
|
|
"ProxyEndpointRead",
|
|
"ProxyRead",
|
|
"ProxyTestResult",
|
|
"ProxyUpdate",
|
|
"RIPEStatASNNeighbour",
|
|
"RIPEStatASOverview",
|
|
"RIPEStatAbuseContact",
|
|
"RIPEStatAnnouncedPrefix",
|
|
"RIPEStatNetworkInfo",
|
|
"RIPEStatPrefixOverview",
|
|
"RIPEStatRelatedPrefix",
|
|
"Report",
|
|
"ReportFont",
|
|
"ReportTemplate",
|
|
"ReportTheme",
|
|
"RulePreview",
|
|
"RuleSuggestion",
|
|
"Scan",
|
|
"ScanActivity",
|
|
"ScanActivityRead",
|
|
"ScanCommand",
|
|
"ScanCommandDetail",
|
|
"ScanCommandRead",
|
|
"ScanContext",
|
|
"ScanContextCreate",
|
|
"ScanContextRead",
|
|
"ScanContextUpdate",
|
|
"ScanCreate",
|
|
"ScanEngine",
|
|
"ScanEngineCreate",
|
|
"ScanEngineRead",
|
|
"ScanEngineUpdate",
|
|
"ScanRead",
|
|
"ScanSchedule",
|
|
"ScanScheduleCreate",
|
|
"ScanScheduleRead",
|
|
"ScanScheduleUpdate",
|
|
"ScanVulnerabilities",
|
|
"ScheduleTargetRef",
|
|
"Subdomain",
|
|
"SubdomainRead",
|
|
"SubdomainSummary",
|
|
"Tag",
|
|
"TagBase",
|
|
"TagCreate",
|
|
"TagRead",
|
|
"TagSummary",
|
|
"TagUpdate",
|
|
"Target",
|
|
"TargetBulkCreate",
|
|
"TargetBulkCreateResponse",
|
|
"TargetCreate",
|
|
"TargetImportItem",
|
|
"TargetImportRequest",
|
|
"TargetImportResult",
|
|
"TargetIpAddressRead",
|
|
"TargetRead",
|
|
"TargetSubdomainRead",
|
|
"TargetType",
|
|
"TargetUpdate",
|
|
"TargetValidationRequest",
|
|
"TargetValidationResponse",
|
|
"TemplateFilter",
|
|
"TemplateLibraryStats",
|
|
"TemplatePage",
|
|
"TemplateSelection",
|
|
"TreeNode",
|
|
"User",
|
|
"UserBase",
|
|
"UserCreate",
|
|
"UserRead",
|
|
"ViewDNSCache",
|
|
"VulnTemplate",
|
|
"VulnTemplateRead",
|
|
"VulnTemplateUpdate",
|
|
"VulnTemplateUploadRequest",
|
|
"Vulnerability",
|
|
"VulnerabilityCoverage",
|
|
"VulnerabilityFacets",
|
|
"VulnerabilityFilter",
|
|
"VulnerabilityPage",
|
|
"VulnerabilityRead",
|
|
"VulnerabilityTriage",
|
|
"Wordlist",
|
|
"WordlistFile",
|
|
"WordlistRead",
|
|
"WordlistUpdate",
|
|
"WordlistUploadRequest",
|
|
"WordlistUploadResult",
|
|
]
|