mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-20 15:21:30 +02:00
Yogesh Ojha
ca80dc8ba0
A Lookalike domains stage in the Analysis group generates permutations of the target's registrable domain with dnstwist, resolves them with dnsx (a, aaaa, mx, ns, one retry of silent names) and keeps the registered ones. A name whose only answer is its TLD's wildcard address is dropped. Registered lookalikes are fetched and compared with the target's page (ssdeep), and the highest ranked are read over RDAP for registrar and creation date. Passive intensity skips the fetch. Each lookalike carries one verdict: linked to target, similar page, MX configured, parked, web content or registered. Linked is a redirect to the apex, non-shared nameservers in common with it, or a project target. Placeholder MX values and domain marketplace landers are not counted as mail or web content. Rows live in lookalike_domains per scan; review state lives in lookalike_triage per target and domain, so it carries across runs. The scan overview and the target page show a lookalike line beside the associated domains, the target page adds a cell, and a sheet lists the lookalikes with the changed characters marked, per-verdict filters, records per row and review actions. dnstwist and ppdeep join the tools package.
543 lines
12 KiB
Python
543 lines
12 KiB
Python
import shared.models._tztypes # patch datetime->timestamptz first
|
|
from shared.models.activity_log import ActivityLog, ActivityLogRead
|
|
from shared.models.ai import AiNarrative
|
|
from shared.models.api_key import (
|
|
APIKey,
|
|
APIKeyCreate,
|
|
APIKeyRead,
|
|
APIKeyUpdate,
|
|
ProviderInfo,
|
|
)
|
|
from shared.models.compare import (
|
|
ChangeField,
|
|
ChangeRow,
|
|
ChangeRows,
|
|
ComparableRun,
|
|
CoverageLine,
|
|
DimensionDelta,
|
|
DimensionVerdict,
|
|
RunDifference,
|
|
RunSide,
|
|
ScanComparison,
|
|
ScreenshotPair,
|
|
SettingDiff,
|
|
StageDiff,
|
|
)
|
|
from shared.models.connector import (
|
|
ActionRead,
|
|
ActionRequest,
|
|
AddTargetRequest,
|
|
CandidatePage,
|
|
CandidateRead,
|
|
Connector,
|
|
ConnectorAction,
|
|
ConnectorCandidate,
|
|
ConnectorCreate,
|
|
ConnectorCreated,
|
|
ConnectorHost,
|
|
ConnectorRead,
|
|
ConnectorScope,
|
|
ConnectorUpdate,
|
|
DiscoveredDomain,
|
|
FindingRecorded,
|
|
FindingReport,
|
|
HostFacts,
|
|
IngestItem,
|
|
IngestRequest,
|
|
IngestResult,
|
|
NoticeRead,
|
|
TargetAdded,
|
|
TargetOption,
|
|
)
|
|
from shared.models.crosslink import CrossLink, CrossLinkPeer
|
|
from shared.models.dns import (
|
|
DnsLookup,
|
|
DnsLookupRead,
|
|
DnsLookupSummary,
|
|
DnsRecord,
|
|
DnsRecordRead,
|
|
)
|
|
from shared.models.domain_posture import (
|
|
DomainPosture,
|
|
DomainPostureRead,
|
|
DomainPostureSummary,
|
|
PostureCheckCount,
|
|
)
|
|
from shared.models.endpoint import (
|
|
Endpoint,
|
|
EndpointCoverage,
|
|
EndpointDetail,
|
|
EndpointFacets,
|
|
EndpointFilter,
|
|
EndpointPage,
|
|
EndpointRead,
|
|
EndpointSummary,
|
|
EndpointTree,
|
|
TreeNode,
|
|
)
|
|
from shared.models.export import Export, ExportCreate, ExportRead
|
|
from shared.models.http_asset import (
|
|
HttpAsset,
|
|
HttpAssetRead,
|
|
HttpAssetSummary,
|
|
)
|
|
from shared.models.instance_settings import (
|
|
InstanceSettings,
|
|
InstanceSettingsRead,
|
|
InstanceSettingsUpdate,
|
|
)
|
|
from shared.models.interest import (
|
|
InterestCatalog,
|
|
InterestDismissal,
|
|
InterestFilter,
|
|
InterestPage,
|
|
InterestRow,
|
|
InterestRule,
|
|
InterestRuleCreate,
|
|
InterestRuleRead,
|
|
InterestRuleUpdate,
|
|
InterestSignal,
|
|
InterestSummary,
|
|
RulePreview,
|
|
RuleSuggestion,
|
|
)
|
|
from shared.models.ip_address import (
|
|
IpAddress,
|
|
IpAddressRead,
|
|
IpAddressSummary,
|
|
TargetIpAddressRead,
|
|
)
|
|
from shared.models.ip_asn_range import IpAsnRange, IpCountryRange
|
|
from shared.models.lookalike import (
|
|
LookalikeDomain,
|
|
LookalikeRead,
|
|
LookalikeSummary,
|
|
LookalikeTriage,
|
|
)
|
|
from shared.models.note import (
|
|
Note,
|
|
NoteCount,
|
|
NoteCreate,
|
|
NoteRead,
|
|
NoteTag,
|
|
NoteUpdate,
|
|
)
|
|
from shared.models.notification import (
|
|
Notification,
|
|
NotificationCreate,
|
|
NotificationRead,
|
|
)
|
|
from shared.models.notification_channel import (
|
|
NotificationChannel,
|
|
NotificationChannelCreate,
|
|
NotificationChannelRead,
|
|
NotificationChannelTestResult,
|
|
NotificationChannelUpdate,
|
|
NotificationPreference,
|
|
)
|
|
from shared.models.organization import (
|
|
Organization,
|
|
OrganizationCreate,
|
|
OrganizationRead,
|
|
OrganizationSummary,
|
|
OrganizationUpdate,
|
|
)
|
|
from shared.models.port import Port, PortRead, PortSummary
|
|
from shared.models.project import Project, ProjectBase, ProjectCreate, ProjectRead
|
|
from shared.models.proxy import (
|
|
Proxy,
|
|
ProxyCreate,
|
|
ProxyEndpoint,
|
|
ProxyEndpointRead,
|
|
ProxyRead,
|
|
ProxyTestResult,
|
|
ProxyUpdate,
|
|
)
|
|
from shared.models.report import Report, ReportFont, ReportTemplate, ReportTheme
|
|
from shared.models.ripestat import (
|
|
RIPEStatAbuseContact,
|
|
RIPEStatAnnouncedPrefix,
|
|
RIPEStatASNNeighbour,
|
|
RIPEStatASOverview,
|
|
RIPEStatNetworkInfo,
|
|
RIPEStatPrefixOverview,
|
|
RIPEStatRelatedPrefix,
|
|
)
|
|
from shared.models.scan import (
|
|
Scan,
|
|
ScanCreate,
|
|
ScanRead,
|
|
)
|
|
from shared.models.scan_activity import ScanActivity, ScanActivityRead
|
|
from shared.models.scan_command import (
|
|
ScanCommand,
|
|
ScanCommandDetail,
|
|
ScanCommandRead,
|
|
)
|
|
from shared.models.scan_context import (
|
|
AuthConfig,
|
|
AuthHeader,
|
|
ScanContext,
|
|
ScanContextCreate,
|
|
ScanContextRead,
|
|
ScanContextUpdate,
|
|
)
|
|
from shared.models.scan_delta import ScanDelta, ScanRetired, ScanRevision
|
|
from shared.models.scan_engine import (
|
|
EngineCatalog,
|
|
ScanEngine,
|
|
ScanEngineCreate,
|
|
ScanEngineRead,
|
|
ScanEngineUpdate,
|
|
)
|
|
from shared.models.scan_schedule import (
|
|
ScanSchedule,
|
|
ScanScheduleCreate,
|
|
ScanScheduleRead,
|
|
ScanScheduleUpdate,
|
|
ScheduleTargetRef,
|
|
)
|
|
from shared.models.scan_surface import (
|
|
AssetSurface,
|
|
ScanSurfaceItem,
|
|
SurfaceItemRead,
|
|
SurfaceSummary,
|
|
SurfaceTierCount,
|
|
)
|
|
from shared.models.secret import (
|
|
Secret,
|
|
SecretCoverage,
|
|
SecretSighting,
|
|
)
|
|
from shared.models.software import (
|
|
NvdCpeMatch,
|
|
NvdCve,
|
|
SoftwareComponentRead,
|
|
SoftwareCoverage,
|
|
SoftwareCve,
|
|
SoftwareCveRead,
|
|
SoftwareFacet,
|
|
SoftwareFacets,
|
|
SoftwareFilter,
|
|
SoftwarePage,
|
|
)
|
|
from shared.models.subdomain import (
|
|
Subdomain,
|
|
SubdomainRead,
|
|
SubdomainSummary,
|
|
TargetSubdomainRead,
|
|
)
|
|
from shared.models.tag import Tag, TagBase, TagCreate, TagRead, TagSummary, TagUpdate
|
|
from shared.models.target import (
|
|
Target,
|
|
TargetBulkCreate,
|
|
TargetBulkCreateResponse,
|
|
TargetCreate,
|
|
TargetImportItem,
|
|
TargetImportRequest,
|
|
TargetImportResult,
|
|
TargetRead,
|
|
TargetType,
|
|
TargetUpdate,
|
|
TargetValidationRequest,
|
|
TargetValidationResponse,
|
|
)
|
|
from shared.models.target_seed import (
|
|
TargetSeed,
|
|
TargetSeedRead,
|
|
TargetSeedResult,
|
|
TargetSeedWrite,
|
|
)
|
|
from shared.models.user import User, UserBase, UserCreate, UserRead
|
|
from shared.models.viewdns import ViewDNSCache
|
|
from shared.models.vuln_template import (
|
|
TemplateFilter,
|
|
TemplateLibraryStats,
|
|
TemplatePage,
|
|
TemplateSelection,
|
|
VulnTemplate,
|
|
VulnTemplateRead,
|
|
VulnTemplateUpdate,
|
|
VulnTemplateUploadRequest,
|
|
)
|
|
from shared.models.vulnerability import (
|
|
ScanVulnerabilities,
|
|
Vulnerability,
|
|
VulnerabilityCoverage,
|
|
VulnerabilityFacets,
|
|
VulnerabilityFilter,
|
|
VulnerabilityPage,
|
|
VulnerabilityRead,
|
|
VulnerabilityTriage,
|
|
)
|
|
from shared.models.watch import (
|
|
ProgramWatch,
|
|
UserMark,
|
|
WatchCreate,
|
|
WatchEvent,
|
|
WatchHost,
|
|
WatchRead,
|
|
WatchUpdate,
|
|
)
|
|
from shared.models.wordlist import (
|
|
Wordlist,
|
|
WordlistFile,
|
|
WordlistRead,
|
|
WordlistUpdate,
|
|
WordlistUploadRequest,
|
|
WordlistUploadResult,
|
|
)
|
|
|
|
TargetRead.model_rebuild()
|
|
|
|
__all__ = [
|
|
"APIKey",
|
|
"APIKeyCreate",
|
|
"APIKeyRead",
|
|
"APIKeyUpdate",
|
|
"ActionRead",
|
|
"ActionRequest",
|
|
"ActivityLog",
|
|
"ActivityLogRead",
|
|
"AddTargetRequest",
|
|
"AiNarrative",
|
|
"AssetSurface",
|
|
"AuthConfig",
|
|
"AuthHeader",
|
|
"CandidatePage",
|
|
"CandidateRead",
|
|
"ChangeField",
|
|
"ChangeRow",
|
|
"ChangeRows",
|
|
"ComparableRun",
|
|
"Connector",
|
|
"ConnectorAction",
|
|
"ConnectorCandidate",
|
|
"ConnectorCreate",
|
|
"ConnectorCreated",
|
|
"ConnectorHost",
|
|
"ConnectorRead",
|
|
"ConnectorScope",
|
|
"ConnectorUpdate",
|
|
"CoverageLine",
|
|
"CrossLink",
|
|
"CrossLinkPeer",
|
|
"DimensionDelta",
|
|
"DimensionVerdict",
|
|
"DiscoveredDomain",
|
|
"DnsLookup",
|
|
"DnsLookupRead",
|
|
"DnsLookupSummary",
|
|
"DnsRecord",
|
|
"DnsRecordRead",
|
|
"DomainPosture",
|
|
"DomainPostureRead",
|
|
"DomainPostureSummary",
|
|
"Endpoint",
|
|
"EndpointCoverage",
|
|
"EndpointDetail",
|
|
"EndpointFacets",
|
|
"EndpointFilter",
|
|
"EndpointPage",
|
|
"EndpointRead",
|
|
"EndpointSummary",
|
|
"EndpointTree",
|
|
"EngineCatalog",
|
|
"Export",
|
|
"ExportCreate",
|
|
"ExportRead",
|
|
"FindingRecorded",
|
|
"FindingReport",
|
|
"HostFacts",
|
|
"HttpAsset",
|
|
"HttpAssetRead",
|
|
"HttpAssetSummary",
|
|
"IngestItem",
|
|
"IngestRequest",
|
|
"IngestResult",
|
|
"InstanceSettings",
|
|
"InstanceSettingsRead",
|
|
"InstanceSettingsUpdate",
|
|
"InterestCatalog",
|
|
"InterestDismissal",
|
|
"InterestFilter",
|
|
"InterestPage",
|
|
"InterestRow",
|
|
"InterestRule",
|
|
"InterestRuleCreate",
|
|
"InterestRuleRead",
|
|
"InterestRuleUpdate",
|
|
"InterestSignal",
|
|
"InterestSummary",
|
|
"IpAddress",
|
|
"IpAddressRead",
|
|
"IpAddressSummary",
|
|
"IpAsnRange",
|
|
"IpCountryRange",
|
|
"LookalikeDomain",
|
|
"LookalikeRead",
|
|
"LookalikeSummary",
|
|
"LookalikeTriage",
|
|
"Note",
|
|
"NoteCount",
|
|
"NoteCreate",
|
|
"NoteRead",
|
|
"NoteTag",
|
|
"NoteUpdate",
|
|
"NoticeRead",
|
|
"Notification",
|
|
"NotificationChannel",
|
|
"NotificationChannelCreate",
|
|
"NotificationChannelRead",
|
|
"NotificationChannelTestResult",
|
|
"NotificationChannelUpdate",
|
|
"NotificationCreate",
|
|
"NotificationPreference",
|
|
"NotificationRead",
|
|
"NvdCpeMatch",
|
|
"NvdCve",
|
|
"Organization",
|
|
"OrganizationCreate",
|
|
"OrganizationRead",
|
|
"OrganizationSummary",
|
|
"OrganizationUpdate",
|
|
"Port",
|
|
"PortRead",
|
|
"PortSummary",
|
|
"PostureCheckCount",
|
|
"ProgramWatch",
|
|
"Project",
|
|
"ProjectBase",
|
|
"ProjectCreate",
|
|
"ProjectRead",
|
|
"ProviderInfo",
|
|
"Proxy",
|
|
"ProxyCreate",
|
|
"ProxyEndpoint",
|
|
"ProxyEndpointRead",
|
|
"ProxyRead",
|
|
"ProxyTestResult",
|
|
"ProxyUpdate",
|
|
"RIPEStatASNNeighbour",
|
|
"RIPEStatASOverview",
|
|
"RIPEStatAbuseContact",
|
|
"RIPEStatAnnouncedPrefix",
|
|
"RIPEStatNetworkInfo",
|
|
"RIPEStatPrefixOverview",
|
|
"RIPEStatRelatedPrefix",
|
|
"Report",
|
|
"ReportFont",
|
|
"ReportTemplate",
|
|
"ReportTheme",
|
|
"RulePreview",
|
|
"RuleSuggestion",
|
|
"RunDifference",
|
|
"RunSide",
|
|
"Scan",
|
|
"ScanActivity",
|
|
"ScanActivityRead",
|
|
"ScanCommand",
|
|
"ScanCommandDetail",
|
|
"ScanCommandRead",
|
|
"ScanComparison",
|
|
"ScanContext",
|
|
"ScanContextCreate",
|
|
"ScanContextRead",
|
|
"ScanContextUpdate",
|
|
"ScanCreate",
|
|
"ScanDelta",
|
|
"ScanEngine",
|
|
"ScanEngineCreate",
|
|
"ScanEngineRead",
|
|
"ScanEngineUpdate",
|
|
"ScanRead",
|
|
"ScanRetired",
|
|
"ScanRevision",
|
|
"ScanSchedule",
|
|
"ScanScheduleCreate",
|
|
"ScanScheduleRead",
|
|
"ScanScheduleUpdate",
|
|
"ScanSurfaceItem",
|
|
"ScanVulnerabilities",
|
|
"ScheduleTargetRef",
|
|
"ScreenshotPair",
|
|
"Secret",
|
|
"SecretCoverage",
|
|
"SecretSighting",
|
|
"SettingDiff",
|
|
"SoftwareComponentRead",
|
|
"SoftwareCoverage",
|
|
"SoftwareCve",
|
|
"SoftwareCveRead",
|
|
"SoftwareFacet",
|
|
"SoftwareFacets",
|
|
"SoftwareFilter",
|
|
"SoftwarePage",
|
|
"StageDiff",
|
|
"Subdomain",
|
|
"SubdomainRead",
|
|
"SubdomainSummary",
|
|
"SurfaceItemRead",
|
|
"SurfaceSummary",
|
|
"SurfaceTierCount",
|
|
"Tag",
|
|
"TagBase",
|
|
"TagCreate",
|
|
"TagRead",
|
|
"TagSummary",
|
|
"TagUpdate",
|
|
"Target",
|
|
"TargetAdded",
|
|
"TargetBulkCreate",
|
|
"TargetBulkCreateResponse",
|
|
"TargetCreate",
|
|
"TargetImportItem",
|
|
"TargetImportRequest",
|
|
"TargetImportResult",
|
|
"TargetIpAddressRead",
|
|
"TargetOption",
|
|
"TargetRead",
|
|
"TargetSeed",
|
|
"TargetSeedRead",
|
|
"TargetSeedResult",
|
|
"TargetSeedWrite",
|
|
"TargetSubdomainRead",
|
|
"TargetType",
|
|
"TargetUpdate",
|
|
"TargetValidationRequest",
|
|
"TargetValidationResponse",
|
|
"TemplateFilter",
|
|
"TemplateLibraryStats",
|
|
"TemplatePage",
|
|
"TemplateSelection",
|
|
"TreeNode",
|
|
"User",
|
|
"UserBase",
|
|
"UserCreate",
|
|
"UserMark",
|
|
"UserRead",
|
|
"ViewDNSCache",
|
|
"VulnTemplate",
|
|
"VulnTemplateRead",
|
|
"VulnTemplateUpdate",
|
|
"VulnTemplateUploadRequest",
|
|
"Vulnerability",
|
|
"VulnerabilityCoverage",
|
|
"VulnerabilityFacets",
|
|
"VulnerabilityFilter",
|
|
"VulnerabilityPage",
|
|
"VulnerabilityRead",
|
|
"VulnerabilityTriage",
|
|
"WatchCreate",
|
|
"WatchEvent",
|
|
"WatchHost",
|
|
"WatchRead",
|
|
"WatchUpdate",
|
|
"Wordlist",
|
|
"WordlistFile",
|
|
"WordlistRead",
|
|
"WordlistUpdate",
|
|
"WordlistUploadRequest",
|
|
"WordlistUploadResult",
|
|
]
|