mirror of
https://github.com/j3ssie/osmedeus.git
synced 2026-09-30 05:24:57 +02:00
- Add db_import_vigolium JS function that routes vigolium JSONL records by envelope type: http_record -> assets, finding -> vulnerabilities (deduped on new finding_hash column), skipping scan/oast_interaction - Add FindingHash field to Vulnerability model with idempotent ALTER TABLE migration and matching index - Suppress run errors when using --silent --empty-target placeholder mode - Retry transient fetchURLContent failures (network errors, 408/429/5xx) with exponential backoff; leave 4xx untouched so GitHub auth fallback can engage - Add unit tests for the vigolium importer (import + idempotency) and the HTTP retry behavior, plus a vigolium juice-shop sample fixture - Bump katana, naabu, kingfisher, bearer registry entries and add vigolium to the direct-fetch registry
180 lines
6.4 KiB
Go
180 lines
6.4 KiB
Go
package functions
|
|
|
|
import "github.com/j3ssie/osmedeus/v5/internal/database"
|
|
|
|
// mergeString returns incoming if non-empty, otherwise existing.
|
|
func mergeString(existing, incoming string) string {
|
|
if incoming != "" {
|
|
return incoming
|
|
}
|
|
return existing
|
|
}
|
|
|
|
// mergeInt returns incoming if non-zero, otherwise existing.
|
|
func mergeInt(existing, incoming int) int {
|
|
if incoming != 0 {
|
|
return incoming
|
|
}
|
|
return existing
|
|
}
|
|
|
|
// mergeInt64 returns incoming if non-zero, otherwise existing.
|
|
func mergeInt64(existing, incoming int64) int64 {
|
|
if incoming != 0 {
|
|
return incoming
|
|
}
|
|
return existing
|
|
}
|
|
|
|
// mergeBool returns true if either value is true (once true, stays true).
|
|
func mergeBool(existing, incoming bool) bool {
|
|
return existing || incoming
|
|
}
|
|
|
|
// mergeStringSlice returns incoming if non-nil and non-empty, otherwise existing.
|
|
func mergeStringSlice(existing, incoming []string) []string {
|
|
if len(incoming) > 0 {
|
|
return incoming
|
|
}
|
|
return existing
|
|
}
|
|
|
|
// mergeAssetFields merges incoming asset fields into existing.
|
|
// Non-zero incoming fields win; zero incoming fields preserve existing values.
|
|
// ID, CreatedAt, Workspace, and AssetValue are always taken from existing.
|
|
func mergeAssetFields(existing, incoming *database.Asset) {
|
|
// Preserve identity fields from existing
|
|
incoming.ID = existing.ID
|
|
incoming.CreatedAt = existing.CreatedAt
|
|
incoming.Workspace = existing.Workspace
|
|
incoming.AssetValue = existing.AssetValue
|
|
|
|
// HTTP data
|
|
incoming.URL = mergeString(existing.URL, incoming.URL)
|
|
incoming.Input = mergeString(existing.Input, incoming.Input)
|
|
incoming.Scheme = mergeString(existing.Scheme, incoming.Scheme)
|
|
incoming.Method = mergeString(existing.Method, incoming.Method)
|
|
incoming.Path = mergeString(existing.Path, incoming.Path)
|
|
|
|
// Response data
|
|
incoming.StatusCode = mergeInt(existing.StatusCode, incoming.StatusCode)
|
|
incoming.ContentType = mergeString(existing.ContentType, incoming.ContentType)
|
|
incoming.ContentLength = mergeInt64(existing.ContentLength, incoming.ContentLength)
|
|
incoming.Title = mergeString(existing.Title, incoming.Title)
|
|
incoming.Words = mergeInt(existing.Words, incoming.Words)
|
|
incoming.Lines = mergeInt(existing.Lines, incoming.Lines)
|
|
|
|
// Network data
|
|
incoming.HostIP = mergeString(existing.HostIP, incoming.HostIP)
|
|
incoming.DnsRecords = mergeStringSlice(existing.DnsRecords, incoming.DnsRecords)
|
|
incoming.TLS = mergeString(existing.TLS, incoming.TLS)
|
|
|
|
// Metadata
|
|
incoming.AssetType = mergeString(existing.AssetType, incoming.AssetType)
|
|
incoming.Technologies = mergeStringSlice(existing.Technologies, incoming.Technologies)
|
|
incoming.ResponseTime = mergeString(existing.ResponseTime, incoming.ResponseTime)
|
|
incoming.Remarks = mergeStringSlice(existing.Remarks, incoming.Remarks)
|
|
incoming.Source = mergeString(existing.Source, incoming.Source)
|
|
incoming.RawJsonData = mergeString(existing.RawJsonData, incoming.RawJsonData)
|
|
incoming.RawResponse = mergeString(existing.RawResponse, incoming.RawResponse)
|
|
incoming.ScreenshotBase64Data = mergeString(existing.ScreenshotBase64Data, incoming.ScreenshotBase64Data)
|
|
|
|
// CDN/WAF classification
|
|
incoming.IsCDN = mergeBool(existing.IsCDN, incoming.IsCDN)
|
|
incoming.IsCloud = mergeBool(existing.IsCloud, incoming.IsCloud)
|
|
incoming.IsWAF = mergeBool(existing.IsWAF, incoming.IsWAF)
|
|
|
|
// Repository/file assets
|
|
incoming.Language = mergeString(existing.Language, incoming.Language)
|
|
incoming.Size = mergeInt64(existing.Size, incoming.Size)
|
|
incoming.LOC = mergeInt64(existing.LOC, incoming.LOC)
|
|
incoming.ExternalURL = mergeString(existing.ExternalURL, incoming.ExternalURL)
|
|
incoming.BlobContent = mergeString(existing.BlobContent, incoming.BlobContent)
|
|
}
|
|
|
|
// mergeVulnFields merges incoming vulnerability fields into existing.
|
|
// Non-zero incoming fields win; zero incoming fields preserve existing values.
|
|
// ID, CreatedAt, and Workspace are always taken from existing.
|
|
func mergeVulnFields(existing, incoming *database.Vulnerability) {
|
|
// Preserve identity fields from existing
|
|
incoming.ID = existing.ID
|
|
incoming.CreatedAt = existing.CreatedAt
|
|
incoming.Workspace = existing.Workspace
|
|
|
|
// Core vulnerability fields
|
|
incoming.VulnInfo = mergeString(existing.VulnInfo, incoming.VulnInfo)
|
|
incoming.VulnTitle = mergeString(existing.VulnTitle, incoming.VulnTitle)
|
|
incoming.VulnDesc = mergeString(existing.VulnDesc, incoming.VulnDesc)
|
|
incoming.VulnPOC = mergeString(existing.VulnPOC, incoming.VulnPOC)
|
|
incoming.Severity = mergeString(existing.Severity, incoming.Severity)
|
|
incoming.Confidence = mergeString(existing.Confidence, incoming.Confidence)
|
|
incoming.FindingHash = mergeString(existing.FindingHash, incoming.FindingHash)
|
|
incoming.AssetType = mergeString(existing.AssetType, incoming.AssetType)
|
|
incoming.AssetValue = mergeString(existing.AssetValue, incoming.AssetValue)
|
|
incoming.Tags = mergeStringSlice(existing.Tags, incoming.Tags)
|
|
|
|
// Detail fields
|
|
incoming.DetailHTTPRequest = mergeString(existing.DetailHTTPRequest, incoming.DetailHTTPRequest)
|
|
incoming.DetailHTTPResponse = mergeString(existing.DetailHTTPResponse, incoming.DetailHTTPResponse)
|
|
incoming.RawVulnJSON = mergeString(existing.RawVulnJSON, incoming.RawVulnJSON)
|
|
}
|
|
|
|
// mergePortAssetFields merges port-specific fields from incoming to existing
|
|
func mergePortAssetFields(existing, incoming *database.Asset) {
|
|
// Preserve identity fields
|
|
incoming.ID = existing.ID
|
|
incoming.CreatedAt = existing.CreatedAt
|
|
incoming.Workspace = existing.Workspace
|
|
incoming.AssetValue = existing.AssetValue
|
|
|
|
// Merge simple fields (non-zero incoming wins)
|
|
if incoming.HostIP == "" {
|
|
incoming.HostIP = existing.HostIP
|
|
}
|
|
if incoming.Source == "" {
|
|
incoming.Source = existing.Source
|
|
}
|
|
|
|
// Keep existing OpenPorts if incoming is empty
|
|
if len(incoming.OpenPorts) == 0 {
|
|
incoming.OpenPorts = existing.OpenPorts
|
|
}
|
|
|
|
// Keep existing PortJsonData if incoming is empty
|
|
if incoming.PortJsonData == "" {
|
|
incoming.PortJsonData = existing.PortJsonData
|
|
}
|
|
}
|
|
|
|
// hasPortAssetChanged checks if port-related fields changed
|
|
func hasPortAssetChanged(existing, new *database.Asset) bool {
|
|
// Check if HostIP changed
|
|
if existing.HostIP != new.HostIP {
|
|
return true
|
|
}
|
|
|
|
// Check if PortJsonData changed
|
|
if existing.PortJsonData != new.PortJsonData {
|
|
return true
|
|
}
|
|
|
|
// Check if OpenPorts array changed
|
|
if len(existing.OpenPorts) != len(new.OpenPorts) {
|
|
return true
|
|
}
|
|
|
|
// Deep compare OpenPorts
|
|
existingPortsSet := make(map[string]bool)
|
|
for _, p := range existing.OpenPorts {
|
|
existingPortsSet[p] = true
|
|
}
|
|
|
|
for _, p := range new.OpenPorts {
|
|
if !existingPortsSet[p] {
|
|
return true
|
|
}
|
|
}
|
|
|
|
return false
|
|
}
|